Push main

app.py

@@ -1,13 +1,32 @@
-from datetime import datetime
 import json
-import gradio as gr
 import re
 import random
 from collections import defaultdict
-import pandas as pd
-import os   
+from datetime import datetime, timezone
+
+from dotenv import load_dotenv
+
+load_dotenv()
+
+import gradio as gr
 from gen_api_answer import get_model_response, parse_model_response
-from common import *
+from db import add_vote, create_db_connection, get_votes
+from utils import Vote
+from common import (
+    POLICY_CONTENT,
+    ACKNOWLEDGEMENTS,
+    DEFAULT_EVAL_PROMPT,
+    DEFAULT_INPUT,
+    DEFAULT_RESPONSE,
+    CSS_STYLES,
+    MAIN_TITLE,
+    HOW_IT_WORKS,
+    BATTLE_RULES,
+    EVAL_DESCRIPTION,
+    VOTING_HEADER,
+)
+from example_metrics import EXAMPLE_METRICS
+
 
 # Model and ELO score data
 DEFAULT_ELO = 1500  # Starting ELO for new models
@@ -15,65 +34,73 @@ K_FACTOR = 32  # Standard chess K-factor, adjust as needed
 elo_scores = defaultdict(lambda: DEFAULT_ELO)
 vote_counts = defaultdict(int)
 
+db = create_db_connection()
+votes_collection = get_votes(db)
+
+current_time = datetime.now()
+
 
 # Load the model_data from JSONL
 def load_model_data():
     model_data = {}
     try:
-        with open('data/models.jsonl', 'r') as f:
+        with open("data/models.jsonl", "r") as f:
             for line in f:
                 model = json.loads(line)
-                model_data[model['name']] = {
-                    'organization': model['organization'],
-                    'license': model['license'],
-                    'api_model': model['api_model']
+                model_data[model["name"]] = {
+                    "organization": model["organization"],
+                    "license": model["license"],
+                    "api_model": model["api_model"],
                 }
     except FileNotFoundError:
         print("Warning: models.jsonl not found")
         return {}
     return model_data
 
+
 model_data = load_model_data()
 
 current_session_id = 0
-voting_data = []
+
 
 def get_new_session_id():
     global current_session_id
     current_session_id += 1
     return f"user{current_session_id}"
 
+
 def store_vote_data(prompt, response_a, response_b, model_a, model_b, winner, judge_id):
-    vote_entry = {
-        "timestamp": datetime.now().isoformat(),
-        "prompt": prompt,
-        "response_a": response_a,
-        "response_b": response_b,
-        "model_a": model_a,
-        "model_b": model_b,
-        "winner": winner,
-        "judge_id": judge_id,
-    }
-    voting_data.append(vote_entry)
-    
-    # Save to file after each vote
-    with open('voting_data.json', 'w') as f:
-        json.dump(voting_data, f, indent=2)
+    vote = Vote(
+        timestamp=datetime.now().isoformat(),
+        prompt=prompt,
+        response_a=response_a,
+        response_b=response_b,
+        model_a=model_a,
+        model_b=model_b,
+        winner=winner,
+        judge_id=judge_id,
+    )
+    add_vote(vote, db)
+
 
 def parse_variables(prompt):
     # Extract variables enclosed in double curly braces
-    variables = re.findall(r'{{(.*?)}}', prompt)
+    variables = re.findall(r"{{(.*?)}}", prompt)
     # Remove duplicates while preserving order
     seen = set()
-    variables = [x.strip() for x in variables if not (x.strip() in seen or seen.add(x.strip()))]
+    variables = [
+        x.strip() for x in variables if not (x.strip() in seen or seen.add(x.strip()))
+    ]
     return variables
 
+
 def get_final_prompt(eval_prompt, variable_values):
     # Replace variables in the eval prompt with their values
     for var, val in variable_values.items():
-        eval_prompt = eval_prompt.replace('{{' + var + '}}', val)
+        eval_prompt = eval_prompt.replace("{{" + var + "}}", val)
     return eval_prompt
 
+
 def submit_prompt(eval_prompt, *variable_values):
     try:
         variables = parse_variables(eval_prompt)
@@ -93,7 +120,8 @@ def submit_prompt(eval_prompt, *variable_values):
             gr.update(visible=True),
             gr.update(visible=True),
             model_a,
-            model_b
+            model_b,
+            final_prompt,
         )
     except Exception as e:
         print(f"Error in submit_prompt: {str(e)}")
@@ -103,10 +131,22 @@ def submit_prompt(eval_prompt, *variable_values):
             gr.update(visible=False),
             gr.update(visible=False),
             None,
-            None
+            None,
+            None,
         )
 
-def vote(choice, model_a, model_b, prompt, response_a, response_b, judge_id):
+
+def vote(
+    choice,
+    model_a,
+    model_b,
+    final_prompt,
+    score_a,
+    critique_a,
+    score_b,
+    critique_b,
+    judge_id,
+):
     # Update ELO scores based on user choice
     elo_a = elo_scores[model_a]
     elo_b = elo_scores[model_b]
@@ -116,9 +156,9 @@ def vote(choice, model_a, model_b, prompt, response_a, response_b, judge_id):
     Eb = 1 / (1 + 10 ** ((elo_a - elo_b) / 400))
 
     # Assign actual scores
-    if choice == 'A':
+    if choice == "A":
         Sa, Sb = 1, 0
-    elif choice == 'B':
+    elif choice == "B":
         Sa, Sb = 0, 1
     else:
         Sa, Sb = 0.5, 0.5
@@ -129,39 +169,103 @@ def vote(choice, model_a, model_b, prompt, response_a, response_b, judge_id):
     vote_counts[model_a] += 1
     vote_counts[model_b] += 1
 
-    # Store the vote data
-    store_vote_data(prompt, response_a, response_b, model_a, model_b, choice, judge_id)
+    # Format the full responses with score and critique
+    response_a = f"""{score_a}
+
+{critique_a}"""
+
+    response_b = f"""{score_b}
+
+{critique_b}"""
+
+    # Store the vote data with the final prompt
+    store_vote_data(
+        final_prompt, response_a, response_b, model_a, model_b, choice, judge_id
+    )
 
     # Return updates for UI components
-    return {
-        action_buttons_row: gr.update(visible=False),
-        model_name_a: gr.update(value=f"*Model: {model_a}*"),
-        model_name_b: gr.update(value=f"*Model: {model_b}*"),
-        send_btn: gr.update(interactive=True),
-        regenerate_button: gr.update(visible=True, interactive=True)
-    }
+    return [
+        gr.update(visible=False),  # action_buttons_row
+        gr.update(value=f"*Model: {model_a}*"),  # model_name_a
+        gr.update(value=f"*Model: {model_b}*"),  # model_name_b
+        gr.update(interactive=True),  # send_btn
+        gr.update(visible=True, interactive=True),  # regenerate_button
+    ]
+
 
+def get_current_votes():
+    """Get current votes from database."""
+    return get_votes(db)
 
 
 def get_leaderboard():
+    """Generate leaderboard data using fresh votes from MongoDB."""
+    # Get fresh voting data
+    voting_data = get_current_votes()
+    print(f"Fetched {len(voting_data)} votes from database")  # Debug log
+
+    # Initialize dictionaries for tracking
+    ratings = defaultdict(lambda: DEFAULT_ELO)
+    matches = defaultdict(int)
+
+    # Process each vote
+    for vote in voting_data:
+        try:
+            model_a = vote.get("model_a")
+            model_b = vote.get("model_b")
+            winner = vote.get("winner")
+
+            # Skip if models aren't in current model_data
+            if (
+                not all([model_a, model_b, winner])
+                or model_a not in model_data
+                or model_b not in model_data
+            ):
+                continue
+
+            # Update match counts
+            matches[model_a] += 1
+            matches[model_b] += 1
+
+            # Calculate ELO changes
+            elo_a = ratings[model_a]
+            elo_b = ratings[model_b]
+
+            # Expected scores
+            expected_a = 1 / (1 + 10 ** ((elo_b - elo_a) / 400))
+            expected_b = 1 - expected_a
+
+            # Actual scores
+            score_a = 1 if winner == "A" else 0 if winner == "B" else 0.5
+            score_b = 1 - score_a
+
+            # Update ratings
+            ratings[model_a] += K_FACTOR * (score_a - expected_a)
+            ratings[model_b] += K_FACTOR * (score_b - expected_b)
+
+        except Exception as e:
+            print(f"Error processing vote: {e}")
+            continue
+
     # Generate leaderboard data
     leaderboard = []
-    for model, elo in elo_scores.items():
-        votes = vote_counts[model]
-        ci = 1.96 * (400 / (votes + 1) ** 0.5)  # Approximate 95% confidence interval
+    for model in model_data.keys():
+        votes = matches[model]
+        elo = ratings[model]
+        ci = 1.96 * (400 / (votes + 1) ** 0.5) if votes > 0 else 0
         data = {
-            'Model': model,
-            'ELO Score': f"{elo:.2f}",
-            '95% CI': f"±{ci:.2f}",
-            '# Votes': votes,
-            'Organization': model_data[model]['organization'],
-            'License': model_data[model]['license'],
+            "Model": model,
+            "ELO Score": f"{elo:.2f}",
+            "95% CI": f"±{ci:.2f}",
+            "# Votes": votes,
+            "Organization": model_data[model]["organization"],
+            "License": model_data[model]["license"],
         }
         leaderboard.append(data)
-    # Sort by ELO score
-    leaderboard.sort(key=lambda x: float(x['ELO Score']), reverse=True)
+
     return leaderboard
 
+
 def regenerate_prompt(model_a, model_b, eval_prompt, *variable_values):
     variables = parse_variables(eval_prompt)
     variable_values_dict = {var: val for var, val in zip(variables, variable_values)}
@@ -169,14 +273,14 @@ def regenerate_prompt(model_a, model_b, eval_prompt, *variable_values):
 
     # Get available models excluding the previous ones
     available_models = [m for m in model_data.keys() if m not in (model_a, model_b)]
-    
+
     # If we have enough models for new pairs
     if len(available_models) >= 2:
         model1, model2 = random.sample(available_models, 2)
     else:
         # Fallback to allowing previous models if necessary
         model1, model2 = random.sample(list(model_data.keys()), 2)
-    
+
     response_a = get_model_response(model1, model_data.get(model1), final_prompt)
     response_b = get_model_response(model2, model_data.get(model2), final_prompt)
 
@@ -185,203 +289,162 @@ def regenerate_prompt(model_a, model_b, eval_prompt, *variable_values):
     score_b, critique_b = parse_model_response(response_b)
 
     return (
-        score_a,              # score_a textbox
-        critique_a,           # critique_a textbox
-        score_b,              # score_b textbox
-        critique_b,           # critique_b textbox
+        score_a,  # score_a textbox
+        critique_a,  # critique_a textbox
+        score_b,  # score_b textbox
+        critique_b,  # critique_b textbox
         gr.update(visible=True),  # action_buttons_row
         gr.update(value="*Model: Unknown*"),  # model_name_a
         gr.update(value="*Model: Unknown*"),  # model_name_b
-        model1,              # model_a_state
-        model2               # model_b_state
+        model1,  # model_a_state
+        model2,  # model_b_state
     )
 
+
 def calculate_elo_change(rating_a, rating_b, winner):
     """Calculate ELO rating changes for both players."""
     expected_a = 1 / (1 + 10 ** ((rating_b - rating_a) / 400))
     expected_b = 1 - expected_a
-    
+
     if winner == "A":
         score_a, score_b = 1, 0
     elif winner == "B":
         score_a, score_b = 0, 1
     else:  # Handle ties
         score_a, score_b = 0.5, 0.5
-    
+
     change_a = K_FACTOR * (score_a - expected_a)
     change_b = K_FACTOR * (score_b - expected_b)
-    
+
     return change_a, change_b
 
+
 def update_leaderboard():
-    """Calculate current ELO ratings from voting history."""
-    ratings = defaultdict(lambda: DEFAULT_ELO)
+    """Generate leaderboard DataFrame using fresh votes from MongoDB."""
+    # Get fresh voting data
+    voting_data = get_current_votes()
+    print(f"Found {len(voting_data)} votes in database")
     matches = defaultdict(int)
-    wins = defaultdict(int)
-    
-    # Load voting data
-    try:
-        with open('voting_data.json', 'r') as f:
-            voting_data = json.load(f)
-    except FileNotFoundError:
-        return pd.DataFrame()
-    
-    # Process each vote
+
+    # Process each vote chronologically
     for vote in voting_data:
-        model_a = vote['model_a']
-        model_b = vote['model_b']
-        winner = vote['winner']
-        
-        # Skip if models aren't in current model_data
-        if model_a not in model_data or model_b not in model_data:
+        # Extract model names from the vote document
+        try:
+            model_a = vote.get("model_a")
+            model_b = vote.get("model_b")
+            winner = vote.get("winner")
+
+            print(f"Processing vote: {model_a} vs {model_b}, winner: {winner}")
+
+            # Skip if any required field is missing or models aren't in current model_data
+            if not all([model_a, model_b, winner]):
+                print(f"Missing required fields in vote: {vote}")
+                continue
+
+            if model_a not in model_data:
+                print(f"Model A '{model_a}' not found in model_data")
+                continue
+
+            if model_b not in model_data:
+                print(f"Model B '{model_b}' not found in model_data")
+                continue
+
+            # Update match counts
+            matches[model_a] += 1
+            matches[model_b] += 1
+            print(
+                f"Updated matches - {model_a}: {matches[model_a]}, {model_b}: {matches[model_b]}"
+            )
+        except Exception as e:
+            print(f"Error processing vote: {e}")
+            print(f"Problematic vote data: {vote}")
             continue
-        
-        # Update match counts
-        matches[model_a] += 1
-        matches[model_b] += 1
-        if winner == "A":
-            wins[model_a] += 1
-        elif winner == "B":
-            wins[model_b] += 1
-        else:  # Handle ties
-            wins[model_a] += 0.5
-            wins[model_b] += 0.5
-        
-        # Update ELO ratings
-        change_a, change_b = calculate_elo_change(ratings[model_a], ratings[model_b], winner)
-        ratings[model_a] += change_a
-        ratings[model_b] += change_b
-    
-    # Create leaderboard DataFrame
-    leaderboard_data = []
-    for model in model_data.keys():  # Only include current models
-        win_rate = (wins[model] / matches[model] * 100) if matches[model] > 0 else 0
-        ci = 1.96 * (400 / (matches[model] + 1) ** 0.5) if matches[model] > 0 else 0  # Confidence interval
-        leaderboard_data.append({
-            'Model': model,
-            'ELO': round(ratings[model], 1),
-            '95% CI': f"±{ci:.1f}",
-            'Matches': matches[model],
-            'Win Rate': f"{win_rate:.1f}%",
-            'Organization': model_data[model]['organization'],
-            'License': model_data[model]['license']
-        })
-    
-    # Sort by ELO rating
-    df = pd.DataFrame(leaderboard_data)
-    return df.sort_values('ELO', ascending=False).reset_index(drop=True)
+
 
 # Update the display_leaderboard function
 def display_leaderboard():
     df = update_leaderboard()
     return gr.DataFrame(
         value=df,
-        headers=['Model', 'ELO', '95% CI', 'Matches', 'Organization', 'License'],
-        datatype=['str', 'number', 'str', 'number', 'str', 'str', 'str'],
-        row_count=(len(df) + 1, 'dynamic'),
+        headers=["Model", "ELO", "95% CI", "Matches", "Organization", "License"],
+        datatype=["str", "number", "str", "number", "str", "str", "str"],
+        row_count=(len(df) + 1, "dynamic"),
     )
 
+
 # Update the leaderboard table definition in the UI
 leaderboard_table = gr.Dataframe(
-    headers=['Model', 'ELO', '95% CI', 'Matches', 'Organization', 'License'],
-    datatype=['str', 'number', 'str', 'number', 'str', 'str', 'str']
+    headers=["Model", "ELO", "95% CI", "Matches", "Organization", "License"],
+    datatype=["str", "number", "str", "number", "str", "str", "str"],
 )
 
+
 def get_leaderboard_stats():
     """Get summary statistics for the leaderboard."""
-    try:
-        with open('voting_data.json', 'r') as f:
-            voting_data = json.load(f)
-        
-        total_votes = len(voting_data)
-        total_models = len(model_data)
-        last_updated = datetime.now().strftime("%Y-%m-%d %H:%M:%S UTC")
-        
-        return f"""
+    now = datetime.now(timezone.utc)
+    total_votes = len(get_current_votes())
+    total_models = len(model_data)
+    last_updated = now.replace(minute=0, second=0, microsecond=0).strftime(
+        "%B %d, %Y at %H:00 UTC"
+    )
+
+    return f"""
 ### Leaderboard Stats
 - **Total Models**: {total_models}
 - **Total Votes**: {total_votes}
 - **Last Updated**: {last_updated}
 """
-    except FileNotFoundError:
-        return "No voting data available"
 
-def initialize_voting_data():
-    """Initialize or clear the voting data file."""
-    empty_data = []
-    with open('voting_data.json', 'w') as f:
-        json.dump(empty_data, f)
-
-# Add this near the start of your app initialization, before the Gradio interface setup
-if __name__ == "__main__":
-    initialize_voting_data()
-    
-    # ... rest of your Gradio app setup ...
-
-# Example evaluation metrics data
-EXAMPLE_METRICS = {
-    "Hallucination": {
-        "prompt": DEFAULT_EVAL_PROMPT,  # We'll replace these with actual examples
-        "input": DEFAULT_INPUT,
-        "response": DEFAULT_RESPONSE
-    },
-    "Precision": {
-        "prompt": DEFAULT_EVAL_PROMPT,
-        "input": DEFAULT_INPUT,
-        "response": DEFAULT_RESPONSE
-    },
-    "Recall": {
-        "prompt": DEFAULT_EVAL_PROMPT,
-        "input": DEFAULT_INPUT,
-        "response": DEFAULT_RESPONSE
-    },
-    "Logical coherence": {
-        "prompt": DEFAULT_EVAL_PROMPT,
-        "input": DEFAULT_INPUT,
-        "response": DEFAULT_RESPONSE
-    },
-    "Faithfulness": {
-        "prompt": DEFAULT_EVAL_PROMPT,
-        "input": DEFAULT_INPUT,
-        "response": DEFAULT_RESPONSE
-    }
-}
 
 def set_example_metric(metric_name):
     if metric_name == "Custom":
-        return [
-            DEFAULT_EVAL_PROMPT,
-            DEFAULT_INPUT,
-            DEFAULT_RESPONSE
-        ]
-    
+        variables = parse_variables(DEFAULT_EVAL_PROMPT)
+        variable_values = []
+        for var in variables:
+            if var == "input":
+                variable_values.append(DEFAULT_INPUT)
+            elif var == "response":
+                variable_values.append(DEFAULT_RESPONSE)
+            else:
+                variable_values.append("")  # Default empty value
+        # Pad variable_values to match the length of variable_rows
+        while len(variable_values) < len(variable_rows):
+            variable_values.append("")
+        return [DEFAULT_EVAL_PROMPT] + variable_values
+
     metric_data = EXAMPLE_METRICS[metric_name]
-    return [
-        metric_data["prompt"],
-        metric_data["input"],
-        metric_data["response"]
-    ]
+    variables = parse_variables(metric_data["prompt"])
+    variable_values = []
+    for var in variables:
+        value = metric_data.get(var, "")  # Default to empty string if not found
+        variable_values.append(value)
+    # Pad variable_values to match the length of variable_rows
+    while len(variable_values) < len(variable_rows):
+        variable_values.append("")
+    return [metric_data["prompt"]] + variable_values
+
 
 # Select random metric at startup
 def get_random_metric():
     metrics = list(EXAMPLE_METRICS.keys())
     return set_example_metric(random.choice(metrics))
 
-with gr.Blocks(theme='default', css=CSS_STYLES) as demo:
+
+with gr.Blocks(theme="default", css=CSS_STYLES) as demo:
     judge_id = gr.State(get_new_session_id())
     gr.Markdown(MAIN_TITLE)
     gr.Markdown(HOW_IT_WORKS)
-    
+
     with gr.Tabs():
         with gr.TabItem("Judge Arena"):
-            
+
             with gr.Row():
                 with gr.Column():
                     gr.Markdown(BATTLE_RULES)
                     gr.Markdown(EVAL_DESCRIPTION)
-            
+
             # Add Example Metrics Section
-            with gr.Accordion("Example evaluation metrics", open=True):
+            with gr.Accordion("Evaluator Prompt Templates", open=False):
                 with gr.Row():
                     custom_btn = gr.Button("Custom", variant="secondary")
                     hallucination_btn = gr.Button("Hallucination")
@@ -399,7 +462,7 @@ with gr.Blocks(theme='default', css=CSS_STYLES) as demo:
                         lines=1,
                         value=DEFAULT_EVAL_PROMPT,
                         placeholder="Type your eval prompt here... denote variables in {{curly brackets}} to be populated on the right.",
-                        show_label=True
+                        show_label=True,
                     )
 
                 # Right column - Variable Mapping
@@ -410,28 +473,25 @@ with gr.Blocks(theme='default', css=CSS_STYLES) as demo:
                     for i in range(5):
                         initial_visibility = True if i < 2 else False
                         with gr.Group(visible=initial_visibility) as var_row:
-                            # Variable input with direct label
-                            initial_value = DEFAULT_INPUT if i == 0 else DEFAULT_RESPONSE
-                            initial_label = "input" if i == 0 else "response" if i == 1 else f"variable_{i+1}"
+                            # Set default labels for the first two inputs
+                            default_label = (
+                                "input" if i == 0 else "response" if i == 1 else ""
+                            )
                             var_input = gr.Textbox(
-                                label=initial_label,
-                                value=initial_value,
-                                container=True
+                                container=True,
+                                label=default_label,  # Add default label here
                             )
                             variable_rows.append((var_row, var_input))
 
             # Send button
             with gr.Row(elem_classes="send-button-row"):
                 send_btn = gr.Button(
-                    value="Test the evaluators",
-                    variant="primary",
-                    size="lg",
-                    scale=1
+                    value="Test the evaluators", variant="primary", size="lg", scale=1
                 )
-            
+
             # Add divider heading for model outputs
             gr.Markdown(VOTING_HEADER)
-            
+
             # Model Responses side-by-side
             with gr.Row():
                 with gr.Column():
@@ -444,23 +504,24 @@ with gr.Blocks(theme='default', css=CSS_STYLES) as demo:
                     score_b = gr.Textbox(label="Score", interactive=False)
                     critique_b = gr.TextArea(label="Critique", lines=8, interactive=False)
                     model_name_b = gr.Markdown("*Model: Unknown*")
-            
+
             # Initially hide vote buttons and regenerate button
             with gr.Row(visible=False) as action_buttons_row:
                 vote_a = gr.Button("Choose A", variant="primary")
                 vote_tie = gr.Button("Tie", variant="secondary")
                 vote_b = gr.Button("Choose B", variant="primary")
-            regenerate_button = gr.Button("Regenerate with different models", variant="secondary", visible=False)
-            
+            regenerate_button = gr.Button(
+                "Regenerate with different models", variant="secondary", visible=False
+            )
+
             # Add spacing and acknowledgements at the bottom
             gr.Markdown(ACKNOWLEDGEMENTS)
 
         with gr.TabItem("Leaderboard"):
-            refresh_button = gr.Button("Refresh")
             stats_display = gr.Markdown()
             leaderboard_table = gr.Dataframe(
-                headers=['Model', 'ELO', '95% CI', 'Matches', 'Organization', 'License'],
-                datatype=['str', 'number', 'str', 'number', 'str', 'str']
+                headers=["Model", "ELO", "95% CI", "Matches", "Organization", "License"],
+                datatype=["str", "number", "str", "number", "str", "str", "str"],
             )
 
         with gr.TabItem("Policy"):
@@ -469,39 +530,50 @@ with gr.Blocks(theme='default', css=CSS_STYLES) as demo:
     # Define state variables for model tracking
     model_a_state = gr.State()
     model_b_state = gr.State()
+    final_prompt_state = gr.State()
 
     # Update variable inputs based on the eval prompt
     def update_variables(eval_prompt):
         variables = parse_variables(eval_prompt)
         updates = []
-        for i in range(5):
+
+        for i in range(len(variable_rows)):
             var_row, var_input = variable_rows[i]
             if i < len(variables):
-                # Set default values for 'input' and 'response', otherwise leave empty
-                if variables[i] == "input":
-                    initial_value = DEFAULT_INPUT
-                elif variables[i] == "response":
-                    initial_value = DEFAULT_RESPONSE
+                var_name = variables[i]
+                # Set the number of lines based on the variable name
+                if var_name == "response":
+                    lines = 4  # Adjust this number as needed
                 else:
-                    initial_value = ""  # Empty for new variables
-
-                updates.extend([
-                    gr.update(visible=True),  # var_row
-                    gr.update(value=initial_value, label=variables[i], visible=True)  # var_input with dynamic label
-                ])
+                    lines = 1  # Default to single line for other variables
+                updates.extend(
+                    [
+                        gr.update(visible=True),  # Show the variable row
+                        gr.update(
+                            label=var_name, visible=True, lines=lines
+                        ),  # Update label and lines
+                    ]
+                )
             else:
-                updates.extend([
-                    gr.update(visible=False),  # var_row
-                    gr.update(value="", visible=False)  # var_input
-                ])
+                updates.extend(
+                    [
+                        gr.update(visible=False),  # Hide the variable row
+                        gr.update(value="", visible=False),  # Clear value when hidden
+                    ]
+                )
         return updates
 
-    eval_prompt.change(fn=update_variables, inputs=eval_prompt, outputs=[item for sublist in variable_rows for item in sublist])
+    eval_prompt.change(
+        fn=update_variables,
+        inputs=eval_prompt,
+        outputs=[item for sublist in variable_rows for item in sublist],
+    )
 
     # Regenerate button functionality
     regenerate_button.click(
         fn=regenerate_prompt,
-        inputs=[model_a_state, model_b_state, eval_prompt] + [var_input for _, var_input in variable_rows],
+        inputs=[model_a_state, model_b_state, eval_prompt]
+        + [var_input for _, var_input in variable_rows],
         outputs=[
             score_a,
             critique_a,
@@ -511,63 +583,120 @@ with gr.Blocks(theme='default', css=CSS_STYLES) as demo:
             model_name_a,
             model_name_b,
             model_a_state,
-            model_b_state
-        ]
+            model_b_state,
+        ],
     )
 
     # Update model names after responses are generated
     def update_model_names(model_a, model_b):
-        return gr.update(value=f"*Model: {model_a}*"), gr.update(value=f"*Model: {model_b}*")
+        return gr.update(value=f"*Model: {model_a}*"), gr.update(
+            value=f"*Model: {model_b}*"
+        )
 
     # Store the last submitted prompt and variables for comparison
     last_submission = gr.State({})
 
-
     # Update the vote button click handlers
     vote_a.click(
-        fn=lambda *args: vote('A', *args),
-        inputs=[model_a_state, model_b_state, eval_prompt, score_a, score_b, judge_id],
-        outputs=[action_buttons_row, model_name_a, model_name_b, send_btn, regenerate_button]
+        fn=lambda *args: vote("A", *args),
+        inputs=[
+            model_a_state,
+            model_b_state,
+            final_prompt_state,
+            score_a,
+            critique_a,
+            score_b,
+            critique_b,
+            judge_id,
+        ],
+        outputs=[
+            action_buttons_row,
+            model_name_a,
+            model_name_b,
+            send_btn,
+            regenerate_button,
+        ],
     )
 
     vote_b.click(
-        fn=lambda *args: vote('B', *args),
-        inputs=[model_a_state, model_b_state, eval_prompt, score_a, score_b, judge_id],
-        outputs=[action_buttons_row, model_name_a, model_name_b, send_btn, regenerate_button]
+        fn=lambda *args: vote("B", *args),
+        inputs=[
+            model_a_state,
+            model_b_state,
+            final_prompt_state,
+            score_a,
+            critique_a,
+            score_b,
+            critique_b,
+            judge_id,
+        ],
+        outputs=[
+            action_buttons_row,
+            model_name_a,
+            model_name_b,
+            send_btn,
+            regenerate_button,
+        ],
     )
 
     vote_tie.click(
-        fn=lambda *args: vote('Tie', *args),
-        inputs=[model_a_state, model_b_state, eval_prompt, score_a, score_b, judge_id],
-        outputs=[action_buttons_row, model_name_a, model_name_b, send_btn, regenerate_button]
+        fn=lambda *args: vote("Tie", *args),
+        inputs=[
+            model_a_state,
+            model_b_state,
+            final_prompt_state,
+            score_a,
+            critique_a,
+            score_b,
+            critique_b,
+            judge_id,
+        ],
+        outputs=[
+            action_buttons_row,
+            model_name_a,
+            model_name_b,
+            send_btn,
+            regenerate_button,
+        ],
     )
 
     # Update the send button handler to store the submitted inputs
     def submit_and_store(prompt, *variables):
         # Create a copy of the current submission
         current_submission = {"prompt": prompt, "variables": variables}
-        
+
         # Get the responses
-        response_a, response_b, buttons_visible, regen_visible, model_a, model_b = submit_prompt(prompt, *variables)
-        
+        (
+            response_a,
+            response_b,
+            buttons_visible,
+            regen_visible,
+            model_a,
+            model_b,
+            final_prompt,
+        ) = submit_prompt(prompt, *variables)
+
         # Parse the responses
         score_a, critique_a = parse_model_response(response_a)
         score_b, critique_b = parse_model_response(response_b)
-        
+
         # Update the last_submission state with the current values
         last_submission.value = current_submission
-        
+
         return (
             score_a,
             critique_a,
             score_b,
             critique_b,
             buttons_visible,
-            gr.update(visible=True, interactive=True),  # Show and enable regenerate button
+            gr.update(
+                visible=True, interactive=True
+            ),  # Show and enable regenerate button
             model_a,
             model_b,
+            final_prompt,  # Add final_prompt to state
+            gr.update(value="*Model: Unknown*"),
             gr.update(value="*Model: Unknown*"),
-            gr.update(value="*Model: Unknown*")
         )
 
     send_btn.click(
@@ -582,9 +711,10 @@ with gr.Blocks(theme='default', css=CSS_STYLES) as demo:
             regenerate_button,
             model_a_state,
             model_b_state,
-            model_name_a,    # Add model name outputs
-            model_name_b
-        ]
+            final_prompt_state,  # Add final_prompt_state to outputs
+            model_name_a,
+            model_name_b,
+        ],
     )
 
     # Update the input change handlers to also disable regenerate button
@@ -594,88 +724,75 @@ with gr.Blocks(theme='default', css=CSS_STYLES) as demo:
         current_inputs = {"prompt": prompt, "variables": variables}
         inputs_changed = last_inputs != current_inputs
         return [
-            gr.update(interactive=True),                    # send button always enabled
-            gr.update(interactive=not inputs_changed)       # regenerate button disabled if inputs changed
+            gr.update(interactive=True),  # send button always enabled
+            gr.update(
+                interactive=not inputs_changed
+            ),  # regenerate button disabled if inputs changed
         ]
 
     # Update the change handlers for prompt and variables
     eval_prompt.change(
         fn=handle_input_changes,
         inputs=[eval_prompt] + [var_input for _, var_input in variable_rows],
-        outputs=[send_btn, regenerate_button]
+        outputs=[send_btn, regenerate_button],
     )
 
     for _, var_input in variable_rows:
         var_input.change(
             fn=handle_input_changes,
             inputs=[eval_prompt] + [var_input for _, var_input in variable_rows],
-            outputs=[send_btn, regenerate_button]
+            outputs=[send_btn, regenerate_button],
         )
 
     # Update the leaderboard
     def refresh_leaderboard():
+        """Refresh the leaderboard data and stats."""
         leaderboard = get_leaderboard()
         data = [
             [
-                entry['Model'],
-                float(entry['ELO Score']),
-                entry['95% CI'],
-                entry['# Votes'],
-                entry['Organization'],
-                entry['License']
-            ] for entry in leaderboard
+                entry["Model"],
+                float(entry["ELO Score"]),
+                entry["95% CI"],
+                entry["# Votes"],
+                entry["Organization"],
+                entry["License"],
+            ]
+            for entry in leaderboard
         ]
         stats = get_leaderboard_stats()
         return [gr.update(value=data), gr.update(value=stats)]
 
-    refresh_button.click(
-        fn=refresh_leaderboard,
-        inputs=None,
-        outputs=[leaderboard_table, stats_display]
-    )
-
     # Add the load event at the very end, just before demo.launch()
     demo.load(
-        fn=refresh_leaderboard,
-        inputs=None,
-        outputs=[leaderboard_table, stats_display]
+        fn=refresh_leaderboard, inputs=None, outputs=[leaderboard_table, stats_display]
     )
 
     # Add click handlers for metric buttons
-    custom_btn.click(
-        fn=lambda: set_example_metric("Custom"),
-        outputs=[eval_prompt, variable_rows[0][1], variable_rows[1][1]]
-    )
+    outputs_list = [eval_prompt] + [var_input for _, var_input in variable_rows]
+
+    custom_btn.click(fn=lambda: set_example_metric("Custom"), outputs=outputs_list)
 
     hallucination_btn.click(
-        fn=lambda: set_example_metric("Hallucination"),
-        outputs=[eval_prompt, variable_rows[0][1], variable_rows[1][1]]
+        fn=lambda: set_example_metric("Hallucination"), outputs=outputs_list
     )
 
-    precision_btn.click(
-        fn=lambda: set_example_metric("Precision"),
-        outputs=[eval_prompt, variable_rows[0][1], variable_rows[1][1]]
-    )
+    precision_btn.click(fn=lambda: set_example_metric("Precision"), outputs=outputs_list)
 
-    recall_btn.click(
-        fn=lambda: set_example_metric("Recall"),
-        outputs=[eval_prompt, variable_rows[0][1], variable_rows[1][1]]
-    )
+    recall_btn.click(fn=lambda: set_example_metric("Recall"), outputs=outputs_list)
 
     coherence_btn.click(
-        fn=lambda: set_example_metric("Logical coherence"),
-        outputs=[eval_prompt, variable_rows[0][1], variable_rows[1][1]]
+        fn=lambda: set_example_metric("Logical_Coherence"), outputs=outputs_list
     )
 
     faithfulness_btn.click(
-        fn=lambda: set_example_metric("Faithfulness"),
-        outputs=[eval_prompt, variable_rows[0][1], variable_rows[1][1]]
+        fn=lambda: set_example_metric("Faithfulness"), outputs=outputs_list
     )
 
-    # Set random metric at startup
+    # Set default metric at startup
     demo.load(
-        fn=get_random_metric,
-        outputs=[eval_prompt, variable_rows[0][1], variable_rows[1][1]]
+        fn=lambda: set_example_metric("Custom"),
+        outputs=[eval_prompt] + [var_input for _, var_input in variable_rows],
     )
 
-demo.launch()
+if __name__ == "__main__":
+    demo.launch()

common.py

@@ -49,7 +49,7 @@ EVAL_DESCRIPTION = """
 <br><br>
 """
 
-DEFAULT_EVAL_PROMPT = """You are assessing a chat bot response to a user's input based on the helpfulness of the response.
+DEFAULT_EVAL_PROMPT = """You are assessing a chat bot response to a user's input based on [INSERT CRITERIA]
 
 Score:
 A score of 1 means that the response's answer meets all of the evaluation criteria.
@@ -101,65 +101,55 @@ By creating advanced evaluation models, we enable AI developers to identify and
 
 ## Overview
 
-Judge Arena is an open-source platform dedicated to improving the standard of evaluation of generative AI models in their role as judges. Users can run evals and assess anonymized responses from two competing model judges, choosing the better judgement or declaring a tie. This policy outlines our commitments and guidelines to ensure a fair, open, and collaborative environment for both users and model providers.
+Judge Arena is an open-source platform dedicated to improving the standard of evaluation of generative AI models in their role as judges. Users can run evals and assess anonymized responses from two competing model judges, choosing the better judgement or declaring a tie. This policy outlines our commitments to maintain a fair, open, and collaborative environment :)
 
 ## Transparency
 
-- **Open-Source**: Judge Arena's code is open-source and available on GitHub. This approach allows anyone to review, replicate, or modify the platform to suit their needs. We use proprietary model provider APIs where provided and Together AI's API to serve leading open-source models.
-- **Community Engagement**: We actively encourage contributions from the community. Feedback, code contributions, and discussions are welcome to improve the platform's functionality, fairness, and transparency.
-- **Methodology**: All processes related to model evaluation, rating calculations, and model selection are openly documented. This transparency ensures that our processes are understandable and reproducible by others.
-- **Data Sharing**: Periodically, we will share 20% of the collected evaluation data with the community. This data includes anonymized prompts, model responses, and aggregated evaluation results.
+- **Open-Source**: Judge Arena's code is open-source and available on GitHub. We encourage contributions from the community and anyone can replicate or modify the platform to suit their needs. We use proprietary model provider APIs where provided and Together AI's API to serve leading open-source models.
+- **Methodology**: All processes related to model evaluation, rating calculations, and model selection are openly documented. We'd like to ensure that our ranking system is understandable and reproducible by others!
+- **Data Sharing**: Periodically, we'll share 20% of the collected evaluation data with the community. The data collected from Judge Arena is restricted to an anonymized user ID, the final prompt sent, the model responses, the user vote, and the timestamp.
 
 ## Model Inclusion Criteria
 
-Judge Arena is specifically designed to assess AI models that function as evaluators (a.k.a judges), including but not limited to powerful general-purpose models and the latest language models designed for evaluation tasks. Models are eligible for inclusion if they meet the following criteria:
+Judge Arena is specifically designed to assess AI models that function as evaluators (a.k.a judges). This includes but is not limited to powerful general-purpose models and the latest language models designed for evaluation tasks. Models are eligible for inclusion if they meet the following criteria:
 
-- **Judge Capability**: The model must possess the ability to score AND critique responses, content, or other models' outputs effectively.
+- **Judge Capability**: The model should possess the ability to score AND critique responses, content, or other models' outputs effectively.
 - **Adaptable:** The model must be prompt-able to be evaluate in different scoring formats, for different criteria.
 - **Accessibility**:
    - **Public API Access**: Models accessible through public APIs without restrictive barriers.
    - **Open-Source Models**: Models with publicly available weights that can be downloaded and run by the community.
 
-## Evaluation Methodology
-
-- **User Participation**: Users run evaluations and select preferred model responses based on quality, relevance, and accuracy contributing to the model's overall rating.
-- **Blind Testing**: All model evaluations are conducted blindly. Users are not informed which model produced which response to eliminate bias.
-- **Data Collection**: We collect sufficient data to ensure statistical significance in our evaluations. We additionally show the 95% confidence interval in the leaderboard to provide a signal of reliability.
-- **Anomaly Detection**: We monitor user activity to detect and mitigate anomalous behavior or voting patterns that could skew results.
-
 ## Leaderboard Management
 
 - **ELO Ranking System**: Models are ranked on a public leaderboard based on aggregated user evaluations. We use an ELO rating system to rank AI judges on the public leaderboard. Each model begins with an initial rating of 1500 (as is used by the International Chess Federation), and we use a K-factor of 32 to determine the maximum rating adjustment after each evaluation.
-- **Minimum Period**: Listed models remain accessible on Judge Arena for a minimum period of two weeks to allow for comprehensive community evaluation.
-- **Deprecation Policy**: Models may be removed from the leaderboard if they become inaccessible, are no longer publicly available.
+- **Minimum Period**: Listed models remain accessible on Judge Arena for a minimum period of two weeks so they can be comprehensively evaluated.
+- **Deprecation Policy**: Models may be removed from the leaderboard if they become inaccessible or are no longer publicly available.
 
-## Privacy and Data Protection
+This policy might be updated to reflect changes in our practices or in response to community feedback.
 
-- **Anonymization**: All shared data is anonymized to prevent the identification of individual users.
+# FAQ
 
-## Policy Updates and Communication
+**Isn't this the same as Chatbot Arena?**
 
-- **Ongoing Revisions**: This policy may be updated to reflect changes in our practices or in response to community feedback.
-- **Notification of Changes**: Policy changes will be communicated to users and stakeholders on this page.
-<br><br>
+We are big fans of what the LMSYS team have done with Chatbot Arena and fully credit them for the inspiration to develop this. We were looking for a dynamic leaderboard that graded on AI judge capabilities and didn't manage to find one, so we created Judge Arena. This UI is designed especially for evals; to match the format of the model-based eval prompts that you would use in your LLM evaluation / monitoring tool.
 
-# FAQ
+**What are the Evaluator Prompt Templates based on?**
 
-**Isn't this the same as Chatbot Arena?**
+As a quick start, we've set up templates that cover the most popular evaluation metrics out there on LLM evaluation / monitoring tools, often known as 'base metrics'. The data samples used in these were randomly picked from popular datasets from academia - [ARC](https://huggingface.co/datasets/allenai/ai2_arc), [Preference Collection](https://huggingface.co/datasets/prometheus-eval/Preference-Collection), [RewardBench](https://huggingface.co/datasets/allenai/reward-bench), [RAGTruth](https://arxiv.org/abs/2401.00396).
 
-- We are big fans of what the LMSYS team have done with Chatbot Arena and fully credit them for the inspiration to develop this. We were looking for a dynamic leaderboard that graded on AI judge capabilities and didn't manage to find one, so we created Judge Arena. This UI is designed especially for evals; to match the format of the model-based eval prompts that you would use in your LLM evaluation / monitoring tool.
+These templates are designed as a starting point to showcase how to interact with the Judge Arena, especially for those less familiar with using LLM judges.
 
-\n\n**Why should I trust this leaderboard?**
+**Why should I trust this leaderboard?**
 
-- We have listed out our efforts to be fully transparent in the policies above. All of the code for this leaderboard is open-source and can be found on our [Github](https://github.com/atla-ai/judge-arena).
+We have listed out our efforts to be fully transparent in the policies above. All of the code for this leaderboard is open-source and can be found on our [Github](https://github.com/atla-ai/judge-arena).
 
-\n\n**Who funds this effort?**
+**Who funds this effort?**
 
-- Atla currently funds this out of our own pocket. We are looking for API credits (with no strings attached) to support this effort - please get in touch if you or someone you know might be able to help.
+Atla currently funds this out of our own pocket. We are looking for API credits (with no strings attached) to support this effort - please get in touch if you or someone you know might be able to help.
 
-\n\n**What is Atla working on?**
+**What is Atla working on?**
 
-- We are training a general-purpose evaluator that you will soon be able to run in this Judge Arena. Our next step will be to open-source a powerful model that the community can use to run fast and accurate evaluations.
+We are training a general-purpose evaluator that you will soon be able to run in this Judge Arena. Our next step will be to open-source a powerful model that the community can use to run fast and accurate evaluations.
 <br><br>
 # Get in touch
-Feel free to email us at [[email protected]](mailto:[email protected]) or leave feedback on our [Github](https://github.com/atla-ai/judge-arena)!"""
+Feel free to email us at [[email protected]](mailto:[email protected]) or leave feedback on our [Github](https://github.com/atla-ai/judge-arena)!"""

db.py

@@ -0,0 +1,33 @@
+import os
+from pymongo import MongoClient
+from pymongo.database import Database
+from utils import get_logger, Vote
+from datetime import datetime, timedelta, timezone
+from typing import List
+
+logger = get_logger()
+
+
+def create_db_connection() -> Database:
+    print(os.getenv("MONGO_URI"))
+    print(os.getenv("MONGO_DB"))
+    db = MongoClient(os.getenv("MONGO_URI")).get_database(os.getenv("MONGO_DB"))
+    return db
+
+
+def add_vote(vote: Vote, db: Database) -> None:
+    try:
+        db.get_collection("votes").insert_one(vote.__dict__)
+        logger.info("Vote added to database")
+    except Exception as e:
+        logger.error("Error adding vote to database")
+        logger.error(e)
+
+
+def get_votes(db: Database) -> List[Vote]:
+    now = datetime.now(timezone.utc)
+    current_hour = now.replace(minute=0, second=0, microsecond=0)
+    votes = list(
+        db.get_collection("votes").find({"timestamp": {"$lte": current_hour.isoformat()}})
+    )
+    return votes

example_metrics.py

@@ -0,0 +1,113 @@
+# Example evaluation metrics data
+
+EXAMPLE_METRICS = {
+    "Hallucination": {
+        "prompt": """You are assessing a chat bot response to a user's input based on whether it contradicts the known ground truth. Focus on factual inconsistencies and opposing statements.\n
+Score:
+A score of 0 means the response contains no contradictions to the ground truth.
+A score of 1 means the response contains contradictions to the ground truth.\n
+Here is the data:
+[BEGIN DATA]
+***
+[User Query]: {{input}}
+***
+[Ground Truth Response]: {{ground_truth}}
+***
+[Response]: {{response}}
+***
+[END DATA]""",
+        "input": """Lichens are symbiotic organisms made of green algae and fungi. What do the green algae supply to the fungi in this symbiotic relationship?
+A) Carbon dioxide 
+B) Food
+C) Protection
+D) Water""",
+        # Example from ARC Evals
+        "ground_truth": "B) Food",
+        "response": "C) Protection",
+    },
+    "Precision": {
+        "prompt": """You are assessing a chat bot response to a user's input based on how completely it captures the key facts and details from the ground truth response.\n
+Score:
+A score of 1 means the response is completely irrelevant or contradicts the ground truth response.
+A score of 2 means the response includes some relevant information but is mostly filled with irrelevant or incorrect details.
+A score of 3 means the response is somewhat relevant with a balanced mix of relevant and irrelevant information.
+A score of 4 means the response is mostly relevant, containing only minor irrelevant details.
+A score of 5 means the response is entirely relevant, perfectly aligning with the ground truth response without any irrelevant information.\n
+Here is the data:
+[BEGIN DATA]
+***
+[User Query]: {{input}}
+***
+[Ground Truth Response]: {{ground_truth}}
+***
+[Response]: {{response}}
+***
+[END DATA]""",
+        # Example from Prometheus
+        "input": "In a city, there are three different colored houses - red, blue, and yellow. The city council has made a rule that no two same colored houses can be adjacent to each other. A query is raised by the city planner who is designing the layout of the houses. The query is, 'Can you help me design a layout where I can place these houses without violating the rule of the city council?' The user didn't provide information about the total number of houses, the layout structure, or the proportion of house colors.",
+        "ground_truth": "In order to provide an accurate layout design, I'll need a bit more information. Could you please specify how many houses in total are to be placed in the city? Also, it would be helpful to know the ratio of the different colored houses. Are there equal numbers of red, blue, and yellow houses, or does one color predominate? Furthermore, understanding the layout structure you have in mind is also important. Is it a linear layout, a circular one, a block-based one, or some other structure? Once I have these details, I'll be able to provide a more precise and useful answer.",
+        "response": "In designing the layout, you could alternate between the three colors - red, blue, and yellow, to ensure no two houses of the same color are next to each other. You could possibly start with a red house, followed by a blue, then a yellow, and then repeat the sequence until all houses are placed. However, the total number of houses and the ratio of the different colored houses could affect the final layout. For instance, if there are more red houses than blue and yellow, some adjustments may need to be made. Also, the structure of the layout could play a role, whether it's a linear or block-based layout, or any other. It might be helpful to get some additional information about these aspects, but based on the information provided, this approach could work.",
+    },
+    "Recall": {
+        "prompt": """You are assessing a chat bot response to a user's input based on how completely it captures the key facts and details from the ground truth response.\n
+Score:
+A score of 1 means the response misses almost all the key facts and details from the ground truth.
+A score of 2 means the response includes only a few of the key facts and details from the ground truth.
+A score of 3 means the response captures a moderate amount of the key facts and details from the ground truth.
+A score of 4 means the response includes most of the key facts and details from the ground truth.
+A score of 5 means the response perfectly captures all key facts and details from the ground truth without omission.\n
+Here is the data:
+[BEGIN DATA]
+***
+[User Query]: {{input}}
+***
+[Ground Truth Response]: {{ground_truth}}
+***
+[Response]: {{response}}
+***
+[END DATA]""",
+        # Example from Prometheus
+        "input": """There's a thing that's bothering me. It's related to a kind of fruit, but not really the usual ones we eat. It's sweet, crunchy, and a lot of people like it, but it's not something you'd normally find in a grocery store. What kind of fruit am I talking about?""",
+        "ground_truth": """Based on your description, it seems like you might be referring to the 'honeycrisp apple'. While it is a type of fruit and is known for its sweetness and crunchiness, it's not as commonly found in grocery stores as other fruit varieties due to its specific growing requirements and seasonality. Its unique taste and texture make it a favorite among many people. However, it's important to note that there could potentially be other fruits that also fit the given description, as taste and texture can vary significantly among fruit varieties.""",
+        "response": """From your clues, it seems that you might be referring to a 'honeycrisp apple'. This fruit is known for its sweetness and crunchiness, but it isn't typically seen in many grocery stores. Many people do enjoy its unique taste and texture. However, without more specific details, there could be other fruits that also match your description. Some fruits can share similar characteristics like sweetness and crunchiness.""",
+    },
+    "Logical_Coherence": {
+        "prompt": """You are assessing a chat bot response to a user's input based on the logical coherence of the response.\n
+Score:
+A score of 1 means the response is illogical, contains internal contradictions, or has significant jumps in reasoning that make it hard to follow.
+A score of 3 means the response follows some logical flow but has minor inconsistencies or gaps in reasoning.
+A score of 5 means the response is completely logically coherent, with clear connections between ideas and a natural progression of thought.\n
+Here is the data:
+[BEGIN DATA]
+***
+[User Query]: {{input}}
+***
+[Response]: {{response}}
+***
+[END DATA]""",
+        # Example from RewardBench
+        "input": "How did US states get their names?",
+        "response": "US states got their names for a variety of reasons. Many are named after early explorers, leaders, or patriots. Some are named after geographical features, such as mountains or rivers. Others are named after plants or animals that are native to the state. Some are even named after historical figures or characters from literature or mythology.",
+    },
+    "Faithfulness": {
+        "prompt": """You are assessing a chat bot response to a user's input based on the faithfulness of the response to the context retrieved.\n
+Score:
+A score of 1 means that the response contains statements that directly contradict the context or are entirely unsupported by it.
+A score of 3 means that the response contains some statements that are not directly supported by the context.
+A score of 5 means that the response is fully grounded in the context, with all statements accurately reflecting the provided information.\n
+Here is the data:
+[BEGIN DATA]
+***
+[User Query]: {{input}}
+***
+[Retrieved Context]: {{context}}
+*** 
+[Response]: {{response}}
+***
+[END DATA]""",
+        # Example from RAGTruth
+        "input": "How to prepare to get an ultrasound?",
+        "context": """Passage 1: 1 If you are just interested in checking the gender, you can go as early as 17 weeks. 2 Any earlier, and the results may not be accurate, leading to an extra coat of paint for your bedroom. 3 To get good pictures, 3D ultrasounds are best performed between 20 and 32 weeks, and the best pictures are between 24 and 30 weeks.""",
+        "response": """ Based on the given passages, here’s how to prepare for an ultrasound: For a gender reveal ultrasound, wait until at least 17 weeks of pregnancy to ensure accurate results. For 3D ultrasounds, schedule the appointment between 20 and 32 weeks of pregnancy for the best pictures""",
+    },
+}

gen_api_answer.py

@@ -11,6 +11,7 @@ together_client = Together()
 
 SYSTEM_PROMPT = """Please act as an impartial judge and evaluate based on the user's instruction. Your output format should strictly adhere to JSON as follows: {"feedback": "<write feedback>", "result": <numerical score>}. Ensure the output is valid JSON, without additional formatting or explanations."""
 
+
 def get_openai_response(model_name, prompt):
     """Get response from OpenAI API"""
     try:
@@ -18,13 +19,14 @@ def get_openai_response(model_name, prompt):
             model=model_name,
             messages=[
                 {"role": "system", "content": SYSTEM_PROMPT},
-                {"role": "user", "content": prompt}
-            ]
+                {"role": "user", "content": prompt},
+            ],
         )
         return response.choices[0].message.content
     except Exception as e:
         return f"Error with OpenAI model {model_name}: {str(e)}"
 
+
 def get_anthropic_response(model_name, prompt):
     """Get response from Anthropic API"""
     try:
@@ -33,14 +35,13 @@ def get_anthropic_response(model_name, prompt):
             max_tokens=1000,
             temperature=0,
             system=SYSTEM_PROMPT,
-            messages=[
-                {"role": "user", "content": [{"type": "text", "text": prompt}]}
-            ]
+            messages=[{"role": "user", "content": [{"type": "text", "text": prompt}]}],
         )
         return response.content[0].text
     except Exception as e:
         return f"Error with Anthropic model {model_name}: {str(e)}"
 
+
 def get_together_response(model_name, prompt):
     """Get response from Together API"""
     try:
@@ -48,52 +49,54 @@ def get_together_response(model_name, prompt):
             model=model_name,
             messages=[
                 {"role": "system", "content": SYSTEM_PROMPT},
-                {"role": "user", "content": prompt}
+                {"role": "user", "content": prompt},
             ],
-            stream=False
+            stream=False,
         )
         return response.choices[0].message.content
     except Exception as e:
         return f"Error with Together model {model_name}: {str(e)}"
 
+
 def get_model_response(model_name, model_info, prompt):
     """Get response from appropriate API based on model organization"""
     if not model_info:
         return "Model not found or unsupported."
-    
-    api_model = model_info['api_model']
-    organization = model_info['organization']
-    
+
+    api_model = model_info["api_model"]
+    organization = model_info["organization"]
+
     try:
-        if organization == 'OpenAI':
+        if organization == "OpenAI":
             return get_openai_response(api_model, prompt)
-        elif organization == 'Anthropic':
+        elif organization == "Anthropic":
             return get_anthropic_response(api_model, prompt)
         else:
             # All other organizations use Together API
             return get_together_response(api_model, prompt)
     except Exception as e:
-        return f"Error with {organization} model {model_name}: {str(e)}" 
+        return f"Error with {organization} model {model_name}: {str(e)}"
+
 
 def parse_model_response(response):
     try:
         # Debug print
         print(f"Raw model response: {response}")
-        
+
         # First try to parse the entire response as JSON
         try:
             data = json.loads(response)
-            return str(data.get('result', 'N/A')), data.get('feedback', 'N/A')
+            return str(data.get("result", "N/A")), data.get("feedback", "N/A")
         except json.JSONDecodeError:
             # If that fails (typically for smaller models), try to find JSON within the response
-            json_match = re.search(r'{.*}', response)
+            json_match = re.search(r"{.*}", response)
             if json_match:
                 data = json.loads(json_match.group(0))
-                return str(data.get('result', 'N/A')), data.get('feedback', 'N/A')
+                return str(data.get("result", "N/A")), data.get("feedback", "N/A")
             else:
-                return 'Error', f"Failed to parse response: {response}"
-                
+                return "Error", f"Failed to parse response: {response}"
+
     except Exception as e:
         # Debug print for error case
         print(f"Failed to parse response: {str(e)}")
-        return 'Error', f"Failed to parse response: {response}"
+        return "Error", f"Failed to parse response: {response}"

requirements.txt

@@ -1,5 +1,6 @@
-gradio>=4.19.2
-openai>=1.12.0
-anthropic>=0.18.1
-pandas>=2.2.1
-together>=0.2.8
+pymongo
+gradio
+python-dotenv
+openai
+anthropic
+together

utils.py

@@ -0,0 +1,27 @@
+from dataclasses import dataclass
+from datetime import datetime
+import logging
+
+
+def get_logger(sink_name: str = "core_utils") -> logging.Logger:
+    logging.basicConfig(
+        format="%(asctime)s,%(msecs)03d %(levelname)-8s "
+        "[%(filename)s:%(lineno)d] %(message)s",
+        datefmt="%Y-%m-%d:%H:%M:%S",
+        level=logging.INFO,
+        force=True,
+    )
+    logger = logging.getLogger(sink_name)
+    return logger
+
+
+@dataclass
+class Vote:
+    timestamp: str
+    prompt: str
+    response_a: str
+    response_b: str
+    model_a: str
+    model_b: str
+    winner: str
+    judge_id: str