update

.gitignore

@@ -1,4 +1,5 @@
 # Byte-compiled / optimized / DLL files
+_/
 __pycache__/
 *.py[cod]
 *$py.class

Makefile

@@ -0,0 +1,13 @@
+.PHONY: style format
+
+
+style:
+	python -m black --line-length 119 .
+	python -m isort .
+	ruff check --fix .
+
+
+quality:
+	python -m black --check --line-length 119 .
+	python -m isort --check-only .
+	ruff check .

README.md

@@ -1,11 +1,18 @@
 ---
-title: BigCodeBench Evaluator
-emoji: 💻
-colorFrom: indigo
+title: BigCodeBench Leaderboard
+emoji: 🥇
+colorFrom: green
 colorTo: indigo
-sdk: docker
+sdk: gradio
+sdk_version: 4.36.1
+app_file: app.py
+disable_embedding: true
 pinned: false
 license: apache-2.0
+tags:
+- leaderboard
+- eval:code
+- test:public
+- judge:auto
 ---
-
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+Paper:arxiv.org/abs/2406.15877

app.py

@@ -1,178 +1,649 @@
-import gradio as gr
-import subprocess
-import sys
 import os
-import threading
+import logging
 import time
-import uuid
-import glob
-import shutil
-from pathlib import Path
+import schedule
+import datetime
+import gradio as gr
+from threading import Thread
+import datasets
+from huggingface_hub import snapshot_download, WebhooksServer, WebhookPayload, RepoCard
+from gradio_leaderboard import Leaderboard, ColumnFilter, SelectColumns
 from apscheduler.schedulers.background import BackgroundScheduler
 
-default_command = "bigcodebench.evaluate"
-is_running = False
-lock = threading.Lock()
+# Start ephemeral Spaces on PRs (see config in README.md)
+from gradio_space_ci.webhook import IS_EPHEMERAL_SPACE, SPACE_ID, configure_space_ci
 
-def generate_command(
-    jsonl_file, split, subset, parallel,
-    min_time_limit, max_as_limit, max_data_limit, max_stack_limit,
-    check_gt_only, no_gt
-):
-    command = [default_command]
-    
-    if jsonl_file is not None:
-        # Copy the uploaded file to the current directory
-        local_filename = os.path.basename(jsonl_file.name)
-        shutil.copy(jsonl_file.name, local_filename)
-        command.extend(["--samples", local_filename])
-    
-    command.extend(["--split", split, "--subset", subset])
-    
-    if parallel is not None and parallel != 0:
-        command.extend(["--parallel", str(int(parallel))])
+from src.display.about import (
+    CITATION_BUTTON_LABEL,
+    CITATION_BUTTON_TEXT,
+    # INTRODUCTION_TEXT,
+    TITLE,
+    ABOUT_TEXT,
+    SUBMISSION_TEXT_3,
+)
+from src.display.css_html_js import custom_css
+from src.display.utils import (
+    COLS,
+    EVAL_COLS,
+    EVAL_TYPES,
+    AutoEvalColumn,
+    fields,
+    EvalQueueColumn
+)
+from src.envs import (
+    API,
+    EVAL_REQUESTS_PATH,
+    RESULT_REPO,
+    DATA_VERSION,
+    DATA_REPO,
+    HARD_RESULT_REPO,
+    ELO_REPO,
+    HARD_ELO_REPO,
+    SOLVE_REPO,
+    HARD_SOLVE_REPO,
+    HF_TOKEN,
+    QUEUE_REPO,
+    REPO_ID,
+    VOTES_REPO,
+    VOTES_PATH,
+    HF_HOME,
+)
+from src.populate import get_evaluation_queue_df, get_leaderboard_df
+from src.execute import generate_command, is_running, lock, stream_logs, find_result_file
+from src.tools.plots import plot_elo_mle, plot_solve_rate
+# from src.voting.vote_system import VoteManager, run_scheduler
+
+# Configure logging
+logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s")
+
+# Start ephemeral Spaces on PRs (see config in README.md)
+from gradio_space_ci.webhook import IS_EPHEMERAL_SPACE, SPACE_ID, configure_space_ci
+
+# Convert the environment variable "LEADERBOARD_FULL_INIT" to a boolean value, defaulting to True if the variable is not set.
+# This controls whether a full initialization should be performed.
+DO_FULL_INIT = True # os.getenv("LEADERBOARD_FULL_INIT", "True") == "True"
+NEW_DATA_ON_LEADERBOARD = True
+LEADERBOARD_DF = None
+HARD_LEADERBOARD_DF = None
+ELO_TASK_DF = None
+ELO_BENCH_DF = None
+HARD_ELO_TASK_DF = None
+HARD_ELO_BENCH_DF = None
+COMPLETE_SOLVE_DF = None
+INSTRUCT_SOLVE_DF = None
+HARD_COMPLETE_SOLVE_DF = None
+HARD_INSTRUCT_SOLVE_DF = None
+
+DATA = datasets.load_dataset(DATA_REPO, "default", cache_dir=HF_HOME, split=DATA_VERSION,
+                             verification_mode="no_checks")
+
+
+def filter_data(data, keyword):
+    if not keyword:
+        return data
+    filtered_data = [item for item in data if keyword.lower() in item['complete_prompt'].lower()]
+    return filtered_data
+
+
+def update_display(search_keyword, index, show_test):
+    filtered_data = filter_data(DATA, search_keyword)
     
-    command.extend([
-        "--min-time-limit", str(min_time_limit),
-        "--max-as-limit", str(int(max_as_limit)),
-        "--max-data-limit", str(int(max_data_limit)),
-        "--max-stack-limit", str(int(max_stack_limit))
-    ])
+    if not filtered_data:
+        return ["No data available. Check the search criteria."] + [""] * 4 + [0, gr.update(maximum=0, value=0)]
     
-    if check_gt_only:
-        command.append("--check-gt-only")
+    max_index = len(filtered_data) - 1
+    index = min(max(0, index), max_index)
     
-    if no_gt:
-        command.append("--no-gt")
+    task_id = filtered_data[index]['task_id']
+    snippet1 = filtered_data[index]['complete_prompt']
+    snippet2 = filtered_data[index]['instruct_prompt']
+    # snippet3 = filtered_data[index]['canonical_solution'] if show_solution else ""
+    snippet4 = filtered_data[index]['test'] if show_test else ""
     
-    return " ".join(command)
+    return [
+        task_id,
+        snippet1,
+        snippet2,
+        # snippet3,
+        snippet4,
+        len(filtered_data),
+        gr.update(maximum=max_index, value=index)
+    ]
 
+def restart_space():
+    API.restart_space(repo_id=REPO_ID, token=HF_TOKEN)
 
-def cleanup_previous_files(jsonl_file):
-    if jsonl_file is not None:
-        file_list = ['Dockerfile', 'app.py', 'README.md', os.path.basename(jsonl_file.name), "__pycache__"]
-    else:
-        file_list = ['Dockerfile', 'app.py', 'README.md', "__pycache__"]
-    for file in glob.glob("*"):
+
+def time_diff_wrapper(func):
+    def wrapper(*args, **kwargs):
+        start_time = time.time()
+        result = func(*args, **kwargs)
+        end_time = time.time()
+        diff = end_time - start_time
+        logging.info(f"Time taken for {func.__name__}: {diff} seconds")
+        return result
+
+    return wrapper
+
+
+@time_diff_wrapper
+def download_dataset(repo_id, local_dir, repo_type="dataset", max_attempts=3, backoff_factor=1.5):
+    """Download dataset with exponential backoff retries."""
+    attempt = 0
+    while attempt < max_attempts:
         try:
-            if file not in file_list:
-                os.remove(file)
-        except Exception as e:
-            print(f"Error during cleanup of {file}: {e}")
-
-def find_result_file():
-    json_files = glob.glob("*.json")
-    if json_files:
-        return max(json_files, key=os.path.getmtime)
-    return None
-
-def run_bigcodebench(command):
-    global is_running
-    with lock:
-        if is_running:
-            yield "A command is already running. Please wait for it to finish.\n"
+            logging.info(f"Downloading {repo_id} to {local_dir}")
+            snapshot_download(
+                repo_id=repo_id,
+                local_dir=local_dir,
+                repo_type=repo_type,
+                tqdm_class=None,
+                etag_timeout=30,
+                max_workers=8,
+            )
+            logging.info("Download successful")
             return
-        is_running = True
+        except Exception as e:
+            wait_time = backoff_factor**attempt
+            logging.error(f"Error downloading {repo_id}: {e}, retrying in {wait_time}s")
+            time.sleep(wait_time)
+            attempt += 1
+    raise Exception(f"Failed to download {repo_id} after {max_attempts} attempts")
 
-    try:
-        yield f"Executing command: {command}\n"
-        
-        process = subprocess.Popen(command.split(), stdout=subprocess.PIPE, stderr=subprocess.STDOUT, universal_newlines=True)
-        
-        for line in process.stdout:
-            yield line
+def get_latest_data_leaderboard(
+    leaderboard_initial_df = None,
+    hard_leaderboard_initial_df = None,
+    elo_task_df = None,
+    elo_bench_df = None,
+    hard_elo_task_df = None,
+    hard_elo_bench_df = None,
+    complete_solve_df = None,
+    instruct_solve_df = None,
+    hard_complete_solve_df = None,
+    hard_instruct_solve_df = None
+    ):
+    global NEW_DATA_ON_LEADERBOARD
+    global LEADERBOARD_DF
+    global HARD_LEADERBOARD_DF
+    global ELO_TASK_DF
+    global ELO_BENCH_DF
+    global HARD_ELO_TASK_DF
+    global HARD_ELO_BENCH_DF
+    global COMPLETE_SOLVE_DF
+    global INSTRUCT_SOLVE_DF
+    global HARD_COMPLETE_SOLVE_DF
+    global HARD_INSTRUCT_SOLVE_DF
+
+    if NEW_DATA_ON_LEADERBOARD:
+        print("Leaderboard updated at reload!")
+        leaderboard_dataset = datasets.load_dataset(
+            RESULT_REPO, 
+            "default", 
+            split="train", 
+            cache_dir=HF_HOME, 
+            download_mode=datasets.DownloadMode.REUSE_DATASET_IF_EXISTS, # Uses the cached dataset 
+            verification_mode="no_checks"
+        )
+        LEADERBOARD_DF = get_leaderboard_df(
+            leaderboard_dataset=leaderboard_dataset, 
+            cols=COLS,
+        )
+        hard_leaderboard_dataset = datasets.load_dataset(
+            HARD_RESULT_REPO, 
+            "default", 
+            split="train", 
+            cache_dir=HF_HOME, 
+            download_mode=datasets.DownloadMode.REUSE_DATASET_IF_EXISTS, # Uses the cached dataset 
+            verification_mode="no_checks"
+        )
+        hard_leaderboard_df = get_leaderboard_df(
+            leaderboard_dataset=hard_leaderboard_dataset, 
+            cols=COLS,
+        )
+        HARD_LEADERBOARD_DF = hard_leaderboard_df
         
-        # process.wait()
+        elo_task_df = datasets.load_dataset(
+            ELO_REPO,
+            "default", 
+            split="task_no_tie", 
+            cache_dir=HF_HOME, 
+            download_mode=datasets.DownloadMode.REUSE_DATASET_IF_EXISTS, # Uses the cached dataset 
+            verification_mode="no_checks"
+        ).to_pandas()
+        elo_bench_df = datasets.load_dataset(
+            ELO_REPO,
+            "default", 
+            split="benchmark_tie", 
+            cache_dir=HF_HOME, 
+            download_mode=datasets.DownloadMode.REUSE_DATASET_IF_EXISTS, # Uses the cached dataset 
+            verification_mode="no_checks"
+        ).to_pandas()
+        ELO_TASK_DF = elo_task_df
+        ELO_BENCH_DF = elo_bench_df
         
-        if process.returncode != 0:
-            yield f"Error: Command exited with status {process.returncode}\n"
+        hard_elo_task_df = datasets.load_dataset(
+            HARD_ELO_REPO,
+            "default", 
+            split="task_no_tie", 
+            cache_dir=HF_HOME, 
+            download_mode=datasets.DownloadMode.REUSE_DATASET_IF_EXISTS, # Uses the cached dataset 
+            verification_mode="no_checks"
+        ).to_pandas()
+        hard_elo_bench_df = datasets.load_dataset(
+            HARD_ELO_REPO,
+            "default", 
+            split="benchmark_tie", 
+            cache_dir=HF_HOME, 
+            download_mode=datasets.DownloadMode.REUSE_DATASET_IF_EXISTS, # Uses the cached dataset 
+            verification_mode="no_checks"
+        ).to_pandas()
+        HARD_ELO_TASK_DF = hard_elo_task_df
+        HARD_ELO_BENCH_DF = hard_elo_bench_df
         
-        yield "Evaluation completed.\n"
+        complete_solve_df = datasets.load_dataset(
+            SOLVE_REPO,
+            "default",
+            split="complete",
+            cache_dir=HF_HOME,
+            download_mode=datasets.DownloadMode.REUSE_DATASET_IF_EXISTS, # Uses the cached dataset
+            verification_mode="no_checks"
+        ).to_pandas()
+        instruct_solve_df = datasets.load_dataset(
+            SOLVE_REPO,
+            "default",
+            split="instruct",
+            cache_dir=HF_HOME,
+            download_mode=datasets.DownloadMode.REUSE_DATASET_IF_EXISTS, # Uses the cached dataset
+            verification_mode="no_checks"
+        ).to_pandas()
+        COMPLETE_SOLVE_DF = complete_solve_df
+        INSTRUCT_SOLVE_DF = instruct_solve_df
         
-        result_file = find_result_file()
-        if result_file:
-            yield f"Result file found: {result_file}\n"
-        else:
-            yield "No result file found.\n"
-    finally:
-        with lock:
-            is_running = False
-
-def stream_logs(command, jsonl_file=None):
-    global is_running
+        hard_complete_solve_df = datasets.load_dataset(
+            HARD_SOLVE_REPO,
+            "default",
+            split="complete",
+            cache_dir=HF_HOME,
+            download_mode=datasets.DownloadMode.REUSE_DATASET_IF_EXISTS, # Uses the cached dataset
+            verification_mode="no_checks"
+        ).to_pandas()
+        hard_instruct_solve_df = datasets.load_dataset(
+            HARD_SOLVE_REPO,
+            "default",
+            split="instruct",
+            cache_dir=HF_HOME,
+            download_mode=datasets.DownloadMode.REUSE_DATASET_IF_EXISTS, # Uses the cached dataset
+            verification_mode="no_checks"
+        ).to_pandas()        
+        HARD_COMPLETE_SOLVE_DF = hard_complete_solve_df
+        HARD_INSTRUCT_SOLVE_DF = hard_instruct_solve_df
         
-    if is_running:
-        yield "A command is already running. Please wait for it to finish.\n"
-        return
-    
-    cleanup_previous_files(jsonl_file)
-    yield "Cleaned up previous files.\n"
+        NEW_DATA_ON_LEADERBOARD = False
 
-    log_content = []
-    for log_line in run_bigcodebench(command):
-        log_content.append(log_line)
-        yield "".join(log_content)
+    else:
+        LEADERBOARD_DF = leaderboard_initial_df
+        # HARD_LEADERBOARD_DF = hard_leaderboard_initial_df
+        ELO_TASK_DF = elo_task_df
+        # ELO_BENCH_DF = elo_bench_df
+        # HARD_ELO_TASK_DF = hard_elo_task_df
+        HARD_ELO_BENCH_DF = hard_elo_bench_df
+        COMPLETE_SOLVE_DF = complete_solve_df
+        # INSTRUCT_SOLVE_DF = instruct_solve_df
+        # HARD_COMPLETE_SOLVE_DF = hard_complete_solve_df
+        HARD_INSTRUCT_SOLVE_DF = hard_instruct_solve_df
         
-with gr.Blocks() as demo:
-    gr.Markdown("# BigCodeBench Evaluator")
-    
-    with gr.Row():
-        jsonl_file = gr.File(label="Upload JSONL file", file_types=[".jsonl"])
-        split = gr.Dropdown(choices=["complete", "instruct"], label="Split", value="complete")
-        subset = gr.Dropdown(choices=["hard"], label="Subset", value="hard")
+    return (LEADERBOARD_DF, HARD_LEADERBOARD_DF, ELO_TASK_DF, ELO_BENCH_DF, HARD_ELO_TASK_DF, HARD_ELO_BENCH_DF, COMPLETE_SOLVE_DF, INSTRUCT_SOLVE_DF, HARD_COMPLETE_SOLVE_DF, HARD_INSTRUCT_SOLVE_DF)
+    # return (HARD_LEADERBOARD_DF, HARD_ELO_TASK_DF, HARD_ELO_BENCH_DF, HARD_COMPLETE_SOLVE_DF, HARD_INSTRUCT_SOLVE_DF)
+
+
+def init_space():
+    """Initializes the application space, loading only necessary data."""
+
+    # Always redownload the leaderboard DataFrame
+    global LEADERBOARD_DF
+    global HARD_LEADERBOARD_DF
+    global ELO_TASK_DF
+    global ELO_BENCH_DF
+    global HARD_ELO_TASK_DF
+    global HARD_ELO_BENCH_DF
+    global COMPLETE_SOLVE_DF
+    global INSTRUCT_SOLVE_DF
+    global HARD_COMPLETE_SOLVE_DF
+    global HARD_INSTRUCT_SOLVE_DF
     
-    with gr.Row():
-        parallel = gr.Number(label="Parallel (optional)", precision=0)
-        min_time_limit = gr.Number(label="Min Time Limit", value=1, precision=1)
-        max_as_limit = gr.Number(label="Max AS Limit", value=25*1024, precision=0)
+    LEADERBOARD_DF, HARD_LEADERBOARD_DF, ELO_TASK_DF, ELO_BENCH_DF, HARD_ELO_TASK_DF, HARD_ELO_BENCH_DF, COMPLETE_SOLVE_DF, INSTRUCT_SOLVE_DF, HARD_COMPLETE_SOLVE_DF, HARD_INSTRUCT_SOLVE_DF = get_latest_data_leaderboard()
+    # HARD_LEADERBOARD_DF, HARD_ELO_TASK_DF, HARD_ELO_BENCH_DF, HARD_COMPLETE_SOLVE_DF, HARD_INSTRUCT_SOLVE_DF = get_latest_data_leaderboard()
+
+    return (LEADERBOARD_DF, HARD_LEADERBOARD_DF, ELO_TASK_DF, ELO_BENCH_DF, HARD_ELO_TASK_DF, HARD_ELO_BENCH_DF, COMPLETE_SOLVE_DF, INSTRUCT_SOLVE_DF, HARD_COMPLETE_SOLVE_DF, HARD_INSTRUCT_SOLVE_DF)
+    # return (HARD_LEADERBOARD_DF, HARD_ELO_TASK_DF, HARD_ELO_BENCH_DF, HARD_COMPLETE_SOLVE_DF, HARD_INSTRUCT_SOLVE_DF)
+
+# Initialize VoteManager
+# vote_manager = VoteManager(VOTES_PATH, EVAL_REQUESTS_PATH, VOTES_REPO)
+
+
+# Schedule the upload_votes method to run every 15 minutes
+# schedule.every(15).minutes.do(vote_manager.upload_votes)
+
+# Start the scheduler in a separate thread
+# scheduler_thread = Thread(target=run_scheduler, args=(vote_manager,), daemon=True)
+# scheduler_thread.start()
+
+# Calls the init_space function with the `full_init` parameter determined by the `do_full_init` variable.
+# This initializes various DataFrames used throughout the application, with the level of initialization detail controlled by the `do_full_init` flag.
+LEADERBOARD_DF, HARD_LEADERBOARD_DF, ELO_TASK_DF, \
+ELO_BENCH_DF, HARD_ELO_TASK_DF, HARD_ELO_BENCH_DF, \
+COMPLETE_SOLVE_DF, INSTRUCT_SOLVE_DF, HARD_COMPLETE_SOLVE_DF, \
+HARD_INSTRUCT_SOLVE_DF = init_space()
+# HARD_LEADERBOARD_DF, HARD_ELO_TASK_DF, HARD_ELO_BENCH_DF, HARD_COMPLETE_SOLVE_DF, HARD_INSTRUCT_SOLVE_DF = init_space()
+
+# Data processing for plots now only on demand in the respective Gradio tab
+# def load_and_create_plots():
+#     plot_df = create_plot_df(create_scores_df(LEADERBOARD_DF))
+#     return plot_df
+
+# Function to check if a user is logged in
+def check_login(profile: gr.OAuthProfile | None) -> bool:
+    if profile is None:
+        return False
+    return True
+
+def init_leaderboard(dataframe):
+    if dataframe is None or dataframe.empty:
+        raise ValueError("Leaderboard DataFrame is empty or None.")
+    return Leaderboard(
+        value=dataframe,
+        datatype=[c.type for c in fields(AutoEvalColumn)],
+        select_columns=SelectColumns(
+            default_selection=[c.name for c in fields(AutoEvalColumn) if c.displayed_by_default],
+            cant_deselect=[c.name for c in fields(AutoEvalColumn) if c.never_hidden or c.dummy],
+            label="Select Columns to Display:",
+        ),
+        search_columns=[AutoEvalColumn.model.name],
+        hide_columns=[c.name for c in fields(AutoEvalColumn) if c.hidden],
+        filter_columns=[
+            ColumnFilter(AutoEvalColumn.type.name, type="checkboxgroup", label="Model Types"),
+            ColumnFilter(AutoEvalColumn.openness.name, type="checkboxgroup", label="Openness"),
+            ColumnFilter(AutoEvalColumn.size_range.name, type="dropdown", label="Model Size"),
+            ColumnFilter(AutoEvalColumn.moe.name, type="checkboxgroup", label="Model Architecture"),
+        ],
+        bool_checkboxgroup_label="Hide models",
+        interactive=False,
+        )
+
+
+def init_others(dataframe):
+    if dataframe is None or dataframe.empty:
+        raise ValueError("Gradio DataFrame is empty or None.")
+    return gr.Dataframe(dataframe, visible=False)
+
+main_block = gr.Blocks(css=custom_css)
+with main_block as demo:
+    with gr.Row(elem_id="header-row"):
+        gr.HTML(TITLE + "<p>Total models: " + str(len(HARD_LEADERBOARD_DF))+ "</p>")
     
-    with gr.Row():
-        max_data_limit = gr.Number(label="Max Data Limit", value=25*1024, precision=0)
-        max_stack_limit = gr.Number(label="Max Stack Limit", value=10, precision=0)
-        check_gt_only = gr.Checkbox(label="Check GT Only")
-        no_gt = gr.Checkbox(label="No GT")
+    # gr.Markdown(INTRODUCTION_TEXT, elem_classes="markdown-text")    
+    with gr.Tabs(elem_classes="tab-buttons") as tabs:
+        with gr.Tab("💎 Hard Set") as hard_tabs:
+            with gr.TabItem("🏅 Benchmark", elem_id="llm-benchmark-tab-table", id="hard_bench"):
+                hard_leaderboard = init_leaderboard(HARD_LEADERBOARD_DF)
+                gr.Markdown(
+                    """
+                **Notes:**
+                - For the efficiency reasons, we only display the Hard Set leaderboard.
+                - _Hard Set_ vs _Full Set_:
+                    - <u>Hard Set</u>: A subset of ~150 BigCodeBench tasks which is more user-facing and challenging.
+                    - <u>Full Set</u>: The full set of 1140 BigCodeBench tasks.
+                - _Complete_ vs _Instruct_:
+                    - <u>Complete</u>: Code Completion based on the (verbose) structured docstring. This split tests if the models are good at coding.
+                    - <u>Instruct</u> (🔥Vibe Check🔥): Code Generation based on the (less verbose) NL-oriented instructions. This split tests if the models are really capable enough to understand human intents to code.
+                - `Complete` and `Instruct` represent the calibrated Pass@1 score on the BigCodeBench benchmark splits.
+                - `Average` is the average of `Complete` and `Instruct` when both are available.
+                - `Elo Rating` represents the task-level Bootstrap of Maximum Likelihood Elo rating on the Complete + Instruct splits. The rating starts from 1000 and is bootstrapped 500 times. We only consider the models having both `Complete` and `Instruct` scores.
+                - `#Act Params (B)` is the number of activated model parameters during inference.
+                - Model providers have the responsibility to avoid data contamination. Models trained on close data can be affected by contamination.
+                - For more details check the 📝 About section.
+                """,
+                    elem_classes="markdown-text",
+                )
+            
+            with gr.TabItem("📊 Elo Rating", id="hard_elo"):
+                with gr.Column():
+                    with gr.Group():
+                        gr.Markdown("## (Task-level, No Tie, BigCodeBench-Complete) -- _Recommended_")
+                        hard_task_elo_map = gr.Plot()
+                        hard_elo_task_gr = init_others(HARD_ELO_TASK_DF)
+                        demo.load(plot_elo_mle, [hard_elo_task_gr],
+                                    hard_task_elo_map)
+                    with gr.Group():
+                        gr.Markdown("## (Benchmark-level, BigCodeBench-Complete)")
+                        hard_bench_elo_map = gr.Plot()
+                        hard_elo_bench_gr = init_others(HARD_ELO_BENCH_DF)
+                        demo.load(plot_elo_mle, [hard_elo_bench_gr],
+                                    hard_bench_elo_map)
+                        
+            with gr.TabItem("🧩 Solve Rate", id="hard_solve"):
+                with gr.Column():
+                    hard_complete_map = gr.Plot()
+                    hard_complete_solve_gr = init_others(HARD_COMPLETE_SOLVE_DF)
+                    demo.load(plot_solve_rate, [hard_complete_solve_gr,
+                                                gr.Textbox("Complete", visible=False),
+                                                gr.Number(10, visible=False),
+                                                gr.Number(16, visible=False),
+                                                ], hard_complete_map)
+                    hard_instruct_map = gr.Plot()
+                    hard_instruct_solve_gr = init_others(HARD_INSTRUCT_SOLVE_DF)
+                    demo.load(plot_solve_rate, [hard_instruct_solve_gr,
+                                                gr.Textbox("Instruct", visible=False),
+                                                gr.Number(10, visible=False),
+                                                gr.Number(16, visible=False),
+                                                ], hard_instruct_map)
+        with gr.Tab("🎯 Full Set") as full_tabs:
+            with gr.TabItem("🏅 Benchmark", elem_id="llm-benchmark-tab-table", id="full_bench"):
+                leaderboard = init_leaderboard(LEADERBOARD_DF)
+                gr.Markdown(
+                    """
+                **Notes:**
+                - _Complete_ vs _Instruct_:
+                    - <u>Complete</u>: Code Completion based on the (verbose) structured docstring. This variant tests if the models are good at coding.
+                    - <u>Instruct</u> (🔥Vibe Check🔥): Code Generation based on the (less verbose) NL-oriented instructions. This variant tests if the models are really capable enough to understand human intents to code.
+                - `complete` and `instruct` represent the calibrated Pass@1 score on the BigCodeBench benchmark variants.
+                - `elo_mle` represents the task-level Bootstrap of Maximum Likelihood Elo rating on the BigCodeBench-Complete split. The rating starts from 1000 and is bootstrapped 500 times.
+                - `size` is the amount of activated model weight during inference.
+                - Model providers have the responsibility to avoid data contamination. Models trained on close data can be affected by contamination.
+                - For more details check the 📝 About section.
+                """,
+                    elem_classes="markdown-text",
+                )
+            
+            with gr.TabItem("📊 Elo Rating", id="full_elo"):
+                with gr.Column():
+                    with gr.Group():
+                        
+                        gr.Markdown("## (Task-level, No Tie, BigCodeBench-Complete) -- _Recommended_")
+                        task_elo_map = gr.Plot()
+                        elo_task_gr = init_others(ELO_TASK_DF)
+                        demo.load(plot_elo_mle, [elo_task_gr], task_elo_map)
+                    with gr.Group():
+                        gr.Markdown("## (Benchmark-level, BigCodeBench-Complete)")
+                        bench_elo_map = gr.Plot()
+                        elo_bench_gr = init_others(ELO_BENCH_DF)
+                        demo.load(plot_elo_mle, [elo_bench_gr], bench_elo_map)
+                    
+            with gr.TabItem("🧩 Solve Rate", id="full_solve"):
+                with gr.Column():
+                    complete_map = gr.Plot()
+                    complete_solve_gr = init_others(COMPLETE_SOLVE_DF)
+                    demo.load(plot_solve_rate, [complete_solve_gr,
+                                                gr.Textbox("Complete", visible=False),
+                                                ], complete_map)
+                    instruct_map = gr.Plot()
+                    instruct_solve_gr = init_others(INSTRUCT_SOLVE_DF)
+                    demo.load(plot_solve_rate, [instruct_solve_gr,
+                                                gr.Textbox("Instruct", visible=False),
+                                                ], instruct_map)
+        with gr.TabItem("📝 About", id=3):
+            gr.Markdown(ABOUT_TEXT, elem_classes="markdown-text")
+        with gr.TabItem("🔎 Data Viewer", id="viewer"):
+            search_input = gr.Textbox(label="Search by keyword")
+            count_output = gr.Number(label="Number of filtered items")
+            index_slider = gr.Slider(minimum=0, maximum=len(DATA)-1, step=1, label="Select Index")
+            # show_solution = gr.Checkbox(label="Show Solution")
+            show_test = gr.Checkbox(label="Show Test Cases")
+            update_button = gr.Button("Update")
+            
+            task_id_output = gr.Textbox(label="Task ID")
+            code_completion = gr.Code(language="python", label="Code Completion")
+            nl_instruction = gr.Code(language="markdown", label="Natural Language Instruction")
+            # solution = gr.Code(language="python", label="Solution")
+            test_cases = gr.Code(language="python", label="Test Cases")
+            
+            update_button.click(
+                update_display, 
+                inputs=[search_input, index_slider, show_test],
+                outputs=[task_id_output, code_completion, nl_instruction, test_cases, count_output, index_slider]
+            )
+
+            # Initial load
+            demo.load(
+                update_display, 
+                inputs=[search_input, index_slider, show_test],
+                outputs=[task_id_output, code_completion, nl_instruction, test_cases, count_output, index_slider]
+            )
+
+        with gr.TabItem("🚀 Request", id=4):
+            gr.Markdown(SUBMISSION_TEXT_3)
+            
+        with gr.TabItem(" Execute", id=5):
+            gr.Markdown("# BigCodeBench Evaluator")
+            
+            with gr.Row():
+                jsonl_file = gr.File(label="Upload JSONL file", file_types=[".jsonl"])
+                split = gr.Dropdown(choices=["complete", "instruct"], label="Split", value="complete")
+                subset = gr.Dropdown(choices=["hard"], label="Subset", value="hard")
+            
+            with gr.Row():
+                parallel = gr.Number(label="Parallel (optional)", precision=0)
+                min_time_limit = gr.Number(label="Min Time Limit", value=1, precision=1)
+                max_as_limit = gr.Number(label="Max AS Limit", value=25*1024, precision=0)
+            
+            with gr.Row():
+                max_data_limit = gr.Number(label="Max Data Limit", value=25*1024, precision=0)
+                max_stack_limit = gr.Number(label="Max Stack Limit", value=10, precision=0)
+                check_gt_only = gr.Checkbox(label="Check GT Only")
+                no_gt = gr.Checkbox(label="No GT")
+            
+            command_output = gr.Textbox(label="Command", value=default_command, interactive=False)
+            with gr.Row():
+                submit_btn = gr.Button("Run Evaluation")
+                download_btn = gr.DownloadButton(label="Download Result")
+            log_output = gr.Textbox(label="Execution Logs", lines=20)
+            
+            input_components = [
+                jsonl_file, split, subset, parallel,
+                min_time_limit, max_as_limit, max_data_limit, max_stack_limit,
+                check_gt_only, no_gt
+            ]
+            
+            for component in input_components:
+                component.change(generate_command, inputs=input_components, outputs=command_output)
+                
+            
+            def start_evaluation(command, jsonl_file, subset, split):
+                extra = subset + "_" if subset != "full" else ""
+                if jsonl_file is not None:
+                    result_path = os.path.basename(jsonl_file.name).replace(".jsonl", f"_{extra}eval_results.json")
+                else:
+                    result_path = None
+
+                for log in stream_logs(command, jsonl_file):
+                    if jsonl_file is not None:
+                        yield log, gr.update(value=result_path, label=result_path), gr.update()
+                    else:
+                        yield log, gr.update(), gr.update()
+                is_running = False
+                result_file = find_result_file()
+                if result_file:
+                    return gr.update(label="Evaluation completed. Result file found."), gr.update(value=result_file)
+                            # gr.Button(visible=False)#,
+                            # gr.DownloadButton(label="Download Result", value=result_file, visible=True))
+                else:
+                    return gr.update(label="Evaluation completed. No result file found."), gr.update(value=result_path)
+                            # gr.Button("Run Evaluation", visible=True),
+                            # gr.DownloadButton(visible=False))
+            submit_btn.click(start_evaluation,
+                        inputs=[command_output, jsonl_file, subset, split],
+                        outputs=[log_output, download_btn])
     
-    command_output = gr.Textbox(label="Command", value=default_command, interactive=False)
     with gr.Row():
-        submit_btn = gr.Button("Run Evaluation")
-        download_btn = gr.DownloadButton(label="Download Result")
-    log_output = gr.Textbox(label="Execution Logs", lines=20)
-    
-    input_components = [
-        jsonl_file, split, subset, parallel,
-        min_time_limit, max_as_limit, max_data_limit, max_stack_limit,
-        check_gt_only, no_gt
-    ]
-    
-    for component in input_components:
-        component.change(generate_command, inputs=input_components, outputs=command_output)
+        with gr.Accordion("📙 Citation", open=False):
+            citation_button = gr.Textbox(
+                value=CITATION_BUTTON_TEXT,
+                label=CITATION_BUTTON_LABEL,
+                lines=20,
+                elem_id="citation-button",
+                show_copy_button=True,
+            )
+                    
+    main_block.load(fn=get_latest_data_leaderboard, inputs=[leaderboard, hard_leaderboard, elo_task_gr, elo_bench_gr, hard_elo_task_gr, hard_elo_bench_gr, complete_solve_gr, instruct_solve_gr, hard_complete_solve_gr, hard_instruct_solve_gr], outputs=[leaderboard, hard_leaderboard, elo_task_gr, elo_bench_gr, hard_elo_task_gr, hard_elo_bench_gr, complete_solve_gr, instruct_solve_gr, hard_complete_solve_gr, hard_instruct_solve_gr])
+    # main_block.load(fn=get_latest_data_leaderboard, inputs=[hard_leaderboard, hard_elo_task_gr, hard_elo_bench_gr, hard_complete_solve_gr, hard_instruct_solve_gr], outputs=[hard_leaderboard, hard_elo_task_gr, hard_elo_bench_gr, hard_complete_solve_gr, hard_instruct_solve_gr])
+    # leaderboard.change(fn=get_latest_data_queue, inputs=None, outputs=[finished_eval_table, running_eval_table, pending_eval_table])
+    # pending_eval_table.change(fn=vote_manager.create_request_vote_df, inputs=[pending_eval_table], outputs=[pending_eval_table_votes])
+
+main_block.queue(default_concurrency_limit=100)
+
+
+def enable_space_ci_and_return_server(ui: gr.Blocks) -> WebhooksServer:
+    # Taken from https://huggingface.co/spaces/Wauplin/gradio-space-ci/blob/075119aee75ab5e7150bf0814eec91c83482e790/src/gradio_space_ci/webhook.py#L61
+    # Compared to original, this one do not monkeypatch Gradio which allows us to define more webhooks.
+    # ht to Lucain!
+    if SPACE_ID is None:
+        print("Not in a Space: Space CI disabled.")
+        return WebhooksServer(ui=main_block)
+
+    if IS_EPHEMERAL_SPACE:
+        print("In an ephemeral Space: Space CI disabled.")
+        return WebhooksServer(ui=main_block)
+
+    card = RepoCard.load(repo_id_or_path=SPACE_ID, repo_type="space")
+    config = card.data.get("space_ci", {})
+    print(f"Enabling Space CI with config from README: {config}")
+
+    return configure_space_ci(
+        blocks=ui,
+        trusted_authors=config.get("trusted_authors"),
+        private=config.get("private", "auto"),
+        variables=config.get("variables", "auto"),
+        secrets=config.get("secrets"),
+        hardware=config.get("hardware"),
+        storage=config.get("storage"),
+    )
+
+# Create webhooks server (with CI url if in Space and not ephemeral)
+webhooks_server = enable_space_ci_and_return_server(ui=main_block)
+
+# Add webhooks
+@webhooks_server.add_webhook
+def update_leaderboard(payload: WebhookPayload) -> None:
+    """Redownloads the leaderboard dataset each time it updates"""
+    if payload.repo.type == "dataset" and payload.event.action == "update":
+        global NEW_DATA_ON_LEADERBOARD
+        if NEW_DATA_ON_LEADERBOARD:
+            return
+        NEW_DATA_ON_LEADERBOARD = True
+
+        for repo in [RESULT_REPO, HARD_RESULT_REPO, ELO_REPO, HARD_ELO_REPO, SOLVE_REPO, HARD_SOLVE_REPO]:
+            datasets.load_dataset(
+                repo, 
+                "default", 
+                cache_dir=HF_HOME, 
+                download_mode=datasets.DownloadMode.FORCE_REDOWNLOAD, 
+                verification_mode="no_checks"
+            )
         
-    
-    def start_evaluation(command, jsonl_file, subset, split):
-        extra = subset + "_" if subset != "full" else ""
-        if jsonl_file is not None:
-            result_path = os.path.basename(jsonl_file.name).replace(".jsonl", f"_{extra}eval_results.json")
-        else:
-            result_path = None
-
-        for log in stream_logs(command, jsonl_file):
-            if jsonl_file is not None:
-                yield log, gr.update(value=result_path, label=result_path), gr.update()
-            else:
-                yield log, gr.update(), gr.update()
-        is_running = False
-        result_file = find_result_file()
-        if result_file:
-            return gr.update(label="Evaluation completed. Result file found."), gr.update(value=result_file)
-                    # gr.Button(visible=False)#,
-                    # gr.DownloadButton(label="Download Result", value=result_file, visible=True))
-        else:
-            return gr.update(label="Evaluation completed. No result file found."), gr.update(value=result_path)
-                    # gr.Button("Run Evaluation", visible=True),
-                    # gr.DownloadButton(visible=False))
-    submit_btn.click(start_evaluation,
-                 inputs=[command_output, jsonl_file, subset, split],
-                 outputs=[log_output, download_btn])
-
-demo.queue(max_size=300).launch(share=True, server_name="0.0.0.0", server_port=7860)
-scheduler = BackgroundScheduler()
+        
+
+webhooks_server.launch()
+
+scheduler = BackgroundScheduler()
+scheduler.add_job(restart_space, "interval", hours=5) # restarted every 3h as backup in case automatic updates are not working
+scheduler.start()

demo.py

@@ -0,0 +1,178 @@
+import gradio as gr
+import subprocess
+import sys
+import os
+import threading
+import time
+import uuid
+import glob
+import shutil
+from pathlib import Path
+from apscheduler.schedulers.background import BackgroundScheduler
+
+default_command = "bigcodebench.evaluate"
+is_running = False
+lock = threading.Lock()
+
+def generate_command(
+    jsonl_file, split, subset, parallel,
+    min_time_limit, max_as_limit, max_data_limit, max_stack_limit,
+    check_gt_only, no_gt
+):
+    command = [default_command]
+    
+    if jsonl_file is not None:
+        # Copy the uploaded file to the current directory
+        local_filename = os.path.basename(jsonl_file.name)
+        shutil.copy(jsonl_file.name, local_filename)
+        command.extend(["--samples", local_filename])
+    
+    command.extend(["--split", split, "--subset", subset])
+    
+    if parallel is not None and parallel != 0:
+        command.extend(["--parallel", str(int(parallel))])
+    
+    command.extend([
+        "--min-time-limit", str(min_time_limit),
+        "--max-as-limit", str(int(max_as_limit)),
+        "--max-data-limit", str(int(max_data_limit)),
+        "--max-stack-limit", str(int(max_stack_limit))
+    ])
+    
+    if check_gt_only:
+        command.append("--check-gt-only")
+    
+    if no_gt:
+        command.append("--no-gt")
+    
+    return " ".join(command)
+
+
+def cleanup_previous_files(jsonl_file):
+    if jsonl_file is not None:
+        file_list = ['Dockerfile', 'app.py', 'README.md', os.path.basename(jsonl_file.name), "__pycache__"]
+    else:
+        file_list = ['Dockerfile', 'app.py', 'README.md', "__pycache__"]
+    for file in glob.glob("*"):
+        try:
+            if file not in file_list:
+                os.remove(file)
+        except Exception as e:
+            print(f"Error during cleanup of {file}: {e}")
+
+def find_result_file():
+    json_files = glob.glob("*.json")
+    if json_files:
+        return max(json_files, key=os.path.getmtime)
+    return None
+
+def run_bigcodebench(command):
+    global is_running
+    with lock:
+        if is_running:
+            yield "A command is already running. Please wait for it to finish.\n"
+            return
+        is_running = True
+
+    try:
+        yield f"Executing command: {command}\n"
+        
+        process = subprocess.Popen(command.split(), stdout=subprocess.PIPE, stderr=subprocess.STDOUT, universal_newlines=True)
+        
+        for line in process.stdout:
+            yield line
+        
+        # process.wait()
+        
+        if process.returncode != 0:
+            yield f"Error: Command exited with status {process.returncode}\n"
+        
+        yield "Evaluation completed.\n"
+        
+        result_file = find_result_file()
+        if result_file:
+            yield f"Result file found: {result_file}\n"
+        else:
+            yield "No result file found.\n"
+    finally:
+        with lock:
+            is_running = False
+
+def stream_logs(command, jsonl_file=None):
+    global is_running
+        
+    if is_running:
+        yield "A command is already running. Please wait for it to finish.\n"
+        return
+    
+    cleanup_previous_files(jsonl_file)
+    yield "Cleaned up previous files.\n"
+
+    log_content = []
+    for log_line in run_bigcodebench(command):
+        log_content.append(log_line)
+        yield "".join(log_content)
+        
+with gr.Blocks() as demo:
+    gr.Markdown("# BigCodeBench Evaluator")
+    
+    with gr.Row():
+        jsonl_file = gr.File(label="Upload JSONL file", file_types=[".jsonl"])
+        split = gr.Dropdown(choices=["complete", "instruct"], label="Split", value="complete")
+        subset = gr.Dropdown(choices=["hard"], label="Subset", value="hard")
+    
+    with gr.Row():
+        parallel = gr.Number(label="Parallel (optional)", precision=0)
+        min_time_limit = gr.Number(label="Min Time Limit", value=1, precision=1)
+        max_as_limit = gr.Number(label="Max AS Limit", value=25*1024, precision=0)
+    
+    with gr.Row():
+        max_data_limit = gr.Number(label="Max Data Limit", value=25*1024, precision=0)
+        max_stack_limit = gr.Number(label="Max Stack Limit", value=10, precision=0)
+        check_gt_only = gr.Checkbox(label="Check GT Only")
+        no_gt = gr.Checkbox(label="No GT")
+    
+    command_output = gr.Textbox(label="Command", value=default_command, interactive=False)
+    with gr.Row():
+        submit_btn = gr.Button("Run Evaluation")
+        download_btn = gr.DownloadButton(label="Download Result")
+    log_output = gr.Textbox(label="Execution Logs", lines=20)
+    
+    input_components = [
+        jsonl_file, split, subset, parallel,
+        min_time_limit, max_as_limit, max_data_limit, max_stack_limit,
+        check_gt_only, no_gt
+    ]
+    
+    for component in input_components:
+        component.change(generate_command, inputs=input_components, outputs=command_output)
+        
+    
+    def start_evaluation(command, jsonl_file, subset, split):
+        extra = subset + "_" if subset != "full" else ""
+        if jsonl_file is not None:
+            result_path = os.path.basename(jsonl_file.name).replace(".jsonl", f"_{extra}eval_results.json")
+        else:
+            result_path = None
+
+        for log in stream_logs(command, jsonl_file):
+            if jsonl_file is not None:
+                yield log, gr.update(value=result_path, label=result_path), gr.update()
+            else:
+                yield log, gr.update(), gr.update()
+        is_running = False
+        result_file = find_result_file()
+        if result_file:
+            return gr.update(label="Evaluation completed. Result file found."), gr.update(value=result_file)
+                    # gr.Button(visible=False)#,
+                    # gr.DownloadButton(label="Download Result", value=result_file, visible=True))
+        else:
+            return gr.update(label="Evaluation completed. No result file found."), gr.update(value=result_path)
+                    # gr.Button("Run Evaluation", visible=True),
+                    # gr.DownloadButton(visible=False))
+    submit_btn.click(start_evaluation,
+                 inputs=[command_output, jsonl_file, subset, split],
+                 outputs=[log_output, download_btn])
+
+demo.queue(max_size=300).launch(share=True, server_name="0.0.0.0", server_port=7860)
+scheduler = BackgroundScheduler()

pyproject.toml

@@ -0,0 +1,13 @@
+[tool.ruff]
+# Enable pycodestyle (`E`) and Pyflakes (`F`) codes by default.
+select = ["E", "F"]
+ignore = ["E501"] # line too long (black is taking care of this)
+line-length = 119
+fixable = ["A", "B", "C", "D", "E", "F", "G", "I", "N", "Q", "S", "T", "W", "ANN", "ARG", "BLE", "COM", "DJ", "DTZ", "EM", "ERA", "EXE", "FBT", "ICN", "INP", "ISC", "NPY", "PD", "PGH", "PIE", "PL", "PT", "PTH", "PYI", "RET", "RSE", "RUF", "SIM", "SLF", "TCH", "TID", "TRY", "UP", "YTT"]
+
+[tool.isort]
+profile = "black"
+line_length = 119
+
+[tool.black]
+line-length = 119

src/display/about.py

@@ -0,0 +1,148 @@
+TITLE = """<div style="text-align: center;"><h1> 🌸<span style='color: #C867B5;'>BigCodeBench</span> Leaderboard</h1></div>\
+            <br>\
+            <p>Inspired from the <a href="https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard">🤗 Open LLM Leaderboard</a> and <a href="https://huggingface.co/spaces/bigcode/bigcode-models-leaderboard">⭐ Big Code Models Leaderboard</a>, we compare performance of LLMs on <a href="https://huggingface.co/datasets/bigcode/bigcodebench">BigCodeBench</a> benchmark.</p>
+            <p>To get started, please check out <a href="https://github.com/bigcode-project/bigcodebench">our GitHub repository</a>.
+            <br>\
+            For more details, please check our <a href="https://huggingface.co/blog/terryyz/bigcodebench-hard">blog on the Hard Set</a>, <a href="https://huggingface.co/blog/leaderboard-bigcodebench">blog on the Full Set</a> and <a href="https://arxiv.org/abs/2406.15877">paper</a>.</p>
+            """
+
+ABOUT_TEXT = """# Context
+We believe that there are three main expectations of a good execution-based programming benchmark:
+1. The benchmark should be easy to use and efficient in evaluating the fundamental capabilities of LLMs. Repo-level and agent-centric benchmarks (e.g., SWE-bench) are not suitable for this purpose.
+2. The benchmark should be practical, covering various programming scenarios. Algo-specific benchmarks (e.g., HumanEval and MBPP) are unsuitable. Domain-specific benchmarks (e.g., DS-1000) are also unsuitable for this purpose.
+3. The benchmark should be challenging, where the tasks require LLMs' strong compositional reasoning capabilities and instruction-following capabilities. The benchmarks with simple tasks (e.g., ODEX) are unsuitable.
+
+BigCodeBench is the first benchmark that meets all three expectations. It is an <u>*__easy-to-use__*</u> benchmark that evaluates LLMs with <u>*__practical__*</u> and <u>*__challenging__*</u> programming tasks, accompanied by an end-to-end evaluation framework [`bigcodebench`](https://github.com/bigcode-project/bigcodebench). We aim to assess how well LLMs can solve programming tasks in an open-ended setting, with the following two focuses:
+
+- Diverse Function Calls: This design requires LLMs to utilize diverse function calls.
+- Complex Instructions: This design requires LLMs to follow complex instructions.
+
+
+### Benchamrks & Prompts
+The dataset has 2 variants: 
+1. `BigCodeBench-Complete`: _Code Completion based on the structured long-context docstrings_.
+1. `BigCodeBench-Instruct`: _Code Generation based on the NL-oriented instructions_.
+
+Figure below shows the example of `Complete` vs `Instruct` prompt. For `Instruct`, we only focus on instruction-tuned LLMs.
+
+<img src="https://github.com/bigcode-bench/bigcode-bench.github.io/blob/main/asset/bigcodebench_prompt.svg?raw=true" alt="OctoCoder vs Base HumanEval prompt" width="800px">
+
+The specific prompt template can be found [here](https://github.com/bigcode-project/bigcodebench/blob/main/bigcodebench/model.py).
+
+There are some edge cases:
+- Due to the training flaws in StarCoder2 and Granite-Code, we additionally strip the trailing newlines for model inference.
+- We have not included the `Instruct` results of Granite-Code-Instruct 8B & 3B as they constantly have empty outputs.
+
+### Evaluation Parameters
+- All models were evaluated with the [bigcodebench](https://github.com/bigcode-project/bigcodebench). You can install the [PyPI package](https://pypi.org/project/bigcodebench/).
+To get started, please first set up the environment:
+
+```bash
+# Install to use bigcodebench.evaluate
+pip install bigcodebench --upgrade
+# If you want to use the evaluate locally, you need to install the requirements
+pip install -I -r https://raw.githubusercontent.com/bigcode-project/bigcodebench/main/Requirements/requirements-eval.txt
+
+# Install to use bigcodebench.generate
+# You are strongly recommended to install the generate dependencies in a separate environment
+pip install bigcodebench[generate] --upgrade
+```
+
+### Scoring and Rankings
+- Models are ranked according to Pass@1 using greedy decoding. Setup details can be found <a href="https://github.com/bigcode-project/bigcodebench/blob/main/bigcodebench/generate.py">here</a>.
+- The code to compute Elo rating is [here](https://github.com/bigcode-project/bigcodebench/blob/main/analysis/get_results.py), which is based on [Chatbot Arena Notebook](https://colab.research.google.com/drive/1RAWb22-PFNI-X1gPVzc927SGUdfr6nsR#scrollTo=JdiJbB6pZB1B&line=2&uniqifier=1). We only compute the Elo rating for the `BigCodeBench-Complete` variant.
+
+### Contact
+If you have any questions, feel free to reach out to us at [[email protected]](mailto:[email protected]) or [[email protected]](mailto:[email protected])
+
+### Citation Information
+
+```bibtex
+@article{zhuo2024bigcodebench,
+    title={BigCodeBench: Benchmarking Code Generation with Diverse Function Calls and Complex Instructions}, 
+    author={Terry Yue Zhuo and Minh Chien Vu and Jenny Chim and Han Hu and Wenhao Yu and Ratnadira Widyasari and Imam Nur Bani Yusuf and Haolan Zhan and Junda He and Indraneil Paul and Simon Brunner and Chen Gong and Thong Hoang and Armel Randy Zebaze and Xiaoheng Hong and Wen-Ding Li and Jean Kaddour and Ming Xu and Zhihan Zhang and Prateek Yadav and Naman Jain and Alex Gu and Zhoujun Cheng and Jiawei Liu and Qian Liu and Zijian Wang and David Lo and Binyuan Hui and Niklas Muennighoff and Daniel Fried and Xiaoning Du and Harm de Vries and Leandro Von Werra},
+    journal={arXiv preprint arXiv:2406.15877},
+    year={2024}
+}
+```
+"""
+
+SUBMISSION_TEXT = """
+<h1 align="center">
+How to submit models/results to the leaderboard?
+</h1>
+We welcome the community to submit evaluation results of new models. We also provide an experimental feature for submitting models that our team will evaluate on the 🤗 cluster.
+
+## Submitting Models (experimental feature)
+Inspired from the Open LLM Leaderboard, we welcome code models submission from the community that will be automatically evaluated. Please note that this is still an experimental feature.
+Below are some guidlines to follow before submitting your model:
+
+#### 1) Make sure you can load your model and tokenizer using AutoClasses:
+```python
+from transformers import AutoConfig, AutoModel, AutoTokenizer
+config = AutoConfig.from_pretrained("your model name", revision=revision)
+model = AutoModel.from_pretrained("your model name", revision=revision)
+tokenizer = AutoTokenizer.from_pretrained("your model name", revision=revision)
+```
+If this step fails, follow the error messages to debug your model before submitting it. It's likely your model has been improperly uploaded.
+Note: make sure your model is public!
+Note: if your model needs `use_remote_code=True`, we do not support this option yet.
+#### 2) Convert your model weights to [safetensors](https://huggingface.co/docs/safetensors/index)
+It's a new format for storing weights which is safer and faster to load and use. It will also allow us to add the number of parameters of your model to the `Extended Viewer`!
+#### 3) Make sure your model has an open license!
+This is a leaderboard for Open LLMs, and we'd love for as many people as possible to know they can use your model 🤗
+#### 4) Fill up your model card
+When we add extra information about models to the leaderboard, it will be automatically taken from the model card.
+"""
+
+SUBMISSION_TEXT_2 = """
+## Sumbitting Results
+You also have the option for running evaluation yourself and submitting results. These results will be added as non-verified, the authors are however required to upload their generations in case other members want to check.
+
+### 1 - Running Evaluation
+
+We wrote a detailed guide for running the evaluation on your model. You can find the it in [bigcode-evaluation-harness/leaderboard](https://github.com/bigcode-project/bigcode-evaluation-harness/tree/main/leaderboard). This will generate a json file summarizing the results, in addition to the raw generations and metric files.
+
+### 2- Submitting Results 🚀
+
+To submit your results create a **Pull Request** in the community tab to add them under the [folder](https://huggingface.co/spaces/bigcode/bigcodebench-code-evals/tree/main/community_results) `community_results` in this repository:
+- Create a folder called `ORG_MODELNAME_USERNAME` for example `bigcode_my_model_terry`
+- Put your json file with grouped scores from the guide, in addition generations folder and metrics folder in it.
+
+The title of the PR should be `[Community Submission] Model: org/model, Username: your_username`, replace org and model with those corresponding to the model you evaluated.
+"""
+
+SUBMISSION_TEXT_3 = """
+<h1 align="center">
+How to submit models/results to the leaderboard?
+</h1>
+We welcome the community to submit evaluation results of new models. These results will be added as non-verified, the authors are however required to upload their generations in case other members want to check.
+
+### 1 - Running Evaluation
+
+We wrote a detailed guide for running the evaluation on your model. You can find the it in [bigcode-evaluation-harness/leaderboard](https://github.com/bigcode-project/bigcode-evaluation-harness/tree/main/leaderboard). This will generate a json file summarizing the results, in addition to the raw generations and metric files.
+
+### 2- Submitting Results 🚀
+
+To submit your results create a **Pull Request** in the community tab to add them under the [folder](https://huggingface.co/spaces/bigcode/multilingual-code-evals/tree/main/community_results) `community_results` in this repository:
+- Create a folder called `ORG_MODELNAME_USERNAME` for example `bigcode_starcoder_loubnabnl`
+- Put your json file with grouped scores from the guide, in addition generations folder and metrics folder in it.
+
+The title of the PR should be `[Community Submission] Model: org/model, Username: your_username`, replace org and model with those corresponding to the model you evaluated.
+"""
+
+CITATION_BUTTON_LABEL = "Copy the following snippet to cite these results"
+
+CITATION_BUTTON_TEXT = r"""
+@article{zhuo2024bigcodebench,
+  title={BigCodeBench: Benchmarking Code Generation with Diverse Function Calls and Complex Instructions},
+  author={Zhuo, Terry Yue and Vu, Minh Chien and Chim, Jenny and Hu, Han and Yu, Wenhao and Widyasari, Ratnadira and Yusuf, Imam Nur Bani and Zhan, Haolan and He, Junda and Paul, Indraneil and others},
+  journal={arXiv preprint arXiv:2406.15877},
+  year={2024}
+}
+"""
+
+SUBMISSION_TEXT_3="""
+## We welcome the community to request for new models to be added to the leaderboard.
+## Please [file an issue](https://github.com/bigcode-project/bigcodebench/issues/new/choose) to add the model to the leaderboard or [start a discussion](https://huggingface.co/spaces/bigcode/bigcodebench-leaderboard/discussions/new) in the community🤗
+"""

src/display/css_html_js.py

@@ -0,0 +1,115 @@
+custom_css = """
+/* Limit the width of the first AutoEvalColumn so that names don't expand too much */
+table td:first-child,
+table th:first-child {
+    max-width: 400px;
+    overflow: auto;
+    white-space: nowrap;
+}
+
+/* Full width space */
+.gradio-container {
+    max-width: 95% !important;
+}
+
+/* Text style and margins */
+.markdown-text {
+    font-size: 16px !important;
+}
+
+#models-to-add-text {
+    font-size: 18px !important;
+}
+
+#citation-button span {
+    font-size: 16px !important;
+}
+
+#citation-button textarea {
+    font-size: 16px !important;
+}
+
+#citation-button > label > button {
+    margin: 6px;
+    transform: scale(1.3);
+}
+
+#search-bar-table-box > div:first-child {
+    background: none;
+    border: none;
+}
+
+#search-bar {
+    padding: 0px;
+}
+
+.tab-buttons button {
+    font-size: 20px;
+}
+
+/* Filters style */
+#filter_type {
+    border: 0;
+    padding-left: 0;
+    padding-top: 0;
+}
+#filter_type label {
+    display: flex;
+}
+#filter_type label > span {
+    margin-top: var(--spacing-lg);
+    margin-right: 0.5em;
+}
+#filter_type label > .wrap {
+    width: 103px;
+}
+#filter_type label > .wrap .wrap-inner {
+    padding: 2px;
+}
+#filter_type label > .wrap .wrap-inner input {
+    width: 1px;
+}
+#filter-columns-type {
+    border: 0;
+    padding: 0.5;
+}
+#filter-columns-size {
+    border: 0;
+    padding: 0.5;
+}
+#box-filter > .form {
+    border: 0;
+}
+
+/* Header styles */
+#header-title {
+    text-align: left;
+    display: inline-block;
+}
+
+#header-row {
+    display: flex;
+    justify-content: space-between;
+    align-items: center;
+}
+
+#header-row .gradio-html {
+    flex-grow: 1;
+}
+
+#oauth-button {
+    height: auto;
+    min-width: max-content;
+    white-space: nowrap;
+    padding: 10px 20px;
+    border-radius: 4px;
+}
+"""
+
+get_window_url_params = """
+    function(url_params) {
+        const params = new URLSearchParams(window.location.search);
+        url_params = Object.fromEntries(params);
+        return url_params;
+    }
+    """

src/display/formatting.py

@@ -0,0 +1,37 @@
+from huggingface_hub import HfApi
+
+API = HfApi()
+
+
+def model_hyperlink(link, model_name):
+    return f'<a target="_blank" href="{link}" style="color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;">{model_name}</a>'
+
+
+def make_clickable_model(df, model_col, link_col):
+    df[model_col] = df.apply(
+        lambda row: model_hyperlink(row[link_col], row[model_col]), axis=1
+    )
+    df["Openness"] = df.apply(
+        lambda row: "Open" if "huggingface.co" in row[link_col] else "Closed", axis=1
+    )
+    return df
+
+
+def styled_error(error):
+    return f"<p style='color: red; font-size: 20px; text-align: center;'>{error}</p>"
+
+
+def styled_warning(warn):
+    return f"<p style='color: orange; font-size: 20px; text-align: center;'>{warn}</p>"
+
+
+def styled_message(message):
+    return f"<p style='color: green; font-size: 20px; text-align: center;'>{message}</p>"
+
+
+def has_no_nan_values(df, columns):
+    return df[columns].notna().all(axis=1)
+
+
+def has_nan_values(df, columns):
+    return df[columns].isna().any(axis=1)

src/display/utils.py

@@ -0,0 +1,142 @@
+from dataclasses import dataclass, make_dataclass
+from enum import Enum
+import json
+import logging
+from datetime import datetime
+import pandas as pd
+
+
+# Configure logging
+logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s")
+
+# Convert ISO 8601 dates to datetime objects for comparison
+def parse_iso8601_datetime(date_str):
+    if date_str.endswith('Z'):
+        date_str = date_str[:-1] + '+00:00'
+    return datetime.fromisoformat(date_str)
+
+def parse_datetime(datetime_str):
+    formats = [
+        "%Y-%m-%dT%H-%M-%S.%f",  # Format with dashes
+        "%Y-%m-%dT%H:%M:%S.%f",  # Standard format with colons
+        "%Y-%m-%dT%H %M %S.%f",  # Spaces as separator
+    ]
+
+    for fmt in formats:
+        try:
+            return datetime.strptime(datetime_str, fmt)
+        except ValueError:
+            continue
+    # in rare cases set unix start time for files with incorrect time (legacy files)
+    logging.error(f"No valid date format found for: {datetime_str}")
+    return datetime(1970, 1, 1)
+
+
+def load_json_data(file_path):
+    """Safely load JSON data from a file."""
+    try:
+        with open(file_path, "r") as file:
+            return json.load(file)
+    except json.JSONDecodeError:
+        print(f"Error reading JSON from {file_path}")
+        return None  # Or raise an exception
+
+
+def fields(raw_class):
+    return [v for k, v in raw_class.__dict__.items() if k[:2] != "__" and k[-2:] != "__"]
+
+
+column_map = {
+    "T": "T",
+    "model": "Model",
+    "type": "Model Type",
+    "size_range": "Size Range",
+    "complete": "Complete",
+    "instruct": "Instruct",
+    "average": "Average",
+    "elo_mle": "Elo Rating",
+    "link": "Link",
+    "act_param": "#Act Params (B)",
+    "size": "#Params (B)",
+    "moe": "MoE",
+    # "lazy": "Lazy",
+    "openness": "Openness",
+    # "direct_complete": "Direct Completion",
+}
+
+type_map = {
+    "🔶": "🔶 Chat Models (RLHF, DPO, IFT, ...)",
+    "🟢": "🟢 Base Models"
+}
+
+moe_map = {
+    True: "MoE",
+    False: "Dense"
+}
+# These classes are for user facing column names,
+# to avoid having to change them all around the code
+# when a modif is needed
+@dataclass(frozen=True)
+class ColumnContent:
+    name: str
+    type: str
+    displayed_by_default: bool
+    hidden: bool = False
+    never_hidden: bool = False
+    dummy: bool = False
+
+
+auto_eval_column_dict = []
+# Init
+auto_eval_column_dict.append(["T", ColumnContent, ColumnContent(column_map["T"], "str", True, never_hidden=True)])
+auto_eval_column_dict.append(["model", ColumnContent, ColumnContent(column_map["model"], "markdown", True, never_hidden=True)])
+auto_eval_column_dict.append(["type", ColumnContent, ColumnContent(column_map["type"], "str", False, True)])
+auto_eval_column_dict.append(["size_range", ColumnContent, ColumnContent(column_map["size_range"], "str", False, True)])
+# Scores
+auto_eval_column_dict.append(["complete", ColumnContent, ColumnContent(column_map["complete"], "number", True)])
+auto_eval_column_dict.append(["instruct", ColumnContent, ColumnContent(column_map["instruct"], "number", True)])
+auto_eval_column_dict.append(["average", ColumnContent, ColumnContent(column_map["average"], "number", True)])
+auto_eval_column_dict.append(["elo_mle", ColumnContent, ColumnContent(column_map["elo_mle"], "number", True)])
+
+# Model information
+auto_eval_column_dict.append(["act_param", ColumnContent, ColumnContent(column_map["act_param"], "number", True)])
+auto_eval_column_dict.append(["link", ColumnContent, ColumnContent(column_map["link"], "str", False, True)])
+auto_eval_column_dict.append(["size", ColumnContent, ColumnContent(column_map["size"], "number", False)])
+# auto_eval_column_dict.append(["lazy", ColumnContent, ColumnContent(column_map["lazy"], "bool", False, True)])
+auto_eval_column_dict.append(["moe", ColumnContent, ColumnContent(column_map["moe"], "str", False, True)])
+auto_eval_column_dict.append(["openness", ColumnContent, ColumnContent(column_map["openness"], "str", False, True)])
+# auto_eval_column_dict.append(["direct_complete", ColumnContent, ColumnContent(column_map["direct_complete"], "bool", False)])
+
+# We use make dataclass to dynamically fill the scores from Tasks
+AutoEvalColumn = make_dataclass("AutoEvalColumn", auto_eval_column_dict, frozen=True)
+
+
+@dataclass(frozen=True)
+class EvalQueueColumn:  # Queue column
+    model_link = ColumnContent("link", "markdown", True)
+    model_name = ColumnContent("model", "str", True)
+
+@dataclass
+class ModelDetails:
+    name: str
+    symbol: str = ""  # emoji, only for the model type
+
+
+# Column selection
+COLS = [c.name for c in fields(AutoEvalColumn)]
+TYPES = [c.type for c in fields(AutoEvalColumn)]
+
+EVAL_COLS = [c.name for c in fields(EvalQueueColumn)]
+EVAL_TYPES = [c.type for c in fields(EvalQueueColumn)]
+
+
+NUMERIC_INTERVALS = {
+    "?": pd.Interval(-1, 0, closed="right"),
+    "~1.5": pd.Interval(0, 2, closed="right"),
+    "~3": pd.Interval(2, 4, closed="right"),
+    "~7": pd.Interval(4, 9, closed="right"),
+    "~13": pd.Interval(9, 20, closed="right"),
+    "~35": pd.Interval(20, 45, closed="right"),
+    "~60": pd.Interval(45, 70, closed="right"),
+    "70+": pd.Interval(70, 10000, closed="right"),
+}

src/envs.py

@@ -0,0 +1,42 @@
+import os
+from huggingface_hub import HfApi
+
+# clone / pull the lmeh eval data
+HF_TOKEN = os.environ.get("HF_TOKEN", None)
+
+DATA_VERSION = "v0.1.0_hf"
+
+REPO_ID = "bigcode/bigcodebench-leaderboard"
+QUEUE_REPO = "bigcode/bigcodebench-requests"
+DATA_REPO = "bigcode/bigcodebench"
+RESULT_REPO = "bigcode/bigcodebench-results"
+HARD_RESULT_REPO = "bigcode/bigcodebench-hard-results"
+
+ELO_REPO = "bigcode/bigcodebench-elo"
+HARD_ELO_REPO = "bigcode/bigcodebench-hard-elo"
+SOLVE_REPO = "bigcode/bigcodebench-solve-rate"
+HARD_SOLVE_REPO = "bigcode/bigcodebench-hard-solve-rate"
+
+VOTES_REPO = "bigcode/bigcodebench-votes"
+
+HF_HOME = os.getenv("HF_HOME", ".")
+
+# Check HF_HOME write access
+print(f"Initial HF_HOME set to: {HF_HOME}")
+
+if not os.access(HF_HOME, os.W_OK):
+    print(f"No write access to HF_HOME: {HF_HOME}. Resetting to current directory.")
+    HF_HOME = "."
+    os.environ["HF_HOME"] = HF_HOME
+else:
+    print("Write access confirmed for HF_HOME")
+
+VOTES_PATH = os.path.join(HF_HOME, "model-votes")
+EVAL_REQUESTS_PATH = os.path.join(HF_HOME, "eval-queue")
+
+# Rate limit variables
+RATE_LIMIT_PERIOD = 7
+RATE_LIMIT_QUOTA = 5
+HAS_HIGHER_RATE_LIMIT = []
+
+API = HfApi(token=HF_TOKEN)

src/execute.py

@@ -0,0 +1,178 @@
+import gradio as gr
+import subprocess
+import sys
+import os
+import threading
+import time
+import uuid
+import glob
+import shutil
+from pathlib import Path
+from apscheduler.schedulers.background import BackgroundScheduler
+
+default_command = "bigcodebench.evaluate"
+is_running = False
+lock = threading.Lock()
+
+def generate_command(
+    jsonl_file, split, subset, parallel,
+    min_time_limit, max_as_limit, max_data_limit, max_stack_limit,
+    check_gt_only, no_gt
+):
+    command = [default_command]
+    
+    if jsonl_file is not None:
+        # Copy the uploaded file to the current directory
+        local_filename = os.path.basename(jsonl_file.name)
+        shutil.copy(jsonl_file.name, local_filename)
+        command.extend(["--samples", local_filename])
+    
+    command.extend(["--split", split, "--subset", subset])
+    
+    if parallel is not None and parallel != 0:
+        command.extend(["--parallel", str(int(parallel))])
+    
+    command.extend([
+        "--min-time-limit", str(min_time_limit),
+        "--max-as-limit", str(int(max_as_limit)),
+        "--max-data-limit", str(int(max_data_limit)),
+        "--max-stack-limit", str(int(max_stack_limit))
+    ])
+    
+    if check_gt_only:
+        command.append("--check-gt-only")
+    
+    if no_gt:
+        command.append("--no-gt")
+    
+    return " ".join(command)
+
+
+def cleanup_previous_files(jsonl_file):
+    if jsonl_file is not None:
+        file_list = ['Dockerfile', 'app.py', 'README.md', os.path.basename(jsonl_file.name), "__pycache__"]
+    else:
+        file_list = ['Dockerfile', 'app.py', 'README.md', "__pycache__"]
+    for file in glob.glob("*"):
+        try:
+            if file not in file_list:
+                os.remove(file)
+        except Exception as e:
+            print(f"Error during cleanup of {file}: {e}")
+
+def find_result_file():
+    json_files = glob.glob("*.json")
+    if json_files:
+        return max(json_files, key=os.path.getmtime)
+    return None
+
+def run_bigcodebench(command):
+    global is_running
+    with lock:
+        if is_running:
+            yield "A command is already running. Please wait for it to finish.\n"
+            return
+        is_running = True
+
+    try:
+        yield f"Executing command: {command}\n"
+        
+        process = subprocess.Popen(command.split(), stdout=subprocess.PIPE, stderr=subprocess.STDOUT, universal_newlines=True)
+        
+        for line in process.stdout:
+            yield line
+        
+        # process.wait()
+        
+        if process.returncode != 0:
+            yield f"Error: Command exited with status {process.returncode}\n"
+        
+        yield "Evaluation completed.\n"
+        
+        result_file = find_result_file()
+        if result_file:
+            yield f"Result file found: {result_file}\n"
+        else:
+            yield "No result file found.\n"
+    finally:
+        with lock:
+            is_running = False
+
+def stream_logs(command, jsonl_file=None):
+    global is_running
+        
+    if is_running:
+        yield "A command is already running. Please wait for it to finish.\n"
+        return
+    
+    cleanup_previous_files(jsonl_file)
+    yield "Cleaned up previous files.\n"
+
+    log_content = []
+    for log_line in run_bigcodebench(command):
+        log_content.append(log_line)
+        yield "".join(log_content)
+        
+with gr.Blocks() as demo:
+    gr.Markdown("# BigCodeBench Evaluator")
+    
+    with gr.Row():
+        jsonl_file = gr.File(label="Upload JSONL file", file_types=[".jsonl"])
+        split = gr.Dropdown(choices=["complete", "instruct"], label="Split", value="complete")
+        subset = gr.Dropdown(choices=["hard"], label="Subset", value="hard")
+    
+    with gr.Row():
+        parallel = gr.Number(label="Parallel (optional)", precision=0)
+        min_time_limit = gr.Number(label="Min Time Limit", value=1, precision=1)
+        max_as_limit = gr.Number(label="Max AS Limit", value=25*1024, precision=0)
+    
+    with gr.Row():
+        max_data_limit = gr.Number(label="Max Data Limit", value=25*1024, precision=0)
+        max_stack_limit = gr.Number(label="Max Stack Limit", value=10, precision=0)
+        check_gt_only = gr.Checkbox(label="Check GT Only")
+        no_gt = gr.Checkbox(label="No GT")
+    
+    command_output = gr.Textbox(label="Command", value=default_command, interactive=False)
+    with gr.Row():
+        submit_btn = gr.Button("Run Evaluation")
+        download_btn = gr.DownloadButton(label="Download Result")
+    log_output = gr.Textbox(label="Execution Logs", lines=20)
+    
+    input_components = [
+        jsonl_file, split, subset, parallel,
+        min_time_limit, max_as_limit, max_data_limit, max_stack_limit,
+        check_gt_only, no_gt
+    ]
+    
+    for component in input_components:
+        component.change(generate_command, inputs=input_components, outputs=command_output)
+        
+    
+    def start_evaluation(command, jsonl_file, subset, split):
+        extra = subset + "_" if subset != "full" else ""
+        if jsonl_file is not None:
+            result_path = os.path.basename(jsonl_file.name).replace(".jsonl", f"_{extra}eval_results.json")
+        else:
+            result_path = None
+
+        for log in stream_logs(command, jsonl_file):
+            if jsonl_file is not None:
+                yield log, gr.update(value=result_path, label=result_path), gr.update()
+            else:
+                yield log, gr.update(), gr.update()
+        is_running = False
+        result_file = find_result_file()
+        if result_file:
+            return gr.update(label="Evaluation completed. Result file found."), gr.update(value=result_file)
+                    # gr.Button(visible=False)#,
+                    # gr.DownloadButton(label="Download Result", value=result_file, visible=True))
+        else:
+            return gr.update(label="Evaluation completed. No result file found."), gr.update(value=result_path)
+                    # gr.Button("Run Evaluation", visible=True),
+                    # gr.DownloadButton(visible=False))
+    submit_btn.click(start_evaluation,
+                 inputs=[command_output, jsonl_file, subset, split],
+                 outputs=[log_output, download_btn])
+
+demo.queue(max_size=300).launch(share=True, server_name="0.0.0.0", server_port=7860)
+scheduler = BackgroundScheduler()

src/populate.py

@@ -0,0 +1,50 @@
+import pathlib
+import pandas as pd
+from datasets import Dataset
+from src.display.formatting import has_no_nan_values, make_clickable_model
+from src.display.utils import AutoEvalColumn, EvalQueueColumn
+from src.display.utils import load_json_data, column_map, type_map, moe_map, NUMERIC_INTERVALS
+
+
+
+def get_evaluation_queue_df(save_path, cols):
+    """Generate dataframes for pending, running, and finished evaluation entries."""
+    save_path = pathlib.Path(save_path)
+    all_evals = []
+
+    for path in save_path.rglob("*.json"):
+        data = load_json_data(path)
+    # Organizing data by status
+    status_map = {
+        "PENDING": ["PENDING", "RERUN"],
+        "RUNNING": ["RUNNING"],
+        "FINISHED": ["FINISHED", "PENDING_NEW_EVAL"],
+    }
+    status_dfs = {status: [] for status in status_map}
+    for eval_data in all_evals:
+        for status, extra_statuses in status_map.items():
+            if eval_data["status"] in extra_statuses:
+                status_dfs[status].append(eval_data)
+
+    return tuple(pd.DataFrame(status_dfs[status], columns=cols) for status in ["FINISHED", "RUNNING", "PENDING"])
+
+
+def get_leaderboard_df(leaderboard_dataset: Dataset, cols: list):
+    """Retrieve and process leaderboard data."""
+    all_data_json = leaderboard_dataset.to_dict()
+    num_items = leaderboard_dataset.num_rows
+    all_data_json_list = [{k: all_data_json[k][ix] for k in all_data_json.keys()} for ix in range(num_items)]
+
+    df = pd.DataFrame.from_records(all_data_json_list)
+    # replace df.moe true to false, false to true
+    # map column names
+    df = df.rename(columns=column_map)
+    df[AutoEvalColumn.moe.name] = df[AutoEvalColumn.moe.name].map(moe_map)
+    df[AutoEvalColumn.T.name] = df[AutoEvalColumn.type.name]
+    df[AutoEvalColumn.type.name] = df[AutoEvalColumn.type.name].map(type_map)
+    df[AutoEvalColumn.average.name] = df.apply(lambda x: round((x[AutoEvalColumn.complete.name] + x[AutoEvalColumn.instruct.name]) / 2, 1) if not pd.isna(x[AutoEvalColumn.complete.name]) and not pd.isna(x[AutoEvalColumn.instruct.name]) else None, axis=1)
+    df[AutoEvalColumn.size_range.name] = df[AutoEvalColumn.size.name].apply(lambda x: next((k for k, v in NUMERIC_INTERVALS.items() if x in v), "?"))
+    df = make_clickable_model(df, AutoEvalColumn.model.name, AutoEvalColumn.link.name)
+    df = df.sort_values(by=[AutoEvalColumn.average.name], ascending=False)
+    df = df[cols].round(decimals=2)
+    return df

src/tools/plots.py

@@ -0,0 +1,72 @@
+import plotly.graph_objects as go
+import plotly.express as px
+import numpy as np
+
+
+def plot_elo_mle(df):
+    fig = px.scatter(df, x="model", y="rating", error_y="error_y",
+                     error_y_minus="error_y_minus",
+                    #  title="Bootstrap of Elo MLE Estimates (BigCodeBench-Complete)"
+                     )
+    fig.update_layout(xaxis_title="Model", 
+                      yaxis_title="Rating",
+                      autosize=True,
+                    #   width=1300,
+                    #   height=900,
+                      )
+    return fig
+
+
+def plot_solve_rate(df, task, rows=30, cols=38):
+    keys = df["task_id"]
+    values = df["solve_rate"]
+    
+    values = np.array(values, dtype=float)  # Ensure values are floats
+
+    # Extract numerical IDs and sort by them
+    ids = [int(key.split('/')[-1]) for key in keys]
+    sorted_indices = np.argsort(ids)
+    keys = np.array(keys)[sorted_indices]
+    values = values[sorted_indices]
+    
+    n = len(values)
+    pad_width = rows * cols - n
+    
+    # Create a masked array
+    masked_values = np.ma.array(np.full(rows * cols, np.nan), mask=True)
+    masked_values[:n] = values
+    masked_values.mask[:n] = False
+    masked_values = masked_values.reshape((rows, cols))
+
+    keys_padded = np.pad(keys, (0, pad_width), 'constant', constant_values='')
+    keys_reshaped = keys_padded.reshape((rows, cols))
+
+    hover_text = np.empty_like(masked_values, dtype=object)
+    for i in range(rows):
+        for j in range(cols):
+            if not masked_values.mask[i, j]:
+                hover_text[i, j] = f"{keys_reshaped[i, j]}<br>Solve Rate: {masked_values[i, j]:.2f}"
+            else:
+                hover_text[i, j] = "NaN"
+
+    upper_solve_rate = round(np.count_nonzero(values) / n * 100, 2)
+    
+    fig = go.Figure(data=go.Heatmap(
+        z=masked_values,
+        text=hover_text,
+        hoverinfo='text',
+        colorscale='teal',
+        zmin=0,
+        zmax=100
+    ))
+
+    fig.update_layout(
+        title=f'BigCodeBench-{task}<br><i>Lowest Upper Limit: {upper_solve_rate}%</i>',
+        xaxis_nticks=cols,
+        yaxis_nticks=rows,
+        xaxis=dict(showticklabels=False),
+        yaxis=dict(showticklabels=False),
+        autosize=True,
+    )
+    
+    return fig

src/voting/vote_system.py

@@ -0,0 +1,150 @@
+import json
+import logging
+import pathlib
+import pandas as pd
+import gradio as gr
+import schedule
+import time
+from datetime import datetime, timezone
+
+from src.envs import API
+
+# Set up logging
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+class VoteManager:
+    def __init__(self, votes_path, eval_requests_path, repo_id):
+        self.votes_path = votes_path
+        self.eval_requests_path = eval_requests_path
+        self.repo_id = repo_id
+        self.vote_dataset = self.read_vote_dataset()
+        self.vote_check_set = self.make_check_set(self.vote_dataset)
+        self.votes_to_upload = []
+
+    def init_vote_dataset(self):
+        self.vote_dataset = self.read_vote_dataset()
+        self.vote_check_set = self.make_check_set(self.vote_dataset)
+
+    def read_vote_dataset(self):
+        result = []
+        votes_file = pathlib.Path(self.votes_path) / "votes_data.jsonl"
+        if votes_file.exists():
+            with open(votes_file, "r") as f:
+                for line in f:
+                    data = json.loads(line.strip())
+                    result.append(data)
+        result = pd.DataFrame(result)
+        return result
+
+    def make_check_set(self, vote_dataset: pd.DataFrame):
+        result = list()
+        for row in vote_dataset.itertuples(index=False, name='vote'):
+            result.append((row.model, row.revision, row.username))
+        return set(result)
+    
+    def get_model_revision(self, selected_model: str) -> str:
+        """Fetch the revision for the given model from the request files."""
+        for user_folder in pathlib.Path(self.eval_requests_path).iterdir():
+            if user_folder.is_dir():
+                for file in user_folder.glob("*.json"):
+                    with open(file, "r") as f:
+                        data = json.load(f)
+                        if data.get("model") == selected_model:
+                            return data.get("revision", "main")
+        return "main"
+
+    def create_request_vote_df(self, pending_models_df: gr.Dataframe):
+        if pending_models_df.empty or not "model_name" in pending_models_df.columns:
+            return pending_models_df
+        self.vote_dataset = self.read_vote_dataset()
+        vote_counts = self.vote_dataset.groupby(['model', 'revision']).size().reset_index(name='vote_count')
+
+        pending_models_df_votes = pd.merge(
+            pending_models_df, 
+            vote_counts, 
+            left_on=["model_name", 'revision'], 
+            right_on=['model', 'revision'], 
+            how='left'
+        )
+        # Filling empty votes
+        pending_models_df_votes['vote_count'] = pending_models_df_votes['vote_count'].fillna(0)
+        pending_models_df_votes = pending_models_df_votes.sort_values(by=["vote_count", "model_name"], ascending=[False, True])
+        # Removing useless columns
+        pending_models_df_votes = pending_models_df_votes.drop(["model_name", "model"], axis=1)
+        return pending_models_df_votes
+
+    # Function to be called when a user votes for a model
+    def add_vote(
+            self,
+            selected_model: str,
+            pending_models_df: gr.Dataframe,
+            profile: gr.OAuthProfile | None
+        ):
+        logger.debug(f"Type of list before usage: {type(list)}")
+        # model_name, revision, user_id, timestamp
+        if selected_model in ["str", ""]:
+            gr.Warning("No model selected")
+            return
+        
+        if profile is None:
+            gr.Warning("Hub Login required")
+            return
+
+        vote_username = profile.username
+        model_revision = self.get_model_revision(selected_model)
+        
+        # tuple (immutable) for checking than already voted for model
+        check_tuple = (selected_model, model_revision, vote_username)
+        if check_tuple in self.vote_check_set:
+            gr.Warning("Already voted for this model")
+            return
+        
+        current_time = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
+
+        vote_obj = {
+            "model": selected_model,
+            "revision": model_revision,
+            "username": vote_username,
+            "timestamp": current_time
+        }
+
+        # Append the vote to the JSONL file
+        try:
+            votes_file = pathlib.Path(self.votes_path) / "votes_data.jsonl"
+            with open(votes_file, "a") as f:
+                f.write(json.dumps(vote_obj) + "\n")
+            logger.info(f"Vote added locally: {vote_obj}")
+
+            self.votes_to_upload.append(vote_obj)
+        except Exception as e:
+            logger.error(f"Failed to write vote to file: {e}")
+            gr.Warning("Failed to record vote. Please try again")
+            return
+        
+        self.vote_check_set.add(check_tuple)
+        gr.Info(f"Voted for {selected_model}")
+
+        return self.create_request_vote_df(pending_models_df)
+
+    def upload_votes(self):
+        if self.votes_to_upload:
+            votes_file = pathlib.Path(self.votes_path) / "votes_data.jsonl"
+            try:
+                with open(votes_file, "rb") as f:
+                    API.upload_file(
+                        path_or_fileobj=f,
+                        path_in_repo="votes_data.jsonl",
+                        repo_id=self.repo_id,
+                        repo_type="dataset",
+                        commit_message="Updating votes_data.jsonl with new votes",
+                    )
+                logger.info("Votes uploaded to votes repository")
+                self.votes_to_upload.clear()
+            except Exception as e:
+                logger.error(f"Failed to upload votes to repository: {e}")
+
+def run_scheduler(vote_manager):
+    while True:
+        schedule.run_pending()
+        time.sleep(1)