app.py

import os
from dataclasses import dataclass

from substrateinterface import Keypair
from wandb.apis.public import Runs, Run

import wandb
import gradio as gr

wandb_api = wandb.Api()

SOURCE_VALIDATOR_UID = int(os.environ["SOURCE_VALIDATOR_UID"])
WANDB_RUN_PATH = os.environ["WANDB_RUN_PATH"]


@dataclass
class LeaderboardEntry:
    uid: int
    model: str
    score: float
    hotkey: str
    previous_day_winner: bool
    rank: int


def is_valid_run(run: Run):
    required_config_keys = ["hotkey", "uid", "contest", "signature"]

    for key in required_config_keys:
        if key not in run.config:
            return False

    uid = run.config["uid"]
    validator_hotkey = run.config["hotkey"]
    contest_name = run.config["contest"]

    signing_message = f"{uid}:{validator_hotkey}:{contest_name}"

    try:
        return Keypair(validator_hotkey).verify(signing_message, run.config["signature"])
    except Exception:
        return False


def main():
    demo = gr.Blocks(css=".typewriter {font-family: 'JMH Typewriter', sans-serif;}")
    with demo:
        with gr.Accordion("Submission Leaderboard"):
            runs: Runs = wandb_api.runs(
                WANDB_RUN_PATH,
                filters={"config.type": "validator", "config.uid": SOURCE_VALIDATOR_UID},
                order="-created_at",
            )

            entries: dict[int, LeaderboardEntry] = {}

            for run in runs:
                if not is_valid_run(run):
                    continue

                for key, value in run.summary.items():
                    if key.startswith("_"):
                        continue

                    try:
                        uid = int(key)

                        entries[uid] = LeaderboardEntry(
                            uid=uid,
                            rank=value["rank"],
                            model=value["model"],
                            score=value["score"],
                            hotkey=value["hotkey"],
                            previous_day_winner=value["multiday_winner"],
                        )
                    except Exception:
                        continue

                leaderboard: list[tuple] = [
                    (entry.uid, entry.model, entry.score, entry.hotkey, entry.previous_day_winner)
                    for entry in sorted(entries.values(), key=lambda entry: entry.rank)
                ]

            gr.components.Dataframe(
                value=leaderboard,
                headers=["Uid", "Model", "Score", "Hotkey", "Previous day winner"],
                datatype=["number", "markdown", "number", "markdown", "bool"],
                elem_id="leaderboard-table",
                interactive=False,
                visible=True,
            )

    demo.launch()


main()