File size: 2,778 Bytes
d06d966 5bb9974 d06d966 a2cb75e d06d966 a2cb75e d06d966 a2cb75e d06d966 a2cb75e d06d966 a2cb75e d06d966 5bb9974 d06d966 b065235 d06d966 5bb9974 d06d966 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 |
import os
from dataclasses import dataclass
from substrateinterface import Keypair
from wandb.apis.public import Runs, Run
import wandb
import gradio as gr
wandb_api = wandb.Api()
SOURCE_VALIDATOR_UID = int(os.environ["SOURCE_VALIDATOR_UID"])
WANDB_RUN_PATH = os.environ["WANDB_RUN_PATH"]
@dataclass
class LeaderboardEntry:
uid: int
model: str
score: float
hotkey: str
previous_day_winner: bool
rank: int
def is_valid_run(run: Run):
required_config_keys = ["hotkey", "uid", "contest", "signature"]
for key in required_config_keys:
if key not in run.config:
return False
uid = run.config["uid"]
validator_hotkey = run.config["hotkey"]
contest_name = run.config["contest"]
signing_message = f"{uid}:{validator_hotkey}:{contest_name}"
try:
return Keypair(validator_hotkey).verify(signing_message, run.config["signature"])
except Exception:
return False
def main():
demo = gr.Blocks(css=".typewriter {font-family: 'JMH Typewriter', sans-serif;}")
with demo:
with gr.Accordion("Submission Leaderboard"):
runs: Runs = wandb_api.runs(
WANDB_RUN_PATH,
filters={"config.type": "validator", "config.uid": SOURCE_VALIDATOR_UID},
order="-created_at",
)
entries: dict[int, LeaderboardEntry] = {}
for run in runs:
if not is_valid_run(run):
continue
for key, value in run.summary.items():
if key.startswith("_"):
continue
try:
uid = int(key)
entries[uid] = LeaderboardEntry(
uid=uid,
rank=value["rank"],
model=value["model"],
score=value["score"],
hotkey=value["hotkey"],
previous_day_winner=value["multiday_winner"],
)
except Exception:
continue
leaderboard: list[tuple] = [
(entry.uid, entry.model, entry.score, entry.hotkey, entry.previous_day_winner)
for entry in sorted(entries.values(), key=lambda entry: entry.rank)
]
gr.components.Dataframe(
value=leaderboard,
headers=["Uid", "Model", "Score", "Hotkey", "Previous day winner"],
datatype=["number", "markdown", "number", "markdown", "bool"],
elem_id="leaderboard-table",
interactive=False,
visible=True,
)
demo.launch()
main()
|