Implement Leaderboard Graph

app.py

@@ -1,8 +1,11 @@
 import os
 import time
 from dataclasses import dataclass
+from datetime import datetime
+from zoneinfo import ZoneInfo
 
 import gradio as gr
+import plotly.graph_objects as go
 import schedule
 import wandb
 from substrateinterface import Keypair
@@ -14,6 +17,10 @@ demo = gr.Blocks(css=".typewriter {font-family: 'JMH Typewriter', sans-serif;}")
 SOURCE_VALIDATOR_UID = int(os.environ["SOURCE_VALIDATOR_UID"])
 WANDB_RUN_PATH = os.environ["WANDB_RUN_PATH"]
 
+BASELINE = 0.0
+GRAPH_HISTORY_DAYS = 30
+MAX_GRAPH_ENTRIES = 5
+
 
 @dataclass
 class LeaderboardEntry:
@@ -25,6 +32,14 @@ class LeaderboardEntry:
     rank: int
 
 
+@dataclass
+class GraphEntry:
+    dates: list[datetime]
+    scores: list[float]
+    models: list[str]
+    max_score: float
+
+
 def is_valid_run(run: Run):
     required_config_keys = ["hotkey", "uid", "contest", "signature"]
 
@@ -44,8 +59,80 @@ def is_valid_run(run: Run):
         return False
 
 
+def get_graph_entries(runs: Runs) -> dict[int, GraphEntry]:
+    graph_entries: dict[int, GraphEntry] = {}
+
+    for run in reversed(runs[:GRAPH_HISTORY_DAYS]):
+        if not is_valid_run(run):
+            continue
+
+        date = datetime.strptime(run.created_at, "%Y-%m-%dT%H:%M:%S")
+
+        for key, value in run.summary.items():
+            if key.startswith("_"):
+                continue
+
+            uid = int(key)
+            score = value["score"]
+            model = value["model"]
+
+            if uid not in graph_entries:
+                graph_entries[uid] = GraphEntry([date], [score], [model], score)
+            else:
+                if score > graph_entries[uid].max_score:
+                    graph_entries[uid].max_score = score
+
+                data = graph_entries[uid]
+                data.dates.append(date)
+                data.scores.append(data.max_score)
+                data.models.append(model)
+
+    return dict(sorted(graph_entries.items(), key=lambda entry: entry[1].max_score, reverse=True)[:MAX_GRAPH_ENTRIES])
+
+
+def create_graph(graph_entries: dict[int, GraphEntry]):
+    fig = go.Figure()
+
+    for uid, data in graph_entries.items():
+        fig.add_trace(go.Scatter(
+            x=data.dates,
+            y=data.scores,
+            customdata=data.models,
+            mode="lines+markers",
+            name=uid,
+            hovertemplate=(
+                    "<b>Date:</b> %{x|%Y-%m-%d}<br>" +
+                    "<b>Score:</b> %{y}<br>" +
+                    "<b>Model:</b> %{customdata}<br>"
+            ),
+        ))
+
+    date_range = max(graph_entries.values(), key=lambda entry: len(entry.dates)).dates
+
+    fig.add_trace(go.Scatter(
+        x=date_range,
+        y=[BASELINE] * len(date_range),
+        line=dict(color="#ff0000", width=3),
+        mode="lines",
+        name="Baseline",
+    ))
+
+    background_color = gr.themes.default.colors.slate.c800
+
+    fig.update_layout(
+        title="Score Improvements",
+        yaxis_title="Score",
+        plot_bgcolor=background_color,
+        paper_bgcolor=background_color,
+        template="plotly_dark"
+    )
+
+    gr.Plot(fig)
+
+
 def refresh_leaderboard():
-    print("Refreshing Leaderboard")
+    now = datetime.now(tz=ZoneInfo("America/New_York"))
+    print(f"Refreshing Leaderboard at {now.strftime('%Y-%m-%d %H:%M:%S')}")
 
     demo.clear()
 
@@ -92,6 +179,8 @@ def refresh_leaderboard():
                 for entry in sorted(entries.values(), key=lambda entry: (entry.score, entry.rank), reverse=True)
             ]
 
+            create_graph(get_graph_entries(runs))
+
             gr.components.Dataframe(
                 value=leaderboard,
                 headers=["Rank", "Uid", "Model", "Score", "Hotkey", "Previous day winner"],

requirements.txt

@@ -3,3 +3,4 @@ gradio
 wandb
 substrate-interface
 schedule
+plotly