File size: 8,306 Bytes
74e3b17
 
 
08ae6c5
74e3b17
 
0811d37
74e3b17
 
 
 
 
 
 
 
95c19d6
74e3b17
 
 
2a5f9fb
74e3b17
 
8c49cb6
74e3b17
95c19d6
2a73469
74e3b17
 
 
22b004e
74e3b17
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1ffc326
74e3b17
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
9f23558
 
 
 
 
 
74e3b17
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
95c19d6
 
74e3b17
 
8b88d2c
d084b26
74e3b17
 
 
0811d37
95c19d6
74e3b17
 
 
 
 
 
 
 
 
 
 
95c19d6
 
8b88d2c
74e3b17
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
95c19d6
8c49cb6
74e3b17
 
 
95c19d6
 
0811d37
74e3b17
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
import fnmatch
import glob
import json
import logging
import os
import pprint

import gradio as gr
import gymnasium as gym
import numpy as np
import pandas as pd
import torch
from apscheduler.schedulers.background import BackgroundScheduler
from huggingface_hub import hf_hub_download, snapshot_download
from huggingface_hub.utils._errors import EntryNotFoundError

from src.css_html_js import dark_mode_gradio_js
from src.envs import API, RESULTS_PATH, RESULTS_REPO, TOKEN
from src.logging import configure_root_logger, setup_logger

logging.getLogger("openai").setLevel(logging.WARNING)
logger = setup_logger(__name__)

configure_root_logger()
logger = setup_logger(__name__)

pp = pprint.PrettyPrinter(width=80)


ALL_ENV_IDS = list(gym.registry.keys())


def model_hyperlink(link, model_id):
    return f'<a target="_blank" href="{link}" style="color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;">{model_id}</a>'


def make_clickable_model(model_id):
    link = f"https://huggingface.co/{model_id}"
    return model_hyperlink(link, model_id)


def pattern_match(patterns, source_list):
    if isinstance(patterns, str):
        patterns = [patterns]

    env_ids = set()
    for pattern in patterns:
        for matching in fnmatch.filter(source_list, pattern):
            env_ids.add(matching)
    return sorted(list(env_ids))


def evaluate(model_id, revision):
    tags = API.model_info(model_id, revision=revision).tags

    # Extract the environment IDs from the tags (usually only one)
    env_ids = pattern_match(tags, ALL_ENV_IDS)
    logger.info(f"Selected environments: {env_ids}")

    results = {}

    # Check if the agent exists
    try:
        agent_path = hf_hub_download(repo_id=model_id, filename="agent.pt")
    except EntryNotFoundError:
        logger.error("Agent not found")
        return None

    # Check safety
    security = next(iter(API.list_files_info(model_id, "agent.pt", expand=True))).security
    if security is None or "safe" not in security:
        logger.error("Agent safety not available")
        return None
    elif not security["safe"]:
        logger.error("Agent not safe")
        return None

    # Load the agent
    try:
        agent = torch.jit.load(agent_path)
    except Exception as e:
        logger.error(f"Error loading agent: {e}")
        return None

    # Evaluate the agent on the environments
    for env_id in env_ids:
        episodic_rewards = []
        env = gym.make(env_id)
        for _ in range(10):
            episodic_reward = 0.0
            observation, info = env.reset()
            done = False
            while not done:
                torch_observation = torch.from_numpy(np.array([observation]))
                action = agent(torch_observation).numpy()[0]
                observation, reward, terminated, truncated, info = env.step(action)
                done = terminated or truncated
                episodic_reward += reward

            episodic_rewards.append(episodic_reward)

        mean_reward = np.mean(episodic_rewards)
        results[env_id] = {"episodic_return": mean_reward}
    return results


def _backend_routine():
    # List only the text classification models
    rl_models = list(API.list_models(filter="reinforcement-learning"))
    logger.info(f"Found {len(rl_models)} RL models")
    compatible_models = []
    for model in rl_models:
        filenames = [sib.rfilename for sib in model.siblings]
        if "agent.pt" in filenames:
            compatible_models.append((model.modelId, model.sha))

    logger.info(f"Found {len(compatible_models)} compatible models")

    # Get the results
    snapshot_download(
        repo_id=RESULTS_REPO,
        revision="main",
        local_dir=RESULTS_PATH,
        repo_type="dataset",
        max_workers=60,
        token=TOKEN,
    )
    json_files = glob.glob(f"{RESULTS_PATH}/**/*.json", recursive=True)

    evaluated_models = set()
    for json_filepath in json_files:
        with open(json_filepath) as fp:
            data = json.load(fp)
        evaluated_models.add((data["config"]["model_id"], data["config"]["model_sha"]))

    # Find the models that are not associated with any results
    pending_models = set(compatible_models) - evaluated_models
    logger.info(f"Found {len(pending_models)} pending models")

    # Run an evaluation on the models
    for model_id, sha in pending_models:
        logger.info(f"Running evaluation on {model_id}")
        report = {"config": {"model_id": model_id, "model_sha": sha}}
        try:
            evaluations = evaluate(model_id, revision=sha)
        except Exception as e:
            logger.error(f"Error evaluating {model_id}: {e}")
            evaluations = None

        if evaluations is not None:
            report["results"] = evaluations
            report["status"] = "DONE"
        else:
            report["status"] = "FAILED"

        # Update the results
        dumped = json.dumps(report, indent=2)
        output_path = os.path.join(RESULTS_PATH, model_id, f"results_{sha}.json")
        os.makedirs(os.path.dirname(output_path), exist_ok=True)
        with open(output_path, "w") as f:
            f.write(dumped)

        # Upload the results to the results repo
        API.upload_file(
            path_or_fileobj=output_path,
            path_in_repo=f"{model_id}/results_{sha}.json",
            repo_id=RESULTS_REPO,
            repo_type="dataset",
        )


def backend_routine():
    try:
        _backend_routine()
    except Exception as e:
        logger.error(f"{e.__class__.__name__}: {str(e)}")


def get_leaderboard_df():
    snapshot_download(
        repo_id=RESULTS_REPO,
        revision="main",
        local_dir=RESULTS_PATH,
        repo_type="dataset",
        max_workers=60,
        token=TOKEN,
    )

    json_files = glob.glob(f"{RESULTS_PATH}/**/*.json", recursive=True)
    data = []

    for json_filepath in json_files:
        with open(json_filepath) as fp:
            report = json.load(fp)
        model_id = report["config"]["model_id"]
        row = {"Agent": model_id, "Status": report["status"]}
        if report["status"] == "DONE":
            results = {env_id: result["episodic_return"] for env_id, result in report["results"].items()}
            row.update(results)
        data.append(row)

    # Create DataFrame
    df = pd.DataFrame(data)
    # Replace NaN values with empty strings
    df = df.fillna("")
    return df


TITLE = """
🚀 Open RL Leaderboard
"""

INTRODUCTION_TEXT = """
Welcome to the Open RL Leaderboard! This is a community-driven benchmark for reinforcement learning models.
"""

ABOUT_TEXT = """
The Open RL Leaderboard is a community-driven benchmark for reinforcement learning models.
"""


def select_column(column_names, data):
    column_names = [col for col in column_names if col in data.columns]
    column_names = ["Agent"] + column_names  # add model name column
    df = data[column_names]

    def check_row(row):
        return not (row.drop("Agent") == "").all()

    mask = df.apply(check_row, axis=1)
    df = df[mask]
    return df


with gr.Blocks(js=dark_mode_gradio_js) as demo:
    gr.HTML(TITLE)
    gr.Markdown(INTRODUCTION_TEXT, elem_classes="markdown-text")
    with gr.Tabs(elem_classes="tab-buttons") as tabs:
        with gr.TabItem("🏅 Leaderboard", elem_id="llm-benchmark-tab-table", id=0):
            full_df = get_leaderboard_df()
            hidden_df = gr.components.Dataframe(full_df, visible=False)  # hidden dataframe

            env_checkboxes = gr.components.CheckboxGroup(
                label="Environments",
                choices=ALL_ENV_IDS,
                value=[ALL_ENV_IDS[0]],
                interactive=True,
            )
            leaderboard = gr.components.Dataframe(select_column([ALL_ENV_IDS[0]], full_df))

            # Events
            env_checkboxes.change(select_column, [env_checkboxes, hidden_df], leaderboard)

        with gr.TabItem("📝 About", elem_id="llm-benchmark-tab-table", id=2):
            gr.Markdown(ABOUT_TEXT)


scheduler = BackgroundScheduler()
scheduler.add_job(func=backend_routine, trigger="interval", seconds=30)
scheduler.start()


if __name__ == "__main__":
    demo.queue().launch()  # server_name="0.0.0.0", show_error=True, server_port=7860)