mujoco

app.py

@@ -67,7 +67,7 @@ def evaluate(model_id, revision):
         return None
 
     # Check safety
-    security = next(iter(API.list_files_info(model_id, "agent.pt", expand=True))).security
+    security = next(iter(API.get_paths_info(model_id, "agent.pt", expand=True))).security
     if security is None or "safe" not in security:
         logger.error("Agent safety not available")
         return None
@@ -100,7 +100,8 @@ def evaluate(model_id, revision):
             episodic_rewards.append(episodic_reward)
 
         mean_reward = np.mean(episodic_rewards)
-        results[env_id] = {"episodic_return": mean_reward}
+        std_reward = np.std(episodic_rewards)
+        results[env_id] = {"episodic_return_mean": mean_reward, "episodic_reward_std": std_reward}
     return results
 
 
@@ -195,7 +196,7 @@ def get_leaderboard_df():
         model_id = report["config"]["model_id"]
         row = {"Agent": model_id, "Status": report["status"]}
         if report["status"] == "DONE":
-            results = {env_id: result["episodic_return"] for env_id, result in report["results"].items()}
+            results = {env_id: result["episodic_return_mean"] for env_id, result in report["results"].items()}
             row.update(results)
         data.append(row)
 
@@ -237,8 +238,7 @@ with gr.Blocks(js=dark_mode_gradio_js) as demo:
     gr.Markdown(INTRODUCTION_TEXT, elem_classes="markdown-text")
     with gr.Tabs(elem_classes="tab-buttons") as tabs:
         with gr.TabItem("🏅 Leaderboard", elem_id="llm-benchmark-tab-table", id=0):
-            full_df = get_leaderboard_df()
-            hidden_df = gr.components.Dataframe(full_df, visible=False)  # hidden dataframe
+            hidden_df = gr.components.Dataframe(get_leaderboard_df, visible=False, every=60)  # hidden dataframe
 
             env_checkboxes = gr.components.CheckboxGroup(
                 label="Environments",
@@ -246,17 +246,19 @@ with gr.Blocks(js=dark_mode_gradio_js) as demo:
                 value=[ALL_ENV_IDS[0]],
                 interactive=True,
             )
-            leaderboard = gr.components.Dataframe(select_column([ALL_ENV_IDS[0]], full_df))
+            leaderboard = gr.components.Dataframe(select_column([ALL_ENV_IDS[0]], get_leaderboard_df()))
 
             # Events
             env_checkboxes.change(select_column, [env_checkboxes, hidden_df], leaderboard)
+            # Update hidden dataframe
+            # hidden_df.change(get_leaderboard_df, [], hidden_df, every=10)
 
         with gr.TabItem("📝 About", elem_id="llm-benchmark-tab-table", id=2):
             gr.Markdown(ABOUT_TEXT)
 
 
 scheduler = BackgroundScheduler()
-scheduler.add_job(func=backend_routine, trigger="interval", seconds=30)
+scheduler.add_job(func=backend_routine, trigger="interval", seconds=60)
 scheduler.start()
 
 

requirements.txt

@@ -13,6 +13,8 @@ python-dateutil==2.8.2
 requests==2.28.2
 torch==2.2.2
 tqdm==4.65.0
+cython<3
+free-mujoco-py
 
 # Log Visualizer
 BeautifulSoup4==4.12.2