Upload 2 files

app.py

@@ -0,0 +1,249 @@
+import json
+import gradio as gr
+import webbrowser
+
+# 已有数据的路径
+data_file_path = '//leaderboard_data.json'
+
+# 加载已有的数据
+def load_data():
+    with open(data_file_path, 'r') as file:
+        return json.load(file)
+
+existing_data = load_data()
+
+# 根据排名生成表格数据
+def generate_table(data):
+    if not data:
+        return [], []  # No data, return empty headers and rows
+
+    # Make sure all entries have a 'Scores' dictionary
+    valid_data = [entry for entry in data if 'Scores' in entry and isinstance(entry['Scores'], dict)]
+
+    # Sort the data based on the 'Average' score in descending order
+    sorted_data = sorted(valid_data, key=lambda x: x['Scores'].get('Average', 0), reverse=True)
+
+    # Now, build the table rows with the sorted data
+    headers = ["Method"] + list(sorted_data[0]['Scores'].keys()) if sorted_data else ["Method"]
+    rows = []
+    for entry in sorted_data:
+        row = [entry['Method']] + [entry['Scores'].get(key, "N/A") for key in headers[1:]]
+        rows.append(row)
+
+    return headers, rows
+
+# 原始数据
+protected_methods = ["GPT-4-Turbo-Preview (DFS)", "GPT-3.5-Turbo-1106 (DFS)", "GPT-4-0613 (DFS)", "GPT-3.5-Turbo-0613 (DFS)", "GPT-4-Turbo-Preview (CoT)", "ToolLLaMA v2 (DFS)", "GPT-4-0613 (CoT)", "GPT-3.5-Turbo-1106 (CoT)", "GPT-3.5-Turbo-0613 (CoT)", "ToolLLaMA v2 (CoT)"]
+
+# 合并上传的数据到已有的数据中
+def merge_data(uploaded_data_json):
+    # No need to call json.loads here because uploaded_data is already a Python dict
+    new_data = uploaded_data_json
+
+    # Define a helper function to merge scores for an entry
+    def merge_scores(existing_scores, new_scores):
+        for key, value in new_scores.items():
+            existing_scores[key] = value
+
+    # Merge 'SolvablePassRateScores'
+    # 更新 'SolvablePassRateScores'
+    for new_entry in new_data["SolvablePassRateScores"]:
+        if new_entry["Method"] not in protected_methods:
+            existing_entry = next(
+                (item for item in existing_data["SolvablePassRateScores"] if item["Method"] == new_entry["Method"]),
+                None)
+            if existing_entry:
+                # 这是一个非保护的方法，更新它
+                merge_scores(existing_entry["Scores"], new_entry["Scores"])
+            else:
+                # 这是一个新的方法，添加到列表中
+                existing_data["SolvablePassRateScores"].append(new_entry)
+
+    # Merge 'SolvableWinRateScores'
+    for new_entry in new_data["SolvableWinRateScores"]:
+        if new_entry["Method"] not in protected_methods:
+            existing_entry = next(
+                (item for item in existing_data["SolvableWinRateScores"] if item["Method"] == new_entry["Method"]),
+                None)
+            if existing_entry:
+                merge_scores(existing_entry["Scores"], new_entry["Scores"])
+            else:
+                existing_data["SolvableWinRateScores"].append(new_entry)
+
+         # After merging the data, write the updated data back to the file
+        with open(data_file_path, 'w') as file:
+            json.dump(existing_data, file, indent=4)
+
+    # No need to sort here since we're doing it in the generate_table function
+    return existing_data
+
+def process_file(file_info):
+    if file_info is not None:
+        # 如果 file_info 是文件的路径字符串，需要使用 'open' 来读取文件
+        with open(file_info, "r") as uploaded_file:
+            data_content = uploaded_file.read()
+            uploaded_data_json = json.loads(data_content)
+        # Merge the uploaded data
+        merge_data(uploaded_data_json)
+        # No need to call generate_table here since we are not returning it
+    # If we don't return anything here, gradio will not update the table
+    # We need to return the new data for the table if it's interactive
+    pass_rate_table = generate_table(existing_data["SolvablePassRateScores"])[1]
+    win_rate_table = generate_table(existing_data["SolvableWinRateScores"])[1]
+    return pass_rate_table, win_rate_table
+
+def refresh_table_data():
+    # 重新加载数据
+    new_data = existing_data
+    # 重新生成表格数据
+    new_pass_rate_data = generate_table(new_data["SolvablePassRateScores"])[1]
+    new_win_rate_data = generate_table(new_data["SolvableWinRateScores"])[1]
+    # 返回新的表格数据
+    return new_pass_rate_data, new_win_rate_data
+
+# 创建界面
+with gr.Blocks() as app:
+    # The large title
+    gr.Markdown("# StableToolBench Leaderboard")
+
+    # The introductory content
+    gr.Markdown("""
+            **Large Language Models (LLMs)** have witnessed remarkable advancements in recent years, prompting the exploration of tool learning, which integrates LLMs with external tools to address diverse real-world challenges. Assessing the capability of LLMs to utilise tools necessitates large-scale and stable benchmarks. However, previous works relied on either hand-crafted online tools with limited scale, or large-scale real online APIs suffering from instability of API status. To address this problem, we introduce StableToolBench, a benchmark evolving from ToolBench, proposing a virtual API server and stable evaluation system. The virtual API server contains a caching system and API simulators which are complementary to alleviate the change in API status. Meanwhile, the stable evaluation system designs solvable pass and win rates using GPT-4 as the automatic evaluator to eliminate the randomness during evaluation. Experimental results demonstrate the stability of StableToolBench, and further discuss the effectiveness of API simulators, the caching system, and the evaluation system.
+            """)
+    gr.Markdown(""" ### For further information, please refer to: """)
+    buttons_html = """
+        <style>
+            .custom-link-button {
+                font-size: 18px !important; /* Adjust the font size as needed */
+                padding: 10px 15px !important; /* Add some padding */
+                margin: 5px !important;
+                color: white !important; /* Text color */
+                background-color: #106BA3 !important; /* Background color */
+                text-decoration: none !important; /* Remove underline from links */
+                display: inline-block !important;
+                border-radius: 5px !important; /* Rounded corners */
+                border: none !important; /* Remove borders */
+                cursor: pointer !important; /* Mouse pointer on hover */
+                text-align: center !important;
+            }
+            .custom-link-button:hover {
+                background-color: #0D5B8F !important;
+            }
+        </style>
+        <a href="https://arxiv.org/pdf/2403.07714.pdf" target="_blank" class="custom-link-button">Paper</a>
+        <a href="https://arxiv.org/abs/2403.07714" target="_blank" class="custom-link-button">arXiv</a>
+        <a href="https://github.com/zhichengg/StableToolBench" target="_blank" class="custom-link-button">Code</a>
+        <a href="https://drive.google.com/file/d/1XUiCMA5NV359UGR-eknF0TcXORuR7RXj/view?pli=1" target="_blank" class="custom-link-button">Cache Data</a>
+        """
+
+    gr.HTML(buttons_html)
+
+    # Markdown 标题
+    gr.Markdown("## Solvable Pass Rate Scores")
+    # 初始生成表格数据
+    headers1, rows1 = generate_table(existing_data["SolvablePassRateScores"])
+    table1 = gr.Dataframe(headers=headers1, value=rows1, interactive=True)
+
+    gr.Markdown("## Solvable Win Rate Scores")
+    # 初始生成表格数据
+    headers2, rows2 = generate_table(existing_data["SolvableWinRateScores"])
+    table2 = gr.Dataframe(headers=headers2, value=rows2, interactive=True)
+
+    refresh_button = gr.Button("Refresh Leaderboards")
+    # 当刷新按钮被点击时，调用 refresh_table_data 函数来更新表格
+    refresh_button.click(
+        fn=refresh_table_data,
+        outputs=[table1, table2]
+    )
+
+    gr.Markdown("## The Stable Evaluation System")
+    gr.Markdown("**Solvable Tasks Filtration**. Since the solvablility of tasks in original ToolBench induces siginificant instability, we filter out the unsolvable tasks in advance. This process is executed using GPT-4, Gemini Pro, and Claude 2. Each task from the dataset is evaluated by these models to determine its solvability through majority voting. A task is classified as solvable if it provides all the necessary and valid information required for completion and can be resolved with the available tools. Human evaluation shows that these models can effectively filter out unsolvable tasks, ensuring the stability of the benchmark.")
+    gr.Markdown("**Metrics (SoPR and SoWR)**. Due to the limitation of gpt-3.5-turbo-16k in tool learning, we uniformly adopt gpt-4-turbo-preview as the automatic evaluator. SoPR is in essence PR with all tasks solvable and only assesses the answers using the same prompt in ToolBench. The evaluator assigns outcomes of answers categorised as Solved, Unsolved, or Unsure, which respectively contribute scores of 1, 0.5, and 0 to the overall SoPR calculation. As for SoWR, when one is solved and the other is unsolved, the solved one wins. Under other circumstances, gpt-4-turbo-preview will be used to make a win-lose decision.")
+
+
+    headers_ex = ["", "I1 Instruction", "I1 Category", "I1 Tool", "I2 Instruction", "I2 Category", "I3 Instruction",
+               "Total"]
+    data_ex = [
+        ["Full", 200, 200, 200, 200, 200, 100, 1100],
+        ["Solvable", 163, 153, 158, 106, 124, 61, 765]
+    ]
+    gr.Markdown("#### Table: Summary of Task Statistics before and after filtration")
+    gr.Dataframe(headers=headers_ex, value=data_ex, interactive=False)
+
+    gr.Markdown("## Upload Your Own Results")
+    gr.Markdown("""
+            If you would like to contribute to the leaderboard, please follow the JSON structure below for your method's scores.
+
+            **Solvable Pass Rate Scores Template:**
+            ```json
+            {
+                "SolvablePassRateScores": [
+                    {
+                        "Method": "Your Method Name",
+                        "Scores": {
+                        "I1 Instruction": 85.5,
+                        "I1 Instruction SE": 1.2,
+                        "I1 Category": 80.0,
+                        "I1 Category SE": 1.0,
+                        "I1 Tool": 88.5,
+                        "I1 Tool SE": 0.8,
+                        "I2 Category": 82.5,
+                        "I2 Category SE": 1.3,
+                        "I2 Instruction": 86.0,
+                        "I2 Instruction SE": 0.5,
+                        "I3 Instruction": 90.0,
+                        "I3 Instruction SE": 0.7,
+                        "Average": 87.5,
+                        "Average SE": 1.1
+                        }
+                    }
+                    // Add more methods here...
+                ],
+                "SolvableWinRateScores": [
+                    {
+                        "Method": "Your Method Name",
+                        "Scores": {
+                            "I1 Instruction": 65.0,
+                            "I1 Category": 68.5,
+                            "I1 Tool": 66.8,
+                            "I2 Category": 70.0,
+                            "I2 Instruction": 69.2,
+                            "I3 Instruction": 71.5,
+                            "Average": 68.5
+                        }
+                    }
+                    // Add more methods here...
+                ]
+            }
+            ```
+
+            Make sure your uploaded JSON file follows this structure.
+        """)
+    # 文件上传组件和提交按钮
+    upload_component = gr.File(label="Upload JSON File")
+    submit_button = gr.Button("Submit")
+
+    submit_button.click(
+        fn=process_file,
+        inputs=upload_component,
+        outputs=[table1, table2]
+    )
+
+    # 引用部分
+    gr.Markdown(" ## If you like our project, please consider cite our work as follows: ")
+    citation_text = """
+        ```
+        @misc{guo2024stabletoolbench,
+          title={StableToolBench: Towards Stable Large-Scale Benchmarking on Tool Learning of Large Language Models}, 
+          author={Zhicheng Guo and Sijie Cheng and Hao Wang and Shihao Liang and Yujia Qin and Peng Li and Zhiyuan Liu and Maosong Sun and Yang Liu},
+          year={2024},
+          eprint={2403.07714},
+          archivePrefix={arXiv},
+          primaryClass={cs.CL}
+        }
+        ```
+        """
+    gr.Markdown(citation_text)
+
+if __name__ == "__main__":
+    app.launch()

leaderboard_data.json

@@ -0,0 +1,366 @@
+{
+    "SolvablePassRateScores": [
+        {
+            "Method": "ToolLLaMA v2 (DFS)",
+            "Scores": {
+                "I1 Instruction": 59.8,
+                "I1 Instruction SE": 1.5,
+                "I1 Category": 59.5,
+                "I1 Category SE": 1.4,
+                "I1 Tool": 65.7,
+                "I1 Tool SE": 1.1,
+                "I2 Category": 56.5,
+                "I2 Category SE": 0.3,
+                "I2 Instruction": 47.6,
+                "I2 Instruction SE": 0.4,
+                "I3 Instruction": 62.8,
+                "I3 Instruction SE": 1.9,
+                "Average": 58.7,
+                "Average SE": 1.1
+            }
+        },
+        {
+            "Method": "ToolLLaMA v2 (CoT)",
+            "Scores": {
+                "I1 Instruction": 37.2,
+                "I1 Instruction SE": 0.1,
+                "I1 Category": 42.3,
+                "I1 Category SE": 0.4,
+                "I1 Tool": 43.0,
+                "I1 Tool SE": 0.5,
+                "I2 Category": 37.4,
+                "I2 Category SE": 0.4,
+                "I2 Instruction": 33.6,
+                "I2 Instruction SE": 1.2,
+                "I3 Instruction": 39.6,
+                "I3 Instruction SE": 1.0,
+                "Average": 38.9,
+                "Average SE": 0.6
+            }
+        },
+        {
+            "Method": "GPT-4-Turbo-Preview (DFS)",
+            "Scores": {
+                "I1 Instruction": 70.8,
+                "I1 Instruction SE": 1.0,
+                "I1 Category": 71.1,
+                "I1 Category SE": 0.7,
+                "I1 Tool": 70.4,
+                "I1 Tool SE": 1.2,
+                "I2 Category": 70.4,
+                "I2 Category SE": 1.3,
+                "I2 Instruction": 71.7,
+                "I2 Instruction SE": 0.4,
+                "I3 Instruction": 84.7,
+                "I3 Instruction SE": 1.7,
+                "Average": 73.2,
+                "Average SE": 1.1
+            }
+        },
+        {
+            "Method": "GPT-4-Turbo-Preview (CoT)",
+            "Scores": {
+                "I1 Instruction": 63.1,
+                "I1 Instruction SE": 1.0,
+                "I1 Category": 64.5,
+                "I1 Category SE": 0.5,
+                "I1 Tool": 55.3,
+                "I1 Tool SE": 0.3,
+                "I2 Category": 63.0,
+                "I2 Category SE": 0.8,
+                "I2 Instruction": 57.3,
+                "I2 Instruction SE": 0.8,
+                "I3 Instruction": 61.7,
+                "I3 Instruction SE": 0.8,
+                "Average": 60.8,
+                "Average SE": 0.7
+            }
+        },
+        {
+            "Method": "GPT-4-0613 (DFS)",
+            "Scores": {
+                "I1 Instruction": 65.5,
+                "I1 Instruction SE": 1.1,
+                "I1 Category": 62.0,
+                "I1 Category SE": 1.7,
+                "I1 Tool": 72.1,
+                "I1 Tool SE": 1.6,
+                "I2 Category": 70.8,
+                "I2 Category SE": 1.3,
+                "I2 Instruction": 73.1,
+                "I2 Instruction SE": 1.4,
+                "I3 Instruction": 74.9,
+                "I3 Instruction SE": 1.5,
+                "Average": 69.7,
+                "Average SE": 1.4
+            }
+        },
+        {
+            "Method": "GPT-4-0613 (CoT)",
+            "Scores": {
+                "I1 Instruction": 50.7,
+                "I1 Instruction SE": 0.4,
+                "I1 Category": 57.1,
+                "I1 Category SE": 0.3,
+                "I1 Tool": 51.9,
+                "I1 Tool SE": 0.3,
+                "I2 Category": 55.0,
+                "I2 Category SE": 1.1,
+                "I2 Instruction": 61.6,
+                "I2 Instruction SE": 0.8,
+                "I3 Instruction": 56.3,
+                "I3 Instruction SE": 0.8,
+                "Average": 55.4,
+                "Average SE": 0.6
+            }
+        },
+        {
+            "Method": "GPT-3.5-Turbo-1106 (DFS)",
+            "Scores": {
+                "I1 Instruction": 67.8,
+                "I1 Instruction SE": 0.9,
+                "I1 Category": 67.2,
+                "I1 Category SE": 0.3,
+                "I1 Tool": 72.9,
+                "I1 Tool SE": 0.7,
+                "I2 Category": 63.2,
+                "I2 Category SE": 1.0,
+                "I2 Instruction": 70.9,
+                "I2 Instruction SE": 0.4,
+                "I3 Instruction": 77.6,
+                "I3 Instruction SE": 0.8,
+                "Average": 69.9,
+                "Average SE": 0.7
+            }
+        },
+        {
+            "Method": "GPT-3.5-Turbo-1106 (CoT)",
+            "Scores": {
+                "I1 Instruction": 51.3,
+                "I1 Instruction SE": 0.6,
+                "I1 Category": 48.8,
+                "I1 Category SE": 0.3,
+                "I1 Tool": 59.9,
+                "I1 Tool SE": 0.8,
+                "I2 Category": 50.8,
+                "I2 Category SE": 0.7,
+                "I2 Instruction": 43.2,
+                "I2 Instruction SE": 0.8,
+                "I3 Instruction": 58.5,
+                "I3 Instruction SE": 0.8,
+                "Average": 52.1,
+                "Average SE": 0.7
+            }
+        },
+        {
+            "Method": "GPT-3.5-Turbo-0613 (DFS)",
+            "Scores": {
+                "I1 Instruction": 66.4,
+                "I1 Instruction SE": 1.5,
+                "I1 Category": 64.3,
+                "I1 Category SE": 1.0,
+                "I1 Tool": 67.2,
+                "I1 Tool SE": 2.4,
+                "I2 Category": 67.7,
+                "I2 Category SE": 0.8,
+                "I2 Instruction": 61.5,
+                "I2 Instruction SE": 1.0,
+                "I3 Instruction": 81.4,
+                "I3 Instruction SE": 1.5,
+                "Average": 68.1,
+                "Average SE": 1.4
+            }
+        },
+        {
+            "Method": "GPT-3.5-Turbo-0613 (CoT)",
+            "Scores": {
+                "I1 Instruction": 55.9,
+                "I1 Instruction SE": 1.0,
+                "I1 Category": 50.8,
+                "I1 Category SE": 0.8,
+                "I1 Tool": 55.9,
+                "I1 Tool SE": 1.0,
+                "I2 Category": 44.1,
+                "I2 Category SE": 0.8,
+                "I2 Instruction": 36.2,
+                "I2 Instruction SE": 0.4,
+                "I3 Instruction": 51.4,
+                "I3 Instruction SE": 1.5,
+                "Average": 49.1,
+                "Average SE": 1.0
+            }
+        },
+        {
+            "Method": "Method Name 1",
+            "Scores": {
+                "I1 Instruction": 85.5,
+                "I1 Instruction SE": 1.2,
+                "I1 Category": 80.0,
+                "I1 Category SE": 1.0,
+                "I1 Tool": 88.5,
+                "I1 Tool SE": 0.8,
+                "I2 Category": 82.5,
+                "I2 Category SE": 1.3,
+                "I2 Instruction": 86.0,
+                "I2 Instruction SE": 0.5,
+                "I3 Instruction": 90.0,
+                "I3 Instruction SE": 0.7,
+                "Average": 87.5,
+                "Average SE": 1.1
+            }
+        },
+        {
+            "Method": "Method Name 2",
+            "Scores": {
+                "I1 Instruction": 75.3,
+                "I1 Instruction SE": 2.1,
+                "I1 Category": 70.2,
+                "I1 Category SE": 1.5,
+                "I1 Tool": 78.4,
+                "I1 Tool SE": 1.3,
+                "I2 Category": 72.9,
+                "I2 Category SE": 1.4,
+                "I2 Instruction": 80.1,
+                "I2 Instruction SE": 0.9,
+                "I3 Instruction": 85.0,
+                "I3 Instruction SE": 1.0,
+                "Average": 77.2,
+                "Average SE": 1.2
+            }
+        }
+    ],
+    "SolvableWinRateScores": [
+        {
+            "Method": "GPT-3.5-Turbo-0613 (DFS)",
+            "Scores": {
+                "I1 Instruction": 57.7,
+                "I1 Category": 60.8,
+                "I1 Tool": 61.4,
+                "I2 Category": 66.1,
+                "I2 Instruction": 63.2,
+                "I3 Instruction": 70.5,
+                "Average": 63.3
+            }
+        },
+        {
+            "Method": "GPT-4-0613 (CoT)",
+            "Scores": {
+                "I1 Instruction": 50.3,
+                "I1 Category": 54.2,
+                "I1 Tool": 50.6,
+                "I2 Category": 50.0,
+                "I2 Instruction": 64.2,
+                "I3 Instruction": 55.7,
+                "Average": 54.2
+            }
+        },
+        {
+            "Method": "GPT-4-0613 (DFS)",
+            "Scores": {
+                "I1 Instruction": 57.1,
+                "I1 Category": 60.1,
+                "I1 Tool": 57.0,
+                "I2 Category": 64.5,
+                "I2 Instruction": 74.5,
+                "I3 Instruction": 72.1,
+                "Average": 64.2
+            }
+        },
+        {
+            "Method": "ToolLLaMA v2 (CoT)",
+            "Scores": {
+                "I1 Instruction": 35.0,
+                "I1 Category": 30.7,
+                "I1 Tool": 37.3,
+                "I2 Category": 31.5,
+                "I2 Instruction": 36.8,
+                "I3 Instruction": 23.0,
+                "Average": 32.4
+            }
+        },
+        {
+            "Method": "ToolLLaMA v2 (DFS)",
+            "SCores": {
+                "I1 Instruction": 43.6,
+                "I1 Category": 45.1,
+                "I1 Tool": 38.6,
+                "I2 Category": 42.7,
+                "I2 Instruction": 53.8,
+                "I3 Instruction": 45.9,
+                "Average": 44.9
+            }
+        },
+        {
+            "Method": "GPT-3.5-Turbo-1106 (CoT)",
+            "Scores": {
+                "I1 Instruction": 46.6,
+                "I1 Category": 45.1,
+                "I1 Tool": 48.1,
+                "I2 Category": 44.4,
+                "I2 Instruction": 37.7,
+                "I3 Instruction": 52.5,
+                "Average": 45.7
+            }
+        },
+        {
+            "Method": "GPT-3.5-Turbo-1106 (DFS)",
+            "Scores": {
+                "I1 Instruction": 56.4,
+                "I1 Category": 54.2,
+                "I1 Tool": 51.9,
+                "I2 Category": 54.0,
+                "I2 Instruction": 62.3,
+                "I3 Instruction": 72.1,
+                "Average": 58.5
+            }
+        },
+        {
+            "Method": "GPT-4-Turbo-Preview (CoT)",
+            "Scores": {
+                "I1 Instruction": 68.7,
+                "I1 Category": 71.9,
+                "I1 Tool": 58.2,
+                "I2 Category": 71.0,
+                "I2 Instruction": 76.4,
+                "I3 Instruction": 73.8,
+                "Average": 70.0
+            }
+        },
+        {
+            "Method": "GPT-4-Turbo-Preview (DFS)",
+            "Scores": {
+                "I1 Instruction": 66.9,
+                "I1 Category": 73.9,
+                "I1 Tool": 68.4,
+                "I2 Category": 72.6,
+                "I2 Instruction": 78.3,
+                "I3 Instruction": 77.0,
+                "Average": 72.9
+            }
+        },
+        {
+            "Method": "Method Name 3",
+            "Scores": {
+                "I1 Instruction": 65.0,
+                "I1 Category": 68.5,
+                "I1 Tool": 66.8,
+                "I2 Category": 70.0,
+                "I2 Instruction": 69.2,
+                "I3 Instruction": 71.5,
+                "Average": 68.5
+            }
+        },
+        {
+            "Method": "Method Name 4",
+            "Scores": {
+                "I1 Instruction": 85.0,
+                "I1 Category": 82.5,
+                "I1 Tool": 88.0,
+                "I2 Category": 84.0,
+                "I2 Instruction": 87.2,
+                "I3 Instruction": 90.5,
+                "Average": 86.2
+            }
+        }
+    ]
+}