dump from mock space

.gitattributes

@@ -25,6 +25,7 @@
 *.safetensors filter=lfs diff=lfs merge=lfs -text
 saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
 *.tflite filter=lfs diff=lfs merge=lfs -text
 *.tgz filter=lfs diff=lfs merge=lfs -text
 *.wasm filter=lfs diff=lfs merge=lfs -text
@@ -32,4 +33,3 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
-scale-hf-logo.png filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -1,13 +1,228 @@
-auto_evals/
-venv/
+# Created by https://www.toptal.com/developers/gitignore/api/python,visualstudiocode,macos
+# Edit at https://www.toptal.com/developers/gitignore?templates=python,visualstudiocode,macos
+
+### macOS ###
+# General
+.DS_Store
+.AppleDouble
+.LSOverride
+
+# Icon must end with two \r
+Icon
+
+
+# Thumbnails
+._*
+
+# Files that might appear in the root of a volume
+.DocumentRevisions-V100
+.fseventsd
+.Spotlight-V100
+.TemporaryItems
+.Trashes
+.VolumeIcon.icns
+.com.apple.timemachine.donotpresent
+
+# Directories potentially created on remote AFP share
+.AppleDB
+.AppleDesktop
+Network Trash Folder
+Temporary Items
+.apdisk
+
+### macOS Patch ###
+# iCloud generated files
+*.icloud
+
+### Python ###
+# Byte-compiled / optimized / DLL files
 __pycache__/
-.env
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
 .ipynb_checkpoints
-*ipynb
-.vscode/
-
-eval-queue/
-eval-results/
-eval-queue-bk/
-eval-results-bk/
-logs/
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# poetry
+#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+#poetry.lock
+
+# pdm
+#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
+#pdm.lock
+#   pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
+#   in version control.
+#   https://pdm.fming.dev/#use-with-ide
+.pdm.toml
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# PyCharm
+#  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
+#  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
+#  and can be added to the global gitignore or merged into this file.  For a more nuclear
+#  option (not recommended) you can uncomment the following to ignore the entire idea folder.
+#.idea/
+
+### Python Patch ###
+# Poetry local configuration file - https://python-poetry.org/docs/configuration/#local-configuration
+poetry.toml
+
+# ruff
+.ruff_cache/
+
+# LSP config files
+pyrightconfig.json
+
+### VisualStudioCode ###
+.vscode/*
+!.vscode/settings.json
+!.vscode/tasks.json
+!.vscode/launch.json
+!.vscode/extensions.json
+!.vscode/*.code-snippets
+
+# Local History for Visual Studio Code
+.history/
+
+# Built Visual Studio Code Extensions
+*.vsix
+
+### VisualStudioCode Patch ###
+# Ignore all local history of files
+.history
+.ionide
+
+# End of https://www.toptal.com/developers/gitignore/api/python,visualstudiocode,macos

README.md

@@ -1,45 +1,12 @@
 ---
-title: MEGA Bench
+title: MEGA-Bench
 emoji: 🥇
-colorFrom: green
+colorFrom: blue
 colorTo: indigo
 sdk: gradio
+sdk_version: 5.1.0
 app_file: app.py
 pinned: true
 license: apache-2.0
-short_description: The space for the leaderboard and information of MEGA-Bench
 ---
 
-# Start the configuration
-
-Most of the variables to change for a default leaderboard are in `src/env.py` (replace the path for your leaderboard) and `src/about.py` (for tasks).
-
-Results files should have the following format and be stored as json files:
-```json
-{
-    "config": {
-        "model_dtype": "torch.float16", # or torch.bfloat16 or 8bit or 4bit
-        "model_name": "path of the model on the hub: org/model",
-        "model_sha": "revision on the hub",
-    },
-    "results": {
-        "task_name": {
-            "metric_name": score,
-        },
-        "task_name2": {
-            "metric_name": score,
-        }
-    }
-}
-```
-
-Request files are created automatically by this tool.
-
-If you encounter problem on the space, don't hesitate to restart it to remove the create eval-queue, eval-queue-bk, eval-results and eval-results-bk created folder.
-
-# Code logic for more complex edits
-
-You'll find 
-- the main table' columns names and properties in `src/display/utils.py`
-- the logic to read all results and request files, then convert them in dataframe lines, in `src/leaderboard/read_evals.py`, and `src/populate.py`
-- the logic to allow or filter submissions in `src/submission/submit.py` and `src/submission/check_validity.py`

app.py

@@ -1,204 +1,79 @@
 import gradio as gr
-from gradio_leaderboard import Leaderboard, ColumnFilter, SelectColumns
-import pandas as pd
-from apscheduler.schedulers.background import BackgroundScheduler
-from huggingface_hub import snapshot_download
-
-from src.about import (
-    CITATION_BUTTON_LABEL,
-    CITATION_BUTTON_TEXT,
-    EVALUATION_QUEUE_TEXT,
-    INTRODUCTION_TEXT,
-    LLM_BENCHMARKS_TEXT,
-    TITLE,
-)
-from src.display.css_html_js import custom_css
-from src.display.utils import (
-    BENCHMARK_COLS,
-    COLS,
-    EVAL_COLS,
-    EVAL_TYPES,
-    AutoEvalColumn,
-    ModelType,
-    fields,
-    WeightType,
-    Precision
-)
-from src.envs import API, EVAL_REQUESTS_PATH, EVAL_RESULTS_PATH, QUEUE_REPO, REPO_ID, RESULTS_REPO, TOKEN
-from src.populate import get_evaluation_queue_df, get_leaderboard_df
-from src.submission.submit import add_new_eval
-
-
-def restart_space():
-    API.restart_space(repo_id=REPO_ID)
-
-### Space initialisation
-try:
-    print(EVAL_REQUESTS_PATH)
-    snapshot_download(
-        repo_id=QUEUE_REPO, local_dir=EVAL_REQUESTS_PATH, repo_type="dataset", tqdm_class=None, etag_timeout=30, token=TOKEN
-    )
-except Exception:
-    restart_space()
-try:
-    print(EVAL_RESULTS_PATH)
-    snapshot_download(
-        repo_id=RESULTS_REPO, local_dir=EVAL_RESULTS_PATH, repo_type="dataset", tqdm_class=None, etag_timeout=30, token=TOKEN
+from utils import get_leaderboard_data, SUPER_GROUPS, MODEL_GROUPS
+import os
+from constants import *
+
+# Get the directory of the current script
+current_dir = os.path.dirname(os.path.abspath(__file__))
+
+# Construct the path to the CSS file
+css_file = os.path.join(current_dir, "static", "css", "style.css")
+
+# Read the CSS file
+with open(css_file, "r") as f:
+    css = f.read()
+
+def update_leaderboard(selected_super_group, selected_model_group):
+    headers, data = get_leaderboard_data(selected_super_group, selected_model_group)
+    return gr.Dataframe(
+        value=data,
+        headers=headers,
+        datatype=["str"] + ["number"] * (len(headers) - 1),
     )
-except Exception:
-    restart_space()
-
 
-LEADERBOARD_DF = get_leaderboard_df(EVAL_RESULTS_PATH, EVAL_REQUESTS_PATH, COLS, BENCHMARK_COLS)
-
-(
-    finished_eval_queue_df,
-    running_eval_queue_df,
-    pending_eval_queue_df,
-) = get_evaluation_queue_df(EVAL_REQUESTS_PATH, EVAL_COLS)
-
-def init_leaderboard(dataframe):
-    if dataframe is None or dataframe.empty:
-        raise ValueError("Leaderboard DataFrame is empty or None.")
-    return Leaderboard(
-        value=dataframe,
-        datatype=[c.type for c in fields(AutoEvalColumn)],
-        select_columns=SelectColumns(
-            default_selection=[c.name for c in fields(AutoEvalColumn) if c.displayed_by_default],
-            cant_deselect=[c.name for c in fields(AutoEvalColumn) if c.never_hidden],
-            label="Select Columns to Display:",
-        ),
-        search_columns=[AutoEvalColumn.model.name, AutoEvalColumn.license.name],
-        hide_columns=[c.name for c in fields(AutoEvalColumn) if c.hidden],
-        filter_columns=[
-            ColumnFilter(AutoEvalColumn.model_type.name, type="checkboxgroup", label="Model types"),
-            ColumnFilter(AutoEvalColumn.precision.name, type="checkboxgroup", label="Precision"),
-            ColumnFilter(
-                AutoEvalColumn.params.name,
-                type="slider",
-                min=0.01,
-                max=150,
-                label="Select the number of parameters (B)",
-            ),
-            ColumnFilter(
-                AutoEvalColumn.still_on_hub.name, type="boolean", label="Deleted/incomplete", default=True
-            ),
-        ],
-        bool_checkboxgroup_label="Hide models",
-        interactive=False,
+with gr.Blocks(css=css) as block:
+    gr.Markdown(
+        LEADERBOARD_INTRODUCTION
     )
-
-
-demo = gr.Blocks(css=custom_css)
-with demo:
-    gr.HTML(TITLE)
-    gr.Markdown(INTRODUCTION_TEXT, elem_classes="markdown-text")
-
     with gr.Tabs(elem_classes="tab-buttons") as tabs:
-        with gr.TabItem("🏅 LLM Benchmark", elem_id="llm-benchmark-tab-table", id=0):
-            leaderboard = init_leaderboard(LEADERBOARD_DF)
-
-        with gr.TabItem("📝 About", elem_id="llm-benchmark-tab-table", id=2):
-            gr.Markdown(LLM_BENCHMARKS_TEXT, elem_classes="markdown-text")
-
-        with gr.TabItem("🚀 Submit here! ", elem_id="llm-benchmark-tab-table", id=3):
-            with gr.Column():
-                with gr.Row():
-                    gr.Markdown(EVALUATION_QUEUE_TEXT, elem_classes="markdown-text")
-
-                with gr.Column():
-                    with gr.Accordion(
-                        f"✅ Finished Evaluations ({len(finished_eval_queue_df)})",
-                        open=False,
-                    ):
-                        with gr.Row():
-                            finished_eval_table = gr.components.Dataframe(
-                                value=finished_eval_queue_df,
-                                headers=EVAL_COLS,
-                                datatype=EVAL_TYPES,
-                                row_count=5,
-                            )
-                    with gr.Accordion(
-                        f"🔄 Running Evaluation Queue ({len(running_eval_queue_df)})",
-                        open=False,
-                    ):
-                        with gr.Row():
-                            running_eval_table = gr.components.Dataframe(
-                                value=running_eval_queue_df,
-                                headers=EVAL_COLS,
-                                datatype=EVAL_TYPES,
-                                row_count=5,
-                            )
+        with gr.TabItem("📊 MEGA-Bench", elem_id="qa-tab-table1", id=1):
+            with gr.Row():
+                with gr.Accordion("Citation", open=False):
+                    citation_button = gr.Textbox(
+                        value=CITATION_BUTTON_TEXT,
+                        label=CITATION_BUTTON_LABEL,
+                        elem_id="citation-button",
+                        lines=10,
+                    )
+            gr.Markdown(
+                TABLE_INTRODUCTION
+            )
 
-                    with gr.Accordion(
-                        f"⏳ Pending Evaluation Queue ({len(pending_eval_queue_df)})",
-                        open=False,
-                    ):
-                        with gr.Row():
-                            pending_eval_table = gr.components.Dataframe(
-                                value=pending_eval_queue_df,
-                                headers=EVAL_COLS,
-                                datatype=EVAL_TYPES,
-                                row_count=5,
-                            )
             with gr.Row():
-                gr.Markdown("# ✉️✨ Submit your model here!", elem_classes="markdown-text")
+                super_group_selector = gr.Radio(
+                    choices=list(SUPER_GROUPS.keys()),
+                    label="Select a dimension to display breakdown results",
+                    value=list(SUPER_GROUPS.keys())[0]
+                )
+                model_group_selector = gr.Radio(
+                    choices=list(MODEL_GROUPS.keys()),
+                    label="Select a model group",
+                    value="All"
+                )
+
+            initial_headers, initial_data = get_leaderboard_data(list(SUPER_GROUPS.keys())[0], "All")
+            data_component = gr.Dataframe(
+                value=initial_data,
+                headers=initial_headers,
+                datatype=["str"] + ["number"] * (len(initial_headers) - 1),
+                interactive=False,
+                elem_classes="custom-dataframe",
+            )
+            refresh_button = gr.Button("Refresh")
+            refresh_button.click(fn=update_leaderboard, inputs=[super_group_selector, model_group_selector], outputs=[data_component])
+            super_group_selector.change(fn=update_leaderboard, inputs=[super_group_selector, model_group_selector], outputs=[data_component])
+            model_group_selector.change(fn=update_leaderboard, inputs=[super_group_selector, model_group_selector], outputs=[data_component])
 
+        with gr.TabItem("📝 Data Information", elem_id="qa-tab-table2", id=2):
+            gr.Markdown(DATA_INFO, elem_classes="markdown-text")
+
+        with gr.TabItem("🚀 Submit here! ", elem_id="submit-tab", id=3):
             with gr.Row():
-                with gr.Column():
-                    model_name_textbox = gr.Textbox(label="Model name")
-                    revision_name_textbox = gr.Textbox(label="Revision commit", placeholder="main")
-                    model_type = gr.Dropdown(
-                        choices=[t.to_str(" : ") for t in ModelType if t != ModelType.Unknown],
-                        label="Model type",
-                        multiselect=False,
-                        value=None,
-                        interactive=True,
-                    )
+                gr.Markdown(SUBMIT_INTRODUCTION, elem_classes="markdown-text")
 
-                with gr.Column():
-                    precision = gr.Dropdown(
-                        choices=[i.value.name for i in Precision if i != Precision.Unknown],
-                        label="Precision",
-                        multiselect=False,
-                        value="float16",
-                        interactive=True,
-                    )
-                    weight_type = gr.Dropdown(
-                        choices=[i.value.name for i in WeightType],
-                        label="Weights type",
-                        multiselect=False,
-                        value="Original",
-                        interactive=True,
-                    )
-                    base_model_name_textbox = gr.Textbox(label="Base model (for delta or adapter weights)")
 
-            submit_button = gr.Button("Submit Eval")
-            submission_result = gr.Markdown()
-            submit_button.click(
-                add_new_eval,
-                [
-                    model_name_textbox,
-                    base_model_name_textbox,
-                    revision_name_textbox,
-                    precision,
-                    weight_type,
-                    model_type,
-                ],
-                submission_result,
-            )
 
-    with gr.Row():
-        with gr.Accordion("📙 Citation", open=False):
-            citation_button = gr.Textbox(
-                value=CITATION_BUTTON_TEXT,
-                label=CITATION_BUTTON_LABEL,
-                lines=20,
-                elem_id="citation-button",
-                show_copy_button=True,
-            )
+if __name__ == "__main__":
+    block.launch(share=True)
+    #block.launch(server_name="127.0.0.1", server_port=7860)
 
-scheduler = BackgroundScheduler()
-scheduler.add_job(restart_space, "interval", seconds=1800)
-scheduler.start()
-demo.queue(default_concurrency_limit=40).launch()

constants.py

@@ -0,0 +1,81 @@
+import os
+
+HF_TOKEN = os.environ.get("HF_TOKEN")
+
+LEADERBOARD_INTRODUCTION = """# MEGA-Bench Leaderboard
+
+## 🚀 Introduction
+
+[MEGA-Bench](https://tiger-ai-lab.github.io/MEGA-Bench/) is a comprehensive benchmark scaling multimodal evaluation to 500+ real-world tasks! 
+
+We aim to provide cost-effective and accurate evaluation for multimodal models, covering a wide range of real-world tasks. You don't have to run models on dozens of benchmarks -- MEGA-Bench delivers a comprehensive performance report in a single benchmark.
+
+## 🧐 Highlights of MEGA-Bench
+
+- 505 diverse tasks evaluating multimodal models across 8 grand application types, 7 input visual formats, 6 output formats, and 10 general multimodal skills, covering single-image, multi-image, and video tasks
+- Moves beyond multiple-choice questions, offering diverse output formats like numbers, code, LATEX, phrases, free-form responses, and more. We developed 45 customized metrics to accurately evaluate these diverse outputs
+- Focuses on task diversity rather than repetitive examples, ensuring cost-efficient evaluation
+- Provides fine-grained capability reports across application type, input/output formats, and required skills
+
+
+## 🔨 Systematic Annotation Process
+
+- Guided by an initial application-driven taxonomy tree
+- 16 expert annotators contributing to a 2-round process to develop 505 tasks
+- Utilizes advanced tools for task design, review, and quality control
+- Ensures high-quality data through continuous refinement and balanced task distribution
+
+
+## 📊🔍 Results & Takeaways from Evaluating Top Models
+
+- GPT4o leads the benchmark, outperforming others by 3.5% over Claude3.5
+- Qwen2VL stands out among open-source models, nearing flagship-level performance
+- Chain-of-Thought (CoT) improves proprietary models but has limited impact on open-source models
+- Efficiency models like Gemini 1.5 Flash perform well but struggle with UI and document tasks
+- Many open-source models face challenges in adhering to output format instructions
+
+## 🎯 Interactive Visualization 
+
+Visit our [project page](https://tiger-ai-lab.github.io/MEGA-Bench/) to explore the interactive task taxonomy and radar maps, offering deep insights into model capabilities across multiple dimensions. Discover a comprehensive breakdown far beyond single-score evaluations.
+
+
+## 📚 More Information
+
+- Our evaluation pipeline will soon be available on our GitHub: https://github.com/TIGER-AI-Lab/MEGA-Bench.
+- Check full details of our paper at [https://arxiv.org/abs/2410.10563](https://arxiv.org/abs/2410.10563)
+- Hugging Face Datasets: [https://huggingface.co/datasets/TIGER-Lab/MEGA-Bench](https://huggingface.co/datasets/TIGER-Lab/MEGA-Bench)
+- GitHub: [https://github.com/TIGER-AI-Lab/MEGA-Bench](https://github.com/TIGER-AI-Lab/MEGA-Bench)
+
+"""
+
+TABLE_INTRODUCTION = """
+"""
+
+DATA_INFO = """
+### Data Sources
+The data source of MEGA-Bench tasks have three main types:
+- **Purely Self-designed:** The task is designed entirely by the annotator, and the annotator looks for the image or video resources from the Internet or even using code/simulator.
+- **Inspired and adapted from existing benchmarks:** The task is inspired by existing benchmarks or datasets. The annotator collects the raw image/video data from existing datasets but does not use the original annotation. The annotator redesigns/repurposes the data by writing concrete task descriptions and creating new questions and answers, or using scripts to re-process the data for the designed task.  
+- **Directly converted from existing benchmarks:** The task is directly converted from existing benchmarks or datasets. The annotator randomly samples a subset from the existing benchmark, directly using its image/video and the annotation without redesign.
+
+In our annotation process, the first two task types are encouraged. The task reviewers strictly control the number of the third type and reject the task if an annotator submits many tasks of the third type.
+
+Please refer to Table 17 of our [paper](https://arxiv.org/abs/2410.10563) for the detailed data source of all tasks in MEGA-Bench.
+"""
+
+CITATION_BUTTON_LABEL = "Copy the following snippet to cite our paper and evaluation results below"
+CITATION_BUTTON_TEXT = r"""
+@article{chen2024mega-bench,
+        title={MEGA-Bench: Scaling Multimodal Evaluation to over 500 Real-World Tasks},
+        author={Chen, Jiacheng and Liang, Tianhao and Siu, Sherman and Wang, Zhengqing and Wang, Kai and Wang, Yubo and Ni, Yuansheng and Zhu, Wang and Jiang, Ziyan and Lyu, Bohan and Jiang, Dongfu and He, Xuan and Liu, Yuan and Hu, Hexiang and Yue, Xiang and Chen, Wenhu},
+        journal={arXiv preprint arXiv:2410.10563},
+        year={2024},
+}
+"""
+
+SUBMIT_INTRODUCTION = """# Submit on MEGA-Bench Leaderboard
+
+We will provide details on how to submitting the results files once our evaluation pipeline is released on our [GitHub repository](https://github.com/TIGER-AI-Lab/MEGA-Bench).
+
+
+"""

requirements.txt

@@ -1,16 +1,18 @@
-APScheduler
-black
-datasets
-gradio
-gradio[oauth]
-gradio_leaderboard==0.0.9
-gradio_client
-huggingface-hub>=0.18.0
-matplotlib
-numpy
-pandas
-python-dateutil
-tqdm
-transformers
+APScheduler==3.10.1
+black==23.11.0
+click==8.1.3
+datasets==2.14.5
+gradio==5.1.0
+gradio_client==1.4.0
+huggingface-hub==0.25.2
+matplotlib==3.7.1
+numpy==1.24.2
+pandas==2.0.0
+python-dateutil==2.8.2
+requests==2.28.2
+tqdm==4.65.0
+transformers==4.35.2
 tokenizers>=0.15.0
+git+https://github.com/EleutherAI/lm-evaluation-harness.git@b281b0921b636bc36ad05c0b0b0763bd6dd43463#egg=lm-eval
+accelerate==0.24.1
 sentencepiece

static/css/style.css

@@ -0,0 +1,62 @@
+.custom-dataframe {
+    width: 100% !important;
+    overflow-x: auto !important;
+}
+
+.custom-dataframe table {
+    width: auto !important;
+    min-width: 100% !important;
+    font-size: 14px !important;
+}
+
+.custom-dataframe thead th {
+    padding: 4px 8px !important;
+    text-align: center !important;
+    vertical-align: middle !important;
+    white-space: nowrap !important;
+    overflow: visible !important;
+    font-size: 12px !important;
+    font-weight: bold !important;
+    line-height: 1.2 !important;
+}
+
+.custom-dataframe tbody td {
+    padding: 4px 8px !important;
+    text-align: right !important;
+    vertical-align: middle !important;
+    white-space: nowrap !important;
+    overflow: visible !important;
+    line-height: 1.2 !important;
+}
+
+.custom-dataframe tbody td:first-child {
+    text-align: left !important;
+}
+
+/* Adjust the sort indicator position */
+.custom-dataframe thead th::after {
+    font-size: 12px !important;
+    line-height: 1 !important;
+    margin-left: 4px !important;
+}
+
+/* Style for global result columns */
+.custom-dataframe thead th:nth-child(-n+4),
+.custom-dataframe tbody td:nth-child(-n+4) {
+    background-color: #f0f8ff !important; /* Light blue background */
+}
+
+/* Style for dimension-specific result columns */
+.custom-dataframe thead th:nth-child(n+5),
+.custom-dataframe tbody td:nth-child(n+5) {
+    background-color: #f0fff0 !important; /* Light green background */
+}
+
+/* Alternating row colors for better readability */
+.custom-dataframe tbody tr:nth-child(even) td:nth-child(-n+4) {
+    background-color: #e6f3ff !important; /* Slightly darker light blue */
+}
+
+.custom-dataframe tbody tr:nth-child(even) td:nth-child(n+5) {
+    background-color: #e6ffe6 !important; /* Slightly darker light green */
+}

static/eval_results/all_summary.json

@@ -0,0 +1,418 @@
+{
+    "GPT_4o": {
+        "core_noncot": {
+            "num_eval_tasks": 440,
+            "num_eval_samples": 6539,
+            "num_not_eval_samples": 0,
+            "num_total_samples": 6961,
+            "macro_mean_score": 0.5187898818829914,
+            "micro_mean_score": 0.5127977300993917
+        },
+        "core_cot": {
+            "num_eval_tasks": 440,
+            "num_eval_samples": 6539,
+            "num_not_eval_samples": 0,
+            "num_total_samples": 6961,
+            "macro_mean_score": 0.5251654337401854,
+            "micro_mean_score": 0.522332974147119
+        },
+        "open": {
+            "num_eval_tasks": 65,
+            "num_eval_samples": 1163,
+            "num_total_samples": 2448,
+            "macro_mean_score": 0.6478225794744895,
+            "micro_mean_score": 0.665391229578676
+        },
+        "overall_score": 0.5409529871515315
+    },
+    "Gemini_1.5_pro_002": {
+        "core_noncot": {
+            "num_eval_tasks": 440,
+            "num_eval_samples": 6539,
+            "num_not_eval_samples": 0,
+            "num_total_samples": 6961,
+            "macro_mean_score": 0.46887846869580546,
+            "micro_mean_score": 0.46403536258864253
+        },
+        "core_cot": {
+            "num_eval_tasks": 440,
+            "num_eval_samples": 6539,
+            "num_not_eval_samples": 0,
+            "num_total_samples": 6961,
+            "macro_mean_score": 0.481393687771543,
+            "micro_mean_score": 0.4756661334397647
+        },
+        "open": {
+            "num_eval_tasks": 65,
+            "num_eval_samples": 1163,
+            "num_total_samples": 2448,
+            "macro_mean_score": 0.5858190649927173,
+            "micro_mean_score": 0.6104901117798793
+        },
+        "overall_score": 0.4948345779089219
+    },
+    "Gemini_1.5_flash_002": {
+        "core_noncot": {
+            "num_eval_tasks": 440,
+            "num_eval_samples": 6539,
+            "num_not_eval_samples": 0,
+            "num_total_samples": 6961,
+            "macro_mean_score": 0.4183865592515826,
+            "micro_mean_score": 0.41216971462683855
+        },
+        "core_cot": {
+            "num_eval_tasks": 440,
+            "num_eval_samples": 6539,
+            "num_not_eval_samples": 0,
+            "num_total_samples": 6961,
+            "macro_mean_score": 0.4183865592515826,
+            "micro_mean_score": 0.41216971462683855
+        },
+        "open": {
+            "num_eval_tasks": 65,
+            "num_eval_samples": 1163,
+            "num_total_samples": 2168,
+            "macro_mean_score": 0.5691365176285039,
+            "micro_mean_score": 0.5987532244196045
+        },
+        "overall_score": 0.4377900192406913
+    },
+    "Claude_3.5": {
+        "core_noncot": {
+            "num_eval_tasks": 440,
+            "num_eval_samples": 6539,
+            "num_not_eval_samples": 0,
+            "num_total_samples": 6961,
+            "macro_mean_score": 0.4863241841253708,
+            "micro_mean_score": 0.4798092874490549
+        },
+        "core_cot": {
+            "num_eval_tasks": 440,
+            "num_eval_samples": 6539,
+            "num_not_eval_samples": 0,
+            "num_total_samples": 6961,
+            "macro_mean_score": 0.5023557473841108,
+            "micro_mean_score": 0.4985442599850241
+        },
+        "open": {
+            "num_eval_tasks": 65,
+            "num_eval_samples": 1163,
+            "num_total_samples": 2288,
+            "macro_mean_score": 0.6373907158949892,
+            "micro_mean_score": 0.6569647463456579
+        },
+        "overall_score": 0.519736485905313
+    },
+    "GPT_4o_mini": {
+        "core_noncot": {
+            "num_eval_tasks": 440,
+            "num_eval_samples": 6539,
+            "num_not_eval_samples": 0,
+            "num_total_samples": 6961,
+            "macro_mean_score": 0.3974259652331149,
+            "micro_mean_score": 0.392578163407945
+        },
+        "core_cot": {
+            "num_eval_tasks": 440,
+            "num_eval_samples": 6539,
+            "num_not_eval_samples": 0,
+            "num_total_samples": 6961,
+            "macro_mean_score": 0.4070959243997505,
+            "micro_mean_score": 0.40376078514357017
+        },
+        "open": {
+            "num_eval_tasks": 65,
+            "num_eval_samples": 1163,
+            "num_total_samples": 1224,
+            "macro_mean_score": 0.586537827213665,
+            "micro_mean_score": 0.6133276010318144
+        },
+        "overall_score": 0.43019240694015537
+    },
+    "Qwen2_VL_72B": {
+        "core_noncot": {
+            "num_eval_tasks": 440,
+            "num_eval_samples": 6539,
+            "num_not_eval_samples": 0,
+            "num_total_samples": 6961,
+            "macro_mean_score": 0.4623988230573754,
+            "micro_mean_score": 0.4568583770401895
+        },
+        "core_cot": {
+            "num_eval_tasks": 440,
+            "num_eval_samples": 6539,
+            "num_not_eval_samples": 0,
+            "num_total_samples": 6961,
+            "macro_mean_score": 0.45284699372478177,
+            "micro_mean_score": 0.4487693487093462
+        },
+        "open": {
+            "num_eval_tasks": 65,
+            "num_eval_samples": 1163,
+            "num_total_samples": 2448,
+            "macro_mean_score": 0.5639771804231668,
+            "micro_mean_score": 0.5835339638865004
+        },
+        "overall_score": 0.4754732650945565
+    },
+    "Qwen2_VL_7B": {
+        "core_noncot": {
+            "num_eval_tasks": 440,
+            "num_eval_samples": 6539,
+            "num_not_eval_samples": 0,
+            "num_total_samples": 6961,
+            "macro_mean_score": 0.34725455697890745,
+            "micro_mean_score": 0.34344091516995323
+        },
+        "core_cot": {
+            "num_eval_tasks": 440,
+            "num_eval_samples": 6539,
+            "num_not_eval_samples": 0,
+            "num_total_samples": 6961,
+            "macro_mean_score": 0.3284357723853296,
+            "micro_mean_score": 0.32443422147119677
+        },
+        "open": {
+            "num_eval_tasks": 65,
+            "num_eval_samples": 1170,
+            "num_total_samples": 2452,
+            "macro_mean_score": 0.43955105763038577,
+            "micro_mean_score": 0.45508547008546996
+        },
+        "overall_score": 0.35913430458751355
+    },
+    "llava_onevision_72B": {
+        "core_noncot": {
+            "num_eval_tasks": 440,
+            "num_eval_samples": 6539,
+            "num_not_eval_samples": 0,
+            "num_total_samples": 6961,
+            "macro_mean_score": 0.31960132549012704,
+            "micro_mean_score": 0.3173848563095166
+        },
+        "core_cot": {
+            "num_eval_tasks": 440,
+            "num_eval_samples": 6539,
+            "num_not_eval_samples": 0,
+            "num_total_samples": 6961,
+            "macro_mean_score": 0.29725827011768174,
+            "micro_mean_score": 0.2954433666362564
+        },
+        "open": {
+            "num_eval_tasks": 65,
+            "num_eval_samples": 1163,
+            "num_total_samples": 1224,
+            "macro_mean_score": 0.4599484231632498,
+            "micro_mean_score": 0.4850386930352536
+        },
+        "overall_score": 0.33766580340844976
+    },
+    "llava_onevision_7B": {
+        "core_noncot": {
+            "num_eval_tasks": 440,
+            "num_eval_samples": 6539,
+            "num_not_eval_samples": 0,
+            "num_total_samples": 6961,
+            "macro_mean_score": 0.2239290419841492,
+            "micro_mean_score": 0.22222171180488767
+        },
+        "core_cot": {
+            "num_eval_tasks": 440,
+            "num_eval_samples": 6539,
+            "num_not_eval_samples": 0,
+            "num_total_samples": 6961,
+            "macro_mean_score": 0.21347545703998197,
+            "micro_mean_score": 0.210586172002703
+        },
+        "open": {
+            "num_eval_tasks": 65,
+            "num_eval_samples": 1163,
+            "num_total_samples": 2448,
+            "macro_mean_score": 0.33979975321921935,
+            "micro_mean_score": 0.36474634565778147
+        },
+        "overall_score": 0.23884309392529685
+    },
+    "InternVL2_76B": {
+        "core_noncot": {
+            "num_eval_tasks": 440,
+            "num_eval_samples": 6539,
+            "num_not_eval_samples": 0,
+            "num_total_samples": 6961,
+            "macro_mean_score": 0.34977582844066846,
+            "micro_mean_score": 0.3452353155814884
+        },
+        "core_cot": {
+            "num_eval_tasks": 440,
+            "num_eval_samples": 6539,
+            "num_not_eval_samples": 0,
+            "num_total_samples": 6961,
+            "macro_mean_score": 0.35539585884136143,
+            "micro_mean_score": 0.35043335903915124
+        },
+        "open": {
+            "num_eval_tasks": 65,
+            "num_eval_samples": 1163,
+            "num_total_samples": 1224,
+            "macro_mean_score": 0.5192997443033639,
+            "micro_mean_score": 0.5421324161650903
+        },
+        "overall_score": 0.37649239855429245
+    },
+    "InternVL2_8B": {
+        "core_noncot": {
+            "num_eval_tasks": 440,
+            "num_eval_samples": 6539,
+            "num_not_eval_samples": 0,
+            "num_total_samples": 6961,
+            "macro_mean_score": 0.25920867490737526,
+            "micro_mean_score": 0.2543416126895087
+        },
+        "core_cot": {
+            "num_eval_tasks": 440,
+            "num_eval_samples": 6539,
+            "num_not_eval_samples": 0,
+            "num_total_samples": 6961,
+            "macro_mean_score": 0.24055897165959364,
+            "micro_mean_score": 0.23784634936127952
+        },
+        "open": {
+            "num_eval_tasks": 65,
+            "num_eval_samples": 1165,
+            "num_total_samples": 2452,
+            "macro_mean_score": 0.3978571701460552,
+            "micro_mean_score": 0.4108583690987125
+        },
+        "overall_score": 0.2770545208291856
+    },
+    "MiniCPM_v2.6": {
+        "core_noncot": {
+            "num_eval_tasks": 440,
+            "num_eval_samples": 6539,
+            "num_not_eval_samples": 0,
+            "num_total_samples": 6961,
+            "macro_mean_score": 0.22838207666977445,
+            "micro_mean_score": 0.22452805919103805
+        },
+        "core_cot": {
+            "num_eval_tasks": 440,
+            "num_eval_samples": 6539,
+            "num_not_eval_samples": 0,
+            "num_total_samples": 6961,
+            "macro_mean_score": 0.22901463640480854,
+            "micro_mean_score": 0.2250606411323753
+        },
+        "open": {
+            "num_eval_tasks": 65,
+            "num_eval_samples": 1163,
+            "num_total_samples": 2448,
+            "macro_mean_score": 0.41728623355613875,
+            "micro_mean_score": 0.43452278589853827
+        },
+        "overall_score": 0.25324761425596987
+    },
+    "Phi-3.5-vision": {
+        "core_noncot": {
+            "num_eval_tasks": 440,
+            "num_eval_samples": 6539,
+            "num_not_eval_samples": 0,
+            "num_total_samples": 6961,
+            "macro_mean_score": 0.23240864879023493,
+            "micro_mean_score": 0.22932978620408923
+        },
+        "core_cot": {
+            "num_eval_tasks": 440,
+            "num_eval_samples": 6539,
+            "num_not_eval_samples": 0,
+            "num_total_samples": 6961,
+            "macro_mean_score": 0.2295097914016776,
+            "micro_mean_score": 0.2266573336398296
+        },
+        "open": {
+            "num_eval_tasks": 65,
+            "num_eval_samples": 1163,
+            "num_total_samples": 2428,
+            "macro_mean_score": 0.3947914647737769,
+            "micro_mean_score": 0.42459157351676696
+        },
+        "overall_score": 0.2533094072831661
+    },
+    "Pixtral_12B": {
+        "core_noncot": {
+            "num_eval_tasks": 440,
+            "num_eval_samples": 6539,
+            "num_not_eval_samples": 0,
+            "num_total_samples": 6961,
+            "macro_mean_score": 0.3186510310643637,
+            "micro_mean_score": 0.3151734861550665
+        },
+        "core_cot": {
+            "num_eval_tasks": 440,
+            "num_eval_samples": 6539,
+            "num_not_eval_samples": 0,
+            "num_total_samples": 6961,
+            "macro_mean_score": 0.3132232487306254,
+            "micro_mean_score": 0.30971424472967524
+        },
+        "open": {
+            "num_eval_tasks": 65,
+            "num_eval_samples": 1163,
+            "num_total_samples": 1224,
+            "macro_mean_score": 0.4566234428542061,
+            "micro_mean_score": 0.4870593293207223
+        },
+        "overall_score": 0.3364098563442444
+    },
+    "Llama_3_2_11B": {
+        "core_noncot": {
+            "num_eval_tasks": 440,
+            "num_eval_samples": 6539,
+            "num_not_eval_samples": 0,
+            "num_total_samples": 6961,
+            "macro_mean_score": 0.10044261716549671,
+            "micro_mean_score": 0.09980638766828835
+        },
+        "core_cot": {
+            "num_eval_tasks": 440,
+            "num_eval_samples": 6539,
+            "num_not_eval_samples": 0,
+            "num_total_samples": 6961,
+            "macro_mean_score": 0.15984490401619783,
+            "micro_mean_score": 0.15794038158731832
+        },
+        "open": {
+            "num_eval_tasks": 65,
+            "num_eval_samples": 1163,
+            "num_total_samples": 1224,
+            "macro_mean_score": 0.3173342406187366,
+            "micro_mean_score": 0.3487962166809973
+        },
+        "overall_score": 0.1801158087274157
+    },
+    "Idefics3": {
+        "core_noncot": {
+            "num_eval_tasks": 440,
+            "num_eval_samples": 6539,
+            "num_not_eval_samples": 0,
+            "num_total_samples": 6961,
+            "macro_mean_score": 0.11118980301103833,
+            "micro_mean_score": 0.11201785633274061
+        },
+        "core_cot": {
+            "num_eval_tasks": 440,
+            "num_eval_samples": 6539,
+            "num_not_eval_samples": 0,
+            "num_total_samples": 6961,
+            "macro_mean_score": 0.08956972487602757,
+            "micro_mean_score": 0.08982225274252693
+        },
+        "open": {
+            "num_eval_tasks": 65,
+            "num_eval_samples": 1163,
+            "num_total_samples": 2448,
+            "macro_mean_score": 0.3210866162255635,
+            "micro_mean_score": 0.35649183147033553
+        },
+        "overall_score": 0.138206224513898
+    }
+}

utils.py

@@ -0,0 +1,148 @@
+import pandas as pd
+import gradio as gr
+import csv
+import json
+import os
+import shutil
+from huggingface_hub import Repository
+import numpy as np
+
+# Load the JSON data
+with open("./static/eval_results/all_model_keywords_stats.json", "r") as f:
+    MODEL_DATA = json.load(f)
+
+with open("./static/eval_results/all_summary.json", "r") as f:
+    SUMMARY_DATA = json.load(f)
+
+
+# Define model name mapping
+MODEL_NAME_MAP = {
+    "GPT_4o": "GPT-4o (0513)",
+    "Claude_3.5": "Claude-3.5-Sonnet",
+    "Gemini_1.5_pro_002": "Gemini-1.5-Pro-002",
+    "InternVL2_76B": "InternVL2-Llama3-76B",
+    "Qwen2_VL_72B": "Qwen2-VL-72B",
+    "llava_onevision_72B": "Llava-OneVision-72B",
+    "GPT_4o_mini": "GPT-4o mini",
+    "Gemini_1.5_flash_002": "Gemini-1.5-Flash-002",
+    "Pixtral_12B": "Pixtral 12B",
+    "Qwen2_VL_7B": "Qwen2-VL-7B",
+    "InternVL2_8B": "InternVL2-8B",
+    "llava_onevision_7B": "Llava-OneVision-7B",
+    "Llama_3_2_11B": "Llama-3.2-11B",
+    "Phi-3.5-vision": "Phi-3.5-Vision",
+    "MiniCPM_v2.6": "MiniCPM-V2.6",
+    "Idefics3": "Idefics3-8B-Llama3",
+}
+
+# Custom name mapping for dimensions and keywords
+DIMENSION_NAME_MAP = {
+    "skills": "Skills",
+    "input_format": "Input Format",
+    "output_format": "Output Format",
+    "input_num": "Visual Input Number",
+    "app": "Application"
+}
+
+KEYWORD_NAME_MAP = {
+    # Skills
+    "Object Recognition and Classification": "Object Recognition",
+    "Text Recognition (OCR)": "OCR",
+    "Language Understanding and Generation": "Language",
+    "Scene and Event Understanding": "Scene/Event",
+    "Mathematical and Logical Reasoning": "Math/Logic",
+    "Commonsense and Social Reasoning": "Commonsense",
+    "Ethical and Safety Reasoning": "Ethics/Safety",
+    "Domain-Specific Knowledge and Skills": "Domain-Specific",
+    "Spatial and Temporal Reasoning": "Spatial/Temporal",
+    "Planning and Decision Making": "Planning/Decision",
+    # Input Format
+    'User Interface Screenshots': "UI related", 
+    'Text-Based Images and Documents': "Documents", 
+    'Diagrams and Data Visualizations': "Infographics", 
+    'Videos': "Videos", 
+    'Artistic and Creative Content': "Arts/Creative", 
+    'Photographs': "Photographs", 
+    '3D Models and Aerial Imagery': "3D related",
+    # Application
+    'Information_Extraction': "Info Extraction", 
+    'Planning' : "Planning", 
+    'Coding': "Coding", 
+    'Perception': "Perception", 
+    'Metrics': "Metrics", 
+    'Science': "Science", 
+    'Knowledge': "Knowledge", 
+    'Mathematics': "Math",
+    # Output format
+    'contextual_formatted_text': "Contexual", 
+    'structured_output': "Structured", 
+    'exact_text': "Exact", 
+    'numerical_data': "Numerical", 
+    'open_ended_output': "Open-ended", 
+    'multiple_choice': "MC",
+    "6-8 images": "6-8 imgs",
+    "1-image": "1 img",
+    "2-3 images": "2-3 imgs",
+    "4-5 images": "4-5 imgs",
+    "9-image or more": "9+ imgs",
+    "video": "Video",
+}
+
+# Extract super groups (dimensions) and their keywords
+SUPER_GROUPS = {DIMENSION_NAME_MAP[dim]: [KEYWORD_NAME_MAP.get(k, k) for k in MODEL_DATA[next(iter(MODEL_DATA))][dim].keys()] 
+                for dim in MODEL_DATA[next(iter(MODEL_DATA))]}
+
+SUBMISSION_NAME = "test_leaderboard_submission"
+SUBMISSION_URL = os.path.join("https://huggingface.co/datasets/cccjc/", SUBMISSION_NAME)
+CSV_DIR = "./test_leaderboard_submission/results.csv"
+
+def get_original_dimension(mapped_dimension):
+    return next(k for k, v in DIMENSION_NAME_MAP.items() if v == mapped_dimension)
+
+def get_original_keyword(mapped_keyword):
+    return next((k for k, v in KEYWORD_NAME_MAP.items() if v == mapped_keyword), mapped_keyword)
+
+# Define model groups
+MODEL_GROUPS = {
+    "All": list(MODEL_DATA.keys()),
+    "Flagship Models": ['GPT_4o', 'Claude_3.5', 'Gemini_1.5_pro_002', 'Qwen2_VL_72B', 'InternVL2_76B', 'llava_onevision_72B'],
+    "Efficienty Models": ['Gemini_1.5_flash_002', 'GPT_4o_mini', 'Qwen2_VL_7B', 'Pixtral_12B', 'InternVL2_8B', 'Phi-3.5-vision', 'MiniCPM_v2.6', 'llava_onevision_7B', 'Llama_3_2_11B', 'Idefics3'],
+    "Proprietary Flagship models": ['GPT_4o', 'Claude_3.5', 'Gemini_1.5_pro_002'],
+    "Open-source Efficienty Models": ['Qwen2_VL_7B', 'Pixtral_12B', 'InternVL2_8B', 'Phi-3.5-vision', 'MiniCPM_v2.6', 'llava_onevision_7B', 'Llama_3_2_11B', 'Idefics3'],
+    "Open-source Flagship Models": ['Qwen2_VL_72B', 'InternVL2_76B', 'llava_onevision_72B'],
+    "Proprietary Efficienty Models": ['Gemini_1.5_flash_002', 'GPT_4o_mini', 'Qwen2_VL_7B', 'Pixtral_12B', 'InternVL2_8B', 'Phi-3.5-vision', 'MiniCPM_v2.6', 'llava_onevision_7B', 'Llama_3_2_11B', 'Idefics3'],
+}
+
+def get_display_model_name(model_name):
+    return MODEL_NAME_MAP.get(model_name, model_name)
+
+def get_df(selected_super_group, selected_model_group):
+    original_dimension = get_original_dimension(selected_super_group)
+    data = []
+    for model in MODEL_GROUPS[selected_model_group]:
+        model_data = MODEL_DATA[model]
+        summary = SUMMARY_DATA[model]
+        core_score = max(summary["core_noncot"]["macro_mean_score"], summary["core_cot"]["macro_mean_score"])
+        row = {
+            "Models": get_display_model_name(model),  # Use the mapped name
+            "Overall": round(summary["overall_score"] * 100, 2),
+            "Core": round(core_score * 100, 2),
+            "Open-ended": round(summary["open"]["macro_mean_score"] * 100, 2)
+        }
+        for keyword in SUPER_GROUPS[selected_super_group]:
+            original_keyword = get_original_keyword(keyword)
+            if original_dimension in model_data and original_keyword in model_data[original_dimension]:
+                row[keyword] = round(model_data[original_dimension][original_keyword]["average_score"] * 100, 2)
+            else:
+                row[keyword] = None
+        data.append(row)
+    
+    df = pd.DataFrame(data)
+    df = df.sort_values(by="Overall", ascending=False)
+    return df
+
+def get_leaderboard_data(selected_super_group, selected_model_group):
+    df = get_df(selected_super_group, selected_model_group)
+    headers = ["Models", "Overall", "Core", "Open-ended"] + SUPER_GROUPS[selected_super_group]
+    data = df[headers].values.tolist()
+    return headers, data