update

app.py

@@ -1,334 +1,233 @@
-import subprocess
+__all__ = ['block', 'make_clickable_model', 'make_clickable_user', 'get_submissions']
+
 import gradio as gr
 import pandas as pd
-from apscheduler.schedulers.background import BackgroundScheduler
-from huggingface_hub import snapshot_download
-
-from src.about import (
-    CITATION_BUTTON_LABEL,
-    CITATION_BUTTON_TEXT,
-    EVALUATION_QUEUE_TEXT,
-    INTRODUCTION_TEXT,
-    LLM_BENCHMARKS_TEXT,
-    TITLE,
-)
-from src.display.css_html_js import custom_css
-from src.display.utils import (
-    BENCHMARK_COLS,
-    COLS,
-    EVAL_COLS,
-    EVAL_TYPES,
-    NUMERIC_INTERVALS,
-    TYPES,
-    AutoEvalColumn,
-    ModelType,
-    fields,
-    WeightType,
-    Precision
-)
-from src.envs import API, EVAL_REQUESTS_PATH, EVAL_RESULTS_PATH, QUEUE_REPO, REPO_ID, RESULTS_REPO, TOKEN
-from src.populate import get_evaluation_queue_df, get_leaderboard_df
-from src.submission.submit import add_new_eval
-
-
-def restart_space():
-    API.restart_space(repo_id=REPO_ID)
-
-try:
-    print(EVAL_REQUESTS_PATH)
-    snapshot_download(
-        repo_id=QUEUE_REPO, local_dir=EVAL_REQUESTS_PATH, repo_type="dataset", tqdm_class=None, etag_timeout=30, token=TOKEN
-    )
-except Exception:
-    restart_space()
-try:
-    print(EVAL_RESULTS_PATH)
-    snapshot_download(
-        repo_id=RESULTS_REPO, local_dir=EVAL_RESULTS_PATH, repo_type="dataset", tqdm_class=None, etag_timeout=30, token=TOKEN
-    )
-except Exception:
-    restart_space()
-
-
-raw_data, original_df = get_leaderboard_df(EVAL_RESULTS_PATH, EVAL_REQUESTS_PATH, COLS, BENCHMARK_COLS)
-leaderboard_df = original_df.copy()
-
-(
-    finished_eval_queue_df,
-    running_eval_queue_df,
-    pending_eval_queue_df,
-) = get_evaluation_queue_df(EVAL_REQUESTS_PATH, EVAL_COLS)
-
-
-# Searching and filtering
-def update_table(
-    hidden_df: pd.DataFrame,
-    columns: list,
-    type_query: list,
-    precision_query: str,
-    size_query: list,
-    show_deleted: bool,
-    query: str,
+import re
+import pdb
+import tempfile
+
+from constants import *
+from src.compute import compute_scores
+
+global data_component, filter_component
+
+
+def validate_model_size(s):
+    pattern = r'^\d+B$|^-$'
+    if re.match(pattern, s):
+        return s
+    else:
+        return '-'
+
+def upload_file(files):
+    file_paths = [file.name for file in files]
+    return file_paths
+
+def add_new_eval(
+    input_file,
+    model_name_textbox: str,
+    revision_name_textbox: str,
+    model_link: str,
+    model_type: str,
+    model_size: str,
 ):
-    filtered_df = filter_models(hidden_df, type_query, size_query, precision_query, show_deleted)
-    filtered_df = filter_queries(query, filtered_df)
-    df = select_columns(filtered_df, columns)
+    if input_file is None:
+        return "Error! Empty file!"
+    else:
+
+        model_size = validate_model_size(model_size)
+
+        input_file = compute_scores(input_file)
+        input_data = input_file[1]
+        input_data = [float(i) for i in input_data]
+
+        csv_data = pd.read_csv(CSV_DIR)
+
+        if revision_name_textbox == '':
+            col = csv_data.shape[0]
+            model_name = model_name_textbox
+            name_list = [name.split(']')[0][1:] if name.endswith(')') else name for name in csv_data['Model']]
+            print(name_list)
+            print(model_name)
+            assert model_name not in name_list
+        else:
+            model_name = revision_name_textbox
+            model_name_list = csv_data['Model']
+            name_list = [name.split(']')[0][1:] if name.endswith(')') else name for name in model_name_list]
+            if revision_name_textbox not in name_list:
+                col = csv_data.shape[0]
+            else:
+                col = name_list.index(revision_name_textbox)
+
+        if model_link == '':
+            model_name = model_name  # no url
+        else:
+            model_name = '[' + model_name + '](' + model_link + ')'
+
+        # add new data
+        new_data = [
+            model_name,
+            model_type,
+            model_size,
+            input_data[0],
+            input_data[1],
+            input_data[2],
+            input_data[3],
+            input_data[4],
+            input_data[5],
+            input_data[6],
+            input_data[7],
+            input_data[8],
+            input_data[9],
+            input_data[10],
+            input_data[11],
+            input_data[12],
+            input_data[13],
+            input_data[14],
+            input_data[15],
+            input_data[16],
+            ]
+        csv_data.loc[col] = new_data 
+        # with open(f'./file/{model_name}.json','w' ,encoding='utf-8') as f:
+        #     json.dump(new_data, f)  
+        csv_data.to_csv(CSV_DIR, index=False)
+    return 0
+
+def get_baseline_df():
+    # pdb.set_trace()
+    df = pd.read_csv(CSV_DIR)
+    df = df.sort_values(by="Avg. All", ascending=False)
+    present_columns = MODEL_INFO + checkbox_group.value
+    df = df[present_columns]
     return df
 
+def get_all_df():
+    df = pd.read_csv(CSV_DIR)
+    df = df.sort_values(by="Avg. All", ascending=False)
+    return df
 
-def search_table(df: pd.DataFrame, query: str) -> pd.DataFrame:
-    return df[(df[AutoEvalColumn.model.name].str.contains(query, case=False))]
-
-
-def select_columns(df: pd.DataFrame, columns: list) -> pd.DataFrame:
-    always_here_cols = [
-        AutoEvalColumn.model_type_symbol.name,
-        AutoEvalColumn.model.name,
-    ]
-    # We use COLS to maintain sorting
-    filtered_df = df[
-        always_here_cols + [c for c in COLS if c in df.columns and c in columns] 
-    ]
-    return filtered_df
+block = gr.Blocks()
 
 
-def filter_queries(query: str, filtered_df: pd.DataFrame) -> pd.DataFrame:
-    final_df = []
-    if query != "":
-        queries = [q.strip() for q in query.split(";")]
-        for _q in queries:
-            _q = _q.strip()
-            if _q != "":
-                temp_filtered_df = search_table(filtered_df, _q)
-                if len(temp_filtered_df) > 0:
-                    final_df.append(temp_filtered_df)
-        if len(final_df) > 0:
-            filtered_df = pd.concat(final_df)
-            filtered_df = filtered_df.drop_duplicates(
-                subset=[AutoEvalColumn.model.name, AutoEvalColumn.precision.name, AutoEvalColumn.revision.name]
+with block:
+    gr.Markdown(
+        LEADERBORAD_INTRODUCTION
+    )
+    with gr.Tabs(elem_classes="tab-buttons") as tabs:
+        with gr.TabItem("🏅 TempCompass Benchmark", elem_id="video-benchmark-tab-table", id=0):
+    
+            gr.Markdown(
+                TABLE_INTRODUCTION
             )
 
-    return filtered_df
-
-
-def filter_models(
-    df: pd.DataFrame, type_query: list, size_query: list, precision_query: list, show_deleted: bool
-) -> pd.DataFrame:
-    # Show all models
-    if show_deleted:
-        filtered_df = df
-    else:  # Show only still on the hub models
-        filtered_df = df[df[AutoEvalColumn.still_on_hub.name] == True]
-
-    type_emoji = [t[0] for t in type_query]
-    filtered_df = filtered_df.loc[df[AutoEvalColumn.model_type_symbol.name].isin(type_emoji)]
-    filtered_df = filtered_df.loc[df[AutoEvalColumn.precision.name].isin(precision_query + ["None"])]
-
-    numeric_interval = pd.IntervalIndex(sorted([NUMERIC_INTERVALS[s] for s in size_query]))
-    params_column = pd.to_numeric(df[AutoEvalColumn.params.name], errors="coerce")
-    mask = params_column.apply(lambda x: any(numeric_interval.contains(x)))
-    filtered_df = filtered_df.loc[mask]
-
-    return filtered_df
-
-
-demo = gr.Blocks(css=custom_css)
-with demo:
-    gr.HTML(TITLE)
-    gr.Markdown(INTRODUCTION_TEXT, elem_classes="markdown-text")
-
-    with gr.Tabs(elem_classes="tab-buttons") as tabs:
-        with gr.TabItem("🏅 TempCompass Benchmark", elem_id="llm-benchmark-tab-table", id=0):
-            with gr.Row():
-                with gr.Column():
-                    with gr.Row():
-                        search_bar = gr.Textbox(
-                            placeholder=" 🔍 Search for your model (separate multiple queries with `;`) and press ENTER...",
-                            show_label=False,
-                            elem_id="search-bar",
-                        )
-                    with gr.Row():
-                        shown_columns = gr.CheckboxGroup(
-                            choices=[
-                                c.name
-                                for c in fields(AutoEvalColumn)
-                                if not c.hidden and not c.never_hidden
-                            ],
-                            value=[
-                                c.name
-                                for c in fields(AutoEvalColumn)
-                                if c.displayed_by_default and not c.hidden and not c.never_hidden
-                            ],
-                            label="Select columns to show",
-                            elem_id="column-select",
-                            interactive=True,
-                        )
-                    with gr.Row():
-                        deleted_models_visibility = gr.Checkbox(
-                            value=False, label="Show gated/private/deleted models", interactive=True
-                        )
-                with gr.Column(min_width=320):
-                    #with gr.Box(elem_id="box-filter"):
-                    filter_columns_type = gr.CheckboxGroup(
-                        label="Model types",
-                        choices=[t.to_str() for t in ModelType],
-                        value=[t.to_str() for t in ModelType],
-                        interactive=True,
-                        elem_id="filter-columns-type",
-                    )
-                    filter_columns_precision = gr.CheckboxGroup(
-                        label="Precision",
-                        choices=[i.value.name for i in Precision],
-                        value=[i.value.name for i in Precision],
-                        interactive=True,
-                        elem_id="filter-columns-precision",
-                    )
-                    filter_columns_size = gr.CheckboxGroup(
-                        label="Model sizes (in billions of parameters)",
-                        choices=list(NUMERIC_INTERVALS.keys()),
-                        value=list(NUMERIC_INTERVALS.keys()),
-                        interactive=True,
-                        elem_id="filter-columns-size",
-                    )
+            # selection for column part:
+            checkbox_group = gr.CheckboxGroup(
+                choices=TASK_INFO_v2,
+                value=AVG_INFO,
+                label="Select options",
+                interactive=True,
+            )
 
-            leaderboard_table = gr.components.Dataframe(
-                value=leaderboard_df[
-                    [c.name for c in fields(AutoEvalColumn) if c.never_hidden]
-                    + shown_columns.value
-                ],
-                headers=[c.name for c in fields(AutoEvalColumn) if c.never_hidden] + shown_columns.value,
-                datatype=TYPES,
-                elem_id="leaderboard-table",
+            # 创建数据帧组件
+            data_component = gr.components.Dataframe(
+                value=get_baseline_df, 
+                headers=COLUMN_NAMES,
+                type="pandas", 
+                datatype=DATA_TITILE_TYPE,
                 interactive=False,
                 visible=True,
-            )
-
-            # Dummy leaderboard for handling the case when the user uses backspace key
-            hidden_leaderboard_table_for_search = gr.components.Dataframe(
-                value=original_df[COLS],
-                headers=COLS,
-                datatype=TYPES,
-                visible=False,
-            )
-            search_bar.submit(
-                update_table,
-                [
-                    hidden_leaderboard_table_for_search,
-                    shown_columns,
-                    filter_columns_type,
-                    filter_columns_precision,
-                    filter_columns_size,
-                    deleted_models_visibility,
-                    search_bar,
-                ],
-                leaderboard_table,
-            )
-            for selector in [shown_columns, filter_columns_type, filter_columns_precision, filter_columns_size, deleted_models_visibility]:
-                selector.change(
-                    update_table,
-                    [
-                        hidden_leaderboard_table_for_search,
-                        shown_columns,
-                        filter_columns_type,
-                        filter_columns_precision,
-                        filter_columns_size,
-                        deleted_models_visibility,
-                        search_bar,
-                    ],
-                    leaderboard_table,
-                    queue=True,
                 )
+    
+            def on_checkbox_group_change(selected_columns):
+                # pdb.set_trace()
+                selected_columns = [item for item in TASK_INFO_v2 if item in selected_columns]
+                present_columns = MODEL_INFO + selected_columns
+                updated_data = get_all_df()[present_columns]
+                updated_data = updated_data.sort_values(by=present_columns[1], ascending=False)
+                updated_headers = present_columns
+                update_datatype = [DATA_TITILE_TYPE[COLUMN_NAMES.index(x)] for x in updated_headers]
+
+                filter_component = gr.components.Dataframe(
+                    value=updated_data, 
+                    headers=updated_headers,
+                    type="pandas", 
+                    datatype=update_datatype,
+                    interactive=False,
+                    visible=True,
+                    )
+                # pdb.set_trace()
+        
+                return filter_component.value
+
+            # 将复选框组关联到处理函数
+            checkbox_group.change(fn=on_checkbox_group_change, inputs=checkbox_group, outputs=data_component)
+        '''
+        # table 2
+        with gr.TabItem("📝 About", elem_id="seed-benchmark-tab-table", id=2):
+            gr.Markdown(LEADERBORAD_INFO, elem_classes="markdown-text")
+        '''
+        # table 3 
+        with gr.TabItem("🚀 Submit here! ", elem_id="seed-benchmark-tab-table", id=3):
+            gr.Markdown(LEADERBORAD_INTRODUCTION, elem_classes="markdown-text")
 
-        # with gr.TabItem("📝 About", elem_id="llm-benchmark-tab-table", id=2):
-        #     gr.Markdown(LLM_BENCHMARKS_TEXT, elem_classes="markdown-text")
-
-        with gr.TabItem("🚀 Submit here! ", elem_id="llm-benchmark-tab-table", id=3):
-            with gr.Column():
-                with gr.Row():
-                    gr.Markdown(EVALUATION_QUEUE_TEXT, elem_classes="markdown-text")
-
-                with gr.Column():
-                    with gr.Accordion(
-                        f"✅ Finished Evaluations ({len(finished_eval_queue_df)})",
-                        open=False,
-                    ):
-                        with gr.Row():
-                            finished_eval_table = gr.components.Dataframe(
-                                value=finished_eval_queue_df,
-                                headers=EVAL_COLS,
-                                datatype=EVAL_TYPES,
-                                row_count=5,
-                            )
-                    with gr.Accordion(
-                        f"🔄 Running Evaluation Queue ({len(running_eval_queue_df)})",
-                        open=False,
-                    ):
-                        with gr.Row():
-                            running_eval_table = gr.components.Dataframe(
-                                value=running_eval_queue_df,
-                                headers=EVAL_COLS,
-                                datatype=EVAL_TYPES,
-                                row_count=5,
-                            )
+            with gr.Row():
+                gr.Markdown(SUBMIT_INTRODUCTION, elem_classes="markdown-text")
 
-                    with gr.Accordion(
-                        f"⏳ Pending Evaluation Queue ({len(pending_eval_queue_df)})",
-                        open=False,
-                    ):
-                        with gr.Row():
-                            pending_eval_table = gr.components.Dataframe(
-                                value=pending_eval_queue_df,
-                                headers=EVAL_COLS,
-                                datatype=EVAL_TYPES,
-                                row_count=5,
-                            )
             with gr.Row():
-                gr.Markdown("# ✉️✨ Submit your model here!", elem_classes="markdown-text")
+                gr.Markdown("# ✉️✨ Submit your model evaluation json file here!", elem_classes="markdown-text")
 
             with gr.Row():
                 with gr.Column():
-                    model_name_textbox = gr.Textbox(label="Model name")
-                    revision_name_textbox = gr.Textbox(label="Revision commit", placeholder="main")
+                    model_name_textbox = gr.Textbox(
+                        label="Model name", placeholder="Chat-UniVi-7B"
+                        )
+                    revision_name_textbox = gr.Textbox(
+                        label="Revision Model Name", placeholder="Chat-UniVi-7B"
+                    )
+                    model_link = gr.Textbox(
+                        label="Model Link", placeholder="https://github.com/PKU-YuanGroup/Chat-UniVi"
+                    )
                     model_type = gr.Dropdown(
-                        choices=[t.to_str(" : ") for t in ModelType if t != ModelType.Unknown],
+                        choices=[                         
+                            "LLM",
+                            "ImageLLM",
+                            "VideoLLM",
+                            "Other", 
+                        ], 
                         label="Model type",
                         multiselect=False,
                         value=None,
                         interactive=True,
                     )
-
-                with gr.Column():
-                    precision = gr.Dropdown(
-                        choices=[i.value.name for i in Precision if i != Precision.Unknown],
-                        label="Precision",
-                        multiselect=False,
-                        value="float16",
-                        interactive=True,
-                    )
-                    weight_type = gr.Dropdown(
-                        choices=[i.value.name for i in WeightType],
-                        label="Weights type",
-                        multiselect=False,
-                        value="Original",
-                        interactive=True,
+                    model_size = gr.Textbox(
+                        label="Model size", placeholder="7B(Input content format must be 'number+B' or '-', default is '-')"
                     )
-                    base_model_name_textbox = gr.Textbox(label="Base model (for delta or adapter weights)")
 
-            submit_button = gr.Button("Submit Eval")
-            submission_result = gr.Markdown()
-            submit_button.click(
-                add_new_eval,
-                [
-                    model_name_textbox,
-                    base_model_name_textbox,
-                    revision_name_textbox,
-                    precision,
-                    weight_type,
-                    model_type,
-                ],
-                submission_result,
-            )
+            with gr.Column():
+
+                input_file = gr.File(label="Click to Upload a json File", type='binary')
+                submit_button = gr.Button("Submit Eval")
+    
+                submission_result = gr.Markdown()
+                submit_button.click(
+                    add_new_eval,
+                    inputs=[
+                        input_file,
+                        model_name_textbox,
+                        revision_name_textbox,
+                        model_link,
+                        model_type,
+                        model_size,
+                    ],
+                    # outputs = submission_result,
+                )
 
+    with gr.Row():
+        data_run = gr.Button("Refresh")
+        data_run.click(
+            get_baseline_df, outputs=data_component
+        )
+    
     with gr.Row():
         with gr.Accordion("📙 Citation", open=False):
             citation_button = gr.Textbox(
@@ -339,7 +238,6 @@ with demo:
                 show_copy_button=True,
             )
 
-scheduler = BackgroundScheduler()
-scheduler.add_job(restart_space, "interval", seconds=1800)
-scheduler.start()
-demo.queue(default_concurrency_limit=40).launch()
+    # block.load(get_baseline_df, outputs=data_title)
+
+block.launch()

constants.py

@@ -0,0 +1,94 @@
+# this is .py for store constants 
+MODEL_INFO = ["Model"]
+
+TASK_INFO_v2 = ["Avg. All", "Avg. Video-Exclusive", "Avg. Prior-Knowledge QA", "Avg. Decision-Making",
+                "ActivityNet", "MSVD", "MSRVTT", "TGIF", "Youcook2", "Ucfcrime", "MOT",
+                "TVQA", "MV", "NBA",
+                "Driving-exam", "Driving-decision-making", "SQA3D"]
+
+AVG_INFO = ["Avg. All", "Avg. Video-Exclusive", "Avg. Prior-Knowledge QA", "Avg. Decision-Making"]
+DATA_TITILE_TYPE = ["markdown",
+                    "number", "number", "number", "number", "number", "number", "number",
+                    "number", "number", "number",
+                    "number", "number", "number",
+                    "number", "number", "number", "number", ]
+CSV_DIR = "./file/result.csv"
+
+# COLUMN_NAMES = MODEL_INFO + TASK_INFO
+COLUMN_NAMES = MODEL_INFO + TASK_INFO_v2
+
+LEADERBORAD_INTRODUCTION = """
+Welcome to the leaderboard of TempCompass! 🏆
+
+TempCompass is a benchmark to evaluate the temporal perception ability of Video LLMs. It consists of 410 videos and 7,540 task instructions, covering 11 temporal aspects and 4 task types. Please refer to [our paper](https://arxiv.org/abs/2403.00476) for more details.
+"""
+
+SUBMIT_INTRODUCTION = """
+# TempCompass Leaderboard
+
+Welcome to the leaderboard of the Video-Bench! 🏆
+
+## Submit Instruction
+Run inference and automatic evaluation according to our [github repository](https://github.com/llyx97/TempCompass?tab=readme-ov-file#-quick-start). 
+
+You will obtain the JSON file `<task_type>.json`, where `<task_type>` correspond to one of the four categories: `multi-choice`, `yes_no`, `caption_matching` and `captioning`. (Example files can be found [here](https://github.com/llyx97/TempCompass/tree/main/auto_eval_results/video-llava))
+
+For `multi-choice`, `yes_no`, `caption_matching`, the evaluation result of each question contains five keys. A specific example is as follows:
+```python
+{
+    "question": "What activity is the monkey engaged in?\\nA. swimming\\nB. running\\nC. climbing\\nD. fighting",
+    "gt-answer": "D. fighting",
+    "video-llm-prediction": "D",
+    "match_success": true,  # whether the video-llm-prediction can be assessed by rule-based matching
+    "rating": 1
+}
+```
+
+For `captioning`, we prompt chatgpt to answer the multi-choice question, using the Video LLM generated caption as context. An example of evalution result is as follows:
+```python
+{
+    "chatgpt-reasoning": "The video description specifically mentions that the man is dribbling a basketball, dunking a basketball, and passing a basketball.",
+    "chatgpt-answer": "B. dribbling a basketball, C. passing a basketball",
+    "video-llm-prediction": "The video showcases a man dribbling a basketball, dunking a basketball, and passing a basketball. The man is seen moving around the court while performing these actions. The video captures the man's movements and the sound of the ball bouncing on the court. The man's dribbling skills are impressive, and he seems to be in control of the ball at all times. The dunking and passing actions are also executed with precision, and the man's movements are fluid and graceful. Overall, the video is a great display of basketball skills and is sure to impress any basketball",
+    "gt-answer": "A. dunking a basketball",
+    "rating": 0
+}
+```
+
+
+### Submit Example
+For example, if you want to submit Video-LLaVA's result in the leaderboard, you need to:
+1. Fill in ‘Video-LLaVA’ in ‘Model Name’ if it is your first time to submit your result (You can leave ‘Revision Model Name’ blank).
+2. Fill in ‘Video-LLaVA’ in ‘Revision Model Name’ if you want to update your result (You can leave ‘Model Name’ blank).
+3. Select ‘ImageLLM’ in ‘Model Type’.
+4. Fill in ‘https://github.com/x/x’ in ‘Model Link’.
+5. Fill in ‘7B’ in ‘Model size’.
+6. Upload `<task_type>.json`.
+7. Click the ‘Submit Eval’ button.
+8. Click ‘Refresh’ to obtain the uploaded leaderboard.
+
+"""
+
+TABLE_INTRODUCTION = """In the table below, we summarize each task performance of all the models.
+        We use accurancy(%) as the primary evaluation metric for each tasks. 
+    """
+
+LEADERBORAD_INFO = """
+      Based on powerful Large Language Models (LLMs), recent generative Multimodal Large Language Models (MLLMs) have gained prominence as a pivotal research area, exhibiting remarkable capability for both comprehension and generation. 
+      In this work, we address the evaluation of generative comprehension in MLLMs as a preliminary step towards a comprehensive assessment of generative models, by introducing a benchmark named SEED-Bench. 
+      SEED-Bench consists of 19K multiple choice questions with accurate human annotations (x6 larger than existing benchmarks), which spans 12 evaluation dimensions including the comprehension of both the image and video modality. 
+      We develop an advanced pipeline for generating multiple-choice questions that target specific evaluation dimensions, integrating both automatic filtering and manual verification processes. 
+      Multiple-choice questions with groundtruth options derived from human annotation enables an objective and efficient assessment of model performance, eliminating the need for human or GPT intervention during evaluation. 
+      We further evaluate the performance of 18 models across all 12 dimensions, covering both the spatial and temporal understanding. 
+      By revealing the limitations of existing MLLMs through evaluation results, we aim for SEED-Bench to provide insights for motivating future research.
+"""
+
+CITATION_BUTTON_LABEL = "Copy the following snippet to cite these results"
+CITATION_BUTTON_TEXT = r"""
+@article{liu2024tempcompass,
+  title   = {TempCompass: Do Video LLMs Really Understand Videos?},
+  author  = {Yuanxin Liu and Shicheng Li and Yi Liu and Yuxiang Wang and Shuhuai Ren and Lei Li and Sishuo Chen and Xu Sun and Lu Hou},
+  year    = {2024},
+  journal = {arXiv preprint arXiv: 2403.00476}
+}
+"""

file/result.csv

@@ -0,0 +1,13 @@
+Model,Avg. All,Avg. Video-Exclusive,Avg. Prior-Knowledge QA,Avg. Decision-Making,ActivityNet,MSVD,MSRVTT,TGIF,Youcook2,Ucfcrime,MOT,TVQA,MV,NBA,Driving-exam,Driving-decision-making,SQA3D
+Random,28.45459441,25.84861538,24.47045673,35.04471112,0.3458,0.26224,0.265,0.22377,0.25,0.25,0.1667,0.2,0.26151895,0.272594752,0.368055556,0.44209,0.25
+[VideoChat-7B](https://github.com/OpenGVLab/Ask-Anything),35.41215477,34.12376923,29.60966667,42.5030284,0.4455,0.4215,0.374,0.33744,0.27663,0.2241,0.27775,0.2615,0.34109,0.2857,0.388888,0.553846,0.31428571
+[Video-ChatGPT-7B](https://github.com/mbzuai-oryx/Video-ChatGPT),38.5186297,39.81651709,29.244,46.495372,0.466,0.575,0.463,0.3559,0.348,0.2413,0.277747222,0.28764,0.3652,0.22448,0.4166666,0.582051,0.372
+[Otter-7B](https://github.com/Luodian/Otter),37.47000387,37.51728162,32.99,41.90273,0.443,0.5495,0.4695,0.34266,0.3265,0.22413,0.166666611,0.2765,0.370635,0.342565,0.5277777,0.4871794,0.2965
+[PandaGPT-7B](https://github.com/yxuansu/PandaGPT),37.52393217,37.53914677,31.98733333,43.0453164,0.449624,0.5042521,0.44594594,0.29663,0.33016,0.3301,0.166665,0.2785,0.37063,0.31049,0.4166,0.5602564,0.30757651
+[Valley-7B](https://github.com/RupertLuo/Valley),33.95521521,28.38772829,29.20933333,44.268584,0.381,0.32032,0.2802802,0.3141,0.2905,0.203448,0.111108278,0.237,0.32587,0.31341,0.41666,0.5653846,0.333
+[mPLUG-owl-7B](https://github.com/X-PLUG/mPLUG-Owl),33.14659856,33.16526701,26.39762867,39.8769,0.41470735,0.4245,0.363,0.31656,0.2705,0.2275862,0.277777611,0.2395,0.3017,0.25072886,0.333333,0.510256,0.32
+[Video-LLaMA-7B](https://github.com/DAMO-NLP-SG/Video-LLaMA),32.83174044,32.48401966,27.79906667,38.212135,0.3985,0.4115,0.3405,0.312766,0.289,0.275862,0.166666556,0.2475,0.324082,0.26239,0.30555555,0.4910256,0.3115
+[Chat-UniVi-7B](https://github.com/PKU-YuanGroup/Chat-UniVi),35.31147004,37.87,27.43,40.64,0.49,0.486,0.4165,0.413,0.29,0.2827,0.166666649,0.2305,0.3357,0.2566,0.3889,0.5308,0.2907
+sphinx-v2,45.53190476,44.22571429,41.81666667,50.55333333,0.5307,0.6845,0.5395,0.5341,0.42,0.2759,0.1111,0.3645,0.4396,0.4504,0.4722,0.5564,0.488
+Gemini,49.598478632478624,50.63076923076923,47.93666666666667,50.228,0.585,0.6179,0.4742,0.5305,0.4769,0.5477,0.1176,0.4656,0.5318,0.4407,0.5285,0.4129
+llava_phi_2.7,43.41644444444445,42.97,37.54333333333334,49.736,0.5785,0.608,0.514,0.4542,0.4345,0.1483,0.1111,0.392,0.4763,0.258,0.5538,0.4535

file/result.csv.bak

@@ -0,0 +1,5 @@
+Model,Avg. All,Avg. Multi-Choice,Avg. Yes/No,Avg. Caption Matching,Avg. Caption Generation,Action. Multi-Choice,Action. Yes/No,Action. Caption Matching,Action. Caption Generation,Direction. Multi-Choice,Direction. Yes/No,Direction. Caption Matching,Direction. Caption Generation,Speed. Multi-Choice,Speed. Yes/No,Speed. Caption Matching,Speed. Caption Generation,Event Order. Multi-Choice,Event Order. Yes/No,Event Order. Caption Matching,Event Order. Caption Generation,Attribute Change. Multi-Choice,Attribute Change. Yes/No,Attribute Change. Caption Matching,Attribute Change. Caption Generation
+Random,48.31,66.71,33.8,61.53,47.24,18.16,30.12,21.56,64.13,83.28,70.82,72.75,72.49,83.65,65.98,60.6,67.75,39.83,10.06,48.97,73.41,28.69,25.93,90.31,65.94
+[VideoChat-7B](https://github.com/OpenGVLab/Ask-Anything),26.47,94.12,42.23,55.56,71.9,35.08,86.8,97.23,95.45,91.23,69.17,19.82,45.5,32.3,48.16,31.83,19.13,44.73,20.71,36.68,61.13,87.71,28.19,26.12,16.33
+Gemini,5.1,61.4,65.71,35.03,50.61,12.5,18.74,33.16,8.16,21.18,3.02,37.25,75.82,87.79,31.66,83.32,41.48,47.26,33.73,54.57,31.64,58.51,4.88,55.22,65.75
+llava_phi_2.7,97.64,81.61,39.3,54.9,17.11,33.57,13.78,76.95,90.81,3.07,5.98,14.63,23.62,15.46,88.03,22.58,21.46,88.25,35.72,85.05,58.54,86.19,74.07,57.24,0.9

merge_eval_result.py

@@ -0,0 +1,14 @@
+import json, os
+
+eval_result_path = "file/example_eval_results"
+eval_result_files = [f for f in os.listdir(eval_result_path) if f.endswith('.json')]
+
+merged_result = {}
+for fn in eval_result_files:
+    task_type = fn.replace('.json', '')
+    with open(f"{eval_result_path}/{fn}", "r") as f:
+        merged_result[task_type] = json.load(f)
+
+merge_file = f"{eval_result_path}/merged_result.json"
+with open(merge_file, "w") as f:
+    json.dump(merged_result, f, indent=4)

src/compute.py

@@ -0,0 +1,121 @@
+import json
+import os
+import glob
+import argparse
+import csv
+
+
+def chatgpt_json(merge_file):
+    # chat results
+    merge_data = merge_file.decode("utf-8")
+    merge_data = eval(merge_data)
+    correct_answer_file = 'file/ANSWER.json'
+    with open(correct_answer_file, 'r', encoding='utf-8') as f:
+        correct_answer_data = json.load(f)
+
+    dataset_scores_dict = {}
+    for dataset_name, item in merge_data.items():
+
+        total_nums = len(item)
+        correct = 0
+        # assert len(item) >= len(correct_answer_data[dataset_name]), f'Video-Bench-Input.json---{dataset_name}---is incomplete!'
+        for id, sub_item in item.items():
+            if sub_item['output_chatgpt_choice'] == correct_answer_data[dataset_name][id]['answer']:
+                correct += 1
+
+        # dataset_scores_dict[dataset_name] = round(correct / total_nums * 100, 2)
+        dataset_scores_dict[dataset_name] = round(correct / total_nums , 4)
+    return dataset_scores_dict
+
+
+def compute_scores(merge_file):
+    dataset_score_dict = chatgpt_json(merge_file)
+    dataset_weight = {
+        1:
+            {
+                "ActivityNet": 1,
+                "MSVD": 1,
+                "MSRVTT": 1,
+                "TGIF": 1,
+                "Youcook2": 1,
+                "Ucfcrime": 1,
+                "MOT": 0.5,
+            },
+
+        2:
+            {
+                "TVQA": 1,
+                "MV": 1,
+                "NBA": 1,
+            },
+
+        3:
+            {
+                "Driving-exam": 0.5,
+                "Driving-decision-making": 1,
+                "SQA3D": 1,
+            }
+
+    }
+
+    # Video-exclusive Understanding score
+    exclusive_understanding_weight = dataset_weight[1]
+    weights_sum = sum(exclusive_understanding_weight.values())
+    exclusive_understanding_score = 0
+    # import ipdb; ipdb.set_trace()
+    for dataset_name, weight in exclusive_understanding_weight.items():
+        exclusive_understanding_score += weight * dataset_score_dict[dataset_name] / weights_sum * 100
+
+    # Prior Knowledge-based Question-answer
+    prior_QA_weight = dataset_weight[2]
+    weights_sum = sum(prior_QA_weight.values())
+    prior_QA_score = 0
+    for dataset_name, weight in prior_QA_weight.items():
+        prior_QA_score += weight * dataset_score_dict[dataset_name] / weights_sum *100
+
+    # Comprehension and Decision-making
+    com_and_dec_QA_weight = dataset_weight[3]
+    weights_sum = sum(com_and_dec_QA_weight.values())
+    com_and_dec_QA_score = 0
+    for dataset_name, weight in com_and_dec_QA_weight.items():
+        com_and_dec_QA_score += weight * dataset_score_dict[dataset_name] / weights_sum *100
+
+    dataset_score_dict['Exclusive_understanding'] = exclusive_understanding_score
+    dataset_score_dict['Prior_Knowledge'] = prior_QA_score
+    dataset_score_dict['Comprehension_and_Decision-making'] = com_and_dec_QA_score
+
+    # final score
+    final_score = sum([exclusive_understanding_score, prior_QA_score, com_and_dec_QA_score]) / 3
+    dataset_score_dict['final_score'] = final_score
+
+    # print(dataset_score_dict)
+    # with open(args.score_output_file, 'w', encoding='utf-8') as f:
+    #   json.dump(dataset_score_dict, f, indent=2)
+    # print(f'{args.score_output_file} is saved!')
+    # ========================
+    data = [
+
+        ["Avg. All", "Avg. Video-Exclusive", "Avg. Prior-Knowledge QA", "Avg. Decision-Making",
+         "ActivityNet", "MSVD", "MSRVTT", "TGIF", "Youcook2", "Ucfcrime",
+         "MOT", "TVQA", "MV", "NBA", "Driving-exam", "Driving-decision-making", "SQA3D"],
+
+        [final_score, exclusive_understanding_score, prior_QA_score, com_and_dec_QA_score,
+         dataset_score_dict['ActivityNet'],
+         dataset_score_dict["MSVD"],
+         dataset_score_dict['MSRVTT'],
+         dataset_score_dict['TGIF'],
+         dataset_score_dict['Youcook2'],
+         dataset_score_dict['Ucfcrime'],
+         dataset_score_dict['MOT'],
+         dataset_score_dict['TVQA'],
+         dataset_score_dict['MV'],
+         dataset_score_dict['NBA'],
+         dataset_score_dict['Driving-exam'],
+         dataset_score_dict['Driving-decision-making'],
+         dataset_score_dict['SQA3D'],
+         ],
+    ]
+
+
+    return data
+