initial commit

app.py

@@ -0,0 +1,82 @@
+import gradio as gr
+from lb_info import load_results, BUILD_L1_DF
+from desc import (
+    LEADERBOARD_INTRODUCTION,
+    LEADERBOARD_MD,
+    CITATION_BUTTON_TEXT,
+    CITATION_BUTTON_LABEL,
+    CINEPILE_ABOUT_MD,
+)
+from urllib.request import urlopen
+
+
+def filter_df(fields):
+    # Use set operations to avoid duplicates
+    headers = (
+        [
+            "Model",
+            "Params (B)",
+            "Average Accuracy",
+        ]
+        + fields
+        + [
+            "Average Rank",
+        ]
+    )
+
+    # Remove duplicates in headers by keeping the earliest entry
+    headers = list(dict.fromkeys(headers))
+    return table[headers]
+
+
+with gr.Blocks(theme=gr.themes.Soft()) as demo:
+    struct = load_results()
+    results = struct
+
+    # Build leaderboard DataFrame for CinePile data
+    table, check_box = BUILD_L1_DF(results)
+
+    N_MODELS = len(table)
+    UP_TS = "20th October 2024"  # Replace with actual timestamp
+
+    gr.Markdown(LEADERBOARD_INTRODUCTION.format(N_MODELS, UP_TS))
+
+    with gr.Tabs(elem_classes="tab-buttons") as tabs:
+        # First Tab: CinePile Leaderboard
+        with gr.TabItem("CinePile Leaderboard", elem_id="main"):
+            gr.Markdown(LEADERBOARD_MD)
+
+            # Checkbox for selecting question categories
+            checkbox_group = gr.CheckboxGroup(
+                choices=check_box["question_categories"],
+                label="Question Categories",
+                interactive=True,
+            )
+
+            # DataFrame component for displaying the leaderboard
+            data_component = gr.DataFrame(
+                value=table[check_box["essential"]],
+                datatype=[check_box["type_map"][x] for x in check_box["essential"]],
+                interactive=False,
+                visible=True,
+            )
+
+            # Update the table when checkbox changes
+            checkbox_group.change(
+                fn=filter_df, inputs=checkbox_group, outputs=data_component
+            )
+
+        # Second Tab: About
+        with gr.TabItem("About CinePile", elem_id="about"):
+            gr.Markdown(urlopen(CINEPILE_ABOUT_MD).read().decode())
+
+        # Add citation support under "About"
+        with gr.Row():
+            with gr.Accordion("Citation", open=False):
+                citation_button = gr.Textbox(
+                    value=CITATION_BUTTON_TEXT,
+                    label=CITATION_BUTTON_LABEL,
+                    elem_id="citation-button",
+                )
+
+demo.launch()

desc.py

@@ -0,0 +1,24 @@
+LEADERBOARD_INTRODUCTION = """# CinePile Leaderboard
+### CinePile is a long video understanding dataset & benchmark. The leaderboard presents the evaluation results of various VLMs on the CinePile benchmark.
+### Currently, CinePile Leaderboard covers {} different VLMs. 
+
+This leaderboard was last updated: {}. 
+"""
+LEADERBOARD_MD = """
+## Main Evaluation Results
+
+- Average Accuracy: The average accuracy on all question categories (normalized to 0 - 100, the higher the better). 
+- Average Rank: The average rank based on the average accuracy (the lower the better). 
+"""
+
+
+# CONSTANTS-CITATION
+CITATION_BUTTON_TEXT = r"""@article{rawal2024cinepile,
+  title={Cinepile: A long video question answering dataset and benchmark},
+  author={Rawal, Ruchit and Saifullah, Khalid and Basri, Ronen and Jacobs, David and Somepalli, Gowthami and Goldstein, Tom},
+  journal={arXiv preprint arXiv:2405.08813},
+  year={2024}
+}"""
+CITATION_BUTTON_LABEL = "Copy the following snippet to cite these results"
+
+CINEPILE_ABOUT_MD = "https://raw.githubusercontent.com/JARVVVIS/cinepile_leaderboard/refs/heads/main/assets/about_cinepile.md"

lb_info.py

@@ -0,0 +1,76 @@
+import pandas as pd
+from collections import defaultdict
+import copy as cp
+import numpy as np
+import json
+import requests
+
+## Load CinePile Data from URL
+RESULTS_URL = "https://raw.githubusercontent.com/JARVVVIS/cinepile_leaderboard/refs/heads/main/assets/cinepile_results.json"
+cinepile_data = json.loads(requests.get(RESULTS_URL).text)
+
+
+# Function to build the leaderboard DataFrame
+def BUILD_L1_DF(data):
+    res = defaultdict(list)
+    for item in data:
+        res["Model"].append(item["Model"])
+        res["Params (B)"].append(item["Params"].split("B")[0])
+        res["Average Accuracy"].append(item["Avg"])
+        res["CRD"].append(item["CRD"])
+        res["NPA"].append(item["NPA"])
+        res["STA"].append(item["STA"])
+        res["TEMP"].append(item["TEMP"])
+        res["TH"].append(item["TH"])
+
+    # Build DataFrame and rank by average score
+    df = pd.DataFrame(res)
+    df["Average Rank"] = df["Average Accuracy"].rank(ascending=False)
+    df = df.sort_values(by="Average Rank")
+
+    check_box = {
+        "essential": [
+            "Model",
+            "Params (B)",
+            "Average Accuracy",
+            "Average Rank",
+        ],
+        "question_categories": ["CRD", "NPA", "STA", "TEMP", "TH"],
+        "required": ["Average Accuracy", "Average Rank"],
+        "all": [
+            "Model",
+            "Params (B)",
+            "Average Accuracy",
+            "CRD",
+            "NPA",
+            "STA",
+            "TEMP",
+            "TH",
+            "Average Rank",
+        ],
+        "type_map": defaultdict(
+            lambda: "number", {"Model": "str", "Params (B)": "str"}
+        ),
+    }
+    return df, check_box
+
+
+def load_results():
+    # Simulate loading CinePile data (replace with actual data loading if necessary)
+    return cinepile_data
+
+
+def format_timestamp(timestamp):
+    return (
+        timestamp[:2]
+        + "."
+        + timestamp[2:4]
+        + "."
+        + timestamp[4:6]
+        + " "
+        + timestamp[6:8]
+        + ":"
+        + timestamp[8:10]
+        + ":"
+        + timestamp[10:12]
+    )