Upload 9 files

app.py

@@ -0,0 +1,304 @@
+
+__all__ = ['block', 'make_clickable_model', 'make_clickable_user', 'get_submissions']
+
+import gradio as gr
+import pandas as pd
+import json
+import pdb
+import tempfile
+
+from constants import *
+from src.auto_leaderboard.model_metadata_type import ModelType
+
+global data_component, filter_component
+
+
+def upload_file(files):
+    file_paths = [file.name for file in files]
+    return file_paths
+
+def prediction_analyse(prediction_content):
+    # pdb.set_trace()
+    predictions = prediction_content.split("\n")
+
+    # 读取 ground_truth JSON 文件
+    with open("./file/SEED-Bench.json", "r") as file:
+        ground_truth_data = json.load(file)["questions"]
+
+    # 将 ground_truth 数据转换为以 question_id 为键的字典
+    ground_truth = {item["question_id"]: item for item in ground_truth_data}
+
+    # 初始化结果统计字典
+    results = {i: {"correct": 0, "total": 0} for i in range(1, 13)}
+
+    # 遍历 predictions，计算每个 question_type_id 的正确预测数和总预测数
+    for prediction in predictions:
+        # pdb.set_trace()
+        prediction = prediction.strip()
+        if not prediction:
+            continue
+        try:
+            prediction = json.loads(prediction)
+        except json.JSONDecodeError:
+            print(f"Warning: Skipping invalid JSON data in line: {prediction}")
+            continue
+        question_id = prediction["question_id"]
+        gt_item = ground_truth[question_id]
+        question_type_id = gt_item["question_type_id"]
+
+        if prediction["prediction"] == gt_item["answer"]:
+            results[question_type_id]["correct"] += 1
+
+        results[question_type_id]["total"] += 1
+    
+    return results
+
+def add_new_eval(
+    input_file,
+    model_name_textbox: str,
+    revision_name_textbox: str,
+    model_type: str,
+    model_link: str,
+    LLM_type: str,
+    LLM_name_textbox: str,
+    Evaluation_dimension: str,
+):
+    if input_file is None:
+        return "Error! Empty file!"
+    else:
+        content = input_file.decode("utf-8")
+        prediction = prediction_analyse(content)
+        csv_data = pd.read_csv(CSV_DIR)
+
+        Start_dimension, End_dimension = 1, 13
+        if Evaluation_dimension == 'Image':
+            End_dimension = 10
+        elif Evaluation_dimension == 'Video':
+            Start_dimension = 10
+        each_task_accuracy = {i: round(prediction[i]["correct"] / prediction[i]["total"] * 100, 1) if i >= Start_dimension and i < End_dimension else 0 for i in range(1, 13)}
+
+        # count for average image\video\all
+        total_correct_image = sum(prediction[i]["correct"] for i in range(1, 10))
+        total_correct_video = sum(prediction[i]["correct"] for i in range(10, 13))
+
+        total_image = sum(prediction[i]["total"] for i in range(1, 10))
+        total_video = sum(prediction[i]["total"] for i in range(10, 13))
+
+        if Evaluation_dimension != 'Video':
+            average_accuracy_image = round(total_correct_image / total_image * 100, 1)
+        else:
+            average_accuracy_image = 0
+        
+        if Evaluation_dimension != 'Image':
+            average_accuracy_video = round(total_correct_video / total_video * 100, 1)
+        else:
+            average_accuracy_video = 0
+        
+        if Evaluation_dimension == 'All':
+            overall_accuracy = round((total_correct_image + total_correct_video) / (total_image + total_video) * 100, 1)
+        else:
+            overall_accuracy = 0
+
+        if LLM_type == 'Other':
+            LLM_name = LLM_name_textbox
+        else:
+            LLM_name = LLM_type
+        
+        if revision_name_textbox == '':
+            col = csv_data.shape[0]
+            model_name = model_name_textbox
+        else:
+            model_name = revision_name_textbox
+            model_name_list = csv_data['Model']
+            name_list = [name.split(']')[0][1:] for name in model_name_list]
+            if revision_name_textbox not in name_list:
+                col = csv_data.shape[0]
+            else:
+                col = name_list.index(revision_name_textbox)    
+        
+        if model_link == '':
+            model_name = model_name  # no url
+        else:
+            model_name = '[' + model_name + '](' + model_link + ')'
+
+        # add new data
+        new_data = [
+            model_type, 
+            model_name, 
+            LLM_name, 
+            each_task_accuracy[1],
+            each_task_accuracy[2],
+            each_task_accuracy[3],
+            each_task_accuracy[4],
+            each_task_accuracy[5],
+            each_task_accuracy[6],
+            each_task_accuracy[7],
+            each_task_accuracy[8],
+            each_task_accuracy[9],
+            average_accuracy_image,
+            each_task_accuracy[10],
+            each_task_accuracy[11],
+            each_task_accuracy[12], 
+            average_accuracy_video, 
+            overall_accuracy]
+        # pdb.set_trace()
+        csv_data.loc[col] = new_data
+        csv_data = csv_data.to_csv(CSV_DIR, index=False)
+    return 0
+
+def get_baseline_df():
+    df = pd.read_csv(CSV_DIR)
+    return df
+
+block = gr.Blocks()
+
+
+with block:
+    gr.Markdown(
+        LEADERBORAD_INTRODUCTION
+    )
+    with gr.Tabs(elem_classes="tab-buttons") as tabs:
+        with gr.TabItem("🏅 SEED Benchmark", elem_id="seed-benchmark-tab-table", id=0):
+            with gr.Row():
+                with gr.Accordion("Citation", open=False):
+                    citation_button = gr.Textbox(
+                        value=CITATION_BUTTON_TEXT,
+                        label=CITATION_BUTTON_LABEL,
+                        elem_id="citation-button",
+                    ).style(show_copy_button=True)
+    
+            gr.Markdown(
+                TABLE_INTRODUCTION
+            )
+
+            # selection for column part:
+            checkbox_group = gr.CheckboxGroup(
+                choices=TASK_INFO,
+                value=TASK_INFO,
+                label="Select options",
+                interactive=True,
+            )
+
+            # 创建数据帧组件
+            data_component = gr.components.Dataframe(
+                value=get_baseline_df, 
+                headers=COLUMN_NAMES,
+                type="pandas", 
+                datatype=DATA_TITILE_TYPE,
+                interactive=False,
+                visible=True,
+                )
+    
+            def on_checkbox_group_change(selected_columns):
+                # pdb.set_trace()
+                selected_columns = [item for item in TASK_INFO if item in selected_columns]
+                present_columns = MODEL_INFO + selected_columns
+                updated_data = get_baseline_df()[present_columns]
+                updated_headers = present_columns
+                update_datatype = [DATA_TITILE_TYPE[COLUMN_NAMES.index(x)] for x in updated_headers]
+
+                filter_component = gr.components.Dataframe(
+                    value=updated_data, 
+                    headers=updated_headers,
+                    type="pandas", 
+                    datatype=update_datatype,
+                    interactive=False,
+                    visible=True,
+                    )
+                # pdb.set_trace()
+        
+                return filter_component.value
+
+            # 将复选框组关联到处理函数
+            checkbox_group.change(fn=on_checkbox_group_change, inputs=checkbox_group, outputs=data_component)
+
+        # table 2
+        with gr.TabItem("📝 About", elem_id="seed-benchmark-tab-table", id=2):
+            gr.Markdown(LEADERBORAD_INFO, elem_classes="markdown-text")
+        
+        # table 3 
+        with gr.TabItem("🚀 Submit here! ", elem_id="seed-benchmark-tab-table", id=3):
+            gr.Markdown(LEADERBORAD_INTRODUCTION, elem_classes="markdown-text")
+
+            with gr.Row():
+                gr.Markdown(SUBMIT_INTRODUCTION, elem_classes="markdown-text")
+
+            with gr.Row():
+                gr.Markdown("# ✉️✨ Submit your model evaluation json file here!", elem_classes="markdown-text")
+
+            with gr.Row():
+                with gr.Column():
+                    model_name_textbox = gr.Textbox(
+                        label="Model name", placeholder="LLaMA-7B"
+                        )
+                    revision_name_textbox = gr.Textbox(
+                        label="Revision Model Name", placeholder="LLaMA-7B"
+                    )
+                    model_type = gr.Dropdown(
+                        choices=[                         
+                            "LLM",
+                            "ImageLLM",
+                            "VideoLLM",
+                            "Other", 
+                        ], 
+                        label="Model type", 
+                        multiselect=False,
+                        value="ImageLLM",
+                        interactive=True,
+                    )
+                    model_link = gr.Textbox(
+                        label="Model Link", placeholder="https://huggingface.co/decapoda-research/llama-7b-hf"
+                    )
+
+                with gr.Column():
+                    
+                    LLM_type = gr.Dropdown(
+                        choices=["Vicuna-7B", "Flan-T5-XL", "LLaMA-7B", "Other"],
+                        label="LLM type", 
+                        multiselect=False,
+                        value="LLaMA-7B",
+                        interactive=True,
+                    )
+                    LLM_name_textbox = gr.Textbox(
+                        label="LLM model (for Other)",
+                        placeholder="LLaMA-13B"
+                    )
+                    Evaluation_dimension = gr.Dropdown(
+                        choices=["All", "Image", "Video"],
+                        label="Evaluation dimension", 
+                        multiselect=False,
+                        value="All",
+                        interactive=True,
+                    )
+
+            with gr.Column():
+
+                input_file = gr.inputs.File(label = "Click to Upload a json File", file_count="single", type='binary')
+                submit_button = gr.Button("Submit Eval")
+    
+                submission_result = gr.Markdown()
+                submit_button.click(
+                    add_new_eval,
+                    inputs = [
+                        input_file,
+                        model_name_textbox,
+                        revision_name_textbox,
+                        model_type,
+                        model_link,
+                        LLM_type,
+                        LLM_name_textbox,
+                        Evaluation_dimension,
+                    ],
+                    # outputs = submission_result,
+                )
+
+
+    with gr.Row():
+        data_run = gr.Button("Refresh")
+        data_run.click(
+            get_baseline_df, outputs=data_component
+        )
+
+    # block.load(get_baseline_df, outputs=data_title)
+
+block.launch()

constants.py

@@ -0,0 +1,67 @@
+# this is .py for store constants 
+MODEL_INFO = ["Model Type", "Model", "Language Model"]
+TASK_INFO = ["Scene Understanding", "Instance Identity", "Instance Attributes", "Instance Localization", "Instance Counting", "Spatial Relation", "Instance Interaction", "Visual Reasoning", "Text Recognition", "Avg. Img", "Action Recognition", "Action Prediction", "Procedure Understanding", "Avg. Video", "Avg. All"]
+AVG_INFO = ["Avg. Img", "Avg. Video", "Avg. All"]
+DATA_TITILE_TYPE = ["markdown", "markdown", "markdown", "number", "number", "number", "number", "number", "number", "number", "number", "number", "number", "number", "number", "number", "number", "number"]
+CSV_DIR = "./file/result.csv"
+
+COLUMN_NAMES = MODEL_INFO + TASK_INFO
+DATA_NUM = [3158, 1831, 4649, 978, 2447, 657, 97, 331, 85, 1740, 2077, 1192]
+
+UNTUNED_MODEL_RESULTS = '''LLM & Flan-T5               & Flan-T5-XL &23.0 &29.0 &32.8 &31.8 &20.5 &31.8 &33.0 &18.2 &19.4 &23.2 &34.9 &25.4 \\
+         LLM                   & Vicuna                & Vicuna-7B  &23.4 &30.7 &29.7 &30.9 &30.8 &28.6 &29.8 &18.5 &13.4 &27.3 &34.5 &23.8 \\
+         LLM                   & LLaMA                 & LLaMA-7B   &26.3 &27.4 &26.2 &28.3 &25.1 &28.8 &19.2 &37.0 & 9.0 &33.0 &23.1 &26.2  \\
+         ImageLLM              & BLIP-2                & Flan-T5-XL &59.1 &53.9 &49.2 &42.3 &43.2 &36.7 &55.7 &45.6 &25.9 &32.6 &47.5 &24.0 \\
+         ImageLLM              & InstructBLIP          & Flan-T5-XL &60.3 &58.5 &63.4 &40.6 &58.4 &38.7 &51.6 &45.9 &25.9 &33.1 &49.1 &27.1 \\
+         ImageLLM              & InstructBLIP-Vicuna   & Vicuna-7B  &60.2 &58.9 &65.6 &43.6 &57.2 &40.3 &52.6 &47.7 &43.5 &34.5 &49.6 &23.1 \\
+         ImageLLM              & LLaVA                 & LLaMA-7B   &42.7 &34.9 &33.5 &28.4 &41.9 &30.8 &27.8 &46.8 &27.7 &29.7 &21.4 &19.1 \\
+         ImageLLM              & MiniGPT-4             & Flan-T5-XL &56.3 &49.2 &45.8 &37.9 &45.3 &32.6 &47.4 &57.1 &11.8 &38.2 &24.5 &27.1 \\
+         ImageLLM              & VPGTrans              & LLaMA-7B   &51.9 &44.1 &39.9 &36.1 &33.7 &36.4 &32.0 &53.2 &30.6 &39.5 &24.3 &31.9 \\
+         ImageLLM              & MultiModal-GPT        & LLaMA-7B   &43.6 &37.9 &31.5 &30.8 &27.3 &30.1 &29.9 &51.4 &18.8 &36.9 &25.8 &24.0 \\
+         ImageLLM              & Otter                 & LLaMA-7B   &44.9 &38.6 &32.2 &30.9 &26.3 &31.8 &32.0 &51.4 &31.8 &37.9 &27.2 &24.8 \\
+         ImageLLM              & OpenFlamingo          & LLaMA-7B   &43.9 &38.1 &31.3 &30.1 &27.3 &30.6 &29.9 &50.2 &20.0 &37.2 &25.4 &24.2 \\
+         ImageLLM              & LLaMA-Adapter V2      & LLaMA-7B   &45.2 &38.5 &29.3 &33.0 &29.7 &35.5 &39.2 &52.0 &24.7 &38.6 &18.5 &19.6 \\
+         ImageLLM              & GVT                   & Vicuna-7B  &41.7 &35.5 &31.8 &29.5 &36.2 &32.0 &32.0 &51.1 &27.1 &33.9 &25.4 &23.0 \\
+         ImageLLM              & mPLUG-Owl             & LLaMA-7B   &49.7 &45.3 &32.5 &36.7 &27.3 &32.7 &44.3 &54.7 &28.8 &26.7 &17.9 &26.5 \\
+         VideoLLM              & VideoChat             & Vicuna-7B  &47.1 &43.8 &34.9 &40.0 &32.8 &34.6 &42.3 &50.5 &17.7 &34.9 &36.4 &27.3 \\
+         VideoLLM              & Video-ChatGPT         & LLaMA-7B   &37.2 &31.4 &33.2 &28.4 &35.5 &29.5 &23.7 &42.3 &25.9 &27.6 &21.3 &21.1 \\
+         VideoLLM              & Valley                & LLaMA-13B  &39.3 &32.9 &31.6 &27.9 &24.2 &30.1 &27.8 &43.8 &11.8 &31.3 &23.2 &20.7 \\'''
+
+
+LEADERBORAD_INTRODUCTION = """# SEED-Bench Leaderboard
+    
+    Welcome to the leaderboard of the SEED-Bench! 🏆 
+    This is a community where participants create multimodal language models and action generation algorithms to generate API function calls based goals described in natural lanugage!
+    Please refer to [our paper](https://arxiv.org/abs/2307.16125) for more details.
+    """
+
+SUBMIT_INTRODUCTION = """# Submit Precautions
+    1. Attain json file from our [github repository](https://github.com/AILab-CVC/SEED-Bench)
+    2. If you want to revision model, please ensure 'Revision Model Name' align with what's in the leaderboard.
+    3. Please ensure for right link for each submittion. Everyone could go to model's repository thought model name in the leaderboard.
+    4. If you don't want to evaluate all dimension, not evaluated dimension performance and its corresponding average performance will set to 0.
+"""
+
+TABLE_INTRODUCTION = """In the table below, we summarize each task performance of all the models.
+        We use accurancy(%) as the primary evaluation metric for each tasks. 
+    """
+
+LEADERBORAD_INFO = """
+      Based on powerful Large Language Models (LLMs), recent generative Multimodal Large Language Models (MLLMs) have gained prominence as a pivotal research area, exhibiting remarkable capability for both comprehension and generation. 
+      In this work, we address the evaluation of generative comprehension in MLLMs as a preliminary step towards a comprehensive assessment of generative models, by introducing a benchmark named SEED-Bench. 
+      SEED-Bench consists of 19K multiple choice questions with accurate human annotations (x6 larger than existing benchmarks), which spans 12 evaluation dimensions including the comprehension of both the image and video modality. 
+      We develop an advanced pipeline for generating multiple-choice questions that target specific evaluation dimensions, integrating both automatic filtering and manual verification processes. 
+      Multiple-choice questions with groundtruth options derived from human annotation enables an objective and efficient assessment of model performance, eliminating the need for human or GPT intervention during evaluation. 
+      We further evaluate the performance of 18 models across all 12 dimensions, covering both the spatial and temporal understanding. 
+      By revealing the limitations of existing MLLMs through evaluation results, we aim for SEED-Bench to provide insights for motivating future research.
+"""
+
+
+
+CITATION_BUTTON_LABEL = "Copy the following snippet to cite these results"
+CITATION_BUTTON_TEXT = r"""@article{li2023seed,
+  title={SEED-Bench: Benchmarking Multimodal LLMs with Generative Comprehension},
+  author={Li, Bohao and Wang, Rui and Wang, Guangzhi and Ge, Yuying and Ge, Yixiao and Shan, Ying},
+  journal={arXiv preprint arXiv:2307.16125},
+  year={2023}
+}"""

file/result.csv

@@ -0,0 +1,22 @@
+Model Type,Model,Language Model,Scene Understanding,Instance Identity,Instance Attributes,Instance Localization,Instance Counting,Spatial Relation,Instance Interaction,Visual Reasoning,Text Recognition,Avg. Img,Action Recognition,Action Prediction,Procedure Understanding,Avg. Video,Avg. All
+LLM,[Flan-T5](https://huggingface.co/google/flan-t5-xl),Flan-T5-XL,23.0,29.0,32.8,31.8,20.5,31.8,33.0,18.2,19.4,27.3,23.2,34.9,25.4,28.6,27.7
+LLM,[Vicuna](https://huggingface.co/lmsys/vicuna-7b-v1.3),Vicuna-7B,23.4,30.7,29.7,30.9,30.8,28.6,29.8,18.5,13.4,28.2,27.3,34.5,23.8,29.5,28.5
+LLM,[LLaMA](https://research.facebook.com/publications/llama-open-and-efficient-foundation-language-models/),LLaMA-7B,26.3,27.4,26.2,28.3,25.1,28.8,19.2,37.0,9.0,26.6,33.0,23.1,26.2,27.3,26.8
+ImageLLM,[BLIP-2](https://github.com/salesforce/LAVIS),Flan-T5-XL,59.1,53.9,49.2,42.3,43.2,36.7,55.7,45.6,25.9,49.7,32.6,47.5,24.0,36.7,46.4
+ImageLLM,[InstructBLIP](https://github.com/salesforce/LAVIS),Flan-T5-XL,60.3,58.5,63.4,40.6,58.4,38.7,51.6,45.9,25.9,57.8,33.1,49.1,27.1,38.3,52.7
+ImageLLM,[InstructBLIP-Vicuna](https://github.com/salesforce/LAVIS),Vicuna-7B,60.2,58.9,65.6,43.6,57.2,40.3,52.6,47.7,43.5,58.8,34.5,49.6,23.1,38.1,53.4
+ImageLLM,[LLaVA](https://github.com/haotian-liu/LLaVA),LLaMA-7B,42.7,34.9,33.5,28.4,41.9,30.8,27.8,46.8,27.7,37.0,29.7,21.4,19.1,23.8,33.5
+ImageLLM,[MiniGPT-4](https://github.com/Vision-CAIR/MiniGPT-4),Flan-T5-XL,56.3,49.2,45.8,37.9,45.3,32.6,47.4,57.1,11.8,47.4,38.2,24.5,27.1,29.9,42.8
+ImageLLM,[VPGTrans](https://github.com/VPGTrans/VPGTrans),LLaMA-7B,51.9,44.1,39.9,36.1,33.7,36.4,32.0,53.2,30.6,41.8,39.5,24.3,31.9,31.4,39.1
+ImageLLM,[MultiModal-GPT](https://github.com/open-mmlab/Multimodal-GPT),LLaMA-7B,43.6,37.9,31.5,30.8,27.3,30.1,29.9,51.4,18.8,34.5,36.9,25.8,24.0,29.2,33.2
+ImageLLM,[Otter](https://github.com/Luodian/Otter),LLaMA-7B,44.9,38.6,32.2,30.9,26.3,31.8,32.0,51.4,31.8,35.2,37.9,27.2,24.8,30.4,33.9
+ImageLLM,[Otter](https://github.com/Luodian/Otter),MPT-7B,51.3,43.5,42.3,34.2,38.4,30.9,40.2,55.3,24.7,42.9,36.8,29.2,23.8,30.6,39.7
+ImageLLM,[OpenFlamingo](https://github.com/mlfoundations/open_flamingo),LLaMA-7B,43.9,38.1,31.3,30.1,27.3,30.6,29.9,50.2,20.0,34.5,37.2,25.4,24.2,29.3,33.1
+ImageLLM,[OpenFlamingo](https://github.com/mlfoundations/open_flamingo),MPT-7B,53.2,45.3,40.0,31.2,39.3,32.6,36.1,51.4,25.9,42.7,42.9,34.7,26.9,35.7,40.9
+ImageLLM,[LLaMA-AdapterV2](https://github.com/OpenGVLab/LLaMA-Adapter),LLaMA-7B,45.2,38.5,29.3,33.0,29.7,35.5,39.2,52.0,24.7,35.2,38.6,18.5,19.6,25.8,32.7
+ImageLLM,[GVT](https://github.com/TencentARC/GVT),Vicuna-7B,41.7,35.5,31.8,29.5,36.2,32.0,32.0,51.1,27.1,35.5,33.9,25.4,23.0,27.8,33.5
+ImageLLM,[mPLUG-Owl](https://github.com/X-PLUG/mPLUG-Owl),LLaMA-7B,49.7,45.3,32.5,36.7,27.3,32.7,44.3,54.7,28.8,37.9,26.7,17.9,26.5,23.0,34.0
+ImageLLM,[Kosmos-2](https://github.com/microsoft/unilm/tree/master/kosmos-2),Decoder Only 1.3B,63.4,57.1,58.5,44.0,41.4,37.9,55.7,60.7,25.9,54.4,41.3,40.4,27.0,37.5,50.0
+VideoLLM,[VideoChat](https://github.com/OpenGVLab/Ask-Anything),Vicuna-7B,47.1,43.8,34.9,40.0,32.8,34.6,42.3,50.5,17.7,39.0,34.9,36.4,27.3,33.7,37.6
+VideoLLM,[Video-ChatGPT](https://github.com/mbzuai-oryx/Video-ChatGPT),LLaMA-7B,37.2,31.4,33.2,28.4,35.5,29.5,23.7,42.3,25.9,33.9,27.6,21.3,21.1,23.5,31.2
+VideoLLM,[Valley](https://github.com/RupertLuo/Valley),LLaMA-13B,39.3,32.9,31.6,27.9,24.2,30.1,27.8,43.8,11.8,32.0,31.3,23.2,20.7,25.4,30.3

requirements.txt

@@ -0,0 +1,70 @@
+aiofiles==23.1.0
+aiohttp==3.8.4
+aiosignal==1.3.1
+altair==4.2.2
+anyio==3.6.2
+APScheduler==3.10.1
+async-timeout==4.0.2
+attrs==23.1.0
+certifi==2022.12.7
+charset-normalizer==3.1.0
+click==8.1.3
+contourpy==1.0.7
+cycler==0.11.0
+datasets==2.12.0
+entrypoints==0.4
+fastapi==0.95.1
+ffmpy==0.3.0
+filelock==3.11.0
+fonttools==4.39.3
+frozenlist==1.3.3
+fsspec==2023.4.0
+gradio==3.27.0
+gradio_client==0.1.3
+h11==0.14.0
+httpcore==0.17.0
+httpx==0.24.0
+huggingface-hub==0.13.4
+idna==3.4
+Jinja2==3.1.2
+jsonschema==4.17.3
+kiwisolver==1.4.4
+linkify-it-py==2.0.0
+markdown-it-py==2.2.0
+MarkupSafe==2.1.2
+matplotlib==3.7.1
+mdit-py-plugins==0.3.3
+mdurl==0.1.2
+multidict==6.0.4
+numpy==1.24.2
+orjson==3.8.10
+packaging==23.1
+pandas==2.0.0
+Pillow==9.5.0
+plotly==5.14.1
+pyarrow==11.0.0
+pydantic==1.10.7
+pydub==0.25.1
+pyparsing==3.0.9
+pyrsistent==0.19.3
+python-dateutil==2.8.2
+python-multipart==0.0.6
+pytz==2023.3
+pytz-deprecation-shim==0.1.0.post0
+PyYAML==6.0
+requests==2.28.2
+semantic-version==2.10.0
+six==1.16.0
+sniffio==1.3.0
+starlette==0.26.1
+toolz==0.12.0
+tqdm==4.65.0
+transformers==4.28.1
+typing_extensions==4.5.0
+tzdata==2023.3
+tzlocal==4.3
+uc-micro-py==1.0.1
+urllib3==1.26.15
+uvicorn==0.21.1
+websockets==11.0.1
+yarl==1.8.2

src/auto_leaderboard/model_metadata_type.py

@@ -0,0 +1,30 @@
+from dataclasses import dataclass
+from enum import Enum
+import glob
+import json
+import os
+from typing import Dict, List
+
+from ..utils_display import AutoEvalColumn
+
+@dataclass
+class ModelInfo:
+    name: str
+    symbol: str # emoji
+
+model_type_symbols = {
+    "LLM": "🟢",
+    "ImageLLM": "🔶",
+    "VideoLLM": "⭕",
+    "Other": "🟦",
+}
+
+class ModelType(Enum):
+    PT = ModelInfo(name="LLM", symbol="🟢")
+    FT = ModelInfo(name="ImageLLM", symbol="🔶")
+    IFT = ModelInfo(name="VideoLLM", symbol="⭕")
+    RL = ModelInfo(name="Other", symbol="🟦")
+
+    def to_str(self, separator = " "):
+        return f"{self.value.symbol}{separator}{self.value.name}" 
+

src/utils_display.py

@@ -0,0 +1,99 @@
+from dataclasses import dataclass
+
+# These classes are for user facing column names, to avoid having to change them 
+# all around the code when a modif is needed 
+@dataclass
+class ColumnContent:
+    name: str
+    type: str 
+    displayed_by_default: bool 
+    hidden: bool = False
+
+def fields(raw_class):
+    return [v for k, v in raw_class.__dict__.items() if k[:2] != "__" and k[-2:] != "__"]
+
+@dataclass(frozen=True)
+class AutoEvalColumn: # Auto evals column
+    model_type_symbol = ColumnContent("T", "str", True)
+    model = ColumnContent("Model", "markdown", True)
+    average = ColumnContent("Average ⬆️", "number", True)
+    arc = ColumnContent("ARC", "number", True)
+    hellaswag = ColumnContent("HellaSwag", "number", True)
+    mmlu = ColumnContent("MMLU", "number", True)
+    truthfulqa = ColumnContent("TruthfulQA", "number", True)
+    model_type = ColumnContent("Type", "str", False)
+    precision = ColumnContent("Precision", "str", False, True)
+    license = ColumnContent("Hub License", "str", False)
+    params = ColumnContent("#Params (B)", "number", False)
+    likes = ColumnContent("Hub ❤️", "number", False)
+    revision = ColumnContent("Model sha", "str", False, False)
+    dummy = ColumnContent("model_name_for_query", "str", True) # dummy col to implement search bar (hidden by custom CSS)
+
+@dataclass(frozen=True)
+class EloEvalColumn: # Elo evals column
+    model = ColumnContent("Model", "markdown", True)
+    gpt4 = ColumnContent("GPT-4 (all)", "number", True)
+    human_all = ColumnContent("Human (all)", "number", True)
+    human_instruct = ColumnContent("Human (instruct)", "number", True)
+    human_code_instruct = ColumnContent("Human (code-instruct)", "number", True)
+
+
+@dataclass(frozen=True)
+class EvalQueueColumn: # Queue column
+    model = ColumnContent("model", "markdown", True)
+    revision = ColumnContent("revision", "str", True)
+    private = ColumnContent("private", "bool", True)
+    precision = ColumnContent("precision", "bool", True)
+    weight_type = ColumnContent("weight_type", "str", "Original")
+    status = ColumnContent("status", "str", True)
+
+LLAMAS = ["huggingface/llama-7b", "huggingface/llama-13b", "huggingface/llama-30b", "huggingface/llama-65b"]
+
+
+KOALA_LINK = "https://huggingface.co/TheBloke/koala-13B-HF"
+VICUNA_LINK = "https://huggingface.co/lmsys/vicuna-13b-delta-v1.1"
+OASST_LINK = "https://huggingface.co/OpenAssistant/oasst-sft-4-pythia-12b-epoch-3.5"
+DOLLY_LINK = "https://huggingface.co/databricks/dolly-v2-12b"
+MODEL_PAGE = "https://huggingface.co/models"
+LLAMA_LINK = "https://ai.facebook.com/blog/large-language-model-llama-meta-ai/"
+VICUNA_LINK = "https://huggingface.co/CarperAI/stable-vicuna-13b-delta"
+ALPACA_LINK = "https://crfm.stanford.edu/2023/03/13/alpaca.html"
+
+
+def model_hyperlink(link, model_name):
+    return f'<a target="_blank" href="{link}" style="color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;">{model_name}</a>'
+
+
+def make_clickable_model(model_name):
+    link = f"https://huggingface.co/{model_name}"
+
+    if model_name in LLAMAS:
+        link = LLAMA_LINK
+        model_name = model_name.split("/")[1]
+    elif model_name == "HuggingFaceH4/stable-vicuna-13b-2904":
+        link = VICUNA_LINK
+        model_name = "stable-vicuna-13b"
+    elif model_name == "HuggingFaceH4/llama-7b-ift-alpaca":
+        link = ALPACA_LINK
+        model_name = "alpaca-13b"
+    if model_name == "dolly-12b":
+        link = DOLLY_LINK
+    elif model_name == "vicuna-13b":
+        link = VICUNA_LINK
+    elif model_name == "koala-13b":
+        link = KOALA_LINK
+    elif model_name == "oasst-12b":
+        link = OASST_LINK
+    #else:
+    #    link = MODEL_PAGE
+  
+    return model_hyperlink(link, model_name)
+
+def styled_error(error):
+    return f"<p style='color: red; font-size: 20px; text-align: center;'>{error}</p>"
+
+def styled_warning(warn):
+    return f"<p style='color: orange; font-size: 20px; text-align: center;'>{warn}</p>"
+
+def styled_message(message):
+    return f"<p style='color: green; font-size: 20px; text-align: center;'>{message}</p>"