update about and submit pages

app.py

@@ -29,7 +29,10 @@ from src.display.utils import (
 from src.envs import API, EVAL_REQUESTS_PATH, EVAL_RESULTS_PATH, QUEUE_REPO, REPO_ID, RESULTS_REPO, TOKEN
 from src.populate import get_evaluation_queue_df, get_leaderboard_df
 from src.submission.submit import add_new_eval
-pd.set_option('display.float_format', '{:.2f}'.format)
+
+def handle_new_eval_submission(model_name, model_zip, model_link):
+    # This is a placeholder for the actual submission logic
+    return "We are not accepting submissions at this time, please check back soon!"
 
 def restart_space():
     API.restart_space(repo_id=REPO_ID)
@@ -155,7 +158,7 @@ with demo:
     gr.Markdown(INTRODUCTION_TEXT, elem_classes="markdown-text")
 
     with gr.Tabs(elem_classes="tab-buttons") as tabs:
-        with gr.TabItem("🏅 LLM Benchmark", elem_id="llm-benchmark-tab-table", id=0):
+        with gr.TabItem("🏅 3D-POPE Benchmark", elem_id="llm-benchmark-tab-table", id=0):
             with gr.Row():
                 with gr.Column():
                     with gr.Row():
@@ -239,87 +242,60 @@ with demo:
                 with gr.Row():
                     gr.Markdown(EVALUATION_QUEUE_TEXT, elem_classes="markdown-text")
 
-                with gr.Column():
-                    with gr.Accordion(
-                        f"✅ Finished Evaluations ({len(finished_eval_queue_df)})",
-                        open=False,
-                    ):
-                        with gr.Row():
-                            finished_eval_table = gr.components.Dataframe(
-                                value=finished_eval_queue_df,
-                                headers=EVAL_COLS,
-                                datatype=EVAL_TYPES,
-                                row_count=5,
-                            )
-                    with gr.Accordion(
-                        f"🔄 Running Evaluation Queue ({len(running_eval_queue_df)})",
-                        open=False,
-                    ):
-                        with gr.Row():
-                            running_eval_table = gr.components.Dataframe(
-                                value=running_eval_queue_df,
-                                headers=EVAL_COLS,
-                                datatype=EVAL_TYPES,
-                                row_count=5,
-                            )
-
-                    with gr.Accordion(
-                        f"⏳ Pending Evaluation Queue ({len(pending_eval_queue_df)})",
-                        open=False,
-                    ):
-                        with gr.Row():
-                            pending_eval_table = gr.components.Dataframe(
-                                value=pending_eval_queue_df,
-                                headers=EVAL_COLS,
-                                datatype=EVAL_TYPES,
-                                row_count=5,
-                            )
+                # with gr.Column():
+                #     with gr.Accordion(
+                #         f"✅ Finished Evaluations ({len(finished_eval_queue_df)})",
+                #         open=False,
+                #     ):
+                #         with gr.Row():
+                #             finished_eval_table = gr.components.Dataframe(
+                #                 value=finished_eval_queue_df,
+                #                 headers=EVAL_COLS,
+                #                 datatype=EVAL_TYPES,
+                #                 row_count=5,
+                #             )
+                #     with gr.Accordion(
+                #         f"🔄 Running Evaluation Queue ({len(running_eval_queue_df)})",
+                #         open=False,
+                #     ):
+                #         with gr.Row():
+                #             running_eval_table = gr.components.Dataframe(
+                #                 value=running_eval_queue_df,
+                #                 headers=EVAL_COLS,
+                #                 datatype=EVAL_TYPES,
+                #                 row_count=5,
+                #             )
+
+                #     with gr.Accordion(
+                #         f"⏳ Pending Evaluation Queue ({len(pending_eval_queue_df)})",
+                #         open=False,
+                #     ):
+                #         with gr.Row():
+                #             pending_eval_table = gr.components.Dataframe(
+                #                 value=pending_eval_queue_df,
+                #                 headers=EVAL_COLS,
+                #                 datatype=EVAL_TYPES,
+                #                 row_count=5,
+                #             )
             with gr.Row():
-                gr.Markdown("# ✉️✨ Submit your model here!", elem_classes="markdown-text")
+                gr.Markdown("# 📋 Submit your results here!", elem_classes="markdown-text")
 
             with gr.Row():
-                with gr.Column():
                     model_name_textbox = gr.Textbox(label="Model name")
-                    revision_name_textbox = gr.Textbox(label="Revision commit", placeholder="main")
-                    model_type = gr.Dropdown(
-                        choices=[t.to_str(" : ") for t in ModelType if t != ModelType.Unknown],
-                        label="Model type",
-                        multiselect=False,
-                        value=None,
-                        interactive=True,
-                    )
-
-                with gr.Column():
-                    precision = gr.Dropdown(
-                        choices=[i.value.name for i in Precision if i != Precision.Unknown],
-                        label="Precision",
-                        multiselect=False,
-                        value="float16",
-                        interactive=True,
-                    )
-                    weight_type = gr.Dropdown(
-                        choices=[i.value.name for i in WeightType],
-                        label="Weights type",
-                        multiselect=False,
-                        value="Original",
-                        interactive=True,
+                    model_zip_file = gr.File(label="Upload model ZIP file")
+                    model_link_textbox = gr.Textbox(label="Model link")
+            with gr.Row():
+                gr.Column()
+                with gr.Column(scale=2):
+                    submit_button = gr.Button("Submit Model")
+                    submission_result = gr.Markdown()
+
+                    submit_button.click(
+                        handle_new_eval_submission,
+                        [model_name_textbox, model_zip_file, model_link_textbox],
+                        submission_result
                     )
-                    base_model_name_textbox = gr.Textbox(label="Base model (for delta or adapter weights)")
-
-            submit_button = gr.Button("Submit Eval")
-            submission_result = gr.Markdown()
-            submit_button.click(
-                add_new_eval,
-                [
-                    model_name_textbox,
-                    base_model_name_textbox,
-                    revision_name_textbox,
-                    precision,
-                    weight_type,
-                    model_type,
-                ],
-                submission_result,
-            )
+                gr.Column()
 
     with gr.Row():
         with gr.Accordion("📙 Citation", open=False):

src/about.py

@@ -39,55 +39,87 @@ NUM_FEWSHOT = 0 # Change with your few shot
 
 
 # Your leaderboard name
-TITLE = """<h1 align="center" id="space-title">3D-POPE Leaderboard</h1>"""
+TITLE = """<h1 align="center" id="space-title">3D-POPE Leaderboard</h1>
+<p><center>
+<a href="https://3d-grand.github.io/" target="_blank">[Project Page]</a>
+<a href="https://www.dropbox.com/scl/fo/5p9nb4kalnz407sbqgemg/AG1KcxeIS_SUoJ1hoLPzv84?rlkey=weunabtbiz17jitfv3f4jpmm1&dl=0" target="_blank">[3D-GRAND Data]</a>
+<a href="https://www.dropbox.com/scl/fo/inemjtgqt2nkckymn65rp/AGi2KSYU9AHbnpuj7TWYihs?rlkey=ldbn36b1z6nqj74yv5ph6cqwc&dl=0" target="_blank">[3D-POPE Data]</a>
+</center></p>
+"""
 
 # What does your leaderboard evaluate?
 INTRODUCTION_TEXT = """
 #### This is the official leaderboard for the 3D Polling-based Object Probing Evaluation (3D-POPE) benchmark.
-
-###### 3D-POPE is designed to assess a model's ability to accurately identify the presence or absence of objects in a given 3D scene.
 """
 
 # Which evaluations are you running? how can people reproduce what you have?
 LLM_BENCHMARKS_TEXT = f"""
-## How it works
+# 3D-POPE: A Benchmark for Evaluating Hallucination in 3D-LLMs
+### To systematically evaluate the hallucination behavior of 3D-LLMs, we introduce the 3D Polling-based Object Probing Evaluation (3D-POPE) benchmark. 3D-POPE is designed to assess a model's ability to accurately identify the presence or absence of objects in a given 3D scene.
 
-## Reproducibility
-To reproduce our results, here is the commands you can run:
+## Dataset
+To facilitate the 3D-POPE benchmark, we curate a dedicated dataset from the ScanNet dataset, utilizing the semantic classes from ScanNet200. Specifically, we use the ScanNet validation set as the foundation for evaluating 3D-LLMs on the 3D-POPE benchmark.
 
-"""
+Benchmark design. 3D-POPE consists of a set of triples, each comprising a 3D scene, a posed question, and a binary answer (“Yes” or “No”) indicating the presence or absence of an object (Fig. 1 middle). To ensure a balanced dataset, we maintain a 1:1 ratio of existent to nonexistent objects when constructing these triples. For the selection of negative samples (i.e., nonexistent objects), we employ three distinct sampling strategies:
 
-EVALUATION_QUEUE_TEXT = """
-## Some good practices before submitting a model
-
-### 1) Make sure you can load your model and tokenizer using AutoClasses:
-```python
-from transformers import AutoConfig, AutoModel, AutoTokenizer
-config = AutoConfig.from_pretrained("your model name", revision=revision)
-model = AutoModel.from_pretrained("your model name", revision=revision)
-tokenizer = AutoTokenizer.from_pretrained("your model name", revision=revision)
-```
-If this step fails, follow the error messages to debug your model before submitting it. It's likely your model has been improperly uploaded.
+• Random Sampling: Nonexistent objects are randomly selected from the set of objects not present in the 3D scene.\n
+• Popular Sampling: We select the top-k most frequent objects not present in the 3D scene, where k equals the number of objects currently in the scene.\n
+• Adversarial Sampling: For each positively identified object in the scene, we rank objects that are not present and have not been used as negative samples based on their frequency of co-occurrence with the positive object in the training dataset. The highest-ranking co-occurring object is then selected as the adversarial sample. This approach differs from the original POPE [41] to avoid adversarial samples mirroring popular samples, as indoor scenes often contain similar objects.\n
+These sampling strategies are designed to challenge the model's robustness and assess its susceptibility to different levels of object hallucination.
 
-Note: make sure your model is public!
-Note: if your model needs `use_remote_code=True`, we do not support this option yet but we are working on adding it, stay posted!
+## Metrics
+To evaluate the model's performance on the 3D-POPE benchmark, we use key metrics including Precision, Recall, F1 Score, Accuracy, and Yes (%). Precision and Recall assess the model's ability to correctly affirm the presence of objects and identify the absence of objects, respectively. Precision is particularly important as it indicates the proportion of non-existing objects generated by the 3D-LLMs. The F1 Score, combining Precision and Recall, offers a balanced view of performance and serves as the primary evaluation metric. Accuracy measures the proportion of correctly answered questions, encompassing both “Yes” and “No” responses. Additionally, the Yes (%) metric reports the ratio of incorrect “Yes” responses to understand the model’s tendencies regarding object hallucination.
 
-### 2) Convert your model weights to [safetensors](https://huggingface.co/docs/safetensors/index)
-It's a new format for storing weights which is safer and faster to load and use. It will also allow us to add the number of parameters of your model to the `Extended Viewer`!
+## Leaderboard 
+We establish a public leaderboard for the 3D-POPE benchmark, allowing researchers to submit their 3D-LLM results and compare their performance against other state-of-the-art models. The leaderboard reports the evaluation metrics for each model under the three sampling strategies, providing a transparent and standardized way to assess the hallucination performance of 3D-LLMs.
+"""
 
-### 3) Make sure your model has an open license!
-This is a leaderboard for Open LLMs, and we'd love for as many people as possible to know they can use your model 🤗
+EVALUATION_QUEUE_TEXT = """
+# Submitting results from your own model
+
+Read the below instructions **carefully** to ensure your submission is properly formatted and complete.
+
+You should submit a total of 12 JSON files, each containing outputs generated by your model on [our dataset](https://www.dropbox.com/scl/fo/inemjtgqt2nkckymn65rp/AGi2KSYU9AHbnpuj7TWYihs?rlkey=ldbn36b1z6nqj74yv5ph6cqwc&dl=0). 
+These files should be organized within a single ZIP file under the folder structure as follows:
+```bash
+❯ tree LEO/json-outputs
+LEO/json-outputs
+├── adversarial_template_1.json
+├── adversarial_template_2.json
+├── adversarial_template_3.json
+├── adversarial_template_4.json
+├── popular_template_1.json
+├── popular_template_2.json
+├── popular_template_3.json
+├── popular_template_4.json
+├── random_template_1.json
+├── random_template_2.json
+├── random_template_3.json
+└── random_template_4.json
+```
 
-### 4) Fill up your model card
-When we add extra information about models to the leaderboard, it will be automatically taken from the model card
+Each JSON file should contain data structured exactly like the following example from random_template_1.json:
+```json
+{
+  "random_template_1": [
+    {
+      "source": "scannet",
+      "scene_id": "scene0011_00",
+      "question": "Are there any people in the room?",
+      "ground_truth_answer": "no",
+      "predicted_answer": "yes",
+      "template": "template_1",
+      "question_type": "random"
+    },
+    ...
+  ]
+}
+```
 
-## In case of model failure
-If your model is displayed in the `FAILED` category, its execution stopped.
-Make sure you have followed the above steps first.
-If everything is done, check you can launch the EleutherAIHarness on your model locally, using the above command without modifications (you can add `--limit` to limit the number of examples per task).
+Submit your properly formatted ZIP file below and your model results will be added to the leaderboard.
 """
 
-CITATION_BUTTON_LABEL = "Copy the following snippet to cite these results"
+CITATION_BUTTON_LABEL = "Copy the following snippet to cite our work"
 CITATION_BUTTON_TEXT = r"""
 @misc{yang20243dgrand,
         title={3D-GRAND: Towards Better Grounding and Less Hallucination for 3D-LLMs},