feat-add-v2405 (#26)

- update part code for v24.05 (257f64d0bddc9d6bac7df327345dac25bebc92c8)
- feat: add v2024.05 (2bce3f3b3285bf8e6b4ab0920c2c52eb43c2cd2d)


Co-authored-by: Jianlyu Chen <[email protected]>

app.py

@@ -131,303 +131,303 @@ with demo:
     gr.Markdown(INTRODUCTION_TEXT, elem_classes="markdown-text")
 
     with gr.Tabs(elem_classes="tab-buttons") as tabs:
-        with gr.TabItem("QA", elem_id="qa-benchmark-tab-table", id=0):
+        with gr.TabItem("Results", elem_id="results-tab-table"):
             with gr.Row():
-                with gr.Column(min_width=320):
-                    # select domain
-                    with gr.Row():
-                        selected_domains = get_domain_dropdown(DOMAIN_COLS_QA, DOMAIN_COLS_QA)
-                    # select language
-                    with gr.Row():
-                        selected_langs = get_language_dropdown(LANG_COLS_QA, LANG_COLS_QA)
-
-                with gr.Column():
-                    with gr.Row():
-                        selected_version = get_version_dropdown()
-                    # select the metric
-                    selected_metric = get_metric_dropdown(METRIC_LIST, DEFAULT_METRIC_QA)
-                    with gr.Row():
-                        show_anonymous = get_anonymous_checkbox()
-                    with gr.Row():
-                        show_revision_and_timestamp = get_revision_and_ts_checkbox()
-            with gr.Tabs(elem_classes="tab-buttons") as sub_tabs:
-                with gr.TabItem("Retrieval + Reranking", id=10):
-                    with gr.Row():
-                        # search retrieval models
-                        with gr.Column():
-                            search_bar = get_search_bar()
-                        # select reranking models
-                        with gr.Column():
-                            selected_rerankings = get_reranking_dropdown(reranking_models)
-                    leaderboard_table = get_leaderboard_table(leaderboard_df_qa, types_qa)
-                    # Dummy leaderboard for handling the case when the user uses backspace key
-                    hidden_leaderboard_table_for_search = get_leaderboard_table(original_df_qa, types_qa, visible=False)
-
-                    set_listeners(
-                        "qa",
-                        leaderboard_table,
-                        hidden_leaderboard_table_for_search,
-                        search_bar,
-                        selected_domains,
-                        selected_langs,
-                        selected_rerankings,
-                        show_anonymous,
-                        show_revision_and_timestamp,
-                    )
-
-                    # set metric listener
-                    selected_metric.change(
-                        update_metric_qa,
-                        [
-                            selected_metric,
+                selected_version = get_version_dropdown()
+
+            with gr.TabItem("QA", elem_id="qa-benchmark-tab-table", id=0):
+                with gr.Row():
+                    with gr.Column(min_width=320):
+                        # select domain
+                        with gr.Row():
+                            selected_domains = get_domain_dropdown(DOMAIN_COLS_QA, DOMAIN_COLS_QA)
+                        # select language
+                        with gr.Row():
+                            selected_langs = get_language_dropdown(LANG_COLS_QA, LANG_COLS_QA)
+
+                    with gr.Column():
+                        # select the metric
+                        selected_metric = get_metric_dropdown(METRIC_LIST, DEFAULT_METRIC_QA)
+                        with gr.Row():
+                            show_anonymous = get_anonymous_checkbox()
+                        with gr.Row():
+                            show_revision_and_timestamp = get_revision_and_ts_checkbox()
+                with gr.Tabs(elem_classes="tab-buttons") as sub_tabs:
+                    with gr.TabItem("Retrieval + Reranking", id=10):
+                        with gr.Row():
+                            # search retrieval models
+                            with gr.Column():
+                                search_bar = get_search_bar()
+                            # select reranking models
+                            with gr.Column():
+                                selected_rerankings = get_reranking_dropdown(reranking_models)
+                        leaderboard_table = get_leaderboard_table(leaderboard_df_qa, types_qa)
+                        # Dummy leaderboard for handling the case when the user uses backspace key
+                        hidden_leaderboard_table_for_search = get_leaderboard_table(original_df_qa, types_qa, visible=False)
+
+                        set_listeners(
+                            "qa",
+                            leaderboard_table,
+                            hidden_leaderboard_table_for_search,
+                            search_bar,
                             selected_domains,
                             selected_langs,
                             selected_rerankings,
-                            search_bar,
                             show_anonymous,
                             show_revision_and_timestamp,
-                        ],
-                        leaderboard_table,
-                        queue=True
-                    )
-                with gr.TabItem("Retrieval Only", id=11):
-                    with gr.Row():
-                        with gr.Column(scale=1):
-                            search_bar_retriever = get_search_bar()
-                        with gr.Column(scale=1):
-                            selected_noreranker = get_noreranking_dropdown()
-                    lb_df_retriever = leaderboard_df_qa[leaderboard_df_qa[COL_NAME_RERANKING_MODEL] == "NoReranker"]
-                    lb_df_retriever = reset_rank(lb_df_retriever)
-                    lb_table_retriever = get_leaderboard_table(lb_df_retriever, types_qa)
-                    # Dummy leaderboard for handling the case when the user uses backspace key
-                    hidden_lb_df_retriever = original_df_qa[original_df_qa[COL_NAME_RERANKING_MODEL] == "NoReranker"]
-                    hidden_lb_df_retriever = reset_rank(hidden_lb_df_retriever)
-                    hidden_lb_table_retriever = get_leaderboard_table(hidden_lb_df_retriever, types_qa, visible=False)
-
-                    set_listeners(
-                        "qa",
-                        lb_table_retriever,
-                        hidden_lb_table_retriever,
-                        search_bar_retriever,
-                        selected_domains,
-                        selected_langs,
-                        selected_noreranker,
-                        show_anonymous,
-                        show_revision_and_timestamp,
-                    )
-
-                    # set metric listener
-                    selected_metric.change(
-                        update_metric_qa,
-                        [
-                            selected_metric,
+                        )
+
+                        # set metric listener
+                        selected_metric.change(
+                            update_metric_qa,
+                            [
+                                selected_metric,
+                                selected_domains,
+                                selected_langs,
+                                selected_rerankings,
+                                search_bar,
+                                show_anonymous,
+                                show_revision_and_timestamp,
+                            ],
+                            leaderboard_table,
+                            queue=True
+                        )
+                    with gr.TabItem("Retrieval Only", id=11):
+                        with gr.Row():
+                            with gr.Column(scale=1):
+                                search_bar_retriever = get_search_bar()
+                            with gr.Column(scale=1):
+                                selected_noreranker = get_noreranking_dropdown()
+                        lb_df_retriever = leaderboard_df_qa[leaderboard_df_qa[COL_NAME_RERANKING_MODEL] == "NoReranker"]
+                        lb_df_retriever = reset_rank(lb_df_retriever)
+                        lb_table_retriever = get_leaderboard_table(lb_df_retriever, types_qa)
+                        # Dummy leaderboard for handling the case when the user uses backspace key
+                        hidden_lb_df_retriever = original_df_qa[original_df_qa[COL_NAME_RERANKING_MODEL] == "NoReranker"]
+                        hidden_lb_df_retriever = reset_rank(hidden_lb_df_retriever)
+                        hidden_lb_table_retriever = get_leaderboard_table(hidden_lb_df_retriever, types_qa, visible=False)
+
+                        set_listeners(
+                            "qa",
+                            lb_table_retriever,
+                            hidden_lb_table_retriever,
+                            search_bar_retriever,
                             selected_domains,
                             selected_langs,
                             selected_noreranker,
-                            search_bar_retriever,
                             show_anonymous,
                             show_revision_and_timestamp,
-                        ],
-                        lb_table_retriever,
-                        queue=True
-                    )
-                with gr.TabItem("Reranking Only", id=12):
-                    lb_df_reranker = leaderboard_df_qa[leaderboard_df_qa[COL_NAME_RETRIEVAL_MODEL] == BM25_LINK]
-                    lb_df_reranker = reset_rank(lb_df_reranker)
-                    reranking_models_reranker = lb_df_reranker[COL_NAME_RERANKING_MODEL].apply(remove_html).unique().tolist()
-                    with gr.Row():
-                        with gr.Column(scale=1):
-                            selected_rerankings_reranker = get_reranking_dropdown(reranking_models_reranker)
-                        with gr.Column(scale=1):
-                            search_bar_reranker = gr.Textbox(show_label=False, visible=False)
-                    lb_table_reranker = get_leaderboard_table(lb_df_reranker, types_qa)
-                    hidden_lb_df_reranker = original_df_qa[original_df_qa[COL_NAME_RETRIEVAL_MODEL] == BM25_LINK]
-                    hidden_lb_df_reranker = reset_rank(hidden_lb_df_reranker)
-                    hidden_lb_table_reranker = get_leaderboard_table(
-                        hidden_lb_df_reranker, types_qa, visible=False
-                    )
-
-                    set_listeners(
-                        "qa",
-                        lb_table_reranker,
-                        hidden_lb_table_reranker,
-                        search_bar_reranker,
-                        selected_domains,
-                        selected_langs,
-                        selected_rerankings_reranker,
-                        show_anonymous,
-                        show_revision_and_timestamp,
-                    )
-                    # set metric listener
-                    selected_metric.change(
-                        update_metric_qa,
-                        [
-                            selected_metric,
+                        )
+
+                        # set metric listener
+                        selected_metric.change(
+                            update_metric_qa,
+                            [
+                                selected_metric,
+                                selected_domains,
+                                selected_langs,
+                                selected_noreranker,
+                                search_bar_retriever,
+                                show_anonymous,
+                                show_revision_and_timestamp,
+                            ],
+                            lb_table_retriever,
+                            queue=True
+                        )
+                    with gr.TabItem("Reranking Only", id=12):
+                        lb_df_reranker = leaderboard_df_qa[leaderboard_df_qa[COL_NAME_RETRIEVAL_MODEL] == BM25_LINK]
+                        lb_df_reranker = reset_rank(lb_df_reranker)
+                        reranking_models_reranker = lb_df_reranker[COL_NAME_RERANKING_MODEL].apply(remove_html).unique().tolist()
+                        with gr.Row():
+                            with gr.Column(scale=1):
+                                selected_rerankings_reranker = get_reranking_dropdown(reranking_models_reranker)
+                            with gr.Column(scale=1):
+                                search_bar_reranker = gr.Textbox(show_label=False, visible=False)
+                        lb_table_reranker = get_leaderboard_table(lb_df_reranker, types_qa)
+                        hidden_lb_df_reranker = original_df_qa[original_df_qa[COL_NAME_RETRIEVAL_MODEL] == BM25_LINK]
+                        hidden_lb_df_reranker = reset_rank(hidden_lb_df_reranker)
+                        hidden_lb_table_reranker = get_leaderboard_table(
+                            hidden_lb_df_reranker, types_qa, visible=False
+                        )
+
+                        set_listeners(
+                            "qa",
+                            lb_table_reranker,
+                            hidden_lb_table_reranker,
+                            search_bar_reranker,
                             selected_domains,
                             selected_langs,
                             selected_rerankings_reranker,
-                            search_bar_reranker,
                             show_anonymous,
                             show_revision_and_timestamp,
-                        ],
-                        lb_table_reranker,
-                        queue=True
-                    )
-        with gr.TabItem("Long Doc", elem_id="long-doc-benchmark-tab-table", id=1):
-            with gr.Row():
-                with gr.Column(min_width=320):
-                    # select domain
-                    with gr.Row():
-                        selected_domains = get_domain_dropdown(DOMAIN_COLS_LONG_DOC, DOMAIN_COLS_LONG_DOC)
-                    # select language
-                    with gr.Row():
-                        selected_langs = get_language_dropdown(
-                            LANG_COLS_LONG_DOC, LANG_COLS_LONG_DOC
                         )
-                with gr.Column():
-                    with gr.Row():
-                        selected_version = get_version_dropdown()
-                    # select the metric
-                    with gr.Row():
-                        selected_metric = get_metric_dropdown(METRIC_LIST, DEFAULT_METRIC_LONG_DOC)
-                    with gr.Row():
-                        show_anonymous = get_anonymous_checkbox()
-                    with gr.Row():
-                        show_revision_and_timestamp = get_revision_and_ts_checkbox()
-            with gr.Tabs(elem_classes="tab-buttons") as sub_tabs:
-                with gr.TabItem("Retrieval + Reranking", id=20):
-                    with gr.Row():
-                        with gr.Column():
-                            search_bar = get_search_bar()
-                        # select reranking model
-                        with gr.Column():
-                            selected_rerankings = get_reranking_dropdown(reranking_models)
-
-                    lb_table = get_leaderboard_table(
-                        leaderboard_df_long_doc, types_long_doc
-                    )
-
-                    # Dummy leaderboard for handling the case when the user uses backspace key
-                    hidden_lb_table_for_search = get_leaderboard_table(
-                        original_df_long_doc, types_long_doc, visible=False
-                    )
-
-                    set_listeners(
-                        "long-doc",
-                        lb_table,
-                        hidden_lb_table_for_search,
-                        search_bar,
-                        selected_domains,
-                        selected_langs,
-                        selected_rerankings,
-                        show_anonymous,
-                        show_revision_and_timestamp,
-                    )
-
-                    # set metric listener
-                    selected_metric.change(
-                        update_metric_long_doc,
-                        [
-                            selected_metric,
+                        # set metric listener
+                        selected_metric.change(
+                            update_metric_qa,
+                            [
+                                selected_metric,
+                                selected_domains,
+                                selected_langs,
+                                selected_rerankings_reranker,
+                                search_bar_reranker,
+                                show_anonymous,
+                                show_revision_and_timestamp,
+                            ],
+                            lb_table_reranker,
+                            queue=True
+                        )
+            with gr.TabItem("Long Doc", elem_id="long-doc-benchmark-tab-table", id=1):
+                with gr.Row():
+                    with gr.Column(min_width=320):
+                        # select domain
+                        with gr.Row():
+                            selected_domains = get_domain_dropdown(DOMAIN_COLS_LONG_DOC, DOMAIN_COLS_LONG_DOC)
+                        # select language
+                        with gr.Row():
+                            selected_langs = get_language_dropdown(
+                                LANG_COLS_LONG_DOC, LANG_COLS_LONG_DOC
+                            )
+                    with gr.Column():
+                        # select the metric
+                        with gr.Row():
+                            selected_metric = get_metric_dropdown(METRIC_LIST, DEFAULT_METRIC_LONG_DOC)
+                        with gr.Row():
+                            show_anonymous = get_anonymous_checkbox()
+                        with gr.Row():
+                            show_revision_and_timestamp = get_revision_and_ts_checkbox()
+                with gr.Tabs(elem_classes="tab-buttons") as sub_tabs:
+                    with gr.TabItem("Retrieval + Reranking", id=20):
+                        with gr.Row():
+                            with gr.Column():
+                                search_bar = get_search_bar()
+                            # select reranking model
+                            with gr.Column():
+                                selected_rerankings = get_reranking_dropdown(reranking_models)
+
+                        lb_table = get_leaderboard_table(
+                            leaderboard_df_long_doc, types_long_doc
+                        )
+
+                        # Dummy leaderboard for handling the case when the user uses backspace key
+                        hidden_lb_table_for_search = get_leaderboard_table(
+                            original_df_long_doc, types_long_doc, visible=False
+                        )
+
+                        set_listeners(
+                            "long-doc",
+                            lb_table,
+                            hidden_lb_table_for_search,
+                            search_bar,
                             selected_domains,
                             selected_langs,
                             selected_rerankings,
-                            search_bar,
                             show_anonymous,
-                            show_revision_and_timestamp
-                        ],
-                        lb_table,
-                        queue=True
-                    )
-                with gr.TabItem("Retrieval Only", id=21):
-                    with gr.Row():
-                        with gr.Column(scale=1):
-                            search_bar_retriever = get_search_bar()
-                        with gr.Column(scale=1):
-                            selected_noreranker = get_noreranking_dropdown()
-                    lb_df_retriever_long_doc = leaderboard_df_long_doc[
-                        leaderboard_df_long_doc[COL_NAME_RERANKING_MODEL] == "NoReranker"
-                    ]
-                    lb_df_retriever_long_doc = reset_rank(lb_df_retriever_long_doc)
-                    hidden_lb_db_retriever_long_doc = original_df_long_doc[
-                        original_df_long_doc[COL_NAME_RERANKING_MODEL] == "NoReranker"
-                    ]
-                    hidden_lb_db_retriever_long_doc = reset_rank(hidden_lb_db_retriever_long_doc)
-                    lb_table_retriever_long_doc = get_leaderboard_table(
-                        lb_df_retriever_long_doc, types_long_doc)
-                    hidden_lb_table_retriever_long_doc = get_leaderboard_table(
-                        hidden_lb_db_retriever_long_doc, types_long_doc, visible=False
-                    )
-
-                    set_listeners(
-                        "long-doc",
-                        lb_table_retriever_long_doc,
-                        hidden_lb_table_retriever_long_doc,
-                        search_bar_retriever,
-                        selected_domains,
-                        selected_langs,
-                        selected_noreranker,
-                        show_anonymous,
-                        show_revision_and_timestamp,
-                    )
-
-                    selected_metric.change(
-                        update_metric_long_doc,
-                        [
-                            selected_metric,
+                            show_revision_and_timestamp,
+                        )
+
+                        # set metric listener
+                        selected_metric.change(
+                            update_metric_long_doc,
+                            [
+                                selected_metric,
+                                selected_domains,
+                                selected_langs,
+                                selected_rerankings,
+                                search_bar,
+                                show_anonymous,
+                                show_revision_and_timestamp
+                            ],
+                            lb_table,
+                            queue=True
+                        )
+                    with gr.TabItem("Retrieval Only", id=21):
+                        with gr.Row():
+                            with gr.Column(scale=1):
+                                search_bar_retriever = get_search_bar()
+                            with gr.Column(scale=1):
+                                selected_noreranker = get_noreranking_dropdown()
+                        lb_df_retriever_long_doc = leaderboard_df_long_doc[
+                            leaderboard_df_long_doc[COL_NAME_RERANKING_MODEL] == "NoReranker"
+                        ]
+                        lb_df_retriever_long_doc = reset_rank(lb_df_retriever_long_doc)
+                        hidden_lb_db_retriever_long_doc = original_df_long_doc[
+                            original_df_long_doc[COL_NAME_RERANKING_MODEL] == "NoReranker"
+                        ]
+                        hidden_lb_db_retriever_long_doc = reset_rank(hidden_lb_db_retriever_long_doc)
+                        lb_table_retriever_long_doc = get_leaderboard_table(
+                            lb_df_retriever_long_doc, types_long_doc)
+                        hidden_lb_table_retriever_long_doc = get_leaderboard_table(
+                            hidden_lb_db_retriever_long_doc, types_long_doc, visible=False
+                        )
+
+                        set_listeners(
+                            "long-doc",
+                            lb_table_retriever_long_doc,
+                            hidden_lb_table_retriever_long_doc,
+                            search_bar_retriever,
                             selected_domains,
                             selected_langs,
                             selected_noreranker,
-                            search_bar_retriever,
                             show_anonymous,
                             show_revision_and_timestamp,
-                        ],
-                        lb_table_retriever_long_doc,
-                        queue=True
-                    )
-                with gr.TabItem("Reranking Only", id=22):
-                    lb_df_reranker_ldoc = leaderboard_df_long_doc[
-                        leaderboard_df_long_doc[COL_NAME_RETRIEVAL_MODEL] == BM25_LINK
-                        ]
-                    lb_df_reranker_ldoc = reset_rank(lb_df_reranker_ldoc)
-                    reranking_models_reranker_ldoc = lb_df_reranker_ldoc[COL_NAME_RERANKING_MODEL].apply(remove_html).unique().tolist()
-                    with gr.Row():
-                        with gr.Column(scale=1):
-                            selected_rerankings_reranker_ldoc = get_reranking_dropdown(reranking_models_reranker_ldoc)
-                        with gr.Column(scale=1):
-                            search_bar_reranker_ldoc = gr.Textbox(show_label=False, visible=False)
-                    lb_table_reranker_ldoc = get_leaderboard_table(lb_df_reranker_ldoc, types_long_doc)
-                    hidden_lb_df_reranker_ldoc = original_df_long_doc[original_df_long_doc[COL_NAME_RETRIEVAL_MODEL] == BM25_LINK]
-                    hidden_lb_df_reranker_ldoc = reset_rank(hidden_lb_df_reranker_ldoc)
-                    hidden_lb_table_reranker_ldoc = get_leaderboard_table(
-                        hidden_lb_df_reranker_ldoc, types_long_doc, visible=False
-                    )
-
-                    set_listeners(
-                        "long-doc",
-                        lb_table_reranker_ldoc,
-                        hidden_lb_table_reranker_ldoc,
-                        search_bar_reranker_ldoc,
-                        selected_domains,
-                        selected_langs,
-                        selected_rerankings_reranker_ldoc,
-                        show_anonymous,
-                        show_revision_and_timestamp,
-                    )
-                    selected_metric.change(
-                        update_metric_long_doc,
-                        [
-                            selected_metric,
+                        )
+
+                        selected_metric.change(
+                            update_metric_long_doc,
+                            [
+                                selected_metric,
+                                selected_domains,
+                                selected_langs,
+                                selected_noreranker,
+                                search_bar_retriever,
+                                show_anonymous,
+                                show_revision_and_timestamp,
+                            ],
+                            lb_table_retriever_long_doc,
+                            queue=True
+                        )
+                    with gr.TabItem("Reranking Only", id=22):
+                        lb_df_reranker_ldoc = leaderboard_df_long_doc[
+                            leaderboard_df_long_doc[COL_NAME_RETRIEVAL_MODEL] == BM25_LINK
+                            ]
+                        lb_df_reranker_ldoc = reset_rank(lb_df_reranker_ldoc)
+                        reranking_models_reranker_ldoc = lb_df_reranker_ldoc[COL_NAME_RERANKING_MODEL].apply(remove_html).unique().tolist()
+                        with gr.Row():
+                            with gr.Column(scale=1):
+                                selected_rerankings_reranker_ldoc = get_reranking_dropdown(reranking_models_reranker_ldoc)
+                            with gr.Column(scale=1):
+                                search_bar_reranker_ldoc = gr.Textbox(show_label=False, visible=False)
+                        lb_table_reranker_ldoc = get_leaderboard_table(lb_df_reranker_ldoc, types_long_doc)
+                        hidden_lb_df_reranker_ldoc = original_df_long_doc[original_df_long_doc[COL_NAME_RETRIEVAL_MODEL] == BM25_LINK]
+                        hidden_lb_df_reranker_ldoc = reset_rank(hidden_lb_df_reranker_ldoc)
+                        hidden_lb_table_reranker_ldoc = get_leaderboard_table(
+                            hidden_lb_df_reranker_ldoc, types_long_doc, visible=False
+                        )
+
+                        set_listeners(
+                            "long-doc",
+                            lb_table_reranker_ldoc,
+                            hidden_lb_table_reranker_ldoc,
+                            search_bar_reranker_ldoc,
                             selected_domains,
                             selected_langs,
                             selected_rerankings_reranker_ldoc,
-                            search_bar_reranker_ldoc,
                             show_anonymous,
                             show_revision_and_timestamp,
-                        ],
-                        lb_table_reranker_ldoc,
-                        queue=True
-                    )
+                        )
+                        selected_metric.change(
+                            update_metric_long_doc,
+                            [
+                                selected_metric,
+                                selected_domains,
+                                selected_langs,
+                                selected_rerankings_reranker_ldoc,
+                                search_bar_reranker_ldoc,
+                                show_anonymous,
+                                show_revision_and_timestamp,
+                            ],
+                            lb_table_reranker_ldoc,
+                            queue=True
+                        )
 
         with gr.TabItem("🚀Submit here!", elem_id="submit-tab-table", id=2):
             with gr.Column():

requirements.txt

@@ -12,3 +12,4 @@ requests>=2.31.0
 tqdm>=4.65.0
 accelerate>=0.24.1
 socksio>=1.0.0
+air-benchmark>=0.0.4

src/about.py

@@ -1,6 +1,6 @@
 # Your leaderboard name
 TITLE = """<h1 align="center" id="space-title">AIR-Bench: Automated Heterogeneous Information Retrieval Benchmark
- (v0.0.3) </h1>"""
+ (v0.1.0.dev) </h1>"""
 
 # What does your leaderboard evaluate?
 INTRODUCTION_TEXT = """
@@ -17,14 +17,14 @@ BENCHMARKS_TEXT = f"""
   - A: Yes, we plan to release new datasets on regular basis. However, the update frequency is to be decided.  
 
 - Q: As you are using models to do the quality control when generating the data, is it biased to the models that are used?
-  - A: Yes, the results is biased to the chosen models. However, we believe the datasets labeled by human are also biased to the human's preference. The key point to verify is whether the model's bias is consistent with the human's. We use our approach to generate test data using the well established MSMARCO datasets. We benchmark different models' performances using the generated dataset and the human-label DEV dataset. Comparing the ranking of different models on these two datasets, we observe the spearman correlation between them is 0.8211 (p-value=5e-5). This indicates that the models' perference is well aligned with the human. Please refer to [here](https://github.com/AIR-Bench/AIR-Bench/blob/main/docs/available_evaluation_results.md#consistency-with-ms-marco) for details
+  - A: Yes, the results is biased to the chosen models. However, we believe the datasets labeled by human are also biased to the human's preference. The key point to verify is whether the model's bias is consistent with the human's. We use our approach to generate test data using the well established MSMARCO datasets. We benchmark different models' performances using the generated dataset and the human-label DEV dataset. Comparing the ranking of different models on these two datasets, we observe the spearman correlation between them is 0.8211 (p-value=5e-5). This indicates that the models' perference is well aligned with the human. Please refer to [here](https://github.com/AIR-Bench/AIR-Bench/blob/main/docs/available_analysis_results.md#consistency-with-human-labeled-data) for details.
 
 """
 
 EVALUATION_QUEUE_TEXT = """
 ## Check out the submission steps at [our GitHub repo](https://github.com/AIR-Bench/AIR-Bench/blob/main/docs/submit_to_leaderboard.md)
 
-## You can find the **STATUS of Your Submission** at the [Backend Space](https://huggingface.co/spaces/AIR-Bench/leaderboard_backend). 
+## You can find the **STATUS of Your Submission** at the [Backend Space](https://huggingface.co/spaces/AIR-Bench/leaderboard_backend)
 
 - If the status is **✔️ Success**, then you can find your results at the [Leaderboard Space](https://huggingface.co/spaces/AIR-Bench/leaderboard) in no more than one hour.
 - If the status is **❌ Failed**, please check your submission steps and try again. If you have any questions, please feel free to open an issue [here](https://github.com/AIR-Bench/AIR-Bench/issues/new).

src/benchmarks.py

@@ -1,5 +1,6 @@
 from dataclasses import dataclass
 from enum import Enum
+from air_benchmark.tasks.tasks import BenchmarkTable
 
 
 def get_safe_name(name: str):
@@ -11,67 +12,6 @@ def get_safe_name(name: str):
         if (character.isalnum() or character == '_'))
 
 
-dataset_dict = {
-    "qa": {
-        "wiki": {
-            "en": ["wikipedia_20240101", ],
-            "zh": ["wikipedia_20240101", ]
-        },
-        "web": {
-            "en": ["mC4", ],
-            "zh": ["mC4", ]
-        },
-        "news": {
-            "en": ["CC-News", ],
-            "zh": ["CC-News", ]
-        },
-        "healthcare": {
-            "en": ["PubMedQA", ],
-            "zh": ["Huatuo-26M", ]
-        },
-        "law": {
-            "en": ["pile-of-law", ],
-             # "zh": ["flk_npc_gov_cn", ]
-        },
-        "finance": {
-            "en": ["Reuters-Financial", ],
-            "zh": ["FinCorpus", ]
-        },
-        "arxiv": {
-            "en": ["Arxiv", ]},
-        "msmarco": {
-            "en": ["MS MARCO", ]},
-    },
-    "long-doc": {
-        "arxiv": {
-            "en": ["gpt3", "llama2", "llm-survey", "gemini"],
-        },
-        "book": {
-            "en": [
-                "origin-of-species_darwin",
-                "a-brief-history-of-time_stephen-hawking"
-            ]
-        },
-        "healthcare": {
-            "en": [
-                "pubmed_100k-200k_1",
-                "pubmed_100k-200k_2",
-                "pubmed_100k-200k_3",
-                "pubmed_40k-50k_5-merged",
-                "pubmed_30k-40k_10-merged"
-            ]
-        },
-        "law": {
-            "en": [
-                "lex_files_300k-400k",
-                "lex_files_400k-500k",
-                "lex_files_500k-600k",
-                "lex_files_600k-700k"
-            ]
-        }
-    }
-}
-
 METRIC_LIST = [
     "ndcg_at_1",
     "ndcg_at_3",
@@ -118,7 +58,7 @@ class Benchmark:
 
 qa_benchmark_dict = {}
 long_doc_benchmark_dict = {}
-for task, domain_dict in dataset_dict.items():
+for task, domain_dict in BenchmarkTable['AIR-Bench_24.04'].items():
     for domain, lang_dict in domain_dict.items():
         for lang, dataset_list in lang_dict.items():
             if task == "qa":

src/envs.py

@@ -27,7 +27,7 @@ BM25_LINK = model_hyperlink("https://github.com/castorini/pyserini", "BM25")
 
 BENCHMARK_VERSION_LIST = [
     "AIR-Bench_24.04",
-    # "AIR-Bench_24.05",
+    "AIR-Bench_24.05",
 ]
 
 LATEST_BENCHMARK_VERSION = BENCHMARK_VERSION_LIST[-1]