fix-24-05-display-issue

.github/workflows/main.yaml

@@ -1,20 +0,0 @@
-name: Sync to Hugging Face hub
-on:
-  push:
-    branches: [main]
-
-  # to run this workflow manually from the Actions tab
-  workflow_dispatch:
-
-jobs:
-  sync-to-hub:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-        with:
-          fetch-depth: 0
-          lfs: true
-      - name: Push to hub
-        env:
-          HF_TOKEN: ${{ secrets.HF_TOKEN }}
-        run: git push https://hanhainebula:[email protected]/spaces/AIR-Bench/leaderboard main

Makefile

@@ -3,21 +3,11 @@
 
 style:
 	python -m black --line-length 119 .
-	python -m black --line-length 119 src
 	python -m isort .
-	python -m isort src
 	ruff check --fix .
-	ruff check --fix src
 
 
 quality:
 	python -m black --check --line-length 119 .
-	python -m black --check --line-length 119 src
 	python -m isort --check-only .
-	python -m isort --check-only src
 	ruff check .
-	ruff check src
-
-
-test:
-	python -m pytest tests

README.md

@@ -10,14 +10,36 @@ pinned: true
 license: apache-2.0
 ---
 
-# The AIR-Bench Leaderboard repository
+# Start the configuration
 
-This repository contains the code for the AIR-Bench Leaderboard.
+Most of the variables to change for a default leaderboard are in `src/env.py` (replace the path for your leaderboard) and `src/about.py` (for tasks).
 
-| Important Links                                              | Description                                    |
-| ------------------------------------------------------------ | ---------------------------------------------- |
-| [AIR-Bench](https://github.com/AIR-Bench/AIR-Bench)          | The main repository for the AIR-Bench project. |
-| [Leaderboard Space](https://huggingface.co/spaces/AIR-Bench/leaderboard) | The leaderboard space on Hugging Face.         |
-| [Leaderboard Backend Space](https://huggingface.co/spaces/AIR-Bench/leaderboard_backend) | The leaderboard backend space on Hugging Face. |
-| [Leaderboard Code](https://github.com/AIR-Bench/leaderboard) | The code for the leaderboard.                  |
-| [Leaderboard Backend Code](https://github.com/AIR-Bench/leaderboard_backend) | The code for the leaderboard backend.          |
+Results files should have the following format and be stored as json files:
+```json
+{
+    "config": {
+        "model_dtype": "torch.float16", # or torch.bfloat16 or 8bit or 4bit
+        "model_name": "path of the model on the hub: org/model",
+        "model_sha": "revision on the hub",
+    },
+    "results": {
+        "task_name": {
+            "metric_name": score,
+        },
+        "task_name2": {
+            "metric_name": score,
+        }
+    }
+}
+```
+
+Request files are created automatically by this tool.
+
+If you encounter problem on the space, don't hesitate to restart it to remove the create eval-queue, eval-queue-bk, eval-results and eval-results-bk created folder.
+
+# Code logic for more complex edits
+
+You'll find 
+- the main table' columns names and properties in `src/display/utils.py`
+- the logic to read all results and request files, then convert them in dataframe lines, in `src/leaderboard/read_evals.py`, and `src/populate.py`
+- teh logic to allow or filter submissions in `src/submission/submit.py` and `src/submission/check_validity.py`

app.py

@@ -1,141 +1,131 @@
-import os
-
 import gradio as gr
 from apscheduler.schedulers.background import BackgroundScheduler
 from huggingface_hub import snapshot_download
 
-from src.about import BENCHMARKS_TEXT, EVALUATION_QUEUE_TEXT, INTRODUCTION_TEXT, TITLE
-from src.benchmarks import LongDocBenchmarks, QABenchmarks
-from src.columns import COL_NAME_RERANKING_MODEL, COL_NAME_RETRIEVAL_MODEL
-from src.components import (
-    get_anonymous_checkbox,
-    get_domain_dropdown,
-    get_language_dropdown,
-    get_leaderboard_table,
-    get_metric_dropdown,
-    get_noreranking_dropdown,
-    get_reranking_dropdown,
-    get_revision_and_ts_checkbox,
-    get_search_bar,
-    get_version_dropdown,
+from src.about import (
+    INTRODUCTION_TEXT,
+    BENCHMARKS_TEXT,
+    TITLE,
+    EVALUATION_QUEUE_TEXT
+)
+from src.benchmarks import (
+    DOMAIN_COLS_QA,
+    LANG_COLS_QA,
+    DOMAIN_COLS_LONG_DOC,
+    LANG_COLS_LONG_DOC,
+    METRIC_LIST,
+    DEFAULT_METRIC_QA,
+    DEFAULT_METRIC_LONG_DOC
+)
+from src.display.css_html_js import custom_css
+from src.display.utils import (
+    COL_NAME_IS_ANONYMOUS,
+    COL_NAME_REVISION,
+    COL_NAME_TIMESTAMP,
+    COL_NAME_RERANKING_MODEL,
+    COL_NAME_RETRIEVAL_MODEL
 )
-from src.css_html_js import custom_css
 from src.envs import (
     API,
-    BENCHMARK_VERSION_LIST,
-    DEFAULT_METRIC_LONG_DOC,
-    DEFAULT_METRIC_QA,
     EVAL_RESULTS_PATH,
-    LATEST_BENCHMARK_VERSION,
-    METRIC_LIST,
     REPO_ID,
     RESULTS_REPO,
     TOKEN,
+    BM25_LINK,
+    BENCHMARK_VERSION_LIST,
+    LATEST_BENCHMARK_VERSION
 )
-from src.loaders import load_eval_results
-from src.models import TaskType, model_hyperlink
-from src.utils import remove_html, reset_rank, set_listeners, submit_results, update_metric, upload_file
-
+from src.read_evals import (
+    get_raw_eval_results,
+    get_leaderboard_df
+)
+from src.utils import (
+    update_metric,
+    upload_file,
+    get_default_cols,
+    submit_results,
+    reset_rank,
+    remove_html
+)
+from src.display.gradio_formatting import (
+    get_version_dropdown,
+    get_search_bar,
+    get_reranking_dropdown,
+    get_metric_dropdown,
+    get_domain_dropdown,
+    get_language_dropdown,
+    get_anonymous_checkbox,
+    get_revision_and_ts_checkbox,
+    get_leaderboard_table,
+    get_noreranking_dropdown
+)
+from src.display.gradio_listener import set_listeners
 
 def restart_space():
     API.restart_space(repo_id=REPO_ID)
 
 
 try:
-    if not os.environ.get("LOCAL_MODE", False):
-        print("Running in local mode")
-        snapshot_download(
-            repo_id=RESULTS_REPO,
-            local_dir=EVAL_RESULTS_PATH,
-            repo_type="dataset",
-            tqdm_class=None,
-            etag_timeout=30,
-            token=TOKEN,
-        )
-except Exception:
-    print("failed to download")
+    snapshot_download(
+        repo_id=RESULTS_REPO, local_dir=EVAL_RESULTS_PATH, repo_type="dataset", tqdm_class=None, etag_timeout=30,
+        token=TOKEN
+    )
+except Exception as e:
+    print(f'failed to download')
     restart_space()
 
-global ds_dict
-ds_dict = load_eval_results(EVAL_RESULTS_PATH)
-global datastore
-datastore = ds_dict[LATEST_BENCHMARK_VERSION]
-
-
-def update_qa_metric(
-    metric: str,
-    domains: list,
-    langs: list,
-    reranking_model: list,
-    query: str,
-    show_anonymous: bool,
-    show_revision_and_timestamp: bool,
-):
-    global datastore
-    return update_metric(
-        datastore,
-        TaskType.qa,
-        metric,
-        domains,
-        langs,
-        reranking_model,
-        query,
-        show_anonymous,
+raw_data = get_raw_eval_results(f"{EVAL_RESULTS_PATH}/{LATEST_BENCHMARK_VERSION}")
+
+original_df_qa = get_leaderboard_df(
+    raw_data, task='qa', metric=DEFAULT_METRIC_QA)
+original_df_long_doc = get_leaderboard_df(
+    raw_data, task='long-doc', metric=DEFAULT_METRIC_LONG_DOC)
+print(f'raw data: {len(raw_data)}')
+print(f'QA data loaded: {original_df_qa.shape}')
+print(f'Long-Doc data loaded: {len(original_df_long_doc)}')
+
+leaderboard_df_qa = original_df_qa.copy()
+# leaderboard_df_qa = leaderboard_df_qa[has_no_nan_values(df, _benchmark_cols)]
+shown_columns_qa, types_qa = get_default_cols(
+    'qa', leaderboard_df_qa.columns, add_fix_cols=True)
+leaderboard_df_qa = leaderboard_df_qa[~leaderboard_df_qa[COL_NAME_IS_ANONYMOUS]][shown_columns_qa]
+leaderboard_df_qa.drop([COL_NAME_REVISION, COL_NAME_TIMESTAMP], axis=1, inplace=True)
+
+leaderboard_df_long_doc = original_df_long_doc.copy()
+shown_columns_long_doc, types_long_doc = get_default_cols(
+    'long-doc', leaderboard_df_long_doc.columns, add_fix_cols=True)
+leaderboard_df_long_doc = leaderboard_df_long_doc[~leaderboard_df_long_doc[COL_NAME_IS_ANONYMOUS]][shown_columns_long_doc]
+leaderboard_df_long_doc.drop([COL_NAME_REVISION, COL_NAME_TIMESTAMP], axis=1, inplace=True)
+
+# select reranking model
+reranking_models = sorted(list(frozenset([eval_result.reranking_model for eval_result in raw_data])))
+
+
+def update_metric_qa(
+        metric: str,
+        domains: list,
+        langs: list,
+        reranking_model: list,
+        query: str,
+        show_anonymous: bool,
         show_revision_and_timestamp,
-    )
-
-
-def update_doc_metric(
-    metric: str,
-    domains: list,
-    langs: list,
-    reranking_model: list,
-    query: str,
-    show_anonymous: bool,
-    show_revision_and_timestamp,
 ):
-    global datastore
-    return update_metric(
-        datastore,
-        TaskType.long_doc,
-        metric,
-        domains,
-        langs,
-        reranking_model,
-        query,
-        show_anonymous,
+    return update_metric(raw_data, 'qa', metric, domains, langs, reranking_model, query, show_anonymous, show_revision_and_timestamp)
+
+def update_metric_long_doc(
+        metric: str,
+        domains: list,
+        langs: list,
+        reranking_model: list,
+        query: str,
+        show_anonymous: bool,
         show_revision_and_timestamp,
-    )
-
-
-def update_qa_version(version):
-    global datastore
-    global ds_dict
-    datastore = ds_dict[version]
-    domain_elem = get_domain_dropdown(QABenchmarks[datastore.slug])
-    lang_elem = get_language_dropdown(QABenchmarks[datastore.slug])
-    model_elem = get_reranking_dropdown(datastore.reranking_models)
-    df_elem = get_leaderboard_table(datastore.qa_fmt_df, datastore.qa_types)
-    hidden_df_elem = get_leaderboard_table(datastore.qa_raw_df, datastore.qa_types, visible=False)
-    return domain_elem, lang_elem, model_elem, df_elem, hidden_df_elem
-
-
-def update_doc_version(version):
-    global datastore
-    global ds_dict
-    datastore = ds_dict[version]
-    domain_elem = get_domain_dropdown(LongDocBenchmarks[datastore.slug])
-    lang_elem = get_language_dropdown(LongDocBenchmarks[datastore.slug])
-    model_elem = get_reranking_dropdown(datastore.reranking_models)
-    df_elem = get_leaderboard_table(datastore.doc_fmt_df, datastore.doc_types)
-    hidden_df_elem = get_leaderboard_table(datastore.doc_raw_df, datastore.doc_types, visible=False)
-    return domain_elem, lang_elem, model_elem, df_elem, hidden_df_elem
+):
+    return update_metric(raw_data, "long-doc", metric, domains, langs, reranking_model, query, show_anonymous, show_revision_and_timestamp)
 
 
 demo = gr.Blocks(css=custom_css)
 
-BM25_LINK = model_hyperlink("https://github.com/castorini/pyserini", "BM25")
-
 with demo:
     gr.HTML(TITLE)
     gr.Markdown(INTRODUCTION_TEXT, elem_classes="markdown-text")
@@ -143,24 +133,25 @@ with demo:
     with gr.Tabs(elem_classes="tab-buttons") as tabs:
         with gr.TabItem("Results", elem_id="results-tab-table"):
             with gr.Row():
-                version = get_version_dropdown()
+                selected_version = get_version_dropdown()
 
             with gr.TabItem("QA", elem_id="qa-benchmark-tab-table", id=0):
                 with gr.Row():
                     with gr.Column(min_width=320):
                         # select domain
                         with gr.Row():
-                            domains = get_domain_dropdown(QABenchmarks[datastore.slug])
+                            selected_domains = get_domain_dropdown(DOMAIN_COLS_QA, DOMAIN_COLS_QA)
                         # select language
                         with gr.Row():
-                            langs = get_language_dropdown(QABenchmarks[datastore.slug])
+                            selected_langs = get_language_dropdown(LANG_COLS_QA, LANG_COLS_QA)
+
                     with gr.Column():
                         # select the metric
-                        metric = get_metric_dropdown(METRIC_LIST, DEFAULT_METRIC_QA)
+                        selected_metric = get_metric_dropdown(METRIC_LIST, DEFAULT_METRIC_QA)
                         with gr.Row():
                             show_anonymous = get_anonymous_checkbox()
                         with gr.Row():
-                            show_rev_ts = get_revision_and_ts_checkbox()
+                            show_revision_and_timestamp = get_revision_and_ts_checkbox()
                 with gr.Tabs(elem_classes="tab-buttons") as sub_tabs:
                     with gr.TabItem("Retrieval + Reranking", id=10):
                         with gr.Row():
@@ -169,327 +160,273 @@ with demo:
                                 search_bar = get_search_bar()
                             # select reranking models
                             with gr.Column():
-                                models = get_reranking_dropdown(datastore.reranking_models)
-                        #  shown_table
-                        qa_df_elem_ret_rerank = get_leaderboard_table(datastore.qa_fmt_df, datastore.qa_types)
+                                selected_rerankings = get_reranking_dropdown(reranking_models)
+                        leaderboard_table = get_leaderboard_table(leaderboard_df_qa, types_qa)
                         # Dummy leaderboard for handling the case when the user uses backspace key
-                        qa_df_elem_ret_rerank_hidden = get_leaderboard_table(
-                            datastore.qa_raw_df, datastore.qa_types, visible=False
-                        )
-
-                        version.change(
-                            update_qa_version,
-                            version,
-                            [domains, langs, models, qa_df_elem_ret_rerank, qa_df_elem_ret_rerank_hidden],
-                        )
+                        hidden_leaderboard_table_for_search = get_leaderboard_table(original_df_qa, types_qa, visible=False)
 
                         set_listeners(
-                            TaskType.qa,
-                            qa_df_elem_ret_rerank,
-                            qa_df_elem_ret_rerank_hidden,
+                            "qa",
+                            leaderboard_table,
+                            hidden_leaderboard_table_for_search,
                             search_bar,
-                            version,
-                            domains,
-                            langs,
-                            models,
+                            selected_domains,
+                            selected_langs,
+                            selected_rerankings,
                             show_anonymous,
-                            show_rev_ts,
+                            show_revision_and_timestamp,
                         )
 
                         # set metric listener
-                        metric.change(
-                            update_qa_metric,
-                            [metric, domains, langs, models, search_bar, show_anonymous, show_rev_ts],
-                            qa_df_elem_ret_rerank,
-                            queue=True,
+                        selected_metric.change(
+                            update_metric_qa,
+                            [
+                                selected_metric,
+                                selected_domains,
+                                selected_langs,
+                                selected_rerankings,
+                                search_bar,
+                                show_anonymous,
+                                show_revision_and_timestamp,
+                            ],
+                            leaderboard_table,
+                            queue=True
                         )
-
                     with gr.TabItem("Retrieval Only", id=11):
                         with gr.Row():
                             with gr.Column(scale=1):
-                                search_bar_ret = get_search_bar()
+                                search_bar_retriever = get_search_bar()
                             with gr.Column(scale=1):
-                                models_ret = get_noreranking_dropdown()
-
-                        _qa_df_ret = datastore.qa_fmt_df[datastore.qa_fmt_df[COL_NAME_RERANKING_MODEL] == "NoReranker"]
-                        _qa_df_ret = reset_rank(_qa_df_ret)
-                        qa_df_elem_ret = get_leaderboard_table(_qa_df_ret, datastore.qa_types)
-
+                                selected_noreranker = get_noreranking_dropdown()
+                        lb_df_retriever = leaderboard_df_qa[leaderboard_df_qa[COL_NAME_RERANKING_MODEL] == "NoReranker"]
+                        lb_df_retriever = reset_rank(lb_df_retriever)
+                        lb_table_retriever = get_leaderboard_table(lb_df_retriever, types_qa)
                         # Dummy leaderboard for handling the case when the user uses backspace key
-                        _qa_df_ret_hidden = datastore.qa_raw_df[
-                            datastore.qa_raw_df[COL_NAME_RERANKING_MODEL] == "NoReranker"
-                        ]
-                        _qa_df_ret_hidden = reset_rank(_qa_df_ret_hidden)
-                        qa_df_elem_ret_hidden = get_leaderboard_table(
-                            _qa_df_ret_hidden, datastore.qa_types, visible=False
-                        )
-
-                        version.change(
-                            update_qa_version,
-                            version,
-                            [
-                                domains,
-                                langs,
-                                models_ret,
-                                qa_df_elem_ret,
-                                qa_df_elem_ret_hidden,
-                            ],
-                        )
+                        hidden_lb_df_retriever = original_df_qa[original_df_qa[COL_NAME_RERANKING_MODEL] == "NoReranker"]
+                        hidden_lb_df_retriever = reset_rank(hidden_lb_df_retriever)
+                        hidden_lb_table_retriever = get_leaderboard_table(hidden_lb_df_retriever, types_qa, visible=False)
 
                         set_listeners(
-                            TaskType.qa,
-                            qa_df_elem_ret,
-                            qa_df_elem_ret_hidden,
-                            search_bar_ret,
-                            version,
-                            domains,
-                            langs,
-                            models_ret,
+                            "qa",
+                            lb_table_retriever,
+                            hidden_lb_table_retriever,
+                            search_bar_retriever,
+                            selected_domains,
+                            selected_langs,
+                            selected_noreranker,
                             show_anonymous,
-                            show_rev_ts,
+                            show_revision_and_timestamp,
                         )
 
-                        metric.change(
-                            update_qa_metric,
+                        # set metric listener
+                        selected_metric.change(
+                            update_metric_qa,
                             [
-                                metric,
-                                domains,
-                                langs,
-                                models_ret,
-                                search_bar_ret,
+                                selected_metric,
+                                selected_domains,
+                                selected_langs,
+                                selected_noreranker,
+                                search_bar_retriever,
                                 show_anonymous,
-                                show_rev_ts,
+                                show_revision_and_timestamp,
                             ],
-                            qa_df_elem_ret,
-                            queue=True,
+                            lb_table_retriever,
+                            queue=True
                         )
-
                     with gr.TabItem("Reranking Only", id=12):
-                        _qa_df_rerank = datastore.qa_fmt_df[datastore.qa_fmt_df[COL_NAME_RETRIEVAL_MODEL] == BM25_LINK]
-                        _qa_df_rerank = reset_rank(_qa_df_rerank)
-                        qa_rerank_models = _qa_df_rerank[COL_NAME_RERANKING_MODEL].apply(remove_html).unique().tolist()
+                        lb_df_reranker = leaderboard_df_qa[leaderboard_df_qa[COL_NAME_RETRIEVAL_MODEL] == BM25_LINK]
+                        lb_df_reranker = reset_rank(lb_df_reranker)
+                        reranking_models_reranker = lb_df_reranker[COL_NAME_RERANKING_MODEL].apply(remove_html).unique().tolist()
                         with gr.Row():
                             with gr.Column(scale=1):
-                                qa_models_rerank = get_reranking_dropdown(qa_rerank_models)
+                                selected_rerankings_reranker = get_reranking_dropdown(reranking_models_reranker)
                             with gr.Column(scale=1):
-                                qa_search_bar_rerank = gr.Textbox(show_label=False, visible=False)
-                        qa_df_elem_rerank = get_leaderboard_table(_qa_df_rerank, datastore.qa_types)
-
-                        _qa_df_rerank_hidden = datastore.qa_raw_df[
-                            datastore.qa_raw_df[COL_NAME_RETRIEVAL_MODEL] == BM25_LINK
-                        ]
-                        _qa_df_rerank_hidden = reset_rank(_qa_df_rerank_hidden)
-                        qa_df_elem_rerank_hidden = get_leaderboard_table(
-                            _qa_df_rerank_hidden, datastore.qa_types, visible=False
-                        )
-
-                        version.change(
-                            update_qa_version,
-                            version,
-                            [domains, langs, qa_models_rerank, qa_df_elem_rerank, qa_df_elem_rerank_hidden],
+                                search_bar_reranker = gr.Textbox(show_label=False, visible=False)
+                        lb_table_reranker = get_leaderboard_table(lb_df_reranker, types_qa)
+                        hidden_lb_df_reranker = original_df_qa[original_df_qa[COL_NAME_RETRIEVAL_MODEL] == BM25_LINK]
+                        hidden_lb_df_reranker = reset_rank(hidden_lb_df_reranker)
+                        hidden_lb_table_reranker = get_leaderboard_table(
+                            hidden_lb_df_reranker, types_qa, visible=False
                         )
 
                         set_listeners(
-                            TaskType.qa,
-                            qa_df_elem_rerank,
-                            qa_df_elem_rerank_hidden,
-                            qa_search_bar_rerank,
-                            version,
-                            domains,
-                            langs,
-                            qa_models_rerank,
+                            "qa",
+                            lb_table_reranker,
+                            hidden_lb_table_reranker,
+                            search_bar_reranker,
+                            selected_domains,
+                            selected_langs,
+                            selected_rerankings_reranker,
                             show_anonymous,
-                            show_rev_ts,
+                            show_revision_and_timestamp,
                         )
-
-                        metric.change(
-                            update_qa_metric,
+                        # set metric listener
+                        selected_metric.change(
+                            update_metric_qa,
                             [
-                                metric,
-                                domains,
-                                langs,
-                                qa_models_rerank,
-                                qa_search_bar_rerank,
+                                selected_metric,
+                                selected_domains,
+                                selected_langs,
+                                selected_rerankings_reranker,
+                                search_bar_reranker,
                                 show_anonymous,
-                                show_rev_ts,
+                                show_revision_and_timestamp,
                             ],
-                            qa_df_elem_rerank,
-                            queue=True,
+                            lb_table_reranker,
+                            queue=True
                         )
             with gr.TabItem("Long Doc", elem_id="long-doc-benchmark-tab-table", id=1):
                 with gr.Row():
                     with gr.Column(min_width=320):
                         # select domain
                         with gr.Row():
-                            domains = get_domain_dropdown(LongDocBenchmarks[datastore.slug])
+                            selected_domains = get_domain_dropdown(DOMAIN_COLS_LONG_DOC, DOMAIN_COLS_LONG_DOC)
                         # select language
                         with gr.Row():
-                            langs = get_language_dropdown(LongDocBenchmarks[datastore.slug])
+                            selected_langs = get_language_dropdown(
+                                LANG_COLS_LONG_DOC, LANG_COLS_LONG_DOC
+                            )
                     with gr.Column():
                         # select the metric
                         with gr.Row():
-                            metric = get_metric_dropdown(METRIC_LIST, DEFAULT_METRIC_LONG_DOC)
+                            selected_metric = get_metric_dropdown(METRIC_LIST, DEFAULT_METRIC_LONG_DOC)
                         with gr.Row():
                             show_anonymous = get_anonymous_checkbox()
                         with gr.Row():
-                            show_rev_ts = get_revision_and_ts_checkbox()
-                with gr.Tabs(elem_classes="tab-buttons"):
+                            show_revision_and_timestamp = get_revision_and_ts_checkbox()
+                with gr.Tabs(elem_classes="tab-buttons") as sub_tabs:
                     with gr.TabItem("Retrieval + Reranking", id=20):
                         with gr.Row():
                             with gr.Column():
                                 search_bar = get_search_bar()
+                            # select reranking model
                             with gr.Column():
-                                models = get_reranking_dropdown(datastore.reranking_models)
-
-                        doc_df_elem_ret_rerank = get_leaderboard_table(datastore.doc_fmt_df, datastore.doc_types)
+                                selected_rerankings = get_reranking_dropdown(reranking_models)
 
-                        # Dummy leaderboard for handling the case when the user uses backspace key
-                        doc_df_elem_ret_rerank_hidden = get_leaderboard_table(
-                            datastore.doc_raw_df, datastore.doc_types, visible=False
+                        lb_table = get_leaderboard_table(
+                            leaderboard_df_long_doc, types_long_doc
                         )
 
-                        version.change(
-                            update_doc_version,
-                            version,
-                            [domains, langs, models, doc_df_elem_ret_rerank, doc_df_elem_ret_rerank_hidden],
+                        # Dummy leaderboard for handling the case when the user uses backspace key
+                        hidden_lb_table_for_search = get_leaderboard_table(
+                            original_df_long_doc, types_long_doc, visible=False
                         )
 
                         set_listeners(
-                            TaskType.long_doc,
-                            doc_df_elem_ret_rerank,
-                            doc_df_elem_ret_rerank_hidden,
+                            "long-doc",
+                            lb_table,
+                            hidden_lb_table_for_search,
                             search_bar,
-                            version,
-                            domains,
-                            langs,
-                            models,
+                            selected_domains,
+                            selected_langs,
+                            selected_rerankings,
                             show_anonymous,
-                            show_rev_ts,
+                            show_revision_and_timestamp,
                         )
 
                         # set metric listener
-                        metric.change(
-                            update_doc_metric,
+                        selected_metric.change(
+                            update_metric_long_doc,
                             [
-                                metric,
-                                domains,
-                                langs,
-                                models,
+                                selected_metric,
+                                selected_domains,
+                                selected_langs,
+                                selected_rerankings,
                                 search_bar,
                                 show_anonymous,
-                                show_rev_ts,
+                                show_revision_and_timestamp
                             ],
-                            doc_df_elem_ret_rerank,
-                            queue=True,
+                            lb_table,
+                            queue=True
                         )
                     with gr.TabItem("Retrieval Only", id=21):
                         with gr.Row():
                             with gr.Column(scale=1):
-                                search_bar_ret = get_search_bar()
+                                search_bar_retriever = get_search_bar()
                             with gr.Column(scale=1):
-                                models_ret = get_noreranking_dropdown()
-
-                        _doc_df_ret = datastore.doc_fmt_df[
-                            datastore.doc_fmt_df[COL_NAME_RERANKING_MODEL] == "NoReranker"
+                                selected_noreranker = get_noreranking_dropdown()
+                        lb_df_retriever_long_doc = leaderboard_df_long_doc[
+                            leaderboard_df_long_doc[COL_NAME_RERANKING_MODEL] == "NoReranker"
                         ]
-                        _doc_df_ret = reset_rank(_doc_df_ret)
-                        doc_df_elem_ret = get_leaderboard_table(_doc_df_ret, datastore.doc_types)
-
-                        _doc_df_ret_hidden = datastore.doc_raw_df[
-                            datastore.doc_raw_df[COL_NAME_RERANKING_MODEL] == "NoReranker"
+                        lb_df_retriever_long_doc = reset_rank(lb_df_retriever_long_doc)
+                        hidden_lb_db_retriever_long_doc = original_df_long_doc[
+                            original_df_long_doc[COL_NAME_RERANKING_MODEL] == "NoReranker"
                         ]
-                        _doc_df_ret_hidden = reset_rank(_doc_df_ret_hidden)
-                        doc_df_elem_ret_hidden = get_leaderboard_table(
-                            _doc_df_ret_hidden, datastore.doc_types, visible=False
-                        )
-
-                        version.change(
-                            update_doc_version,
-                            version,
-                            [domains, langs, models_ret, doc_df_elem_ret, doc_df_elem_ret_hidden],
+                        hidden_lb_db_retriever_long_doc = reset_rank(hidden_lb_db_retriever_long_doc)
+                        lb_table_retriever_long_doc = get_leaderboard_table(
+                            lb_df_retriever_long_doc, types_long_doc)
+                        hidden_lb_table_retriever_long_doc = get_leaderboard_table(
+                            hidden_lb_db_retriever_long_doc, types_long_doc, visible=False
                         )
 
                         set_listeners(
-                            TaskType.long_doc,
-                            doc_df_elem_ret,
-                            doc_df_elem_ret_hidden,
-                            search_bar_ret,
-                            version,
-                            domains,
-                            langs,
-                            models_ret,
+                            "long-doc",
+                            lb_table_retriever_long_doc,
+                            hidden_lb_table_retriever_long_doc,
+                            search_bar_retriever,
+                            selected_domains,
+                            selected_langs,
+                            selected_noreranker,
                             show_anonymous,
-                            show_rev_ts,
+                            show_revision_and_timestamp,
                         )
 
-                        metric.change(
-                            update_doc_metric,
+                        selected_metric.change(
+                            update_metric_long_doc,
                             [
-                                metric,
-                                domains,
-                                langs,
-                                models_ret,
-                                search_bar_ret,
+                                selected_metric,
+                                selected_domains,
+                                selected_langs,
+                                selected_noreranker,
+                                search_bar_retriever,
                                 show_anonymous,
-                                show_rev_ts,
+                                show_revision_and_timestamp,
                             ],
-                            doc_df_elem_ret,
-                            queue=True,
+                            lb_table_retriever_long_doc,
+                            queue=True
                         )
                     with gr.TabItem("Reranking Only", id=22):
-                        _doc_df_rerank = datastore.doc_fmt_df[
-                            datastore.doc_fmt_df[COL_NAME_RETRIEVAL_MODEL] == BM25_LINK
-                        ]
-                        _doc_df_rerank = reset_rank(_doc_df_rerank)
-                        doc_rerank_models = (
-                            _doc_df_rerank[COL_NAME_RERANKING_MODEL].apply(remove_html).unique().tolist()
-                        )
+                        lb_df_reranker_ldoc = leaderboard_df_long_doc[
+                            leaderboard_df_long_doc[COL_NAME_RETRIEVAL_MODEL] == BM25_LINK
+                            ]
+                        lb_df_reranker_ldoc = reset_rank(lb_df_reranker_ldoc)
+                        reranking_models_reranker_ldoc = lb_df_reranker_ldoc[COL_NAME_RERANKING_MODEL].apply(remove_html).unique().tolist()
                         with gr.Row():
                             with gr.Column(scale=1):
-                                doc_models_rerank = get_reranking_dropdown(doc_rerank_models)
+                                selected_rerankings_reranker_ldoc = get_reranking_dropdown(reranking_models_reranker_ldoc)
                             with gr.Column(scale=1):
-                                doc_search_bar_rerank = gr.Textbox(show_label=False, visible=False)
-                        doc_df_elem_rerank = get_leaderboard_table(_doc_df_rerank, datastore.doc_types)
-                        _doc_df_rerank_hidden = datastore.doc_raw_df[
-                            datastore.doc_raw_df[COL_NAME_RETRIEVAL_MODEL] == BM25_LINK
-                        ]
-                        _doc_df_rerank_hidden = reset_rank(_doc_df_rerank_hidden)
-                        doc_df_elem_rerank_hidden = get_leaderboard_table(
-                            _doc_df_rerank_hidden, datastore.doc_types, visible=False
-                        )
-
-                        version.change(
-                            update_doc_version,
-                            version,
-                            [domains, langs, doc_models_rerank, doc_df_elem_rerank, doc_df_elem_rerank_hidden],
+                                search_bar_reranker_ldoc = gr.Textbox(show_label=False, visible=False)
+                        lb_table_reranker_ldoc = get_leaderboard_table(lb_df_reranker_ldoc, types_long_doc)
+                        hidden_lb_df_reranker_ldoc = original_df_long_doc[original_df_long_doc[COL_NAME_RETRIEVAL_MODEL] == BM25_LINK]
+                        hidden_lb_df_reranker_ldoc = reset_rank(hidden_lb_df_reranker_ldoc)
+                        hidden_lb_table_reranker_ldoc = get_leaderboard_table(
+                            hidden_lb_df_reranker_ldoc, types_long_doc, visible=False
                         )
 
                         set_listeners(
-                            TaskType.long_doc,
-                            doc_df_elem_rerank,
-                            doc_df_elem_rerank_hidden,
-                            doc_search_bar_rerank,
-                            version,
-                            domains,
-                            langs,
-                            doc_models_rerank,
+                            "long-doc",
+                            lb_table_reranker_ldoc,
+                            hidden_lb_table_reranker_ldoc,
+                            search_bar_reranker_ldoc,
+                            selected_domains,
+                            selected_langs,
+                            selected_rerankings_reranker_ldoc,
                             show_anonymous,
-                            show_rev_ts,
+                            show_revision_and_timestamp,
                         )
-
-                        metric.change(
-                            update_doc_metric,
+                        selected_metric.change(
+                            update_metric_long_doc,
                             [
-                                metric,
-                                domains,
-                                langs,
-                                doc_models_rerank,
-                                doc_search_bar_rerank,
+                                selected_metric,
+                                selected_domains,
+                                selected_langs,
+                                selected_rerankings_reranker_ldoc,
+                                search_bar_reranker_ldoc,
                                 show_anonymous,
-                                show_rev_ts,
+                                show_revision_and_timestamp,
                             ],
-                            doc_df_elem_rerank,
-                            queue=True,
+                            lb_table_reranker_ldoc,
+                            queue=True
                         )
 
         with gr.TabItem("🚀Submit here!", elem_id="submit-tab-table", id=2):
@@ -506,18 +443,23 @@ with demo:
                 with gr.Row():
                     with gr.Column():
                         reranking_model_name = gr.Textbox(
-                            label="Reranking Model name", info="Optional", value="NoReranker"
+                            label="Reranking Model name",
+                            info="Optional",
+                            value="NoReranker"
                         )
                     with gr.Column():
-                        reranking_model_url = gr.Textbox(label="Reranking Model URL", info="Optional", value="")
+                        reranking_model_url = gr.Textbox(
+                            label="Reranking Model URL",
+                            info="Optional",
+                            value=""
+                        )
                 with gr.Row():
                     with gr.Column():
                         benchmark_version = gr.Dropdown(
                             BENCHMARK_VERSION_LIST,
                             value=LATEST_BENCHMARK_VERSION,
                             interactive=True,
-                            label="AIR-Bench Version",
-                        )
+                            label="AIR-Bench Version")
                 with gr.Row():
                     upload_button = gr.UploadButton("Click to upload search results", file_count="single")
                 with gr.Row():
@@ -526,8 +468,7 @@ with demo:
                     is_anonymous = gr.Checkbox(
                         label="Nope. I want to submit anonymously 🥷",
                         value=False,
-                        info="Do you want to shown on the leaderboard by default?",
-                    )
+                        info="Do you want to shown on the leaderboard by default?")
                 with gr.Row():
                     submit_button = gr.Button("Submit")
                 with gr.Row():
@@ -537,8 +478,7 @@ with demo:
                     [
                         upload_button,
                     ],
-                    file_output,
-                )
+                    file_output)
                 submit_button.click(
                     submit_results,
                     [
@@ -548,10 +488,10 @@ with demo:
                         reranking_model_name,
                         reranking_model_url,
                         benchmark_version,
-                        is_anonymous,
+                        is_anonymous
                     ],
                     submission_result,
-                    show_progress="hidden",
+                    show_progress="hidden"
                 )
 
         with gr.TabItem("📝 About", elem_id="llm-benchmark-tab-table", id=3):

pyproject.toml

@@ -1,9 +1,9 @@
 [tool.ruff]
 # Enable pycodestyle (`E`) and Pyflakes (`F`) codes by default.
-lint.select = ["E", "F"]
-lint.ignore = ["E501"] # line too long (black is taking care of this)
+select = ["E", "F"]
+ignore = ["E501"] # line too long (black is taking care of this)
 line-length = 119
-lint.fixable = ["A", "B", "C", "D", "E", "F", "G", "I", "N", "Q", "S", "T", "W", "ANN", "ARG", "BLE", "COM", "DJ", "DTZ", "EM", "ERA", "EXE", "FBT", "ICN", "INP", "ISC", "NPY", "PD", "PGH", "PIE", "PL", "PT", "PTH", "PYI", "RET", "RSE", "RUF", "SIM", "SLF", "TCH", "TID", "TRY", "UP", "YTT"]
+fixable = ["A", "B", "C", "D", "E", "F", "G", "I", "N", "Q", "S", "T", "W", "ANN", "ARG", "BLE", "COM", "DJ", "DTZ", "EM", "ERA", "EXE", "FBT", "ICN", "INP", "ISC", "NPY", "PD", "PGH", "PIE", "PL", "PT", "PTH", "PYI", "RET", "RSE", "RUF", "SIM", "SLF", "TCH", "TID", "TRY", "UP", "YTT"]
 
 [tool.isort]
 profile = "black"

requirements.txt

@@ -2,7 +2,7 @@ APScheduler>=3.10.1
 black>=23.11.0
 click>=8.1.3
 datasets>=2.14.5
-gradio<5.0.0
+gradio>=4.29.0
 gradio_client>=0.16.1
 huggingface-hub>=0.18.0
 numpy>=1.24.2
@@ -12,4 +12,4 @@ requests>=2.31.0
 tqdm>=4.65.0
 accelerate>=0.24.1
 socksio>=1.0.0
-air-benchmark>=0.1.0
+air-benchmark>=0.0.4

src/about.py

@@ -8,7 +8,7 @@ INTRODUCTION_TEXT = """
 """
 
 # Which evaluations are you running? how can people reproduce what you have?
-BENCHMARKS_TEXT = """
+BENCHMARKS_TEXT = f"""
 ## How the test data are generated?
 ### Find more information at [our GitHub repo](https://github.com/AIR-Bench/AIR-Bench/blob/main/docs/data_generation.md)
 

src/benchmarks.py

@@ -1,71 +1,92 @@
 from dataclasses import dataclass
 from enum import Enum
-
 from air_benchmark.tasks.tasks import BenchmarkTable
 
-from src.envs import BENCHMARK_VERSION_LIST, METRIC_LIST
-from src.models import TaskType, get_safe_name
+
+def get_safe_name(name: str):
+    """Get RFC 1123 compatible safe name"""
+    name = name.replace('-', '_')
+    return ''.join(
+        character.lower()
+        for character in name
+        if (character.isalnum() or character == '_'))
+
+
+METRIC_LIST = [
+    "ndcg_at_1",
+    "ndcg_at_3",
+    "ndcg_at_5",
+    "ndcg_at_10",
+    "ndcg_at_100",
+    "ndcg_at_1000",
+    "map_at_1",
+    "map_at_3",
+    "map_at_5",
+    "map_at_10",
+    "map_at_100",
+    "map_at_1000",
+    "recall_at_1",
+    "recall_at_3",
+    "recall_at_5",
+    "recall_at_10",
+    "recall_at_100",
+    "recall_at_1000",
+    "precision_at_1",
+    "precision_at_3",
+    "precision_at_5",
+    "precision_at_10",
+    "precision_at_100",
+    "precision_at_1000",
+    "mrr_at_1",
+    "mrr_at_3",
+    "mrr_at_5",
+    "mrr_at_10",
+    "mrr_at_100",
+    "mrr_at_1000"
+]
 
 
 @dataclass
 class Benchmark:
     name: str  # [domain]_[language]_[metric], task_key in the json file,
-    metric: str  # metric_key in the json file
+    metric: str  # ndcg_at_1 ,metric_key in the json file
     col_name: str  # [domain]_[language], name to display in the leaderboard
     domain: str
     lang: str
     task: str
 
 
-# create a function return an enum class containing all the benchmarks
-def get_qa_benchmarks_dict(version: str):
-    benchmark_dict = {}
-    for task, domain_dict in BenchmarkTable[version].items():
-        if task != TaskType.qa.value:
-            continue
-        for domain, lang_dict in domain_dict.items():
-            for lang, dataset_list in lang_dict.items():
-                benchmark_name = get_safe_name(f"{domain}_{lang}")
+qa_benchmark_dict = {}
+long_doc_benchmark_dict = {}
+for task, domain_dict in BenchmarkTable['AIR-Bench_24.04'].items():
+    for domain, lang_dict in domain_dict.items():
+        for lang, dataset_list in lang_dict.items():
+            if task == "qa":
+                benchmark_name = f"{domain}_{lang}"
+                benchmark_name = get_safe_name(benchmark_name)
                 col_name = benchmark_name
                 for metric in dataset_list:
-                    if "test" not in dataset_list[metric]["splits"]:
-                        continue
-                    benchmark_dict[benchmark_name] = Benchmark(benchmark_name, metric, col_name, domain, lang, task)
-    return benchmark_dict
-
-
-def get_doc_benchmarks_dict(version: str):
-    benchmark_dict = {}
-    for task, domain_dict in BenchmarkTable[version].items():
-        if task != TaskType.long_doc.value:
-            continue
-        for domain, lang_dict in domain_dict.items():
-            for lang, dataset_list in lang_dict.items():
+                    qa_benchmark_dict[benchmark_name] = Benchmark(benchmark_name, metric, col_name, domain, lang, task)
+            elif task == "long-doc":
                 for dataset in dataset_list:
                     benchmark_name = f"{domain}_{lang}_{dataset}"
                     benchmark_name = get_safe_name(benchmark_name)
                     col_name = benchmark_name
-                    if "test" not in dataset_list[dataset]["splits"]:
-                        continue
                     for metric in METRIC_LIST:
-                        benchmark_dict[benchmark_name] = Benchmark(
-                            benchmark_name, metric, col_name, domain, lang, task
-                        )
-    return benchmark_dict
+                        long_doc_benchmark_dict[benchmark_name] = Benchmark(benchmark_name, metric, col_name, domain,
+                                                                            lang, task)
 
+BenchmarksQA = Enum('BenchmarksQA', qa_benchmark_dict)
+BenchmarksLongDoc = Enum('BenchmarksLongDoc', long_doc_benchmark_dict)
 
-_qa_benchmark_dict = {}
-for version in BENCHMARK_VERSION_LIST:
-    safe_version_name = get_safe_name(version)
-    _qa_benchmark_dict[safe_version_name] = Enum(f"QABenchmarks_{safe_version_name}", get_qa_benchmarks_dict(version))
+BENCHMARK_COLS_QA = [c.col_name for c in qa_benchmark_dict.values()]
+BENCHMARK_COLS_LONG_DOC = [c.col_name for c in long_doc_benchmark_dict.values()]
 
-_doc_benchmark_dict = {}
-for version in BENCHMARK_VERSION_LIST:
-    safe_version_name = get_safe_name(version)
-    _doc_benchmark_dict[safe_version_name] = Enum(
-        f"LongDocBenchmarks_{safe_version_name}", get_doc_benchmarks_dict(version)
-    )
+DOMAIN_COLS_QA = list(frozenset([c.domain for c in qa_benchmark_dict.values()]))
+LANG_COLS_QA = list(frozenset([c.lang for c in qa_benchmark_dict.values()]))
 
+DOMAIN_COLS_LONG_DOC = list(frozenset([c.domain for c in long_doc_benchmark_dict.values()]))
+LANG_COLS_LONG_DOC = list(frozenset([c.lang for c in long_doc_benchmark_dict.values()]))
 
-QABenchmarks = Enum("QABenchmarks", _qa_benchmark_dict)
-LongDocBenchmarks = Enum("LongDocBenchmarks", _doc_benchmark_dict)
+DEFAULT_METRIC_QA = "ndcg_at_10"
+DEFAULT_METRIC_LONG_DOC = "recall_at_10"

src/display/formatting.py

@@ -0,0 +1,29 @@
+def model_hyperlink(link, model_name):
+    return f'<a target="_blank" href="{link}" style="color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;">{model_name}</a>'
+
+
+def make_clickable_model(model_name: str, model_link: str):
+    # link = f"https://huggingface.co/{model_name}"
+    if not model_link or not model_link.startswith("https://"):
+        return model_name
+    return model_hyperlink(model_link, model_name)
+
+
+def styled_error(error):
+    return f"<p style='color: red; font-size: 20px; text-align: center;'>{error}</p>"
+
+
+def styled_warning(warn):
+    return f"<p style='color: orange; font-size: 20px; text-align: center;'>{warn}</p>"
+
+
+def styled_message(message):
+    return f"<p style='color: green; font-size: 20px; text-align: center;'>{message}</p>"
+
+
+def has_no_nan_values(df, columns):
+    return df[columns].notna().all(axis=1)
+
+
+def has_nan_values(df, columns):
+    return df[columns].isna().any(axis=1)

src/{components.py → display/gradio_formatting.py}

@@ -1,14 +1,12 @@
 import gradio as gr
-
 from src.envs import BENCHMARK_VERSION_LIST, LATEST_BENCHMARK_VERSION
 
-
 def get_version_dropdown():
     return gr.Dropdown(
         choices=BENCHMARK_VERSION_LIST,
         value=LATEST_BENCHMARK_VERSION,
         label="Select the version of AIR-Bench",
-        interactive=True,
+        interactive=True
     )
 
 
@@ -16,25 +14,26 @@ def get_search_bar():
     return gr.Textbox(
         placeholder=" 🔍 Search for retrieval methods (separate multiple queries with `;`) and press ENTER...",
         show_label=False,
-        info="Search the retrieval methods",
+        info="Search the retrieval methods"
     )
 
 
 def get_reranking_dropdown(model_list):
-    return gr.Dropdown(choices=model_list, label="Select the reranking models", interactive=True, multiselect=True)
+    return gr.Dropdown(
+        choices=model_list,
+        label="Select the reranking models",
+        interactive=True,
+        multiselect=True
+    )
 
 
 def get_noreranking_dropdown():
     return gr.Dropdown(
-        choices=[
-            "NoReranker",
-        ],
-        value=[
-            "NoReranker",
-        ],
+        choices=["NoReranker", ],
+        value=["NoReranker", ],
         interactive=False,
         multiselect=True,
-        visible=False,
+        visible=False
     )
 
 
@@ -53,10 +52,7 @@ def get_metric_dropdown(metric_list, default_metrics):
     )
 
 
-def get_domain_dropdown(benchmarks, default_domains=None):
-    domain_list = list(frozenset([c.value.domain for c in list(benchmarks.value)]))
-    if default_domains is None:
-        default_domains = domain_list
+def get_domain_dropdown(domain_list, default_domains):
     return gr.CheckboxGroup(
         choices=domain_list,
         value=default_domains,
@@ -65,16 +61,13 @@ def get_domain_dropdown(benchmarks, default_domains=None):
     )
 
 
-def get_language_dropdown(benchmarks, default_languages=None):
-    language_list = list(frozenset([c.value.lang for c in list(benchmarks.value)]))
-    if default_languages is None:
-        default_languages = language_list
+def get_language_dropdown(language_list, default_languages):
     return gr.Dropdown(
         choices=language_list,
-        value=default_languages,
+        value=language_list,
         label="Select the languages",
         multiselect=True,
-        interactive=True,
+        interactive=True
     )
 
 
@@ -82,13 +75,15 @@ def get_anonymous_checkbox():
     return gr.Checkbox(
         label="Show anonymous submissions",
         value=False,
-        info="The anonymous submissions might have invalid model information.",
+        info="The anonymous submissions might have invalid model information."
     )
 
 
 def get_revision_and_ts_checkbox():
     return gr.Checkbox(
-        label="Show submission details", value=False, info="Show the revision and timestamp information of submissions"
+        label="Show submission details",
+        value=False,
+        info="Show the revision and timestamp information of submissions"
     )
 
 

src/display/gradio_listener.py

@@ -0,0 +1,53 @@
+from src.utils import update_table, update_table_long_doc
+
+
+def set_listeners(
+        task,
+        displayed_leaderboard,
+        hidden_leaderboard,
+        search_bar,
+        selected_domains,
+        selected_langs,
+        selected_rerankings,
+        show_anonymous,
+        show_revision_and_timestamp,
+
+):
+    if task == "qa":
+        update_table_func = update_table
+    elif task == "long-doc":
+        update_table_func = update_table_long_doc
+    else:
+        raise NotImplementedError
+    # Set search_bar listener
+    search_bar.submit(
+        update_table_func,
+        [
+            hidden_leaderboard,  #  hidden_leaderboard_table_for_search,
+            selected_domains,
+            selected_langs,
+            selected_rerankings,
+            search_bar,
+            show_anonymous,
+        ],
+        displayed_leaderboard
+    )
+
+    # Set column-wise listener
+    for selector in [
+        selected_domains, selected_langs, show_anonymous, show_revision_and_timestamp, selected_rerankings
+    ]:
+        selector.change(
+            update_table_func,
+            [
+                hidden_leaderboard,
+                selected_domains,
+                selected_langs,
+                selected_rerankings,
+                search_bar,
+                show_anonymous,
+                show_revision_and_timestamp
+            ],
+            displayed_leaderboard,
+            queue=True,
+        )

src/{columns.py → display/utils.py}

@@ -1,7 +1,9 @@
 from dataclasses import dataclass, make_dataclass
 
+from src.benchmarks import BenchmarksQA, BenchmarksLongDoc
 
-def _fields(raw_class):
+
+def fields(raw_class):
     return [v for k, v in raw_class.__dict__.items() if k[:2] != "__" and k[-2:] != "__"]
 
 
@@ -17,22 +19,28 @@ class ColumnContent:
     never_hidden: bool = False
 
 
+COL_NAME_AVG = "Average ⬆️"
+COL_NAME_RETRIEVAL_MODEL = "Retrieval Method"
+COL_NAME_RERANKING_MODEL = "Reranking Model"
+COL_NAME_RETRIEVAL_MODEL_LINK = "Retrieval Model LINK"
+COL_NAME_RERANKING_MODEL_LINK = "Reranking Model LINK"
+COL_NAME_RANK = "Rank 🏆"
+COL_NAME_REVISION = "Revision"
+COL_NAME_TIMESTAMP = "Submission Date"
+COL_NAME_IS_ANONYMOUS = "Anonymous Submission"
+
+
 def get_default_auto_eval_column_dict():
     auto_eval_column_dict = []
-    auto_eval_column_dict.append(["rank", ColumnContent, ColumnContent(COL_NAME_RANK, "number", True)])
+    # Init
     auto_eval_column_dict.append(
-        [
-            "retrieval_model",
-            ColumnContent,
-            ColumnContent(COL_NAME_RETRIEVAL_MODEL, "markdown", True, never_hidden=True),
-        ]
+        ["rank", ColumnContent, ColumnContent(COL_NAME_RANK, "number", True)]
     )
     auto_eval_column_dict.append(
-        [
-            "reranking_model",
-            ColumnContent,
-            ColumnContent(COL_NAME_RERANKING_MODEL, "markdown", True, never_hidden=True),
-        ]
+        ["retrieval_model", ColumnContent, ColumnContent(COL_NAME_RETRIEVAL_MODEL, "markdown", True, hidden=False, never_hidden=True)]
+    )
+    auto_eval_column_dict.append(
+        ["reranking_model", ColumnContent, ColumnContent(COL_NAME_RERANKING_MODEL, "markdown", True, hidden=False, never_hidden=True)]
     )
     auto_eval_column_dict.append(
         ["revision", ColumnContent, ColumnContent(COL_NAME_REVISION, "markdown", True, never_hidden=True)]
@@ -40,30 +48,14 @@ def get_default_auto_eval_column_dict():
     auto_eval_column_dict.append(
         ["timestamp", ColumnContent, ColumnContent(COL_NAME_TIMESTAMP, "date", True, never_hidden=True)]
     )
-    auto_eval_column_dict.append(["average", ColumnContent, ColumnContent(COL_NAME_AVG, "number", True)])
     auto_eval_column_dict.append(
-        [
-            "retrieval_model_link",
-            ColumnContent,
-            ColumnContent(
-                COL_NAME_RETRIEVAL_MODEL_LINK,
-                "markdown",
-                False,
-                hidden=True,
-            ),
-        ]
+        ["average", ColumnContent, ColumnContent(COL_NAME_AVG, "number", True)]
+    )
+    auto_eval_column_dict.append(
+        ["retrieval_model_link", ColumnContent, ColumnContent(COL_NAME_RETRIEVAL_MODEL_LINK, "markdown", False, hidden=True, never_hidden=False)]
     )
     auto_eval_column_dict.append(
-        [
-            "reranking_model_link",
-            ColumnContent,
-            ColumnContent(
-                COL_NAME_RERANKING_MODEL_LINK,
-                "markdown",
-                False,
-                hidden=True,
-            ),
-        ]
+        ["reranking_model_link", ColumnContent, ColumnContent(COL_NAME_RERANKING_MODEL_LINK, "markdown", False, hidden=True, never_hidden=False)]
     )
     auto_eval_column_dict.append(
         ["is_anonymous", ColumnContent, ColumnContent(COL_NAME_IS_ANONYMOUS, "bool", False, hidden=True)]
@@ -71,10 +63,10 @@ def get_default_auto_eval_column_dict():
     return auto_eval_column_dict
 
 
-def make_autoevalcolumn(cls_name, benchmarks):
+def make_autoevalcolumn(cls_name="BenchmarksQA", benchmarks=BenchmarksQA):
     auto_eval_column_dict = get_default_auto_eval_column_dict()
-    # Leaderboard columns
-    for benchmark in list(benchmarks.value):
+    ## Leaderboard columns
+    for benchmark in benchmarks:
         auto_eval_column_dict.append(
             [benchmark.name, ColumnContent, ColumnContent(benchmark.value.col_name, "number", True)]
         )
@@ -83,24 +75,19 @@ def make_autoevalcolumn(cls_name, benchmarks):
     return make_dataclass(cls_name, auto_eval_column_dict, frozen=True)
 
 
-def get_default_col_names_and_types(benchmarks):
-    AutoEvalColumn = make_autoevalcolumn("AutoEvalColumn", benchmarks)
-    col_names = [c.name for c in _fields(AutoEvalColumn) if not c.hidden]
-    col_types = [c.type for c in _fields(AutoEvalColumn) if not c.hidden]
-    return col_names, col_types
+AutoEvalColumnQA = make_autoevalcolumn(
+    "AutoEvalColumnQA", BenchmarksQA)
+AutoEvalColumnLongDoc = make_autoevalcolumn(
+    "AutoEvalColumnLongDoc", BenchmarksLongDoc)
 
 
-def get_fixed_col_names_and_types():
-    fixed_cols = get_default_auto_eval_column_dict()[:-3]
-    return [c.name for _, _, c in fixed_cols], [c.type for _, _, c in fixed_cols]
+# Column selection
+COLS_QA = [c.name for c in fields(AutoEvalColumnQA) if not c.hidden]
+COLS_LONG_DOC = [c.name for c in fields(AutoEvalColumnLongDoc) if not c.hidden]
+TYPES_QA = [c.type for c in fields(AutoEvalColumnQA) if not c.hidden]
+TYPES_LONG_DOC = [c.type for c in fields(AutoEvalColumnLongDoc) if not c.hidden]
+COLS_LITE = [c.name for c in fields(AutoEvalColumnQA) if c.displayed_by_default and not c.hidden]
 
+QA_BENCHMARK_COLS = [t.value.col_name for t in BenchmarksQA]
 
-COL_NAME_AVG = "Average ⬆️"
-COL_NAME_RETRIEVAL_MODEL = "Retrieval Method"
-COL_NAME_RERANKING_MODEL = "Reranking Model"
-COL_NAME_RETRIEVAL_MODEL_LINK = "Retrieval Model LINK"
-COL_NAME_RERANKING_MODEL_LINK = "Reranking Model LINK"
-COL_NAME_RANK = "Rank 🏆"
-COL_NAME_REVISION = "Revision"
-COL_NAME_TIMESTAMP = "Submission Date"
-COL_NAME_IS_ANONYMOUS = "Anonymous Submission"
+LONG_DOC_BENCHMARK_COLS = [t.value.col_name for t in BenchmarksLongDoc]

src/envs.py

@@ -1,14 +1,12 @@
 import os
-
+from src.display.formatting import model_hyperlink
 from huggingface_hub import HfApi
 
 # Info to change for your repository
 # ----------------------------------
 TOKEN = os.environ.get("TOKEN", "")  # A read/write token for your org
 
-OWNER = (
-    "AIR-Bench"  # Change to your org - don't forget to create a results and request dataset, with the correct format!
-)
+OWNER = "AIR-Bench"  # "nan"  # Change to your org - don't forget to create a results and request dataset, with the correct format!
 # ----------------------------------
 
 REPO_ID = f"{OWNER}/leaderboard"
@@ -17,7 +15,7 @@ RESULTS_REPO = f"{OWNER}/eval_results"
 # repo for submitting the evaluation
 SEARCH_RESULTS_REPO = f"{OWNER}/search_results"
 
-# If you set up a cache later, just change HF_HOME
+# If you setup a cache later, just change HF_HOME
 CACHE_PATH = os.getenv("HF_HOME", ".")
 
 # Local caches
@@ -25,43 +23,11 @@ EVAL_RESULTS_PATH = os.path.join(CACHE_PATH, "eval_results")
 
 API = HfApi(token=TOKEN)
 
+BM25_LINK = model_hyperlink("https://github.com/castorini/pyserini", "BM25")
+
 BENCHMARK_VERSION_LIST = [
     "AIR-Bench_24.04",
     "AIR-Bench_24.05",
 ]
 
-LATEST_BENCHMARK_VERSION = BENCHMARK_VERSION_LIST[0]
-DEFAULT_METRIC_QA = "ndcg_at_10"
-DEFAULT_METRIC_LONG_DOC = "recall_at_10"
-METRIC_LIST = [
-    "ndcg_at_1",
-    "ndcg_at_3",
-    "ndcg_at_5",
-    "ndcg_at_10",
-    "ndcg_at_100",
-    "ndcg_at_1000",
-    "map_at_1",
-    "map_at_3",
-    "map_at_5",
-    "map_at_10",
-    "map_at_100",
-    "map_at_1000",
-    "recall_at_1",
-    "recall_at_3",
-    "recall_at_5",
-    "recall_at_10",
-    "recall_at_100",
-    "recall_at_1000",
-    "precision_at_1",
-    "precision_at_3",
-    "precision_at_5",
-    "precision_at_10",
-    "precision_at_100",
-    "precision_at_1000",
-    "mrr_at_1",
-    "mrr_at_3",
-    "mrr_at_5",
-    "mrr_at_10",
-    "mrr_at_100",
-    "mrr_at_1000",
-]
+LATEST_BENCHMARK_VERSION = BENCHMARK_VERSION_LIST[-1]

src/loaders.py

@@ -1,88 +0,0 @@
-import os.path
-from pathlib import Path
-from typing import Dict, List, Union
-
-import pandas as pd
-
-from src.columns import COL_NAME_IS_ANONYMOUS, COL_NAME_REVISION, COL_NAME_TIMESTAMP
-from src.envs import BENCHMARK_VERSION_LIST, DEFAULT_METRIC_LONG_DOC, DEFAULT_METRIC_QA
-from src.models import FullEvalResult, LeaderboardDataStore, TaskType, get_safe_name
-from src.utils import get_default_cols, get_leaderboard_df, reset_rank
-
-pd.options.mode.copy_on_write = True
-
-
-def load_raw_eval_results(results_path: Union[Path, str]) -> List[FullEvalResult]:
-    """
-    Load the evaluation results from a json file
-    """
-    model_result_filepaths = []
-    for root, dirs, files in os.walk(results_path):
-        if len(files) == 0:
-            continue
-
-        # select the latest results
-        for file in files:
-            if not (file.startswith("results") and file.endswith(".json")):
-                print(f"skip {file}")
-                continue
-            model_result_filepaths.append(os.path.join(root, file))
-
-    eval_results = {}
-    for model_result_filepath in model_result_filepaths:
-        # create evaluation results
-        try:
-            eval_result = FullEvalResult.init_from_json_file(model_result_filepath)
-        except UnicodeDecodeError:
-            print(f"loading file failed. {model_result_filepath}")
-            continue
-        print(f"file loaded: {model_result_filepath}")
-        timestamp = eval_result.timestamp
-        eval_results[timestamp] = eval_result
-
-    results = []
-    for k, v in eval_results.items():
-        try:
-            v.to_dict()
-            results.append(v)
-        except KeyError:
-            print(f"loading failed: {k}")
-            continue
-    return results
-
-
-def load_leaderboard_datastore(file_path, version) -> LeaderboardDataStore:
-    ds = LeaderboardDataStore(version, get_safe_name(version))
-    ds.raw_data = load_raw_eval_results(file_path)
-    print(f"raw data: {len(ds.raw_data)}")
-
-    ds.qa_raw_df = get_leaderboard_df(ds, TaskType.qa, DEFAULT_METRIC_QA)
-    print(f"QA data loaded: {ds.qa_raw_df.shape}")
-    ds.qa_fmt_df = ds.qa_raw_df.copy()
-    qa_cols, ds.qa_types = get_default_cols(TaskType.qa, ds.slug, add_fix_cols=True)
-    # by default, drop the anonymous submissions
-    ds.qa_fmt_df = ds.qa_fmt_df[~ds.qa_fmt_df[COL_NAME_IS_ANONYMOUS]][qa_cols]
-    # reset the rank after dropping the anonymous submissions
-    ds.qa_fmt_df = reset_rank(ds.qa_fmt_df)
-    ds.qa_fmt_df.drop([COL_NAME_REVISION, COL_NAME_TIMESTAMP], axis=1, inplace=True)
-
-    ds.doc_raw_df = get_leaderboard_df(ds, TaskType.long_doc, DEFAULT_METRIC_LONG_DOC)
-    print(f"Long-Doc data loaded: {len(ds.doc_raw_df)}")
-    ds.doc_fmt_df = ds.doc_raw_df.copy()
-    doc_cols, ds.doc_types = get_default_cols(TaskType.long_doc, ds.slug, add_fix_cols=True)
-    # by default, drop the anonymous submissions
-    ds.doc_fmt_df = ds.doc_fmt_df[~ds.doc_fmt_df[COL_NAME_IS_ANONYMOUS]][doc_cols]
-    # reset the rank after dropping the anonymous submissions
-    ds.doc_fmt_df = reset_rank(ds.doc_fmt_df)
-    ds.doc_fmt_df.drop([COL_NAME_REVISION, COL_NAME_TIMESTAMP], axis=1, inplace=True)
-
-    ds.reranking_models = sorted(list(frozenset([eval_result.reranking_model for eval_result in ds.raw_data])))
-    return ds
-
-
-def load_eval_results(file_path: Union[str, Path]) -> Dict[str, LeaderboardDataStore]:
-    output = {}
-    for version in BENCHMARK_VERSION_LIST:
-        fn = f"{file_path}/{version}"
-        output[version] = load_leaderboard_datastore(fn, version)
-    return output

src/{models.py → read_evals.py}

@@ -1,21 +1,38 @@
 import json
+import os.path
 from collections import defaultdict
 from dataclasses import dataclass
-from enum import Enum
 from typing import List
 
 import pandas as pd
 
-from src.columns import (
-    COL_NAME_IS_ANONYMOUS,
+from src.benchmarks import get_safe_name
+from src.display.utils import (
     COL_NAME_RERANKING_MODEL,
-    COL_NAME_RERANKING_MODEL_LINK,
     COL_NAME_RETRIEVAL_MODEL,
+    COL_NAME_RERANKING_MODEL_LINK,
     COL_NAME_RETRIEVAL_MODEL_LINK,
     COL_NAME_REVISION,
     COL_NAME_TIMESTAMP,
+    COL_NAME_IS_ANONYMOUS,
+    COLS_QA,
+    QA_BENCHMARK_COLS,
+    COLS_LONG_DOC,
+    LONG_DOC_BENCHMARK_COLS,
+    COL_NAME_AVG,
+    COL_NAME_RANK
 )
 
+from src.display.formatting import make_clickable_model
+
+pd.options.mode.copy_on_write = True
+
+def calculate_mean(row):
+    if pd.isna(row).any():
+        return -1
+    else:
+        return row.mean()
+
 
 @dataclass
 class EvalResult:
@@ -23,7 +40,6 @@ class EvalResult:
     Evaluation result of a single embedding model with a specific reranking model on benchmarks over different
     domains, languages, and datasets
     """
-
     eval_name: str  # name of the evaluation, [retrieval_model]_[reranking_model]_[metric]
     retrieval_model: str
     reranking_model: str
@@ -40,7 +56,6 @@ class FullEvalResult:
     """
     Evaluation result of a single embedding model with a specific reranking model on benchmarks over different tasks
     """
-
     eval_name: str  # name of the evaluation, [retrieval_model]_[reranking_model]
     retrieval_model: str
     reranking_model: str
@@ -64,6 +79,7 @@ class FullEvalResult:
         result_list = []
         retrieval_model_link = ""
         reranking_model_link = ""
+        revision = ""
         for item in model_data:
             config = item.get("config", {})
             # eval results for different metrics
@@ -82,26 +98,24 @@ class FullEvalResult:
                 metric=config["metric"],
                 timestamp=config.get("timestamp", "2024-05-12T12:24:02Z"),
                 revision=config.get("revision", "3a2ba9dcad796a48a02ca1147557724e"),
-                is_anonymous=config.get("is_anonymous", False),
+                is_anonymous=config.get("is_anonymous", False)
             )
             result_list.append(eval_result)
-        eval_result = result_list[0]
         return cls(
-            eval_name=f"{eval_result.retrieval_model}_{eval_result.reranking_model}",
-            retrieval_model=eval_result.retrieval_model,
-            reranking_model=eval_result.reranking_model,
+            eval_name=f"{result_list[0].retrieval_model}_{result_list[0].reranking_model}",
+            retrieval_model=result_list[0].retrieval_model,
+            reranking_model=result_list[0].reranking_model,
             retrieval_model_link=retrieval_model_link,
             reranking_model_link=reranking_model_link,
             results=result_list,
-            timestamp=eval_result.timestamp,
-            revision=eval_result.revision,
-            is_anonymous=eval_result.is_anonymous,
+            timestamp=result_list[0].timestamp,
+            revision=result_list[0].revision,
+            is_anonymous=result_list[0].is_anonymous
         )
 
-    def to_dict(self, task="qa", metric="ndcg_at_3") -> List:
+    def to_dict(self, task='qa', metric='ndcg_at_3') -> List:
         """
-        Convert the results in all the EvalResults over different tasks and metrics.
-        The output is a list of dict compatible with the dataframe UI
+        Convert the results in all the EvalResults over different tasks and metrics. The output is a list of dict compatible with the dataframe UI
         """
         results = defaultdict(dict)
         for eval_result in self.results:
@@ -109,66 +123,106 @@ class FullEvalResult:
                 continue
             if eval_result.task != task:
                 continue
-            eval_name = eval_result.eval_name
-            results[eval_name]["eval_name"] = eval_name
-            results[eval_name][COL_NAME_RETRIEVAL_MODEL] = make_clickable_model(
-                self.retrieval_model, self.retrieval_model_link
-            )
-            results[eval_name][COL_NAME_RERANKING_MODEL] = make_clickable_model(
-                self.reranking_model, self.reranking_model_link
-            )
-            results[eval_name][COL_NAME_RETRIEVAL_MODEL_LINK] = self.retrieval_model_link
-            results[eval_name][COL_NAME_RERANKING_MODEL_LINK] = self.reranking_model_link
-            results[eval_name][COL_NAME_REVISION] = self.revision
-            results[eval_name][COL_NAME_TIMESTAMP] = self.timestamp
-            results[eval_name][COL_NAME_IS_ANONYMOUS] = self.is_anonymous
-
+            results[eval_result.eval_name]["eval_name"] = eval_result.eval_name
+            results[eval_result.eval_name][COL_NAME_RETRIEVAL_MODEL] = (
+                make_clickable_model(self.retrieval_model, self.retrieval_model_link))
+            results[eval_result.eval_name][COL_NAME_RERANKING_MODEL] = (
+                make_clickable_model(self.reranking_model, self.reranking_model_link))
+            results[eval_result.eval_name][COL_NAME_RETRIEVAL_MODEL_LINK] = self.retrieval_model_link
+            results[eval_result.eval_name][COL_NAME_RERANKING_MODEL_LINK] = self.reranking_model_link
+            results[eval_result.eval_name][COL_NAME_REVISION] = self.revision
+            results[eval_result.eval_name][COL_NAME_TIMESTAMP] = self.timestamp
+            results[eval_result.eval_name][COL_NAME_IS_ANONYMOUS] = self.is_anonymous
+
+            # print(f'result loaded: {eval_result.eval_name}')
             for result in eval_result.results:
                 # add result for each domain, language, and dataset
                 domain = result["domain"]
                 lang = result["lang"]
                 dataset = result["dataset"]
                 value = result["value"] * 100
-                if dataset == "default":
+                if dataset == 'default':
                     benchmark_name = f"{domain}_{lang}"
                 else:
                     benchmark_name = f"{domain}_{lang}_{dataset}"
-                results[eval_name][get_safe_name(benchmark_name)] = value
+                results[eval_result.eval_name][get_safe_name(benchmark_name)] = value
         return [v for v in results.values()]
 
 
-@dataclass
-class LeaderboardDataStore:
-    version: str
-    slug: str
-    raw_data: list = None
-    qa_raw_df: pd.DataFrame = pd.DataFrame()
-    doc_raw_df: pd.DataFrame = pd.DataFrame()
-    qa_fmt_df: pd.DataFrame = pd.DataFrame()
-    doc_fmt_df: pd.DataFrame = pd.DataFrame()
-    reranking_models: list = None
-    qa_types: list = None
-    doc_types: list = None
-
-
-# Define an enum class with the name `TaskType`. There are two types of tasks, `qa` and `long-doc`.
-class TaskType(Enum):
-    qa = "qa"
-    long_doc = "long-doc"
-
-
-def make_clickable_model(model_name: str, model_link: str):
-    # link = f"https://huggingface.co/{model_name}"
-    if not model_link or not model_link.startswith("https://"):
-        return model_name
-    return model_hyperlink(model_link, model_name)
-
-
-def model_hyperlink(link, model_name):
-    return f'<a target="_blank" href="{link}" style="color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;">{model_name}</a>'
-
-
-def get_safe_name(name: str):
-    """Get RFC 1123 compatible safe name"""
-    name = name.replace("-", "_")
-    return "".join(character.lower() for character in name if (character.isalnum() or character == "_"))
+def get_raw_eval_results(results_path: str) -> List[FullEvalResult]:
+    """
+    Load the evaluation results from a json file
+    """
+    model_result_filepaths = []
+    for root, dirs, files in os.walk(results_path):
+        if len(files) == 0:
+            continue
+
+        # select the latest results
+        for file in files:
+            if not (file.startswith("results") and file.endswith(".json")):
+                print(f'skip {file}')
+                continue
+            model_result_filepaths.append(os.path.join(root, file))
+
+    eval_results = {}
+    for model_result_filepath in model_result_filepaths:
+        # create evaluation results
+        try:
+            eval_result = FullEvalResult.init_from_json_file(model_result_filepath)
+        except UnicodeDecodeError as e:
+            print(f"loading file failed. {model_result_filepath}")
+            continue
+        print(f'file loaded: {model_result_filepath}')
+        timestamp = eval_result.timestamp
+        eval_results[timestamp] = eval_result
+
+    results = []
+    for k, v in eval_results.items():
+        try:
+            v.to_dict()
+            results.append(v)
+        except KeyError:
+            print(f"loading failed: {k}")
+            continue
+    return results
+
+
+def get_leaderboard_df(raw_data: List[FullEvalResult], task: str, metric: str) -> pd.DataFrame:
+    """
+    Creates a dataframe from all the individual experiment results
+    """
+    cols = [COL_NAME_IS_ANONYMOUS, ]
+    if task == "qa":
+        cols += COLS_QA
+        benchmark_cols = QA_BENCHMARK_COLS
+    elif task == "long-doc":
+        cols += COLS_LONG_DOC
+        benchmark_cols = LONG_DOC_BENCHMARK_COLS
+    else:
+        raise NotImplemented
+    all_data_json = []
+    for v in raw_data:
+        all_data_json += v.to_dict(task=task, metric=metric)
+    df = pd.DataFrame.from_records(all_data_json)
+    # print(f'dataframe created: {df.shape}')
+
+    _benchmark_cols = frozenset(benchmark_cols).intersection(frozenset(df.columns.to_list()))
+
+    # calculate the average score for selected benchmarks
+    df[COL_NAME_AVG] = df[list(_benchmark_cols)].apply(calculate_mean, axis=1).round(decimals=2)
+    df.sort_values(by=[COL_NAME_AVG], ascending=False, inplace=True)
+    df.reset_index(inplace=True, drop=True)
+
+    _cols = frozenset(cols).intersection(frozenset(df.columns.to_list()))
+    df = df[_cols].round(decimals=2)
+
+    # filter out if any of the benchmarks have not been produced
+    df[COL_NAME_RANK] = df[COL_NAME_AVG].rank(ascending=False, method="min")
+
+    # shorten the revision
+    df[COL_NAME_REVISION] = df[COL_NAME_REVISION].str[:6]
+    
+    # # replace "0" with "-" for average score
+    # df[COL_NAME_AVG] = df[COL_NAME_AVG].replace(0, "-")
+    return df

src/utils.py

@@ -1,37 +1,24 @@
-import hashlib
 import json
-import re
+import hashlib
 from datetime import datetime, timezone
 from pathlib import Path
+from typing import List
 
 import pandas as pd
 
-from src.benchmarks import LongDocBenchmarks, QABenchmarks
-from src.columns import (
-    COL_NAME_AVG,
-    COL_NAME_IS_ANONYMOUS,
-    COL_NAME_RANK,
-    COL_NAME_RERANKING_MODEL,
-    COL_NAME_RETRIEVAL_MODEL,
-    COL_NAME_REVISION,
-    COL_NAME_TIMESTAMP,
-    get_default_col_names_and_types,
-    get_fixed_col_names_and_types,
-)
-from src.envs import API, LATEST_BENCHMARK_VERSION, SEARCH_RESULTS_REPO
-from src.models import TaskType, get_safe_name
-
-
-def calculate_mean(row):
-    if pd.isna(row).any():
-        return -1
-    else:
-        return row.mean()
+from src.benchmarks import BENCHMARK_COLS_QA, BENCHMARK_COLS_LONG_DOC, BenchmarksQA, BenchmarksLongDoc
+from src.display.formatting import styled_message, styled_error
+from src.display.utils import COLS_QA, TYPES_QA, COLS_LONG_DOC, TYPES_LONG_DOC, COL_NAME_RANK, COL_NAME_AVG, \
+    COL_NAME_RERANKING_MODEL, COL_NAME_RETRIEVAL_MODEL, COL_NAME_IS_ANONYMOUS, COL_NAME_TIMESTAMP, COL_NAME_REVISION, get_default_auto_eval_column_dict
+from src.envs import API, SEARCH_RESULTS_REPO, LATEST_BENCHMARK_VERSION
+from src.read_evals import FullEvalResult, get_leaderboard_df, calculate_mean
+
+import re
 
 
 def remove_html(input_str):
     # Regular expression for finding HTML tags
-    clean = re.sub(r"<.*?>", "", input_str)
+    clean = re.sub(r'<.*?>', '', input_str)
     return clean
 
 
@@ -68,152 +55,160 @@ def search_table(df: pd.DataFrame, query: str) -> pd.DataFrame:
     return df[(df[COL_NAME_RETRIEVAL_MODEL].str.contains(query, case=False))]
 
 
-def get_default_cols(task: TaskType, version_slug, add_fix_cols: bool = True) -> tuple:
+def get_default_cols(task: str, columns: list=[], add_fix_cols: bool=True) -> list:
     cols = []
     types = []
-    if task == TaskType.qa:
-        benchmarks = QABenchmarks[version_slug]
-    elif task == TaskType.long_doc:
-        benchmarks = LongDocBenchmarks[version_slug]
+    if task == "qa":
+        cols_list = COLS_QA
+        types_list = TYPES_QA
+        benchmark_list = BENCHMARK_COLS_QA
+    elif task == "long-doc":
+        cols_list = COLS_LONG_DOC
+        types_list = TYPES_LONG_DOC
+        benchmark_list = BENCHMARK_COLS_LONG_DOC
     else:
-        raise NotImplementedError
-    cols_list, types_list = get_default_col_names_and_types(benchmarks)
-    benchmark_list = [c.value.col_name for c in list(benchmarks.value)]
+        raise NotImplemented
     for col_name, col_type in zip(cols_list, types_list):
         if col_name not in benchmark_list:
             continue
+        if len(columns) > 0 and col_name not in columns:
+            continue
         cols.append(col_name)
         types.append(col_type)
+
     if add_fix_cols:
         _cols = []
         _types = []
-        fixed_cols, fixed_cols_types = get_fixed_col_names_and_types()
         for col_name, col_type in zip(cols, types):
-            if col_name in fixed_cols:
+            if col_name in FIXED_COLS:
                 continue
             _cols.append(col_name)
             _types.append(col_type)
-        cols = fixed_cols + _cols
-        types = fixed_cols_types + _types
+        cols = FIXED_COLS + _cols
+        types = FIXED_COLS_TYPES + _types
     return cols, types
 
 
-def get_selected_cols(task, version_slug, domains, languages):
-    cols, _ = get_default_cols(task=task, version_slug=version_slug, add_fix_cols=False)
+fixed_cols = get_default_auto_eval_column_dict()[:-3]
+
+FIXED_COLS = [c.name for _, _, c in fixed_cols]
+FIXED_COLS_TYPES = [c.type for _, _, c in fixed_cols]
+
+
+def select_columns(
+        df: pd.DataFrame,
+        domain_query: list,
+        language_query: list,
+        task: str = "qa",
+        reset_ranking: bool = True
+) -> pd.DataFrame:
+    cols, _ = get_default_cols(task=task, columns=df.columns, add_fix_cols=False)
     selected_cols = []
     for c in cols:
-        if task == TaskType.qa:
-            eval_col = QABenchmarks[version_slug].value[c].value
-        elif task == TaskType.long_doc:
-            eval_col = LongDocBenchmarks[version_slug].value[c].value
-        else:
-            raise NotImplementedError
-        if eval_col.domain not in domains:
+        if task == "qa":
+            eval_col = BenchmarksQA[c].value
+        elif task == "long-doc":
+            eval_col = BenchmarksLongDoc[c].value
+        if eval_col.domain not in domain_query:
             continue
-        if eval_col.lang not in languages:
+        if eval_col.lang not in language_query:
             continue
         selected_cols.append(c)
     # We use COLS to maintain sorting
-    return selected_cols
-
-
-def select_columns(
-    df: pd.DataFrame,
-    domains: list,
-    languages: list,
-    task: TaskType = TaskType.qa,
-    reset_ranking: bool = True,
-    version_slug: str = None,
-) -> pd.DataFrame:
-    selected_cols = get_selected_cols(task, version_slug, domains, languages)
-    fixed_cols, _ = get_fixed_col_names_and_types()
-    filtered_df = df[fixed_cols + selected_cols]
-    filtered_df.replace({"": pd.NA}, inplace=True)
+    filtered_df = df[FIXED_COLS + selected_cols]
     if reset_ranking:
         filtered_df[COL_NAME_AVG] = filtered_df[selected_cols].apply(calculate_mean, axis=1).round(decimals=2)
         filtered_df.sort_values(by=[COL_NAME_AVG], ascending=False, inplace=True)
         filtered_df.reset_index(inplace=True, drop=True)
         filtered_df = reset_rank(filtered_df)
+
     return filtered_df
 
 
-def _update_df_elem(
-    task: TaskType,
-    version: str,
-    source_df: pd.DataFrame,
-    domains: list,
-    langs: list,
-    reranking_query: list,
-    query: str,
-    show_anonymous: bool,
-    reset_ranking: bool = True,
-    show_revision_and_timestamp: bool = False,
+def _update_table(
+        task: str,
+        hidden_df: pd.DataFrame,
+        domains: list,
+        langs: list,
+        reranking_query: list,
+        query: str,
+        show_anonymous: bool,
+        reset_ranking: bool = True,
+        show_revision_and_timestamp: bool = False
 ):
-    filtered_df = source_df.copy()
+    filtered_df = hidden_df.copy()
     if not show_anonymous:
         filtered_df = filtered_df[~filtered_df[COL_NAME_IS_ANONYMOUS]]
     filtered_df = filter_models(filtered_df, reranking_query)
     filtered_df = filter_queries(query, filtered_df)
-    filtered_df = select_columns(filtered_df, domains, langs, task, reset_ranking, get_safe_name(version))
+    filtered_df = select_columns(filtered_df, domains, langs, task, reset_ranking)
     if not show_revision_and_timestamp:
         filtered_df.drop([COL_NAME_REVISION, COL_NAME_TIMESTAMP], axis=1, inplace=True)
     return filtered_df
 
 
-def update_doc_df_elem(
-    version: str,
-    hidden_df: pd.DataFrame,
-    domains: list,
-    langs: list,
-    reranking_query: list,
-    query: str,
-    show_anonymous: bool,
-    show_revision_and_timestamp: bool = False,
-    reset_ranking: bool = True,
+def update_table(
+        hidden_df: pd.DataFrame,
+        domains: list,
+        langs: list,
+        reranking_query: list,
+        query: str,
+        show_anonymous: bool,
+        show_revision_and_timestamp: bool = False,
+        reset_ranking: bool = True
 ):
-    return _update_df_elem(
-        TaskType.long_doc,
-        version,
-        hidden_df,
-        domains,
-        langs,
-        reranking_query,
-        query,
-        show_anonymous,
-        reset_ranking,
-        show_revision_and_timestamp,
-    )
+    return _update_table(
+        "qa", hidden_df, domains, langs, reranking_query, query, show_anonymous, reset_ranking, show_revision_and_timestamp)
+
+
+def update_table_long_doc(
+        hidden_df: pd.DataFrame,
+        domains: list,
+        langs: list,
+        reranking_query: list,
+        query: str,
+        show_anonymous: bool,
+        show_revision_and_timestamp: bool = False,
+        reset_ranking: bool = True
+
+):
+    return _update_table(
+        "long-doc", hidden_df, domains, langs, reranking_query, query, show_anonymous, reset_ranking, show_revision_and_timestamp)
 
 
 def update_metric(
-    datastore,
-    task: TaskType,
-    metric: str,
-    domains: list,
-    langs: list,
-    reranking_model: list,
-    query: str,
-    show_anonymous: bool = False,
-    show_revision_and_timestamp: bool = False,
+        raw_data: List[FullEvalResult],
+        task: str,
+        metric: str,
+        domains: list,
+        langs: list,
+        reranking_model: list,
+        query: str,
+        show_anonymous: bool = False,
+        show_revision_and_timestamp: bool = False,
 ) -> pd.DataFrame:
-    if task == TaskType.qa:
-        update_func = update_qa_df_elem
-    elif task == TaskType.long_doc:
-        update_func = update_doc_df_elem
-    else:
-        raise NotImplementedError
-    df_elem = get_leaderboard_df(datastore, task=task, metric=metric)
-    version = datastore.version
-    return update_func(
-        version,
-        df_elem,
-        domains,
-        langs,
-        reranking_model,
-        query,
-        show_anonymous,
-        show_revision_and_timestamp,
-    )
+    if task == 'qa':
+        leaderboard_df = get_leaderboard_df(raw_data, task=task, metric=metric)
+        return update_table(
+            leaderboard_df,
+            domains,
+            langs,
+            reranking_model,
+            query,
+            show_anonymous,
+            show_revision_and_timestamp
+        )
+    elif task == "long-doc":
+        leaderboard_df = get_leaderboard_df(raw_data, task=task, metric=metric)
+        return update_table_long_doc(
+            leaderboard_df,
+            domains,
+            langs,
+            reranking_model,
+            query,
+            show_anonymous,
+            show_revision_and_timestamp
+        )
 
 
 def upload_file(filepath: str):
@@ -223,6 +218,7 @@ def upload_file(filepath: str):
     return filepath
 
 
+
 def get_iso_format_timestamp():
     # Get the current timestamp with UTC as the timezone
     current_timestamp = datetime.now(timezone.utc)
@@ -231,15 +227,15 @@ def get_iso_format_timestamp():
     current_timestamp = current_timestamp.replace(microsecond=0)
 
     # Convert to ISO 8601 format and replace the offset with 'Z'
-    iso_format_timestamp = current_timestamp.isoformat().replace("+00:00", "Z")
-    filename_friendly_timestamp = current_timestamp.strftime("%Y%m%d%H%M%S")
+    iso_format_timestamp = current_timestamp.isoformat().replace('+00:00', 'Z')
+    filename_friendly_timestamp = current_timestamp.strftime('%Y%m%d%H%M%S')
     return iso_format_timestamp, filename_friendly_timestamp
 
 
 def calculate_file_md5(file_path):
     md5 = hashlib.md5()
 
-    with open(file_path, "rb") as f:
+    with open(file_path, 'rb') as f:
         while True:
             data = f.read(4096)
             if not data:
@@ -250,14 +246,13 @@ def calculate_file_md5(file_path):
 
 
 def submit_results(
-    filepath: str,
-    model: str,
-    model_url: str,
-    reranking_model: str = "",
-    reranking_model_url: str = "",
-    version: str = LATEST_BENCHMARK_VERSION,
-    is_anonymous=False,
-):
+        filepath: str,
+        model: str,
+        model_url: str,
+        reranking_model: str="",
+        reranking_model_url: str="",
+        version: str=LATEST_BENCHMARK_VERSION,
+        is_anonymous=False):
     if not filepath.endswith(".zip"):
         return styled_error(f"file uploading aborted. wrong file type: {filepath}")
 
@@ -270,13 +265,11 @@ def submit_results(
         if not model_url.startswith("https://") and not model_url.startswith("http://"):
             # TODO: retrieve the model page and find the model name on the page
             return styled_error(
-                f"failed to submit. Model url must start with `https://` or `http://`. Illegal model url: {model_url}"
-            )
+                f"failed to submit. Model url must start with `https://` or `http://`. Illegal model url: {model_url}")
         if reranking_model != "NoReranker":
             if not reranking_model_url.startswith("https://") and not reranking_model_url.startswith("http://"):
                 return styled_error(
-                    f"failed to submit. Model url must start with `https://` or `http://`. Illegal model url: {model_url}"
-                )
+                    f"failed to submit. Model url must start with `https://` or `http://`. Illegal model url: {model_url}")
 
     # rename the uploaded file
     input_fp = Path(filepath)
@@ -286,15 +279,14 @@ def submit_results(
     input_folder_path = input_fp.parent
 
     if not reranking_model:
-        reranking_model = "NoReranker"
-
+        reranking_model = 'NoReranker'
+    
     API.upload_file(
         path_or_fileobj=filepath,
         path_in_repo=f"{version}/{model}/{reranking_model}/{output_fn}",
         repo_id=SEARCH_RESULTS_REPO,
         repo_type="dataset",
-        commit_message=f"feat: submit {model} to evaluate",
-    )
+        commit_message=f"feat: submit {model} to evaluate")
 
     output_config_fn = f"{output_fn.removesuffix('.zip')}.json"
     output_config = {
@@ -305,7 +297,7 @@ def submit_results(
         "version": f"{version}",
         "is_anonymous": is_anonymous,
         "revision": f"{revision}",
-        "timestamp": f"{timestamp_config}",
+        "timestamp": f"{timestamp_config}"
     }
     with open(input_folder_path / output_config_fn, "w") as f:
         json.dump(output_config, f, indent=4, ensure_ascii=False)
@@ -314,8 +306,7 @@ def submit_results(
         path_in_repo=f"{version}/{model}/{reranking_model}/{output_config_fn}",
         repo_id=SEARCH_RESULTS_REPO,
         repo_type="dataset",
-        commit_message=f"feat: submit {model} + {reranking_model} config",
-    )
+        commit_message=f"feat: submit {model} + {reranking_model} config")
     return styled_message(
         f"Thanks for submission!\n"
         f"Retrieval method: {model}\nReranking model: {reranking_model}\nSubmission revision: {revision}"
@@ -325,125 +316,3 @@ def submit_results(
 def reset_rank(df):
     df[COL_NAME_RANK] = df[COL_NAME_AVG].rank(ascending=False, method="min")
     return df
-
-
-def get_leaderboard_df(datastore, task: TaskType, metric: str) -> pd.DataFrame:
-    """
-    Creates a dataframe from all the individual experiment results
-    """
-    # load the selected metrics into a DataFrame from the raw json
-    all_data_json = []
-    for v in datastore.raw_data:
-        all_data_json += v.to_dict(task=task.value, metric=metric)
-    df = pd.DataFrame.from_records(all_data_json)
-
-    # calculate the average scores for selected task
-    if task == TaskType.qa:
-        benchmarks = QABenchmarks[datastore.slug]
-    elif task == TaskType.long_doc:
-        benchmarks = LongDocBenchmarks[datastore.slug]
-    else:
-        raise NotImplementedError
-    valid_cols = frozenset(df.columns.to_list())
-    benchmark_cols = []
-    for t in list(benchmarks.value):
-        if t.value.col_name not in valid_cols:
-            continue
-        benchmark_cols.append(t.value.col_name)
-
-    # filter out the columns that are not in the data
-    df[COL_NAME_AVG] = df[list(benchmark_cols)].apply(calculate_mean, axis=1).round(decimals=2)
-    df.sort_values(by=[COL_NAME_AVG], ascending=False, inplace=True)
-    df.reset_index(inplace=True, drop=True)
-
-    # filter out columns that are not in the data
-    display_cols = [COL_NAME_IS_ANONYMOUS, COL_NAME_AVG]
-    default_cols, _ = get_default_col_names_and_types(benchmarks)
-    for col in default_cols:
-        if col in valid_cols:
-            display_cols.append(col)
-    df = df[display_cols].round(decimals=2)
-
-    # rank the scores
-    df = reset_rank(df)
-
-    # shorten the revision
-    df[COL_NAME_REVISION] = df[COL_NAME_REVISION].str[:6]
-
-    return df
-
-
-def set_listeners(
-    task: TaskType,
-    target_df,
-    source_df,
-    search_bar,
-    version,
-    selected_domains,
-    selected_langs,
-    selected_rerankings,
-    show_anonymous,
-    show_revision_and_timestamp,
-):
-    if task == TaskType.qa:
-        update_table_func = update_qa_df_elem
-    elif task == TaskType.long_doc:
-        update_table_func = update_doc_df_elem
-    else:
-        raise NotImplementedError
-    selector_list = [selected_domains, selected_langs, selected_rerankings, search_bar, show_anonymous]
-    search_bar_args = [
-        source_df,
-        version,
-    ] + selector_list
-    selector_args = (
-        [version, source_df]
-        + selector_list
-        + [
-            show_revision_and_timestamp,
-        ]
-    )
-    # Set search_bar listener
-    search_bar.submit(update_table_func, search_bar_args, target_df)
-
-    # Set column-wise listener
-    for selector in selector_list:
-        selector.change(
-            update_table_func,
-            selector_args,
-            target_df,
-            queue=True,
-        )
-
-
-def update_qa_df_elem(
-    version: str,
-    hidden_df: pd.DataFrame,
-    domains: list,
-    langs: list,
-    reranking_query: list,
-    query: str,
-    show_anonymous: bool,
-    show_revision_and_timestamp: bool = False,
-    reset_ranking: bool = True,
-):
-    return _update_df_elem(
-        TaskType.qa,
-        version,
-        hidden_df,
-        domains,
-        langs,
-        reranking_query,
-        query,
-        show_anonymous,
-        reset_ranking,
-        show_revision_and_timestamp,
-    )
-
-
-def styled_error(error):
-    return f"<p style='color: red; font-size: 20px; text-align: center;'>{error}</p>"
-
-
-def styled_message(message):
-    return f"<p style='color: green; font-size: 20px; text-align: center;'>{message}</p>"

tests/src/display/test_utils.py

@@ -0,0 +1,23 @@
+import pytest
+from src.display.utils import fields, AutoEvalColumnQA, COLS_QA, COLS_LONG_DOC, COLS_LITE, TYPES_QA, TYPES_LONG_DOC, QA_BENCHMARK_COLS, LONG_DOC_BENCHMARK_COLS, get_default_auto_eval_column_dict
+
+
+def test_fields():
+    for c in fields(AutoEvalColumnQA):
+        print(c)
+
+
+def test_macro_variables():
+    print(f'COLS_QA: {COLS_QA}')
+    print(f'COLS_LONG_DOC: {COLS_LONG_DOC}')
+    print(f'COLS_LITE: {COLS_LITE}')
+    print(f'TYPES_QA: {TYPES_QA}')
+    print(f'TYPES_LONG_DOC: {TYPES_LONG_DOC}')
+    print(f'QA_BENCHMARK_COLS: {QA_BENCHMARK_COLS}')
+    print(f'LONG_DOC_BENCHMARK_COLS: {LONG_DOC_BENCHMARK_COLS}')
+
+
+def test_get_default_auto_eval_column_dict():
+    auto_eval_column_dict_list = get_default_auto_eval_column_dict()
+    assert len(auto_eval_column_dict_list) == 9
+

tests/src/test_benchmarks.py

@@ -1,33 +1,9 @@
-import pytest
+from src.benchmarks import BenchmarksQA, BenchmarksLongDoc
 
-from src.benchmarks import LongDocBenchmarks, QABenchmarks
-from src.envs import BENCHMARK_VERSION_LIST
 
-# Ref: https://github.com/AIR-Bench/AIR-Bench/blob/4b27b8a8f2047a963805fcf6fb9d74be51ec440c/docs/available_tasks.md
-# 24.05
-# | Task | dev | test |
-# | ---- | --- | ---- |
-# | Long-Doc | 4 | 11 |
-# | QA | 54 | 53 |
-#
-# 24.04
-# | Task | test |
-# | ---- | ---- |
-# | Long-Doc | 15 |
-# | QA | 13 |
+def test_qabenchmarks():
+    print(list(BenchmarksQA))
 
 
-@pytest.mark.parametrize("num_datasets_dict", [{"air_bench_2404": 13, "air_bench_2405": 53}])
-def test_qa_benchmarks(num_datasets_dict):
-    assert len(QABenchmarks) == len(BENCHMARK_VERSION_LIST)
-    for benchmark_list in list(QABenchmarks):
-        version_slug = benchmark_list.name
-        assert num_datasets_dict[version_slug] == len(benchmark_list.value)
-
-
-@pytest.mark.parametrize("num_datasets_dict", [{"air_bench_2404": 15, "air_bench_2405": 11}])
-def test_doc_benchmarks(num_datasets_dict):
-    assert len(LongDocBenchmarks) == len(BENCHMARK_VERSION_LIST)
-    for benchmark_list in list(LongDocBenchmarks):
-        version_slug = benchmark_list.name
-        assert num_datasets_dict[version_slug] == len(benchmark_list.value)
+def test_longdocbenchmarks():
+    print(list(BenchmarksLongDoc))

tests/src/test_columns.py

@@ -1,119 +0,0 @@
-import pytest
-
-from src.benchmarks import LongDocBenchmarks, QABenchmarks
-from src.columns import (
-    COL_NAME_AVG,
-    COL_NAME_RANK,
-    COL_NAME_RERANKING_MODEL,
-    COL_NAME_RETRIEVAL_MODEL,
-    COL_NAME_REVISION,
-    COL_NAME_TIMESTAMP,
-    get_default_auto_eval_column_dict,
-    get_default_col_names_and_types,
-    get_fixed_col_names_and_types,
-    make_autoevalcolumn,
-)
-
-# Ref: https://github.com/AIR-Bench/AIR-Bench/blob/4b27b8a8f2047a963805fcf6fb9d74be51ec440c/docs/available_tasks.md
-# 24.05
-# | Task | dev | test |
-# | ---- | --- | ---- |
-# | Long-Doc | 4 | 11 |
-# | QA | 54 | 53 |
-#
-# 24.04
-# | Task | test |
-# | ---- | ---- |
-# | Long-Doc | 15 |
-# | QA | 13 |
-
-
-@pytest.fixture()
-def expected_col_names():
-    return [
-        "rank",
-        "retrieval_model",
-        "reranking_model",
-        "revision",
-        "timestamp",
-        "average",
-        "retrieval_model_link",
-        "reranking_model_link",
-        "is_anonymous",
-    ]
-
-
-@pytest.fixture()
-def expected_hidden_col_names():
-    return [
-        "retrieval_model_link",
-        "reranking_model_link",
-        "is_anonymous",
-    ]
-
-
-def test_get_default_auto_eval_column_dict(expected_col_names, expected_hidden_col_names):
-    col_list = get_default_auto_eval_column_dict()
-    assert len(col_list) == 9
-    hidden_cols = []
-    for col_tuple, expected_col in zip(col_list, expected_col_names):
-        col, _, col_content = col_tuple
-        assert col == expected_col
-        if col_content.hidden:
-            hidden_cols.append(col)
-    assert hidden_cols == expected_hidden_col_names
-
-
-def test_get_fixed_col_names_and_types():
-    col_names, col_types = get_fixed_col_names_and_types()
-    assert len(col_names) == 6
-    assert len(col_types) == 6
-    expected_col_and_type = [
-        (COL_NAME_RANK, "number"),
-        (COL_NAME_RETRIEVAL_MODEL, "markdown"),
-        (COL_NAME_RERANKING_MODEL, "markdown"),
-        (COL_NAME_REVISION, "markdown"),
-        (COL_NAME_TIMESTAMP, "date"),
-        (COL_NAME_AVG, "number"),
-    ]
-    for col_name, col_type, (c_name, c_type) in zip(col_names, col_types, expected_col_and_type):
-        assert col_name == c_name
-        assert col_type == c_type
-
-
-@pytest.mark.parametrize(
-    "benchmarks, expected_benchmark_len",
-    [
-        (QABenchmarks, {"air_bench_2404": 13, "air_bench_2405": 53}),
-        (LongDocBenchmarks, {"air_bench_2404": 15, "air_bench_2405": 11}),
-    ],
-)
-def test_make_autoevalcolumn(benchmarks, expected_benchmark_len, expected_col_names):
-    expected_default_attrs = frozenset(expected_col_names)
-    for benchmark in benchmarks:
-        TestEvalColumn = make_autoevalcolumn("TestEvalColumn", benchmark)
-        attrs = []
-        for k, v in TestEvalColumn.__dict__.items():
-            if not k.startswith("__"):
-                attrs.append(k)
-        attrs = frozenset(attrs)
-        assert expected_default_attrs.issubset(attrs)
-        benchmark_attrs = attrs.difference(expected_default_attrs)
-        assert len(benchmark_attrs) == expected_benchmark_len[benchmark.name]
-
-
-@pytest.mark.parametrize(
-    "benchmarks, expected_benchmark_len",
-    [
-        (QABenchmarks, {"air_bench_2404": 13, "air_bench_2405": 53}),
-        (LongDocBenchmarks, {"air_bench_2404": 15, "air_bench_2405": 11}),
-    ],
-)
-def test_get_default_col_names_and_types(
-    benchmarks, expected_benchmark_len, expected_col_names, expected_hidden_col_names
-):
-    default_col_len = len(expected_col_names)
-    hidden_col_len = len(expected_hidden_col_names)
-    for benchmark in benchmarks:
-        col_names, col_types = get_default_col_names_and_types(benchmark)
-        assert len(col_names) == expected_benchmark_len[benchmark.name] + default_col_len - hidden_col_len

tests/src/test_envs.py

@@ -1,14 +0,0 @@
-from air_benchmark.tasks import BenchmarkTable
-
-from src.envs import BENCHMARK_VERSION_LIST, DEFAULT_METRIC_LONG_DOC, DEFAULT_METRIC_QA, METRIC_LIST
-
-
-def test_benchmark_version_list():
-    leaderboard_versions = frozenset(BENCHMARK_VERSION_LIST)
-    available_versions = frozenset([k for k in BenchmarkTable.keys()])
-    assert leaderboard_versions.issubset(available_versions)
-
-
-def test_default_metrics():
-    assert DEFAULT_METRIC_QA in METRIC_LIST
-    assert DEFAULT_METRIC_LONG_DOC in METRIC_LIST

tests/src/test_loaders.py

@@ -1,46 +0,0 @@
-from pathlib import Path
-
-import pandas as pd
-import pytest
-
-from src.loaders import load_eval_results, load_leaderboard_datastore, load_raw_eval_results
-
-cur_fp = Path(__file__)
-
-
-@pytest.mark.parametrize("version", ["AIR-Bench_24.04", "AIR-Bench_24.05"])
-def test_load_raw_eval_results(version):
-    raw_data = load_raw_eval_results(cur_fp.parents[1] / f"toydata/eval_results/{version}")
-    assert len(raw_data) == 1
-    full_eval_result = raw_data[0]
-    expected_attr = [
-        "eval_name",
-        "retrieval_model",
-        "reranking_model",
-        "retrieval_model_link",
-        "reranking_model_link",
-        "results",
-        "timestamp",
-        "revision",
-        "is_anonymous",
-    ]
-    result_attr = [k for k in full_eval_result.__dict__.keys() if k[:2] != "__" and k[-2:] != "__"]
-    assert sorted(expected_attr) == sorted(result_attr)
-
-
-@pytest.mark.parametrize("version", ["AIR-Bench_24.04", "AIR-Bench_24.05"])
-def test_load_leaderboard_datastore(version):
-    file_path = cur_fp.parents[1] / f"toydata/eval_results/{version}"
-    datastore = load_leaderboard_datastore(file_path, version)
-    for k, v in datastore.__dict__.items():
-        if k[:2] != "__" and k[-2:] != "__":
-            if isinstance(v, list):
-                assert v
-            elif isinstance(v, pd.DataFrame):
-                assert not v.empty
-
-
-def test_load_eval_results():
-    file_path = cur_fp.parents[1] / "toydata/eval_results/"
-    datastore_dict = load_eval_results(file_path)
-    assert len(datastore_dict) == 2

tests/src/test_models.py

@@ -1,89 +0,0 @@
-from pathlib import Path
-
-import pytest
-
-from src.models import EvalResult, FullEvalResult
-
-cur_fp = Path(__file__)
-
-
-# Ref: https://github.com/AIR-Bench/AIR-Bench/blob/4b27b8a8f2047a963805fcf6fb9d74be51ec440c/docs/available_tasks.md
-# 24.05
-# | Task | dev | test |
-# | ---- | --- | ---- |
-# | Long-Doc | 4 | 11 |
-# | QA | 54 | 53 |
-#
-# 24.04
-# | Task | test |
-# | ---- | ---- |
-# | Long-Doc | 15 |
-# | QA | 13 |
-NUM_QA_BENCHMARKS_24_05 = 53
-NUM_DOC_BENCHMARKS_24_05 = 11
-NUM_QA_BENCHMARKS_24_04 = 13
-NUM_DOC_BENCHMARKS_24_04 = 15
-
-
-def test_eval_result():
-    EvalResult(
-        eval_name="eval_name",
-        retrieval_model="bge-m3",
-        reranking_model="NoReranking",
-        results=[{"domain": "law", "lang": "en", "dataset": "lex_files_500K-600K", "value": 0.45723}],
-        task="qa",
-        metric="ndcg_at_3",
-        timestamp="2024-05-14T03:09:08Z",
-        revision="1e243f14bd295ccdea7a118fe847399d",
-        is_anonymous=True,
-    )
-
-
-@pytest.mark.parametrize(
-    "file_path",
-    [
-        "AIR-Bench_24.04/bge-m3/jina-reranker-v2-base-multilingual/results.json",
-        "AIR-Bench_24.05/bge-m3/NoReranker/results.json",
-    ],
-)
-def test_full_eval_result_init_from_json_file(file_path):
-    json_fp = cur_fp.parents[1] / "toydata/eval_results/" / file_path
-    full_eval_result = FullEvalResult.init_from_json_file(json_fp)
-    assert json_fp.parents[0].stem == full_eval_result.reranking_model
-    assert json_fp.parents[1].stem == full_eval_result.retrieval_model
-    assert len(full_eval_result.results) == 70
-
-
-@pytest.mark.parametrize(
-    "file_path, task, expected_num_results",
-    [
-        ("AIR-Bench_24.04/bge-m3/jina-reranker-v2-base-multilingual/results.json", "qa", NUM_QA_BENCHMARKS_24_04),
-        (
-            "AIR-Bench_24.04/bge-m3/jina-reranker-v2-base-multilingual/results.json",
-            "long-doc",
-            NUM_DOC_BENCHMARKS_24_04,
-        ),
-        ("AIR-Bench_24.05/bge-m3/NoReranker/results.json", "qa", NUM_QA_BENCHMARKS_24_05),
-        ("AIR-Bench_24.05/bge-m3/NoReranker/results.json", "long-doc", NUM_DOC_BENCHMARKS_24_05),
-    ],
-)
-def test_full_eval_result_to_dict(file_path, task, expected_num_results):
-    json_fp = cur_fp.parents[1] / "toydata/eval_results/" / file_path
-    full_eval_result = FullEvalResult.init_from_json_file(json_fp)
-    result_dict_list = full_eval_result.to_dict(task)
-    assert len(result_dict_list) == 1
-    result = result_dict_list[0]
-    attr_list = frozenset(
-        [
-            "eval_name",
-            "Retrieval Method",
-            "Reranking Model",
-            "Retrieval Model LINK",
-            "Reranking Model LINK",
-            "Revision",
-            "Submission Date",
-            "Anonymous Submission",
-        ]
-    )
-    result_cols = list(result.keys())
-    assert len(result_cols) == (expected_num_results + len(attr_list))

tests/src/test_read_evals.py

@@ -0,0 +1,68 @@
+from pathlib import Path
+
+from src.read_evals import FullEvalResult, get_raw_eval_results, get_leaderboard_df
+
+cur_fp = Path(__file__)
+
+
+def test_init_from_json_file():
+    json_fp = cur_fp.parents[2] / "toydata" / "test_data.json"
+    full_eval_result = FullEvalResult.init_from_json_file(json_fp)
+    num_different_task_domain_lang_metric_dataset_combination = 6
+    assert len(full_eval_result.results) == \
+           num_different_task_domain_lang_metric_dataset_combination
+    assert full_eval_result.retrieval_model == "bge-m3"
+    assert full_eval_result.reranking_model == "bge-reranker-v2-m3"
+
+
+def test_to_dict():
+    json_fp = cur_fp.parents[2] / "toydata" / "test_data.json"
+    full_eval_result = FullEvalResult.init_from_json_file(json_fp)
+    result_list = full_eval_result.to_dict(task='qa', metric='ndcg_at_1')
+    assert len(result_list) == 1
+    result_dict = result_list[0]
+    assert result_dict["Retrieval Model"] == "bge-m3"
+    assert result_dict["Reranking Model"] == "bge-reranker-v2-m3"
+    assert result_dict["wiki_en"] is not None
+    assert result_dict["wiki_zh"] is not None
+
+
+def test_get_raw_eval_results():
+    results_path = cur_fp.parents[2] / "toydata" / "eval_results" / "AIR-Bench_24.04"
+    results = get_raw_eval_results(results_path)
+    # only load the latest results
+    assert len(results) == 4
+    assert results[0].eval_name == "bge-base-en-v1.5_NoReranker"
+    assert len(results[0].results) == 70
+    assert results[0].eval_name == "bge-base-en-v1.5_bge-reranker-v2-m3"
+    assert len(results[1].results) == 70
+
+
+def test_get_leaderboard_df():
+    results_path = cur_fp.parents[2] / "toydata" / "eval_results" / "AIR-Bench_24.04"
+    raw_data = get_raw_eval_results(results_path)
+    df = get_leaderboard_df(raw_data, 'qa', 'ndcg_at_10')
+    assert df.shape[0] == 4
+    # the results contain only one embedding model
+    # for i in range(4):
+    #     assert df["Retrieval Model"][i] == "bge-m3"
+    # # the results contain only two reranking model
+    # assert df["Reranking Model"][0] == "bge-reranker-v2-m3"
+    # assert df["Reranking Model"][1] == "NoReranker"
+    # assert df["Average ⬆️"][0] > df["Average ⬆️"][1]
+    # assert not df[['Average ⬆️', 'wiki_en', 'wiki_zh', ]].isnull().values.any()
+
+
+def test_get_leaderboard_df_long_doc():
+    results_path = cur_fp.parents[2] / "toydata" / "test_results"
+    raw_data = get_raw_eval_results(results_path)
+    df = get_leaderboard_df(raw_data, 'long-doc', 'ndcg_at_1')
+    assert df.shape[0] == 2
+    # the results contain only one embedding model
+    for i in range(2):
+        assert df["Retrieval Model"][i] == "bge-m3"
+    # the results contains only two reranking model
+    assert df["Reranking Model"][0] == "bge-reranker-v2-m3"
+    assert df["Reranking Model"][1] == "NoReranker"
+    assert df["Average ⬆️"][0] > df["Average ⬆️"][1]
+    assert not df[['Average ⬆️', 'law_en_lex_files_500k_600k', ]].isnull().values.any()

tests/src/test_utils.py

@@ -1,237 +0,0 @@
-from pathlib import Path
-
-import pandas as pd
-import pytest
-
-from src.columns import COL_NAME_RERANKING_MODEL, COL_NAME_RETRIEVAL_MODEL
-from src.models import TaskType, model_hyperlink
-from src.utils import (
-    _update_df_elem,
-    calculate_mean,
-    filter_models,
-    filter_queries,
-    get_default_cols,
-    get_leaderboard_df,
-    get_selected_cols,
-    remove_html,
-    select_columns,
-)
-
-cur_fp = Path(__file__)
-
-NUM_QA_BENCHMARKS_24_05 = 53
-NUM_DOC_BENCHMARKS_24_05 = 11
-NUM_QA_BENCHMARKS_24_04 = 13
-NUM_DOC_BENCHMARKS_24_04 = 15
-
-
-@pytest.fixture
-def toy_df():
-    return pd.DataFrame(
-        {
-            "Retrieval Method": ["bge-m3", "bge-m3", "jina-embeddings-v2-base", "jina-embeddings-v2-base"],
-            "Reranking Model": ["bge-reranker-v2-m3", "NoReranker", "bge-reranker-v2-m3", "NoReranker"],
-            "Rank 🏆": [1, 2, 3, 4],
-            "Revision": ["123", "234", "345", "456"],
-            "Submission Date": ["", "", "", ""],
-            "Average ⬆️": [0.6, 0.4, 0.3, 0.2],
-            "wiki_en": [0.8, 0.7, 0.2, 0.1],
-            "wiki_zh": [0.4, 0.1, 0.4, 0.3],
-            "news_en": [0.8, 0.7, 0.2, 0.1],
-            "news_zh": [0.4, 0.1, 0.2, 0.3],
-            "Anonymous Submission": [False, False, False, True],
-        }
-    )
-
-
-def test_remove_html():
-    model_name = "jina-embeddings-v3"
-    html_str = model_hyperlink("https://jina.ai", model_name)
-    output_str = remove_html(html_str)
-    assert output_str == model_name
-
-
-def test_calculate_mean():
-    valid_row = [1, 3]
-    invalid_row = [2, pd.NA]
-    df = pd.DataFrame([valid_row, invalid_row], columns=["a", "b"])
-    result = list(df.apply(calculate_mean, axis=1))
-    assert result[0] == sum(valid_row) / 2
-    assert result[1] == -1
-
-
-@pytest.mark.parametrize(
-    "models, expected",
-    [
-        (["model1", "model3"], 2),
-        (["model1", "model_missing"], 1),
-        (["model1", "model2", "model3"], 3),
-        (
-            [
-                "model1",
-            ],
-            1,
-        ),
-        ([], 3),
-    ],
-)
-def test_filter_models(models, expected):
-    df = pd.DataFrame(
-        {
-            COL_NAME_RERANKING_MODEL: [
-                "model1",
-                "model2",
-                "model3",
-            ],
-            "col2": [1, 2, 3],
-        }
-    )
-    output_df = filter_models(df, models)
-    assert len(output_df) == expected
-
-
-@pytest.mark.parametrize(
-    "query, expected",
-    [
-        ("model1;model3", 2),
-        ("model1;model4", 1),
-        ("model1;model2;model3", 3),
-        ("model1", 1),
-        ("", 3),
-    ],
-)
-def test_filter_queries(query, expected):
-    df = pd.DataFrame(
-        {
-            COL_NAME_RETRIEVAL_MODEL: [
-                "model1",
-                "model2",
-                "model3",
-            ],
-            COL_NAME_RERANKING_MODEL: [
-                "model4",
-                "model5",
-                "model6",
-            ],
-        }
-    )
-    output_df = filter_queries(query, df)
-    assert len(output_df) == expected
-
-
-@pytest.mark.parametrize(
-    "task_type, slug, add_fix_cols, expected",
-    [
-        (TaskType.qa, "air_bench_2404", True, NUM_QA_BENCHMARKS_24_04),
-        (TaskType.long_doc, "air_bench_2404", True, NUM_DOC_BENCHMARKS_24_04),
-        (TaskType.qa, "air_bench_2405", False, NUM_QA_BENCHMARKS_24_05),
-        (TaskType.long_doc, "air_bench_2405", False, NUM_DOC_BENCHMARKS_24_05),
-    ],
-)
-def test_get_default_cols(task_type, slug, add_fix_cols, expected):
-    attr_cols = ["Rank 🏆", "Retrieval Method", "Reranking Model", "Revision", "Submission Date", "Average ⬆️"]
-    cols, types = get_default_cols(task_type, slug)
-    cols_set = frozenset(cols)
-    attrs_set = frozenset(attr_cols)
-    if add_fix_cols:
-        assert attrs_set.issubset(cols_set)
-    benchmark_cols = list(cols_set.difference(attrs_set))
-    assert len(benchmark_cols) == expected
-
-
-@pytest.mark.parametrize(
-    "task_type, domains, languages, expected",
-    [
-        (
-            TaskType.qa,
-            ["wiki", "news"],
-            [
-                "zh",
-            ],
-            ["wiki_zh", "news_zh"],
-        ),
-        (
-            TaskType.qa,
-            [
-                "law",
-            ],
-            ["zh", "en"],
-            ["law_en"],
-        ),
-        (
-            TaskType.long_doc,
-            ["healthcare"],
-            ["zh", "en"],
-            [
-                "healthcare_en_pubmed_100k_200k_1",
-                "healthcare_en_pubmed_100k_200k_2",
-                "healthcare_en_pubmed_100k_200k_3",
-                "healthcare_en_pubmed_40k_50k_5_merged",
-                "healthcare_en_pubmed_30k_40k_10_merged",
-            ],
-        ),
-    ],
-)
-def test_get_selected_cols(task_type, domains, languages, expected):
-    slug = "air_bench_2404"
-    cols = get_selected_cols(task_type, slug, domains, languages)
-    assert sorted(cols) == sorted(expected)
-
-
-@pytest.mark.parametrize("reset_rank", [False])
-def test_select_columns(toy_df, reset_rank):
-    expected = [
-        "Rank 🏆",
-        "Retrieval Method",
-        "Reranking Model",
-        "Revision",
-        "Submission Date",
-        "Average ⬆️",
-        "news_zh",
-    ]
-    df_result = select_columns(toy_df, ["news"], ["zh"], version_slug="air_bench_2404", reset_ranking=reset_rank)
-    assert len(df_result.columns) == len(expected)
-    if reset_rank:
-        assert df_result["Average ⬆️"].equals(df_result["news_zh"])
-    else:
-        assert df_result["Average ⬆️"].equals(toy_df["Average ⬆️"])
-
-
-@pytest.mark.parametrize(
-    "reset_rank, show_anony",
-    [
-        (False, True),
-        (True, True),
-        (True, False),
-    ],
-)
-def test__update_df_elem(toy_df, reset_rank, show_anony):
-    df = _update_df_elem(TaskType.qa, "AIR-Bench_24.04", toy_df, ["news"], ["zh"], [], "", show_anony, reset_rank)
-    if show_anony:
-        assert df.shape[0] == 4
-    else:
-        assert df.shape[0] == 3
-    if show_anony:
-        if reset_rank:
-            assert df["Average ⬆️"].equals(df["news_zh"])
-        else:
-            assert df["Average ⬆️"].equals(toy_df["Average ⬆️"])
-
-
-@pytest.mark.parametrize(
-    "version, task_type",
-    [
-        ("AIR-Bench_24.04", TaskType.qa),
-        ("AIR-Bench_24.04", TaskType.long_doc),
-        ("AIR-Bench_24.05", TaskType.qa),
-        ("AIR-Bench_24.05", TaskType.long_doc),
-    ],
-)
-def test_get_leaderboard_df(version, task_type):
-    from src.loaders import load_raw_eval_results
-    from src.models import LeaderboardDataStore, get_safe_name
-
-    raw_data = load_raw_eval_results(cur_fp.parents[1] / f"toydata/eval_results/{version}")
-    ds = LeaderboardDataStore(version, get_safe_name(version), raw_data=raw_data)
-    df = get_leaderboard_df(ds, task_type, "ndcg_at_10")
-    assert df.shape[0] == 1

tests/test_utils.py

@@ -0,0 +1,115 @@
+import pandas as pd
+import pytest
+
+from src.utils import filter_models, search_table, filter_queries, select_columns, update_table_long_doc, get_iso_format_timestamp, get_default_cols, update_table
+from src.display.utils import COL_NAME_IS_ANONYMOUS, COL_NAME_REVISION, COL_NAME_TIMESTAMP, COL_NAME_RERANKING_MODEL, COL_NAME_RETRIEVAL_MODEL, COL_NAME_RANK, COL_NAME_AVG
+
+
+@pytest.fixture
+def toy_df():
+    return pd.DataFrame(
+        {
+            "Retrieval Model": [
+                "bge-m3",
+                "bge-m3",
+                "jina-embeddings-v2-base",
+                "jina-embeddings-v2-base"
+            ],
+            "Reranking Model": [
+                "bge-reranker-v2-m3",
+                "NoReranker",
+                "bge-reranker-v2-m3",
+                "NoReranker"
+            ],
+            "Average ⬆️": [0.6, 0.4, 0.3, 0.2],
+            "wiki_en": [0.8, 0.7, 0.2, 0.1],
+            "wiki_zh": [0.4, 0.1, 0.4, 0.3],
+            "news_en": [0.8, 0.7, 0.2, 0.1],
+            "news_zh": [0.4, 0.1, 0.4, 0.3],
+        }
+    )
+
+
+@pytest.fixture
+def toy_df_long_doc():
+    return pd.DataFrame(
+        {
+            "Retrieval Model": [
+                "bge-m3",
+                "bge-m3",
+                "jina-embeddings-v2-base",
+                "jina-embeddings-v2-base"
+            ],
+            "Reranking Model": [
+                "bge-reranker-v2-m3",
+                "NoReranker",
+                "bge-reranker-v2-m3",
+                "NoReranker"
+            ],
+            "Average ⬆️": [0.6, 0.4, 0.3, 0.2],
+            "law_en_lex_files_300k_400k": [0.4, 0.1, 0.4, 0.3],
+            "law_en_lex_files_400k_500k": [0.8, 0.7, 0.2, 0.1],
+            "law_en_lex_files_500k_600k": [0.8, 0.7, 0.2, 0.1],
+            "law_en_lex_files_600k_700k": [0.4, 0.1, 0.4, 0.3],
+        }
+    )
+def test_filter_models(toy_df):
+    df_result = filter_models(toy_df, ["bge-reranker-v2-m3", ])
+    assert len(df_result) == 2
+    assert df_result.iloc[0]["Reranking Model"] == "bge-reranker-v2-m3"
+
+
+def test_search_table(toy_df):
+    df_result = search_table(toy_df, "jina")
+    assert len(df_result) == 2
+    assert df_result.iloc[0]["Retrieval Model"] == "jina-embeddings-v2-base"
+
+
+def test_filter_queries(toy_df):
+    df_result = filter_queries("jina", toy_df)
+    assert len(df_result) == 2
+    assert df_result.iloc[0]["Retrieval Model"] == "jina-embeddings-v2-base"
+
+
+def test_select_columns(toy_df):
+    df_result = select_columns(toy_df, ['news',], ['zh',])
+    assert len(df_result.columns) == 4
+    assert df_result['Average ⬆️'].equals(df_result['news_zh'])
+
+
+def test_update_table_long_doc(toy_df_long_doc):
+    df_result = update_table_long_doc(toy_df_long_doc, ['law',], ['en',], ["bge-reranker-v2-m3", ], "jina")
+    print(df_result)
+
+
+def test_get_iso_format_timestamp():
+    timestamp_config, timestamp_fn = get_iso_format_timestamp()
+    assert len(timestamp_fn) == 14
+    assert len(timestamp_config) == 20
+    assert timestamp_config[-1] == "Z"
+
+
+def test_get_default_cols():
+    cols, types = get_default_cols("qa")
+    for c, t in zip(cols, types):
+        print(f"type({c}): {t}")
+    assert len(frozenset(cols)) == len(cols)
+
+
+def test_update_table():
+    df = pd.DataFrame(
+        {
+            COL_NAME_IS_ANONYMOUS: [False, False, False],
+            COL_NAME_REVISION: ["a1", "a2", "a3"],
+            COL_NAME_TIMESTAMP: ["2024-05-12T12:24:02Z"] * 3,
+            COL_NAME_RERANKING_MODEL: ["NoReranker"] * 3,
+            COL_NAME_RETRIEVAL_MODEL: ["Foo"] * 3,
+            COL_NAME_RANK: [1, 2, 3],
+            COL_NAME_AVG: [0.1, 0.2, 0.3],  # unsorted values
+            "wiki_en": [0.1, 0.2, 0.3]
+        }
+    )
+    results = update_table(df, "wiki", "en", ["NoReranker"], "", show_anonymous=False, reset_ranking=False, show_revision_and_timestamp=False)
+    # keep the RANK as the same regardless of the unsorted averages
+    assert results[COL_NAME_RANK].to_list() == [1, 2, 3]
+

tests/toydata/test_data.json

@@ -0,0 +1,98 @@
+[
+  {
+    "config": {
+      "retrieval_model": "bge-m3",
+      "reranking_model": "bge-reranker-v2-m3",
+      "task": "long_doc",
+      "metric": "ndcg_at_1"
+    },
+    "results": [
+      {
+        "domain": "law",
+        "lang": "en",
+        "dataset": "lex_files_500K-600K",
+        "value": 0.75723
+      }
+    ]
+  },
+  {
+    "config": {
+      "retrieval_model": "bge-m3",
+      "reranking_model": "bge-reranker-v2-m3",
+      "task": "long_doc",
+      "metric": "ndcg_at_3"
+    },
+    "results": [
+      {
+        "domain": "law",
+        "lang": "en",
+        "dataset": "lex_files_500K-600K",
+        "value": 0.69909
+      }
+    ]
+  },
+  {
+    "config": {
+      "retrieval_model": "bge-m3",
+      "reranking_model": "bge-reranker-v2-m3",
+      "task": "qa",
+      "metric": "ndcg_at_1"
+    },
+    "results": [
+      {
+        "domain": "wiki",
+        "lang": "en",
+        "dataset": "unknown",
+        "value": 0.69083
+      }
+    ]
+  },
+  {
+    "config": {
+      "retrieval_model": "bge-m3",
+      "reranking_model": "bge-reranker-v2-m3",
+      "task": "qa",
+      "metric": "ndcg_at_3"
+    },
+    "results": [
+      {
+        "domain": "wiki",
+        "lang": "en",
+        "dataset": "unknown",
+        "value": 0.73359
+      }
+    ]
+  },
+  {
+    "config": {
+      "retrieval_model": "bge-m3",
+      "reranking_model": "bge-reranker-v2-m3",
+      "task": "qa",
+      "metric": "ndcg_at_1"
+    },
+    "results": [
+      {
+        "domain": "wiki",
+        "lang": "zh",
+        "dataset": "unknown",
+        "value": 0.78358
+      }
+    ]
+  },
+  {
+    "config": {
+      "retrieval_model": "bge-m3",
+      "reranking_model": "bge-reranker-v2-m3",
+      "task": "qa",
+      "metric": "ndcg_at_3"
+    },
+    "results": [
+      {
+        "domain": "wiki",
+        "lang": "zh",
+        "dataset": "unknown",
+        "value": 0.78358
+      }
+    ]
+  }
+]

tests/toydata/test_results/bge-m3/NoReranker/results_2023-11-21T18-10-08.json

@@ -0,0 +1,98 @@
+[
+  {
+    "config": {
+      "retrieval_model": "bge-m3",
+      "reranking_model": "NoReranker",
+      "task": "long_doc",
+      "metric": "ndcg_at_1"
+    },
+    "results": [
+      {
+        "domain": "law",
+        "lang": "en",
+        "dataset": "lex_files_500K-600K",
+        "value": 0.45723
+      }
+    ]
+  },
+  {
+    "config": {
+      "retrieval_model": "bge-m3",
+      "reranking_model": "NoReranker",
+      "task": "long_doc",
+      "metric": "ndcg_at_3"
+    },
+    "results": [
+      {
+        "domain": "law",
+        "lang": "en",
+        "dataset": "lex_files_500K-600K",
+        "value": 0.49909
+      }
+    ]
+  },
+  {
+    "config": {
+      "retrieval_model": "bge-m3",
+      "reranking_model": "NoReranker",
+      "task": "qa",
+      "metric": "ndcg_at_1"
+    },
+    "results": [
+      {
+        "domain": "wiki",
+        "lang": "en",
+        "dataset": "unknown",
+        "value": 0.49083
+      }
+    ]
+  },
+  {
+    "config": {
+      "retrieval_model": "bge-m3",
+      "reranking_model": "NoReranker",
+      "task": "qa",
+      "metric": "ndcg_at_3"
+    },
+    "results": [
+      {
+        "domain": "wiki",
+        "lang": "en",
+        "dataset": "unknown",
+        "value": 0.43359
+      }
+    ]
+  },
+  {
+    "config": {
+      "retrieval_model": "bge-m3",
+      "reranking_model": "NoReranker",
+      "task": "qa",
+      "metric": "ndcg_at_1"
+    },
+    "results": [
+      {
+        "domain": "wiki",
+        "lang": "zh",
+        "dataset": "unknown",
+        "value": 0.78358
+      }
+    ]
+  },
+  {
+    "config": {
+      "retrieval_model": "bge-m3",
+      "reranking_model": "NoReranker",
+      "task": "qa",
+      "metric": "ndcg_at_3"
+    },
+    "results": [
+      {
+        "domain": "wiki",
+        "lang": "zh",
+        "dataset": "unknown",
+        "value": 0.78358
+      }
+    ]
+  }
+]

tests/toydata/test_results/bge-m3/bge-reranker-v2-m3/results_2023-11-21T18-10-08.json

@@ -0,0 +1,98 @@
+[
+  {
+    "config": {
+      "retrieval_model": "bge-m3",
+      "reranking_model": "bge-reranker-v2-m3",
+      "task": "long_doc",
+      "metric": "ndcg_at_1"
+    },
+    "results": [
+      {
+        "domain": "law",
+        "lang": "en",
+        "dataset": "lex_files_500K-600K",
+        "value": 0.75723
+      }
+    ]
+  },
+  {
+    "config": {
+      "retrieval_model": "bge-m3",
+      "reranking_model": "bge-reranker-v2-m3",
+      "task": "long_doc",
+      "metric": "ndcg_at_3"
+    },
+    "results": [
+      {
+        "domain": "law",
+        "lang": "en",
+        "dataset": "lex_files_500K-600K",
+        "value": 0.69909
+      }
+    ]
+  },
+  {
+    "config": {
+      "retrieval_model": "bge-m3",
+      "reranking_model": "bge-reranker-v2-m3",
+      "task": "qa",
+      "metric": "ndcg_at_1"
+    },
+    "results": [
+      {
+        "domain": "wiki",
+        "lang": "en",
+        "dataset": "unknown",
+        "value": 0.69083
+      }
+    ]
+  },
+  {
+    "config": {
+      "retrieval_model": "bge-m3",
+      "reranking_model": "bge-reranker-v2-m3",
+      "task": "qa",
+      "metric": "ndcg_at_3"
+    },
+    "results": [
+      {
+        "domain": "wiki",
+        "lang": "en",
+        "dataset": "unknown",
+        "value": 0.73359
+      }
+    ]
+  },
+  {
+    "config": {
+      "retrieval_model": "bge-m3",
+      "reranking_model": "bge-reranker-v2-m3",
+      "task": "qa",
+      "metric": "ndcg_at_1"
+    },
+    "results": [
+      {
+        "domain": "wiki",
+        "lang": "zh",
+        "dataset": "unknown",
+        "value": 0.78358
+      }
+    ]
+  },
+  {
+    "config": {
+      "retrieval_model": "bge-m3",
+      "reranking_model": "bge-reranker-v2-m3",
+      "task": "qa",
+      "metric": "ndcg_at_3"
+    },
+    "results": [
+      {
+        "domain": "wiki",
+        "lang": "zh",
+        "dataset": "unknown",
+        "value": 0.78358
+      }
+    ]
+  }
+]