feat-add-versions-to-benchmarks (#28)

- feat: use dataclass to manage the dataframes (59fa204055748670e8c233d868c45c69efe200cb)
- feat: refactor the data loading function (ebf3ceb222b424e57c7d349208c4ed4dfbd64f51)
- refactor: refactor the data loading part (2508d96450b4e8d1fa91e597122bfa2161e3afcd)
- refactor: move the column names to a seperated file (e2d3123b00ca5941b9256763391bc8683f1312ab)
- refactor: remove the unnecessary variables (592bb629777e9935559fa6673c9df7ca9983ab3e)
- refactor: move the data model (4eb64b412adff3fdb1d0ffac94650d33ef9d0cc8)
- refactor: refactor the benchmarks (649e0fbb68f339bdb81d780ae9c9b3ca6cf582a3)
- fix: add the missing files (2d272e25de4e66b18f4b4ebb54f234be955217dc)
- refactor: refactor the column settings (a7c0332d093c93dd5f8cc1a8aa8e44fb773c996d)
- refactor: refactor the envs (ba13e25c5f99b7ed76231bf031df874bcb77ad5e)
- refactor: refactor the envs (4791ac52051c86c18ab5098a03e1ea6a5f4a165e)
- fix: fix the format (5e03e4ade3ec0f6839717e0aabdb057c1c14bbcf)
- refactor: refactor the benchmarks (3fcf957e76ce6bc9d9fad97d8872a89f959d9fc5)
- refactor: rename the benchmarks enum (270c122e6fd8edebb31b9e7d174559b624a8bd0e)
- refactor: refactor the codes (32ee53fe91f4740070e5f9af64b14afa41c2e8d9)
- feat: implement the version selector for qa (7845083916bb2581c03d08412041d098e91b895d)
- feat: implement the version selector for qa retrieval only (e050b3942277d336e94d8a95b0b20452a74b168e)
- feat: add versioning for the qa retrieval (23b35432008a2a53d8e21d0cefc83f3f63c6c73a)
- feat: add versioning for the long-doc (bf586e31834eeead8d48187698d5dddae9887135)
- fix: fix the bug in loading selected reranker (f70b03c277e928e4dad4b8c1817652cb65d3df00)
- feat: implement versioning for long-doc retrieval only (67024a07d6fddb2a3a09124711e3da171d2303a3)
- feat: implement versioning for long-doc reranker only (f7654928f561b41d757316b50beaf123b5ee1302)
- Merge branch 'feat-add-versions-to-benchmarks-1015' into pr/28 (1199e4cdd3124ee573a8f91bf99d296fdfa58bb7)
- feat: revert the comments (34e2886581faa397eebf10ba168e3306cc2295da)
- feat: add the local mode (a3fa5e451a79ab414bb653beff6bafbc1ac4d990)
- refactor: remove the legacy imports (1a22df4c26216d18a158d9b6407bea649f4ae47a)
- refactor: reformat with black (ec8e2d4eead1e298a8c128634dddd29d020e7219)
- refactor: refactor formatting (d62c98e3693b068ed87941d6863e9c397d028c70)
- refactor: rename the variables (254881e66d3fcc43126b92d489832806ff1a5a5d)
- refactor: refactor the naming (26e4c47758e8dfec0ee9b9ed6c1d63ea8e2034bf)
- refactor: refactor the naming (5e11615b89e31db79ad01ceaa4e9cda3bfb520bb)
- refactor: refactor the naming (fe361b71cf24ddb4cb6bfed863953b81a08e6630)
- refactor: refactor the naming (0401aeb236a147508130514d6dcc9b29c3d96063)
- refactor: use enum class for the task type (6f9f6495c3e1d61953deded75a1bbbffb58373fb)
- refactor: remove the dependency in env (83bdd4e68bdbfc37e6af4b024f6f2de06c20fe95)
- refactor: restructure the files (f29af77717b7502e3b3545884e369ca011bf3079)
- refactor: restructure the files (98e75e721971319bcca9fadadd950037f1f0a553)
- refactor: reformat (a3d4c8d04da8a7467f6332cb76e70ab8a0512930)
- refactor: remove the duplicate func (2bee5cb7f85f721dd5e452734ce072d4a490f2d3)
- refactor: reformatting the codes (ace0540817215b81bbdeee603225868c07528526)
- fix: update the requirements (e9203ffa05be69769b9628d956614192a0fe7710)
- fix: unify the slug naming (7a743ddec86d0cac763b734c317035de4b00e964)
- test: add unit tests for columns (729aa2a29851fa17e410c064b2a070c03f184832)
- test: add unit tests for envs (0af261c499c573041ba5fd3bc2f61296c035354e)
- test: add unit tests for benchmarks (a50e2119b1a7ffe9797e493e8c2286bbb602bef9)
- test: add unit tests for models (9fcf2679c62faa2692d439f184ed996ffbe62ead)
- test: add unit tests for models (f997dd602daf0027f39899327be2cd11da49ff80)
- feat: update makefile (6b461df314a8c375e76549887f557befd99c830b)
- test: add unit tests for utils (b6713371bfcdd6d58040d9e8f83ea62b86938bce)
- test: add unit tests for utils (30141472956371382677b3798121846c65a58ad4)
- test: add unit tests for utils (2961737bfcb542a3b137753cce536505b597fed4)
- test: add unit tests for loaders (dfb867fd0770db1784e63c18453f9ea84a21469f)
- test: minor fix (34b8881fd5f032979b64684c44e8b3c979cd3928)
- style: reformat the styles (1f175676249074a1370d86e0c4ab0b03117e5d4d)

Makefile

@@ -3,11 +3,21 @@
 
 style:
 	python -m black --line-length 119 .
+	python -m black --line-length 119 src
 	python -m isort .
+	python -m isort src
 	ruff check --fix .
+	ruff check --fix src
 
 
 quality:
 	python -m black --check --line-length 119 .
+	python -m black --check --line-length 119 src
 	python -m isort --check-only .
+	python -m isort --check-only src
 	ruff check .
+	ruff check src
+
+
+test:
+	python -m pytest tests

app.py

@@ -1,131 +1,141 @@
+import os
+
 import gradio as gr
 from apscheduler.schedulers.background import BackgroundScheduler
 from huggingface_hub import snapshot_download
 
-from src.about import (
-    INTRODUCTION_TEXT,
-    BENCHMARKS_TEXT,
-    TITLE,
-    EVALUATION_QUEUE_TEXT
-)
-from src.benchmarks import (
-    DOMAIN_COLS_QA,
-    LANG_COLS_QA,
-    DOMAIN_COLS_LONG_DOC,
-    LANG_COLS_LONG_DOC,
-    METRIC_LIST,
-    DEFAULT_METRIC_QA,
-    DEFAULT_METRIC_LONG_DOC
-)
-from src.display.css_html_js import custom_css
-from src.display.utils import (
-    COL_NAME_IS_ANONYMOUS,
-    COL_NAME_REVISION,
-    COL_NAME_TIMESTAMP,
-    COL_NAME_RERANKING_MODEL,
-    COL_NAME_RETRIEVAL_MODEL
+from src.about import BENCHMARKS_TEXT, EVALUATION_QUEUE_TEXT, INTRODUCTION_TEXT, TITLE
+from src.benchmarks import LongDocBenchmarks, QABenchmarks
+from src.columns import COL_NAME_RERANKING_MODEL, COL_NAME_RETRIEVAL_MODEL
+from src.components import (
+    get_anonymous_checkbox,
+    get_domain_dropdown,
+    get_language_dropdown,
+    get_leaderboard_table,
+    get_metric_dropdown,
+    get_noreranking_dropdown,
+    get_reranking_dropdown,
+    get_revision_and_ts_checkbox,
+    get_search_bar,
+    get_version_dropdown,
 )
+from src.css_html_js import custom_css
 from src.envs import (
     API,
+    BENCHMARK_VERSION_LIST,
+    DEFAULT_METRIC_LONG_DOC,
+    DEFAULT_METRIC_QA,
     EVAL_RESULTS_PATH,
+    LATEST_BENCHMARK_VERSION,
+    METRIC_LIST,
     REPO_ID,
     RESULTS_REPO,
     TOKEN,
-    BM25_LINK,
-    BENCHMARK_VERSION_LIST,
-    LATEST_BENCHMARK_VERSION
-)
-from src.read_evals import (
-    get_raw_eval_results,
-    get_leaderboard_df
-)
-from src.utils import (
-    update_metric,
-    upload_file,
-    get_default_cols,
-    submit_results,
-    reset_rank,
-    remove_html
-)
-from src.display.gradio_formatting import (
-    get_version_dropdown,
-    get_search_bar,
-    get_reranking_dropdown,
-    get_metric_dropdown,
-    get_domain_dropdown,
-    get_language_dropdown,
-    get_anonymous_checkbox,
-    get_revision_and_ts_checkbox,
-    get_leaderboard_table,
-    get_noreranking_dropdown
 )
-from src.display.gradio_listener import set_listeners
+from src.loaders import load_eval_results
+from src.models import TaskType, model_hyperlink
+from src.utils import remove_html, reset_rank, set_listeners, submit_results, update_metric, upload_file
+
 
 def restart_space():
     API.restart_space(repo_id=REPO_ID)
 
 
 try:
-    snapshot_download(
-        repo_id=RESULTS_REPO, local_dir=EVAL_RESULTS_PATH, repo_type="dataset", tqdm_class=None, etag_timeout=30,
-        token=TOKEN
-    )
-except Exception as e:
-    print(f'failed to download')
+    if not os.environ.get("LOCAL_MODE", False):
+        print("Running in local mode")
+        snapshot_download(
+            repo_id=RESULTS_REPO,
+            local_dir=EVAL_RESULTS_PATH,
+            repo_type="dataset",
+            tqdm_class=None,
+            etag_timeout=30,
+            token=TOKEN,
+        )
+except Exception:
+    print("failed to download")
     restart_space()
 
-raw_data = get_raw_eval_results(f"{EVAL_RESULTS_PATH}/{LATEST_BENCHMARK_VERSION}")
-
-original_df_qa = get_leaderboard_df(
-    raw_data, task='qa', metric=DEFAULT_METRIC_QA)
-original_df_long_doc = get_leaderboard_df(
-    raw_data, task='long-doc', metric=DEFAULT_METRIC_LONG_DOC)
-print(f'raw data: {len(raw_data)}')
-print(f'QA data loaded: {original_df_qa.shape}')
-print(f'Long-Doc data loaded: {len(original_df_long_doc)}')
-
-leaderboard_df_qa = original_df_qa.copy()
-# leaderboard_df_qa = leaderboard_df_qa[has_no_nan_values(df, _benchmark_cols)]
-shown_columns_qa, types_qa = get_default_cols(
-    'qa', leaderboard_df_qa.columns, add_fix_cols=True)
-leaderboard_df_qa = leaderboard_df_qa[~leaderboard_df_qa[COL_NAME_IS_ANONYMOUS]][shown_columns_qa]
-leaderboard_df_qa.drop([COL_NAME_REVISION, COL_NAME_TIMESTAMP], axis=1, inplace=True)
-
-leaderboard_df_long_doc = original_df_long_doc.copy()
-shown_columns_long_doc, types_long_doc = get_default_cols(
-    'long-doc', leaderboard_df_long_doc.columns, add_fix_cols=True)
-leaderboard_df_long_doc = leaderboard_df_long_doc[~leaderboard_df_long_doc[COL_NAME_IS_ANONYMOUS]][shown_columns_long_doc]
-leaderboard_df_long_doc.drop([COL_NAME_REVISION, COL_NAME_TIMESTAMP], axis=1, inplace=True)
-
-# select reranking model
-reranking_models = sorted(list(frozenset([eval_result.reranking_model for eval_result in raw_data])))
-
-
-def update_metric_qa(
-        metric: str,
-        domains: list,
-        langs: list,
-        reranking_model: list,
-        query: str,
-        show_anonymous: bool,
-        show_revision_and_timestamp,
+global ds_dict
+ds_dict = load_eval_results(EVAL_RESULTS_PATH)
+global datastore
+datastore = ds_dict[LATEST_BENCHMARK_VERSION]
+
+
+def update_qa_metric(
+    metric: str,
+    domains: list,
+    langs: list,
+    reranking_model: list,
+    query: str,
+    show_anonymous: bool,
+    show_revision_and_timestamp: bool,
 ):
-    return update_metric(raw_data, 'qa', metric, domains, langs, reranking_model, query, show_anonymous, show_revision_and_timestamp)
-
-def update_metric_long_doc(
-        metric: str,
-        domains: list,
-        langs: list,
-        reranking_model: list,
-        query: str,
-        show_anonymous: bool,
+    global datastore
+    return update_metric(
+        datastore,
+        TaskType.qa,
+        metric,
+        domains,
+        langs,
+        reranking_model,
+        query,
+        show_anonymous,
         show_revision_and_timestamp,
+    )
+
+
+def update_doc_metric(
+    metric: str,
+    domains: list,
+    langs: list,
+    reranking_model: list,
+    query: str,
+    show_anonymous: bool,
+    show_revision_and_timestamp,
 ):
-    return update_metric(raw_data, "long-doc", metric, domains, langs, reranking_model, query, show_anonymous, show_revision_and_timestamp)
+    global datastore
+    return update_metric(
+        datastore,
+        TaskType.long_doc,
+        metric,
+        domains,
+        langs,
+        reranking_model,
+        query,
+        show_anonymous,
+        show_revision_and_timestamp,
+    )
+
+
+def update_qa_version(version):
+    global datastore
+    global ds_dict
+    datastore = ds_dict[version]
+    domain_elem = get_domain_dropdown(QABenchmarks[datastore.slug])
+    lang_elem = get_language_dropdown(QABenchmarks[datastore.slug])
+    model_elem = get_reranking_dropdown(datastore.reranking_models)
+    df_elem = get_leaderboard_table(datastore.qa_fmt_df, datastore.qa_types)
+    hidden_df_elem = get_leaderboard_table(datastore.qa_raw_df, datastore.qa_types, visible=False)
+    return domain_elem, lang_elem, model_elem, df_elem, hidden_df_elem
+
+
+def update_doc_version(version):
+    global datastore
+    global ds_dict
+    datastore = ds_dict[version]
+    domain_elem = get_domain_dropdown(LongDocBenchmarks[datastore.slug])
+    lang_elem = get_language_dropdown(LongDocBenchmarks[datastore.slug])
+    model_elem = get_reranking_dropdown(datastore.reranking_models)
+    df_elem = get_leaderboard_table(datastore.doc_fmt_df, datastore.doc_types)
+    hidden_df_elem = get_leaderboard_table(datastore.doc_raw_df, datastore.doc_types, visible=False)
+    return domain_elem, lang_elem, model_elem, df_elem, hidden_df_elem
 
 
 demo = gr.Blocks(css=custom_css)
 
+BM25_LINK = model_hyperlink("https://github.com/castorini/pyserini", "BM25")
+
 with demo:
     gr.HTML(TITLE)
     gr.Markdown(INTRODUCTION_TEXT, elem_classes="markdown-text")
@@ -133,25 +143,24 @@ with demo:
     with gr.Tabs(elem_classes="tab-buttons") as tabs:
         with gr.TabItem("Results", elem_id="results-tab-table"):
             with gr.Row():
-                selected_version = get_version_dropdown()
+                version = get_version_dropdown()
 
             with gr.TabItem("QA", elem_id="qa-benchmark-tab-table", id=0):
                 with gr.Row():
                     with gr.Column(min_width=320):
                         # select domain
                         with gr.Row():
-                            selected_domains = get_domain_dropdown(DOMAIN_COLS_QA, DOMAIN_COLS_QA)
+                            domains = get_domain_dropdown(QABenchmarks[datastore.slug])
                         # select language
                         with gr.Row():
-                            selected_langs = get_language_dropdown(LANG_COLS_QA, LANG_COLS_QA)
-
+                            langs = get_language_dropdown(QABenchmarks[datastore.slug])
                     with gr.Column():
                         # select the metric
-                        selected_metric = get_metric_dropdown(METRIC_LIST, DEFAULT_METRIC_QA)
+                        metric = get_metric_dropdown(METRIC_LIST, DEFAULT_METRIC_QA)
                         with gr.Row():
                             show_anonymous = get_anonymous_checkbox()
                         with gr.Row():
-                            show_revision_and_timestamp = get_revision_and_ts_checkbox()
+                            show_rev_ts = get_revision_and_ts_checkbox()
                 with gr.Tabs(elem_classes="tab-buttons") as sub_tabs:
                     with gr.TabItem("Retrieval + Reranking", id=10):
                         with gr.Row():
@@ -160,273 +169,327 @@ with demo:
                                 search_bar = get_search_bar()
                             # select reranking models
                             with gr.Column():
-                                selected_rerankings = get_reranking_dropdown(reranking_models)
-                        leaderboard_table = get_leaderboard_table(leaderboard_df_qa, types_qa)
+                                models = get_reranking_dropdown(datastore.reranking_models)
+                        #  shown_table
+                        qa_df_elem_ret_rerank = get_leaderboard_table(datastore.qa_fmt_df, datastore.qa_types)
                         # Dummy leaderboard for handling the case when the user uses backspace key
-                        hidden_leaderboard_table_for_search = get_leaderboard_table(original_df_qa, types_qa, visible=False)
+                        qa_df_elem_ret_rerank_hidden = get_leaderboard_table(
+                            datastore.qa_raw_df, datastore.qa_types, visible=False
+                        )
+
+                        version.change(
+                            update_qa_version,
+                            version,
+                            [domains, langs, models, qa_df_elem_ret_rerank, qa_df_elem_ret_rerank_hidden],
+                        )
 
                         set_listeners(
-                            "qa",
-                            leaderboard_table,
-                            hidden_leaderboard_table_for_search,
+                            TaskType.qa,
+                            qa_df_elem_ret_rerank,
+                            qa_df_elem_ret_rerank_hidden,
                             search_bar,
-                            selected_domains,
-                            selected_langs,
-                            selected_rerankings,
+                            version,
+                            domains,
+                            langs,
+                            models,
                             show_anonymous,
-                            show_revision_and_timestamp,
+                            show_rev_ts,
                         )
 
                         # set metric listener
-                        selected_metric.change(
-                            update_metric_qa,
-                            [
-                                selected_metric,
-                                selected_domains,
-                                selected_langs,
-                                selected_rerankings,
-                                search_bar,
-                                show_anonymous,
-                                show_revision_and_timestamp,
-                            ],
-                            leaderboard_table,
-                            queue=True
+                        metric.change(
+                            update_qa_metric,
+                            [metric, domains, langs, models, search_bar, show_anonymous, show_rev_ts],
+                            qa_df_elem_ret_rerank,
+                            queue=True,
                         )
+
                     with gr.TabItem("Retrieval Only", id=11):
                         with gr.Row():
                             with gr.Column(scale=1):
-                                search_bar_retriever = get_search_bar()
+                                search_bar_ret = get_search_bar()
                             with gr.Column(scale=1):
-                                selected_noreranker = get_noreranking_dropdown()
-                        lb_df_retriever = leaderboard_df_qa[leaderboard_df_qa[COL_NAME_RERANKING_MODEL] == "NoReranker"]
-                        lb_df_retriever = reset_rank(lb_df_retriever)
-                        lb_table_retriever = get_leaderboard_table(lb_df_retriever, types_qa)
+                                models_ret = get_noreranking_dropdown()
+
+                        _qa_df_ret = datastore.qa_fmt_df[datastore.qa_fmt_df[COL_NAME_RERANKING_MODEL] == "NoReranker"]
+                        _qa_df_ret = reset_rank(_qa_df_ret)
+                        qa_df_elem_ret = get_leaderboard_table(_qa_df_ret, datastore.qa_types)
+
                         # Dummy leaderboard for handling the case when the user uses backspace key
-                        hidden_lb_df_retriever = original_df_qa[original_df_qa[COL_NAME_RERANKING_MODEL] == "NoReranker"]
-                        hidden_lb_df_retriever = reset_rank(hidden_lb_df_retriever)
-                        hidden_lb_table_retriever = get_leaderboard_table(hidden_lb_df_retriever, types_qa, visible=False)
+                        _qa_df_ret_hidden = datastore.qa_raw_df[
+                            datastore.qa_raw_df[COL_NAME_RERANKING_MODEL] == "NoReranker"
+                        ]
+                        _qa_df_ret_hidden = reset_rank(_qa_df_ret_hidden)
+                        qa_df_elem_ret_hidden = get_leaderboard_table(
+                            _qa_df_ret_hidden, datastore.qa_types, visible=False
+                        )
+
+                        version.change(
+                            update_qa_version,
+                            version,
+                            [
+                                domains,
+                                langs,
+                                models_ret,
+                                qa_df_elem_ret,
+                                qa_df_elem_ret_hidden,
+                            ],
+                        )
 
                         set_listeners(
-                            "qa",
-                            lb_table_retriever,
-                            hidden_lb_table_retriever,
-                            search_bar_retriever,
-                            selected_domains,
-                            selected_langs,
-                            selected_noreranker,
+                            TaskType.qa,
+                            qa_df_elem_ret,
+                            qa_df_elem_ret_hidden,
+                            search_bar_ret,
+                            version,
+                            domains,
+                            langs,
+                            models_ret,
                             show_anonymous,
-                            show_revision_and_timestamp,
+                            show_rev_ts,
                         )
 
-                        # set metric listener
-                        selected_metric.change(
-                            update_metric_qa,
+                        metric.change(
+                            update_qa_metric,
                             [
-                                selected_metric,
-                                selected_domains,
-                                selected_langs,
-                                selected_noreranker,
-                                search_bar_retriever,
+                                metric,
+                                domains,
+                                langs,
+                                models_ret,
+                                search_bar_ret,
                                 show_anonymous,
-                                show_revision_and_timestamp,
+                                show_rev_ts,
                             ],
-                            lb_table_retriever,
-                            queue=True
+                            qa_df_elem_ret,
+                            queue=True,
                         )
+
                     with gr.TabItem("Reranking Only", id=12):
-                        lb_df_reranker = leaderboard_df_qa[leaderboard_df_qa[COL_NAME_RETRIEVAL_MODEL] == BM25_LINK]
-                        lb_df_reranker = reset_rank(lb_df_reranker)
-                        reranking_models_reranker = lb_df_reranker[COL_NAME_RERANKING_MODEL].apply(remove_html).unique().tolist()
+                        _qa_df_rerank = datastore.qa_fmt_df[datastore.qa_fmt_df[COL_NAME_RETRIEVAL_MODEL] == BM25_LINK]
+                        _qa_df_rerank = reset_rank(_qa_df_rerank)
+                        qa_rerank_models = _qa_df_rerank[COL_NAME_RERANKING_MODEL].apply(remove_html).unique().tolist()
                         with gr.Row():
                             with gr.Column(scale=1):
-                                selected_rerankings_reranker = get_reranking_dropdown(reranking_models_reranker)
+                                qa_models_rerank = get_reranking_dropdown(qa_rerank_models)
                             with gr.Column(scale=1):
-                                search_bar_reranker = gr.Textbox(show_label=False, visible=False)
-                        lb_table_reranker = get_leaderboard_table(lb_df_reranker, types_qa)
-                        hidden_lb_df_reranker = original_df_qa[original_df_qa[COL_NAME_RETRIEVAL_MODEL] == BM25_LINK]
-                        hidden_lb_df_reranker = reset_rank(hidden_lb_df_reranker)
-                        hidden_lb_table_reranker = get_leaderboard_table(
-                            hidden_lb_df_reranker, types_qa, visible=False
+                                qa_search_bar_rerank = gr.Textbox(show_label=False, visible=False)
+                        qa_df_elem_rerank = get_leaderboard_table(_qa_df_rerank, datastore.qa_types)
+
+                        _qa_df_rerank_hidden = datastore.qa_raw_df[
+                            datastore.qa_raw_df[COL_NAME_RETRIEVAL_MODEL] == BM25_LINK
+                        ]
+                        _qa_df_rerank_hidden = reset_rank(_qa_df_rerank_hidden)
+                        qa_df_elem_rerank_hidden = get_leaderboard_table(
+                            _qa_df_rerank_hidden, datastore.qa_types, visible=False
+                        )
+
+                        version.change(
+                            update_qa_version,
+                            version,
+                            [domains, langs, qa_models_rerank, qa_df_elem_rerank, qa_df_elem_rerank_hidden],
                         )
 
                         set_listeners(
-                            "qa",
-                            lb_table_reranker,
-                            hidden_lb_table_reranker,
-                            search_bar_reranker,
-                            selected_domains,
-                            selected_langs,
-                            selected_rerankings_reranker,
+                            TaskType.qa,
+                            qa_df_elem_rerank,
+                            qa_df_elem_rerank_hidden,
+                            qa_search_bar_rerank,
+                            version,
+                            domains,
+                            langs,
+                            qa_models_rerank,
                             show_anonymous,
-                            show_revision_and_timestamp,
+                            show_rev_ts,
                         )
-                        # set metric listener
-                        selected_metric.change(
-                            update_metric_qa,
+
+                        metric.change(
+                            update_qa_metric,
                             [
-                                selected_metric,
-                                selected_domains,
-                                selected_langs,
-                                selected_rerankings_reranker,
-                                search_bar_reranker,
+                                metric,
+                                domains,
+                                langs,
+                                qa_models_rerank,
+                                qa_search_bar_rerank,
                                 show_anonymous,
-                                show_revision_and_timestamp,
+                                show_rev_ts,
                             ],
-                            lb_table_reranker,
-                            queue=True
+                            qa_df_elem_rerank,
+                            queue=True,
                         )
             with gr.TabItem("Long Doc", elem_id="long-doc-benchmark-tab-table", id=1):
                 with gr.Row():
                     with gr.Column(min_width=320):
                         # select domain
                         with gr.Row():
-                            selected_domains = get_domain_dropdown(DOMAIN_COLS_LONG_DOC, DOMAIN_COLS_LONG_DOC)
+                            domains = get_domain_dropdown(LongDocBenchmarks[datastore.slug])
                         # select language
                         with gr.Row():
-                            selected_langs = get_language_dropdown(
-                                LANG_COLS_LONG_DOC, LANG_COLS_LONG_DOC
-                            )
+                            langs = get_language_dropdown(LongDocBenchmarks[datastore.slug])
                     with gr.Column():
                         # select the metric
                         with gr.Row():
-                            selected_metric = get_metric_dropdown(METRIC_LIST, DEFAULT_METRIC_LONG_DOC)
+                            metric = get_metric_dropdown(METRIC_LIST, DEFAULT_METRIC_LONG_DOC)
                         with gr.Row():
                             show_anonymous = get_anonymous_checkbox()
                         with gr.Row():
-                            show_revision_and_timestamp = get_revision_and_ts_checkbox()
-                with gr.Tabs(elem_classes="tab-buttons") as sub_tabs:
+                            show_rev_ts = get_revision_and_ts_checkbox()
+                with gr.Tabs(elem_classes="tab-buttons"):
                     with gr.TabItem("Retrieval + Reranking", id=20):
                         with gr.Row():
                             with gr.Column():
                                 search_bar = get_search_bar()
-                            # select reranking model
                             with gr.Column():
-                                selected_rerankings = get_reranking_dropdown(reranking_models)
+                                models = get_reranking_dropdown(datastore.reranking_models)
 
-                        lb_table = get_leaderboard_table(
-                            leaderboard_df_long_doc, types_long_doc
-                        )
+                        doc_df_elem_ret_rerank = get_leaderboard_table(datastore.doc_fmt_df, datastore.doc_types)
 
                         # Dummy leaderboard for handling the case when the user uses backspace key
-                        hidden_lb_table_for_search = get_leaderboard_table(
-                            original_df_long_doc, types_long_doc, visible=False
+                        doc_df_elem_ret_rerank_hidden = get_leaderboard_table(
+                            datastore.doc_raw_df, datastore.doc_types, visible=False
+                        )
+
+                        version.change(
+                            update_doc_version,
+                            version,
+                            [domains, langs, models, doc_df_elem_ret_rerank, doc_df_elem_ret_rerank_hidden],
                         )
 
                         set_listeners(
-                            "long-doc",
-                            lb_table,
-                            hidden_lb_table_for_search,
+                            TaskType.long_doc,
+                            doc_df_elem_ret_rerank,
+                            doc_df_elem_ret_rerank_hidden,
                             search_bar,
-                            selected_domains,
-                            selected_langs,
-                            selected_rerankings,
+                            version,
+                            domains,
+                            langs,
+                            models,
                             show_anonymous,
-                            show_revision_and_timestamp,
+                            show_rev_ts,
                         )
 
                         # set metric listener
-                        selected_metric.change(
-                            update_metric_long_doc,
+                        metric.change(
+                            update_doc_metric,
                             [
-                                selected_metric,
-                                selected_domains,
-                                selected_langs,
-                                selected_rerankings,
+                                metric,
+                                domains,
+                                langs,
+                                models,
                                 search_bar,
                                 show_anonymous,
-                                show_revision_and_timestamp
+                                show_rev_ts,
                             ],
-                            lb_table,
-                            queue=True
+                            doc_df_elem_ret_rerank,
+                            queue=True,
                         )
                     with gr.TabItem("Retrieval Only", id=21):
                         with gr.Row():
                             with gr.Column(scale=1):
-                                search_bar_retriever = get_search_bar()
+                                search_bar_ret = get_search_bar()
                             with gr.Column(scale=1):
-                                selected_noreranker = get_noreranking_dropdown()
-                        lb_df_retriever_long_doc = leaderboard_df_long_doc[
-                            leaderboard_df_long_doc[COL_NAME_RERANKING_MODEL] == "NoReranker"
+                                models_ret = get_noreranking_dropdown()
+
+                        _doc_df_ret = datastore.doc_fmt_df[
+                            datastore.doc_fmt_df[COL_NAME_RERANKING_MODEL] == "NoReranker"
                         ]
-                        lb_df_retriever_long_doc = reset_rank(lb_df_retriever_long_doc)
-                        hidden_lb_db_retriever_long_doc = original_df_long_doc[
-                            original_df_long_doc[COL_NAME_RERANKING_MODEL] == "NoReranker"
+                        _doc_df_ret = reset_rank(_doc_df_ret)
+                        doc_df_elem_ret = get_leaderboard_table(_doc_df_ret, datastore.doc_types)
+
+                        _doc_df_ret_hidden = datastore.doc_raw_df[
+                            datastore.doc_raw_df[COL_NAME_RERANKING_MODEL] == "NoReranker"
                         ]
-                        hidden_lb_db_retriever_long_doc = reset_rank(hidden_lb_db_retriever_long_doc)
-                        lb_table_retriever_long_doc = get_leaderboard_table(
-                            lb_df_retriever_long_doc, types_long_doc)
-                        hidden_lb_table_retriever_long_doc = get_leaderboard_table(
-                            hidden_lb_db_retriever_long_doc, types_long_doc, visible=False
+                        _doc_df_ret_hidden = reset_rank(_doc_df_ret_hidden)
+                        doc_df_elem_ret_hidden = get_leaderboard_table(
+                            _doc_df_ret_hidden, datastore.doc_types, visible=False
+                        )
+
+                        version.change(
+                            update_doc_version,
+                            version,
+                            [domains, langs, models_ret, doc_df_elem_ret, doc_df_elem_ret_hidden],
                         )
 
                         set_listeners(
-                            "long-doc",
-                            lb_table_retriever_long_doc,
-                            hidden_lb_table_retriever_long_doc,
-                            search_bar_retriever,
-                            selected_domains,
-                            selected_langs,
-                            selected_noreranker,
+                            TaskType.long_doc,
+                            doc_df_elem_ret,
+                            doc_df_elem_ret_hidden,
+                            search_bar_ret,
+                            version,
+                            domains,
+                            langs,
+                            models_ret,
                             show_anonymous,
-                            show_revision_and_timestamp,
+                            show_rev_ts,
                         )
 
-                        selected_metric.change(
-                            update_metric_long_doc,
+                        metric.change(
+                            update_doc_metric,
                             [
-                                selected_metric,
-                                selected_domains,
-                                selected_langs,
-                                selected_noreranker,
-                                search_bar_retriever,
+                                metric,
+                                domains,
+                                langs,
+                                models_ret,
+                                search_bar_ret,
                                 show_anonymous,
-                                show_revision_and_timestamp,
+                                show_rev_ts,
                             ],
-                            lb_table_retriever_long_doc,
-                            queue=True
+                            doc_df_elem_ret,
+                            queue=True,
                         )
                     with gr.TabItem("Reranking Only", id=22):
-                        lb_df_reranker_ldoc = leaderboard_df_long_doc[
-                            leaderboard_df_long_doc[COL_NAME_RETRIEVAL_MODEL] == BM25_LINK
-                            ]
-                        lb_df_reranker_ldoc = reset_rank(lb_df_reranker_ldoc)
-                        reranking_models_reranker_ldoc = lb_df_reranker_ldoc[COL_NAME_RERANKING_MODEL].apply(remove_html).unique().tolist()
+                        _doc_df_rerank = datastore.doc_fmt_df[
+                            datastore.doc_fmt_df[COL_NAME_RETRIEVAL_MODEL] == BM25_LINK
+                        ]
+                        _doc_df_rerank = reset_rank(_doc_df_rerank)
+                        doc_rerank_models = (
+                            _doc_df_rerank[COL_NAME_RERANKING_MODEL].apply(remove_html).unique().tolist()
+                        )
                         with gr.Row():
                             with gr.Column(scale=1):
-                                selected_rerankings_reranker_ldoc = get_reranking_dropdown(reranking_models_reranker_ldoc)
+                                doc_models_rerank = get_reranking_dropdown(doc_rerank_models)
                             with gr.Column(scale=1):
-                                search_bar_reranker_ldoc = gr.Textbox(show_label=False, visible=False)
-                        lb_table_reranker_ldoc = get_leaderboard_table(lb_df_reranker_ldoc, types_long_doc)
-                        hidden_lb_df_reranker_ldoc = original_df_long_doc[original_df_long_doc[COL_NAME_RETRIEVAL_MODEL] == BM25_LINK]
-                        hidden_lb_df_reranker_ldoc = reset_rank(hidden_lb_df_reranker_ldoc)
-                        hidden_lb_table_reranker_ldoc = get_leaderboard_table(
-                            hidden_lb_df_reranker_ldoc, types_long_doc, visible=False
+                                doc_search_bar_rerank = gr.Textbox(show_label=False, visible=False)
+                        doc_df_elem_rerank = get_leaderboard_table(_doc_df_rerank, datastore.doc_types)
+                        _doc_df_rerank_hidden = datastore.doc_raw_df[
+                            datastore.doc_raw_df[COL_NAME_RETRIEVAL_MODEL] == BM25_LINK
+                        ]
+                        _doc_df_rerank_hidden = reset_rank(_doc_df_rerank_hidden)
+                        doc_df_elem_rerank_hidden = get_leaderboard_table(
+                            _doc_df_rerank_hidden, datastore.doc_types, visible=False
+                        )
+
+                        version.change(
+                            update_doc_version,
+                            version,
+                            [domains, langs, doc_models_rerank, doc_df_elem_rerank, doc_df_elem_rerank_hidden],
                         )
 
                         set_listeners(
-                            "long-doc",
-                            lb_table_reranker_ldoc,
-                            hidden_lb_table_reranker_ldoc,
-                            search_bar_reranker_ldoc,
-                            selected_domains,
-                            selected_langs,
-                            selected_rerankings_reranker_ldoc,
+                            TaskType.long_doc,
+                            doc_df_elem_rerank,
+                            doc_df_elem_rerank_hidden,
+                            doc_search_bar_rerank,
+                            version,
+                            domains,
+                            langs,
+                            doc_models_rerank,
                             show_anonymous,
-                            show_revision_and_timestamp,
+                            show_rev_ts,
                         )
-                        selected_metric.change(
-                            update_metric_long_doc,
+
+                        metric.change(
+                            update_doc_metric,
                             [
-                                selected_metric,
-                                selected_domains,
-                                selected_langs,
-                                selected_rerankings_reranker_ldoc,
-                                search_bar_reranker_ldoc,
+                                metric,
+                                domains,
+                                langs,
+                                doc_models_rerank,
+                                doc_search_bar_rerank,
                                 show_anonymous,
-                                show_revision_and_timestamp,
+                                show_rev_ts,
                             ],
-                            lb_table_reranker_ldoc,
-                            queue=True
+                            doc_df_elem_rerank,
+                            queue=True,
                         )
 
         with gr.TabItem("🚀Submit here!", elem_id="submit-tab-table", id=2):
@@ -443,23 +506,18 @@ with demo:
                 with gr.Row():
                     with gr.Column():
                         reranking_model_name = gr.Textbox(
-                            label="Reranking Model name",
-                            info="Optional",
-                            value="NoReranker"
+                            label="Reranking Model name", info="Optional", value="NoReranker"
                         )
                     with gr.Column():
-                        reranking_model_url = gr.Textbox(
-                            label="Reranking Model URL",
-                            info="Optional",
-                            value=""
-                        )
+                        reranking_model_url = gr.Textbox(label="Reranking Model URL", info="Optional", value="")
                 with gr.Row():
                     with gr.Column():
                         benchmark_version = gr.Dropdown(
                             BENCHMARK_VERSION_LIST,
                             value=LATEST_BENCHMARK_VERSION,
                             interactive=True,
-                            label="AIR-Bench Version")
+                            label="AIR-Bench Version",
+                        )
                 with gr.Row():
                     upload_button = gr.UploadButton("Click to upload search results", file_count="single")
                 with gr.Row():
@@ -468,7 +526,8 @@ with demo:
                     is_anonymous = gr.Checkbox(
                         label="Nope. I want to submit anonymously 🥷",
                         value=False,
-                        info="Do you want to shown on the leaderboard by default?")
+                        info="Do you want to shown on the leaderboard by default?",
+                    )
                 with gr.Row():
                     submit_button = gr.Button("Submit")
                 with gr.Row():
@@ -478,7 +537,8 @@ with demo:
                     [
                         upload_button,
                     ],
-                    file_output)
+                    file_output,
+                )
                 submit_button.click(
                     submit_results,
                     [
@@ -488,10 +548,10 @@ with demo:
                         reranking_model_name,
                         reranking_model_url,
                         benchmark_version,
-                        is_anonymous
+                        is_anonymous,
                     ],
                     submission_result,
-                    show_progress="hidden"
+                    show_progress="hidden",
                 )
 
         with gr.TabItem("📝 About", elem_id="llm-benchmark-tab-table", id=3):

pyproject.toml

@@ -1,9 +1,9 @@
 [tool.ruff]
 # Enable pycodestyle (`E`) and Pyflakes (`F`) codes by default.
-select = ["E", "F"]
-ignore = ["E501"] # line too long (black is taking care of this)
+lint.select = ["E", "F"]
+lint.ignore = ["E501"] # line too long (black is taking care of this)
 line-length = 119
-fixable = ["A", "B", "C", "D", "E", "F", "G", "I", "N", "Q", "S", "T", "W", "ANN", "ARG", "BLE", "COM", "DJ", "DTZ", "EM", "ERA", "EXE", "FBT", "ICN", "INP", "ISC", "NPY", "PD", "PGH", "PIE", "PL", "PT", "PTH", "PYI", "RET", "RSE", "RUF", "SIM", "SLF", "TCH", "TID", "TRY", "UP", "YTT"]
+lint.fixable = ["A", "B", "C", "D", "E", "F", "G", "I", "N", "Q", "S", "T", "W", "ANN", "ARG", "BLE", "COM", "DJ", "DTZ", "EM", "ERA", "EXE", "FBT", "ICN", "INP", "ISC", "NPY", "PD", "PGH", "PIE", "PL", "PT", "PTH", "PYI", "RET", "RSE", "RUF", "SIM", "SLF", "TCH", "TID", "TRY", "UP", "YTT"]
 
 [tool.isort]
 profile = "black"

requirements.txt

@@ -2,7 +2,7 @@ APScheduler>=3.10.1
 black>=23.11.0
 click>=8.1.3
 datasets>=2.14.5
-gradio>=4.29.0
+gradio<5.0.0
 gradio_client>=0.16.1
 huggingface-hub>=0.18.0
 numpy>=1.24.2
@@ -12,4 +12,4 @@ requests>=2.31.0
 tqdm>=4.65.0
 accelerate>=0.24.1
 socksio>=1.0.0
-air-benchmark>=0.0.4
+air-benchmark>=0.1.0

src/about.py

@@ -8,7 +8,7 @@ INTRODUCTION_TEXT = """
 """
 
 # Which evaluations are you running? how can people reproduce what you have?
-BENCHMARKS_TEXT = f"""
+BENCHMARKS_TEXT = """
 ## How the test data are generated?
 ### Find more information at [our GitHub repo](https://github.com/AIR-Bench/AIR-Bench/blob/main/docs/data_generation.md)
 

src/benchmarks.py

@@ -1,92 +1,71 @@
 from dataclasses import dataclass
 from enum import Enum
-from air_benchmark.tasks.tasks import BenchmarkTable
-
-
-def get_safe_name(name: str):
-    """Get RFC 1123 compatible safe name"""
-    name = name.replace('-', '_')
-    return ''.join(
-        character.lower()
-        for character in name
-        if (character.isalnum() or character == '_'))
 
+from air_benchmark.tasks.tasks import BenchmarkTable
 
-METRIC_LIST = [
-    "ndcg_at_1",
-    "ndcg_at_3",
-    "ndcg_at_5",
-    "ndcg_at_10",
-    "ndcg_at_100",
-    "ndcg_at_1000",
-    "map_at_1",
-    "map_at_3",
-    "map_at_5",
-    "map_at_10",
-    "map_at_100",
-    "map_at_1000",
-    "recall_at_1",
-    "recall_at_3",
-    "recall_at_5",
-    "recall_at_10",
-    "recall_at_100",
-    "recall_at_1000",
-    "precision_at_1",
-    "precision_at_3",
-    "precision_at_5",
-    "precision_at_10",
-    "precision_at_100",
-    "precision_at_1000",
-    "mrr_at_1",
-    "mrr_at_3",
-    "mrr_at_5",
-    "mrr_at_10",
-    "mrr_at_100",
-    "mrr_at_1000"
-]
+from src.envs import BENCHMARK_VERSION_LIST, METRIC_LIST
+from src.models import TaskType, get_safe_name
 
 
 @dataclass
 class Benchmark:
     name: str  # [domain]_[language]_[metric], task_key in the json file,
-    metric: str  # ndcg_at_1 ,metric_key in the json file
+    metric: str  # metric_key in the json file
     col_name: str  # [domain]_[language], name to display in the leaderboard
     domain: str
     lang: str
     task: str
 
 
-qa_benchmark_dict = {}
-long_doc_benchmark_dict = {}
-for task, domain_dict in BenchmarkTable['AIR-Bench_24.04'].items():
-    for domain, lang_dict in domain_dict.items():
-        for lang, dataset_list in lang_dict.items():
-            if task == "qa":
-                benchmark_name = f"{domain}_{lang}"
-                benchmark_name = get_safe_name(benchmark_name)
+# create a function return an enum class containing all the benchmarks
+def get_qa_benchmarks_dict(version: str):
+    benchmark_dict = {}
+    for task, domain_dict in BenchmarkTable[version].items():
+        if task != TaskType.qa.value:
+            continue
+        for domain, lang_dict in domain_dict.items():
+            for lang, dataset_list in lang_dict.items():
+                benchmark_name = get_safe_name(f"{domain}_{lang}")
                 col_name = benchmark_name
                 for metric in dataset_list:
-                    qa_benchmark_dict[benchmark_name] = Benchmark(benchmark_name, metric, col_name, domain, lang, task)
-            elif task == "long-doc":
+                    if "test" not in dataset_list[metric]["splits"]:
+                        continue
+                    benchmark_dict[benchmark_name] = Benchmark(benchmark_name, metric, col_name, domain, lang, task)
+    return benchmark_dict
+
+
+def get_doc_benchmarks_dict(version: str):
+    benchmark_dict = {}
+    for task, domain_dict in BenchmarkTable[version].items():
+        if task != TaskType.long_doc.value:
+            continue
+        for domain, lang_dict in domain_dict.items():
+            for lang, dataset_list in lang_dict.items():
                 for dataset in dataset_list:
                     benchmark_name = f"{domain}_{lang}_{dataset}"
                     benchmark_name = get_safe_name(benchmark_name)
                     col_name = benchmark_name
+                    if "test" not in dataset_list[dataset]["splits"]:
+                        continue
                     for metric in METRIC_LIST:
-                        long_doc_benchmark_dict[benchmark_name] = Benchmark(benchmark_name, metric, col_name, domain,
-                                                                            lang, task)
+                        benchmark_dict[benchmark_name] = Benchmark(
+                            benchmark_name, metric, col_name, domain, lang, task
+                        )
+    return benchmark_dict
 
-BenchmarksQA = Enum('BenchmarksQA', qa_benchmark_dict)
-BenchmarksLongDoc = Enum('BenchmarksLongDoc', long_doc_benchmark_dict)
 
-BENCHMARK_COLS_QA = [c.col_name for c in qa_benchmark_dict.values()]
-BENCHMARK_COLS_LONG_DOC = [c.col_name for c in long_doc_benchmark_dict.values()]
+_qa_benchmark_dict = {}
+for version in BENCHMARK_VERSION_LIST:
+    safe_version_name = get_safe_name(version)
+    _qa_benchmark_dict[safe_version_name] = Enum(f"QABenchmarks_{safe_version_name}", get_qa_benchmarks_dict(version))
 
-DOMAIN_COLS_QA = list(frozenset([c.domain for c in qa_benchmark_dict.values()]))
-LANG_COLS_QA = list(frozenset([c.lang for c in qa_benchmark_dict.values()]))
+_doc_benchmark_dict = {}
+for version in BENCHMARK_VERSION_LIST:
+    safe_version_name = get_safe_name(version)
+    _doc_benchmark_dict[safe_version_name] = Enum(
+        f"LongDocBenchmarks_{safe_version_name}", get_doc_benchmarks_dict(version)
+    )
 
-DOMAIN_COLS_LONG_DOC = list(frozenset([c.domain for c in long_doc_benchmark_dict.values()]))
-LANG_COLS_LONG_DOC = list(frozenset([c.lang for c in long_doc_benchmark_dict.values()]))
 
-DEFAULT_METRIC_QA = "ndcg_at_10"
-DEFAULT_METRIC_LONG_DOC = "recall_at_10"
+QABenchmarks = Enum("QABenchmarks", _qa_benchmark_dict)
+LongDocBenchmarks = Enum("LongDocBenchmarks", _doc_benchmark_dict)

src/{display/utils.py → columns.py}

@@ -1,9 +1,7 @@
 from dataclasses import dataclass, make_dataclass
 
-from src.benchmarks import BenchmarksQA, BenchmarksLongDoc
 
-
-def fields(raw_class):
+def _fields(raw_class):
     return [v for k, v in raw_class.__dict__.items() if k[:2] != "__" and k[-2:] != "__"]
 
 
@@ -19,28 +17,22 @@ class ColumnContent:
     never_hidden: bool = False
 
 
-COL_NAME_AVG = "Average ⬆️"
-COL_NAME_RETRIEVAL_MODEL = "Retrieval Method"
-COL_NAME_RERANKING_MODEL = "Reranking Model"
-COL_NAME_RETRIEVAL_MODEL_LINK = "Retrieval Model LINK"
-COL_NAME_RERANKING_MODEL_LINK = "Reranking Model LINK"
-COL_NAME_RANK = "Rank 🏆"
-COL_NAME_REVISION = "Revision"
-COL_NAME_TIMESTAMP = "Submission Date"
-COL_NAME_IS_ANONYMOUS = "Anonymous Submission"
-
-
 def get_default_auto_eval_column_dict():
     auto_eval_column_dict = []
-    # Init
+    auto_eval_column_dict.append(["rank", ColumnContent, ColumnContent(COL_NAME_RANK, "number", True)])
     auto_eval_column_dict.append(
-        ["rank", ColumnContent, ColumnContent(COL_NAME_RANK, "number", True)]
+        [
+            "retrieval_model",
+            ColumnContent,
+            ColumnContent(COL_NAME_RETRIEVAL_MODEL, "markdown", True, never_hidden=True),
+        ]
     )
     auto_eval_column_dict.append(
-        ["retrieval_model", ColumnContent, ColumnContent(COL_NAME_RETRIEVAL_MODEL, "markdown", True, hidden=False, never_hidden=True)]
-    )
-    auto_eval_column_dict.append(
-        ["reranking_model", ColumnContent, ColumnContent(COL_NAME_RERANKING_MODEL, "markdown", True, hidden=False, never_hidden=True)]
+        [
+            "reranking_model",
+            ColumnContent,
+            ColumnContent(COL_NAME_RERANKING_MODEL, "markdown", True, never_hidden=True),
+        ]
     )
     auto_eval_column_dict.append(
         ["revision", ColumnContent, ColumnContent(COL_NAME_REVISION, "markdown", True, never_hidden=True)]
@@ -48,14 +40,30 @@ def get_default_auto_eval_column_dict():
     auto_eval_column_dict.append(
         ["timestamp", ColumnContent, ColumnContent(COL_NAME_TIMESTAMP, "date", True, never_hidden=True)]
     )
+    auto_eval_column_dict.append(["average", ColumnContent, ColumnContent(COL_NAME_AVG, "number", True)])
     auto_eval_column_dict.append(
-        ["average", ColumnContent, ColumnContent(COL_NAME_AVG, "number", True)]
-    )
-    auto_eval_column_dict.append(
-        ["retrieval_model_link", ColumnContent, ColumnContent(COL_NAME_RETRIEVAL_MODEL_LINK, "markdown", False, hidden=True, never_hidden=False)]
+        [
+            "retrieval_model_link",
+            ColumnContent,
+            ColumnContent(
+                COL_NAME_RETRIEVAL_MODEL_LINK,
+                "markdown",
+                False,
+                hidden=True,
+            ),
+        ]
     )
     auto_eval_column_dict.append(
-        ["reranking_model_link", ColumnContent, ColumnContent(COL_NAME_RERANKING_MODEL_LINK, "markdown", False, hidden=True, never_hidden=False)]
+        [
+            "reranking_model_link",
+            ColumnContent,
+            ColumnContent(
+                COL_NAME_RERANKING_MODEL_LINK,
+                "markdown",
+                False,
+                hidden=True,
+            ),
+        ]
     )
     auto_eval_column_dict.append(
         ["is_anonymous", ColumnContent, ColumnContent(COL_NAME_IS_ANONYMOUS, "bool", False, hidden=True)]
@@ -63,10 +71,10 @@ def get_default_auto_eval_column_dict():
     return auto_eval_column_dict
 
 
-def make_autoevalcolumn(cls_name="BenchmarksQA", benchmarks=BenchmarksQA):
+def make_autoevalcolumn(cls_name, benchmarks):
     auto_eval_column_dict = get_default_auto_eval_column_dict()
-    ## Leaderboard columns
-    for benchmark in benchmarks:
+    # Leaderboard columns
+    for benchmark in list(benchmarks.value):
         auto_eval_column_dict.append(
             [benchmark.name, ColumnContent, ColumnContent(benchmark.value.col_name, "number", True)]
         )
@@ -75,19 +83,24 @@ def make_autoevalcolumn(cls_name="BenchmarksQA", benchmarks=BenchmarksQA):
     return make_dataclass(cls_name, auto_eval_column_dict, frozen=True)
 
 
-AutoEvalColumnQA = make_autoevalcolumn(
-    "AutoEvalColumnQA", BenchmarksQA)
-AutoEvalColumnLongDoc = make_autoevalcolumn(
-    "AutoEvalColumnLongDoc", BenchmarksLongDoc)
+def get_default_col_names_and_types(benchmarks):
+    AutoEvalColumn = make_autoevalcolumn("AutoEvalColumn", benchmarks)
+    col_names = [c.name for c in _fields(AutoEvalColumn) if not c.hidden]
+    col_types = [c.type for c in _fields(AutoEvalColumn) if not c.hidden]
+    return col_names, col_types
 
 
-# Column selection
-COLS_QA = [c.name for c in fields(AutoEvalColumnQA) if not c.hidden]
-COLS_LONG_DOC = [c.name for c in fields(AutoEvalColumnLongDoc) if not c.hidden]
-TYPES_QA = [c.type for c in fields(AutoEvalColumnQA) if not c.hidden]
-TYPES_LONG_DOC = [c.type for c in fields(AutoEvalColumnLongDoc) if not c.hidden]
-COLS_LITE = [c.name for c in fields(AutoEvalColumnQA) if c.displayed_by_default and not c.hidden]
+def get_fixed_col_names_and_types():
+    fixed_cols = get_default_auto_eval_column_dict()[:-3]
+    return [c.name for _, _, c in fixed_cols], [c.type for _, _, c in fixed_cols]
 
-QA_BENCHMARK_COLS = [t.value.col_name for t in BenchmarksQA]
 
-LONG_DOC_BENCHMARK_COLS = [t.value.col_name for t in BenchmarksLongDoc]
+COL_NAME_AVG = "Average ⬆️"
+COL_NAME_RETRIEVAL_MODEL = "Retrieval Method"
+COL_NAME_RERANKING_MODEL = "Reranking Model"
+COL_NAME_RETRIEVAL_MODEL_LINK = "Retrieval Model LINK"
+COL_NAME_RERANKING_MODEL_LINK = "Reranking Model LINK"
+COL_NAME_RANK = "Rank 🏆"
+COL_NAME_REVISION = "Revision"
+COL_NAME_TIMESTAMP = "Submission Date"
+COL_NAME_IS_ANONYMOUS = "Anonymous Submission"

src/{display/gradio_formatting.py → components.py}

@@ -1,12 +1,14 @@
 import gradio as gr
+
 from src.envs import BENCHMARK_VERSION_LIST, LATEST_BENCHMARK_VERSION
 
+
 def get_version_dropdown():
     return gr.Dropdown(
         choices=BENCHMARK_VERSION_LIST,
         value=LATEST_BENCHMARK_VERSION,
         label="Select the version of AIR-Bench",
-        interactive=True
+        interactive=True,
     )
 
 
@@ -14,26 +16,25 @@ def get_search_bar():
     return gr.Textbox(
         placeholder=" 🔍 Search for retrieval methods (separate multiple queries with `;`) and press ENTER...",
         show_label=False,
-        info="Search the retrieval methods"
+        info="Search the retrieval methods",
     )
 
 
 def get_reranking_dropdown(model_list):
-    return gr.Dropdown(
-        choices=model_list,
-        label="Select the reranking models",
-        interactive=True,
-        multiselect=True
-    )
+    return gr.Dropdown(choices=model_list, label="Select the reranking models", interactive=True, multiselect=True)
 
 
 def get_noreranking_dropdown():
     return gr.Dropdown(
-        choices=["NoReranker", ],
-        value=["NoReranker", ],
+        choices=[
+            "NoReranker",
+        ],
+        value=[
+            "NoReranker",
+        ],
         interactive=False,
         multiselect=True,
-        visible=False
+        visible=False,
     )
 
 
@@ -52,7 +53,10 @@ def get_metric_dropdown(metric_list, default_metrics):
     )
 
 
-def get_domain_dropdown(domain_list, default_domains):
+def get_domain_dropdown(benchmarks, default_domains=None):
+    domain_list = list(frozenset([c.value.domain for c in list(benchmarks.value)]))
+    if default_domains is None:
+        default_domains = domain_list
     return gr.CheckboxGroup(
         choices=domain_list,
         value=default_domains,
@@ -61,13 +65,16 @@ def get_domain_dropdown(domain_list, default_domains):
     )
 
 
-def get_language_dropdown(language_list, default_languages):
+def get_language_dropdown(benchmarks, default_languages=None):
+    language_list = list(frozenset([c.value.lang for c in list(benchmarks.value)]))
+    if default_languages is None:
+        default_languages = language_list
     return gr.Dropdown(
         choices=language_list,
-        value=language_list,
+        value=default_languages,
         label="Select the languages",
         multiselect=True,
-        interactive=True
+        interactive=True,
     )
 
 
@@ -75,15 +82,13 @@ def get_anonymous_checkbox():
     return gr.Checkbox(
         label="Show anonymous submissions",
         value=False,
-        info="The anonymous submissions might have invalid model information."
+        info="The anonymous submissions might have invalid model information.",
     )
 
 
 def get_revision_and_ts_checkbox():
     return gr.Checkbox(
-        label="Show submission details",
-        value=False,
-        info="Show the revision and timestamp information of submissions"
+        label="Show submission details", value=False, info="Show the revision and timestamp information of submissions"
     )
 
 

src/display/formatting.py

@@ -1,29 +0,0 @@
-def model_hyperlink(link, model_name):
-    return f'<a target="_blank" href="{link}" style="color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;">{model_name}</a>'
-
-
-def make_clickable_model(model_name: str, model_link: str):
-    # link = f"https://huggingface.co/{model_name}"
-    if not model_link or not model_link.startswith("https://"):
-        return model_name
-    return model_hyperlink(model_link, model_name)
-
-
-def styled_error(error):
-    return f"<p style='color: red; font-size: 20px; text-align: center;'>{error}</p>"
-
-
-def styled_warning(warn):
-    return f"<p style='color: orange; font-size: 20px; text-align: center;'>{warn}</p>"
-
-
-def styled_message(message):
-    return f"<p style='color: green; font-size: 20px; text-align: center;'>{message}</p>"
-
-
-def has_no_nan_values(df, columns):
-    return df[columns].notna().all(axis=1)
-
-
-def has_nan_values(df, columns):
-    return df[columns].isna().any(axis=1)

src/display/gradio_listener.py

@@ -1,53 +0,0 @@
-from src.utils import update_table, update_table_long_doc
-
-
-def set_listeners(
-        task,
-        displayed_leaderboard,
-        hidden_leaderboard,
-        search_bar,
-        selected_domains,
-        selected_langs,
-        selected_rerankings,
-        show_anonymous,
-        show_revision_and_timestamp,
-
-):
-    if task == "qa":
-        update_table_func = update_table
-    elif task == "long-doc":
-        update_table_func = update_table_long_doc
-    else:
-        raise NotImplementedError
-    # Set search_bar listener
-    search_bar.submit(
-        update_table_func,
-        [
-            hidden_leaderboard,  #  hidden_leaderboard_table_for_search,
-            selected_domains,
-            selected_langs,
-            selected_rerankings,
-            search_bar,
-            show_anonymous,
-        ],
-        displayed_leaderboard
-    )
-
-    # Set column-wise listener
-    for selector in [
-        selected_domains, selected_langs, show_anonymous, show_revision_and_timestamp, selected_rerankings
-    ]:
-        selector.change(
-            update_table_func,
-            [
-                hidden_leaderboard,
-                selected_domains,
-                selected_langs,
-                selected_rerankings,
-                search_bar,
-                show_anonymous,
-                show_revision_and_timestamp
-            ],
-            displayed_leaderboard,
-            queue=True,
-        )

src/envs.py

@@ -1,12 +1,14 @@
 import os
-from src.display.formatting import model_hyperlink
+
 from huggingface_hub import HfApi
 
 # Info to change for your repository
 # ----------------------------------
 TOKEN = os.environ.get("TOKEN", "")  # A read/write token for your org
 
-OWNER = "AIR-Bench"  # "nan"  # Change to your org - don't forget to create a results and request dataset, with the correct format!
+OWNER = (
+    "AIR-Bench"  # Change to your org - don't forget to create a results and request dataset, with the correct format!
+)
 # ----------------------------------
 
 REPO_ID = f"{OWNER}/leaderboard"
@@ -15,7 +17,7 @@ RESULTS_REPO = f"{OWNER}/eval_results"
 # repo for submitting the evaluation
 SEARCH_RESULTS_REPO = f"{OWNER}/search_results"
 
-# If you setup a cache later, just change HF_HOME
+# If you set up a cache later, just change HF_HOME
 CACHE_PATH = os.getenv("HF_HOME", ".")
 
 # Local caches
@@ -23,11 +25,43 @@ EVAL_RESULTS_PATH = os.path.join(CACHE_PATH, "eval_results")
 
 API = HfApi(token=TOKEN)
 
-BM25_LINK = model_hyperlink("https://github.com/castorini/pyserini", "BM25")
-
 BENCHMARK_VERSION_LIST = [
     "AIR-Bench_24.04",
-    # "AIR-Bench_24.05",
+    "AIR-Bench_24.05",
 ]
 
-LATEST_BENCHMARK_VERSION = BENCHMARK_VERSION_LIST[-1]
+LATEST_BENCHMARK_VERSION = BENCHMARK_VERSION_LIST[0]
+DEFAULT_METRIC_QA = "ndcg_at_10"
+DEFAULT_METRIC_LONG_DOC = "recall_at_10"
+METRIC_LIST = [
+    "ndcg_at_1",
+    "ndcg_at_3",
+    "ndcg_at_5",
+    "ndcg_at_10",
+    "ndcg_at_100",
+    "ndcg_at_1000",
+    "map_at_1",
+    "map_at_3",
+    "map_at_5",
+    "map_at_10",
+    "map_at_100",
+    "map_at_1000",
+    "recall_at_1",
+    "recall_at_3",
+    "recall_at_5",
+    "recall_at_10",
+    "recall_at_100",
+    "recall_at_1000",
+    "precision_at_1",
+    "precision_at_3",
+    "precision_at_5",
+    "precision_at_10",
+    "precision_at_100",
+    "precision_at_1000",
+    "mrr_at_1",
+    "mrr_at_3",
+    "mrr_at_5",
+    "mrr_at_10",
+    "mrr_at_100",
+    "mrr_at_1000",
+]

src/loaders.py

@@ -0,0 +1,88 @@
+import os.path
+from pathlib import Path
+from typing import Dict, List, Union
+
+import pandas as pd
+
+from src.columns import COL_NAME_IS_ANONYMOUS, COL_NAME_REVISION, COL_NAME_TIMESTAMP
+from src.envs import BENCHMARK_VERSION_LIST, DEFAULT_METRIC_LONG_DOC, DEFAULT_METRIC_QA
+from src.models import FullEvalResult, LeaderboardDataStore, TaskType, get_safe_name
+from src.utils import get_default_cols, get_leaderboard_df, reset_rank
+
+pd.options.mode.copy_on_write = True
+
+
+def load_raw_eval_results(results_path: Union[Path, str]) -> List[FullEvalResult]:
+    """
+    Load the evaluation results from a json file
+    """
+    model_result_filepaths = []
+    for root, dirs, files in os.walk(results_path):
+        if len(files) == 0:
+            continue
+
+        # select the latest results
+        for file in files:
+            if not (file.startswith("results") and file.endswith(".json")):
+                print(f"skip {file}")
+                continue
+            model_result_filepaths.append(os.path.join(root, file))
+
+    eval_results = {}
+    for model_result_filepath in model_result_filepaths:
+        # create evaluation results
+        try:
+            eval_result = FullEvalResult.init_from_json_file(model_result_filepath)
+        except UnicodeDecodeError:
+            print(f"loading file failed. {model_result_filepath}")
+            continue
+        print(f"file loaded: {model_result_filepath}")
+        timestamp = eval_result.timestamp
+        eval_results[timestamp] = eval_result
+
+    results = []
+    for k, v in eval_results.items():
+        try:
+            v.to_dict()
+            results.append(v)
+        except KeyError:
+            print(f"loading failed: {k}")
+            continue
+    return results
+
+
+def load_leaderboard_datastore(file_path, version) -> LeaderboardDataStore:
+    ds = LeaderboardDataStore(version, get_safe_name(version))
+    ds.raw_data = load_raw_eval_results(file_path)
+    print(f"raw data: {len(ds.raw_data)}")
+
+    ds.qa_raw_df = get_leaderboard_df(ds, TaskType.qa, DEFAULT_METRIC_QA)
+    print(f"QA data loaded: {ds.qa_raw_df.shape}")
+    ds.qa_fmt_df = ds.qa_raw_df.copy()
+    qa_cols, ds.qa_types = get_default_cols(TaskType.qa, ds.slug, add_fix_cols=True)
+    # by default, drop the anonymous submissions
+    ds.qa_fmt_df = ds.qa_fmt_df[~ds.qa_fmt_df[COL_NAME_IS_ANONYMOUS]][qa_cols]
+    # reset the rank after dropping the anonymous submissions
+    ds.qa_fmt_df = reset_rank(ds.qa_fmt_df)
+    ds.qa_fmt_df.drop([COL_NAME_REVISION, COL_NAME_TIMESTAMP], axis=1, inplace=True)
+
+    ds.doc_raw_df = get_leaderboard_df(ds, TaskType.long_doc, DEFAULT_METRIC_LONG_DOC)
+    print(f"Long-Doc data loaded: {len(ds.doc_raw_df)}")
+    ds.doc_fmt_df = ds.doc_raw_df.copy()
+    doc_cols, ds.doc_types = get_default_cols(TaskType.long_doc, ds.slug, add_fix_cols=True)
+    # by default, drop the anonymous submissions
+    ds.doc_fmt_df = ds.doc_fmt_df[~ds.doc_fmt_df[COL_NAME_IS_ANONYMOUS]][doc_cols]
+    # reset the rank after dropping the anonymous submissions
+    ds.doc_fmt_df = reset_rank(ds.doc_fmt_df)
+    ds.doc_fmt_df.drop([COL_NAME_REVISION, COL_NAME_TIMESTAMP], axis=1, inplace=True)
+
+    ds.reranking_models = sorted(list(frozenset([eval_result.reranking_model for eval_result in ds.raw_data])))
+    return ds
+
+
+def load_eval_results(file_path: Union[str, Path]) -> Dict[str, LeaderboardDataStore]:
+    output = {}
+    for version in BENCHMARK_VERSION_LIST:
+        fn = f"{file_path}/{version}"
+        output[version] = load_leaderboard_datastore(fn, version)
+    return output

src/{read_evals.py → models.py}

@@ -1,38 +1,21 @@
 import json
-import os.path
 from collections import defaultdict
 from dataclasses import dataclass
+from enum import Enum
 from typing import List
 
 import pandas as pd
 
-from src.benchmarks import get_safe_name
-from src.display.utils import (
+from src.columns import (
+    COL_NAME_IS_ANONYMOUS,
     COL_NAME_RERANKING_MODEL,
-    COL_NAME_RETRIEVAL_MODEL,
     COL_NAME_RERANKING_MODEL_LINK,
+    COL_NAME_RETRIEVAL_MODEL,
     COL_NAME_RETRIEVAL_MODEL_LINK,
     COL_NAME_REVISION,
     COL_NAME_TIMESTAMP,
-    COL_NAME_IS_ANONYMOUS,
-    COLS_QA,
-    QA_BENCHMARK_COLS,
-    COLS_LONG_DOC,
-    LONG_DOC_BENCHMARK_COLS,
-    COL_NAME_AVG,
-    COL_NAME_RANK
 )
 
-from src.display.formatting import make_clickable_model
-
-pd.options.mode.copy_on_write = True
-
-def calculate_mean(row):
-    if pd.isna(row).any():
-        return -1
-    else:
-        return row.mean()
-
 
 @dataclass
 class EvalResult:
@@ -40,6 +23,7 @@ class EvalResult:
     Evaluation result of a single embedding model with a specific reranking model on benchmarks over different
     domains, languages, and datasets
     """
+
     eval_name: str  # name of the evaluation, [retrieval_model]_[reranking_model]_[metric]
     retrieval_model: str
     reranking_model: str
@@ -56,6 +40,7 @@ class FullEvalResult:
     """
     Evaluation result of a single embedding model with a specific reranking model on benchmarks over different tasks
     """
+
     eval_name: str  # name of the evaluation, [retrieval_model]_[reranking_model]
     retrieval_model: str
     reranking_model: str
@@ -79,7 +64,6 @@ class FullEvalResult:
         result_list = []
         retrieval_model_link = ""
         reranking_model_link = ""
-        revision = ""
         for item in model_data:
             config = item.get("config", {})
             # eval results for different metrics
@@ -98,24 +82,26 @@ class FullEvalResult:
                 metric=config["metric"],
                 timestamp=config.get("timestamp", "2024-05-12T12:24:02Z"),
                 revision=config.get("revision", "3a2ba9dcad796a48a02ca1147557724e"),
-                is_anonymous=config.get("is_anonymous", False)
+                is_anonymous=config.get("is_anonymous", False),
             )
             result_list.append(eval_result)
+        eval_result = result_list[0]
         return cls(
-            eval_name=f"{result_list[0].retrieval_model}_{result_list[0].reranking_model}",
-            retrieval_model=result_list[0].retrieval_model,
-            reranking_model=result_list[0].reranking_model,
+            eval_name=f"{eval_result.retrieval_model}_{eval_result.reranking_model}",
+            retrieval_model=eval_result.retrieval_model,
+            reranking_model=eval_result.reranking_model,
             retrieval_model_link=retrieval_model_link,
             reranking_model_link=reranking_model_link,
             results=result_list,
-            timestamp=result_list[0].timestamp,
-            revision=result_list[0].revision,
-            is_anonymous=result_list[0].is_anonymous
+            timestamp=eval_result.timestamp,
+            revision=eval_result.revision,
+            is_anonymous=eval_result.is_anonymous,
         )
 
-    def to_dict(self, task='qa', metric='ndcg_at_3') -> List:
+    def to_dict(self, task="qa", metric="ndcg_at_3") -> List:
         """
-        Convert the results in all the EvalResults over different tasks and metrics. The output is a list of dict compatible with the dataframe UI
+        Convert the results in all the EvalResults over different tasks and metrics.
+        The output is a list of dict compatible with the dataframe UI
         """
         results = defaultdict(dict)
         for eval_result in self.results:
@@ -123,106 +109,66 @@ class FullEvalResult:
                 continue
             if eval_result.task != task:
                 continue
-            results[eval_result.eval_name]["eval_name"] = eval_result.eval_name
-            results[eval_result.eval_name][COL_NAME_RETRIEVAL_MODEL] = (
-                make_clickable_model(self.retrieval_model, self.retrieval_model_link))
-            results[eval_result.eval_name][COL_NAME_RERANKING_MODEL] = (
-                make_clickable_model(self.reranking_model, self.reranking_model_link))
-            results[eval_result.eval_name][COL_NAME_RETRIEVAL_MODEL_LINK] = self.retrieval_model_link
-            results[eval_result.eval_name][COL_NAME_RERANKING_MODEL_LINK] = self.reranking_model_link
-            results[eval_result.eval_name][COL_NAME_REVISION] = self.revision
-            results[eval_result.eval_name][COL_NAME_TIMESTAMP] = self.timestamp
-            results[eval_result.eval_name][COL_NAME_IS_ANONYMOUS] = self.is_anonymous
-
-            # print(f'result loaded: {eval_result.eval_name}')
+            eval_name = eval_result.eval_name
+            results[eval_name]["eval_name"] = eval_name
+            results[eval_name][COL_NAME_RETRIEVAL_MODEL] = make_clickable_model(
+                self.retrieval_model, self.retrieval_model_link
+            )
+            results[eval_name][COL_NAME_RERANKING_MODEL] = make_clickable_model(
+                self.reranking_model, self.reranking_model_link
+            )
+            results[eval_name][COL_NAME_RETRIEVAL_MODEL_LINK] = self.retrieval_model_link
+            results[eval_name][COL_NAME_RERANKING_MODEL_LINK] = self.reranking_model_link
+            results[eval_name][COL_NAME_REVISION] = self.revision
+            results[eval_name][COL_NAME_TIMESTAMP] = self.timestamp
+            results[eval_name][COL_NAME_IS_ANONYMOUS] = self.is_anonymous
+
             for result in eval_result.results:
                 # add result for each domain, language, and dataset
                 domain = result["domain"]
                 lang = result["lang"]
                 dataset = result["dataset"]
                 value = result["value"] * 100
-                if dataset == 'default':
+                if dataset == "default":
                     benchmark_name = f"{domain}_{lang}"
                 else:
                     benchmark_name = f"{domain}_{lang}_{dataset}"
-                results[eval_result.eval_name][get_safe_name(benchmark_name)] = value
+                results[eval_name][get_safe_name(benchmark_name)] = value
         return [v for v in results.values()]
 
 
-def get_raw_eval_results(results_path: str) -> List[FullEvalResult]:
-    """
-    Load the evaluation results from a json file
-    """
-    model_result_filepaths = []
-    for root, dirs, files in os.walk(results_path):
-        if len(files) == 0:
-            continue
-
-        # select the latest results
-        for file in files:
-            if not (file.startswith("results") and file.endswith(".json")):
-                print(f'skip {file}')
-                continue
-            model_result_filepaths.append(os.path.join(root, file))
-
-    eval_results = {}
-    for model_result_filepath in model_result_filepaths:
-        # create evaluation results
-        try:
-            eval_result = FullEvalResult.init_from_json_file(model_result_filepath)
-        except UnicodeDecodeError as e:
-            print(f"loading file failed. {model_result_filepath}")
-            continue
-        print(f'file loaded: {model_result_filepath}')
-        timestamp = eval_result.timestamp
-        eval_results[timestamp] = eval_result
-
-    results = []
-    for k, v in eval_results.items():
-        try:
-            v.to_dict()
-            results.append(v)
-        except KeyError:
-            print(f"loading failed: {k}")
-            continue
-    return results
-
-
-def get_leaderboard_df(raw_data: List[FullEvalResult], task: str, metric: str) -> pd.DataFrame:
-    """
-    Creates a dataframe from all the individual experiment results
-    """
-    cols = [COL_NAME_IS_ANONYMOUS, ]
-    if task == "qa":
-        cols += COLS_QA
-        benchmark_cols = QA_BENCHMARK_COLS
-    elif task == "long-doc":
-        cols += COLS_LONG_DOC
-        benchmark_cols = LONG_DOC_BENCHMARK_COLS
-    else:
-        raise NotImplemented
-    all_data_json = []
-    for v in raw_data:
-        all_data_json += v.to_dict(task=task, metric=metric)
-    df = pd.DataFrame.from_records(all_data_json)
-    # print(f'dataframe created: {df.shape}')
-
-    _benchmark_cols = frozenset(benchmark_cols).intersection(frozenset(df.columns.to_list()))
-
-    # calculate the average score for selected benchmarks
-    df[COL_NAME_AVG] = df[list(_benchmark_cols)].apply(calculate_mean, axis=1).round(decimals=2)
-    df.sort_values(by=[COL_NAME_AVG], ascending=False, inplace=True)
-    df.reset_index(inplace=True, drop=True)
-
-    _cols = frozenset(cols).intersection(frozenset(df.columns.to_list()))
-    df = df[_cols].round(decimals=2)
-
-    # filter out if any of the benchmarks have not been produced
-    df[COL_NAME_RANK] = df[COL_NAME_AVG].rank(ascending=False, method="min")
-
-    # shorten the revision
-    df[COL_NAME_REVISION] = df[COL_NAME_REVISION].str[:6]
-    
-    # # replace "0" with "-" for average score
-    # df[COL_NAME_AVG] = df[COL_NAME_AVG].replace(0, "-")
-    return df
+@dataclass
+class LeaderboardDataStore:
+    version: str
+    slug: str
+    raw_data: list = None
+    qa_raw_df: pd.DataFrame = pd.DataFrame()
+    doc_raw_df: pd.DataFrame = pd.DataFrame()
+    qa_fmt_df: pd.DataFrame = pd.DataFrame()
+    doc_fmt_df: pd.DataFrame = pd.DataFrame()
+    reranking_models: list = None
+    qa_types: list = None
+    doc_types: list = None
+
+
+# Define an enum class with the name `TaskType`. There are two types of tasks, `qa` and `long-doc`.
+class TaskType(Enum):
+    qa = "qa"
+    long_doc = "long-doc"
+
+
+def make_clickable_model(model_name: str, model_link: str):
+    # link = f"https://huggingface.co/{model_name}"
+    if not model_link or not model_link.startswith("https://"):
+        return model_name
+    return model_hyperlink(model_link, model_name)
+
+
+def model_hyperlink(link, model_name):
+    return f'<a target="_blank" href="{link}" style="color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;">{model_name}</a>'
+
+
+def get_safe_name(name: str):
+    """Get RFC 1123 compatible safe name"""
+    name = name.replace("-", "_")
+    return "".join(character.lower() for character in name if (character.isalnum() or character == "_"))

src/utils.py

@@ -1,24 +1,37 @@
-import json
 import hashlib
+import json
+import re
 from datetime import datetime, timezone
 from pathlib import Path
-from typing import List
 
 import pandas as pd
 
-from src.benchmarks import BENCHMARK_COLS_QA, BENCHMARK_COLS_LONG_DOC, BenchmarksQA, BenchmarksLongDoc
-from src.display.formatting import styled_message, styled_error
-from src.display.utils import COLS_QA, TYPES_QA, COLS_LONG_DOC, TYPES_LONG_DOC, COL_NAME_RANK, COL_NAME_AVG, \
-    COL_NAME_RERANKING_MODEL, COL_NAME_RETRIEVAL_MODEL, COL_NAME_IS_ANONYMOUS, COL_NAME_TIMESTAMP, COL_NAME_REVISION, get_default_auto_eval_column_dict
-from src.envs import API, SEARCH_RESULTS_REPO, LATEST_BENCHMARK_VERSION
-from src.read_evals import FullEvalResult, get_leaderboard_df, calculate_mean
-
-import re
+from src.benchmarks import LongDocBenchmarks, QABenchmarks
+from src.columns import (
+    COL_NAME_AVG,
+    COL_NAME_IS_ANONYMOUS,
+    COL_NAME_RANK,
+    COL_NAME_RERANKING_MODEL,
+    COL_NAME_RETRIEVAL_MODEL,
+    COL_NAME_REVISION,
+    COL_NAME_TIMESTAMP,
+    get_default_col_names_and_types,
+    get_fixed_col_names_and_types,
+)
+from src.envs import API, LATEST_BENCHMARK_VERSION, SEARCH_RESULTS_REPO
+from src.models import TaskType, get_safe_name
+
+
+def calculate_mean(row):
+    if pd.isna(row).any():
+        return -1
+    else:
+        return row.mean()
 
 
 def remove_html(input_str):
     # Regular expression for finding HTML tags
-    clean = re.sub(r'<.*?>', '', input_str)
+    clean = re.sub(r"<.*?>", "", input_str)
     return clean
 
 
@@ -55,160 +68,152 @@ def search_table(df: pd.DataFrame, query: str) -> pd.DataFrame:
     return df[(df[COL_NAME_RETRIEVAL_MODEL].str.contains(query, case=False))]
 
 
-def get_default_cols(task: str, columns: list=[], add_fix_cols: bool=True) -> list:
+def get_default_cols(task: TaskType, version_slug, add_fix_cols: bool = True) -> tuple:
     cols = []
     types = []
-    if task == "qa":
-        cols_list = COLS_QA
-        types_list = TYPES_QA
-        benchmark_list = BENCHMARK_COLS_QA
-    elif task == "long-doc":
-        cols_list = COLS_LONG_DOC
-        types_list = TYPES_LONG_DOC
-        benchmark_list = BENCHMARK_COLS_LONG_DOC
+    if task == TaskType.qa:
+        benchmarks = QABenchmarks[version_slug]
+    elif task == TaskType.long_doc:
+        benchmarks = LongDocBenchmarks[version_slug]
     else:
-        raise NotImplemented
+        raise NotImplementedError
+    cols_list, types_list = get_default_col_names_and_types(benchmarks)
+    benchmark_list = [c.value.col_name for c in list(benchmarks.value)]
     for col_name, col_type in zip(cols_list, types_list):
         if col_name not in benchmark_list:
             continue
-        if len(columns) > 0 and col_name not in columns:
-            continue
         cols.append(col_name)
         types.append(col_type)
-
     if add_fix_cols:
         _cols = []
         _types = []
+        fixed_cols, fixed_cols_types = get_fixed_col_names_and_types()
         for col_name, col_type in zip(cols, types):
-            if col_name in FIXED_COLS:
+            if col_name in fixed_cols:
                 continue
             _cols.append(col_name)
             _types.append(col_type)
-        cols = FIXED_COLS + _cols
-        types = FIXED_COLS_TYPES + _types
+        cols = fixed_cols + _cols
+        types = fixed_cols_types + _types
     return cols, types
 
 
-fixed_cols = get_default_auto_eval_column_dict()[:-3]
-
-FIXED_COLS = [c.name for _, _, c in fixed_cols]
-FIXED_COLS_TYPES = [c.type for _, _, c in fixed_cols]
-
-
-def select_columns(
-        df: pd.DataFrame,
-        domain_query: list,
-        language_query: list,
-        task: str = "qa",
-        reset_ranking: bool = True
-) -> pd.DataFrame:
-    cols, _ = get_default_cols(task=task, columns=df.columns, add_fix_cols=False)
+def get_selected_cols(task, version_slug, domains, languages):
+    cols, _ = get_default_cols(task=task, version_slug=version_slug, add_fix_cols=False)
     selected_cols = []
     for c in cols:
-        if task == "qa":
-            eval_col = BenchmarksQA[c].value
-        elif task == "long-doc":
-            eval_col = BenchmarksLongDoc[c].value
-        if eval_col.domain not in domain_query:
+        if task == TaskType.qa:
+            eval_col = QABenchmarks[version_slug].value[c].value
+        elif task == TaskType.long_doc:
+            eval_col = LongDocBenchmarks[version_slug].value[c].value
+        else:
+            raise NotImplementedError
+        if eval_col.domain not in domains:
             continue
-        if eval_col.lang not in language_query:
+        if eval_col.lang not in languages:
             continue
         selected_cols.append(c)
     # We use COLS to maintain sorting
-    filtered_df = df[FIXED_COLS + selected_cols]
+    return selected_cols
+
+
+def select_columns(
+    df: pd.DataFrame,
+    domains: list,
+    languages: list,
+    task: TaskType = TaskType.qa,
+    reset_ranking: bool = True,
+    version_slug: str = None,
+) -> pd.DataFrame:
+    selected_cols = get_selected_cols(task, version_slug, domains, languages)
+    fixed_cols, _ = get_fixed_col_names_and_types()
+    filtered_df = df[fixed_cols + selected_cols]
+    filtered_df.replace({"": pd.NA}, inplace=True)
     if reset_ranking:
         filtered_df[COL_NAME_AVG] = filtered_df[selected_cols].apply(calculate_mean, axis=1).round(decimals=2)
         filtered_df.sort_values(by=[COL_NAME_AVG], ascending=False, inplace=True)
         filtered_df.reset_index(inplace=True, drop=True)
         filtered_df = reset_rank(filtered_df)
-
     return filtered_df
 
 
-def _update_table(
-        task: str,
-        hidden_df: pd.DataFrame,
-        domains: list,
-        langs: list,
-        reranking_query: list,
-        query: str,
-        show_anonymous: bool,
-        reset_ranking: bool = True,
-        show_revision_and_timestamp: bool = False
+def _update_df_elem(
+    task: TaskType,
+    version: str,
+    source_df: pd.DataFrame,
+    domains: list,
+    langs: list,
+    reranking_query: list,
+    query: str,
+    show_anonymous: bool,
+    reset_ranking: bool = True,
+    show_revision_and_timestamp: bool = False,
 ):
-    filtered_df = hidden_df.copy()
+    filtered_df = source_df.copy()
     if not show_anonymous:
         filtered_df = filtered_df[~filtered_df[COL_NAME_IS_ANONYMOUS]]
     filtered_df = filter_models(filtered_df, reranking_query)
     filtered_df = filter_queries(query, filtered_df)
-    filtered_df = select_columns(filtered_df, domains, langs, task, reset_ranking)
+    filtered_df = select_columns(filtered_df, domains, langs, task, reset_ranking, get_safe_name(version))
     if not show_revision_and_timestamp:
         filtered_df.drop([COL_NAME_REVISION, COL_NAME_TIMESTAMP], axis=1, inplace=True)
     return filtered_df
 
 
-def update_table(
-        hidden_df: pd.DataFrame,
-        domains: list,
-        langs: list,
-        reranking_query: list,
-        query: str,
-        show_anonymous: bool,
-        show_revision_and_timestamp: bool = False,
-        reset_ranking: bool = True
-):
-    return _update_table(
-        "qa", hidden_df, domains, langs, reranking_query, query, show_anonymous, reset_ranking, show_revision_and_timestamp)
-
-
-def update_table_long_doc(
-        hidden_df: pd.DataFrame,
-        domains: list,
-        langs: list,
-        reranking_query: list,
-        query: str,
-        show_anonymous: bool,
-        show_revision_and_timestamp: bool = False,
-        reset_ranking: bool = True
-
+def update_doc_df_elem(
+    version: str,
+    hidden_df: pd.DataFrame,
+    domains: list,
+    langs: list,
+    reranking_query: list,
+    query: str,
+    show_anonymous: bool,
+    show_revision_and_timestamp: bool = False,
+    reset_ranking: bool = True,
 ):
-    return _update_table(
-        "long-doc", hidden_df, domains, langs, reranking_query, query, show_anonymous, reset_ranking, show_revision_and_timestamp)
+    return _update_df_elem(
+        TaskType.long_doc,
+        version,
+        hidden_df,
+        domains,
+        langs,
+        reranking_query,
+        query,
+        show_anonymous,
+        reset_ranking,
+        show_revision_and_timestamp,
+    )
 
 
 def update_metric(
-        raw_data: List[FullEvalResult],
-        task: str,
-        metric: str,
-        domains: list,
-        langs: list,
-        reranking_model: list,
-        query: str,
-        show_anonymous: bool = False,
-        show_revision_and_timestamp: bool = False,
+    datastore,
+    task: TaskType,
+    metric: str,
+    domains: list,
+    langs: list,
+    reranking_model: list,
+    query: str,
+    show_anonymous: bool = False,
+    show_revision_and_timestamp: bool = False,
 ) -> pd.DataFrame:
-    if task == 'qa':
-        leaderboard_df = get_leaderboard_df(raw_data, task=task, metric=metric)
-        return update_table(
-            leaderboard_df,
-            domains,
-            langs,
-            reranking_model,
-            query,
-            show_anonymous,
-            show_revision_and_timestamp
-        )
-    elif task == "long-doc":
-        leaderboard_df = get_leaderboard_df(raw_data, task=task, metric=metric)
-        return update_table_long_doc(
-            leaderboard_df,
-            domains,
-            langs,
-            reranking_model,
-            query,
-            show_anonymous,
-            show_revision_and_timestamp
-        )
+    if task == TaskType.qa:
+        update_func = update_qa_df_elem
+    elif task == TaskType.long_doc:
+        update_func = update_doc_df_elem
+    else:
+        raise NotImplementedError
+    df_elem = get_leaderboard_df(datastore, task=task, metric=metric)
+    version = datastore.version
+    return update_func(
+        version,
+        df_elem,
+        domains,
+        langs,
+        reranking_model,
+        query,
+        show_anonymous,
+        show_revision_and_timestamp,
+    )
 
 
 def upload_file(filepath: str):
@@ -218,7 +223,6 @@ def upload_file(filepath: str):
     return filepath
 
 
-
 def get_iso_format_timestamp():
     # Get the current timestamp with UTC as the timezone
     current_timestamp = datetime.now(timezone.utc)
@@ -227,15 +231,15 @@ def get_iso_format_timestamp():
     current_timestamp = current_timestamp.replace(microsecond=0)
 
     # Convert to ISO 8601 format and replace the offset with 'Z'
-    iso_format_timestamp = current_timestamp.isoformat().replace('+00:00', 'Z')
-    filename_friendly_timestamp = current_timestamp.strftime('%Y%m%d%H%M%S')
+    iso_format_timestamp = current_timestamp.isoformat().replace("+00:00", "Z")
+    filename_friendly_timestamp = current_timestamp.strftime("%Y%m%d%H%M%S")
     return iso_format_timestamp, filename_friendly_timestamp
 
 
 def calculate_file_md5(file_path):
     md5 = hashlib.md5()
 
-    with open(file_path, 'rb') as f:
+    with open(file_path, "rb") as f:
         while True:
             data = f.read(4096)
             if not data:
@@ -246,13 +250,14 @@ def calculate_file_md5(file_path):
 
 
 def submit_results(
-        filepath: str,
-        model: str,
-        model_url: str,
-        reranking_model: str="",
-        reranking_model_url: str="",
-        version: str=LATEST_BENCHMARK_VERSION,
-        is_anonymous=False):
+    filepath: str,
+    model: str,
+    model_url: str,
+    reranking_model: str = "",
+    reranking_model_url: str = "",
+    version: str = LATEST_BENCHMARK_VERSION,
+    is_anonymous=False,
+):
     if not filepath.endswith(".zip"):
         return styled_error(f"file uploading aborted. wrong file type: {filepath}")
 
@@ -265,11 +270,13 @@ def submit_results(
         if not model_url.startswith("https://") and not model_url.startswith("http://"):
             # TODO: retrieve the model page and find the model name on the page
             return styled_error(
-                f"failed to submit. Model url must start with `https://` or `http://`. Illegal model url: {model_url}")
+                f"failed to submit. Model url must start with `https://` or `http://`. Illegal model url: {model_url}"
+            )
         if reranking_model != "NoReranker":
             if not reranking_model_url.startswith("https://") and not reranking_model_url.startswith("http://"):
                 return styled_error(
-                    f"failed to submit. Model url must start with `https://` or `http://`. Illegal model url: {model_url}")
+                    f"failed to submit. Model url must start with `https://` or `http://`. Illegal model url: {model_url}"
+                )
 
     # rename the uploaded file
     input_fp = Path(filepath)
@@ -279,14 +286,15 @@ def submit_results(
     input_folder_path = input_fp.parent
 
     if not reranking_model:
-        reranking_model = 'NoReranker'
-    
+        reranking_model = "NoReranker"
+
     API.upload_file(
         path_or_fileobj=filepath,
         path_in_repo=f"{version}/{model}/{reranking_model}/{output_fn}",
         repo_id=SEARCH_RESULTS_REPO,
         repo_type="dataset",
-        commit_message=f"feat: submit {model} to evaluate")
+        commit_message=f"feat: submit {model} to evaluate",
+    )
 
     output_config_fn = f"{output_fn.removesuffix('.zip')}.json"
     output_config = {
@@ -297,7 +305,7 @@ def submit_results(
         "version": f"{version}",
         "is_anonymous": is_anonymous,
         "revision": f"{revision}",
-        "timestamp": f"{timestamp_config}"
+        "timestamp": f"{timestamp_config}",
     }
     with open(input_folder_path / output_config_fn, "w") as f:
         json.dump(output_config, f, indent=4, ensure_ascii=False)
@@ -306,7 +314,8 @@ def submit_results(
         path_in_repo=f"{version}/{model}/{reranking_model}/{output_config_fn}",
         repo_id=SEARCH_RESULTS_REPO,
         repo_type="dataset",
-        commit_message=f"feat: submit {model} + {reranking_model} config")
+        commit_message=f"feat: submit {model} + {reranking_model} config",
+    )
     return styled_message(
         f"Thanks for submission!\n"
         f"Retrieval method: {model}\nReranking model: {reranking_model}\nSubmission revision: {revision}"
@@ -316,3 +325,125 @@ def submit_results(
 def reset_rank(df):
     df[COL_NAME_RANK] = df[COL_NAME_AVG].rank(ascending=False, method="min")
     return df
+
+
+def get_leaderboard_df(datastore, task: TaskType, metric: str) -> pd.DataFrame:
+    """
+    Creates a dataframe from all the individual experiment results
+    """
+    # load the selected metrics into a DataFrame from the raw json
+    all_data_json = []
+    for v in datastore.raw_data:
+        all_data_json += v.to_dict(task=task.value, metric=metric)
+    df = pd.DataFrame.from_records(all_data_json)
+
+    # calculate the average scores for selected task
+    if task == TaskType.qa:
+        benchmarks = QABenchmarks[datastore.slug]
+    elif task == TaskType.long_doc:
+        benchmarks = LongDocBenchmarks[datastore.slug]
+    else:
+        raise NotImplementedError
+    valid_cols = frozenset(df.columns.to_list())
+    benchmark_cols = []
+    for t in list(benchmarks.value):
+        if t.value.col_name not in valid_cols:
+            continue
+        benchmark_cols.append(t.value.col_name)
+
+    # filter out the columns that are not in the data
+    df[COL_NAME_AVG] = df[list(benchmark_cols)].apply(calculate_mean, axis=1).round(decimals=2)
+    df.sort_values(by=[COL_NAME_AVG], ascending=False, inplace=True)
+    df.reset_index(inplace=True, drop=True)
+
+    # filter out columns that are not in the data
+    display_cols = [COL_NAME_IS_ANONYMOUS, COL_NAME_AVG]
+    default_cols, _ = get_default_col_names_and_types(benchmarks)
+    for col in default_cols:
+        if col in valid_cols:
+            display_cols.append(col)
+    df = df[display_cols].round(decimals=2)
+
+    # rank the scores
+    df = reset_rank(df)
+
+    # shorten the revision
+    df[COL_NAME_REVISION] = df[COL_NAME_REVISION].str[:6]
+
+    return df
+
+
+def set_listeners(
+    task: TaskType,
+    target_df,
+    source_df,
+    search_bar,
+    version,
+    selected_domains,
+    selected_langs,
+    selected_rerankings,
+    show_anonymous,
+    show_revision_and_timestamp,
+):
+    if task == TaskType.qa:
+        update_table_func = update_qa_df_elem
+    elif task == TaskType.long_doc:
+        update_table_func = update_doc_df_elem
+    else:
+        raise NotImplementedError
+    selector_list = [selected_domains, selected_langs, selected_rerankings, search_bar, show_anonymous]
+    search_bar_args = [
+        source_df,
+        version,
+    ] + selector_list
+    selector_args = (
+        [version, source_df]
+        + selector_list
+        + [
+            show_revision_and_timestamp,
+        ]
+    )
+    # Set search_bar listener
+    search_bar.submit(update_table_func, search_bar_args, target_df)
+
+    # Set column-wise listener
+    for selector in selector_list:
+        selector.change(
+            update_table_func,
+            selector_args,
+            target_df,
+            queue=True,
+        )
+
+
+def update_qa_df_elem(
+    version: str,
+    hidden_df: pd.DataFrame,
+    domains: list,
+    langs: list,
+    reranking_query: list,
+    query: str,
+    show_anonymous: bool,
+    show_revision_and_timestamp: bool = False,
+    reset_ranking: bool = True,
+):
+    return _update_df_elem(
+        TaskType.qa,
+        version,
+        hidden_df,
+        domains,
+        langs,
+        reranking_query,
+        query,
+        show_anonymous,
+        reset_ranking,
+        show_revision_and_timestamp,
+    )
+
+
+def styled_error(error):
+    return f"<p style='color: red; font-size: 20px; text-align: center;'>{error}</p>"
+
+
+def styled_message(message):
+    return f"<p style='color: green; font-size: 20px; text-align: center;'>{message}</p>"

tests/src/display/test_utils.py

@@ -1,23 +0,0 @@
-import pytest
-from src.display.utils import fields, AutoEvalColumnQA, COLS_QA, COLS_LONG_DOC, COLS_LITE, TYPES_QA, TYPES_LONG_DOC, QA_BENCHMARK_COLS, LONG_DOC_BENCHMARK_COLS, get_default_auto_eval_column_dict
-
-
-def test_fields():
-    for c in fields(AutoEvalColumnQA):
-        print(c)
-
-
-def test_macro_variables():
-    print(f'COLS_QA: {COLS_QA}')
-    print(f'COLS_LONG_DOC: {COLS_LONG_DOC}')
-    print(f'COLS_LITE: {COLS_LITE}')
-    print(f'TYPES_QA: {TYPES_QA}')
-    print(f'TYPES_LONG_DOC: {TYPES_LONG_DOC}')
-    print(f'QA_BENCHMARK_COLS: {QA_BENCHMARK_COLS}')
-    print(f'LONG_DOC_BENCHMARK_COLS: {LONG_DOC_BENCHMARK_COLS}')
-
-
-def test_get_default_auto_eval_column_dict():
-    auto_eval_column_dict_list = get_default_auto_eval_column_dict()
-    assert len(auto_eval_column_dict_list) == 9
-

tests/src/test_benchmarks.py

@@ -1,9 +1,33 @@
-from src.benchmarks import BenchmarksQA, BenchmarksLongDoc
+import pytest
 
+from src.benchmarks import LongDocBenchmarks, QABenchmarks
+from src.envs import BENCHMARK_VERSION_LIST
 
-def test_qabenchmarks():
-    print(list(BenchmarksQA))
+# Ref: https://github.com/AIR-Bench/AIR-Bench/blob/4b27b8a8f2047a963805fcf6fb9d74be51ec440c/docs/available_tasks.md
+# 24.05
+# | Task | dev | test |
+# | ---- | --- | ---- |
+# | Long-Doc | 4 | 11 |
+# | QA | 54 | 53 |
+#
+# 24.04
+# | Task | test |
+# | ---- | ---- |
+# | Long-Doc | 15 |
+# | QA | 13 |
 
 
-def test_longdocbenchmarks():
-    print(list(BenchmarksLongDoc))
+@pytest.mark.parametrize("num_datasets_dict", [{"air_bench_2404": 13, "air_bench_2405": 53}])
+def test_qa_benchmarks(num_datasets_dict):
+    assert len(QABenchmarks) == len(BENCHMARK_VERSION_LIST)
+    for benchmark_list in list(QABenchmarks):
+        version_slug = benchmark_list.name
+        assert num_datasets_dict[version_slug] == len(benchmark_list.value)
+
+
+@pytest.mark.parametrize("num_datasets_dict", [{"air_bench_2404": 15, "air_bench_2405": 11}])
+def test_doc_benchmarks(num_datasets_dict):
+    assert len(LongDocBenchmarks) == len(BENCHMARK_VERSION_LIST)
+    for benchmark_list in list(LongDocBenchmarks):
+        version_slug = benchmark_list.name
+        assert num_datasets_dict[version_slug] == len(benchmark_list.value)

tests/src/test_columns.py

@@ -0,0 +1,119 @@
+import pytest
+
+from src.benchmarks import LongDocBenchmarks, QABenchmarks
+from src.columns import (
+    COL_NAME_AVG,
+    COL_NAME_RANK,
+    COL_NAME_RERANKING_MODEL,
+    COL_NAME_RETRIEVAL_MODEL,
+    COL_NAME_REVISION,
+    COL_NAME_TIMESTAMP,
+    get_default_auto_eval_column_dict,
+    get_default_col_names_and_types,
+    get_fixed_col_names_and_types,
+    make_autoevalcolumn,
+)
+
+# Ref: https://github.com/AIR-Bench/AIR-Bench/blob/4b27b8a8f2047a963805fcf6fb9d74be51ec440c/docs/available_tasks.md
+# 24.05
+# | Task | dev | test |
+# | ---- | --- | ---- |
+# | Long-Doc | 4 | 11 |
+# | QA | 54 | 53 |
+#
+# 24.04
+# | Task | test |
+# | ---- | ---- |
+# | Long-Doc | 15 |
+# | QA | 13 |
+
+
+@pytest.fixture()
+def expected_col_names():
+    return [
+        "rank",
+        "retrieval_model",
+        "reranking_model",
+        "revision",
+        "timestamp",
+        "average",
+        "retrieval_model_link",
+        "reranking_model_link",
+        "is_anonymous",
+    ]
+
+
+@pytest.fixture()
+def expected_hidden_col_names():
+    return [
+        "retrieval_model_link",
+        "reranking_model_link",
+        "is_anonymous",
+    ]
+
+
+def test_get_default_auto_eval_column_dict(expected_col_names, expected_hidden_col_names):
+    col_list = get_default_auto_eval_column_dict()
+    assert len(col_list) == 9
+    hidden_cols = []
+    for col_tuple, expected_col in zip(col_list, expected_col_names):
+        col, _, col_content = col_tuple
+        assert col == expected_col
+        if col_content.hidden:
+            hidden_cols.append(col)
+    assert hidden_cols == expected_hidden_col_names
+
+
+def test_get_fixed_col_names_and_types():
+    col_names, col_types = get_fixed_col_names_and_types()
+    assert len(col_names) == 6
+    assert len(col_types) == 6
+    expected_col_and_type = [
+        (COL_NAME_RANK, "number"),
+        (COL_NAME_RETRIEVAL_MODEL, "markdown"),
+        (COL_NAME_RERANKING_MODEL, "markdown"),
+        (COL_NAME_REVISION, "markdown"),
+        (COL_NAME_TIMESTAMP, "date"),
+        (COL_NAME_AVG, "number"),
+    ]
+    for col_name, col_type, (c_name, c_type) in zip(col_names, col_types, expected_col_and_type):
+        assert col_name == c_name
+        assert col_type == c_type
+
+
+@pytest.mark.parametrize(
+    "benchmarks, expected_benchmark_len",
+    [
+        (QABenchmarks, {"air_bench_2404": 13, "air_bench_2405": 53}),
+        (LongDocBenchmarks, {"air_bench_2404": 15, "air_bench_2405": 11}),
+    ],
+)
+def test_make_autoevalcolumn(benchmarks, expected_benchmark_len, expected_col_names):
+    expected_default_attrs = frozenset(expected_col_names)
+    for benchmark in benchmarks:
+        TestEvalColumn = make_autoevalcolumn("TestEvalColumn", benchmark)
+        attrs = []
+        for k, v in TestEvalColumn.__dict__.items():
+            if not k.startswith("__"):
+                attrs.append(k)
+        attrs = frozenset(attrs)
+        assert expected_default_attrs.issubset(attrs)
+        benchmark_attrs = attrs.difference(expected_default_attrs)
+        assert len(benchmark_attrs) == expected_benchmark_len[benchmark.name]
+
+
+@pytest.mark.parametrize(
+    "benchmarks, expected_benchmark_len",
+    [
+        (QABenchmarks, {"air_bench_2404": 13, "air_bench_2405": 53}),
+        (LongDocBenchmarks, {"air_bench_2404": 15, "air_bench_2405": 11}),
+    ],
+)
+def test_get_default_col_names_and_types(
+    benchmarks, expected_benchmark_len, expected_col_names, expected_hidden_col_names
+):
+    default_col_len = len(expected_col_names)
+    hidden_col_len = len(expected_hidden_col_names)
+    for benchmark in benchmarks:
+        col_names, col_types = get_default_col_names_and_types(benchmark)
+        assert len(col_names) == expected_benchmark_len[benchmark.name] + default_col_len - hidden_col_len

tests/src/test_envs.py

@@ -0,0 +1,14 @@
+from air_benchmark.tasks import BenchmarkTable
+
+from src.envs import BENCHMARK_VERSION_LIST, DEFAULT_METRIC_LONG_DOC, DEFAULT_METRIC_QA, METRIC_LIST
+
+
+def test_benchmark_version_list():
+    leaderboard_versions = frozenset(BENCHMARK_VERSION_LIST)
+    available_versions = frozenset([k for k in BenchmarkTable.keys()])
+    assert leaderboard_versions.issubset(available_versions)
+
+
+def test_default_metrics():
+    assert DEFAULT_METRIC_QA in METRIC_LIST
+    assert DEFAULT_METRIC_LONG_DOC in METRIC_LIST

tests/src/test_loaders.py

@@ -0,0 +1,46 @@
+from pathlib import Path
+
+import pandas as pd
+import pytest
+
+from src.loaders import load_eval_results, load_leaderboard_datastore, load_raw_eval_results
+
+cur_fp = Path(__file__)
+
+
+@pytest.mark.parametrize("version", ["AIR-Bench_24.04", "AIR-Bench_24.05"])
+def test_load_raw_eval_results(version):
+    raw_data = load_raw_eval_results(cur_fp.parents[1] / f"toydata/eval_results/{version}")
+    assert len(raw_data) == 1
+    full_eval_result = raw_data[0]
+    expected_attr = [
+        "eval_name",
+        "retrieval_model",
+        "reranking_model",
+        "retrieval_model_link",
+        "reranking_model_link",
+        "results",
+        "timestamp",
+        "revision",
+        "is_anonymous",
+    ]
+    result_attr = [k for k in full_eval_result.__dict__.keys() if k[:2] != "__" and k[-2:] != "__"]
+    assert sorted(expected_attr) == sorted(result_attr)
+
+
+@pytest.mark.parametrize("version", ["AIR-Bench_24.04", "AIR-Bench_24.05"])
+def test_load_leaderboard_datastore(version):
+    file_path = cur_fp.parents[1] / f"toydata/eval_results/{version}"
+    datastore = load_leaderboard_datastore(file_path, version)
+    for k, v in datastore.__dict__.items():
+        if k[:2] != "__" and k[-2:] != "__":
+            if isinstance(v, list):
+                assert v
+            elif isinstance(v, pd.DataFrame):
+                assert not v.empty
+
+
+def test_load_eval_results():
+    file_path = cur_fp.parents[1] / "toydata/eval_results/"
+    datastore_dict = load_eval_results(file_path)
+    assert len(datastore_dict) == 2

tests/src/test_models.py

@@ -0,0 +1,89 @@
+from pathlib import Path
+
+import pytest
+
+from src.models import EvalResult, FullEvalResult
+
+cur_fp = Path(__file__)
+
+
+# Ref: https://github.com/AIR-Bench/AIR-Bench/blob/4b27b8a8f2047a963805fcf6fb9d74be51ec440c/docs/available_tasks.md
+# 24.05
+# | Task | dev | test |
+# | ---- | --- | ---- |
+# | Long-Doc | 4 | 11 |
+# | QA | 54 | 53 |
+#
+# 24.04
+# | Task | test |
+# | ---- | ---- |
+# | Long-Doc | 15 |
+# | QA | 13 |
+NUM_QA_BENCHMARKS_24_05 = 53
+NUM_DOC_BENCHMARKS_24_05 = 11
+NUM_QA_BENCHMARKS_24_04 = 13
+NUM_DOC_BENCHMARKS_24_04 = 15
+
+
+def test_eval_result():
+    EvalResult(
+        eval_name="eval_name",
+        retrieval_model="bge-m3",
+        reranking_model="NoReranking",
+        results=[{"domain": "law", "lang": "en", "dataset": "lex_files_500K-600K", "value": 0.45723}],
+        task="qa",
+        metric="ndcg_at_3",
+        timestamp="2024-05-14T03:09:08Z",
+        revision="1e243f14bd295ccdea7a118fe847399d",
+        is_anonymous=True,
+    )
+
+
+@pytest.mark.parametrize(
+    "file_path",
+    [
+        "AIR-Bench_24.04/bge-m3/jina-reranker-v2-base-multilingual/results.json",
+        "AIR-Bench_24.05/bge-m3/NoReranker/results.json",
+    ],
+)
+def test_full_eval_result_init_from_json_file(file_path):
+    json_fp = cur_fp.parents[1] / "toydata/eval_results/" / file_path
+    full_eval_result = FullEvalResult.init_from_json_file(json_fp)
+    assert json_fp.parents[0].stem == full_eval_result.reranking_model
+    assert json_fp.parents[1].stem == full_eval_result.retrieval_model
+    assert len(full_eval_result.results) == 70
+
+
+@pytest.mark.parametrize(
+    "file_path, task, expected_num_results",
+    [
+        ("AIR-Bench_24.04/bge-m3/jina-reranker-v2-base-multilingual/results.json", "qa", NUM_QA_BENCHMARKS_24_04),
+        (
+            "AIR-Bench_24.04/bge-m3/jina-reranker-v2-base-multilingual/results.json",
+            "long-doc",
+            NUM_DOC_BENCHMARKS_24_04,
+        ),
+        ("AIR-Bench_24.05/bge-m3/NoReranker/results.json", "qa", NUM_QA_BENCHMARKS_24_05),
+        ("AIR-Bench_24.05/bge-m3/NoReranker/results.json", "long-doc", NUM_DOC_BENCHMARKS_24_05),
+    ],
+)
+def test_full_eval_result_to_dict(file_path, task, expected_num_results):
+    json_fp = cur_fp.parents[1] / "toydata/eval_results/" / file_path
+    full_eval_result = FullEvalResult.init_from_json_file(json_fp)
+    result_dict_list = full_eval_result.to_dict(task)
+    assert len(result_dict_list) == 1
+    result = result_dict_list[0]
+    attr_list = frozenset(
+        [
+            "eval_name",
+            "Retrieval Method",
+            "Reranking Model",
+            "Retrieval Model LINK",
+            "Reranking Model LINK",
+            "Revision",
+            "Submission Date",
+            "Anonymous Submission",
+        ]
+    )
+    result_cols = list(result.keys())
+    assert len(result_cols) == (expected_num_results + len(attr_list))

tests/src/test_read_evals.py

@@ -1,68 +0,0 @@
-from pathlib import Path
-
-from src.read_evals import FullEvalResult, get_raw_eval_results, get_leaderboard_df
-
-cur_fp = Path(__file__)
-
-
-def test_init_from_json_file():
-    json_fp = cur_fp.parents[2] / "toydata" / "test_data.json"
-    full_eval_result = FullEvalResult.init_from_json_file(json_fp)
-    num_different_task_domain_lang_metric_dataset_combination = 6
-    assert len(full_eval_result.results) == \
-           num_different_task_domain_lang_metric_dataset_combination
-    assert full_eval_result.retrieval_model == "bge-m3"
-    assert full_eval_result.reranking_model == "bge-reranker-v2-m3"
-
-
-def test_to_dict():
-    json_fp = cur_fp.parents[2] / "toydata" / "test_data.json"
-    full_eval_result = FullEvalResult.init_from_json_file(json_fp)
-    result_list = full_eval_result.to_dict(task='qa', metric='ndcg_at_1')
-    assert len(result_list) == 1
-    result_dict = result_list[0]
-    assert result_dict["Retrieval Model"] == "bge-m3"
-    assert result_dict["Reranking Model"] == "bge-reranker-v2-m3"
-    assert result_dict["wiki_en"] is not None
-    assert result_dict["wiki_zh"] is not None
-
-
-def test_get_raw_eval_results():
-    results_path = cur_fp.parents[2] / "toydata" / "eval_results" / "AIR-Bench_24.04"
-    results = get_raw_eval_results(results_path)
-    # only load the latest results
-    assert len(results) == 4
-    assert results[0].eval_name == "bge-base-en-v1.5_NoReranker"
-    assert len(results[0].results) == 70
-    assert results[0].eval_name == "bge-base-en-v1.5_bge-reranker-v2-m3"
-    assert len(results[1].results) == 70
-
-
-def test_get_leaderboard_df():
-    results_path = cur_fp.parents[2] / "toydata" / "eval_results" / "AIR-Bench_24.04"
-    raw_data = get_raw_eval_results(results_path)
-    df = get_leaderboard_df(raw_data, 'qa', 'ndcg_at_10')
-    assert df.shape[0] == 4
-    # the results contain only one embedding model
-    # for i in range(4):
-    #     assert df["Retrieval Model"][i] == "bge-m3"
-    # # the results contain only two reranking model
-    # assert df["Reranking Model"][0] == "bge-reranker-v2-m3"
-    # assert df["Reranking Model"][1] == "NoReranker"
-    # assert df["Average ⬆️"][0] > df["Average ⬆️"][1]
-    # assert not df[['Average ⬆️', 'wiki_en', 'wiki_zh', ]].isnull().values.any()
-
-
-def test_get_leaderboard_df_long_doc():
-    results_path = cur_fp.parents[2] / "toydata" / "test_results"
-    raw_data = get_raw_eval_results(results_path)
-    df = get_leaderboard_df(raw_data, 'long-doc', 'ndcg_at_1')
-    assert df.shape[0] == 2
-    # the results contain only one embedding model
-    for i in range(2):
-        assert df["Retrieval Model"][i] == "bge-m3"
-    # the results contains only two reranking model
-    assert df["Reranking Model"][0] == "bge-reranker-v2-m3"
-    assert df["Reranking Model"][1] == "NoReranker"
-    assert df["Average ⬆️"][0] > df["Average ⬆️"][1]
-    assert not df[['Average ⬆️', 'law_en_lex_files_500k_600k', ]].isnull().values.any()

tests/src/test_utils.py

@@ -0,0 +1,237 @@
+from pathlib import Path
+
+import pandas as pd
+import pytest
+
+from src.columns import COL_NAME_RERANKING_MODEL, COL_NAME_RETRIEVAL_MODEL
+from src.models import TaskType, model_hyperlink
+from src.utils import (
+    _update_df_elem,
+    calculate_mean,
+    filter_models,
+    filter_queries,
+    get_default_cols,
+    get_leaderboard_df,
+    get_selected_cols,
+    remove_html,
+    select_columns,
+)
+
+cur_fp = Path(__file__)
+
+NUM_QA_BENCHMARKS_24_05 = 53
+NUM_DOC_BENCHMARKS_24_05 = 11
+NUM_QA_BENCHMARKS_24_04 = 13
+NUM_DOC_BENCHMARKS_24_04 = 15
+
+
+@pytest.fixture
+def toy_df():
+    return pd.DataFrame(
+        {
+            "Retrieval Method": ["bge-m3", "bge-m3", "jina-embeddings-v2-base", "jina-embeddings-v2-base"],
+            "Reranking Model": ["bge-reranker-v2-m3", "NoReranker", "bge-reranker-v2-m3", "NoReranker"],
+            "Rank 🏆": [1, 2, 3, 4],
+            "Revision": ["123", "234", "345", "456"],
+            "Submission Date": ["", "", "", ""],
+            "Average ⬆️": [0.6, 0.4, 0.3, 0.2],
+            "wiki_en": [0.8, 0.7, 0.2, 0.1],
+            "wiki_zh": [0.4, 0.1, 0.4, 0.3],
+            "news_en": [0.8, 0.7, 0.2, 0.1],
+            "news_zh": [0.4, 0.1, 0.2, 0.3],
+            "Anonymous Submission": [False, False, False, True],
+        }
+    )
+
+
+def test_remove_html():
+    model_name = "jina-embeddings-v3"
+    html_str = model_hyperlink("https://jina.ai", model_name)
+    output_str = remove_html(html_str)
+    assert output_str == model_name
+
+
+def test_calculate_mean():
+    valid_row = [1, 3]
+    invalid_row = [2, pd.NA]
+    df = pd.DataFrame([valid_row, invalid_row], columns=["a", "b"])
+    result = list(df.apply(calculate_mean, axis=1))
+    assert result[0] == sum(valid_row) / 2
+    assert result[1] == -1
+
+
+@pytest.mark.parametrize(
+    "models, expected",
+    [
+        (["model1", "model3"], 2),
+        (["model1", "model_missing"], 1),
+        (["model1", "model2", "model3"], 3),
+        (
+            [
+                "model1",
+            ],
+            1,
+        ),
+        ([], 3),
+    ],
+)
+def test_filter_models(models, expected):
+    df = pd.DataFrame(
+        {
+            COL_NAME_RERANKING_MODEL: [
+                "model1",
+                "model2",
+                "model3",
+            ],
+            "col2": [1, 2, 3],
+        }
+    )
+    output_df = filter_models(df, models)
+    assert len(output_df) == expected
+
+
+@pytest.mark.parametrize(
+    "query, expected",
+    [
+        ("model1;model3", 2),
+        ("model1;model4", 1),
+        ("model1;model2;model3", 3),
+        ("model1", 1),
+        ("", 3),
+    ],
+)
+def test_filter_queries(query, expected):
+    df = pd.DataFrame(
+        {
+            COL_NAME_RETRIEVAL_MODEL: [
+                "model1",
+                "model2",
+                "model3",
+            ],
+            COL_NAME_RERANKING_MODEL: [
+                "model4",
+                "model5",
+                "model6",
+            ],
+        }
+    )
+    output_df = filter_queries(query, df)
+    assert len(output_df) == expected
+
+
+@pytest.mark.parametrize(
+    "task_type, slug, add_fix_cols, expected",
+    [
+        (TaskType.qa, "air_bench_2404", True, NUM_QA_BENCHMARKS_24_04),
+        (TaskType.long_doc, "air_bench_2404", True, NUM_DOC_BENCHMARKS_24_04),
+        (TaskType.qa, "air_bench_2405", False, NUM_QA_BENCHMARKS_24_05),
+        (TaskType.long_doc, "air_bench_2405", False, NUM_DOC_BENCHMARKS_24_05),
+    ],
+)
+def test_get_default_cols(task_type, slug, add_fix_cols, expected):
+    attr_cols = ["Rank 🏆", "Retrieval Method", "Reranking Model", "Revision", "Submission Date", "Average ⬆️"]
+    cols, types = get_default_cols(task_type, slug)
+    cols_set = frozenset(cols)
+    attrs_set = frozenset(attr_cols)
+    if add_fix_cols:
+        assert attrs_set.issubset(cols_set)
+    benchmark_cols = list(cols_set.difference(attrs_set))
+    assert len(benchmark_cols) == expected
+
+
+@pytest.mark.parametrize(
+    "task_type, domains, languages, expected",
+    [
+        (
+            TaskType.qa,
+            ["wiki", "news"],
+            [
+                "zh",
+            ],
+            ["wiki_zh", "news_zh"],
+        ),
+        (
+            TaskType.qa,
+            [
+                "law",
+            ],
+            ["zh", "en"],
+            ["law_en"],
+        ),
+        (
+            TaskType.long_doc,
+            ["healthcare"],
+            ["zh", "en"],
+            [
+                "healthcare_en_pubmed_100k_200k_1",
+                "healthcare_en_pubmed_100k_200k_2",
+                "healthcare_en_pubmed_100k_200k_3",
+                "healthcare_en_pubmed_40k_50k_5_merged",
+                "healthcare_en_pubmed_30k_40k_10_merged",
+            ],
+        ),
+    ],
+)
+def test_get_selected_cols(task_type, domains, languages, expected):
+    slug = "air_bench_2404"
+    cols = get_selected_cols(task_type, slug, domains, languages)
+    assert sorted(cols) == sorted(expected)
+
+
+@pytest.mark.parametrize("reset_rank", [False])
+def test_select_columns(toy_df, reset_rank):
+    expected = [
+        "Rank 🏆",
+        "Retrieval Method",
+        "Reranking Model",
+        "Revision",
+        "Submission Date",
+        "Average ⬆️",
+        "news_zh",
+    ]
+    df_result = select_columns(toy_df, ["news"], ["zh"], version_slug="air_bench_2404", reset_ranking=reset_rank)
+    assert len(df_result.columns) == len(expected)
+    if reset_rank:
+        assert df_result["Average ⬆️"].equals(df_result["news_zh"])
+    else:
+        assert df_result["Average ⬆️"].equals(toy_df["Average ⬆️"])
+
+
+@pytest.mark.parametrize(
+    "reset_rank, show_anony",
+    [
+        (False, True),
+        (True, True),
+        (True, False),
+    ],
+)
+def test__update_df_elem(toy_df, reset_rank, show_anony):
+    df = _update_df_elem(TaskType.qa, "AIR-Bench_24.04", toy_df, ["news"], ["zh"], [], "", show_anony, reset_rank)
+    if show_anony:
+        assert df.shape[0] == 4
+    else:
+        assert df.shape[0] == 3
+    if show_anony:
+        if reset_rank:
+            assert df["Average ⬆️"].equals(df["news_zh"])
+        else:
+            assert df["Average ⬆️"].equals(toy_df["Average ⬆️"])
+
+
+@pytest.mark.parametrize(
+    "version, task_type",
+    [
+        ("AIR-Bench_24.04", TaskType.qa),
+        ("AIR-Bench_24.04", TaskType.long_doc),
+        ("AIR-Bench_24.05", TaskType.qa),
+        ("AIR-Bench_24.05", TaskType.long_doc),
+    ],
+)
+def test_get_leaderboard_df(version, task_type):
+    from src.loaders import load_raw_eval_results
+    from src.models import LeaderboardDataStore, get_safe_name
+
+    raw_data = load_raw_eval_results(cur_fp.parents[1] / f"toydata/eval_results/{version}")
+    ds = LeaderboardDataStore(version, get_safe_name(version), raw_data=raw_data)
+    df = get_leaderboard_df(ds, task_type, "ndcg_at_10")
+    assert df.shape[0] == 1

tests/test_utils.py

@@ -1,115 +0,0 @@
-import pandas as pd
-import pytest
-
-from src.utils import filter_models, search_table, filter_queries, select_columns, update_table_long_doc, get_iso_format_timestamp, get_default_cols, update_table
-from src.display.utils import COL_NAME_IS_ANONYMOUS, COL_NAME_REVISION, COL_NAME_TIMESTAMP, COL_NAME_RERANKING_MODEL, COL_NAME_RETRIEVAL_MODEL, COL_NAME_RANK, COL_NAME_AVG
-
-
-@pytest.fixture
-def toy_df():
-    return pd.DataFrame(
-        {
-            "Retrieval Model": [
-                "bge-m3",
-                "bge-m3",
-                "jina-embeddings-v2-base",
-                "jina-embeddings-v2-base"
-            ],
-            "Reranking Model": [
-                "bge-reranker-v2-m3",
-                "NoReranker",
-                "bge-reranker-v2-m3",
-                "NoReranker"
-            ],
-            "Average ⬆️": [0.6, 0.4, 0.3, 0.2],
-            "wiki_en": [0.8, 0.7, 0.2, 0.1],
-            "wiki_zh": [0.4, 0.1, 0.4, 0.3],
-            "news_en": [0.8, 0.7, 0.2, 0.1],
-            "news_zh": [0.4, 0.1, 0.4, 0.3],
-        }
-    )
-
-
-@pytest.fixture
-def toy_df_long_doc():
-    return pd.DataFrame(
-        {
-            "Retrieval Model": [
-                "bge-m3",
-                "bge-m3",
-                "jina-embeddings-v2-base",
-                "jina-embeddings-v2-base"
-            ],
-            "Reranking Model": [
-                "bge-reranker-v2-m3",
-                "NoReranker",
-                "bge-reranker-v2-m3",
-                "NoReranker"
-            ],
-            "Average ⬆️": [0.6, 0.4, 0.3, 0.2],
-            "law_en_lex_files_300k_400k": [0.4, 0.1, 0.4, 0.3],
-            "law_en_lex_files_400k_500k": [0.8, 0.7, 0.2, 0.1],
-            "law_en_lex_files_500k_600k": [0.8, 0.7, 0.2, 0.1],
-            "law_en_lex_files_600k_700k": [0.4, 0.1, 0.4, 0.3],
-        }
-    )
-def test_filter_models(toy_df):
-    df_result = filter_models(toy_df, ["bge-reranker-v2-m3", ])
-    assert len(df_result) == 2
-    assert df_result.iloc[0]["Reranking Model"] == "bge-reranker-v2-m3"
-
-
-def test_search_table(toy_df):
-    df_result = search_table(toy_df, "jina")
-    assert len(df_result) == 2
-    assert df_result.iloc[0]["Retrieval Model"] == "jina-embeddings-v2-base"
-
-
-def test_filter_queries(toy_df):
-    df_result = filter_queries("jina", toy_df)
-    assert len(df_result) == 2
-    assert df_result.iloc[0]["Retrieval Model"] == "jina-embeddings-v2-base"
-
-
-def test_select_columns(toy_df):
-    df_result = select_columns(toy_df, ['news',], ['zh',])
-    assert len(df_result.columns) == 4
-    assert df_result['Average ⬆️'].equals(df_result['news_zh'])
-
-
-def test_update_table_long_doc(toy_df_long_doc):
-    df_result = update_table_long_doc(toy_df_long_doc, ['law',], ['en',], ["bge-reranker-v2-m3", ], "jina")
-    print(df_result)
-
-
-def test_get_iso_format_timestamp():
-    timestamp_config, timestamp_fn = get_iso_format_timestamp()
-    assert len(timestamp_fn) == 14
-    assert len(timestamp_config) == 20
-    assert timestamp_config[-1] == "Z"
-
-
-def test_get_default_cols():
-    cols, types = get_default_cols("qa")
-    for c, t in zip(cols, types):
-        print(f"type({c}): {t}")
-    assert len(frozenset(cols)) == len(cols)
-
-
-def test_update_table():
-    df = pd.DataFrame(
-        {
-            COL_NAME_IS_ANONYMOUS: [False, False, False],
-            COL_NAME_REVISION: ["a1", "a2", "a3"],
-            COL_NAME_TIMESTAMP: ["2024-05-12T12:24:02Z"] * 3,
-            COL_NAME_RERANKING_MODEL: ["NoReranker"] * 3,
-            COL_NAME_RETRIEVAL_MODEL: ["Foo"] * 3,
-            COL_NAME_RANK: [1, 2, 3],
-            COL_NAME_AVG: [0.1, 0.2, 0.3],  # unsorted values
-            "wiki_en": [0.1, 0.2, 0.3]
-        }
-    )
-    results = update_table(df, "wiki", "en", ["NoReranker"], "", show_anonymous=False, reset_ranking=False, show_revision_and_timestamp=False)
-    # keep the RANK as the same regardless of the unsorted averages
-    assert results[COL_NAME_RANK].to_list() == [1, 2, 3]
-

tests/toydata/test_data.json

@@ -1,98 +0,0 @@
-[
-  {
-    "config": {
-      "retrieval_model": "bge-m3",
-      "reranking_model": "bge-reranker-v2-m3",
-      "task": "long_doc",
-      "metric": "ndcg_at_1"
-    },
-    "results": [
-      {
-        "domain": "law",
-        "lang": "en",
-        "dataset": "lex_files_500K-600K",
-        "value": 0.75723
-      }
-    ]
-  },
-  {
-    "config": {
-      "retrieval_model": "bge-m3",
-      "reranking_model": "bge-reranker-v2-m3",
-      "task": "long_doc",
-      "metric": "ndcg_at_3"
-    },
-    "results": [
-      {
-        "domain": "law",
-        "lang": "en",
-        "dataset": "lex_files_500K-600K",
-        "value": 0.69909
-      }
-    ]
-  },
-  {
-    "config": {
-      "retrieval_model": "bge-m3",
-      "reranking_model": "bge-reranker-v2-m3",
-      "task": "qa",
-      "metric": "ndcg_at_1"
-    },
-    "results": [
-      {
-        "domain": "wiki",
-        "lang": "en",
-        "dataset": "unknown",
-        "value": 0.69083
-      }
-    ]
-  },
-  {
-    "config": {
-      "retrieval_model": "bge-m3",
-      "reranking_model": "bge-reranker-v2-m3",
-      "task": "qa",
-      "metric": "ndcg_at_3"
-    },
-    "results": [
-      {
-        "domain": "wiki",
-        "lang": "en",
-        "dataset": "unknown",
-        "value": 0.73359
-      }
-    ]
-  },
-  {
-    "config": {
-      "retrieval_model": "bge-m3",
-      "reranking_model": "bge-reranker-v2-m3",
-      "task": "qa",
-      "metric": "ndcg_at_1"
-    },
-    "results": [
-      {
-        "domain": "wiki",
-        "lang": "zh",
-        "dataset": "unknown",
-        "value": 0.78358
-      }
-    ]
-  },
-  {
-    "config": {
-      "retrieval_model": "bge-m3",
-      "reranking_model": "bge-reranker-v2-m3",
-      "task": "qa",
-      "metric": "ndcg_at_3"
-    },
-    "results": [
-      {
-        "domain": "wiki",
-        "lang": "zh",
-        "dataset": "unknown",
-        "value": 0.78358
-      }
-    ]
-  }
-]

tests/toydata/test_results/bge-m3/NoReranker/results_2023-11-21T18-10-08.json

@@ -1,98 +0,0 @@
-[
-  {
-    "config": {
-      "retrieval_model": "bge-m3",
-      "reranking_model": "NoReranker",
-      "task": "long_doc",
-      "metric": "ndcg_at_1"
-    },
-    "results": [
-      {
-        "domain": "law",
-        "lang": "en",
-        "dataset": "lex_files_500K-600K",
-        "value": 0.45723
-      }
-    ]
-  },
-  {
-    "config": {
-      "retrieval_model": "bge-m3",
-      "reranking_model": "NoReranker",
-      "task": "long_doc",
-      "metric": "ndcg_at_3"
-    },
-    "results": [
-      {
-        "domain": "law",
-        "lang": "en",
-        "dataset": "lex_files_500K-600K",
-        "value": 0.49909
-      }
-    ]
-  },
-  {
-    "config": {
-      "retrieval_model": "bge-m3",
-      "reranking_model": "NoReranker",
-      "task": "qa",
-      "metric": "ndcg_at_1"
-    },
-    "results": [
-      {
-        "domain": "wiki",
-        "lang": "en",
-        "dataset": "unknown",
-        "value": 0.49083
-      }
-    ]
-  },
-  {
-    "config": {
-      "retrieval_model": "bge-m3",
-      "reranking_model": "NoReranker",
-      "task": "qa",
-      "metric": "ndcg_at_3"
-    },
-    "results": [
-      {
-        "domain": "wiki",
-        "lang": "en",
-        "dataset": "unknown",
-        "value": 0.43359
-      }
-    ]
-  },
-  {
-    "config": {
-      "retrieval_model": "bge-m3",
-      "reranking_model": "NoReranker",
-      "task": "qa",
-      "metric": "ndcg_at_1"
-    },
-    "results": [
-      {
-        "domain": "wiki",
-        "lang": "zh",
-        "dataset": "unknown",
-        "value": 0.78358
-      }
-    ]
-  },
-  {
-    "config": {
-      "retrieval_model": "bge-m3",
-      "reranking_model": "NoReranker",
-      "task": "qa",
-      "metric": "ndcg_at_3"
-    },
-    "results": [
-      {
-        "domain": "wiki",
-        "lang": "zh",
-        "dataset": "unknown",
-        "value": 0.78358
-      }
-    ]
-  }
-]

tests/toydata/test_results/bge-m3/bge-reranker-v2-m3/results_2023-11-21T18-10-08.json

@@ -1,98 +0,0 @@
-[
-  {
-    "config": {
-      "retrieval_model": "bge-m3",
-      "reranking_model": "bge-reranker-v2-m3",
-      "task": "long_doc",
-      "metric": "ndcg_at_1"
-    },
-    "results": [
-      {
-        "domain": "law",
-        "lang": "en",
-        "dataset": "lex_files_500K-600K",
-        "value": 0.75723
-      }
-    ]
-  },
-  {
-    "config": {
-      "retrieval_model": "bge-m3",
-      "reranking_model": "bge-reranker-v2-m3",
-      "task": "long_doc",
-      "metric": "ndcg_at_3"
-    },
-    "results": [
-      {
-        "domain": "law",
-        "lang": "en",
-        "dataset": "lex_files_500K-600K",
-        "value": 0.69909
-      }
-    ]
-  },
-  {
-    "config": {
-      "retrieval_model": "bge-m3",
-      "reranking_model": "bge-reranker-v2-m3",
-      "task": "qa",
-      "metric": "ndcg_at_1"
-    },
-    "results": [
-      {
-        "domain": "wiki",
-        "lang": "en",
-        "dataset": "unknown",
-        "value": 0.69083
-      }
-    ]
-  },
-  {
-    "config": {
-      "retrieval_model": "bge-m3",
-      "reranking_model": "bge-reranker-v2-m3",
-      "task": "qa",
-      "metric": "ndcg_at_3"
-    },
-    "results": [
-      {
-        "domain": "wiki",
-        "lang": "en",
-        "dataset": "unknown",
-        "value": 0.73359
-      }
-    ]
-  },
-  {
-    "config": {
-      "retrieval_model": "bge-m3",
-      "reranking_model": "bge-reranker-v2-m3",
-      "task": "qa",
-      "metric": "ndcg_at_1"
-    },
-    "results": [
-      {
-        "domain": "wiki",
-        "lang": "zh",
-        "dataset": "unknown",
-        "value": 0.78358
-      }
-    ]
-  },
-  {
-    "config": {
-      "retrieval_model": "bge-m3",
-      "reranking_model": "bge-reranker-v2-m3",
-      "task": "qa",
-      "metric": "ndcg_at_3"
-    },
-    "results": [
-      {
-        "domain": "wiki",
-        "lang": "zh",
-        "dataset": "unknown",
-        "value": 0.78358
-      }
-    ]
-  }
-]