initial commit

.gitignore

@@ -0,0 +1 @@
+results

README.md

@@ -3,10 +3,10 @@ title: Test
 emoji: 🔥
 colorFrom: blue
 colorTo: green
-sdk: gradio
-sdk_version: 3.2
+sdk: streamlit
+sdk_version: 1.10.0
 app_file: app.py
 pinned: false
 ---
 
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+Check out the configuration reference at <https://huggingface.co/docs/hub/spaces-config-reference>

app.py

@@ -0,0 +1,75 @@
+import numpy as np
+import pandas as pd
+import requests
+import streamlit as st
+from tasks import TASKS
+from huggingface_hub import HfApi
+import datasets
+import os
+
+FORMATTED_TASK_NAMES = TASKS
+api = HfApi()
+
+def download_submissions():
+    submissions = api.list_datasets(
+        filter=("benchmark:mteb"), use_auth_token=os.getenv("HF_HUB_TOKEN")
+    )
+    return submissions
+
+def format_submissions(submissions):
+    submission_data = {**{"Submitter": []}, **{"Submission Name": []}, **{"Submission Date": []}, **{t: [] for t in TASKS}}
+    # The following picks the latest submissions which adhere to the model card schema
+    for submission in submissions:
+        submission_id = submission.id
+        try:
+            data = list(datasets.load_dataset(submission_id, streaming=True, data_files="*csv").values())[0]
+        except FileNotFoundError:
+            print("FileNotFoundError")
+            continue
+        for line in data:
+            if line['dataset'] in submission_data:
+                submission_data[line['dataset']].append(line['value'])
+            if len(submission_data['Submission Name']) == 0 or line['model'] != submission_data['Submission Name'][-1]:
+                submission_data['Submission Name'].append(line['model'])
+        while len(submission_data['Submission Name']) > len(submission_data['Submitter']):
+            submission_data['Submitter'].append(submission.author)
+            submission_data["Submission Date"].append(pd.to_datetime(submission.lastModified).strftime("%b %d, %Y"))
+
+
+    df = pd.DataFrame(submission_data)
+    df.insert(3, "Overall", df[TASKS].mean(axis=1))
+    df = df.copy().sort_values("Overall", ascending=False)
+    df.rename(columns={k: v for k, v in zip(TASKS, FORMATTED_TASK_NAMES)}, inplace=True)
+    # Start ranking from 1
+    df.insert(0, "Rank", np.arange(1, len(df) + 1))
+    return df
+
+
+###########
+### APP ###
+###########
+st.set_page_config(layout="wide")
+st.title("MTEB: Massive Text Embedding Benchmark")
+st.markdown(
+    """
+    ...
+"""
+)
+submissions = download_submissions()
+df = format_submissions(submissions)
+styler = df.style.set_precision(3).set_properties(**{"white-space": "pre-wrap", "text-align": "center"})
+# hack to remove index column: https://discuss.streamlit.io/t/questions-on-st-table/6878/3
+st.markdown(
+    """
+<style>
+table td:nth-child(1) {
+    display: none
+}
+table th:nth-child(1) {
+    display: none
+}
+</style>
+""",
+    unsafe_allow_html=True,
+)
+st.table(styler)

tasks.py

@@ -0,0 +1,82 @@
+TASK_LIST_CLASSIFICATION = [
+    "AmazonCounterfactualClassification",
+    "AmazonPolarityClassification",
+    "AmazonReviewsClassification",
+    "Banking77Classification",
+    "EmotionClassification",
+    "ImdbClassification",
+    "MassiveIntentClassification",
+    "MassiveScenarioClassification",
+    "MTOPDomainClassification",
+    "MTOPIntentClassification",
+    "ToxicConversationsClassification",
+    "TweetSentimentExtractionClassification",
+]
+
+TASK_LIST_CLUSTERING = [
+    "ArxivClusteringP2P",
+    "ArxivClusteringS2S",
+    "BiorxivClusteringP2P",
+    "BiorxivClusteringS2S",
+    "MedrxivClusteringP2P",
+    "MedrxivClusteringS2S",
+    "RedditClustering",
+    "RedditClusteringP2P",
+    "StackExchangeClustering",
+    "StackExchangeClusteringP2P",
+    "TwentyNewsgroupsClustering",
+]
+
+TASK_LIST_PAIR_CLASSIFICATION = [
+    "SprintDuplicateQuestions",
+    "TwitterSemEval2015",
+    "TwitterURLCorpus",
+]
+
+TASK_LIST_RERANKING = [
+    "AskUbuntuDupQuestions",
+    "MindSmallReranking",
+    "SciDocsRR",
+    "StackOverflowDupQuestions",
+]
+
+TASK_LIST_RETRIEVAL = [
+    "ArguAna",
+    "ClimateFEVER",
+    "CQADupstackRetrieval",
+    "DBPedia",
+    "FEVER",
+    "FiQA2018",
+    "HotpotQA",
+    "MSMARCO",
+    "NFCorpus",
+    "NQ",
+    "QuoraRetrieval",
+    "SCIDOCS",
+    "SciFact",
+    "Touche2020",
+    "TRECCOVID",
+]
+
+TASK_LIST_STS = [
+    "BIOSSES",
+    "SICK-R",
+    "STS12",
+    "STS13",
+    "STS14",
+    "STS15",
+    "STS16",
+    "STS17",
+    "STS22",
+    "STSBenchmark",
+    "SummEval",
+]
+
+TASKS = (
+    TASK_LIST_CLASSIFICATION
+    + TASK_LIST_CLUSTERING
+    + TASK_LIST_PAIR_CLASSIFICATION
+    + TASK_LIST_RERANKING
+    + TASK_LIST_RETRIEVAL
+    + TASK_LIST_STS
+)