Better model with bs=256

README.md

@@ -14,7 +14,7 @@ For usage instructions, refer to our codebase: https://github.com/Muennighoff/sg
 
 ## Evaluation Results
 
-For eval results, refer to our paper: https://arxiv.org/abs/2202.08904
+For eval results, refer to the eval folder or our paper: https://arxiv.org/abs/2202.08904
 
 ## Training
 The model was trained with the parameters:

config.json

@@ -67,7 +67,7 @@
   },
   "tokenizer_class": "GPT2Tokenizer",
   "torch_dtype": "float32",
-  "transformers_version": "4.11.3",
+  "transformers_version": "4.20.0.dev0",
   "use_cache": true,
   "vocab_size": 50259,
   "window_size": 256

config_sentence_transformers.json

@@ -1,7 +1,7 @@
 {
   "__version__": {
     "sentence_transformers": "2.1.0",
-    "transformers": "4.11.3",
-    "pytorch": "1.10.1"
+    "transformers": "4.20.0.dev0",
+    "pytorch": "1.10.2"
   }
 }

eval/beir.json

@@ -0,0 +1,276 @@
+{
+    "SGPT-1.3B-weightedmean-msmarco-specb-bitfit": {
+        "cqadupstack_android": {
+            "NDCG@1": 0.29185,
+            "NDCG@3": 0.32944,
+            "NDCG@5": 0.35929,
+            "NDCG@10": 0.38723,
+            "NDCG@100": 0.44145,
+            "NDCG@1000": 0.46656
+        },
+        "cqadupstack_english": {
+            "NDCG@1": 0.30764,
+            "NDCG@3": 0.35197,
+            "NDCG@5": 0.37105,
+            "NDCG@10": 0.39067,
+            "NDCG@100": 0.43427,
+            "NDCG@1000": 0.4586
+        },
+        "cqadupstack_gaming": {
+            "NDCG@1": 0.35987,
+            "NDCG@3": 0.42514,
+            "NDCG@5": 0.44992,
+            "NDCG@10": 0.47974,
+            "NDCG@100": 0.52508,
+            "NDCG@1000": 0.54051
+        },
+        "cqadupstack_gis": {
+            "NDCG@1": 0.2,
+            "NDCG@3": 0.24039,
+            "NDCG@5": 0.26361,
+            "NDCG@10": 0.28866,
+            "NDCG@100": 0.34303,
+            "NDCG@1000": 0.36929
+        },
+        "cqadupstack_mathematica": {
+            "NDCG@1": 0.1393,
+            "NDCG@3": 0.18372,
+            "NDCG@5": 0.20196,
+            "NDCG@10": 0.23076,
+            "NDCG@100": 0.28674,
+            "NDCG@1000": 0.32014
+        },
+        "cqadupstack_physics": {
+            "NDCG@1": 0.26756,
+            "NDCG@3": 0.3194,
+            "NDCG@5": 0.34338,
+            "NDCG@10": 0.3694,
+            "NDCG@100": 0.42578,
+            "NDCG@1000": 0.45088
+        },
+        "cqadupstack_programmers": {
+            "NDCG@1": 0.25799,
+            "NDCG@3": 0.29731,
+            "NDCG@5": 0.32163,
+            "NDCG@10": 0.34766,
+            "NDCG@100": 0.4065,
+            "NDCG@1000": 0.43174
+        },
+        "cqadupstack_stats": {
+            "NDCG@1": 0.18712,
+            "NDCG@3": 0.22737,
+            "NDCG@5": 0.24022,
+            "NDCG@10": 0.25908,
+            "NDCG@100": 0.31008,
+            "NDCG@1000": 0.3382
+        },
+        "cqadupstack_wordpress": {
+            "NDCG@1": 0.17745,
+            "NDCG@3": 0.208,
+            "NDCG@5": 0.22922,
+            "NDCG@10": 0.249,
+            "NDCG@100": 0.30064,
+            "NDCG@1000": 0.33151
+        },
+        "cqadupstack_webmasters": {
+            "NDCG@1": 0.23715,
+            "NDCG@3": 0.28019,
+            "NDCG@5": 0.30105,
+            "NDCG@10": 0.32438,
+            "NDCG@100": 0.38253,
+            "NDCG@1000": 0.41166
+        },
+        "cqadupstack_unix": {
+            "NDCG@1": 0.22761,
+            "NDCG@3": 0.27169,
+            "NDCG@5": 0.28825,
+            "NDCG@10": 0.31209,
+            "NDCG@100": 0.36454,
+            "NDCG@1000": 0.39443
+        },
+        "cqadupstack_tex": {
+            "NDCG@1": 0.1404,
+            "NDCG@3": 0.16974,
+            "NDCG@5": 0.18636,
+            "NDCG@10": 0.20547,
+            "NDCG@100": 0.25368,
+            "NDCG@1000": 0.28579
+        },
+        "cqadupstack": {
+            "NDCG@1": 0.2328283333333333,
+            "NDCG@3": 0.27536333333333335,
+            "NDCG@5": 0.2963283333333333,
+            "NDCG@10": 0.320345,
+            "NDCG@100": 0.37285999999999997,
+            "NDCG@1000": 0.3999425
+        },
+        "trec-covid": {
+            "NDCG@1": 0.88,
+            "NDCG@3": 0.84693,
+            "NDCG@5": 0.82035,
+            "NDCG@10": 0.78527,
+            "NDCG@100": 0.55017,
+            "NDCG@1000": 0.45909
+        },
+        "trec-news": {
+            "NDCG@1": 0.47515,
+            "NDCG@3": 0.4709,
+            "NDCG@5": 0.44046,
+            "NDCG@10": 0.42399,
+            "NDCG@100": 0.4355,
+            "NDCG@1000": 0.53977
+        },
+        "signal1m": {
+            "NDCG@1": 0.40722,
+            "NDCG@3": 0.32186,
+            "NDCG@5": 0.29096,
+            "NDCG@10": 0.25005,
+            "NDCG@100": 0.26756,
+            "NDCG@1000": 0.32664
+        },
+        "robust04": {
+            "NDCG@1": 0.54418,
+            "NDCG@3": 0.49172,
+            "NDCG@5": 0.46235,
+            "NDCG@10": 0.421,
+            "NDCG@100": 0.34684,
+            "NDCG@1000": 0.43416
+        },
+        "nfcorpus": {
+            "NDCG@1": 0.4195,
+            "NDCG@3": 0.3794,
+            "NDCG@5": 0.35456,
+            "NDCG@10": 0.32101,
+            "NDCG@100": 0.30051,
+            "NDCG@1000": 0.38669
+        },
+        "msmarco": {
+            "NDCG@1": 0.19427,
+            "NDCG@3": 0.28633,
+            "NDCG@5": 0.32367,
+            "NDCG@10": 0.3605,
+            "NDCG@100": 0.42155,
+            "NDCG@1000": 0.43777
+        },
+        "nq": {
+            "NDCG@1": 0.26738,
+            "NDCG@3": 0.3542,
+            "NDCG@5": 0.39376,
+            "NDCG@10": 0.42945,
+            "NDCG@100": 0.48308,
+            "NDCG@1000": 0.49665
+        },
+        "average": {
+            "NDCG@1": 0.43378157407407403,
+            "NDCG@3": 0.42776351851851846,
+            "NDCG@5": 0.42827157407407407,
+            "NDCG@10": 0.4332547222222223,
+            "NDCG@100": 0.4522683333333333,
+            "NDCG@1000": 0.48897124999999997
+        },
+        "subaverage": {
+            "NDCG@1": 0.48321,
+            "NDCG@3": 0.4798981818181818,
+            "NDCG@5": 0.48613909090909097,
+            "NDCG@10": 0.4950427272727273,
+            "NDCG@100": 0.51009,
+            "NDCG@1000": 0.5356790909090909
+        },
+        "subsubaverage": {
+            "NDCG@1": 0.342878,
+            "NDCG@3": 0.365358,
+            "NDCG@5": 0.372162,
+            "NDCG@10": 0.39230800000000005,
+            "NDCG@100": 0.42882799999999993,
+            "NDCG@1000": 0.46609199999999995
+        },
+        "bioasq": {
+            "NDCG@1": 0.376,
+            "NDCG@3": 0.34795,
+            "NDCG@5": 0.34274,
+            "NDCG@10": 0.34726,
+            "NDCG@100": 0.39707,
+            "NDCG@1000": 0.43191
+        },
+        "climate-fever": {
+            "NDCG@1": 0.22997,
+            "NDCG@3": 0.20755,
+            "NDCG@5": 0.22802,
+            "NDCG@10": 0.26636,
+            "NDCG@100": 0.33596,
+            "NDCG@1000": 0.36954
+        },
+        "dbpedia-entity": {
+            "NDCG@1": 0.43125,
+            "NDCG@3": 0.35102,
+            "NDCG@5": 0.33354,
+            "NDCG@10": 0.31476,
+            "NDCG@100": 0.34557,
+            "NDCG@1000": 0.41241
+        },
+        "webis-touche2020": {
+            "NDCG@1": 0.30612,
+            "NDCG@3": 0.28528,
+            "NDCG@5": 0.25768,
+            "NDCG@10": 0.2445,
+            "NDCG@100": 0.35871,
+            "NDCG@1000": 0.47273
+        },
+        "scifact": {
+            "NDCG@1": 0.57,
+            "NDCG@3": 0.62656,
+            "NDCG@5": 0.65523,
+            "NDCG@10": 0.68287,
+            "NDCG@100": 0.70985,
+            "NDCG@1000": 0.71695
+        },
+        "scidocs": {
+            "NDCG@1": 0.19,
+            "NDCG@3": 0.15887,
+            "NDCG@5": 0.13476,
+            "NDCG@10": 0.16102,
+            "NDCG@100": 0.22693,
+            "NDCG@1000": 0.27994
+        },
+        "fiqa": {
+            "NDCG@1": 0.27315,
+            "NDCG@3": 0.26342,
+            "NDCG@5": 0.2768,
+            "NDCG@10": 0.29987,
+            "NDCG@100": 0.3641,
+            "NDCG@1000": 0.40185
+        },
+        "arguana": {
+            "NDCG@1": 0.26174,
+            "NDCG@3": 0.39854,
+            "NDCG@5": 0.43946,
+            "NDCG@10": 0.49677,
+            "NDCG@100": 0.54275,
+            "NDCG@1000": 0.54503
+        },
+        "quora": {
+            "NDCG@1": 0.7756,
+            "NDCG@3": 0.81975,
+            "NDCG@5": 0.83738,
+            "NDCG@10": 0.85293,
+            "NDCG@100": 0.86816,
+            "NDCG@1000": 0.87021
+        },
+        "hotpotqa": {
+            "NDCG@1": 0.61742,
+            "NDCG@3": 0.4611,
+            "NDCG@5": 0.48147,
+            "NDCG@10": 0.49923,
+            "NDCG@100": 0.53178,
+            "NDCG@1000": 0.54947
+        },
+        "fever": {
+            "NDCG@1": 0.55056,
+            "NDCG@3": 0.63933,
+            "NDCG@5": 0.66304,
+            "NDCG@10": 0.6819,
+            "NDCG@100": 0.70343,
+            "NDCG@1000": 0.7085
+        }
+    }
+}

pytorch_model.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:70c84fed2517a31d4ff19880d4b6ec22644f2be2a617e8f53b68019f59a8bb72
-size 5363113217
+oid sha256:71d5e1599703e7c4795a70cef20fb032d9c252bf27a99a7b3edc7d3bb13b09a2
+size 5363097985

tokenizer_config.json

@@ -1 +1 @@
-{"unk_token": {"content": "<|endoftext|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "__type": "AddedToken"}, "bos_token": {"content": "<|endoftext|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "__type": "AddedToken"}, "eos_token": {"content": "<|endoftext|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "__type": "AddedToken"}, "add_prefix_space": false, "model_max_length": 2048, "special_tokens_map_file": null, "name_or_path": "EleutherAI/gpt-neo-1.3B", "errors": "replace", "tokenizer_class": "GPT2Tokenizer"}
+{"unk_token": {"content": "<|endoftext|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "__type": "AddedToken"}, "bos_token": {"content": "<|endoftext|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "__type": "AddedToken"}, "eos_token": {"content": "<|endoftext|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "__type": "AddedToken"}, "add_prefix_space": false, "model_max_length": 2048, "special_tokens_map_file": null, "name_or_path": "EleutherAI/gpt-neo-1.3B", "errors": "replace", "pad_token": null, "add_bos_token": false, "tokenizer_class": "GPT2Tokenizer"}