Hugging Face's logo

memray
/
AugTriever-TQGen-AbSum

PyTorch
Model card Files Community
memray commited on
Commit
e1b3d32
•
1 Parent(s): 69be45b

Upload 130 files

Browse files
This view is limited to 50 files because it contains too many changes.   See raw diff
Files changed (50) hide show
  1. .gitattributes +4 -0
  2. beir_output/arguana.csv +8 -0
  3. beir_output/arguana.json +86 -0
  4. beir_output/beir.json +26 -0
  5. beir_output/climate-fever.csv +8 -0
  6. beir_output/climate-fever.json +86 -0
  7. beir_output/cqadupstack.csv +8 -0
  8. beir_output/cqadupstack.json +86 -0
  9. beir_output/dbpedia-entity.csv +8 -0
  10. beir_output/dbpedia-entity.json +86 -0
  11. beir_output/fever.csv +8 -0
  12. beir_output/fever.json +86 -0
  13. beir_output/fiqa.csv +8 -0
  14. beir_output/fiqa.json +86 -0
  15. beir_output/hotpotqa.csv +8 -0
  16. beir_output/hotpotqa.json +86 -0
  17. beir_output/msmarco.csv +8 -0
  18. beir_output/msmarco.json +86 -0
  19. beir_output/nfcorpus.csv +8 -0
  20. beir_output/nfcorpus.json +86 -0
  21. beir_output/nq.csv +8 -0
  22. beir_output/nq.json +86 -0
  23. beir_output/quora.csv +8 -0
  24. beir_output/quora.json +86 -0
  25. beir_output/scidocs.csv +8 -0
  26. beir_output/scidocs.json +86 -0
  27. beir_output/scifact.csv +8 -0
  28. beir_output/scifact.json +86 -0
  29. beir_output/senteval-core.json +5 -0
  30. beir_output/senteval.json +99 -0
  31. beir_output/trec-covid.csv +8 -0
  32. beir_output/trec-covid.json +86 -0
  33. beir_output/webis-touche2020.csv +8 -0
  34. beir_output/webis-touche2020.json +86 -0
  35. cc.moco.absum50.bs2048.gpu8.sh +29 -0
  36. eval_output/.DS_Store +0 -0
  37. eval_output/checkpoint-100000/arguana.csv +8 -0
  38. eval_output/checkpoint-100000/arguana.json +86 -0
  39. eval_output/checkpoint-100000/beir.json +46 -0
  40. eval_output/checkpoint-100000/cqadupstack.csv +8 -0
  41. eval_output/checkpoint-100000/cqadupstack.json +86 -0
  42. eval_output/checkpoint-100000/fiqa.csv +8 -0
  43. eval_output/checkpoint-100000/fiqa.json +86 -0
  44. eval_output/checkpoint-100000/nfcorpus.csv +8 -0
  45. eval_output/checkpoint-100000/nfcorpus.json +86 -0
  46. eval_output/checkpoint-100000/nq.csv +8 -0
  47. eval_output/checkpoint-100000/nq.json +86 -0
  48. eval_output/checkpoint-100000/quora.csv +8 -0
  49. eval_output/checkpoint-100000/quora.json +86 -0
  50. eval_output/checkpoint-100000/scidocs.csv +8 -0
.gitattributes CHANGED
@@ -32,3 +32,7 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
32
  *.zip filter=lfs diff=lfs merge=lfs -text
33
  *.zst filter=lfs diff=lfs merge=lfs -text
34
  *tfevents* filter=lfs diff=lfs merge=lfs -text
 
 
 
 
 
32
  *.zip filter=lfs diff=lfs merge=lfs -text
33
  *.zst filter=lfs diff=lfs merge=lfs -text
34
  *tfevents* filter=lfs diff=lfs merge=lfs -text
35
+ qa_output/nq-test/results.jsonl filter=lfs diff=lfs merge=lfs -text
36
+ qa_output/squad1-test/results.jsonl filter=lfs diff=lfs merge=lfs -text
37
+ qa_output/trivia-test/results.jsonl filter=lfs diff=lfs merge=lfs -text
38
+ qa_output/webq-test/results.jsonl filter=lfs diff=lfs merge=lfs -text
beir_output/arguana.csv ADDED
@@ -0,0 +1,8 @@
 
 
 
 
 
 
 
 
 
1
+ metric,@1,@3,@5,@10,@20,@50,@100,@200,@1000
2
+ ndcg,0.16358,0.27535,0.32609,0.38653,0.41902,0.43899,0.44613,0.44978,0.45272
3
+ map,0.16358,0.24656,0.27459,0.2995,0.30857,0.31185,0.31248,0.31266,0.31273
4
+ precision,0.16358,0.11972,0.09659,0.067,0.03986,0.01794,0.00941,0.00484,0.00099
5
+ recall,0.16358,0.35917,0.48293,0.66999,0.7973,0.89687,0.94097,0.96728,0.99289
6
+ mrr,0.16927,0.24846,0.27723,0.30182,0.31095,0.31424,0.31486,0.31504,0.31511
7
+ recall_cap,0.16927,0.35704,0.48435,0.66927,0.7973,0.89687,0.94097,0.96728,0.99289
8
+ hole,0.69488,0.68563,0.68777,0.68563,0.72461,0.8025,0.84213,0.86538,0.87804
beir_output/arguana.json ADDED
@@ -0,0 +1,86 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "dataset": "arguana",
3
+ "split": "test",
4
+ "metric": "dot",
5
+ "norm_query": false,
6
+ "norm_doc": false,
7
+ "scores": {
8
+ "ndcg": {
9
+ "NDCG@1": 0.16358,
10
+ "NDCG@3": 0.27535,
11
+ "NDCG@5": 0.32609,
12
+ "NDCG@10": 0.38653,
13
+ "NDCG@20": 0.41902,
14
+ "NDCG@50": 0.43899,
15
+ "NDCG@100": 0.44613,
16
+ "NDCG@200": 0.44978,
17
+ "NDCG@1000": 0.45272
18
+ },
19
+ "map": {
20
+ "MAP@1": 0.16358,
21
+ "MAP@3": 0.24656,
22
+ "MAP@5": 0.27459,
23
+ "MAP@10": 0.2995,
24
+ "MAP@20": 0.30857,
25
+ "MAP@50": 0.31185,
26
+ "MAP@100": 0.31248,
27
+ "MAP@200": 0.31266,
28
+ "MAP@1000": 0.31273
29
+ },
30
+ "precision": {
31
+ "P@1": 0.16358,
32
+ "P@3": 0.11972,
33
+ "P@5": 0.09659,
34
+ "P@10": 0.067,
35
+ "P@20": 0.03986,
36
+ "P@50": 0.01794,
37
+ "P@100": 0.00941,
38
+ "P@200": 0.00484,
39
+ "P@1000": 0.00099
40
+ },
41
+ "recall": {
42
+ "Recall@1": 0.16358,
43
+ "Recall@3": 0.35917,
44
+ "Recall@5": 0.48293,
45
+ "Recall@10": 0.66999,
46
+ "Recall@20": 0.7973,
47
+ "Recall@50": 0.89687,
48
+ "Recall@100": 0.94097,
49
+ "Recall@200": 0.96728,
50
+ "Recall@1000": 0.99289
51
+ },
52
+ "mrr": {
53
+ "MRR@1": 0.16927,
54
+ "MRR@3": 0.24846,
55
+ "MRR@5": 0.27723,
56
+ "MRR@10": 0.30182,
57
+ "MRR@20": 0.31095,
58
+ "MRR@50": 0.31424,
59
+ "MRR@100": 0.31486,
60
+ "MRR@200": 0.31504,
61
+ "MRR@1000": 0.31511
62
+ },
63
+ "recall_cap": {
64
+ "R_cap@1": 0.16927,
65
+ "R_cap@3": 0.35704,
66
+ "R_cap@5": 0.48435,
67
+ "R_cap@10": 0.66927,
68
+ "R_cap@20": 0.7973,
69
+ "R_cap@50": 0.89687,
70
+ "R_cap@100": 0.94097,
71
+ "R_cap@200": 0.96728,
72
+ "R_cap@1000": 0.99289
73
+ },
74
+ "hole": {
75
+ "Hole@1": 0.69488,
76
+ "Hole@3": 0.68563,
77
+ "Hole@5": 0.68777,
78
+ "Hole@10": 0.68563,
79
+ "Hole@20": 0.72461,
80
+ "Hole@50": 0.8025,
81
+ "Hole@100": 0.84213,
82
+ "Hole@200": 0.86538,
83
+ "Hole@1000": 0.87804
84
+ }
85
+ }
86
+ }
beir_output/beir.json ADDED
@@ -0,0 +1,26 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "eval_beir-msmarco_ndcg@10": 0.23291,
3
+ "eval_beir-msmarco_recall@10": 0.39965,
4
+ "eval_beir-msmarco_recall@20": 0.50404,
5
+ "eval_beir-msmarco_recall@100": 0.70873,
6
+ "eval_beir-dbpedia-entity_ndcg@10": 0.31098,
7
+ "eval_beir-dbpedia-entity_recall@10": 0.20823,
8
+ "eval_beir-dbpedia-entity_recall@20": 0.26649,
9
+ "eval_beir-dbpedia-entity_recall@100": 0.45154,
10
+ "eval_beir-fever_ndcg@10": 0.63587,
11
+ "eval_beir-fever_recall@10": 0.80061,
12
+ "eval_beir-fever_recall@20": 0.85087,
13
+ "eval_beir-fever_recall@100": 0.91485,
14
+ "eval_beir-climate-fever_ndcg@10": 0.17804,
15
+ "eval_beir-climate-fever_recall@10": 0.21169,
16
+ "eval_beir-climate-fever_recall@20": 0.27416,
17
+ "eval_beir-climate-fever_recall@100": 0.47264,
18
+ "eval_beir-hotpotqa_ndcg@10": 0.5397,
19
+ "eval_beir-hotpotqa_recall@10": 0.56394,
20
+ "eval_beir-hotpotqa_recall@20": 0.6106,
21
+ "eval_beir-hotpotqa_recall@100": 0.70702,
22
+ "eval_beir-avg_ndcg@10": 0.3795,
23
+ "eval_beir-avg_recall@10": 0.436824,
24
+ "eval_beir-avg_recall@20": 0.501232,
25
+ "eval_beir-avg_recall@100": 0.6509560000000001
26
+ }
beir_output/climate-fever.csv ADDED
@@ -0,0 +1,8 @@
 
 
 
 
 
 
 
 
 
1
+ metric,@1,@3,@5,@10,@20,@50,@100,@200,@1000
2
+ ndcg,0.16743,0.14972,0.15913,0.17804,0.19992,0.22738,0.24917,0.26591,0.29405
3
+ map,0.07941,0.10945,0.11807,0.12604,0.13227,0.13731,0.13978,0.14106,0.14203
4
+ precision,0.16743,0.10793,0.08091,0.05309,0.03583,0.02021,0.01302,0.00783,0.00212
5
+ recall,0.07941,0.13864,0.16938,0.21169,0.27416,0.37543,0.47264,0.55744,0.72948
6
+ mrr,0.16743,0.22215,0.23609,0.24606,0.2536,0.2579,0.25925,0.25972,0.25998
7
+ recall_cap,0.16743,0.14864,0.16938,0.21169,0.27416,0.37543,0.47264,0.55744,0.72948
8
+ hole,0.54332,0.60673,0.64274,0.6916,0.72743,0.77664,0.81063,0.84452,0.91304
beir_output/climate-fever.json ADDED
@@ -0,0 +1,86 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "dataset": "climate-fever",
3
+ "split": "test",
4
+ "metric": "dot",
5
+ "norm_query": false,
6
+ "norm_doc": false,
7
+ "scores": {
8
+ "ndcg": {
9
+ "NDCG@1": 0.16743,
10
+ "NDCG@3": 0.14972,
11
+ "NDCG@5": 0.15913,
12
+ "NDCG@10": 0.17804,
13
+ "NDCG@20": 0.19992,
14
+ "NDCG@50": 0.22738,
15
+ "NDCG@100": 0.24917,
16
+ "NDCG@200": 0.26591,
17
+ "NDCG@1000": 0.29405
18
+ },
19
+ "map": {
20
+ "MAP@1": 0.07941,
21
+ "MAP@3": 0.10945,
22
+ "MAP@5": 0.11807,
23
+ "MAP@10": 0.12604,
24
+ "MAP@20": 0.13227,
25
+ "MAP@50": 0.13731,
26
+ "MAP@100": 0.13978,
27
+ "MAP@200": 0.14106,
28
+ "MAP@1000": 0.14203
29
+ },
30
+ "precision": {
31
+ "P@1": 0.16743,
32
+ "P@3": 0.10793,
33
+ "P@5": 0.08091,
34
+ "P@10": 0.05309,
35
+ "P@20": 0.03583,
36
+ "P@50": 0.02021,
37
+ "P@100": 0.01302,
38
+ "P@200": 0.00783,
39
+ "P@1000": 0.00212
40
+ },
41
+ "recall": {
42
+ "Recall@1": 0.07941,
43
+ "Recall@3": 0.13864,
44
+ "Recall@5": 0.16938,
45
+ "Recall@10": 0.21169,
46
+ "Recall@20": 0.27416,
47
+ "Recall@50": 0.37543,
48
+ "Recall@100": 0.47264,
49
+ "Recall@200": 0.55744,
50
+ "Recall@1000": 0.72948
51
+ },
52
+ "mrr": {
53
+ "MRR@1": 0.16743,
54
+ "MRR@3": 0.22215,
55
+ "MRR@5": 0.23609,
56
+ "MRR@10": 0.24606,
57
+ "MRR@20": 0.2536,
58
+ "MRR@50": 0.2579,
59
+ "MRR@100": 0.25925,
60
+ "MRR@200": 0.25972,
61
+ "MRR@1000": 0.25998
62
+ },
63
+ "recall_cap": {
64
+ "R_cap@1": 0.16743,
65
+ "R_cap@3": 0.14864,
66
+ "R_cap@5": 0.16938,
67
+ "R_cap@10": 0.21169,
68
+ "R_cap@20": 0.27416,
69
+ "R_cap@50": 0.37543,
70
+ "R_cap@100": 0.47264,
71
+ "R_cap@200": 0.55744,
72
+ "R_cap@1000": 0.72948
73
+ },
74
+ "hole": {
75
+ "Hole@1": 0.54332,
76
+ "Hole@3": 0.60673,
77
+ "Hole@5": 0.64274,
78
+ "Hole@10": 0.6916,
79
+ "Hole@20": 0.72743,
80
+ "Hole@50": 0.77664,
81
+ "Hole@100": 0.81063,
82
+ "Hole@200": 0.84452,
83
+ "Hole@1000": 0.91304
84
+ }
85
+ }
86
+ }
beir_output/cqadupstack.csv ADDED
@@ -0,0 +1,8 @@
 
 
 
 
 
 
 
 
 
1
+ metric,@1,@3,@5,@10,@20,@50,@100,@200,@1000
2
+ ndcg,0.21238333333333334,0.25052250000000004,0.26944,0.2928333333333334,0.31152583333333334,0.3318433333333333,0.3445375,0.35541416666666664,0.37370333333333333
3
+ map,0.17943833333333337,0.22491000000000003,0.23740666666666665,0.24825000000000005,0.2539983333333333,0.25779500000000005,0.2591683333333334,0.2598941666666667,0.2604566666666667
4
+ precision,0.21238333333333334,0.11569750000000001,0.08366416666666666,0.05258166666666667,0.032051666666666666,0.016111666666666666,0.009380833333333335,0.0053683333333333335,0.0013716666666666667
5
+ recall,0.17943833333333337,0.2751608333333333,0.32309249999999995,0.3925783333333334,0.4612349999999999,0.5543025,0.6243891666666667,0.6937216666666668,0.8333191666666667
6
+ mrr,0.21238333333333334,0.26051166666666664,0.2725008333333333,0.28266749999999996,0.2877675,0.290765,0.29172166666666666,0.2921791666666667,0.29249583333333334
7
+ recall_cap,0.21238333333333334,0.28125416666666664,0.32585833333333336,0.39377083333333335,0.4617558333333332,0.5544891666666666,0.6244641666666667,0.6937291666666666,0.8333191666666667
8
+ hole,0.7077300000000001,0.7993166666666666,0.8298199999999999,0.8623316666666666,0.8835099999999999,0.9013991666666669,0.9103258333333333,0.9166075,0.925735
beir_output/cqadupstack.json ADDED
@@ -0,0 +1,86 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "dataset": "cqadupstack",
3
+ "split": "test",
4
+ "metric": "dot",
5
+ "norm_query": false,
6
+ "norm_doc": false,
7
+ "scores": {
8
+ "ndcg": {
9
+ "NDCG@1": 0.21238333333333334,
10
+ "NDCG@3": 0.25052250000000004,
11
+ "NDCG@5": 0.26944,
12
+ "NDCG@10": 0.2928333333333334,
13
+ "NDCG@20": 0.31152583333333334,
14
+ "NDCG@50": 0.3318433333333333,
15
+ "NDCG@100": 0.3445375,
16
+ "NDCG@200": 0.35541416666666664,
17
+ "NDCG@1000": 0.37370333333333333
18
+ },
19
+ "map": {
20
+ "MAP@1": 0.17943833333333337,
21
+ "MAP@3": 0.22491000000000003,
22
+ "MAP@5": 0.23740666666666665,
23
+ "MAP@10": 0.24825000000000005,
24
+ "MAP@20": 0.2539983333333333,
25
+ "MAP@50": 0.25779500000000005,
26
+ "MAP@100": 0.2591683333333334,
27
+ "MAP@200": 0.2598941666666667,
28
+ "MAP@1000": 0.2604566666666667
29
+ },
30
+ "precision": {
31
+ "P@1": 0.21238333333333334,
32
+ "P@3": 0.11569750000000001,
33
+ "P@5": 0.08366416666666666,
34
+ "P@10": 0.05258166666666667,
35
+ "P@20": 0.032051666666666666,
36
+ "P@50": 0.016111666666666666,
37
+ "P@100": 0.009380833333333335,
38
+ "P@200": 0.0053683333333333335,
39
+ "P@1000": 0.0013716666666666667
40
+ },
41
+ "recall": {
42
+ "Recall@1": 0.17943833333333337,
43
+ "Recall@3": 0.2751608333333333,
44
+ "Recall@5": 0.32309249999999995,
45
+ "Recall@10": 0.3925783333333334,
46
+ "Recall@20": 0.4612349999999999,
47
+ "Recall@50": 0.5543025,
48
+ "Recall@100": 0.6243891666666667,
49
+ "Recall@200": 0.6937216666666668,
50
+ "Recall@1000": 0.8333191666666667
51
+ },
52
+ "mrr": {
53
+ "MRR@1": 0.21238333333333334,
54
+ "MRR@3": 0.26051166666666664,
55
+ "MRR@5": 0.2725008333333333,
56
+ "MRR@10": 0.28266749999999996,
57
+ "MRR@20": 0.2877675,
58
+ "MRR@50": 0.290765,
59
+ "MRR@100": 0.29172166666666666,
60
+ "MRR@200": 0.2921791666666667,
61
+ "MRR@1000": 0.29249583333333334
62
+ },
63
+ "recall_cap": {
64
+ "R_cap@1": 0.21238333333333334,
65
+ "R_cap@3": 0.28125416666666664,
66
+ "R_cap@5": 0.32585833333333336,
67
+ "R_cap@10": 0.39377083333333335,
68
+ "R_cap@20": 0.4617558333333332,
69
+ "R_cap@50": 0.5544891666666666,
70
+ "R_cap@100": 0.6244641666666667,
71
+ "R_cap@200": 0.6937291666666666,
72
+ "R_cap@1000": 0.8333191666666667
73
+ },
74
+ "hole": {
75
+ "Hole@1": 0.7077300000000001,
76
+ "Hole@3": 0.7993166666666666,
77
+ "Hole@5": 0.8298199999999999,
78
+ "Hole@10": 0.8623316666666666,
79
+ "Hole@20": 0.8835099999999999,
80
+ "Hole@50": 0.9013991666666669,
81
+ "Hole@100": 0.9103258333333333,
82
+ "Hole@200": 0.9166075,
83
+ "Hole@1000": 0.925735
84
+ }
85
+ }
86
+ }
beir_output/dbpedia-entity.csv ADDED
@@ -0,0 +1,8 @@
 
 
 
 
 
 
 
 
 
1
+ metric,@1,@3,@5,@10,@20,@50,@100,@200,@1000
2
+ ndcg,0.37625,0.33357,0.32254,0.31098,0.31096,0.3355,0.36694,0.39383,0.44367
3
+ map,0.0666,0.10441,0.12252,0.14517,0.16753,0.19349,0.20652,0.21367,0.22134
4
+ precision,0.49,0.37583,0.3225,0.24875,0.19162,0.1265,0.08685,0.05551,0.01938
5
+ recall,0.0666,0.12451,0.15934,0.20823,0.26649,0.3662,0.45154,0.53488,0.70457
6
+ mrr,0.49,0.59,0.60338,0.60992,0.61379,0.61554,0.61576,0.61586,0.61592
7
+ recall_cap,0.49,0.3975,0.36588,0.33175,0.32552,0.37796,0.45692,0.53814,0.70522
8
+ hole,0.3275,0.4025,0.452,0.531,0.61462,0.7237,0.79307,0.85231,0.93407
beir_output/dbpedia-entity.json ADDED
@@ -0,0 +1,86 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "dataset": "dbpedia-entity",
3
+ "split": "test",
4
+ "metric": "dot",
5
+ "norm_query": false,
6
+ "norm_doc": false,
7
+ "scores": {
8
+ "ndcg": {
9
+ "NDCG@1": 0.37625,
10
+ "NDCG@3": 0.33357,
11
+ "NDCG@5": 0.32254,
12
+ "NDCG@10": 0.31098,
13
+ "NDCG@20": 0.31096,
14
+ "NDCG@50": 0.3355,
15
+ "NDCG@100": 0.36694,
16
+ "NDCG@200": 0.39383,
17
+ "NDCG@1000": 0.44367
18
+ },
19
+ "map": {
20
+ "MAP@1": 0.0666,
21
+ "MAP@3": 0.10441,
22
+ "MAP@5": 0.12252,
23
+ "MAP@10": 0.14517,
24
+ "MAP@20": 0.16753,
25
+ "MAP@50": 0.19349,
26
+ "MAP@100": 0.20652,
27
+ "MAP@200": 0.21367,
28
+ "MAP@1000": 0.22134
29
+ },
30
+ "precision": {
31
+ "P@1": 0.49,
32
+ "P@3": 0.37583,
33
+ "P@5": 0.3225,
34
+ "P@10": 0.24875,
35
+ "P@20": 0.19162,
36
+ "P@50": 0.1265,
37
+ "P@100": 0.08685,
38
+ "P@200": 0.05551,
39
+ "P@1000": 0.01938
40
+ },
41
+ "recall": {
42
+ "Recall@1": 0.0666,
43
+ "Recall@3": 0.12451,
44
+ "Recall@5": 0.15934,
45
+ "Recall@10": 0.20823,
46
+ "Recall@20": 0.26649,
47
+ "Recall@50": 0.3662,
48
+ "Recall@100": 0.45154,
49
+ "Recall@200": 0.53488,
50
+ "Recall@1000": 0.70457
51
+ },
52
+ "mrr": {
53
+ "MRR@1": 0.49,
54
+ "MRR@3": 0.59,
55
+ "MRR@5": 0.60338,
56
+ "MRR@10": 0.60992,
57
+ "MRR@20": 0.61379,
58
+ "MRR@50": 0.61554,
59
+ "MRR@100": 0.61576,
60
+ "MRR@200": 0.61586,
61
+ "MRR@1000": 0.61592
62
+ },
63
+ "recall_cap": {
64
+ "R_cap@1": 0.49,
65
+ "R_cap@3": 0.3975,
66
+ "R_cap@5": 0.36588,
67
+ "R_cap@10": 0.33175,
68
+ "R_cap@20": 0.32552,
69
+ "R_cap@50": 0.37796,
70
+ "R_cap@100": 0.45692,
71
+ "R_cap@200": 0.53814,
72
+ "R_cap@1000": 0.70522
73
+ },
74
+ "hole": {
75
+ "Hole@1": 0.3275,
76
+ "Hole@3": 0.4025,
77
+ "Hole@5": 0.452,
78
+ "Hole@10": 0.531,
79
+ "Hole@20": 0.61462,
80
+ "Hole@50": 0.7237,
81
+ "Hole@100": 0.79307,
82
+ "Hole@200": 0.85231,
83
+ "Hole@1000": 0.93407
84
+ }
85
+ }
86
+ }
beir_output/fever.csv ADDED
@@ -0,0 +1,8 @@
 
 
 
 
 
 
 
 
 
1
+ metric,@1,@3,@5,@10,@20,@50,@100,@200,@1000
2
+ ndcg,0.4808,0.58408,0.61253,0.63587,0.64918,0.65852,0.66196,0.66398,0.66756
3
+ map,0.44897,0.54478,0.56123,0.57125,0.57513,0.5768,0.57715,0.57728,0.57738
4
+ precision,0.4808,0.23817,0.15845,0.08714,0.04658,0.01973,0.01013,0.00517,0.00108
5
+ recall,0.44897,0.6609,0.73008,0.80061,0.85087,0.89514,0.91485,0.92795,0.95532
6
+ mrr,0.4808,0.58093,0.59743,0.60719,0.61072,0.61213,0.61238,0.61245,0.61251
7
+ recall_cap,0.4808,0.66227,0.73036,0.80062,0.85087,0.89514,0.91485,0.92795,0.95532
8
+ hole,0.4877,0.73212,0.81578,0.89344,0.93963,0.9723,0.98453,0.99112,0.99706
beir_output/fever.json ADDED
@@ -0,0 +1,86 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "dataset": "fever",
3
+ "split": "test",
4
+ "metric": "dot",
5
+ "norm_query": false,
6
+ "norm_doc": false,
7
+ "scores": {
8
+ "ndcg": {
9
+ "NDCG@1": 0.4808,
10
+ "NDCG@3": 0.58408,
11
+ "NDCG@5": 0.61253,
12
+ "NDCG@10": 0.63587,
13
+ "NDCG@20": 0.64918,
14
+ "NDCG@50": 0.65852,
15
+ "NDCG@100": 0.66196,
16
+ "NDCG@200": 0.66398,
17
+ "NDCG@1000": 0.66756
18
+ },
19
+ "map": {
20
+ "MAP@1": 0.44897,
21
+ "MAP@3": 0.54478,
22
+ "MAP@5": 0.56123,
23
+ "MAP@10": 0.57125,
24
+ "MAP@20": 0.57513,
25
+ "MAP@50": 0.5768,
26
+ "MAP@100": 0.57715,
27
+ "MAP@200": 0.57728,
28
+ "MAP@1000": 0.57738
29
+ },
30
+ "precision": {
31
+ "P@1": 0.4808,
32
+ "P@3": 0.23817,
33
+ "P@5": 0.15845,
34
+ "P@10": 0.08714,
35
+ "P@20": 0.04658,
36
+ "P@50": 0.01973,
37
+ "P@100": 0.01013,
38
+ "P@200": 0.00517,
39
+ "P@1000": 0.00108
40
+ },
41
+ "recall": {
42
+ "Recall@1": 0.44897,
43
+ "Recall@3": 0.6609,
44
+ "Recall@5": 0.73008,
45
+ "Recall@10": 0.80061,
46
+ "Recall@20": 0.85087,
47
+ "Recall@50": 0.89514,
48
+ "Recall@100": 0.91485,
49
+ "Recall@200": 0.92795,
50
+ "Recall@1000": 0.95532
51
+ },
52
+ "mrr": {
53
+ "MRR@1": 0.4808,
54
+ "MRR@3": 0.58093,
55
+ "MRR@5": 0.59743,
56
+ "MRR@10": 0.60719,
57
+ "MRR@20": 0.61072,
58
+ "MRR@50": 0.61213,
59
+ "MRR@100": 0.61238,
60
+ "MRR@200": 0.61245,
61
+ "MRR@1000": 0.61251
62
+ },
63
+ "recall_cap": {
64
+ "R_cap@1": 0.4808,
65
+ "R_cap@3": 0.66227,
66
+ "R_cap@5": 0.73036,
67
+ "R_cap@10": 0.80062,
68
+ "R_cap@20": 0.85087,
69
+ "R_cap@50": 0.89514,
70
+ "R_cap@100": 0.91485,
71
+ "R_cap@200": 0.92795,
72
+ "R_cap@1000": 0.95532
73
+ },
74
+ "hole": {
75
+ "Hole@1": 0.4877,
76
+ "Hole@3": 0.73212,
77
+ "Hole@5": 0.81578,
78
+ "Hole@10": 0.89344,
79
+ "Hole@20": 0.93963,
80
+ "Hole@50": 0.9723,
81
+ "Hole@100": 0.98453,
82
+ "Hole@200": 0.99112,
83
+ "Hole@1000": 0.99706
84
+ }
85
+ }
86
+ }
beir_output/fiqa.csv ADDED
@@ -0,0 +1,8 @@
 
 
 
 
 
 
 
 
 
1
+ metric,@1,@3,@5,@10,@20,@50,@100,@200,@1000
2
+ ndcg,0.24537,0.22733,0.23826,0.26763,0.29383,0.32214,0.33978,0.35479,0.37946
3
+ map,0.1295,0.17366,0.18773,0.20398,0.21291,0.21873,0.22082,0.22202,0.22294
4
+ precision,0.24537,0.14712,0.1108,0.07593,0.04838,0.02519,0.01481,0.00853,0.00219
5
+ recall,0.1295,0.20816,0.25341,0.33485,0.41649,0.52773,0.6111,0.69263,0.85218
6
+ mrr,0.24537,0.29475,0.30818,0.32292,0.32917,0.33263,0.33371,0.33411,0.33432
7
+ recall_cap,0.24537,0.23174,0.26019,0.33527,0.41649,0.52773,0.6111,0.69263,0.85218
8
+ hole,0.69599,0.79424,0.82654,0.86466,0.88935,0.91136,0.92133,0.92652,0.93546
beir_output/fiqa.json ADDED
@@ -0,0 +1,86 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "dataset": "fiqa",
3
+ "split": "test",
4
+ "metric": "dot",
5
+ "norm_query": false,
6
+ "norm_doc": false,
7
+ "scores": {
8
+ "ndcg": {
9
+ "NDCG@1": 0.24537,
10
+ "NDCG@3": 0.22733,
11
+ "NDCG@5": 0.23826,
12
+ "NDCG@10": 0.26763,
13
+ "NDCG@20": 0.29383,
14
+ "NDCG@50": 0.32214,
15
+ "NDCG@100": 0.33978,
16
+ "NDCG@200": 0.35479,
17
+ "NDCG@1000": 0.37946
18
+ },
19
+ "map": {
20
+ "MAP@1": 0.1295,
21
+ "MAP@3": 0.17366,
22
+ "MAP@5": 0.18773,
23
+ "MAP@10": 0.20398,
24
+ "MAP@20": 0.21291,
25
+ "MAP@50": 0.21873,
26
+ "MAP@100": 0.22082,
27
+ "MAP@200": 0.22202,
28
+ "MAP@1000": 0.22294
29
+ },
30
+ "precision": {
31
+ "P@1": 0.24537,
32
+ "P@3": 0.14712,
33
+ "P@5": 0.1108,
34
+ "P@10": 0.07593,
35
+ "P@20": 0.04838,
36
+ "P@50": 0.02519,
37
+ "P@100": 0.01481,
38
+ "P@200": 0.00853,
39
+ "P@1000": 0.00219
40
+ },
41
+ "recall": {
42
+ "Recall@1": 0.1295,
43
+ "Recall@3": 0.20816,
44
+ "Recall@5": 0.25341,
45
+ "Recall@10": 0.33485,
46
+ "Recall@20": 0.41649,
47
+ "Recall@50": 0.52773,
48
+ "Recall@100": 0.6111,
49
+ "Recall@200": 0.69263,
50
+ "Recall@1000": 0.85218
51
+ },
52
+ "mrr": {
53
+ "MRR@1": 0.24537,
54
+ "MRR@3": 0.29475,
55
+ "MRR@5": 0.30818,
56
+ "MRR@10": 0.32292,
57
+ "MRR@20": 0.32917,
58
+ "MRR@50": 0.33263,
59
+ "MRR@100": 0.33371,
60
+ "MRR@200": 0.33411,
61
+ "MRR@1000": 0.33432
62
+ },
63
+ "recall_cap": {
64
+ "R_cap@1": 0.24537,
65
+ "R_cap@3": 0.23174,
66
+ "R_cap@5": 0.26019,
67
+ "R_cap@10": 0.33527,
68
+ "R_cap@20": 0.41649,
69
+ "R_cap@50": 0.52773,
70
+ "R_cap@100": 0.6111,
71
+ "R_cap@200": 0.69263,
72
+ "R_cap@1000": 0.85218
73
+ },
74
+ "hole": {
75
+ "Hole@1": 0.69599,
76
+ "Hole@3": 0.79424,
77
+ "Hole@5": 0.82654,
78
+ "Hole@10": 0.86466,
79
+ "Hole@20": 0.88935,
80
+ "Hole@50": 0.91136,
81
+ "Hole@100": 0.92133,
82
+ "Hole@200": 0.92652,
83
+ "Hole@1000": 0.93546
84
+ }
85
+ }
86
+ }
beir_output/hotpotqa.csv ADDED
@@ -0,0 +1,8 @@
 
 
 
 
 
 
 
 
 
1
+ metric,@1,@3,@5,@10,@20,@50,@100,@200,@1000
2
+ ndcg,0.63862,0.49334,0.51824,0.5397,0.55425,0.56808,0.57607,0.5818,0.59216
3
+ map,0.31931,0.41375,0.43127,0.44279,0.4482,0.45138,0.4524,0.45283,0.45317
4
+ precision,0.63862,0.30722,0.204,0.11279,0.06106,0.02668,0.01414,0.0074,0.00163
5
+ recall,0.31931,0.46084,0.50999,0.56394,0.6106,0.66705,0.70702,0.74045,0.81431
6
+ mrr,0.63862,0.7007,0.71086,0.71684,0.71919,0.7202,0.72049,0.7206,0.72065
7
+ recall_cap,0.63862,0.46084,0.50999,0.56394,0.6106,0.66705,0.70702,0.74045,0.81431
8
+ hole,0.35665,0.68368,0.78617,0.87699,0.92882,0.96392,0.97707,0.98433,0.99142
beir_output/hotpotqa.json ADDED
@@ -0,0 +1,86 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "dataset": "hotpotqa",
3
+ "split": "test",
4
+ "metric": "dot",
5
+ "norm_query": false,
6
+ "norm_doc": false,
7
+ "scores": {
8
+ "ndcg": {
9
+ "NDCG@1": 0.63862,
10
+ "NDCG@3": 0.49334,
11
+ "NDCG@5": 0.51824,
12
+ "NDCG@10": 0.5397,
13
+ "NDCG@20": 0.55425,
14
+ "NDCG@50": 0.56808,
15
+ "NDCG@100": 0.57607,
16
+ "NDCG@200": 0.5818,
17
+ "NDCG@1000": 0.59216
18
+ },
19
+ "map": {
20
+ "MAP@1": 0.31931,
21
+ "MAP@3": 0.41375,
22
+ "MAP@5": 0.43127,
23
+ "MAP@10": 0.44279,
24
+ "MAP@20": 0.4482,
25
+ "MAP@50": 0.45138,
26
+ "MAP@100": 0.4524,
27
+ "MAP@200": 0.45283,
28
+ "MAP@1000": 0.45317
29
+ },
30
+ "precision": {
31
+ "P@1": 0.63862,
32
+ "P@3": 0.30722,
33
+ "P@5": 0.204,
34
+ "P@10": 0.11279,
35
+ "P@20": 0.06106,
36
+ "P@50": 0.02668,
37
+ "P@100": 0.01414,
38
+ "P@200": 0.0074,
39
+ "P@1000": 0.00163
40
+ },
41
+ "recall": {
42
+ "Recall@1": 0.31931,
43
+ "Recall@3": 0.46084,
44
+ "Recall@5": 0.50999,
45
+ "Recall@10": 0.56394,
46
+ "Recall@20": 0.6106,
47
+ "Recall@50": 0.66705,
48
+ "Recall@100": 0.70702,
49
+ "Recall@200": 0.74045,
50
+ "Recall@1000": 0.81431
51
+ },
52
+ "mrr": {
53
+ "MRR@1": 0.63862,
54
+ "MRR@3": 0.7007,
55
+ "MRR@5": 0.71086,
56
+ "MRR@10": 0.71684,
57
+ "MRR@20": 0.71919,
58
+ "MRR@50": 0.7202,
59
+ "MRR@100": 0.72049,
60
+ "MRR@200": 0.7206,
61
+ "MRR@1000": 0.72065
62
+ },
63
+ "recall_cap": {
64
+ "R_cap@1": 0.63862,
65
+ "R_cap@3": 0.46084,
66
+ "R_cap@5": 0.50999,
67
+ "R_cap@10": 0.56394,
68
+ "R_cap@20": 0.6106,
69
+ "R_cap@50": 0.66705,
70
+ "R_cap@100": 0.70702,
71
+ "R_cap@200": 0.74045,
72
+ "R_cap@1000": 0.81431
73
+ },
74
+ "hole": {
75
+ "Hole@1": 0.35665,
76
+ "Hole@3": 0.68368,
77
+ "Hole@5": 0.78617,
78
+ "Hole@10": 0.87699,
79
+ "Hole@20": 0.92882,
80
+ "Hole@50": 0.96392,
81
+ "Hole@100": 0.97707,
82
+ "Hole@200": 0.98433,
83
+ "Hole@1000": 0.99142
84
+ }
85
+ }
86
+ }
beir_output/msmarco.csv ADDED
@@ -0,0 +1,8 @@
 
 
 
 
 
 
 
 
 
1
+ metric,@1,@3,@5,@10,@20,@50,@100,@200,@1000
2
+ ndcg,0.09785,0.16617,0.1991,0.23291,0.2597,0.28403,0.29794,0.30908,0.32387
3
+ map,0.09563,0.14746,0.1658,0.17988,0.18732,0.19133,0.19257,0.19317,0.19352
4
+ precision,0.09785,0.07459,0.06138,0.04155,0.0263,0.0131,0.00745,0.00415,0.00097
5
+ recall,0.09563,0.21692,0.29617,0.39965,0.50404,0.62426,0.70873,0.78668,0.91224
6
+ mrr,0.09785,0.15107,0.16939,0.18328,0.19065,0.19455,0.19576,0.19632,0.19665
7
+ recall_cap,0.09785,0.21707,0.29617,0.39965,0.50404,0.62426,0.70873,0.78668,0.91224
8
+ hole,0.90086,0.92407,0.93756,0.95732,0.97256,0.9858,0.9914,0.99474,0.99799
beir_output/msmarco.json ADDED
@@ -0,0 +1,86 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "dataset": "msmarco",
3
+ "split": "dev",
4
+ "metric": "dot",
5
+ "norm_query": false,
6
+ "norm_doc": false,
7
+ "scores": {
8
+ "ndcg": {
9
+ "NDCG@1": 0.09785,
10
+ "NDCG@3": 0.16617,
11
+ "NDCG@5": 0.1991,
12
+ "NDCG@10": 0.23291,
13
+ "NDCG@20": 0.2597,
14
+ "NDCG@50": 0.28403,
15
+ "NDCG@100": 0.29794,
16
+ "NDCG@200": 0.30908,
17
+ "NDCG@1000": 0.32387
18
+ },
19
+ "map": {
20
+ "MAP@1": 0.09563,
21
+ "MAP@3": 0.14746,
22
+ "MAP@5": 0.1658,
23
+ "MAP@10": 0.17988,
24
+ "MAP@20": 0.18732,
25
+ "MAP@50": 0.19133,
26
+ "MAP@100": 0.19257,
27
+ "MAP@200": 0.19317,
28
+ "MAP@1000": 0.19352
29
+ },
30
+ "precision": {
31
+ "P@1": 0.09785,
32
+ "P@3": 0.07459,
33
+ "P@5": 0.06138,
34
+ "P@10": 0.04155,
35
+ "P@20": 0.0263,
36
+ "P@50": 0.0131,
37
+ "P@100": 0.00745,
38
+ "P@200": 0.00415,
39
+ "P@1000": 0.00097
40
+ },
41
+ "recall": {
42
+ "Recall@1": 0.09563,
43
+ "Recall@3": 0.21692,
44
+ "Recall@5": 0.29617,
45
+ "Recall@10": 0.39965,
46
+ "Recall@20": 0.50404,
47
+ "Recall@50": 0.62426,
48
+ "Recall@100": 0.70873,
49
+ "Recall@200": 0.78668,
50
+ "Recall@1000": 0.91224
51
+ },
52
+ "mrr": {
53
+ "MRR@1": 0.09785,
54
+ "MRR@3": 0.15107,
55
+ "MRR@5": 0.16939,
56
+ "MRR@10": 0.18328,
57
+ "MRR@20": 0.19065,
58
+ "MRR@50": 0.19455,
59
+ "MRR@100": 0.19576,
60
+ "MRR@200": 0.19632,
61
+ "MRR@1000": 0.19665
62
+ },
63
+ "recall_cap": {
64
+ "R_cap@1": 0.09785,
65
+ "R_cap@3": 0.21707,
66
+ "R_cap@5": 0.29617,
67
+ "R_cap@10": 0.39965,
68
+ "R_cap@20": 0.50404,
69
+ "R_cap@50": 0.62426,
70
+ "R_cap@100": 0.70873,
71
+ "R_cap@200": 0.78668,
72
+ "R_cap@1000": 0.91224
73
+ },
74
+ "hole": {
75
+ "Hole@1": 0.90086,
76
+ "Hole@3": 0.92407,
77
+ "Hole@5": 0.93756,
78
+ "Hole@10": 0.95732,
79
+ "Hole@20": 0.97256,
80
+ "Hole@50": 0.9858,
81
+ "Hole@100": 0.9914,
82
+ "Hole@200": 0.99474,
83
+ "Hole@1000": 0.99799
84
+ }
85
+ }
86
+ }
beir_output/nfcorpus.csv ADDED
@@ -0,0 +1,8 @@
 
 
 
 
 
 
 
 
 
1
+ metric,@1,@3,@5,@10,@20,@50,@100,@200,@1000
2
+ ndcg,0.37926,0.34843,0.32746,0.29881,0.27649,0.26898,0.2763,0.29387,0.36529
3
+ map,0.04368,0.07888,0.09235,0.10625,0.11699,0.12725,0.13272,0.13745,0.14607
4
+ precision,0.39628,0.3323,0.28483,0.21858,0.16037,0.10433,0.07133,0.04858,0.02022
5
+ recall,0.04368,0.09208,0.11936,0.15184,0.17956,0.23425,0.28269,0.35094,0.59763
6
+ mrr,0.40248,0.47575,0.49154,0.50011,0.50126,0.50372,0.50427,0.50457,0.50477
7
+ recall_cap,0.40248,0.34881,0.32255,0.28298,0.25421,0.26373,0.29608,0.3556,0.59763
8
+ hole,0.05882,0.07843,0.0904,0.09907,0.11285,0.12322,0.12424,0.12882,0.1328
beir_output/nfcorpus.json ADDED
@@ -0,0 +1,86 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "dataset": "nfcorpus",
3
+ "split": "test",
4
+ "metric": "dot",
5
+ "norm_query": false,
6
+ "norm_doc": false,
7
+ "scores": {
8
+ "ndcg": {
9
+ "NDCG@1": 0.37926,
10
+ "NDCG@3": 0.34843,
11
+ "NDCG@5": 0.32746,
12
+ "NDCG@10": 0.29881,
13
+ "NDCG@20": 0.27649,
14
+ "NDCG@50": 0.26898,
15
+ "NDCG@100": 0.2763,
16
+ "NDCG@200": 0.29387,
17
+ "NDCG@1000": 0.36529
18
+ },
19
+ "map": {
20
+ "MAP@1": 0.04368,
21
+ "MAP@3": 0.07888,
22
+ "MAP@5": 0.09235,
23
+ "MAP@10": 0.10625,
24
+ "MAP@20": 0.11699,
25
+ "MAP@50": 0.12725,
26
+ "MAP@100": 0.13272,
27
+ "MAP@200": 0.13745,
28
+ "MAP@1000": 0.14607
29
+ },
30
+ "precision": {
31
+ "P@1": 0.39628,
32
+ "P@3": 0.3323,
33
+ "P@5": 0.28483,
34
+ "P@10": 0.21858,
35
+ "P@20": 0.16037,
36
+ "P@50": 0.10433,
37
+ "P@100": 0.07133,
38
+ "P@200": 0.04858,
39
+ "P@1000": 0.02022
40
+ },
41
+ "recall": {
42
+ "Recall@1": 0.04368,
43
+ "Recall@3": 0.09208,
44
+ "Recall@5": 0.11936,
45
+ "Recall@10": 0.15184,
46
+ "Recall@20": 0.17956,
47
+ "Recall@50": 0.23425,
48
+ "Recall@100": 0.28269,
49
+ "Recall@200": 0.35094,
50
+ "Recall@1000": 0.59763
51
+ },
52
+ "mrr": {
53
+ "MRR@1": 0.40248,
54
+ "MRR@3": 0.47575,
55
+ "MRR@5": 0.49154,
56
+ "MRR@10": 0.50011,
57
+ "MRR@20": 0.50126,
58
+ "MRR@50": 0.50372,
59
+ "MRR@100": 0.50427,
60
+ "MRR@200": 0.50457,
61
+ "MRR@1000": 0.50477
62
+ },
63
+ "recall_cap": {
64
+ "R_cap@1": 0.40248,
65
+ "R_cap@3": 0.34881,
66
+ "R_cap@5": 0.32255,
67
+ "R_cap@10": 0.28298,
68
+ "R_cap@20": 0.25421,
69
+ "R_cap@50": 0.26373,
70
+ "R_cap@100": 0.29608,
71
+ "R_cap@200": 0.3556,
72
+ "R_cap@1000": 0.59763
73
+ },
74
+ "hole": {
75
+ "Hole@1": 0.05882,
76
+ "Hole@3": 0.07843,
77
+ "Hole@5": 0.0904,
78
+ "Hole@10": 0.09907,
79
+ "Hole@20": 0.11285,
80
+ "Hole@50": 0.12322,
81
+ "Hole@100": 0.12424,
82
+ "Hole@200": 0.12882,
83
+ "Hole@1000": 0.1328
84
+ }
85
+ }
86
+ }
beir_output/nq.csv ADDED
@@ -0,0 +1,8 @@
 
 
 
 
 
 
 
 
 
1
+ metric,@1,@3,@5,@10,@20,@50,@100,@200,@1000
2
+ ndcg,0.14977,0.21812,0.25843,0.29837,0.33082,0.35938,0.37296,0.38187,0.39152
3
+ map,0.13222,0.19374,0.21665,0.23398,0.24349,0.24859,0.24986,0.25038,0.25064
4
+ precision,0.14977,0.10361,0.0847,0.05678,0.03596,0.01772,0.00985,0.00531,0.00116
5
+ recall,0.13222,0.26794,0.36233,0.48047,0.60291,0.73877,0.81825,0.87768,0.95597
6
+ mrr,0.15006,0.21461,0.23693,0.25261,0.26078,0.26498,0.26609,0.26649,0.26668
7
+ recall_cap,0.15006,0.26801,0.36233,0.48047,0.60291,0.73877,0.81825,0.87768,0.95597
8
+ hole,0.84791,0.89446,0.91321,0.94041,0.96088,0.97933,0.9873,0.99177,0.99618
beir_output/nq.json ADDED
@@ -0,0 +1,86 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "dataset": "nq",
3
+ "split": "test",
4
+ "metric": "dot",
5
+ "norm_query": false,
6
+ "norm_doc": false,
7
+ "scores": {
8
+ "ndcg": {
9
+ "NDCG@1": 0.14977,
10
+ "NDCG@3": 0.21812,
11
+ "NDCG@5": 0.25843,
12
+ "NDCG@10": 0.29837,
13
+ "NDCG@20": 0.33082,
14
+ "NDCG@50": 0.35938,
15
+ "NDCG@100": 0.37296,
16
+ "NDCG@200": 0.38187,
17
+ "NDCG@1000": 0.39152
18
+ },
19
+ "map": {
20
+ "MAP@1": 0.13222,
21
+ "MAP@3": 0.19374,
22
+ "MAP@5": 0.21665,
23
+ "MAP@10": 0.23398,
24
+ "MAP@20": 0.24349,
25
+ "MAP@50": 0.24859,
26
+ "MAP@100": 0.24986,
27
+ "MAP@200": 0.25038,
28
+ "MAP@1000": 0.25064
29
+ },
30
+ "precision": {
31
+ "P@1": 0.14977,
32
+ "P@3": 0.10361,
33
+ "P@5": 0.0847,
34
+ "P@10": 0.05678,
35
+ "P@20": 0.03596,
36
+ "P@50": 0.01772,
37
+ "P@100": 0.00985,
38
+ "P@200": 0.00531,
39
+ "P@1000": 0.00116
40
+ },
41
+ "recall": {
42
+ "Recall@1": 0.13222,
43
+ "Recall@3": 0.26794,
44
+ "Recall@5": 0.36233,
45
+ "Recall@10": 0.48047,
46
+ "Recall@20": 0.60291,
47
+ "Recall@50": 0.73877,
48
+ "Recall@100": 0.81825,
49
+ "Recall@200": 0.87768,
50
+ "Recall@1000": 0.95597
51
+ },
52
+ "mrr": {
53
+ "MRR@1": 0.15006,
54
+ "MRR@3": 0.21461,
55
+ "MRR@5": 0.23693,
56
+ "MRR@10": 0.25261,
57
+ "MRR@20": 0.26078,
58
+ "MRR@50": 0.26498,
59
+ "MRR@100": 0.26609,
60
+ "MRR@200": 0.26649,
61
+ "MRR@1000": 0.26668
62
+ },
63
+ "recall_cap": {
64
+ "R_cap@1": 0.15006,
65
+ "R_cap@3": 0.26801,
66
+ "R_cap@5": 0.36233,
67
+ "R_cap@10": 0.48047,
68
+ "R_cap@20": 0.60291,
69
+ "R_cap@50": 0.73877,
70
+ "R_cap@100": 0.81825,
71
+ "R_cap@200": 0.87768,
72
+ "R_cap@1000": 0.95597
73
+ },
74
+ "hole": {
75
+ "Hole@1": 0.84791,
76
+ "Hole@3": 0.89446,
77
+ "Hole@5": 0.91321,
78
+ "Hole@10": 0.94041,
79
+ "Hole@20": 0.96088,
80
+ "Hole@50": 0.97933,
81
+ "Hole@100": 0.9873,
82
+ "Hole@200": 0.99177,
83
+ "Hole@1000": 0.99618
84
+ }
85
+ }
86
+ }
beir_output/quora.csv ADDED
@@ -0,0 +1,8 @@
 
 
 
 
 
 
 
 
 
1
+ metric,@1,@3,@5,@10,@20,@50,@100,@200,@1000
2
+ ndcg,0.6801,0.73881,0.76011,0.78119,0.79347,0.80197,0.80506,0.80666,0.80835
3
+ map,0.59232,0.69716,0.7166,0.7303,0.73595,0.73848,0.73903,0.73922,0.73933
4
+ precision,0.6801,0.32447,0.21682,0.12183,0.0662,0.0285,0.01474,0.00753,0.00156
5
+ recall,0.59232,0.77203,0.82842,0.88917,0.93078,0.96438,0.97876,0.98715,0.99725
6
+ mrr,0.6798,0.74905,0.7596,0.76606,0.76819,0.76893,0.76907,0.7691,0.76912
7
+ recall_cap,0.6798,0.78502,0.83355,0.89063,0.93106,0.9644,0.97876,0.98715,0.99725
8
+ hole,0.2983,0.6418,0.74436,0.83671,0.89097,0.92861,0.94355,0.9524,0.96129
beir_output/quora.json ADDED
@@ -0,0 +1,86 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "dataset": "quora",
3
+ "split": "test",
4
+ "metric": "dot",
5
+ "norm_query": false,
6
+ "norm_doc": false,
7
+ "scores": {
8
+ "ndcg": {
9
+ "NDCG@1": 0.6801,
10
+ "NDCG@3": 0.73881,
11
+ "NDCG@5": 0.76011,
12
+ "NDCG@10": 0.78119,
13
+ "NDCG@20": 0.79347,
14
+ "NDCG@50": 0.80197,
15
+ "NDCG@100": 0.80506,
16
+ "NDCG@200": 0.80666,
17
+ "NDCG@1000": 0.80835
18
+ },
19
+ "map": {
20
+ "MAP@1": 0.59232,
21
+ "MAP@3": 0.69716,
22
+ "MAP@5": 0.7166,
23
+ "MAP@10": 0.7303,
24
+ "MAP@20": 0.73595,
25
+ "MAP@50": 0.73848,
26
+ "MAP@100": 0.73903,
27
+ "MAP@200": 0.73922,
28
+ "MAP@1000": 0.73933
29
+ },
30
+ "precision": {
31
+ "P@1": 0.6801,
32
+ "P@3": 0.32447,
33
+ "P@5": 0.21682,
34
+ "P@10": 0.12183,
35
+ "P@20": 0.0662,
36
+ "P@50": 0.0285,
37
+ "P@100": 0.01474,
38
+ "P@200": 0.00753,
39
+ "P@1000": 0.00156
40
+ },
41
+ "recall": {
42
+ "Recall@1": 0.59232,
43
+ "Recall@3": 0.77203,
44
+ "Recall@5": 0.82842,
45
+ "Recall@10": 0.88917,
46
+ "Recall@20": 0.93078,
47
+ "Recall@50": 0.96438,
48
+ "Recall@100": 0.97876,
49
+ "Recall@200": 0.98715,
50
+ "Recall@1000": 0.99725
51
+ },
52
+ "mrr": {
53
+ "MRR@1": 0.6798,
54
+ "MRR@3": 0.74905,
55
+ "MRR@5": 0.7596,
56
+ "MRR@10": 0.76606,
57
+ "MRR@20": 0.76819,
58
+ "MRR@50": 0.76893,
59
+ "MRR@100": 0.76907,
60
+ "MRR@200": 0.7691,
61
+ "MRR@1000": 0.76912
62
+ },
63
+ "recall_cap": {
64
+ "R_cap@1": 0.6798,
65
+ "R_cap@3": 0.78502,
66
+ "R_cap@5": 0.83355,
67
+ "R_cap@10": 0.89063,
68
+ "R_cap@20": 0.93106,
69
+ "R_cap@50": 0.9644,
70
+ "R_cap@100": 0.97876,
71
+ "R_cap@200": 0.98715,
72
+ "R_cap@1000": 0.99725
73
+ },
74
+ "hole": {
75
+ "Hole@1": 0.2983,
76
+ "Hole@3": 0.6418,
77
+ "Hole@5": 0.74436,
78
+ "Hole@10": 0.83671,
79
+ "Hole@20": 0.89097,
80
+ "Hole@50": 0.92861,
81
+ "Hole@100": 0.94355,
82
+ "Hole@200": 0.9524,
83
+ "Hole@1000": 0.96129
84
+ }
85
+ }
86
+ }
beir_output/scidocs.csv ADDED
@@ -0,0 +1,8 @@
 
 
 
 
 
 
 
 
 
1
+ metric,@1,@3,@5,@10,@20,@50,@100,@200,@1000
2
+ ndcg,0.196,0.16049,0.13802,0.16627,0.19205,0.21944,0.23828,0.25562,0.29203
3
+ map,0.03988,0.07149,0.08285,0.09675,0.10559,0.11173,0.11416,0.11562,0.11705
4
+ precision,0.196,0.15,0.1198,0.0855,0.05765,0.03108,0.01894,0.01128,0.00318
5
+ recall,0.03988,0.09123,0.12133,0.17298,0.23343,0.31485,0.38372,0.45717,0.64643
6
+ mrr,0.196,0.26317,0.28087,0.2931,0.30043,0.30405,0.3052,0.30566,0.30591
7
+ recall_cap,0.196,0.15,0.12133,0.17298,0.23343,0.31485,0.38372,0.45717,0.64643
8
+ hole,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
beir_output/scidocs.json ADDED
@@ -0,0 +1,86 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "dataset": "scidocs",
3
+ "split": "test",
4
+ "metric": "dot",
5
+ "norm_query": false,
6
+ "norm_doc": false,
7
+ "scores": {
8
+ "ndcg": {
9
+ "NDCG@1": 0.196,
10
+ "NDCG@3": 0.16049,
11
+ "NDCG@5": 0.13802,
12
+ "NDCG@10": 0.16627,
13
+ "NDCG@20": 0.19205,
14
+ "NDCG@50": 0.21944,
15
+ "NDCG@100": 0.23828,
16
+ "NDCG@200": 0.25562,
17
+ "NDCG@1000": 0.29203
18
+ },
19
+ "map": {
20
+ "MAP@1": 0.03988,
21
+ "MAP@3": 0.07149,
22
+ "MAP@5": 0.08285,
23
+ "MAP@10": 0.09675,
24
+ "MAP@20": 0.10559,
25
+ "MAP@50": 0.11173,
26
+ "MAP@100": 0.11416,
27
+ "MAP@200": 0.11562,
28
+ "MAP@1000": 0.11705
29
+ },
30
+ "precision": {
31
+ "P@1": 0.196,
32
+ "P@3": 0.15,
33
+ "P@5": 0.1198,
34
+ "P@10": 0.0855,
35
+ "P@20": 0.05765,
36
+ "P@50": 0.03108,
37
+ "P@100": 0.01894,
38
+ "P@200": 0.01128,
39
+ "P@1000": 0.00318
40
+ },
41
+ "recall": {
42
+ "Recall@1": 0.03988,
43
+ "Recall@3": 0.09123,
44
+ "Recall@5": 0.12133,
45
+ "Recall@10": 0.17298,
46
+ "Recall@20": 0.23343,
47
+ "Recall@50": 0.31485,
48
+ "Recall@100": 0.38372,
49
+ "Recall@200": 0.45717,
50
+ "Recall@1000": 0.64643
51
+ },
52
+ "mrr": {
53
+ "MRR@1": 0.196,
54
+ "MRR@3": 0.26317,
55
+ "MRR@5": 0.28087,
56
+ "MRR@10": 0.2931,
57
+ "MRR@20": 0.30043,
58
+ "MRR@50": 0.30405,
59
+ "MRR@100": 0.3052,
60
+ "MRR@200": 0.30566,
61
+ "MRR@1000": 0.30591
62
+ },
63
+ "recall_cap": {
64
+ "R_cap@1": 0.196,
65
+ "R_cap@3": 0.15,
66
+ "R_cap@5": 0.12133,
67
+ "R_cap@10": 0.17298,
68
+ "R_cap@20": 0.23343,
69
+ "R_cap@50": 0.31485,
70
+ "R_cap@100": 0.38372,
71
+ "R_cap@200": 0.45717,
72
+ "R_cap@1000": 0.64643
73
+ },
74
+ "hole": {
75
+ "Hole@1": 0.0,
76
+ "Hole@3": 0.0,
77
+ "Hole@5": 0.0,
78
+ "Hole@10": 0.0,
79
+ "Hole@20": 0.0,
80
+ "Hole@50": 0.0,
81
+ "Hole@100": 0.0,
82
+ "Hole@200": 0.0,
83
+ "Hole@1000": 0.0
84
+ }
85
+ }
86
+ }
beir_output/scifact.csv ADDED
@@ -0,0 +1,8 @@
 
 
 
 
 
 
 
 
 
1
+ metric,@1,@3,@5,@10,@20,@50,@100,@200,@1000
2
+ ndcg,0.50333,0.58634,0.62066,0.65554,0.66294,0.66815,0.67401,0.67644,0.6834
3
+ map,0.48039,0.55722,0.58018,0.59603,0.5984,0.59935,0.59993,0.60006,0.60024
4
+ precision,0.50333,0.23333,0.162,0.09267,0.04817,0.01987,0.01033,0.00527,0.00112
5
+ recall,0.48039,0.64061,0.72583,0.82744,0.85478,0.87989,0.91489,0.93156,0.98933
6
+ mrr,0.50333,0.57667,0.59383,0.60824,0.60984,0.61062,0.61104,0.61115,0.61131
7
+ recall_cap,0.50333,0.64389,0.72583,0.82744,0.85478,0.87989,0.91489,0.93156,0.98933
8
+ hole,0.48667,0.74111,0.80267,0.859,0.89533,0.92507,0.93613,0.94043,0.94373
beir_output/scifact.json ADDED
@@ -0,0 +1,86 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "dataset": "scifact",
3
+ "split": "test",
4
+ "metric": "dot",
5
+ "norm_query": false,
6
+ "norm_doc": false,
7
+ "scores": {
8
+ "ndcg": {
9
+ "NDCG@1": 0.50333,
10
+ "NDCG@3": 0.58634,
11
+ "NDCG@5": 0.62066,
12
+ "NDCG@10": 0.65554,
13
+ "NDCG@20": 0.66294,
14
+ "NDCG@50": 0.66815,
15
+ "NDCG@100": 0.67401,
16
+ "NDCG@200": 0.67644,
17
+ "NDCG@1000": 0.6834
18
+ },
19
+ "map": {
20
+ "MAP@1": 0.48039,
21
+ "MAP@3": 0.55722,
22
+ "MAP@5": 0.58018,
23
+ "MAP@10": 0.59603,
24
+ "MAP@20": 0.5984,
25
+ "MAP@50": 0.59935,
26
+ "MAP@100": 0.59993,
27
+ "MAP@200": 0.60006,
28
+ "MAP@1000": 0.60024
29
+ },
30
+ "precision": {
31
+ "P@1": 0.50333,
32
+ "P@3": 0.23333,
33
+ "P@5": 0.162,
34
+ "P@10": 0.09267,
35
+ "P@20": 0.04817,
36
+ "P@50": 0.01987,
37
+ "P@100": 0.01033,
38
+ "P@200": 0.00527,
39
+ "P@1000": 0.00112
40
+ },
41
+ "recall": {
42
+ "Recall@1": 0.48039,
43
+ "Recall@3": 0.64061,
44
+ "Recall@5": 0.72583,
45
+ "Recall@10": 0.82744,
46
+ "Recall@20": 0.85478,
47
+ "Recall@50": 0.87989,
48
+ "Recall@100": 0.91489,
49
+ "Recall@200": 0.93156,
50
+ "Recall@1000": 0.98933
51
+ },
52
+ "mrr": {
53
+ "MRR@1": 0.50333,
54
+ "MRR@3": 0.57667,
55
+ "MRR@5": 0.59383,
56
+ "MRR@10": 0.60824,
57
+ "MRR@20": 0.60984,
58
+ "MRR@50": 0.61062,
59
+ "MRR@100": 0.61104,
60
+ "MRR@200": 0.61115,
61
+ "MRR@1000": 0.61131
62
+ },
63
+ "recall_cap": {
64
+ "R_cap@1": 0.50333,
65
+ "R_cap@3": 0.64389,
66
+ "R_cap@5": 0.72583,
67
+ "R_cap@10": 0.82744,
68
+ "R_cap@20": 0.85478,
69
+ "R_cap@50": 0.87989,
70
+ "R_cap@100": 0.91489,
71
+ "R_cap@200": 0.93156,
72
+ "R_cap@1000": 0.98933
73
+ },
74
+ "hole": {
75
+ "Hole@1": 0.48667,
76
+ "Hole@3": 0.74111,
77
+ "Hole@5": 0.80267,
78
+ "Hole@10": 0.859,
79
+ "Hole@20": 0.89533,
80
+ "Hole@50": 0.92507,
81
+ "Hole@100": 0.93613,
82
+ "Hole@200": 0.94043,
83
+ "Hole@1000": 0.94373
84
+ }
85
+ }
86
+ }
beir_output/senteval-core.json ADDED
@@ -0,0 +1,5 @@
 
 
 
 
 
 
1
+ {
2
+ "eval_senteval-stsb_spearman": 0.759890982808152,
3
+ "eval_senteval-sickr_spearman": 0.7337127308506983,
4
+ "eval_senteval-avg_sts": 0.7468018568294252
5
+ }
beir_output/senteval.json ADDED
@@ -0,0 +1,99 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "STSBenchmark": {
3
+ "train": {
4
+ "pearson": [
5
+ 0.73125161607048,
6
+ 0.0
7
+ ],
8
+ "spearman": [
9
+ 0.7150765613453124,
10
+ 0.0
11
+ ],
12
+ "nsamples": 5749
13
+ },
14
+ "dev": {
15
+ "pearson": [
16
+ 0.7443000568532008,
17
+ 6.40691279840157e-265
18
+ ],
19
+ "spearman": [
20
+ 0.759890982808152,
21
+ 1.6901750172343795e-282
22
+ ],
23
+ "nsamples": 1500
24
+ },
25
+ "test": {
26
+ "pearson": [
27
+ 0.7216210553290073,
28
+ 3.539646733532031e-222
29
+ ],
30
+ "spearman": [
31
+ 0.7304939228930519,
32
+ 2.4834600870262812e-230
33
+ ],
34
+ "nsamples": 1379
35
+ },
36
+ "all": {
37
+ "pearson": {
38
+ "all": 0.7322719424112719,
39
+ "mean": 0.7323909094175627,
40
+ "wmean": 0.7319808833295888
41
+ },
42
+ "spearman": {
43
+ "all": 0.7295849518142924,
44
+ "mean": 0.7351538223488387,
45
+ "wmean": 0.7253317970625808
46
+ }
47
+ }
48
+ },
49
+ "SICKRelatedness": {
50
+ "train": {
51
+ "pearson": [
52
+ 0.7596859741218331,
53
+ 0.0
54
+ ],
55
+ "spearman": [
56
+ 0.7018231072465838,
57
+ 0.0
58
+ ],
59
+ "nsamples": 4500
60
+ },
61
+ "dev": {
62
+ "pearson": [
63
+ 0.777647576791967,
64
+ 1.9353259230163057e-102
65
+ ],
66
+ "spearman": [
67
+ 0.7337127308506983,
68
+ 1.2717377929862408e-85
69
+ ],
70
+ "nsamples": 500
71
+ },
72
+ "test": {
73
+ "pearson": [
74
+ 0.7566490759566877,
75
+ 0.0
76
+ ],
77
+ "spearman": [
78
+ 0.6936198984184592,
79
+ 0.0
80
+ ],
81
+ "nsamples": 4927
82
+ },
83
+ "all": {
84
+ "pearson": {
85
+ "all": 0.7589921709943909,
86
+ "mean": 0.7646608756234959,
87
+ "wmean": 0.7590833755598704
88
+ },
89
+ "spearman": {
90
+ "all": 0.6992260214830822,
91
+ "mean": 0.7097185788385804,
92
+ "wmean": 0.6993578712141356
93
+ }
94
+ }
95
+ },
96
+ "eval_senteval-stsb_spearman": 0.759890982808152,
97
+ "eval_senteval-sickr_spearman": 0.7337127308506983,
98
+ "eval_senteval-avg_sts": 0.7468018568294252
99
+ }
beir_output/trec-covid.csv ADDED
@@ -0,0 +1,8 @@
 
 
 
 
 
 
 
 
 
1
+ metric,@1,@3,@5,@10,@20,@50,@100,@200,@1000
2
+ ndcg,0.57,0.55022,0.54679,0.52738,0.51181,0.46694,0.41673,0.36577,0.39368
3
+ map,0.00178,0.00485,0.00745,0.01289,0.02273,0.0443,0.07001,0.10421,0.17671
4
+ precision,0.62,0.59333,0.596,0.574,0.552,0.4988,0.4358,0.3675,0.18164
5
+ recall,0.00178,0.00511,0.00822,0.01494,0.02835,0.06063,0.10192,0.16568,0.37739
6
+ mrr,0.62,0.71,0.733,0.733,0.73482,0.73741,0.73741,0.73741,0.73741
7
+ recall_cap,0.62,0.59333,0.596,0.574,0.552,0.4988,0.4358,0.37071,0.37901
8
+ hole,0.12,0.17333,0.176,0.168,0.195,0.2468,0.2766,0.3277,0.4544
beir_output/trec-covid.json ADDED
@@ -0,0 +1,86 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "dataset": "trec-covid",
3
+ "split": "test",
4
+ "metric": "dot",
5
+ "norm_query": false,
6
+ "norm_doc": false,
7
+ "scores": {
8
+ "ndcg": {
9
+ "NDCG@1": 0.57,
10
+ "NDCG@3": 0.55022,
11
+ "NDCG@5": 0.54679,
12
+ "NDCG@10": 0.52738,
13
+ "NDCG@20": 0.51181,
14
+ "NDCG@50": 0.46694,
15
+ "NDCG@100": 0.41673,
16
+ "NDCG@200": 0.36577,
17
+ "NDCG@1000": 0.39368
18
+ },
19
+ "map": {
20
+ "MAP@1": 0.00178,
21
+ "MAP@3": 0.00485,
22
+ "MAP@5": 0.00745,
23
+ "MAP@10": 0.01289,
24
+ "MAP@20": 0.02273,
25
+ "MAP@50": 0.0443,
26
+ "MAP@100": 0.07001,
27
+ "MAP@200": 0.10421,
28
+ "MAP@1000": 0.17671
29
+ },
30
+ "precision": {
31
+ "P@1": 0.62,
32
+ "P@3": 0.59333,
33
+ "P@5": 0.596,
34
+ "P@10": 0.574,
35
+ "P@20": 0.552,
36
+ "P@50": 0.4988,
37
+ "P@100": 0.4358,
38
+ "P@200": 0.3675,
39
+ "P@1000": 0.18164
40
+ },
41
+ "recall": {
42
+ "Recall@1": 0.00178,
43
+ "Recall@3": 0.00511,
44
+ "Recall@5": 0.00822,
45
+ "Recall@10": 0.01494,
46
+ "Recall@20": 0.02835,
47
+ "Recall@50": 0.06063,
48
+ "Recall@100": 0.10192,
49
+ "Recall@200": 0.16568,
50
+ "Recall@1000": 0.37739
51
+ },
52
+ "mrr": {
53
+ "MRR@1": 0.62,
54
+ "MRR@3": 0.71,
55
+ "MRR@5": 0.733,
56
+ "MRR@10": 0.733,
57
+ "MRR@20": 0.73482,
58
+ "MRR@50": 0.73741,
59
+ "MRR@100": 0.73741,
60
+ "MRR@200": 0.73741,
61
+ "MRR@1000": 0.73741
62
+ },
63
+ "recall_cap": {
64
+ "R_cap@1": 0.62,
65
+ "R_cap@3": 0.59333,
66
+ "R_cap@5": 0.596,
67
+ "R_cap@10": 0.574,
68
+ "R_cap@20": 0.552,
69
+ "R_cap@50": 0.4988,
70
+ "R_cap@100": 0.4358,
71
+ "R_cap@200": 0.37071,
72
+ "R_cap@1000": 0.37901
73
+ },
74
+ "hole": {
75
+ "Hole@1": 0.12,
76
+ "Hole@3": 0.17333,
77
+ "Hole@5": 0.176,
78
+ "Hole@10": 0.168,
79
+ "Hole@20": 0.195,
80
+ "Hole@50": 0.2468,
81
+ "Hole@100": 0.2766,
82
+ "Hole@200": 0.3277,
83
+ "Hole@1000": 0.4544
84
+ }
85
+ }
86
+ }
beir_output/webis-touche2020.csv ADDED
@@ -0,0 +1,8 @@
 
 
 
 
 
 
 
 
 
1
+ metric,@1,@3,@5,@10,@20,@50,@100,@200,@1000
2
+ ndcg,0.23469,0.19546,0.20774,0.20204,0.21577,0.27115,0.31444,0.35927,0.43626
3
+ map,0.02245,0.04169,0.05849,0.07998,0.09998,0.1207,0.13034,0.13799,0.14633
4
+ precision,0.2449,0.21088,0.22449,0.18367,0.14286,0.09714,0.06653,0.04449,0.01453
5
+ recall,0.02245,0.05107,0.09042,0.1418,0.21284,0.32885,0.429,0.54296,0.79353
6
+ mrr,0.2449,0.34014,0.38401,0.39492,0.40261,0.4037,0.40457,0.40476,0.4048
7
+ recall_cap,0.2449,0.21088,0.22449,0.20515,0.21951,0.32885,0.429,0.54296,0.79353
8
+ hole,0.67347,0.72109,0.73061,0.77551,0.82347,0.86327,0.90245,0.93031,0.969
beir_output/webis-touche2020.json ADDED
@@ -0,0 +1,86 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "dataset": "webis-touche2020",
3
+ "split": "test",
4
+ "metric": "dot",
5
+ "norm_query": false,
6
+ "norm_doc": false,
7
+ "scores": {
8
+ "ndcg": {
9
+ "NDCG@1": 0.23469,
10
+ "NDCG@3": 0.19546,
11
+ "NDCG@5": 0.20774,
12
+ "NDCG@10": 0.20204,
13
+ "NDCG@20": 0.21577,
14
+ "NDCG@50": 0.27115,
15
+ "NDCG@100": 0.31444,
16
+ "NDCG@200": 0.35927,
17
+ "NDCG@1000": 0.43626
18
+ },
19
+ "map": {
20
+ "MAP@1": 0.02245,
21
+ "MAP@3": 0.04169,
22
+ "MAP@5": 0.05849,
23
+ "MAP@10": 0.07998,
24
+ "MAP@20": 0.09998,
25
+ "MAP@50": 0.1207,
26
+ "MAP@100": 0.13034,
27
+ "MAP@200": 0.13799,
28
+ "MAP@1000": 0.14633
29
+ },
30
+ "precision": {
31
+ "P@1": 0.2449,
32
+ "P@3": 0.21088,
33
+ "P@5": 0.22449,
34
+ "P@10": 0.18367,
35
+ "P@20": 0.14286,
36
+ "P@50": 0.09714,
37
+ "P@100": 0.06653,
38
+ "P@200": 0.04449,
39
+ "P@1000": 0.01453
40
+ },
41
+ "recall": {
42
+ "Recall@1": 0.02245,
43
+ "Recall@3": 0.05107,
44
+ "Recall@5": 0.09042,
45
+ "Recall@10": 0.1418,
46
+ "Recall@20": 0.21284,
47
+ "Recall@50": 0.32885,
48
+ "Recall@100": 0.429,
49
+ "Recall@200": 0.54296,
50
+ "Recall@1000": 0.79353
51
+ },
52
+ "mrr": {
53
+ "MRR@1": 0.2449,
54
+ "MRR@3": 0.34014,
55
+ "MRR@5": 0.38401,
56
+ "MRR@10": 0.39492,
57
+ "MRR@20": 0.40261,
58
+ "MRR@50": 0.4037,
59
+ "MRR@100": 0.40457,
60
+ "MRR@200": 0.40476,
61
+ "MRR@1000": 0.4048
62
+ },
63
+ "recall_cap": {
64
+ "R_cap@1": 0.2449,
65
+ "R_cap@3": 0.21088,
66
+ "R_cap@5": 0.22449,
67
+ "R_cap@10": 0.20515,
68
+ "R_cap@20": 0.21951,
69
+ "R_cap@50": 0.32885,
70
+ "R_cap@100": 0.429,
71
+ "R_cap@200": 0.54296,
72
+ "R_cap@1000": 0.79353
73
+ },
74
+ "hole": {
75
+ "Hole@1": 0.67347,
76
+ "Hole@3": 0.72109,
77
+ "Hole@5": 0.73061,
78
+ "Hole@10": 0.77551,
79
+ "Hole@20": 0.82347,
80
+ "Hole@50": 0.86327,
81
+ "Hole@100": 0.90245,
82
+ "Hole@200": 0.93031,
83
+ "Hole@1000": 0.969
84
+ }
85
+ }
86
+ }
cc.moco.absum50.bs2048.gpu8.sh ADDED
@@ -0,0 +1,29 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/usr/bin/env bash
2
+
3
+ export CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7
4
+ export LOCAL_RANK=0
5
+ export WORLD_SIZE=8
6
+
7
+ export TOKENIZERS_PARALLELISM=false
8
+ export NUM_WORKER=4
9
+ export MAX_STEPS=100000
10
+
11
+ export CUDA_LAUNCH_BLOCKING=1
12
+ export NCCL_DEBUG=INFO
13
+
14
+ export EXP_NAME=cc.T03b_absum50.moco-2e14.contriever256-special50.bert-base-uncased.avg.dot.q128d256.step100k.bs1024.lr5e5
15
+ export PROJECT_DIR=output_dir/$EXP_NAME
16
+ mkdir -p $PROJECT_DIR
17
+ cp "$0" $PROJECT_DIR # copy bash to project dir
18
+ echo $PROJECT_DIR
19
+
20
+ export WANDB_NAME=$EXP_NAME
21
+ export WANDB_PROJECT=unsup_retrieval_cc
22
+ export WANDB_DIR=$PROJECT_DIR
23
+ mkdir -p $WANDB_DIR/wandb
24
+
25
+ nohup python -m torch.distributed.launch --nproc_per_node=8 --master_port=31133 --max_restarts=0 train.py --model_name_or_path bert-base-uncased --arch_type moco --train_file /export/home/data/search/upr/cc/T03B_PileCC_absum.json --dev_file /export/home/data/pretrain/wiki2021_structure/wiki_psgs_w100.dev.tail2e13.tsv --data_type hf --data_pipeline_name contriever256-special50% --remove_unused_columns False --sim_type dot --queue_size 16384 --momentum 0.9995 --output_dir $PROJECT_DIR --cache_dir /export/home/data/pretrain/.cache --max_steps $MAX_STEPS --warmup_steps 10000 --logging_steps 100 --eval_steps 10000 --save_steps 1000000 --per_device_train_batch_size 256 --per_device_eval_batch_size 256 --dataloader_num_workers $NUM_WORKER --learning_rate 5e-5 --max_q_tokens 128 --max_d_tokens 256 --evaluation_strategy steps --load_best_model_at_end --overwrite_output_dir --do_train --do_eval --run_name $EXP_NAME --fp16 --seed 42 --report_to wandb --wiki_passage_path /export/home/data/search/nq/psgs_w100.tsv --qa_datasets_path /export/home/data/search/nq/qas/*-test.csv,/export/home/data/search/nq/qas/entityqs/test/P*.test.json > $PROJECT_DIR/nohup.log 2>&1 & echo $! > run.pid
26
+
27
+ #export WANDB_RUN_ID=13h4vrrq
28
+ #nohup python -m torch.distributed.launch --nproc_per_node=8 --master_port=3112 --max_restarts=0 train.py --model_name_or_path bert-base-uncased --arch_type inbatch --reload_model_from $PROJECT_DIR --remove_unused_columns False --sim_type dot --output_dir $PROJECT_DIR --cache_dir /export/home/data/pretrain/.cache --max_steps $MAX_STEPS --do_eval --run_name $EXP_NAME --fp16 --seed 477 --report_to wandb --wiki_passage_path /export/home/data/search/nq/psgs_w100.tsv --qa_datasets_path /export/home/data/search/nq/qas/*-test.csv,/export/home/data/search/nq/qas/entityqs/test/P*.test.json > $PROJECT_DIR/nohup-eval.log 2>&1 & echo $! > run.pid
29
+
eval_output/.DS_Store ADDED
Binary file (8.2 kB). View file
 
eval_output/checkpoint-100000/arguana.csv ADDED
@@ -0,0 +1,8 @@
 
 
 
 
 
 
 
 
 
1
+ metric,@1,@3,@5,@10,@20,@50,@100,@200,@1000
2
+ ndcg,0.16358,0.27535,0.32609,0.38653,0.41902,0.43899,0.44613,0.44978,0.45272
3
+ map,0.16358,0.24656,0.27459,0.2995,0.30857,0.31185,0.31248,0.31266,0.31273
4
+ precision,0.16358,0.11972,0.09659,0.067,0.03986,0.01794,0.00941,0.00484,0.00099
5
+ recall,0.16358,0.35917,0.48293,0.66999,0.7973,0.89687,0.94097,0.96728,0.99289
6
+ mrr,0.16927,0.24846,0.27723,0.30182,0.31095,0.31424,0.31486,0.31504,0.31511
7
+ recall_cap,0.16927,0.35704,0.48435,0.66927,0.7973,0.89687,0.94097,0.96728,0.99289
8
+ hole,0.69488,0.68563,0.68777,0.68563,0.72461,0.8025,0.84213,0.86538,0.87804
eval_output/checkpoint-100000/arguana.json ADDED
@@ -0,0 +1,86 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "dataset": "arguana",
3
+ "split": "test",
4
+ "metric": "dot",
5
+ "norm_query": false,
6
+ "norm_doc": false,
7
+ "scores": {
8
+ "ndcg": {
9
+ "NDCG@1": 0.16358,
10
+ "NDCG@3": 0.27535,
11
+ "NDCG@5": 0.32609,
12
+ "NDCG@10": 0.38653,
13
+ "NDCG@20": 0.41902,
14
+ "NDCG@50": 0.43899,
15
+ "NDCG@100": 0.44613,
16
+ "NDCG@200": 0.44978,
17
+ "NDCG@1000": 0.45272
18
+ },
19
+ "map": {
20
+ "MAP@1": 0.16358,
21
+ "MAP@3": 0.24656,
22
+ "MAP@5": 0.27459,
23
+ "MAP@10": 0.2995,
24
+ "MAP@20": 0.30857,
25
+ "MAP@50": 0.31185,
26
+ "MAP@100": 0.31248,
27
+ "MAP@200": 0.31266,
28
+ "MAP@1000": 0.31273
29
+ },
30
+ "precision": {
31
+ "P@1": 0.16358,
32
+ "P@3": 0.11972,
33
+ "P@5": 0.09659,
34
+ "P@10": 0.067,
35
+ "P@20": 0.03986,
36
+ "P@50": 0.01794,
37
+ "P@100": 0.00941,
38
+ "P@200": 0.00484,
39
+ "P@1000": 0.00099
40
+ },
41
+ "recall": {
42
+ "Recall@1": 0.16358,
43
+ "Recall@3": 0.35917,
44
+ "Recall@5": 0.48293,
45
+ "Recall@10": 0.66999,
46
+ "Recall@20": 0.7973,
47
+ "Recall@50": 0.89687,
48
+ "Recall@100": 0.94097,
49
+ "Recall@200": 0.96728,
50
+ "Recall@1000": 0.99289
51
+ },
52
+ "mrr": {
53
+ "MRR@1": 0.16927,
54
+ "MRR@3": 0.24846,
55
+ "MRR@5": 0.27723,
56
+ "MRR@10": 0.30182,
57
+ "MRR@20": 0.31095,
58
+ "MRR@50": 0.31424,
59
+ "MRR@100": 0.31486,
60
+ "MRR@200": 0.31504,
61
+ "MRR@1000": 0.31511
62
+ },
63
+ "recall_cap": {
64
+ "R_cap@1": 0.16927,
65
+ "R_cap@3": 0.35704,
66
+ "R_cap@5": 0.48435,
67
+ "R_cap@10": 0.66927,
68
+ "R_cap@20": 0.7973,
69
+ "R_cap@50": 0.89687,
70
+ "R_cap@100": 0.94097,
71
+ "R_cap@200": 0.96728,
72
+ "R_cap@1000": 0.99289
73
+ },
74
+ "hole": {
75
+ "Hole@1": 0.69488,
76
+ "Hole@3": 0.68563,
77
+ "Hole@5": 0.68777,
78
+ "Hole@10": 0.68563,
79
+ "Hole@20": 0.72461,
80
+ "Hole@50": 0.8025,
81
+ "Hole@100": 0.84213,
82
+ "Hole@200": 0.86538,
83
+ "Hole@1000": 0.87804
84
+ }
85
+ }
86
+ }
eval_output/checkpoint-100000/beir.json ADDED
@@ -0,0 +1,46 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "eval_beir-nfcorpus_ndcg@10": 0.29881,
3
+ "eval_beir-nfcorpus_recall@10": 0.15184,
4
+ "eval_beir-nfcorpus_recall@20": 0.17956,
5
+ "eval_beir-nfcorpus_recall@100": 0.28269,
6
+ "eval_beir-fiqa_ndcg@10": 0.26763,
7
+ "eval_beir-fiqa_recall@10": 0.33485,
8
+ "eval_beir-fiqa_recall@20": 0.41649,
9
+ "eval_beir-fiqa_recall@100": 0.6111,
10
+ "eval_beir-arguana_ndcg@10": 0.38653,
11
+ "eval_beir-arguana_recall@10": 0.66999,
12
+ "eval_beir-arguana_recall@20": 0.7973,
13
+ "eval_beir-arguana_recall@100": 0.94097,
14
+ "eval_beir-scidocs_ndcg@10": 0.16627,
15
+ "eval_beir-scidocs_recall@10": 0.17298,
16
+ "eval_beir-scidocs_recall@20": 0.23343,
17
+ "eval_beir-scidocs_recall@100": 0.38372,
18
+ "eval_beir-scifact_ndcg@10": 0.65554,
19
+ "eval_beir-scifact_recall@10": 0.82744,
20
+ "eval_beir-scifact_recall@20": 0.85478,
21
+ "eval_beir-scifact_recall@100": 0.91489,
22
+ "eval_beir-webis-touche2020_ndcg@10": 0.20204,
23
+ "eval_beir-webis-touche2020_recall@10": 0.1418,
24
+ "eval_beir-webis-touche2020_recall@20": 0.21284,
25
+ "eval_beir-webis-touche2020_recall@100": 0.429,
26
+ "eval_beir-cqadupstack_ndcg@10": 0.2928333333333334,
27
+ "eval_beir-cqadupstack_recall@10": 0.3925783333333334,
28
+ "eval_beir-cqadupstack_recall@20": 0.4612349999999999,
29
+ "eval_beir-cqadupstack_recall@100": 0.6243891666666667,
30
+ "eval_beir-trec-covid_ndcg@10": 0.52738,
31
+ "eval_beir-trec-covid_recall@10": 0.574,
32
+ "eval_beir-trec-covid_recall@20": 0.552,
33
+ "eval_beir-trec-covid_recall@100": 0.4358,
34
+ "eval_beir-quora_ndcg@10": 0.78119,
35
+ "eval_beir-quora_recall@10": 0.88917,
36
+ "eval_beir-quora_recall@20": 0.93078,
37
+ "eval_beir-quora_recall@100": 0.97876,
38
+ "eval_beir-nq_ndcg@10": 0.29837,
39
+ "eval_beir-nq_recall@10": 0.48047,
40
+ "eval_beir-nq_recall@20": 0.60291,
41
+ "eval_beir-nq_recall@100": 0.81825,
42
+ "eval_beir-avg_ndcg@10": 0.3876593333333333,
43
+ "eval_beir-avg_recall@10": 0.46351183333333335,
44
+ "eval_beir-avg_recall@20": 0.5241325,
45
+ "eval_beir-avg_recall@100": 0.6419569166666668
46
+ }
eval_output/checkpoint-100000/cqadupstack.csv ADDED
@@ -0,0 +1,8 @@
 
 
 
 
 
 
 
 
 
1
+ metric,@1,@3,@5,@10,@20,@50,@100,@200,@1000
2
+ ndcg,0.21238333333333334,0.25052250000000004,0.26944,0.2928333333333334,0.31152583333333334,0.3318433333333333,0.3445375,0.35541416666666664,0.37370333333333333
3
+ map,0.17943833333333337,0.22491000000000003,0.23740666666666665,0.24825000000000005,0.2539983333333333,0.25779500000000005,0.2591683333333334,0.2598941666666667,0.2604566666666667
4
+ precision,0.21238333333333334,0.11569750000000001,0.08366416666666666,0.05258166666666667,0.032051666666666666,0.016111666666666666,0.009380833333333335,0.0053683333333333335,0.0013716666666666667
5
+ recall,0.17943833333333337,0.2751608333333333,0.32309249999999995,0.3925783333333334,0.4612349999999999,0.5543025,0.6243891666666667,0.6937216666666668,0.8333191666666667
6
+ mrr,0.21238333333333334,0.26051166666666664,0.2725008333333333,0.28266749999999996,0.2877675,0.290765,0.29172166666666666,0.2921791666666667,0.29249583333333334
7
+ recall_cap,0.21238333333333334,0.28125416666666664,0.32585833333333336,0.39377083333333335,0.4617558333333332,0.5544891666666666,0.6244641666666667,0.6937291666666666,0.8333191666666667
8
+ hole,0.7077300000000001,0.7993166666666666,0.8298199999999999,0.8623316666666666,0.8835099999999999,0.9013991666666669,0.9103258333333333,0.9166075,0.925735
eval_output/checkpoint-100000/cqadupstack.json ADDED
@@ -0,0 +1,86 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "dataset": "cqadupstack",
3
+ "split": "test",
4
+ "metric": "dot",
5
+ "norm_query": false,
6
+ "norm_doc": false,
7
+ "scores": {
8
+ "ndcg": {
9
+ "NDCG@1": 0.21238333333333334,
10
+ "NDCG@3": 0.25052250000000004,
11
+ "NDCG@5": 0.26944,
12
+ "NDCG@10": 0.2928333333333334,
13
+ "NDCG@20": 0.31152583333333334,
14
+ "NDCG@50": 0.3318433333333333,
15
+ "NDCG@100": 0.3445375,
16
+ "NDCG@200": 0.35541416666666664,
17
+ "NDCG@1000": 0.37370333333333333
18
+ },
19
+ "map": {
20
+ "MAP@1": 0.17943833333333337,
21
+ "MAP@3": 0.22491000000000003,
22
+ "MAP@5": 0.23740666666666665,
23
+ "MAP@10": 0.24825000000000005,
24
+ "MAP@20": 0.2539983333333333,
25
+ "MAP@50": 0.25779500000000005,
26
+ "MAP@100": 0.2591683333333334,
27
+ "MAP@200": 0.2598941666666667,
28
+ "MAP@1000": 0.2604566666666667
29
+ },
30
+ "precision": {
31
+ "P@1": 0.21238333333333334,
32
+ "P@3": 0.11569750000000001,
33
+ "P@5": 0.08366416666666666,
34
+ "P@10": 0.05258166666666667,
35
+ "P@20": 0.032051666666666666,
36
+ "P@50": 0.016111666666666666,
37
+ "P@100": 0.009380833333333335,
38
+ "P@200": 0.0053683333333333335,
39
+ "P@1000": 0.0013716666666666667
40
+ },
41
+ "recall": {
42
+ "Recall@1": 0.17943833333333337,
43
+ "Recall@3": 0.2751608333333333,
44
+ "Recall@5": 0.32309249999999995,
45
+ "Recall@10": 0.3925783333333334,
46
+ "Recall@20": 0.4612349999999999,
47
+ "Recall@50": 0.5543025,
48
+ "Recall@100": 0.6243891666666667,
49
+ "Recall@200": 0.6937216666666668,
50
+ "Recall@1000": 0.8333191666666667
51
+ },
52
+ "mrr": {
53
+ "MRR@1": 0.21238333333333334,
54
+ "MRR@3": 0.26051166666666664,
55
+ "MRR@5": 0.2725008333333333,
56
+ "MRR@10": 0.28266749999999996,
57
+ "MRR@20": 0.2877675,
58
+ "MRR@50": 0.290765,
59
+ "MRR@100": 0.29172166666666666,
60
+ "MRR@200": 0.2921791666666667,
61
+ "MRR@1000": 0.29249583333333334
62
+ },
63
+ "recall_cap": {
64
+ "R_cap@1": 0.21238333333333334,
65
+ "R_cap@3": 0.28125416666666664,
66
+ "R_cap@5": 0.32585833333333336,
67
+ "R_cap@10": 0.39377083333333335,
68
+ "R_cap@20": 0.4617558333333332,
69
+ "R_cap@50": 0.5544891666666666,
70
+ "R_cap@100": 0.6244641666666667,
71
+ "R_cap@200": 0.6937291666666666,
72
+ "R_cap@1000": 0.8333191666666667
73
+ },
74
+ "hole": {
75
+ "Hole@1": 0.7077300000000001,
76
+ "Hole@3": 0.7993166666666666,
77
+ "Hole@5": 0.8298199999999999,
78
+ "Hole@10": 0.8623316666666666,
79
+ "Hole@20": 0.8835099999999999,
80
+ "Hole@50": 0.9013991666666669,
81
+ "Hole@100": 0.9103258333333333,
82
+ "Hole@200": 0.9166075,
83
+ "Hole@1000": 0.925735
84
+ }
85
+ }
86
+ }
eval_output/checkpoint-100000/fiqa.csv ADDED
@@ -0,0 +1,8 @@
 
 
 
 
 
 
 
 
 
1
+ metric,@1,@3,@5,@10,@20,@50,@100,@200,@1000
2
+ ndcg,0.24537,0.22733,0.23826,0.26763,0.29383,0.32214,0.33978,0.35479,0.37946
3
+ map,0.1295,0.17366,0.18773,0.20398,0.21291,0.21873,0.22082,0.22202,0.22294
4
+ precision,0.24537,0.14712,0.1108,0.07593,0.04838,0.02519,0.01481,0.00853,0.00219
5
+ recall,0.1295,0.20816,0.25341,0.33485,0.41649,0.52773,0.6111,0.69263,0.85218
6
+ mrr,0.24537,0.29475,0.30818,0.32292,0.32917,0.33263,0.33371,0.33411,0.33432
7
+ recall_cap,0.24537,0.23174,0.26019,0.33527,0.41649,0.52773,0.6111,0.69263,0.85218
8
+ hole,0.69599,0.79424,0.82654,0.86466,0.88935,0.91136,0.92133,0.92652,0.93546
eval_output/checkpoint-100000/fiqa.json ADDED
@@ -0,0 +1,86 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "dataset": "fiqa",
3
+ "split": "test",
4
+ "metric": "dot",
5
+ "norm_query": false,
6
+ "norm_doc": false,
7
+ "scores": {
8
+ "ndcg": {
9
+ "NDCG@1": 0.24537,
10
+ "NDCG@3": 0.22733,
11
+ "NDCG@5": 0.23826,
12
+ "NDCG@10": 0.26763,
13
+ "NDCG@20": 0.29383,
14
+ "NDCG@50": 0.32214,
15
+ "NDCG@100": 0.33978,
16
+ "NDCG@200": 0.35479,
17
+ "NDCG@1000": 0.37946
18
+ },
19
+ "map": {
20
+ "MAP@1": 0.1295,
21
+ "MAP@3": 0.17366,
22
+ "MAP@5": 0.18773,
23
+ "MAP@10": 0.20398,
24
+ "MAP@20": 0.21291,
25
+ "MAP@50": 0.21873,
26
+ "MAP@100": 0.22082,
27
+ "MAP@200": 0.22202,
28
+ "MAP@1000": 0.22294
29
+ },
30
+ "precision": {
31
+ "P@1": 0.24537,
32
+ "P@3": 0.14712,
33
+ "P@5": 0.1108,
34
+ "P@10": 0.07593,
35
+ "P@20": 0.04838,
36
+ "P@50": 0.02519,
37
+ "P@100": 0.01481,
38
+ "P@200": 0.00853,
39
+ "P@1000": 0.00219
40
+ },
41
+ "recall": {
42
+ "Recall@1": 0.1295,
43
+ "Recall@3": 0.20816,
44
+ "Recall@5": 0.25341,
45
+ "Recall@10": 0.33485,
46
+ "Recall@20": 0.41649,
47
+ "Recall@50": 0.52773,
48
+ "Recall@100": 0.6111,
49
+ "Recall@200": 0.69263,
50
+ "Recall@1000": 0.85218
51
+ },
52
+ "mrr": {
53
+ "MRR@1": 0.24537,
54
+ "MRR@3": 0.29475,
55
+ "MRR@5": 0.30818,
56
+ "MRR@10": 0.32292,
57
+ "MRR@20": 0.32917,
58
+ "MRR@50": 0.33263,
59
+ "MRR@100": 0.33371,
60
+ "MRR@200": 0.33411,
61
+ "MRR@1000": 0.33432
62
+ },
63
+ "recall_cap": {
64
+ "R_cap@1": 0.24537,
65
+ "R_cap@3": 0.23174,
66
+ "R_cap@5": 0.26019,
67
+ "R_cap@10": 0.33527,
68
+ "R_cap@20": 0.41649,
69
+ "R_cap@50": 0.52773,
70
+ "R_cap@100": 0.6111,
71
+ "R_cap@200": 0.69263,
72
+ "R_cap@1000": 0.85218
73
+ },
74
+ "hole": {
75
+ "Hole@1": 0.69599,
76
+ "Hole@3": 0.79424,
77
+ "Hole@5": 0.82654,
78
+ "Hole@10": 0.86466,
79
+ "Hole@20": 0.88935,
80
+ "Hole@50": 0.91136,
81
+ "Hole@100": 0.92133,
82
+ "Hole@200": 0.92652,
83
+ "Hole@1000": 0.93546
84
+ }
85
+ }
86
+ }
eval_output/checkpoint-100000/nfcorpus.csv ADDED
@@ -0,0 +1,8 @@
 
 
 
 
 
 
 
 
 
1
+ metric,@1,@3,@5,@10,@20,@50,@100,@200,@1000
2
+ ndcg,0.37926,0.34843,0.32746,0.29881,0.27649,0.26898,0.2763,0.29387,0.36529
3
+ map,0.04368,0.07888,0.09235,0.10625,0.11699,0.12725,0.13272,0.13745,0.14607
4
+ precision,0.39628,0.3323,0.28483,0.21858,0.16037,0.10433,0.07133,0.04858,0.02022
5
+ recall,0.04368,0.09208,0.11936,0.15184,0.17956,0.23425,0.28269,0.35094,0.59763
6
+ mrr,0.40248,0.47575,0.49154,0.50011,0.50126,0.50372,0.50427,0.50457,0.50477
7
+ recall_cap,0.40248,0.34881,0.32255,0.28298,0.25421,0.26373,0.29608,0.3556,0.59763
8
+ hole,0.05882,0.07843,0.0904,0.09907,0.11285,0.12322,0.12424,0.12882,0.1328
eval_output/checkpoint-100000/nfcorpus.json ADDED
@@ -0,0 +1,86 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "dataset": "nfcorpus",
3
+ "split": "test",
4
+ "metric": "dot",
5
+ "norm_query": false,
6
+ "norm_doc": false,
7
+ "scores": {
8
+ "ndcg": {
9
+ "NDCG@1": 0.37926,
10
+ "NDCG@3": 0.34843,
11
+ "NDCG@5": 0.32746,
12
+ "NDCG@10": 0.29881,
13
+ "NDCG@20": 0.27649,
14
+ "NDCG@50": 0.26898,
15
+ "NDCG@100": 0.2763,
16
+ "NDCG@200": 0.29387,
17
+ "NDCG@1000": 0.36529
18
+ },
19
+ "map": {
20
+ "MAP@1": 0.04368,
21
+ "MAP@3": 0.07888,
22
+ "MAP@5": 0.09235,
23
+ "MAP@10": 0.10625,
24
+ "MAP@20": 0.11699,
25
+ "MAP@50": 0.12725,
26
+ "MAP@100": 0.13272,
27
+ "MAP@200": 0.13745,
28
+ "MAP@1000": 0.14607
29
+ },
30
+ "precision": {
31
+ "P@1": 0.39628,
32
+ "P@3": 0.3323,
33
+ "P@5": 0.28483,
34
+ "P@10": 0.21858,
35
+ "P@20": 0.16037,
36
+ "P@50": 0.10433,
37
+ "P@100": 0.07133,
38
+ "P@200": 0.04858,
39
+ "P@1000": 0.02022
40
+ },
41
+ "recall": {
42
+ "Recall@1": 0.04368,
43
+ "Recall@3": 0.09208,
44
+ "Recall@5": 0.11936,
45
+ "Recall@10": 0.15184,
46
+ "Recall@20": 0.17956,
47
+ "Recall@50": 0.23425,
48
+ "Recall@100": 0.28269,
49
+ "Recall@200": 0.35094,
50
+ "Recall@1000": 0.59763
51
+ },
52
+ "mrr": {
53
+ "MRR@1": 0.40248,
54
+ "MRR@3": 0.47575,
55
+ "MRR@5": 0.49154,
56
+ "MRR@10": 0.50011,
57
+ "MRR@20": 0.50126,
58
+ "MRR@50": 0.50372,
59
+ "MRR@100": 0.50427,
60
+ "MRR@200": 0.50457,
61
+ "MRR@1000": 0.50477
62
+ },
63
+ "recall_cap": {
64
+ "R_cap@1": 0.40248,
65
+ "R_cap@3": 0.34881,
66
+ "R_cap@5": 0.32255,
67
+ "R_cap@10": 0.28298,
68
+ "R_cap@20": 0.25421,
69
+ "R_cap@50": 0.26373,
70
+ "R_cap@100": 0.29608,
71
+ "R_cap@200": 0.3556,
72
+ "R_cap@1000": 0.59763
73
+ },
74
+ "hole": {
75
+ "Hole@1": 0.05882,
76
+ "Hole@3": 0.07843,
77
+ "Hole@5": 0.0904,
78
+ "Hole@10": 0.09907,
79
+ "Hole@20": 0.11285,
80
+ "Hole@50": 0.12322,
81
+ "Hole@100": 0.12424,
82
+ "Hole@200": 0.12882,
83
+ "Hole@1000": 0.1328
84
+ }
85
+ }
86
+ }
eval_output/checkpoint-100000/nq.csv ADDED
@@ -0,0 +1,8 @@
 
 
 
 
 
 
 
 
 
1
+ metric,@1,@3,@5,@10,@20,@50,@100,@200,@1000
2
+ ndcg,0.14977,0.21812,0.25843,0.29837,0.33082,0.35938,0.37296,0.38187,0.39152
3
+ map,0.13222,0.19374,0.21665,0.23398,0.24349,0.24859,0.24986,0.25038,0.25064
4
+ precision,0.14977,0.10361,0.0847,0.05678,0.03596,0.01772,0.00985,0.00531,0.00116
5
+ recall,0.13222,0.26794,0.36233,0.48047,0.60291,0.73877,0.81825,0.87768,0.95597
6
+ mrr,0.15006,0.21461,0.23693,0.25261,0.26078,0.26498,0.26609,0.26649,0.26668
7
+ recall_cap,0.15006,0.26801,0.36233,0.48047,0.60291,0.73877,0.81825,0.87768,0.95597
8
+ hole,0.84791,0.89446,0.91321,0.94041,0.96088,0.97933,0.9873,0.99177,0.99618
eval_output/checkpoint-100000/nq.json ADDED
@@ -0,0 +1,86 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "dataset": "nq",
3
+ "split": "test",
4
+ "metric": "dot",
5
+ "norm_query": false,
6
+ "norm_doc": false,
7
+ "scores": {
8
+ "ndcg": {
9
+ "NDCG@1": 0.14977,
10
+ "NDCG@3": 0.21812,
11
+ "NDCG@5": 0.25843,
12
+ "NDCG@10": 0.29837,
13
+ "NDCG@20": 0.33082,
14
+ "NDCG@50": 0.35938,
15
+ "NDCG@100": 0.37296,
16
+ "NDCG@200": 0.38187,
17
+ "NDCG@1000": 0.39152
18
+ },
19
+ "map": {
20
+ "MAP@1": 0.13222,
21
+ "MAP@3": 0.19374,
22
+ "MAP@5": 0.21665,
23
+ "MAP@10": 0.23398,
24
+ "MAP@20": 0.24349,
25
+ "MAP@50": 0.24859,
26
+ "MAP@100": 0.24986,
27
+ "MAP@200": 0.25038,
28
+ "MAP@1000": 0.25064
29
+ },
30
+ "precision": {
31
+ "P@1": 0.14977,
32
+ "P@3": 0.10361,
33
+ "P@5": 0.0847,
34
+ "P@10": 0.05678,
35
+ "P@20": 0.03596,
36
+ "P@50": 0.01772,
37
+ "P@100": 0.00985,
38
+ "P@200": 0.00531,
39
+ "P@1000": 0.00116
40
+ },
41
+ "recall": {
42
+ "Recall@1": 0.13222,
43
+ "Recall@3": 0.26794,
44
+ "Recall@5": 0.36233,
45
+ "Recall@10": 0.48047,
46
+ "Recall@20": 0.60291,
47
+ "Recall@50": 0.73877,
48
+ "Recall@100": 0.81825,
49
+ "Recall@200": 0.87768,
50
+ "Recall@1000": 0.95597
51
+ },
52
+ "mrr": {
53
+ "MRR@1": 0.15006,
54
+ "MRR@3": 0.21461,
55
+ "MRR@5": 0.23693,
56
+ "MRR@10": 0.25261,
57
+ "MRR@20": 0.26078,
58
+ "MRR@50": 0.26498,
59
+ "MRR@100": 0.26609,
60
+ "MRR@200": 0.26649,
61
+ "MRR@1000": 0.26668
62
+ },
63
+ "recall_cap": {
64
+ "R_cap@1": 0.15006,
65
+ "R_cap@3": 0.26801,
66
+ "R_cap@5": 0.36233,
67
+ "R_cap@10": 0.48047,
68
+ "R_cap@20": 0.60291,
69
+ "R_cap@50": 0.73877,
70
+ "R_cap@100": 0.81825,
71
+ "R_cap@200": 0.87768,
72
+ "R_cap@1000": 0.95597
73
+ },
74
+ "hole": {
75
+ "Hole@1": 0.84791,
76
+ "Hole@3": 0.89446,
77
+ "Hole@5": 0.91321,
78
+ "Hole@10": 0.94041,
79
+ "Hole@20": 0.96088,
80
+ "Hole@50": 0.97933,
81
+ "Hole@100": 0.9873,
82
+ "Hole@200": 0.99177,
83
+ "Hole@1000": 0.99618
84
+ }
85
+ }
86
+ }
eval_output/checkpoint-100000/quora.csv ADDED
@@ -0,0 +1,8 @@
 
 
 
 
 
 
 
 
 
1
+ metric,@1,@3,@5,@10,@20,@50,@100,@200,@1000
2
+ ndcg,0.6801,0.73881,0.76011,0.78119,0.79347,0.80197,0.80506,0.80666,0.80835
3
+ map,0.59232,0.69716,0.7166,0.7303,0.73595,0.73848,0.73903,0.73922,0.73933
4
+ precision,0.6801,0.32447,0.21682,0.12183,0.0662,0.0285,0.01474,0.00753,0.00156
5
+ recall,0.59232,0.77203,0.82842,0.88917,0.93078,0.96438,0.97876,0.98715,0.99725
6
+ mrr,0.6798,0.74905,0.7596,0.76606,0.76819,0.76893,0.76907,0.7691,0.76912
7
+ recall_cap,0.6798,0.78502,0.83355,0.89063,0.93106,0.9644,0.97876,0.98715,0.99725
8
+ hole,0.2983,0.6418,0.74436,0.83671,0.89097,0.92861,0.94355,0.9524,0.96129
eval_output/checkpoint-100000/quora.json ADDED
@@ -0,0 +1,86 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "dataset": "quora",
3
+ "split": "test",
4
+ "metric": "dot",
5
+ "norm_query": false,
6
+ "norm_doc": false,
7
+ "scores": {
8
+ "ndcg": {
9
+ "NDCG@1": 0.6801,
10
+ "NDCG@3": 0.73881,
11
+ "NDCG@5": 0.76011,
12
+ "NDCG@10": 0.78119,
13
+ "NDCG@20": 0.79347,
14
+ "NDCG@50": 0.80197,
15
+ "NDCG@100": 0.80506,
16
+ "NDCG@200": 0.80666,
17
+ "NDCG@1000": 0.80835
18
+ },
19
+ "map": {
20
+ "MAP@1": 0.59232,
21
+ "MAP@3": 0.69716,
22
+ "MAP@5": 0.7166,
23
+ "MAP@10": 0.7303,
24
+ "MAP@20": 0.73595,
25
+ "MAP@50": 0.73848,
26
+ "MAP@100": 0.73903,
27
+ "MAP@200": 0.73922,
28
+ "MAP@1000": 0.73933
29
+ },
30
+ "precision": {
31
+ "P@1": 0.6801,
32
+ "P@3": 0.32447,
33
+ "P@5": 0.21682,
34
+ "P@10": 0.12183,
35
+ "P@20": 0.0662,
36
+ "P@50": 0.0285,
37
+ "P@100": 0.01474,
38
+ "P@200": 0.00753,
39
+ "P@1000": 0.00156
40
+ },
41
+ "recall": {
42
+ "Recall@1": 0.59232,
43
+ "Recall@3": 0.77203,
44
+ "Recall@5": 0.82842,
45
+ "Recall@10": 0.88917,
46
+ "Recall@20": 0.93078,
47
+ "Recall@50": 0.96438,
48
+ "Recall@100": 0.97876,
49
+ "Recall@200": 0.98715,
50
+ "Recall@1000": 0.99725
51
+ },
52
+ "mrr": {
53
+ "MRR@1": 0.6798,
54
+ "MRR@3": 0.74905,
55
+ "MRR@5": 0.7596,
56
+ "MRR@10": 0.76606,
57
+ "MRR@20": 0.76819,
58
+ "MRR@50": 0.76893,
59
+ "MRR@100": 0.76907,
60
+ "MRR@200": 0.7691,
61
+ "MRR@1000": 0.76912
62
+ },
63
+ "recall_cap": {
64
+ "R_cap@1": 0.6798,
65
+ "R_cap@3": 0.78502,
66
+ "R_cap@5": 0.83355,
67
+ "R_cap@10": 0.89063,
68
+ "R_cap@20": 0.93106,
69
+ "R_cap@50": 0.9644,
70
+ "R_cap@100": 0.97876,
71
+ "R_cap@200": 0.98715,
72
+ "R_cap@1000": 0.99725
73
+ },
74
+ "hole": {
75
+ "Hole@1": 0.2983,
76
+ "Hole@3": 0.6418,
77
+ "Hole@5": 0.74436,
78
+ "Hole@10": 0.83671,
79
+ "Hole@20": 0.89097,
80
+ "Hole@50": 0.92861,
81
+ "Hole@100": 0.94355,
82
+ "Hole@200": 0.9524,
83
+ "Hole@1000": 0.96129
84
+ }
85
+ }
86
+ }
eval_output/checkpoint-100000/scidocs.csv ADDED
@@ -0,0 +1,8 @@
 
 
 
 
 
 
 
 
 
1
+ metric,@1,@3,@5,@10,@20,@50,@100,@200,@1000
2
+ ndcg,0.196,0.16049,0.13802,0.16627,0.19205,0.21944,0.23828,0.25562,0.29203
3
+ map,0.03988,0.07149,0.08285,0.09675,0.10559,0.11173,0.11416,0.11562,0.11705
4
+ precision,0.196,0.15,0.1198,0.0855,0.05765,0.03108,0.01894,0.01128,0.00318
5
+ recall,0.03988,0.09123,0.12133,0.17298,0.23343,0.31485,0.38372,0.45717,0.64643
6
+ mrr,0.196,0.26317,0.28087,0.2931,0.30043,0.30405,0.3052,0.30566,0.30591
7
+ recall_cap,0.196,0.15,0.12133,0.17298,0.23343,0.31485,0.38372,0.45717,0.64643
8
+ hole,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0