add cqa metrics

README.md

@@ -56,6 +56,903 @@ model-index:
       value: 28.12
     - type: f1
       value: 26.904734434317966
+  - task:
+      type: Retrieval
+    dataset:
+      type: BeIR/cqadupstack
+      name: MTEB CQADupstackAndroidRetrieval
+      config: default
+      split: test
+      revision: None
+    metrics:
+    - type: map_at_1
+      value: 36.635
+    - type: map_at_10
+      value: 48.291000000000004
+    - type: map_at_100
+      value: 49.833
+    - type: map_at_1000
+      value: 49.944
+    - type: map_at_3
+      value: 44.362
+    - type: map_at_5
+      value: 46.678
+    - type: mrr_at_1
+      value: 44.349
+    - type: mrr_at_10
+      value: 54.35
+    - type: mrr_at_100
+      value: 54.995000000000005
+    - type: mrr_at_1000
+      value: 55.03
+    - type: mrr_at_3
+      value: 52.074
+    - type: mrr_at_5
+      value: 53.433
+    - type: ndcg_at_1
+      value: 44.349
+    - type: ndcg_at_10
+      value: 54.876999999999995
+    - type: ndcg_at_100
+      value: 59.663
+    - type: ndcg_at_1000
+      value: 61.23
+    - type: ndcg_at_3
+      value: 49.727
+    - type: ndcg_at_5
+      value: 52.271
+    - type: precision_at_1
+      value: 44.349
+    - type: precision_at_10
+      value: 10.485999999999999
+    - type: precision_at_100
+      value: 1.6209999999999998
+    - type: precision_at_1000
+      value: 0.208
+    - type: precision_at_3
+      value: 23.653
+    - type: precision_at_5
+      value: 17.282
+    - type: recall_at_1
+      value: 36.635
+    - type: recall_at_10
+      value: 66.878
+    - type: recall_at_100
+      value: 86.239
+    - type: recall_at_1000
+      value: 96.14200000000001
+    - type: recall_at_3
+      value: 51.793
+    - type: recall_at_5
+      value: 58.943999999999996
+  - task:
+      type: Retrieval
+    dataset:
+      type: BeIR/cqadupstack
+      name: MTEB CQADupstackEnglishRetrieval
+      config: default
+      split: test
+      revision: None
+    metrics:
+    - type: map_at_1
+      value: 31.323
+    - type: map_at_10
+      value: 42.39
+    - type: map_at_100
+      value: 43.741
+    - type: map_at_1000
+      value: 43.872
+    - type: map_at_3
+      value: 39.109
+    - type: map_at_5
+      value: 40.961999999999996
+    - type: mrr_at_1
+      value: 39.617999999999995
+    - type: mrr_at_10
+      value: 48.595
+    - type: mrr_at_100
+      value: 49.236000000000004
+    - type: mrr_at_1000
+      value: 49.278
+    - type: mrr_at_3
+      value: 46.274
+    - type: mrr_at_5
+      value: 47.72
+    - type: ndcg_at_1
+      value: 39.617999999999995
+    - type: ndcg_at_10
+      value: 48.455
+    - type: ndcg_at_100
+      value: 52.949999999999996
+    - type: ndcg_at_1000
+      value: 54.93599999999999
+    - type: ndcg_at_3
+      value: 44.038
+    - type: ndcg_at_5
+      value: 46.154
+    - type: precision_at_1
+      value: 39.617999999999995
+    - type: precision_at_10
+      value: 9.318
+    - type: precision_at_100
+      value: 1.4869999999999999
+    - type: precision_at_1000
+      value: 0.19499999999999998
+    - type: precision_at_3
+      value: 21.614
+    - type: precision_at_5
+      value: 15.376000000000001
+    - type: recall_at_1
+      value: 31.323
+    - type: recall_at_10
+      value: 59.114999999999995
+    - type: recall_at_100
+      value: 77.98
+    - type: recall_at_1000
+      value: 90.561
+    - type: recall_at_3
+      value: 45.713
+    - type: recall_at_5
+      value: 51.842999999999996
+  - task:
+      type: Retrieval
+    dataset:
+      type: BeIR/cqadupstack
+      name: MTEB CQADupstackGamingRetrieval
+      config: default
+      split: test
+      revision: None
+    metrics:
+    - type: map_at_1
+      value: 40.858
+    - type: map_at_10
+      value: 53.477
+    - type: map_at_100
+      value: 54.47
+    - type: map_at_1000
+      value: 54.522999999999996
+    - type: map_at_3
+      value: 50.407999999999994
+    - type: map_at_5
+      value: 52.114000000000004
+    - type: mrr_at_1
+      value: 46.708
+    - type: mrr_at_10
+      value: 56.855999999999995
+    - type: mrr_at_100
+      value: 57.472
+    - type: mrr_at_1000
+      value: 57.498000000000005
+    - type: mrr_at_3
+      value: 54.45100000000001
+    - type: mrr_at_5
+      value: 55.781000000000006
+    - type: ndcg_at_1
+      value: 46.708
+    - type: ndcg_at_10
+      value: 59.299
+    - type: ndcg_at_100
+      value: 63.138000000000005
+    - type: ndcg_at_1000
+      value: 64.189
+    - type: ndcg_at_3
+      value: 54.125
+    - type: ndcg_at_5
+      value: 56.57600000000001
+    - type: precision_at_1
+      value: 46.708
+    - type: precision_at_10
+      value: 9.48
+    - type: precision_at_100
+      value: 1.234
+    - type: precision_at_1000
+      value: 0.136
+    - type: precision_at_3
+      value: 24.221999999999998
+    - type: precision_at_5
+      value: 16.414
+    - type: recall_at_1
+      value: 40.858
+    - type: recall_at_10
+      value: 73.1
+    - type: recall_at_100
+      value: 89.447
+    - type: recall_at_1000
+      value: 97.00999999999999
+    - type: recall_at_3
+      value: 59.092999999999996
+    - type: recall_at_5
+      value: 65.275
+  - task:
+      type: Retrieval
+    dataset:
+      type: BeIR/cqadupstack
+      name: MTEB CQADupstackGisRetrieval
+      config: default
+      split: test
+      revision: None
+    metrics:
+    - type: map_at_1
+      value: 27.400000000000002
+    - type: map_at_10
+      value: 36.878
+    - type: map_at_100
+      value: 37.993
+    - type: map_at_1000
+      value: 38.074000000000005
+    - type: map_at_3
+      value: 34.147
+    - type: map_at_5
+      value: 35.703
+    - type: mrr_at_1
+      value: 29.378999999999998
+    - type: mrr_at_10
+      value: 38.921
+    - type: mrr_at_100
+      value: 39.865
+    - type: mrr_at_1000
+      value: 39.92
+    - type: mrr_at_3
+      value: 36.29
+    - type: mrr_at_5
+      value: 37.878
+    - type: ndcg_at_1
+      value: 29.378999999999998
+    - type: ndcg_at_10
+      value: 42.205
+    - type: ndcg_at_100
+      value: 47.333
+    - type: ndcg_at_1000
+      value: 49.258
+    - type: ndcg_at_3
+      value: 36.83
+    - type: ndcg_at_5
+      value: 39.525
+    - type: precision_at_1
+      value: 29.378999999999998
+    - type: precision_at_10
+      value: 6.4750000000000005
+    - type: precision_at_100
+      value: 0.947
+    - type: precision_at_1000
+      value: 0.11499999999999999
+    - type: precision_at_3
+      value: 15.631
+    - type: precision_at_5
+      value: 10.983
+    - type: recall_at_1
+      value: 27.400000000000002
+    - type: recall_at_10
+      value: 56.61000000000001
+    - type: recall_at_100
+      value: 79.475
+    - type: recall_at_1000
+      value: 93.714
+    - type: recall_at_3
+      value: 42.064
+    - type: recall_at_5
+      value: 48.526
+  - task:
+      type: Retrieval
+    dataset:
+      type: BeIR/cqadupstack
+      name: MTEB CQADupstackMathematicaRetrieval
+      config: default
+      split: test
+      revision: None
+    metrics:
+    - type: map_at_1
+      value: 16.184
+    - type: map_at_10
+      value: 24.157
+    - type: map_at_100
+      value: 25.339
+    - type: map_at_1000
+      value: 25.454
+    - type: map_at_3
+      value: 21.426000000000002
+    - type: map_at_5
+      value: 22.792
+    - type: mrr_at_1
+      value: 19.776
+    - type: mrr_at_10
+      value: 28.53
+    - type: mrr_at_100
+      value: 29.463
+    - type: mrr_at_1000
+      value: 29.532000000000004
+    - type: mrr_at_3
+      value: 26.016000000000002
+    - type: mrr_at_5
+      value: 27.359
+    - type: ndcg_at_1
+      value: 19.776
+    - type: ndcg_at_10
+      value: 29.482000000000003
+    - type: ndcg_at_100
+      value: 35.132999999999996
+    - type: ndcg_at_1000
+      value: 38.048
+    - type: ndcg_at_3
+      value: 24.519
+    - type: ndcg_at_5
+      value: 26.541999999999998
+    - type: precision_at_1
+      value: 19.776
+    - type: precision_at_10
+      value: 5.5969999999999995
+    - type: precision_at_100
+      value: 0.9780000000000001
+    - type: precision_at_1000
+      value: 0.136
+    - type: precision_at_3
+      value: 12.065
+    - type: precision_at_5
+      value: 8.756
+    - type: recall_at_1
+      value: 16.184
+    - type: recall_at_10
+      value: 41.506
+    - type: recall_at_100
+      value: 66.322
+    - type: recall_at_1000
+      value: 87.40299999999999
+    - type: recall_at_3
+      value: 27.618
+    - type: recall_at_5
+      value: 32.81
+  - task:
+      type: Retrieval
+    dataset:
+      type: BeIR/cqadupstack
+      name: MTEB CQADupstackPhysicsRetrieval
+      config: default
+      split: test
+      revision: None
+    metrics:
+    - type: map_at_1
+      value: 28.79
+    - type: map_at_10
+      value: 39.475
+    - type: map_at_100
+      value: 40.864
+    - type: map_at_1000
+      value: 40.967
+    - type: map_at_3
+      value: 36.394999999999996
+    - type: map_at_5
+      value: 38.101
+    - type: mrr_at_1
+      value: 35.611
+    - type: mrr_at_10
+      value: 45.32
+    - type: mrr_at_100
+      value: 46.160000000000004
+    - type: mrr_at_1000
+      value: 46.205
+    - type: mrr_at_3
+      value: 42.717
+    - type: mrr_at_5
+      value: 44.233
+    - type: ndcg_at_1
+      value: 35.611
+    - type: ndcg_at_10
+      value: 45.513999999999996
+    - type: ndcg_at_100
+      value: 51.163000000000004
+    - type: ndcg_at_1000
+      value: 53.099
+    - type: ndcg_at_3
+      value: 40.602
+    - type: ndcg_at_5
+      value: 42.933
+    - type: precision_at_1
+      value: 35.611
+    - type: precision_at_10
+      value: 8.219
+    - type: precision_at_100
+      value: 1.302
+    - type: precision_at_1000
+      value: 0.166
+    - type: precision_at_3
+      value: 19.281000000000002
+    - type: precision_at_5
+      value: 13.550999999999998
+    - type: recall_at_1
+      value: 28.79
+    - type: recall_at_10
+      value: 57.708000000000006
+    - type: recall_at_100
+      value: 80.965
+    - type: recall_at_1000
+      value: 93.60000000000001
+    - type: recall_at_3
+      value: 43.766
+    - type: recall_at_5
+      value: 50.003
+  - task:
+      type: Retrieval
+    dataset:
+      type: BeIR/cqadupstack
+      name: MTEB CQADupstackProgrammersRetrieval
+      config: default
+      split: test
+      revision: None
+    metrics:
+    - type: map_at_1
+      value: 27.392
+    - type: map_at_10
+      value: 37.213
+    - type: map_at_100
+      value: 38.513999999999996
+    - type: map_at_1000
+      value: 38.629999999999995
+    - type: map_at_3
+      value: 33.844
+    - type: map_at_5
+      value: 35.791000000000004
+    - type: mrr_at_1
+      value: 33.676
+    - type: mrr_at_10
+      value: 42.58
+    - type: mrr_at_100
+      value: 43.472
+    - type: mrr_at_1000
+      value: 43.519999999999996
+    - type: mrr_at_3
+      value: 40.011
+    - type: mrr_at_5
+      value: 41.575
+    - type: ndcg_at_1
+      value: 33.676
+    - type: ndcg_at_10
+      value: 42.949
+    - type: ndcg_at_100
+      value: 48.542
+    - type: ndcg_at_1000
+      value: 50.804
+    - type: ndcg_at_3
+      value: 37.631
+    - type: ndcg_at_5
+      value: 40.226
+    - type: precision_at_1
+      value: 33.676
+    - type: precision_at_10
+      value: 7.785
+    - type: precision_at_100
+      value: 1.229
+    - type: precision_at_1000
+      value: 0.16199999999999998
+    - type: precision_at_3
+      value: 17.694
+    - type: precision_at_5
+      value: 12.763
+    - type: recall_at_1
+      value: 27.392
+    - type: recall_at_10
+      value: 54.82599999999999
+    - type: recall_at_100
+      value: 78.61
+    - type: recall_at_1000
+      value: 93.78800000000001
+    - type: recall_at_3
+      value: 40.019
+    - type: recall_at_5
+      value: 46.866
+  - task:
+      type: Retrieval
+    dataset:
+      type: BeIR/cqadupstack
+      name: MTEB CQADupstackRetrieval
+      config: default
+      split: test
+      revision: None
+    metrics:
+    - type: map_at_1
+      value: 27.550666666666668
+    - type: map_at_10
+      value: 37.07508333333333
+    - type: map_at_100
+      value: 38.31308333333333
+    - type: map_at_1000
+      value: 38.427166666666665
+    - type: map_at_3
+      value: 34.14741666666667
+    - type: map_at_5
+      value: 35.72416666666667
+    - type: mrr_at_1
+      value: 32.63183333333333
+    - type: mrr_at_10
+      value: 41.346999999999994
+    - type: mrr_at_100
+      value: 42.17225
+    - type: mrr_at_1000
+      value: 42.22475
+    - type: mrr_at_3
+      value: 38.903999999999996
+    - type: mrr_at_5
+      value: 40.27291666666667
+    - type: ndcg_at_1
+      value: 32.63183333333333
+    - type: ndcg_at_10
+      value: 42.61841666666667
+    - type: ndcg_at_100
+      value: 47.742
+    - type: ndcg_at_1000
+      value: 49.869416666666666
+    - type: ndcg_at_3
+      value: 37.73925
+    - type: ndcg_at_5
+      value: 39.925666666666665
+    - type: precision_at_1
+      value: 32.63183333333333
+    - type: precision_at_10
+      value: 7.504000000000001
+    - type: precision_at_100
+      value: 1.1986666666666668
+    - type: precision_at_1000
+      value: 0.15758333333333333
+    - type: precision_at_3
+      value: 17.415666666666667
+    - type: precision_at_5
+      value: 12.297749999999999
+    - type: recall_at_1
+      value: 27.550666666666668
+    - type: recall_at_10
+      value: 54.68383333333333
+    - type: recall_at_100
+      value: 77.01691666666667
+    - type: recall_at_1000
+      value: 91.71175000000001
+    - type: recall_at_3
+      value: 40.91866666666667
+    - type: recall_at_5
+      value: 46.669000000000004
+  - task:
+      type: Retrieval
+    dataset:
+      type: BeIR/cqadupstack
+      name: MTEB CQADupstackStatsRetrieval
+      config: default
+      split: test
+      revision: None
+    metrics:
+    - type: map_at_1
+      value: 24.91
+    - type: map_at_10
+      value: 32.053
+    - type: map_at_100
+      value: 33.086
+    - type: map_at_1000
+      value: 33.176
+    - type: map_at_3
+      value: 29.768
+    - type: map_at_5
+      value: 30.842000000000002
+    - type: mrr_at_1
+      value: 27.607
+    - type: mrr_at_10
+      value: 34.732
+    - type: mrr_at_100
+      value: 35.589
+    - type: mrr_at_1000
+      value: 35.65
+    - type: mrr_at_3
+      value: 32.566
+    - type: mrr_at_5
+      value: 33.556000000000004
+    - type: ndcg_at_1
+      value: 27.607
+    - type: ndcg_at_10
+      value: 36.579
+    - type: ndcg_at_100
+      value: 41.646
+    - type: ndcg_at_1000
+      value: 43.845
+    - type: ndcg_at_3
+      value: 32.132
+    - type: ndcg_at_5
+      value: 33.825
+    - type: precision_at_1
+      value: 27.607
+    - type: precision_at_10
+      value: 5.827999999999999
+    - type: precision_at_100
+      value: 0.928
+    - type: precision_at_1000
+      value: 0.12
+    - type: precision_at_3
+      value: 13.804
+    - type: precision_at_5
+      value: 9.447999999999999
+    - type: recall_at_1
+      value: 24.91
+    - type: recall_at_10
+      value: 47.924
+    - type: recall_at_100
+      value: 70.88799999999999
+    - type: recall_at_1000
+      value: 87.087
+    - type: recall_at_3
+      value: 35.169
+    - type: recall_at_5
+      value: 39.497
+  - task:
+      type: Retrieval
+    dataset:
+      type: BeIR/cqadupstack
+      name: MTEB CQADupstackTexRetrieval
+      config: default
+      split: test
+      revision: None
+    metrics:
+    - type: map_at_1
+      value: 18.19
+    - type: map_at_10
+      value: 25.765
+    - type: map_at_100
+      value: 26.882
+    - type: map_at_1000
+      value: 27.012999999999998
+    - type: map_at_3
+      value: 23.378
+    - type: map_at_5
+      value: 24.587
+    - type: mrr_at_1
+      value: 22.505
+    - type: mrr_at_10
+      value: 29.948999999999998
+    - type: mrr_at_100
+      value: 30.871
+    - type: mrr_at_1000
+      value: 30.947999999999997
+    - type: mrr_at_3
+      value: 27.764
+    - type: mrr_at_5
+      value: 28.951999999999998
+    - type: ndcg_at_1
+      value: 22.505
+    - type: ndcg_at_10
+      value: 30.593999999999998
+    - type: ndcg_at_100
+      value: 35.983
+    - type: ndcg_at_1000
+      value: 38.869
+    - type: ndcg_at_3
+      value: 26.369
+    - type: ndcg_at_5
+      value: 28.124
+    - type: precision_at_1
+      value: 22.505
+    - type: precision_at_10
+      value: 5.575
+    - type: precision_at_100
+      value: 0.9860000000000001
+    - type: precision_at_1000
+      value: 0.14200000000000002
+    - type: precision_at_3
+      value: 12.423
+    - type: precision_at_5
+      value: 8.878
+    - type: recall_at_1
+      value: 18.19
+    - type: recall_at_10
+      value: 41.032000000000004
+    - type: recall_at_100
+      value: 65.32900000000001
+    - type: recall_at_1000
+      value: 85.702
+    - type: recall_at_3
+      value: 29.136
+    - type: recall_at_5
+      value: 33.711
+  - task:
+      type: Retrieval
+    dataset:
+      type: BeIR/cqadupstack
+      name: MTEB CQADupstackUnixRetrieval
+      config: default
+      split: test
+      revision: None
+    metrics:
+    - type: map_at_1
+      value: 28.304000000000002
+    - type: map_at_10
+      value: 37.153000000000006
+    - type: map_at_100
+      value: 38.317
+    - type: map_at_1000
+      value: 38.422
+    - type: map_at_3
+      value: 34.317
+    - type: map_at_5
+      value: 35.801
+    - type: mrr_at_1
+      value: 33.675
+    - type: mrr_at_10
+      value: 41.302
+    - type: mrr_at_100
+      value: 42.202
+    - type: mrr_at_1000
+      value: 42.264
+    - type: mrr_at_3
+      value: 38.759
+    - type: mrr_at_5
+      value: 40.215
+    - type: ndcg_at_1
+      value: 33.675
+    - type: ndcg_at_10
+      value: 42.35
+    - type: ndcg_at_100
+      value: 47.653
+    - type: ndcg_at_1000
+      value: 49.964999999999996
+    - type: ndcg_at_3
+      value: 37.372
+    - type: ndcg_at_5
+      value: 39.544000000000004
+    - type: precision_at_1
+      value: 33.675
+    - type: precision_at_10
+      value: 7.136000000000001
+    - type: precision_at_100
+      value: 1.097
+    - type: precision_at_1000
+      value: 0.14100000000000001
+    - type: precision_at_3
+      value: 16.915
+    - type: precision_at_5
+      value: 11.884
+    - type: recall_at_1
+      value: 28.304000000000002
+    - type: recall_at_10
+      value: 54.083000000000006
+    - type: recall_at_100
+      value: 77.167
+    - type: recall_at_1000
+      value: 93.151
+    - type: recall_at_3
+      value: 40.441
+    - type: recall_at_5
+      value: 45.95
+  - task:
+      type: Retrieval
+    dataset:
+      type: BeIR/cqadupstack
+      name: MTEB CQADupstackWebmastersRetrieval
+      config: default
+      split: test
+      revision: None
+    metrics:
+    - type: map_at_1
+      value: 29.575000000000003
+    - type: map_at_10
+      value: 39.089
+    - type: map_at_100
+      value: 40.813
+    - type: map_at_1000
+      value: 41.032000000000004
+    - type: map_at_3
+      value: 36.153999999999996
+    - type: map_at_5
+      value: 37.518
+    - type: mrr_at_1
+      value: 35.573
+    - type: mrr_at_10
+      value: 43.891000000000005
+    - type: mrr_at_100
+      value: 44.777
+    - type: mrr_at_1000
+      value: 44.812999999999995
+    - type: mrr_at_3
+      value: 41.337
+    - type: mrr_at_5
+      value: 42.533
+    - type: ndcg_at_1
+      value: 35.573
+    - type: ndcg_at_10
+      value: 45.275999999999996
+    - type: ndcg_at_100
+      value: 50.94
+    - type: ndcg_at_1000
+      value: 52.893
+    - type: ndcg_at_3
+      value: 40.693
+    - type: ndcg_at_5
+      value: 42.198
+    - type: precision_at_1
+      value: 35.573
+    - type: precision_at_10
+      value: 8.715
+    - type: precision_at_100
+      value: 1.7209999999999999
+    - type: precision_at_1000
+      value: 0.252
+    - type: precision_at_3
+      value: 19.302
+    - type: precision_at_5
+      value: 13.439
+    - type: recall_at_1
+      value: 29.575000000000003
+    - type: recall_at_10
+      value: 56.65599999999999
+    - type: recall_at_100
+      value: 81.999
+    - type: recall_at_1000
+      value: 93.999
+    - type: recall_at_3
+      value: 42.768
+    - type: recall_at_5
+      value: 47.54
+  - task:
+      type: Retrieval
+    dataset:
+      type: BeIR/cqadupstack
+      name: MTEB CQADupstackWordpressRetrieval
+      config: default
+      split: test
+      revision: None
+    metrics:
+    - type: map_at_1
+      value: 21.047
+    - type: map_at_10
+      value: 28.96
+    - type: map_at_100
+      value: 29.904999999999998
+    - type: map_at_1000
+      value: 30.019000000000002
+    - type: map_at_3
+      value: 26.461000000000002
+    - type: map_at_5
+      value: 27.801
+    - type: mrr_at_1
+      value: 23.105
+    - type: mrr_at_10
+      value: 31.137999999999998
+    - type: mrr_at_100
+      value: 31.965
+    - type: mrr_at_1000
+      value: 32.039
+    - type: mrr_at_3
+      value: 28.589
+    - type: mrr_at_5
+      value: 30.04
+    - type: ndcg_at_1
+      value: 23.105
+    - type: ndcg_at_10
+      value: 33.841
+    - type: ndcg_at_100
+      value: 38.76
+    - type: ndcg_at_1000
+      value: 41.297
+    - type: ndcg_at_3
+      value: 28.833
+    - type: ndcg_at_5
+      value: 31.19
+    - type: precision_at_1
+      value: 23.105
+    - type: precision_at_10
+      value: 5.434
+    - type: precision_at_100
+      value: 0.8540000000000001
+    - type: precision_at_1000
+      value: 0.11800000000000001
+    - type: precision_at_3
+      value: 12.384
+    - type: precision_at_5
+      value: 8.799
+    - type: recall_at_1
+      value: 21.047
+    - type: recall_at_10
+      value: 46.768
+    - type: recall_at_100
+      value: 69.782
+    - type: recall_at_1000
+      value: 88.384
+    - type: recall_at_3
+      value: 33.444
+    - type: recall_at_5
+      value: 39.062999999999995
   - task:
       type: Retrieval
     dataset: