metadata
license: bigscience-bloom-rail-1.0
language:
- ak
- ar
- as
- bm
- bn
- ca
- code
- en
- es
- eu
- fon
- fr
- gu
- hi
- id
- ig
- ki
- kn
- lg
- ln
- ml
- mr
- ne
- nso
- ny
- or
- pa
- pt
- rn
- rw
- sn
- st
- sw
- ta
- te
- tn
- ts
- tum
- tw
- ur
- vi
- wo
- xh
- yo
- zh
- zhs
- zht
- zu
tags:
- mteb
model-index:
- name: udever-bloom-3b
results:
- task:
type: STS
dataset:
type: C-MTEB/AFQMC
name: MTEB AFQMC
config: default
split: validation
revision: None
metrics:
- type: cos_sim_pearson
value: 30.0892025910701
- type: cos_sim_spearman
value: 30.549960550731782
- type: euclidean_pearson
value: 29.68940732194022
- type: euclidean_spearman
value: 30.254869740623715
- type: manhattan_pearson
value: 29.693089299297732
- type: manhattan_spearman
value: 30.21293218369479
- task:
type: STS
dataset:
type: C-MTEB/ATEC
name: MTEB ATEC
config: default
split: test
revision: None
metrics:
- type: cos_sim_pearson
value: 36.469490571108054
- type: cos_sim_spearman
value: 37.34843946308442
- type: euclidean_pearson
value: 39.697664194640886
- type: euclidean_spearman
value: 37.623976566242334
- type: manhattan_pearson
value: 39.8389981955552
- type: manhattan_spearman
value: 37.689111419556
- task:
type: Classification
dataset:
type: mteb/amazon_counterfactual
name: MTEB AmazonCounterfactualClassification (en)
config: en
split: test
revision: e8379541af4e31359cca9fbcf4b00f2671dba205
metrics:
- type: accuracy
value: 78.8955223880597
- type: ap
value: 43.270679598956285
- type: f1
value: 73.10740489387823
- task:
type: Classification
dataset:
type: mteb/amazon_polarity
name: MTEB AmazonPolarityClassification
config: default
split: test
revision: e2d317d38cd51312af73b3d32a06d1a08b442046
metrics:
- type: accuracy
value: 87.981225
- type: ap
value: 83.55047186016726
- type: f1
value: 87.95185650917034
- task:
type: Classification
dataset:
type: mteb/amazon_reviews_multi
name: MTEB AmazonReviewsClassification (en)
config: en
split: test
revision: 1399c76144fd37290681b995c656ef9b2e06e26d
metrics:
- type: accuracy
value: 42.58
- type: f1
value: 42.011158109228425
- task:
type: Retrieval
dataset:
type: arguana
name: MTEB ArguAna
config: default
split: test
revision: None
metrics:
- type: map_at_1
value: 22.688
- type: map_at_10
value: 38.855000000000004
- type: map_at_100
value: 39.859
- type: map_at_1000
value: 39.871
- type: map_at_3
value: 33.428000000000004
- type: map_at_5
value: 36.571999999999996
- type: mrr_at_1
value: 23.044
- type: mrr_at_10
value: 39.022
- type: mrr_at_100
value: 40.019
- type: mrr_at_1000
value: 40.03
- type: mrr_at_3
value: 33.642
- type: mrr_at_5
value: 36.707
- type: ndcg_at_1
value: 22.688
- type: ndcg_at_10
value: 48.33
- type: ndcg_at_100
value: 52.616
- type: ndcg_at_1000
value: 52.891999999999996
- type: ndcg_at_3
value: 37.104
- type: ndcg_at_5
value: 42.764
- type: precision_at_1
value: 22.688
- type: precision_at_10
value: 7.881
- type: precision_at_100
value: 0.975
- type: precision_at_1000
value: 0.1
- type: precision_at_3
value: 15.931999999999999
- type: precision_at_5
value: 12.304
- type: recall_at_1
value: 22.688
- type: recall_at_10
value: 78.805
- type: recall_at_100
value: 97.51100000000001
- type: recall_at_1000
value: 99.644
- type: recall_at_3
value: 47.795
- type: recall_at_5
value: 61.522
- task:
type: Clustering
dataset:
type: mteb/arxiv-clustering-p2p
name: MTEB ArxivClusteringP2P
config: default
split: test
revision: a122ad7f3f0291bf49cc6f4d32aa80929df69d5d
metrics:
- type: v_measure
value: 45.37384003345981
- task:
type: Clustering
dataset:
type: mteb/arxiv-clustering-s2s
name: MTEB ArxivClusteringS2S
config: default
split: test
revision: f910caf1a6075f7329cdf8c1a6135696f37dbd53
metrics:
- type: v_measure
value: 36.52143615051018
- task:
type: Reranking
dataset:
type: mteb/askubuntudupquestions-reranking
name: MTEB AskUbuntuDupQuestions
config: default
split: test
revision: 2000358ca161889fa9c082cb41daa8dcfb161a54
metrics:
- type: map
value: 59.91826882625199
- type: mrr
value: 73.30530273051049
- task:
type: STS
dataset:
type: mteb/biosses-sts
name: MTEB BIOSSES
config: default
split: test
revision: d3fb88f8f02e40887cd149695127462bbcf29b4a
metrics:
- type: cos_sim_pearson
value: 86.80556032491437
- type: cos_sim_spearman
value: 84.81639043031876
- type: euclidean_pearson
value: 84.20426417923026
- type: euclidean_spearman
value: 83.53503593258247
- type: manhattan_pearson
value: 84.25387997667964
- type: manhattan_spearman
value: 83.11394200032217
- task:
type: STS
dataset:
type: C-MTEB/BQ
name: MTEB BQ
config: default
split: test
revision: None
metrics:
- type: cos_sim_pearson
value: 47.017986848644625
- type: cos_sim_spearman
value: 47.16708658456057
- type: euclidean_pearson
value: 47.81098065168003
- type: euclidean_spearman
value: 48.01014499886206
- type: manhattan_pearson
value: 48.013333352251244
- type: manhattan_spearman
value: 48.252964666749016
- task:
type: BitextMining
dataset:
type: mteb/bucc-bitext-mining
name: MTEB BUCC (de-en)
config: de-en
split: test
revision: d51519689f32196a32af33b075a01d0e7c51e252
metrics:
- type: accuracy
value: 71.78496868475992
- type: f1
value: 71.05715215634456
- type: precision
value: 70.7532208520454
- type: recall
value: 71.78496868475992
- task:
type: BitextMining
dataset:
type: mteb/bucc-bitext-mining
name: MTEB BUCC (fr-en)
config: fr-en
split: test
revision: d51519689f32196a32af33b075a01d0e7c51e252
metrics:
- type: accuracy
value: 98.34910851860005
- type: f1
value: 98.16751045564604
- type: precision
value: 98.07762858610317
- type: recall
value: 98.34910851860005
- task:
type: BitextMining
dataset:
type: mteb/bucc-bitext-mining
name: MTEB BUCC (ru-en)
config: ru-en
split: test
revision: d51519689f32196a32af33b075a01d0e7c51e252
metrics:
- type: accuracy
value: 59.965361967440245
- type: f1
value: 58.44898687503467
- type: precision
value: 57.83301194437321
- type: recall
value: 59.965361967440245
- task:
type: BitextMining
dataset:
type: mteb/bucc-bitext-mining
name: MTEB BUCC (zh-en)
config: zh-en
split: test
revision: d51519689f32196a32af33b075a01d0e7c51e252
metrics:
- type: accuracy
value: 98.63085834649816
- type: f1
value: 98.59575215025451
- type: precision
value: 98.5781990521327
- type: recall
value: 98.63085834649816
- task:
type: Classification
dataset:
type: mteb/banking77
name: MTEB Banking77Classification
config: default
split: test
revision: 0fd18e25b25c072e09e0d92ab615fda904d66300
metrics:
- type: accuracy
value: 84.15584415584416
- type: f1
value: 84.1389435939967
- task:
type: Clustering
dataset:
type: mteb/biorxiv-clustering-p2p
name: MTEB BiorxivClusteringP2P
config: default
split: test
revision: 65b79d1d13f80053f67aca9498d9402c2d9f1f40
metrics:
- type: v_measure
value: 36.52184607783334
- task:
type: Clustering
dataset:
type: mteb/biorxiv-clustering-s2s
name: MTEB BiorxivClusteringS2S
config: default
split: test
revision: 258694dd0231531bc1fd9de6ceb52a0853c6d908
metrics:
- type: v_measure
value: 31.976191171733653
- task:
type: Clustering
dataset:
type: C-MTEB/CLSClusteringP2P
name: MTEB CLSClusteringP2P
config: default
split: test
revision: None
metrics:
- type: v_measure
value: 36.733774048381484
- task:
type: Clustering
dataset:
type: C-MTEB/CLSClusteringS2S
name: MTEB CLSClusteringS2S
config: default
split: test
revision: None
metrics:
- type: v_measure
value: 36.451952183379056
- task:
type: Reranking
dataset:
type: C-MTEB/CMedQAv1-reranking
name: MTEB CMedQAv1
config: default
split: test
revision: None
metrics:
- type: map
value: 68.9131612041328
- type: mrr
value: 73.47626984126985
- task:
type: Reranking
dataset:
type: C-MTEB/CMedQAv2-reranking
name: MTEB CMedQAv2
config: default
split: test
revision: None
metrics:
- type: map
value: 69.42233467142258
- type: mrr
value: 74.22722222222221
- task:
type: Retrieval
dataset:
type: BeIR/cqadupstack
name: MTEB CQADupstackAndroidRetrieval
config: default
split: test
revision: None
metrics:
- type: map_at_1
value: 32.943
- type: map_at_10
value: 42.796
- type: map_at_100
value: 44.141999999999996
- type: map_at_1000
value: 44.277
- type: map_at_3
value: 39.201
- type: map_at_5
value: 41.262
- type: mrr_at_1
value: 41.488
- type: mrr_at_10
value: 49.214999999999996
- type: mrr_at_100
value: 50.02799999999999
- type: mrr_at_1000
value: 50.075
- type: mrr_at_3
value: 46.733000000000004
- type: mrr_at_5
value: 48.171
- type: ndcg_at_1
value: 41.488
- type: ndcg_at_10
value: 48.619
- type: ndcg_at_100
value: 53.868
- type: ndcg_at_1000
value: 56.027
- type: ndcg_at_3
value: 43.765
- type: ndcg_at_5
value: 45.974
- type: precision_at_1
value: 41.488
- type: precision_at_10
value: 9.07
- type: precision_at_100
value: 1.4460000000000002
- type: precision_at_1000
value: 0.19499999999999998
- type: precision_at_3
value: 20.649
- type: precision_at_5
value: 14.878
- type: recall_at_1
value: 32.943
- type: recall_at_10
value: 59.217
- type: recall_at_100
value: 81.337
- type: recall_at_1000
value: 95.185
- type: recall_at_3
value: 44.377
- type: recall_at_5
value: 51.088
- task:
type: Retrieval
dataset:
type: BeIR/cqadupstack
name: MTEB CQADupstackEnglishRetrieval
config: default
split: test
revision: None
metrics:
- type: map_at_1
value: 26.412999999999997
- type: map_at_10
value: 34.766999999999996
- type: map_at_100
value: 35.774
- type: map_at_1000
value: 35.894999999999996
- type: map_at_3
value: 31.935000000000002
- type: map_at_5
value: 33.661
- type: mrr_at_1
value: 33.248
- type: mrr_at_10
value: 40.274
- type: mrr_at_100
value: 40.92
- type: mrr_at_1000
value: 40.977000000000004
- type: mrr_at_3
value: 38.004
- type: mrr_at_5
value: 39.425
- type: ndcg_at_1
value: 33.248
- type: ndcg_at_10
value: 39.828
- type: ndcg_at_100
value: 43.863
- type: ndcg_at_1000
value: 46.228
- type: ndcg_at_3
value: 35.643
- type: ndcg_at_5
value: 37.851
- type: precision_at_1
value: 33.248
- type: precision_at_10
value: 7.4079999999999995
- type: precision_at_100
value: 1.162
- type: precision_at_1000
value: 0.168
- type: precision_at_3
value: 16.964000000000002
- type: precision_at_5
value: 12.267999999999999
- type: recall_at_1
value: 26.412999999999997
- type: recall_at_10
value: 48.93
- type: recall_at_100
value: 66.437
- type: recall_at_1000
value: 81.68900000000001
- type: recall_at_3
value: 36.822
- type: recall_at_5
value: 42.925000000000004
- task:
type: Retrieval
dataset:
type: BeIR/cqadupstack
name: MTEB CQADupstackGamingRetrieval
config: default
split: test
revision: None
metrics:
- type: map_at_1
value: 37.07
- type: map_at_10
value: 49.051
- type: map_at_100
value: 50.13999999999999
- type: map_at_1000
value: 50.2
- type: map_at_3
value: 46.01
- type: map_at_5
value: 47.711
- type: mrr_at_1
value: 42.32
- type: mrr_at_10
value: 52.32
- type: mrr_at_100
value: 53.068000000000005
- type: mrr_at_1000
value: 53.09700000000001
- type: mrr_at_3
value: 49.864000000000004
- type: mrr_at_5
value: 51.312000000000005
- type: ndcg_at_1
value: 42.32
- type: ndcg_at_10
value: 54.727000000000004
- type: ndcg_at_100
value: 59.153
- type: ndcg_at_1000
value: 60.373
- type: ndcg_at_3
value: 49.478
- type: ndcg_at_5
value: 51.998999999999995
- type: precision_at_1
value: 42.32
- type: precision_at_10
value: 8.802999999999999
- type: precision_at_100
value: 1.196
- type: precision_at_1000
value: 0.135
- type: precision_at_3
value: 22.006
- type: precision_at_5
value: 15.072
- type: recall_at_1
value: 37.07
- type: recall_at_10
value: 68.221
- type: recall_at_100
value: 87.22999999999999
- type: recall_at_1000
value: 95.929
- type: recall_at_3
value: 54.321
- type: recall_at_5
value: 60.358000000000004
- task:
type: Retrieval
dataset:
type: BeIR/cqadupstack
name: MTEB CQADupstackGisRetrieval
config: default
split: test
revision: None
metrics:
- type: map_at_1
value: 23.055
- type: map_at_10
value: 31.163999999999998
- type: map_at_100
value: 32.213
- type: map_at_1000
value: 32.303
- type: map_at_3
value: 28.610000000000003
- type: map_at_5
value: 30.091
- type: mrr_at_1
value: 24.972
- type: mrr_at_10
value: 32.981
- type: mrr_at_100
value: 33.948
- type: mrr_at_1000
value: 34.015
- type: mrr_at_3
value: 30.546
- type: mrr_at_5
value: 31.959
- type: ndcg_at_1
value: 24.972
- type: ndcg_at_10
value: 35.806
- type: ndcg_at_100
value: 40.991
- type: ndcg_at_1000
value: 43.296
- type: ndcg_at_3
value: 30.849
- type: ndcg_at_5
value: 33.334
- type: precision_at_1
value: 24.972
- type: precision_at_10
value: 5.571000000000001
- type: precision_at_100
value: 0.853
- type: precision_at_1000
value: 0.109
- type: precision_at_3
value: 12.956999999999999
- type: precision_at_5
value: 9.333
- type: recall_at_1
value: 23.055
- type: recall_at_10
value: 48.301
- type: recall_at_100
value: 72.051
- type: recall_at_1000
value: 89.408
- type: recall_at_3
value: 35.315000000000005
- type: recall_at_5
value: 41.031
- task:
type: Retrieval
dataset:
type: BeIR/cqadupstack
name: MTEB CQADupstackMathematicaRetrieval
config: default
split: test
revision: None
metrics:
- type: map_at_1
value: 14.782
- type: map_at_10
value: 21.94
- type: map_at_100
value: 23.172
- type: map_at_1000
value: 23.302999999999997
- type: map_at_3
value: 19.911
- type: map_at_5
value: 20.998
- type: mrr_at_1
value: 18.407999999999998
- type: mrr_at_10
value: 25.936999999999998
- type: mrr_at_100
value: 27.035999999999998
- type: mrr_at_1000
value: 27.118
- type: mrr_at_3
value: 23.983999999999998
- type: mrr_at_5
value: 25.141000000000002
- type: ndcg_at_1
value: 18.407999999999998
- type: ndcg_at_10
value: 26.387
- type: ndcg_at_100
value: 32.606
- type: ndcg_at_1000
value: 35.744
- type: ndcg_at_3
value: 22.686999999999998
- type: ndcg_at_5
value: 24.375
- type: precision_at_1
value: 18.407999999999998
- type: precision_at_10
value: 4.801
- type: precision_at_100
value: 0.9299999999999999
- type: precision_at_1000
value: 0.134
- type: precision_at_3
value: 10.945
- type: precision_at_5
value: 7.811
- type: recall_at_1
value: 14.782
- type: recall_at_10
value: 36.018
- type: recall_at_100
value: 63.552
- type: recall_at_1000
value: 85.857
- type: recall_at_3
value: 25.898
- type: recall_at_5
value: 30.081999999999997
- task:
type: Retrieval
dataset:
type: BeIR/cqadupstack
name: MTEB CQADupstackPhysicsRetrieval
config: default
split: test
revision: None
metrics:
- type: map_at_1
value: 26.369
- type: map_at_10
value: 37.704
- type: map_at_100
value: 39.018
- type: map_at_1000
value: 39.134
- type: map_at_3
value: 34.243
- type: map_at_5
value: 36.083
- type: mrr_at_1
value: 32.916000000000004
- type: mrr_at_10
value: 43.488
- type: mrr_at_100
value: 44.29
- type: mrr_at_1000
value: 44.336999999999996
- type: mrr_at_3
value: 40.696
- type: mrr_at_5
value: 42.289
- type: ndcg_at_1
value: 32.916000000000004
- type: ndcg_at_10
value: 44.362
- type: ndcg_at_100
value: 49.730999999999995
- type: ndcg_at_1000
value: 51.857
- type: ndcg_at_3
value: 38.683
- type: ndcg_at_5
value: 41.249
- type: precision_at_1
value: 32.916000000000004
- type: precision_at_10
value: 8.412
- type: precision_at_100
value: 1.2970000000000002
- type: precision_at_1000
value: 0.166
- type: precision_at_3
value: 18.895999999999997
- type: precision_at_5
value: 13.550999999999998
- type: recall_at_1
value: 26.369
- type: recall_at_10
value: 58.464000000000006
- type: recall_at_100
value: 80.884
- type: recall_at_1000
value: 94.676
- type: recall_at_3
value: 42.485
- type: recall_at_5
value: 49.262
- task:
type: Retrieval
dataset:
type: BeIR/cqadupstack
name: MTEB CQADupstackProgrammersRetrieval
config: default
split: test
revision: None
metrics:
- type: map_at_1
value: 23.896
- type: map_at_10
value: 33.384
- type: map_at_100
value: 34.683
- type: map_at_1000
value: 34.807
- type: map_at_3
value: 30.724
- type: map_at_5
value: 32.339
- type: mrr_at_1
value: 29.909000000000002
- type: mrr_at_10
value: 38.395
- type: mrr_at_100
value: 39.339
- type: mrr_at_1000
value: 39.404
- type: mrr_at_3
value: 36.339
- type: mrr_at_5
value: 37.618
- type: ndcg_at_1
value: 29.909000000000002
- type: ndcg_at_10
value: 38.688
- type: ndcg_at_100
value: 44.399
- type: ndcg_at_1000
value: 46.942
- type: ndcg_at_3
value: 34.548
- type: ndcg_at_5
value: 36.605
- type: precision_at_1
value: 29.909000000000002
- type: precision_at_10
value: 7.066
- type: precision_at_100
value: 1.174
- type: precision_at_1000
value: 0.155
- type: precision_at_3
value: 16.819
- type: precision_at_5
value: 11.872
- type: recall_at_1
value: 23.896
- type: recall_at_10
value: 49.531
- type: recall_at_100
value: 73.977
- type: recall_at_1000
value: 91.393
- type: recall_at_3
value: 37.53
- type: recall_at_5
value: 43.373
- task:
type: Retrieval
dataset:
type: BeIR/cqadupstack
name: MTEB CQADupstackRetrieval
config: default
split: test
revision: None
metrics:
- type: map_at_1
value: 24.153166666666667
- type: map_at_10
value: 32.7705
- type: map_at_100
value: 33.93133333333334
- type: map_at_1000
value: 34.052499999999995
- type: map_at_3
value: 30.158500000000004
- type: map_at_5
value: 31.595916666666664
- type: mrr_at_1
value: 28.87725
- type: mrr_at_10
value: 36.86358333333333
- type: mrr_at_100
value: 37.74550000000001
- type: mrr_at_1000
value: 37.80916666666666
- type: mrr_at_3
value: 34.634499999999996
- type: mrr_at_5
value: 35.926750000000006
- type: ndcg_at_1
value: 28.87725
- type: ndcg_at_10
value: 37.82341666666667
- type: ndcg_at_100
value: 42.98408333333333
- type: ndcg_at_1000
value: 45.44883333333333
- type: ndcg_at_3
value: 33.41875000000001
- type: ndcg_at_5
value: 35.45158333333333
- type: precision_at_1
value: 28.87725
- type: precision_at_10
value: 6.638249999999999
- type: precision_at_100
value: 1.0863333333333334
- type: precision_at_1000
value: 0.14858333333333335
- type: precision_at_3
value: 15.481
- type: precision_at_5
value: 10.953916666666668
- type: recall_at_1
value: 24.153166666666667
- type: recall_at_10
value: 48.796499999999995
- type: recall_at_100
value: 71.53716666666666
- type: recall_at_1000
value: 88.72158333333333
- type: recall_at_3
value: 36.419583333333335
- type: recall_at_5
value: 41.735833333333325
- task:
type: Retrieval
dataset:
type: BeIR/cqadupstack
name: MTEB CQADupstackStatsRetrieval
config: default
split: test
revision: None
metrics:
- type: map_at_1
value: 21.523
- type: map_at_10
value: 28.915000000000003
- type: map_at_100
value: 29.808
- type: map_at_1000
value: 29.910999999999998
- type: map_at_3
value: 26.863999999999997
- type: map_at_5
value: 27.801
- type: mrr_at_1
value: 24.387
- type: mrr_at_10
value: 31.703
- type: mrr_at_100
value: 32.481
- type: mrr_at_1000
value: 32.559
- type: mrr_at_3
value: 29.805999999999997
- type: mrr_at_5
value: 30.688
- type: ndcg_at_1
value: 24.387
- type: ndcg_at_10
value: 33.272
- type: ndcg_at_100
value: 37.79
- type: ndcg_at_1000
value: 40.428
- type: ndcg_at_3
value: 29.409000000000002
- type: ndcg_at_5
value: 30.813000000000002
- type: precision_at_1
value: 24.387
- type: precision_at_10
value: 5.337
- type: precision_at_100
value: 0.8240000000000001
- type: precision_at_1000
value: 0.11299999999999999
- type: precision_at_3
value: 13.19
- type: precision_at_5
value: 8.926
- type: recall_at_1
value: 21.523
- type: recall_at_10
value: 44.054
- type: recall_at_100
value: 64.80900000000001
- type: recall_at_1000
value: 84.265
- type: recall_at_3
value: 33.019999999999996
- type: recall_at_5
value: 36.561
- task:
type: Retrieval
dataset:
type: BeIR/cqadupstack
name: MTEB CQADupstackTexRetrieval
config: default
split: test
revision: None
metrics:
- type: map_at_1
value: 15.461
- type: map_at_10
value: 21.802
- type: map_at_100
value: 22.825
- type: map_at_1000
value: 22.95
- type: map_at_3
value: 19.79
- type: map_at_5
value: 20.828
- type: mrr_at_1
value: 18.789
- type: mrr_at_10
value: 25.373
- type: mrr_at_100
value: 26.269
- type: mrr_at_1000
value: 26.355
- type: mrr_at_3
value: 23.394000000000002
- type: mrr_at_5
value: 24.451999999999998
- type: ndcg_at_1
value: 18.789
- type: ndcg_at_10
value: 25.948
- type: ndcg_at_100
value: 30.926
- type: ndcg_at_1000
value: 33.938
- type: ndcg_at_3
value: 22.281000000000002
- type: ndcg_at_5
value: 23.818
- type: precision_at_1
value: 18.789
- type: precision_at_10
value: 4.766
- type: precision_at_100
value: 0.848
- type: precision_at_1000
value: 0.127
- type: precision_at_3
value: 10.633
- type: precision_at_5
value: 7.6259999999999994
- type: recall_at_1
value: 15.461
- type: recall_at_10
value: 34.967999999999996
- type: recall_at_100
value: 57.25900000000001
- type: recall_at_1000
value: 78.738
- type: recall_at_3
value: 24.495
- type: recall_at_5
value: 28.510999999999996
- task:
type: Retrieval
dataset:
type: BeIR/cqadupstack
name: MTEB CQADupstackUnixRetrieval
config: default
split: test
revision: None
metrics:
- type: map_at_1
value: 25.165
- type: map_at_10
value: 32.66
- type: map_at_100
value: 33.842
- type: map_at_1000
value: 33.952
- type: map_at_3
value: 30.503999999999998
- type: map_at_5
value: 31.546000000000003
- type: mrr_at_1
value: 29.851
- type: mrr_at_10
value: 37.112
- type: mrr_at_100
value: 38.057
- type: mrr_at_1000
value: 38.119
- type: mrr_at_3
value: 35.106
- type: mrr_at_5
value: 36.22
- type: ndcg_at_1
value: 29.851
- type: ndcg_at_10
value: 37.395
- type: ndcg_at_100
value: 42.906
- type: ndcg_at_1000
value: 45.427
- type: ndcg_at_3
value: 33.465
- type: ndcg_at_5
value: 35.02
- type: precision_at_1
value: 29.851
- type: precision_at_10
value: 6.166
- type: precision_at_100
value: 1.005
- type: precision_at_1000
value: 0.132
- type: precision_at_3
value: 15.235999999999999
- type: precision_at_5
value: 10.354
- type: recall_at_1
value: 25.165
- type: recall_at_10
value: 47.439
- type: recall_at_100
value: 71.56099999999999
- type: recall_at_1000
value: 89.435
- type: recall_at_3
value: 36.275
- type: recall_at_5
value: 40.435
- task:
type: Retrieval
dataset:
type: BeIR/cqadupstack
name: MTEB CQADupstackWebmastersRetrieval
config: default
split: test
revision: None
metrics:
- type: map_at_1
value: 25.589000000000002
- type: map_at_10
value: 33.729
- type: map_at_100
value: 35.306
- type: map_at_1000
value: 35.552
- type: map_at_3
value: 30.988
- type: map_at_5
value: 32.406
- type: mrr_at_1
value: 30.830000000000002
- type: mrr_at_10
value: 38.446999999999996
- type: mrr_at_100
value: 39.478
- type: mrr_at_1000
value: 39.544000000000004
- type: mrr_at_3
value: 36.034
- type: mrr_at_5
value: 37.546
- type: ndcg_at_1
value: 30.830000000000002
- type: ndcg_at_10
value: 39.22
- type: ndcg_at_100
value: 45.004
- type: ndcg_at_1000
value: 47.837
- type: ndcg_at_3
value: 34.811
- type: ndcg_at_5
value: 36.831
- type: precision_at_1
value: 30.830000000000002
- type: precision_at_10
value: 7.489999999999999
- type: precision_at_100
value: 1.534
- type: precision_at_1000
value: 0.241
- type: precision_at_3
value: 16.14
- type: precision_at_5
value: 11.66
- type: recall_at_1
value: 25.589000000000002
- type: recall_at_10
value: 49.238
- type: recall_at_100
value: 74.893
- type: recall_at_1000
value: 92.902
- type: recall_at_3
value: 36.75
- type: recall_at_5
value: 42.256
- task:
type: Retrieval
dataset:
type: BeIR/cqadupstack
name: MTEB CQADupstackWordpressRetrieval
config: default
split: test
revision: None
metrics:
- type: map_at_1
value: 17.572
- type: map_at_10
value: 25.334
- type: map_at_100
value: 26.253
- type: map_at_1000
value: 26.346000000000004
- type: map_at_3
value: 23.122
- type: map_at_5
value: 24.425
- type: mrr_at_1
value: 19.409000000000002
- type: mrr_at_10
value: 27.118
- type: mrr_at_100
value: 28.032
- type: mrr_at_1000
value: 28.110000000000003
- type: mrr_at_3
value: 25.108000000000004
- type: mrr_at_5
value: 26.3
- type: ndcg_at_1
value: 19.409000000000002
- type: ndcg_at_10
value: 29.629
- type: ndcg_at_100
value: 34.572
- type: ndcg_at_1000
value: 37.289
- type: ndcg_at_3
value: 25.406000000000002
- type: ndcg_at_5
value: 27.55
- type: precision_at_1
value: 19.409000000000002
- type: precision_at_10
value: 4.769
- type: precision_at_100
value: 0.767
- type: precision_at_1000
value: 0.108
- type: precision_at_3
value: 11.337
- type: precision_at_5
value: 8.096
- type: recall_at_1
value: 17.572
- type: recall_at_10
value: 41.177
- type: recall_at_100
value: 64.456
- type: recall_at_1000
value: 85.182
- type: recall_at_3
value: 29.747
- type: recall_at_5
value: 34.948
- task:
type: Retrieval
dataset:
type: climate-fever
name: MTEB ClimateFEVER
config: default
split: test
revision: None
metrics:
- type: map_at_1
value: 9.264
- type: map_at_10
value: 16.09
- type: map_at_100
value: 17.717
- type: map_at_1000
value: 17.903
- type: map_at_3
value: 13.422
- type: map_at_5
value: 14.78
- type: mrr_at_1
value: 20.326
- type: mrr_at_10
value: 31.274
- type: mrr_at_100
value: 32.312999999999995
- type: mrr_at_1000
value: 32.365
- type: mrr_at_3
value: 27.959
- type: mrr_at_5
value: 29.877
- type: ndcg_at_1
value: 20.326
- type: ndcg_at_10
value: 23.358
- type: ndcg_at_100
value: 30.36
- type: ndcg_at_1000
value: 33.883
- type: ndcg_at_3
value: 18.704
- type: ndcg_at_5
value: 20.374
- type: precision_at_1
value: 20.326
- type: precision_at_10
value: 7.303
- type: precision_at_100
value: 1.488
- type: precision_at_1000
value: 0.214
- type: precision_at_3
value: 13.811000000000002
- type: precision_at_5
value: 10.84
- type: recall_at_1
value: 9.264
- type: recall_at_10
value: 29.177999999999997
- type: recall_at_100
value: 53.61900000000001
- type: recall_at_1000
value: 73.48400000000001
- type: recall_at_3
value: 17.738
- type: recall_at_5
value: 22.279
- task:
type: Retrieval
dataset:
type: C-MTEB/CmedqaRetrieval
name: MTEB CmedqaRetrieval
config: default
split: dev
revision: None
metrics:
- type: map_at_1
value: 14.494000000000002
- type: map_at_10
value: 21.37
- type: map_at_100
value: 22.741
- type: map_at_1000
value: 22.911
- type: map_at_3
value: 18.929000000000002
- type: map_at_5
value: 20.244
- type: mrr_at_1
value: 23.105999999999998
- type: mrr_at_10
value: 29.137999999999998
- type: mrr_at_100
value: 30.064
- type: mrr_at_1000
value: 30.152
- type: mrr_at_3
value: 27.119
- type: mrr_at_5
value: 28.301
- type: ndcg_at_1
value: 23.105999999999998
- type: ndcg_at_10
value: 26.182
- type: ndcg_at_100
value: 32.396
- type: ndcg_at_1000
value: 36.177
- type: ndcg_at_3
value: 22.708000000000002
- type: ndcg_at_5
value: 24.137
- type: precision_at_1
value: 23.105999999999998
- type: precision_at_10
value: 6.0040000000000004
- type: precision_at_100
value: 1.119
- type: precision_at_1000
value: 0.161
- type: precision_at_3
value: 13.028
- type: precision_at_5
value: 9.557
- type: recall_at_1
value: 14.494000000000002
- type: recall_at_10
value: 32.910000000000004
- type: recall_at_100
value: 59.202999999999996
- type: recall_at_1000
value: 85.61
- type: recall_at_3
value: 22.397
- type: recall_at_5
value: 26.900000000000002
- task:
type: PairClassification
dataset:
type: C-MTEB/CMNLI
name: MTEB Cmnli
config: default
split: validation
revision: None
metrics:
- type: cos_sim_accuracy
value: 74.91280817799158
- type: cos_sim_ap
value: 83.32013347926805
- type: cos_sim_f1
value: 76.57387580299788
- type: cos_sim_precision
value: 70.63006122852063
- type: cos_sim_recall
value: 83.61000701426234
- type: dot_accuracy
value: 70.5832832230908
- type: dot_ap
value: 75.9647326130666
- type: dot_f1
value: 73.65528072241852
- type: dot_precision
value: 63.47487734731856
- type: dot_recall
value: 87.72504091653029
- type: euclidean_accuracy
value: 74.51593505712569
- type: euclidean_ap
value: 83.04382773676555
- type: euclidean_f1
value: 75.7739770513098
- type: euclidean_precision
value: 70.5502922797823
- type: euclidean_recall
value: 81.83306055646482
- type: manhattan_accuracy
value: 74.73241130487071
- type: manhattan_ap
value: 83.32768114935021
- type: manhattan_f1
value: 76.09116319071167
- type: manhattan_precision
value: 70.42786069651741
- type: manhattan_recall
value: 82.74491465980827
- type: max_accuracy
value: 74.91280817799158
- type: max_ap
value: 83.32768114935021
- type: max_f1
value: 76.57387580299788
- task:
type: Retrieval
dataset:
type: C-MTEB/CovidRetrieval
name: MTEB CovidRetrieval
config: default
split: dev
revision: None
metrics:
- type: map_at_1
value: 55.032000000000004
- type: map_at_10
value: 63.517
- type: map_at_100
value: 64.159
- type: map_at_1000
value: 64.17699999999999
- type: map_at_3
value: 61.503
- type: map_at_5
value: 62.741
- type: mrr_at_1
value: 55.111
- type: mrr_at_10
value: 63.50900000000001
- type: mrr_at_100
value: 64.13499999999999
- type: mrr_at_1000
value: 64.153
- type: mrr_at_3
value: 61.521
- type: mrr_at_5
value: 62.759
- type: ndcg_at_1
value: 55.216
- type: ndcg_at_10
value: 67.569
- type: ndcg_at_100
value: 70.71
- type: ndcg_at_1000
value: 71.211
- type: ndcg_at_3
value: 63.543000000000006
- type: ndcg_at_5
value: 65.718
- type: precision_at_1
value: 55.216
- type: precision_at_10
value: 8.093
- type: precision_at_100
value: 0.96
- type: precision_at_1000
value: 0.1
- type: precision_at_3
value: 23.253
- type: precision_at_5
value: 15.026
- type: recall_at_1
value: 55.032000000000004
- type: recall_at_10
value: 80.163
- type: recall_at_100
value: 94.94200000000001
- type: recall_at_1000
value: 98.946
- type: recall_at_3
value: 69.231
- type: recall_at_5
value: 74.49900000000001
- task:
type: Retrieval
dataset:
type: dbpedia-entity
name: MTEB DBPedia
config: default
split: test
revision: None
metrics:
- type: map_at_1
value: 8.391
- type: map_at_10
value: 16.381999999999998
- type: map_at_100
value: 21.262
- type: map_at_1000
value: 22.461000000000002
- type: map_at_3
value: 12.471
- type: map_at_5
value: 14.016
- type: mrr_at_1
value: 62.25000000000001
- type: mrr_at_10
value: 69.64099999999999
- type: mrr_at_100
value: 70.114
- type: mrr_at_1000
value: 70.128
- type: mrr_at_3
value: 67.958
- type: mrr_at_5
value: 68.996
- type: ndcg_at_1
value: 50.375
- type: ndcg_at_10
value: 34.542
- type: ndcg_at_100
value: 37.265
- type: ndcg_at_1000
value: 44.324000000000005
- type: ndcg_at_3
value: 40.113
- type: ndcg_at_5
value: 37.177
- type: precision_at_1
value: 62.25000000000001
- type: precision_at_10
value: 26.05
- type: precision_at_100
value: 7.632999999999999
- type: precision_at_1000
value: 1.6209999999999998
- type: precision_at_3
value: 42.5
- type: precision_at_5
value: 35.199999999999996
- type: recall_at_1
value: 8.391
- type: recall_at_10
value: 21.099
- type: recall_at_100
value: 40.886
- type: recall_at_1000
value: 63.805
- type: recall_at_3
value: 13.766
- type: recall_at_5
value: 16.128
- task:
type: Retrieval
dataset:
type: C-MTEB/DuRetrieval
name: MTEB DuRetrieval
config: default
split: dev
revision: None
metrics:
- type: map_at_1
value: 21.933
- type: map_at_10
value: 65.739
- type: map_at_100
value: 69.245
- type: map_at_1000
value: 69.33399999999999
- type: map_at_3
value: 44.874
- type: map_at_5
value: 56.242999999999995
- type: mrr_at_1
value: 78.95
- type: mrr_at_10
value: 85.37700000000001
- type: mrr_at_100
value: 85.474
- type: mrr_at_1000
value: 85.481
- type: mrr_at_3
value: 84.63300000000001
- type: mrr_at_5
value: 85.141
- type: ndcg_at_1
value: 78.95
- type: ndcg_at_10
value: 75.81599999999999
- type: ndcg_at_100
value: 80.42399999999999
- type: ndcg_at_1000
value: 81.357
- type: ndcg_at_3
value: 73.821
- type: ndcg_at_5
value: 72.497
- type: precision_at_1
value: 78.95
- type: precision_at_10
value: 37.285000000000004
- type: precision_at_100
value: 4.589
- type: precision_at_1000
value: 0.481
- type: precision_at_3
value: 66.333
- type: precision_at_5
value: 55.879999999999995
- type: recall_at_1
value: 21.933
- type: recall_at_10
value: 77.943
- type: recall_at_100
value: 92.17
- type: recall_at_1000
value: 96.986
- type: recall_at_3
value: 48.079
- type: recall_at_5
value: 62.65500000000001
- task:
type: Retrieval
dataset:
type: C-MTEB/EcomRetrieval
name: MTEB EcomRetrieval
config: default
split: dev
revision: None
metrics:
- type: map_at_1
value: 38.2
- type: map_at_10
value: 46.785
- type: map_at_100
value: 47.635
- type: map_at_1000
value: 47.675
- type: map_at_3
value: 44.583
- type: map_at_5
value: 45.848
- type: mrr_at_1
value: 38.2
- type: mrr_at_10
value: 46.785
- type: mrr_at_100
value: 47.635
- type: mrr_at_1000
value: 47.675
- type: mrr_at_3
value: 44.583
- type: mrr_at_5
value: 45.848
- type: ndcg_at_1
value: 38.2
- type: ndcg_at_10
value: 51.282000000000004
- type: ndcg_at_100
value: 55.608000000000004
- type: ndcg_at_1000
value: 56.726
- type: ndcg_at_3
value: 46.763
- type: ndcg_at_5
value: 49.035000000000004
- type: precision_at_1
value: 38.2
- type: precision_at_10
value: 6.550000000000001
- type: precision_at_100
value: 0.8619999999999999
- type: precision_at_1000
value: 0.095
- type: precision_at_3
value: 17.7
- type: precision_at_5
value: 11.72
- type: recall_at_1
value: 38.2
- type: recall_at_10
value: 65.5
- type: recall_at_100
value: 86.2
- type: recall_at_1000
value: 95.1
- type: recall_at_3
value: 53.1
- type: recall_at_5
value: 58.599999999999994
- task:
type: Classification
dataset:
type: mteb/emotion
name: MTEB EmotionClassification
config: default
split: test
revision: 4f58c6b202a23cf9a4da393831edf4f9183cad37
metrics:
- type: accuracy
value: 47.88
- type: f1
value: 43.30537129784135
- task:
type: Retrieval
dataset:
type: fever
name: MTEB FEVER
config: default
split: test
revision: None
metrics:
- type: map_at_1
value: 54.423
- type: map_at_10
value: 66.136
- type: map_at_100
value: 66.557
- type: map_at_1000
value: 66.57300000000001
- type: map_at_3
value: 64.042
- type: map_at_5
value: 65.366
- type: mrr_at_1
value: 58.745999999999995
- type: mrr_at_10
value: 70.456
- type: mrr_at_100
value: 70.801
- type: mrr_at_1000
value: 70.809
- type: mrr_at_3
value: 68.504
- type: mrr_at_5
value: 69.746
- type: ndcg_at_1
value: 58.745999999999995
- type: ndcg_at_10
value: 71.96000000000001
- type: ndcg_at_100
value: 73.83
- type: ndcg_at_1000
value: 74.17
- type: ndcg_at_3
value: 68.033
- type: ndcg_at_5
value: 70.22
- type: precision_at_1
value: 58.745999999999995
- type: precision_at_10
value: 9.397
- type: precision_at_100
value: 1.043
- type: precision_at_1000
value: 0.108
- type: precision_at_3
value: 27.208
- type: precision_at_5
value: 17.561
- type: recall_at_1
value: 54.423
- type: recall_at_10
value: 85.703
- type: recall_at_100
value: 93.989
- type: recall_at_1000
value: 96.35000000000001
- type: recall_at_3
value: 75.05
- type: recall_at_5
value: 80.447
- task:
type: Retrieval
dataset:
type: fiqa
name: MTEB FiQA2018
config: default
split: test
revision: None
metrics:
- type: map_at_1
value: 16.286
- type: map_at_10
value: 27.499000000000002
- type: map_at_100
value: 29.176999999999996
- type: map_at_1000
value: 29.354999999999997
- type: map_at_3
value: 23.684
- type: map_at_5
value: 25.544
- type: mrr_at_1
value: 32.87
- type: mrr_at_10
value: 41.906
- type: mrr_at_100
value: 42.739
- type: mrr_at_1000
value: 42.78
- type: mrr_at_3
value: 38.992
- type: mrr_at_5
value: 40.535
- type: ndcg_at_1
value: 32.87
- type: ndcg_at_10
value: 35.124
- type: ndcg_at_100
value: 41.638
- type: ndcg_at_1000
value: 44.869
- type: ndcg_at_3
value: 30.975
- type: ndcg_at_5
value: 32.112
- type: precision_at_1
value: 32.87
- type: precision_at_10
value: 10.062
- type: precision_at_100
value: 1.653
- type: precision_at_1000
value: 0.22599999999999998
- type: precision_at_3
value: 20.833
- type: precision_at_5
value: 15.340000000000002
- type: recall_at_1
value: 16.286
- type: recall_at_10
value: 42.734
- type: recall_at_100
value: 67.582
- type: recall_at_1000
value: 86.735
- type: recall_at_3
value: 28.438000000000002
- type: recall_at_5
value: 33.944
- task:
type: Retrieval
dataset:
type: hotpotqa
name: MTEB HotpotQA
config: default
split: test
revision: None
metrics:
- type: map_at_1
value: 33.606
- type: map_at_10
value: 46.085
- type: map_at_100
value: 46.796
- type: map_at_1000
value: 46.866
- type: map_at_3
value: 43.614000000000004
- type: map_at_5
value: 45.094
- type: mrr_at_1
value: 67.211
- type: mrr_at_10
value: 73.447
- type: mrr_at_100
value: 73.734
- type: mrr_at_1000
value: 73.752
- type: mrr_at_3
value: 72.233
- type: mrr_at_5
value: 72.982
- type: ndcg_at_1
value: 67.211
- type: ndcg_at_10
value: 55.125
- type: ndcg_at_100
value: 57.904999999999994
- type: ndcg_at_1000
value: 59.40800000000001
- type: ndcg_at_3
value: 51.283
- type: ndcg_at_5
value: 53.32599999999999
- type: precision_at_1
value: 67.211
- type: precision_at_10
value: 11.198
- type: precision_at_100
value: 1.34
- type: precision_at_1000
value: 0.154
- type: precision_at_3
value: 31.631999999999998
- type: precision_at_5
value: 20.591
- type: recall_at_1
value: 33.606
- type: recall_at_10
value: 55.989
- type: recall_at_100
value: 67.01599999999999
- type: recall_at_1000
value: 77.076
- type: recall_at_3
value: 47.448
- type: recall_at_5
value: 51.479
- task:
type: Classification
dataset:
type: C-MTEB/IFlyTek-classification
name: MTEB IFlyTek
config: default
split: validation
revision: None
metrics:
- type: accuracy
value: 45.02500961908426
- type: f1
value: 36.80024928040335
- task:
type: Classification
dataset:
type: mteb/imdb
name: MTEB ImdbClassification
config: default
split: test
revision: 3d86128a09e091d6018b6d26cad27f2739fc2db7
metrics:
- type: accuracy
value: 77.698
- type: ap
value: 72.08492726312224
- type: f1
value: 77.57721549038352
- task:
type: Classification
dataset:
type: C-MTEB/JDReview-classification
name: MTEB JDReview
config: default
split: test
revision: None
metrics:
- type: accuracy
value: 83.63977485928706
- type: ap
value: 48.33680179995013
- type: f1
value: 77.42875376726259
- task:
type: STS
dataset:
type: C-MTEB/LCQMC
name: MTEB LCQMC
config: default
split: test
revision: None
metrics:
- type: cos_sim_pearson
value: 67.71826986847978
- type: cos_sim_spearman
value: 75.31951271324436
- type: euclidean_pearson
value: 73.99129929755692
- type: euclidean_spearman
value: 75.50510874612128
- type: manhattan_pearson
value: 74.1581557667118
- type: manhattan_spearman
value: 75.62495446886778
- task:
type: Retrieval
dataset:
type: C-MTEB/MMarcoRetrieval
name: MTEB MMarcoRetrieval
config: default
split: dev
revision: None
metrics:
- type: map_at_1
value: 64.305
- type: map_at_10
value: 73.286
- type: map_at_100
value: 73.661
- type: map_at_1000
value: 73.675
- type: map_at_3
value: 71.433
- type: map_at_5
value: 72.596
- type: mrr_at_1
value: 66.562
- type: mrr_at_10
value: 73.932
- type: mrr_at_100
value: 74.265
- type: mrr_at_1000
value: 74.278
- type: mrr_at_3
value: 72.333
- type: mrr_at_5
value: 73.322
- type: ndcg_at_1
value: 66.562
- type: ndcg_at_10
value: 76.998
- type: ndcg_at_100
value: 78.684
- type: ndcg_at_1000
value: 79.038
- type: ndcg_at_3
value: 73.491
- type: ndcg_at_5
value: 75.436
- type: precision_at_1
value: 66.562
- type: precision_at_10
value: 9.34
- type: precision_at_100
value: 1.018
- type: precision_at_1000
value: 0.105
- type: precision_at_3
value: 27.683999999999997
- type: precision_at_5
value: 17.645
- type: recall_at_1
value: 64.305
- type: recall_at_10
value: 87.825
- type: recall_at_100
value: 95.451
- type: recall_at_1000
value: 98.17
- type: recall_at_3
value: 78.522
- type: recall_at_5
value: 83.146
- task:
type: Retrieval
dataset:
type: msmarco
name: MTEB MSMARCO
config: default
split: dev
revision: None
metrics:
- type: map_at_1
value: 21.862000000000002
- type: map_at_10
value: 33.635999999999996
- type: map_at_100
value: 34.833
- type: map_at_1000
value: 34.886
- type: map_at_3
value: 29.916999999999998
- type: map_at_5
value: 32.042
- type: mrr_at_1
value: 22.493
- type: mrr_at_10
value: 34.217999999999996
- type: mrr_at_100
value: 35.365
- type: mrr_at_1000
value: 35.411
- type: mrr_at_3
value: 30.585
- type: mrr_at_5
value: 32.659
- type: ndcg_at_1
value: 22.493
- type: ndcg_at_10
value: 40.247
- type: ndcg_at_100
value: 46.025
- type: ndcg_at_1000
value: 47.343
- type: ndcg_at_3
value: 32.696999999999996
- type: ndcg_at_5
value: 36.476
- type: precision_at_1
value: 22.493
- type: precision_at_10
value: 6.334
- type: precision_at_100
value: 0.922
- type: precision_at_1000
value: 0.104
- type: precision_at_3
value: 13.863
- type: precision_at_5
value: 10.232
- type: recall_at_1
value: 21.862000000000002
- type: recall_at_10
value: 60.56700000000001
- type: recall_at_100
value: 87.261
- type: recall_at_1000
value: 97.365
- type: recall_at_3
value: 40.081
- type: recall_at_5
value: 49.16
- task:
type: Classification
dataset:
type: mteb/mtop_domain
name: MTEB MTOPDomainClassification (en)
config: en
split: test
revision: d80d48c1eb48d3562165c59d59d0034df9fff0bf
metrics:
- type: accuracy
value: 92.34154126766987
- type: f1
value: 92.05415284766352
- task:
type: Classification
dataset:
type: mteb/mtop_intent
name: MTEB MTOPIntentClassification (en)
config: en
split: test
revision: ae001d0e6b1228650b7bd1c2c65fb50ad11a8aba
metrics:
- type: accuracy
value: 70.63155494756043
- type: f1
value: 53.392602505424435
- task:
type: Classification
dataset:
type: mteb/amazon_massive_intent
name: MTEB MassiveIntentClassification (en)
config: en
split: test
revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7
metrics:
- type: accuracy
value: 70.39340954942837
- type: f1
value: 68.85705470713275
- task:
type: Classification
dataset:
type: mteb/amazon_massive_scenario
name: MTEB MassiveScenarioClassification (en)
config: en
split: test
revision: 7d571f92784cd94a019292a1f45445077d0ef634
metrics:
- type: accuracy
value: 77.18897108271688
- type: f1
value: 77.36699772115247
- task:
type: Retrieval
dataset:
type: C-MTEB/MedicalRetrieval
name: MTEB MedicalRetrieval
config: default
split: dev
revision: None
metrics:
- type: map_at_1
value: 40.699999999999996
- type: map_at_10
value: 45.304
- type: map_at_100
value: 45.862
- type: map_at_1000
value: 45.923
- type: map_at_3
value: 44.433
- type: map_at_5
value: 44.753
- type: mrr_at_1
value: 40.8
- type: mrr_at_10
value: 45.354
- type: mrr_at_100
value: 45.912
- type: mrr_at_1000
value: 45.973000000000006
- type: mrr_at_3
value: 44.483
- type: mrr_at_5
value: 44.803
- type: ndcg_at_1
value: 40.699999999999996
- type: ndcg_at_10
value: 47.477999999999994
- type: ndcg_at_100
value: 50.51
- type: ndcg_at_1000
value: 52.367
- type: ndcg_at_3
value: 45.609
- type: ndcg_at_5
value: 46.186
- type: precision_at_1
value: 40.699999999999996
- type: precision_at_10
value: 5.43
- type: precision_at_100
value: 0.692
- type: precision_at_1000
value: 0.084
- type: precision_at_3
value: 16.333000000000002
- type: precision_at_5
value: 10.08
- type: recall_at_1
value: 40.699999999999996
- type: recall_at_10
value: 54.300000000000004
- type: recall_at_100
value: 69.19999999999999
- type: recall_at_1000
value: 84.3
- type: recall_at_3
value: 49
- type: recall_at_5
value: 50.4
- task:
type: Clustering
dataset:
type: mteb/medrxiv-clustering-p2p
name: MTEB MedrxivClusteringP2P
config: default
split: test
revision: e7a26af6f3ae46b30dde8737f02c07b1505bcc73
metrics:
- type: v_measure
value: 31.70883822617504
- task:
type: Clustering
dataset:
type: mteb/medrxiv-clustering-s2s
name: MTEB MedrxivClusteringS2S
config: default
split: test
revision: 35191c8c0dca72d8ff3efcd72aa802307d469663
metrics:
- type: v_measure
value: 28.801248513598072
- task:
type: Reranking
dataset:
type: mteb/mind_small
name: MTEB MindSmallReranking
config: default
split: test
revision: 3bdac13927fdc888b903db93b2ffdbd90b295a69
metrics:
- type: map
value: 30.97227673339198
- type: mrr
value: 32.03205560232119
- task:
type: Reranking
dataset:
type: C-MTEB/Mmarco-reranking
name: MTEB MMarcoReranking
config: default
split: dev
revision: None
metrics:
- type: map
value: 25.89977615357687
- type: mrr
value: 24.192857142857143
- task:
type: Classification
dataset:
type: C-MTEB/MultilingualSentiment-classification
name: MTEB MultilingualSentiment
config: default
split: validation
revision: None
metrics:
- type: accuracy
value: 67.16666666666666
- type: f1
value: 67.15765577091656
- task:
type: Retrieval
dataset:
type: nfcorpus
name: MTEB NFCorpus
config: default
split: test
revision: None
metrics:
- type: map_at_1
value: 5.079000000000001
- type: map_at_10
value: 12.04
- type: map_at_100
value: 15.375
- type: map_at_1000
value: 16.878
- type: map_at_3
value: 8.851
- type: map_at_5
value: 10.23
- type: mrr_at_1
value: 43.963
- type: mrr_at_10
value: 52.886
- type: mrr_at_100
value: 53.498000000000005
- type: mrr_at_1000
value: 53.54
- type: mrr_at_3
value: 50.876999999999995
- type: mrr_at_5
value: 52.254999999999995
- type: ndcg_at_1
value: 42.415000000000006
- type: ndcg_at_10
value: 33.660000000000004
- type: ndcg_at_100
value: 31.008000000000003
- type: ndcg_at_1000
value: 40.016
- type: ndcg_at_3
value: 39.329
- type: ndcg_at_5
value: 36.687999999999995
- type: precision_at_1
value: 43.963
- type: precision_at_10
value: 25.356
- type: precision_at_100
value: 8.245
- type: precision_at_1000
value: 2.106
- type: precision_at_3
value: 37.255
- type: precision_at_5
value: 31.95
- type: recall_at_1
value: 5.079000000000001
- type: recall_at_10
value: 15.838
- type: recall_at_100
value: 32.159
- type: recall_at_1000
value: 64.91799999999999
- type: recall_at_3
value: 10.152999999999999
- type: recall_at_5
value: 12.4
- task:
type: Retrieval
dataset:
type: nq
name: MTEB NQ
config: default
split: test
revision: None
metrics:
- type: map_at_1
value: 29.605999999999998
- type: map_at_10
value: 43.518
- type: map_at_100
value: 44.583
- type: map_at_1000
value: 44.622
- type: map_at_3
value: 39.673
- type: map_at_5
value: 41.897
- type: mrr_at_1
value: 33.604
- type: mrr_at_10
value: 46.156000000000006
- type: mrr_at_100
value: 46.974
- type: mrr_at_1000
value: 47.002
- type: mrr_at_3
value: 42.907000000000004
- type: mrr_at_5
value: 44.792
- type: ndcg_at_1
value: 33.575
- type: ndcg_at_10
value: 50.61600000000001
- type: ndcg_at_100
value: 55.129
- type: ndcg_at_1000
value: 56.084
- type: ndcg_at_3
value: 43.297999999999995
- type: ndcg_at_5
value: 46.979
- type: precision_at_1
value: 33.575
- type: precision_at_10
value: 8.297
- type: precision_at_100
value: 1.083
- type: precision_at_1000
value: 0.117
- type: precision_at_3
value: 19.602
- type: precision_at_5
value: 13.934
- type: recall_at_1
value: 29.605999999999998
- type: recall_at_10
value: 69.718
- type: recall_at_100
value: 89.352
- type: recall_at_1000
value: 96.543
- type: recall_at_3
value: 50.617999999999995
- type: recall_at_5
value: 59.031
- task:
type: PairClassification
dataset:
type: C-MTEB/OCNLI
name: MTEB Ocnli
config: default
split: validation
revision: None
metrics:
- type: cos_sim_accuracy
value: 65.83649160801299
- type: cos_sim_ap
value: 69.86408265006916
- type: cos_sim_f1
value: 70.50709939148074
- type: cos_sim_precision
value: 57.2463768115942
- type: cos_sim_recall
value: 91.76346356916578
- type: dot_accuracy
value: 61.93827828911749
- type: dot_ap
value: 64.26140500313572
- type: dot_f1
value: 68.97081413210446
- type: dot_precision
value: 54.19432709716355
- type: dot_recall
value: 94.82576557550159
- type: euclidean_accuracy
value: 66.32376827287493
- type: euclidean_ap
value: 70.58216586017075
- type: euclidean_f1
value: 71.31782945736435
- type: euclidean_precision
value: 58.11170212765957
- type: euclidean_recall
value: 92.29144667370645
- type: manhattan_accuracy
value: 66.54033567948024
- type: manhattan_ap
value: 70.88996923294056
- type: manhattan_f1
value: 71.45256087321579
- type: manhattan_precision
value: 59.30313588850174
- type: manhattan_recall
value: 89.86272439281943
- type: max_accuracy
value: 66.54033567948024
- type: max_ap
value: 70.88996923294056
- type: max_f1
value: 71.45256087321579
- task:
type: Classification
dataset:
type: C-MTEB/OnlineShopping-classification
name: MTEB OnlineShopping
config: default
split: test
revision: None
metrics:
- type: accuracy
value: 90.41
- type: ap
value: 88.15736492425235
- type: f1
value: 90.40118324200982
- task:
type: STS
dataset:
type: C-MTEB/PAWSX
name: MTEB PAWSX
config: default
split: test
revision: None
metrics:
- type: cos_sim_pearson
value: 14.718326697461064
- type: cos_sim_spearman
value: 17.458017383716168
- type: euclidean_pearson
value: 19.416710995216608
- type: euclidean_spearman
value: 17.87886266073602
- type: manhattan_pearson
value: 19.508696307778063
- type: manhattan_spearman
value: 18.026398724663487
- task:
type: STS
dataset:
type: C-MTEB/QBQTC
name: MTEB QBQTC
config: default
split: test
revision: None
metrics:
- type: cos_sim_pearson
value: 31.330102731068386
- type: cos_sim_spearman
value: 33.69612492132476
- type: euclidean_pearson
value: 33.83912666711584
- type: euclidean_spearman
value: 35.58666712573462
- type: manhattan_pearson
value: 34.257595977157706
- type: manhattan_spearman
value: 36.08587604692898
- task:
type: Retrieval
dataset:
type: quora
name: MTEB QuoraRetrieval
config: default
split: test
revision: None
metrics:
- type: map_at_1
value: 70.37
- type: map_at_10
value: 84.22699999999999
- type: map_at_100
value: 84.871
- type: map_at_1000
value: 84.88900000000001
- type: map_at_3
value: 81.277
- type: map_at_5
value: 83.16799999999999
- type: mrr_at_1
value: 80.97
- type: mrr_at_10
value: 87.24300000000001
- type: mrr_at_100
value: 87.346
- type: mrr_at_1000
value: 87.347
- type: mrr_at_3
value: 86.258
- type: mrr_at_5
value: 86.914
- type: ndcg_at_1
value: 81
- type: ndcg_at_10
value: 88.009
- type: ndcg_at_100
value: 89.251
- type: ndcg_at_1000
value: 89.374
- type: ndcg_at_3
value: 85.169
- type: ndcg_at_5
value: 86.75399999999999
- type: precision_at_1
value: 81
- type: precision_at_10
value: 13.343
- type: precision_at_100
value: 1.526
- type: precision_at_1000
value: 0.157
- type: precision_at_3
value: 37.25
- type: precision_at_5
value: 24.504
- type: recall_at_1
value: 70.37
- type: recall_at_10
value: 95.158
- type: recall_at_100
value: 99.39
- type: recall_at_1000
value: 99.98
- type: recall_at_3
value: 86.942
- type: recall_at_5
value: 91.446
- task:
type: Clustering
dataset:
type: mteb/reddit-clustering
name: MTEB RedditClustering
config: default
split: test
revision: 24640382cdbf8abc73003fb0fa6d111a705499eb
metrics:
- type: v_measure
value: 49.71370818375339
- task:
type: Clustering
dataset:
type: mteb/reddit-clustering-p2p
name: MTEB RedditClusteringP2P
config: default
split: test
revision: 282350215ef01743dc01b456c7f5241fa8937f16
metrics:
- type: v_measure
value: 55.07451965473589
- task:
type: Retrieval
dataset:
type: scidocs
name: MTEB SCIDOCS
config: default
split: test
revision: None
metrics:
- type: map_at_1
value: 4.508
- type: map_at_10
value: 10.825
- type: map_at_100
value: 12.598
- type: map_at_1000
value: 12.854
- type: map_at_3
value: 7.892
- type: map_at_5
value: 9.349
- type: mrr_at_1
value: 22.2
- type: mrr_at_10
value: 32.611000000000004
- type: mrr_at_100
value: 33.61
- type: mrr_at_1000
value: 33.671
- type: mrr_at_3
value: 29.15
- type: mrr_at_5
value: 31.225
- type: ndcg_at_1
value: 22.2
- type: ndcg_at_10
value: 18.502
- type: ndcg_at_100
value: 25.424999999999997
- type: ndcg_at_1000
value: 30.233999999999998
- type: ndcg_at_3
value: 17.711
- type: ndcg_at_5
value: 15.501000000000001
- type: precision_at_1
value: 22.2
- type: precision_at_10
value: 9.49
- type: precision_at_100
value: 1.941
- type: precision_at_1000
value: 0.31
- type: precision_at_3
value: 16.433
- type: precision_at_5
value: 13.54
- type: recall_at_1
value: 4.508
- type: recall_at_10
value: 19.243
- type: recall_at_100
value: 39.407
- type: recall_at_1000
value: 62.953
- type: recall_at_3
value: 9.993
- type: recall_at_5
value: 13.733
- task:
type: STS
dataset:
type: mteb/sickr-sts
name: MTEB SICK-R
config: default
split: test
revision: a6ea5a8cab320b040a23452cc28066d9beae2cee
metrics:
- type: cos_sim_pearson
value: 85.88096352325879
- type: cos_sim_spearman
value: 80.84882728439892
- type: euclidean_pearson
value: 82.89512161923362
- type: euclidean_spearman
value: 80.69723454935396
- type: manhattan_pearson
value: 82.94365287299226
- type: manhattan_spearman
value: 80.64700541831023
- task:
type: STS
dataset:
type: mteb/sts12-sts
name: MTEB STS12
config: default
split: test
revision: a0d554a64d88156834ff5ae9920b964011b16384
metrics:
- type: cos_sim_pearson
value: 84.09030569824817
- type: cos_sim_spearman
value: 76.10288448289813
- type: euclidean_pearson
value: 82.19317617787483
- type: euclidean_spearman
value: 78.51206398528993
- type: manhattan_pearson
value: 82.50688072451729
- type: manhattan_spearman
value: 78.71694597298867
- task:
type: STS
dataset:
type: mteb/sts13-sts
name: MTEB STS13
config: default
split: test
revision: 7e90230a92c190f1bf69ae9002b8cea547a64cca
metrics:
- type: cos_sim_pearson
value: 85.04298066236511
- type: cos_sim_spearman
value: 85.49051395372348
- type: euclidean_pearson
value: 85.7369561800059
- type: euclidean_spearman
value: 86.35626949911497
- type: manhattan_pearson
value: 85.86766305481635
- type: manhattan_spearman
value: 86.5115276036124
- task:
type: STS
dataset:
type: mteb/sts14-sts
name: MTEB STS14
config: default
split: test
revision: 6031580fec1f6af667f0bd2da0a551cf4f0b2375
metrics:
- type: cos_sim_pearson
value: 83.98107748125086
- type: cos_sim_spearman
value: 80.43502071880916
- type: euclidean_pearson
value: 82.24603130661005
- type: euclidean_spearman
value: 80.94302742946145
- type: manhattan_pearson
value: 82.4215619893203
- type: manhattan_spearman
value: 81.13824893869541
- task:
type: STS
dataset:
type: mteb/sts15-sts
name: MTEB STS15
config: default
split: test
revision: ae752c7c21bf194d8b67fd573edf7ae58183cbe3
metrics:
- type: cos_sim_pearson
value: 86.95857345426359
- type: cos_sim_spearman
value: 87.7540379885978
- type: euclidean_pearson
value: 87.86433964223119
- type: euclidean_spearman
value: 88.43585275816753
- type: manhattan_pearson
value: 87.90915813062988
- type: manhattan_spearman
value: 88.49038031429657
- task:
type: STS
dataset:
type: mteb/sts16-sts
name: MTEB STS16
config: default
split: test
revision: 4d8694f8f0e0100860b497b999b3dbed754a0513
metrics:
- type: cos_sim_pearson
value: 83.84530028548023
- type: cos_sim_spearman
value: 85.42197371225963
- type: euclidean_pearson
value: 84.12042159341938
- type: euclidean_spearman
value: 84.69864997658445
- type: manhattan_pearson
value: 84.09772815909784
- type: manhattan_spearman
value: 84.63986468736967
- task:
type: STS
dataset:
type: mteb/sts17-crosslingual-sts
name: MTEB STS17 (en-en)
config: en-en
split: test
revision: af5e6fb845001ecf41f4c1e033ce921939a2a68d
metrics:
- type: cos_sim_pearson
value: 89.89281017946413
- type: cos_sim_spearman
value: 89.94783195991867
- type: euclidean_pearson
value: 89.19342633226815
- type: euclidean_spearman
value: 88.6692137120815
- type: manhattan_pearson
value: 89.19006596701496
- type: manhattan_spearman
value: 88.65041672073397
- task:
type: STS
dataset:
type: mteb/sts22-crosslingual-sts
name: MTEB STS22 (en)
config: en
split: test
revision: 6d1ba47164174a496b7fa5d3569dae26a6813b80
metrics:
- type: cos_sim_pearson
value: 65.05176237336566
- type: cos_sim_spearman
value: 65.12758602746149
- type: euclidean_pearson
value: 67.44468889455905
- type: euclidean_spearman
value: 67.42836832904808
- type: manhattan_pearson
value: 67.99438187200471
- type: manhattan_spearman
value: 67.96190936270705
- task:
type: STS
dataset:
type: C-MTEB/STSB
name: MTEB STSB
config: default
split: test
revision: None
metrics:
- type: cos_sim_pearson
value: 81.36171514729287
- type: cos_sim_spearman
value: 81.51752389848613
- type: euclidean_pearson
value: 81.14136234145765
- type: euclidean_spearman
value: 81.27609983297867
- type: manhattan_pearson
value: 81.44966268348165
- type: manhattan_spearman
value: 81.53484018091312
- task:
type: STS
dataset:
type: mteb/stsbenchmark-sts
name: MTEB STSBenchmark
config: default
split: test
revision: b0fddb56ed78048fa8b90373c8a3cfc37b684831
metrics:
- type: cos_sim_pearson
value: 86.92195724268996
- type: cos_sim_spearman
value: 87.70682082313391
- type: euclidean_pearson
value: 86.24220109166684
- type: euclidean_spearman
value: 86.51998671092596
- type: manhattan_pearson
value: 86.17577571663554
- type: manhattan_spearman
value: 86.45961101071687
- task:
type: Reranking
dataset:
type: mteb/scidocs-reranking
name: MTEB SciDocsRR
config: default
split: test
revision: d3c5e1fc0b855ab6097bf1cda04dd73947d7caab
metrics:
- type: map
value: 78.62106635785725
- type: mrr
value: 93.84658279266121
- task:
type: Retrieval
dataset:
type: scifact
name: MTEB SciFact
config: default
split: test
revision: None
metrics:
- type: map_at_1
value: 53.761
- type: map_at_10
value: 64.56
- type: map_at_100
value: 65.243
- type: map_at_1000
value: 65.269
- type: map_at_3
value: 62.156
- type: map_at_5
value: 63.55
- type: mrr_at_1
value: 56.667
- type: mrr_at_10
value: 66.084
- type: mrr_at_100
value: 66.58500000000001
- type: mrr_at_1000
value: 66.61
- type: mrr_at_3
value: 64.333
- type: mrr_at_5
value: 65.3
- type: ndcg_at_1
value: 56.667
- type: ndcg_at_10
value: 69.43
- type: ndcg_at_100
value: 72.031
- type: ndcg_at_1000
value: 72.75
- type: ndcg_at_3
value: 65.282
- type: ndcg_at_5
value: 67.24900000000001
- type: precision_at_1
value: 56.667
- type: precision_at_10
value: 9.3
- type: precision_at_100
value: 1.0670000000000002
- type: precision_at_1000
value: 0.11199999999999999
- type: precision_at_3
value: 25.778000000000002
- type: precision_at_5
value: 16.866999999999997
- type: recall_at_1
value: 53.761
- type: recall_at_10
value: 82.678
- type: recall_at_100
value: 93.667
- type: recall_at_1000
value: 99.333
- type: recall_at_3
value: 71.578
- type: recall_at_5
value: 76.25
- task:
type: PairClassification
dataset:
type: mteb/sprintduplicatequestions-pairclassification
name: MTEB SprintDuplicateQuestions
config: default
split: test
revision: d66bd1f72af766a5cc4b0ca5e00c162f89e8cc46
metrics:
- type: cos_sim_accuracy
value: 99.80594059405941
- type: cos_sim_ap
value: 95.35711574476811
- type: cos_sim_f1
value: 90.12096774193547
- type: cos_sim_precision
value: 90.85365853658537
- type: cos_sim_recall
value: 89.4
- type: dot_accuracy
value: 99.76732673267327
- type: dot_ap
value: 93.20624501431367
- type: dot_f1
value: 87.74126238914971
- type: dot_precision
value: 91.71210468920393
- type: dot_recall
value: 84.1
- type: euclidean_accuracy
value: 99.80594059405941
- type: euclidean_ap
value: 95.35758863966429
- type: euclidean_f1
value: 90.15075376884421
- type: euclidean_precision
value: 90.6060606060606
- type: euclidean_recall
value: 89.7
- type: manhattan_accuracy
value: 99.80990099009901
- type: manhattan_ap
value: 95.48335466728275
- type: manhattan_f1
value: 90.2672718103883
- type: manhattan_precision
value: 91.04781281790437
- type: manhattan_recall
value: 89.5
- type: max_accuracy
value: 99.80990099009901
- type: max_ap
value: 95.48335466728275
- type: max_f1
value: 90.2672718103883
- task:
type: Clustering
dataset:
type: mteb/stackexchange-clustering
name: MTEB StackExchangeClustering
config: default
split: test
revision: 6cbc1f7b2bc0622f2e39d2c77fa502909748c259
metrics:
- type: v_measure
value: 59.422562431402845
- task:
type: Clustering
dataset:
type: mteb/stackexchange-clustering-p2p
name: MTEB StackExchangeClusteringP2P
config: default
split: test
revision: 815ca46b2622cec33ccafc3735d572c266efdb44
metrics:
- type: v_measure
value: 31.695493629721373
- task:
type: Reranking
dataset:
type: mteb/stackoverflowdupquestions-reranking
name: MTEB StackOverflowDupQuestions
config: default
split: test
revision: e185fbe320c72810689fc5848eb6114e1ef5ec69
metrics:
- type: map
value: 50.070077950465965
- type: mrr
value: 50.72293311263899
- task:
type: Summarization
dataset:
type: mteb/summeval
name: MTEB SummEval
config: default
split: test
revision: cda12ad7615edc362dbf25a00fdd61d3b1eaf93c
metrics:
- type: cos_sim_pearson
value: 30.59608436984981
- type: cos_sim_spearman
value: 30.617289383193103
- type: dot_pearson
value: 30.78715584903813
- type: dot_spearman
value: 31.269245492805283
- task:
type: Reranking
dataset:
type: C-MTEB/T2Reranking
name: MTEB T2Reranking
config: default
split: dev
revision: None
metrics:
- type: map
value: 66.49332760690612
- type: mrr
value: 76.52668294806075
- task:
type: Retrieval
dataset:
type: C-MTEB/T2Retrieval
name: MTEB T2Retrieval
config: default
split: dev
revision: None
metrics:
- type: map_at_1
value: 24.607
- type: map_at_10
value: 67.009
- type: map_at_100
value: 70.838
- type: map_at_1000
value: 70.954
- type: map_at_3
value: 47.573
- type: map_at_5
value: 58.10999999999999
- type: mrr_at_1
value: 84.333
- type: mrr_at_10
value: 87.822
- type: mrr_at_100
value: 87.969
- type: mrr_at_1000
value: 87.97500000000001
- type: mrr_at_3
value: 87.16000000000001
- type: mrr_at_5
value: 87.587
- type: ndcg_at_1
value: 84.333
- type: ndcg_at_10
value: 76.303
- type: ndcg_at_100
value: 81.05499999999999
- type: ndcg_at_1000
value: 82.218
- type: ndcg_at_3
value: 78.691
- type: ndcg_at_5
value: 76.66
- type: precision_at_1
value: 84.333
- type: precision_at_10
value: 38.019999999999996
- type: precision_at_100
value: 4.7669999999999995
- type: precision_at_1000
value: 0.505
- type: precision_at_3
value: 68.939
- type: precision_at_5
value: 57.306999999999995
- type: recall_at_1
value: 24.607
- type: recall_at_10
value: 74.971
- type: recall_at_100
value: 90.108
- type: recall_at_1000
value: 95.917
- type: recall_at_3
value: 49.586000000000006
- type: recall_at_5
value: 62.232
- task:
type: Classification
dataset:
type: C-MTEB/TNews-classification
name: MTEB TNews
config: default
split: validation
revision: None
metrics:
- type: accuracy
value: 47.702
- type: f1
value: 46.274469606672426
- task:
type: Retrieval
dataset:
type: trec-covid
name: MTEB TRECCOVID
config: default
split: test
revision: None
metrics:
- type: map_at_1
value: 0.252
- type: map_at_10
value: 2.178
- type: map_at_100
value: 12.781999999999998
- type: map_at_1000
value: 29.494999999999997
- type: map_at_3
value: 0.73
- type: map_at_5
value: 1.169
- type: mrr_at_1
value: 94
- type: mrr_at_10
value: 97
- type: mrr_at_100
value: 97
- type: mrr_at_1000
value: 97
- type: mrr_at_3
value: 97
- type: mrr_at_5
value: 97
- type: ndcg_at_1
value: 88
- type: ndcg_at_10
value: 83.21
- type: ndcg_at_100
value: 63.31
- type: ndcg_at_1000
value: 54.734
- type: ndcg_at_3
value: 87.408
- type: ndcg_at_5
value: 86.20100000000001
- type: precision_at_1
value: 94
- type: precision_at_10
value: 88.2
- type: precision_at_100
value: 64.68
- type: precision_at_1000
value: 23.966
- type: precision_at_3
value: 93.333
- type: precision_at_5
value: 91.60000000000001
- type: recall_at_1
value: 0.252
- type: recall_at_10
value: 2.307
- type: recall_at_100
value: 15.703
- type: recall_at_1000
value: 51.111
- type: recall_at_3
value: 0.749
- type: recall_at_5
value: 1.212
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (sqi-eng)
config: sqi-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 16.8
- type: f1
value: 13.168299935527422
- type: precision
value: 12.209559281760876
- type: recall
value: 16.8
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (fry-eng)
config: fry-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 35.83815028901734
- type: f1
value: 29.0852500101055
- type: precision
value: 26.965317919075147
- type: recall
value: 35.83815028901734
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (kur-eng)
config: kur-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 15.121951219512194
- type: f1
value: 11.844149203614325
- type: precision
value: 11.042929292929294
- type: recall
value: 15.121951219512194
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (tur-eng)
config: tur-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 9.9
- type: f1
value: 7.1396348187007215
- type: precision
value: 6.501835713997978
- type: recall
value: 9.9
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (deu-eng)
config: deu-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 76.6
- type: f1
value: 72.73241758241758
- type: precision
value: 71.18867647058823
- type: recall
value: 76.6
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (nld-eng)
config: nld-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 42
- type: f1
value: 36.81003102453103
- type: precision
value: 35.19870269535562
- type: recall
value: 42
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (ron-eng)
config: ron-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 35.3
- type: f1
value: 30.353777056277053
- type: precision
value: 28.773956778515604
- type: recall
value: 35.3
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (ang-eng)
config: ang-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 35.82089552238806
- type: f1
value: 27.44136460554371
- type: precision
value: 24.340796019900495
- type: recall
value: 35.82089552238806
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (ido-eng)
config: ido-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 51.800000000000004
- type: f1
value: 45.82491836793846
- type: precision
value: 43.729303094622864
- type: recall
value: 51.800000000000004
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (jav-eng)
config: jav-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 25.853658536585368
- type: f1
value: 19.79869362796192
- type: precision
value: 18.250680214094846
- type: recall
value: 25.853658536585368
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (isl-eng)
config: isl-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 9
- type: f1
value: 6.926590762281661
- type: precision
value: 6.507185696775364
- type: recall
value: 9
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (slv-eng)
config: slv-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 14.33778857837181
- type: f1
value: 10.888963524130242
- type: precision
value: 10.189272116928368
- type: recall
value: 14.33778857837181
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (cym-eng)
config: cym-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 11.304347826086957
- type: f1
value: 8.459121175343064
- type: precision
value: 7.7218644669759975
- type: recall
value: 11.304347826086957
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (kaz-eng)
config: kaz-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 8.521739130434783
- type: f1
value: 6.751744703151353
- type: precision
value: 6.387004921960017
- type: recall
value: 8.521739130434783
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (est-eng)
config: est-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 7.3
- type: f1
value: 5.626766011766011
- type: precision
value: 5.1270385799923
- type: recall
value: 7.3
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (heb-eng)
config: heb-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 3.2
- type: f1
value: 1.91950282507703
- type: precision
value: 1.6684431360304504
- type: recall
value: 3.2
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (gla-eng)
config: gla-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 5.790108564535585
- type: f1
value: 4.128499324411468
- type: precision
value: 3.8151453928788914
- type: recall
value: 5.790108564535585
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (mar-eng)
config: mar-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 70.3
- type: f1
value: 65.18318181818181
- type: precision
value: 63.126911976911984
- type: recall
value: 70.3
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (lat-eng)
config: lat-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 45.300000000000004
- type: f1
value: 38.339152873270514
- type: precision
value: 36.130903304212126
- type: recall
value: 45.300000000000004
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (bel-eng)
config: bel-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 16
- type: f1
value: 12.172850459161385
- type: precision
value: 11.27855570316309
- type: recall
value: 16
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (pms-eng)
config: pms-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 37.714285714285715
- type: f1
value: 32.188793178089945
- type: precision
value: 30.457500778089013
- type: recall
value: 37.714285714285715
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (gle-eng)
config: gle-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 6.5
- type: f1
value: 4.528544131928126
- type: precision
value: 4.171387799947767
- type: recall
value: 6.5
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (pes-eng)
config: pes-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 21
- type: f1
value: 17.006564035803166
- type: precision
value: 15.844832112332114
- type: recall
value: 21
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (nob-eng)
config: nob-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 25.5
- type: f1
value: 22.79430820164996
- type: precision
value: 21.938476924594045
- type: recall
value: 25.5
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (bul-eng)
config: bul-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 33.7
- type: f1
value: 26.898922166422164
- type: precision
value: 24.939117884031678
- type: recall
value: 33.7
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (cbk-eng)
config: cbk-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 69
- type: f1
value: 63.68992285492286
- type: precision
value: 61.72837301587302
- type: recall
value: 69
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (hun-eng)
config: hun-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 7.3999999999999995
- type: f1
value: 5.5655686223658565
- type: precision
value: 5.119921502146487
- type: recall
value: 7.3999999999999995
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (uig-eng)
config: uig-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 1.5
- type: f1
value: 1.001208686507139
- type: precision
value: 0.9683730903243098
- type: recall
value: 1.5
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (rus-eng)
config: rus-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 69
- type: f1
value: 62.61056277056276
- type: precision
value: 59.96357142857143
- type: recall
value: 69
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (spa-eng)
config: spa-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 98.3
- type: f1
value: 97.76666666666668
- type: precision
value: 97.51666666666668
- type: recall
value: 98.3
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (hye-eng)
config: hye-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 2.0215633423180592
- type: f1
value: 1.5634923413129036
- type: precision
value: 1.4895885785373653
- type: recall
value: 2.0215633423180592
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (tel-eng)
config: tel-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 83.33333333333334
- type: f1
value: 79.3019943019943
- type: precision
value: 77.45726495726495
- type: recall
value: 83.33333333333334
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (afr-eng)
config: afr-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 23.400000000000002
- type: f1
value: 18.655079988631996
- type: precision
value: 17.338269096494905
- type: recall
value: 23.400000000000002
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (mon-eng)
config: mon-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 6.363636363636363
- type: f1
value: 4.48376251469035
- type: precision
value: 4.071778641679957
- type: recall
value: 6.363636363636363
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (arz-eng)
config: arz-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 77.56813417190776
- type: f1
value: 73.16561844863732
- type: precision
value: 71.3440484509667
- type: recall
value: 77.56813417190776
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (hrv-eng)
config: hrv-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 17.299999999999997
- type: f1
value: 13.693204564375854
- type: precision
value: 12.830651358081276
- type: recall
value: 17.299999999999997
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (nov-eng)
config: nov-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 59.92217898832685
- type: f1
value: 53.29591938541354
- type: precision
value: 50.58736335000926
- type: recall
value: 59.92217898832685
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (gsw-eng)
config: gsw-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 25.64102564102564
- type: f1
value: 19.31404777558624
- type: precision
value: 17.413105413105416
- type: recall
value: 25.64102564102564
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (nds-eng)
config: nds-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 29.7
- type: f1
value: 24.44977050316952
- type: precision
value: 22.798075396825396
- type: recall
value: 29.7
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (ukr-eng)
config: ukr-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 32.2
- type: f1
value: 25.423187804627435
- type: precision
value: 23.404003309492442
- type: recall
value: 32.2
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (uzb-eng)
config: uzb-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 9.11214953271028
- type: f1
value: 5.910063827286792
- type: precision
value: 5.296401380795872
- type: recall
value: 9.11214953271028
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (lit-eng)
config: lit-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 7.199999999999999
- type: f1
value: 5.816726797396153
- type: precision
value: 5.508698718788661
- type: recall
value: 7.199999999999999
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (ina-eng)
config: ina-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 87.2
- type: f1
value: 83.88333333333333
- type: precision
value: 82.42833333333333
- type: recall
value: 87.2
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (lfn-eng)
config: lfn-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 53.7
- type: f1
value: 48.25312435500516
- type: precision
value: 46.34107401656314
- type: recall
value: 53.7
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (zsm-eng)
config: zsm-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 88.1
- type: f1
value: 85.21690476190476
- type: precision
value: 83.96761904761905
- type: recall
value: 88.1
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (ita-eng)
config: ita-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 78.10000000000001
- type: f1
value: 73.38746031746032
- type: precision
value: 71.47583333333334
- type: recall
value: 78.10000000000001
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (cmn-eng)
config: cmn-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 96.1
- type: f1
value: 95.08333333333333
- type: precision
value: 94.58333333333334
- type: recall
value: 96.1
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (lvs-eng)
config: lvs-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 9
- type: f1
value: 6.952605595133894
- type: precision
value: 6.457724621713984
- type: recall
value: 9
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (glg-eng)
config: glg-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 84.7
- type: f1
value: 80.97880952380953
- type: precision
value: 79.36428571428571
- type: recall
value: 84.7
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (ceb-eng)
config: ceb-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 10.5
- type: f1
value: 8.146458694813958
- type: precision
value: 7.618942433110826
- type: recall
value: 10.5
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (bre-eng)
config: bre-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 8.4
- type: f1
value: 6.144921607886653
- type: precision
value: 5.5261043562899586
- type: recall
value: 8.4
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (ben-eng)
config: ben-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 84.39999999999999
- type: f1
value: 80.65333333333334
- type: precision
value: 78.97833333333332
- type: recall
value: 84.39999999999999
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (swg-eng)
config: swg-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 28.57142857142857
- type: f1
value: 22.767379679144387
- type: precision
value: 21.2016369047619
- type: recall
value: 28.57142857142857
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (arq-eng)
config: arq-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 34.24807903402854
- type: f1
value: 29.241572730305222
- type: precision
value: 27.6428310072657
- type: recall
value: 34.24807903402854
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (kab-eng)
config: kab-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 2.9000000000000004
- type: f1
value: 1.9156734696693711
- type: precision
value: 1.7528460881307182
- type: recall
value: 2.9000000000000004
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (fra-eng)
config: fra-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 94.89999999999999
- type: f1
value: 93.53333333333332
- type: precision
value: 92.90666666666667
- type: recall
value: 94.89999999999999
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (por-eng)
config: por-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 95
- type: f1
value: 93.61666666666666
- type: precision
value: 92.93333333333332
- type: recall
value: 95
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (tat-eng)
config: tat-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 6.3
- type: f1
value: 4.920070356472795
- type: precision
value: 4.565811270125224
- type: recall
value: 6.3
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (oci-eng)
config: oci-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 47.4
- type: f1
value: 41.08392857142857
- type: precision
value: 38.999704968944094
- type: recall
value: 47.4
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (pol-eng)
config: pol-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 18.2
- type: f1
value: 14.826165036734295
- type: precision
value: 13.988559330454489
- type: recall
value: 18.2
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (war-eng)
config: war-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 13.3
- type: f1
value: 10.73451225789461
- type: precision
value: 10.06524508030025
- type: recall
value: 13.3
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (aze-eng)
config: aze-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 9.3
- type: f1
value: 7.613044370901514
- type: precision
value: 7.184100384035204
- type: recall
value: 9.3
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (vie-eng)
config: vie-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 97
- type: f1
value: 96.05
- type: precision
value: 95.58333333333334
- type: recall
value: 97
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (nno-eng)
config: nno-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 19.8
- type: f1
value: 16.070523504273503
- type: precision
value: 14.848185626325227
- type: recall
value: 19.8
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (cha-eng)
config: cha-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 29.1970802919708
- type: f1
value: 22.579707397225647
- type: precision
value: 20.792945550165477
- type: recall
value: 29.1970802919708
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (mhr-eng)
config: mhr-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 4.3
- type: f1
value: 2.884495496452018
- type: precision
value: 2.6280916815877506
- type: recall
value: 4.3
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (dan-eng)
config: dan-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 28.7
- type: f1
value: 24.9056519214062
- type: precision
value: 23.800155414494334
- type: recall
value: 28.7
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (ell-eng)
config: ell-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 9.5
- type: f1
value: 6.723431537130878
- type: precision
value: 6.078266616597544
- type: recall
value: 9.5
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (amh-eng)
config: amh-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 1.7857142857142856
- type: f1
value: 0.4579590594653929
- type: precision
value: 0.32939943654229364
- type: recall
value: 1.7857142857142856
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (pam-eng)
config: pam-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 9.1
- type: f1
value: 7.1794182614770845
- type: precision
value: 6.81138018671376
- type: recall
value: 9.1
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (hsb-eng)
config: hsb-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 15.113871635610765
- type: f1
value: 12.353104530336957
- type: precision
value: 11.66106754766342
- type: recall
value: 15.113871635610765
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (srp-eng)
config: srp-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 18.4
- type: f1
value: 15.091645001025805
- type: precision
value: 14.200823959052217
- type: recall
value: 18.4
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (epo-eng)
config: epo-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 33.2
- type: f1
value: 28.066634199134192
- type: precision
value: 26.54372717117398
- type: recall
value: 33.2
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (kzj-eng)
config: kzj-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 7.6
- type: f1
value: 5.992580343865051
- type: precision
value: 5.7409125738839055
- type: recall
value: 7.6
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (awa-eng)
config: awa-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 52.81385281385281
- type: f1
value: 46.86834810211434
- type: precision
value: 45.13687899402185
- type: recall
value: 52.81385281385281
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (fao-eng)
config: fao-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 16.030534351145036
- type: f1
value: 12.902313597194603
- type: precision
value: 12.19757977391565
- type: recall
value: 16.030534351145036
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (mal-eng)
config: mal-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 94.75982532751091
- type: f1
value: 93.11984473556527
- type: precision
value: 92.3216885007278
- type: recall
value: 94.75982532751091
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (ile-eng)
config: ile-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 70.19999999999999
- type: f1
value: 64.41237595737596
- type: precision
value: 62.074285714285715
- type: recall
value: 70.19999999999999
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (bos-eng)
config: bos-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 19.2090395480226
- type: f1
value: 14.986259497894084
- type: precision
value: 14.08083152750014
- type: recall
value: 19.2090395480226
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (cor-eng)
config: cor-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 5.800000000000001
- type: f1
value: 4.004811414639001
- type: precision
value: 3.611296721493974
- type: recall
value: 5.800000000000001
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (cat-eng)
config: cat-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 93.10000000000001
- type: f1
value: 91.17333333333335
- type: precision
value: 90.27833333333334
- type: recall
value: 93.10000000000001
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (eus-eng)
config: eus-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 68.2
- type: f1
value: 63.805870279146134
- type: precision
value: 62.064924029458915
- type: recall
value: 68.2
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (yue-eng)
config: yue-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 88.9
- type: f1
value: 86.38250000000001
- type: precision
value: 85.345
- type: recall
value: 88.9
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (swe-eng)
config: swe-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 26.3
- type: f1
value: 21.72601907540825
- type: precision
value: 20.3161132602622
- type: recall
value: 26.3
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (dtp-eng)
config: dtp-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 6.6000000000000005
- type: f1
value: 5.4107919446503585
- type: precision
value: 5.143205186348676
- type: recall
value: 6.6000000000000005
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (kat-eng)
config: kat-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 1.2064343163538873
- type: f1
value: 0.7118331023204635
- type: precision
value: 0.6930197065411955
- type: recall
value: 1.2064343163538873
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (jpn-eng)
config: jpn-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 78
- type: f1
value: 73.95134920634919
- type: precision
value: 72.3770634920635
- type: recall
value: 78
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (csb-eng)
config: csb-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 12.648221343873518
- type: f1
value: 10.259994816302727
- type: precision
value: 9.677206851119895
- type: recall
value: 12.648221343873518
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (xho-eng)
config: xho-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 10.56338028169014
- type: f1
value: 7.792644757433489
- type: precision
value: 7.299087316692951
- type: recall
value: 10.56338028169014
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (orv-eng)
config: orv-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 8.1437125748503
- type: f1
value: 5.6113303405098724
- type: precision
value: 5.156075980223929
- type: recall
value: 8.1437125748503
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (ind-eng)
config: ind-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 92.5
- type: f1
value: 90.53999999999999
- type: precision
value: 89.64500000000001
- type: recall
value: 92.5
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (tuk-eng)
config: tuk-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 8.374384236453201
- type: f1
value: 5.831645092728836
- type: precision
value: 5.241568776051535
- type: recall
value: 8.374384236453201
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (max-eng)
config: max-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 45.42253521126761
- type: f1
value: 40.878561970111264
- type: precision
value: 39.52681669728516
- type: recall
value: 45.42253521126761
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (swh-eng)
config: swh-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 32.05128205128205
- type: f1
value: 25.433010420698523
- type: precision
value: 23.545685308843208
- type: recall
value: 32.05128205128205
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (hin-eng)
config: hin-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 94.6
- type: f1
value: 92.86666666666666
- type: precision
value: 92.01666666666667
- type: recall
value: 94.6
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (dsb-eng)
config: dsb-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 14.822546972860126
- type: f1
value: 12.439321820122155
- type: precision
value: 11.940341857811413
- type: recall
value: 14.822546972860126
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (ber-eng)
config: ber-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 6.7
- type: f1
value: 5.534443298607457
- type: precision
value: 5.299107273391812
- type: recall
value: 6.7
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (tam-eng)
config: tam-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 87.94788273615634
- type: f1
value: 84.65798045602605
- type: precision
value: 83.2084690553746
- type: recall
value: 87.94788273615634
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (slk-eng)
config: slk-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 13.8
- type: f1
value: 11.356912127897372
- type: precision
value: 10.778191051205624
- type: recall
value: 13.8
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (tgl-eng)
config: tgl-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 13.700000000000001
- type: f1
value: 10.74774895608627
- type: precision
value: 9.966243757837463
- type: recall
value: 13.700000000000001
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (ast-eng)
config: ast-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 76.37795275590551
- type: f1
value: 71.24671916010499
- type: precision
value: 69.20697412823397
- type: recall
value: 76.37795275590551
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (mkd-eng)
config: mkd-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 18.099999999999998
- type: f1
value: 13.934122253809159
- type: precision
value: 12.815974391105971
- type: recall
value: 18.099999999999998
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (khm-eng)
config: khm-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 0.6925207756232686
- type: f1
value: 0.08966600365830146
- type: precision
value: 0.05066184676394412
- type: recall
value: 0.6925207756232686
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (ces-eng)
config: ces-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 11.1
- type: f1
value: 8.28646043238052
- type: precision
value: 7.686198801198802
- type: recall
value: 11.1
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (tzl-eng)
config: tzl-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 38.46153846153847
- type: f1
value: 31.640899949723472
- type: precision
value: 29.298878205128204
- type: recall
value: 38.46153846153847
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (urd-eng)
config: urd-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 81.2
- type: f1
value: 76.77103174603175
- type: precision
value: 74.96511904761905
- type: recall
value: 81.2
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (ara-eng)
config: ara-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 90.60000000000001
- type: f1
value: 88.20666666666665
- type: precision
value: 87.14833333333334
- type: recall
value: 90.60000000000001
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (kor-eng)
config: kor-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 35.699999999999996
- type: f1
value: 29.159127620745267
- type: precision
value: 27.109529030910608
- type: recall
value: 35.699999999999996
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (yid-eng)
config: yid-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 0.9433962264150944
- type: f1
value: 0.28088681664921333
- type: precision
value: 0.22694150916099465
- type: recall
value: 0.9433962264150944
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (fin-eng)
config: fin-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 7.5
- type: f1
value: 5.825362182391272
- type: precision
value: 5.526187577939453
- type: recall
value: 7.5
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (tha-eng)
config: tha-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 4.197080291970803
- type: f1
value: 3.079215618580677
- type: precision
value: 2.8501768792419
- type: recall
value: 4.197080291970803
- task:
type: BitextMining
dataset:
type: mteb/tatoeba-bitext-mining
name: MTEB Tatoeba (wuu-eng)
config: wuu-eng
split: test
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553
metrics:
- type: accuracy
value: 87.9
- type: f1
value: 84.60499999999999
- type: precision
value: 83.11428571428571
- type: recall
value: 87.9
- task:
type: Clustering
dataset:
type: C-MTEB/ThuNewsClusteringP2P
name: MTEB ThuNewsClusteringP2P
config: default
split: test
revision: None
metrics:
- type: v_measure
value: 50.23655676494653
- task:
type: Clustering
dataset:
type: C-MTEB/ThuNewsClusteringS2S
name: MTEB ThuNewsClusteringS2S
config: default
split: test
revision: None
metrics:
- type: v_measure
value: 49.54033078256682
- task:
type: Retrieval
dataset:
type: webis-touche2020
name: MTEB Touche2020
config: default
split: test
revision: None
metrics:
- type: map_at_1
value: 2.299
- type: map_at_10
value: 9.232999999999999
- type: map_at_100
value: 15.156
- type: map_at_1000
value: 16.63
- type: map_at_3
value: 4.2250000000000005
- type: map_at_5
value: 6.078
- type: mrr_at_1
value: 30.612000000000002
- type: mrr_at_10
value: 45.158
- type: mrr_at_100
value: 45.9
- type: mrr_at_1000
value: 45.910000000000004
- type: mrr_at_3
value: 39.456
- type: mrr_at_5
value: 42.925000000000004
- type: ndcg_at_1
value: 29.592000000000002
- type: ndcg_at_10
value: 25.166
- type: ndcg_at_100
value: 35.35
- type: ndcg_at_1000
value: 46.67
- type: ndcg_at_3
value: 24.545
- type: ndcg_at_5
value: 25.112000000000002
- type: precision_at_1
value: 30.612000000000002
- type: precision_at_10
value: 23.673
- type: precision_at_100
value: 7.428999999999999
- type: precision_at_1000
value: 1.482
- type: precision_at_3
value: 23.810000000000002
- type: precision_at_5
value: 25.306
- type: recall_at_1
value: 2.299
- type: recall_at_10
value: 16.801
- type: recall_at_100
value: 45.506
- type: recall_at_1000
value: 79.985
- type: recall_at_3
value: 5.069
- type: recall_at_5
value: 8.863999999999999
- task:
type: Classification
dataset:
type: mteb/toxic_conversations_50k
name: MTEB ToxicConversationsClassification
config: default
split: test
revision: d7c0de2777da35d6aae2200a62c6e0e5af397c4c
metrics:
- type: accuracy
value: 72.1314
- type: ap
value: 14.605968497007712
- type: f1
value: 55.37284214772282
- task:
type: Classification
dataset:
type: mteb/tweet_sentiment_extraction
name: MTEB TweetSentimentExtractionClassification
config: default
split: test
revision: d604517c81ca91fe16a244d1248fc021f9ecee7a
metrics:
- type: accuracy
value: 61.044142614601014
- type: f1
value: 61.30028928459138
- task:
type: Clustering
dataset:
type: mteb/twentynewsgroups-clustering
name: MTEB TwentyNewsgroupsClustering
config: default
split: test
revision: 6125ec4e24fa026cec8a478383ee943acfbd5449
metrics:
- type: v_measure
value: 41.28707371610032
- task:
type: PairClassification
dataset:
type: mteb/twittersemeval2015-pairclassification
name: MTEB TwitterSemEval2015
config: default
split: test
revision: 70970daeab8776df92f5ea462b6173c0b46fd2d1
metrics:
- type: cos_sim_accuracy
value: 85.09864695714371
- type: cos_sim_ap
value: 70.63738634684302
- type: cos_sim_f1
value: 66.12903225806453
- type: cos_sim_precision
value: 64.22178020885131
- type: cos_sim_recall
value: 68.15303430079156
- type: dot_accuracy
value: 83.59063002920665
- type: dot_ap
value: 66.68356189934075
- type: dot_f1
value: 63.27201851626264
- type: dot_precision
value: 58.76895225164064
- type: dot_recall
value: 68.52242744063325
- type: euclidean_accuracy
value: 85.027120462538
- type: euclidean_ap
value: 69.99328290454234
- type: euclidean_f1
value: 65.23797657612758
- type: euclidean_precision
value: 61.803588290840416
- type: euclidean_recall
value: 69.07651715039577
- type: manhattan_accuracy
value: 85.02115992132086
- type: manhattan_ap
value: 69.91284274429754
- type: manhattan_f1
value: 65.19297407097623
- type: manhattan_precision
value: 59.5763267088884
- type: manhattan_recall
value: 71.97889182058047
- type: max_accuracy
value: 85.09864695714371
- type: max_ap
value: 70.63738634684302
- type: max_f1
value: 66.12903225806453
- task:
type: PairClassification
dataset:
type: mteb/twitterurlcorpus-pairclassification
name: MTEB TwitterURLCorpus
config: default
split: test
revision: 8b6510b0b1fa4e4c4f879467980e9be563ec1cdf
metrics:
- type: cos_sim_accuracy
value: 89.119804400978
- type: cos_sim_ap
value: 86.1777422918812
- type: cos_sim_f1
value: 78.57841293719444
- type: cos_sim_precision
value: 76.80488163505366
- type: cos_sim_recall
value: 80.4357868801971
- type: dot_accuracy
value: 88.86366282454303
- type: dot_ap
value: 84.1891332504211
- type: dot_f1
value: 78.31691507672025
- type: dot_precision
value: 74.67700258397933
- type: dot_recall
value: 82.32984293193716
- type: euclidean_accuracy
value: 88.74141343578997
- type: euclidean_ap
value: 85.60421594792011
- type: euclidean_f1
value: 77.79556879538262
- type: euclidean_precision
value: 75.32991995384727
- type: euclidean_recall
value: 80.42808746535263
- type: manhattan_accuracy
value: 88.7782822990647
- type: manhattan_ap
value: 85.61374819166252
- type: manhattan_f1
value: 77.78237795927583
- type: manhattan_precision
value: 76.08423532876813
- type: manhattan_recall
value: 79.55805358792732
- type: max_accuracy
value: 89.119804400978
- type: max_ap
value: 86.1777422918812
- type: max_f1
value: 78.57841293719444
- task:
type: Retrieval
dataset:
type: C-MTEB/VideoRetrieval
name: MTEB VideoRetrieval
config: default
split: dev
revision: None
metrics:
- type: map_at_1
value: 41.8
- type: map_at_10
value: 51.456999999999994
- type: map_at_100
value: 52.107000000000006
- type: map_at_1000
value: 52.141999999999996
- type: map_at_3
value: 48.717
- type: map_at_5
value: 50.452
- type: mrr_at_1
value: 41.8
- type: mrr_at_10
value: 51.441
- type: mrr_at_100
value: 52.091
- type: mrr_at_1000
value: 52.125
- type: mrr_at_3
value: 48.699999999999996
- type: mrr_at_5
value: 50.434999999999995
- type: ndcg_at_1
value: 41.8
- type: ndcg_at_10
value: 56.537000000000006
- type: ndcg_at_100
value: 59.901
- type: ndcg_at_1000
value: 60.889
- type: ndcg_at_3
value: 51.019999999999996
- type: ndcg_at_5
value: 54.106
- type: precision_at_1
value: 41.8
- type: precision_at_10
value: 7.26
- type: precision_at_100
value: 0.8880000000000001
- type: precision_at_1000
value: 0.097
- type: precision_at_3
value: 19.233
- type: precision_at_5
value: 13.020000000000001
- type: recall_at_1
value: 41.8
- type: recall_at_10
value: 72.6
- type: recall_at_100
value: 88.8
- type: recall_at_1000
value: 96.7
- type: recall_at_3
value: 57.699999999999996
- type: recall_at_5
value: 65.10000000000001
- task:
type: Classification
dataset:
type: C-MTEB/waimai-classification
name: MTEB Waimai
config: default
split: test
revision: None
metrics:
- type: accuracy
value: 84.07
- type: ap
value: 65.23766736490957
- type: f1
value: 82.17794239849368
Model Card for udever-bloom
udever-bloom-3b is finetuned from bigscience/bloom-3b via BitFit on MS MARCO Passage Ranking, SNLI and MultiNLI data.
It is a universal embedding model across tasks, natural and programming languages.
(From the technical view, udever is merely with some minor improvements to sgpt-bloom)
Model Details
Model Description
- Developed by: Alibaba Group
- Model type: Transformer-based Language Model (decoder-only)
- Language(s) (NLP): Multiple; see bloom training data
- Finetuned from model : bigscience/bloom-3b
Model Sources
- Repository: github.com/izhx/uni-rep
- Paper : Language Models are Universal Embedders
- Training Date : 2023-06
How to Get Started with the Model
Use the code below to get started with the model.
import torch
from transformers import AutoTokenizer, BloomModel
tokenizer = AutoTokenizer.from_pretrained('izhx/udever-bloom-3b')
model = BloomModel.from_pretrained('izhx/udever-bloom-3b')
boq, eoq, bod, eod = '[BOQ]', '[EOQ]', '[BOD]', '[EOD]'
eoq_id, eod_id = tokenizer.convert_tokens_to_ids([eoq, eod])
if tokenizer.padding_side != 'left':
print('!!!', tokenizer.padding_side)
tokenizer.padding_side = 'left'
def encode(texts: list, is_query: bool = True, max_length=300):
bos = boq if is_query else bod
eos_id = eoq_id if is_query else eod_id
texts = [bos + t for t in texts]
encoding = tokenizer(
texts, truncation=True, max_length=max_length - 1, padding=True
)
for ids, mask in zip(encoding['input_ids'], encoding['attention_mask']):
ids.append(eos_id)
mask.append(1)
inputs = tokenizer.pad(encoding, return_tensors='pt')
with torch.inference_mode():
outputs = model(**inputs)
embeds = outputs.last_hidden_state[:, -1]
return embeds
encode(['I am Bert', 'You are Elmo'])
Training Details
Training Data
- MS MARCO Passage Ranking, retrieved by (https://github.com/UKPLab/sentence-transformers/blob/master/examples/training/ms_marco/train_bi-encoder_mnrl.py#L86)
- SNLI and MultiNLI (https://sbert.net/datasets/AllNLI.tsv.gz)
Training Procedure
Preprocessing
MS MARCO hard negatives provided by (https://github.com/UKPLab/sentence-transformers/blob/master/examples/training/ms_marco/train_bi-encoder_mnrl.py#L86). Negatives for SNLI and MultiNLI are randomly sampled.
Training Hyperparameters
- Training regime: tf32, BitFit
- Batch size: 1024
- Epochs: 3
- Optimizer: AdamW
- Learning rate: 1e-4
- Scheduler: constant with warmup.
- Warmup: 0.25 epoch
Evaluation
Table 1: Massive Text Embedding Benchmark MTEB
| MTEB | Avg. | Class. | Clust. | PairClass. | Rerank. | Retr. | STS | Summ. |
|---|---|---|---|---|---|---|---|---|
| #Datasets ➡️ | 56 | 12 | 11 | 3 | 4 | 15 | 10 | 1 |
| bge-large-en-v1.5 | 64.23 | 75.97 | 46.08 | 87.12 | 60.03 | 54.29 | 83.11 | 31.61 |
| bge-base-en-v1.5 | 63.55 | 75.53 | 45.77 | 86.55 | 58.86 | 53.25 | 82.4 | 31.07 |
| gte-large | 63.13 | 73.33 | 46.84 | 85 | 59.13 | 52.22 | 83.35 | 31.66 |
| gte-base | 62.39 | 73.01 | 46.2 | 84.57 | 58.61 | 51.14 | 82.3 | 31.17 |
| e5-large-v2 | 62.25 | 75.24 | 44.49 | 86.03 | 56.61 | 50.56 | 82.05 | 30.19 |
| instructor-xl | 61.79 | 73.12 | 44.74 | 86.62 | 57.29 | 49.26 | 83.06 | 32.32 |
| instructor-large | 61.59 | 73.86 | 45.29 | 85.89 | 57.54 | 47.57 | 83.15 | 31.84 |
| e5-base-v2 | 61.5 | 73.84 | 43.8 | 85.73 | 55.91 | 50.29 | 81.05 | 30.28 |
| e5-large | 61.42 | 73.14 | 43.33 | 85.94 | 56.53 | 49.99 | 82.06 | 30.97 |
| text-embedding-ada-002 (OpenAI API) | 60.99 | 70.93 | 45.9 | 84.89 | 56.32 | 49.25 | 80.97 | 30.8 |
| e5-base | 60.44 | 72.63 | 42.11 | 85.09 | 55.7 | 48.75 | 80.96 | 31.01 |
| SGPT-5.8B-msmarco | 58.93 | 68.13 | 40.34 | 82 | 56.56 | 50.25 | 78.1 | 31.46 |
| sgpt-bloom-7b1-msmarco | 57.59 | 66.19 | 38.93 | 81.9 | 55.65 | 48.22 | 77.74 | 33.6 |
| Udever-bloom-560m | 55.80 | 68.04 | 36.89 | 81.05 | 52.60 | 41.19 | 79.93 | 32.06 |
| Udever-bloom-1b1 | 58.28 | 70.18 | 39.11 | 83.11 | 54.28 | 45.27 | 81.52 | 31.10 |
| Udever-bloom-3b | 59.86 | 71.91 | 40.74 | 84.06 | 54.90 | 47.67 | 82.37 | 30.62 |
| Udever-bloom-7b1 | 60.63 | 72.13 | 40.81 | 85.40 | 55.91 | 49.34 | 83.01 | 30.97 |
Table 2: CodeSearchNet
| CodeSearchNet | Go | Ruby | Python | Java | JS | PHP | Avg. |
|---|---|---|---|---|---|---|---|
| CodeBERT | 69.3 | 70.6 | 84.0 | 86.8 | 74.8 | 70.6 | 76.0 |
| GraphCodeBERT | 84.1 | 73.2 | 87.9 | 75.7 | 71.1 | 72.5 | 77.4 |
| cpt-code S | 97.7 | 86.3 | 99.8 | 94.0 | 86.0 | 96.7 | 93.4 |
| cpt-code M | 97.5 | 85.5 | 99.9 | 94.4 | 86.5 | 97.2 | 93.5 |
| sgpt-bloom-7b1-msmarco | 76.79 | 69.25 | 95.68 | 77.93 | 70.35 | 73.45 | 77.24 |
| Udever-bloom-560m | 75.38 | 66.67 | 96.23 | 78.99 | 69.39 | 73.69 | 76.73 |
| Udever-bloom-1b1 | 78.76 | 72.85 | 97.67 | 82.77 | 74.38 | 78.97 | 80.90 |
| Udever-bloom-3b | 80.63 | 75.40 | 98.02 | 83.88 | 76.18 | 79.67 | 82.29 |
| Udever-bloom-7b1 | 79.37 | 76.59 | 98.38 | 84.68 | 77.49 | 80.03 | 82.76 |
Table 3: Chinese multi-domain retrieval Multi-cpr
| E-commerce | Entertainment video | Medical | ||||||
|---|---|---|---|---|---|---|---|---|
| Model | Train | Backbone | MRR@10 | Recall@1k | MRR@10 | Recall@1k | MRR@10 | Recall@1k |
| BM25 | - | - | 0.225 | 0.815 | 0.225 | 0.780 | 0.187 | 0.482 |
| Doc2Query | - | - | 0.239 | 0.826 | 0.238 | 0.794 | 0.210 | 0.505 |
| DPR-1 | In-Domain | BERT | 0.270 | 0.921 | 0.254 | 0.934 | 0.327 | 0.747 |
| DPR-2 | In-Domain | BERT-CT | 0.289 | 0.926 | 0.263 | 0.935 | 0.339 | 0.769 |
| text-embedding-ada-002 | General | GPT | 0.183 | 0.825 | 0.159 | 0.786 | 0.245 | 0.593 |
| sgpt-bloom-7b1-msmarco | General | BLOOM | 0.242 | 0.840 | 0.227 | 0.829 | 0.311 | 0.675 |
| Udever-bloom-560m | General | BLOOM | 0.156 | 0.802 | 0.149 | 0.749 | 0.245 | 0.571 |
| Udever-bloom-1b1 | General | BLOOM | 0.244 | 0.863 | 0.208 | 0.815 | 0.241 | 0.557 |
| Udever-bloom-3b | General | BLOOM | 0.267 | 0.871 | 0.228 | 0.836 | 0.288 | 0.619 |
| Udever-bloom-7b1 | General | BLOOM | 0.296 | 0.889 | 0.267 | 0.907 | 0.343 | 0.705 |
More results refer to paper section 3.
Technical Specifications
Model Architecture and Objective
- Model: bigscience/bloom-3b.
- Objective: Constrastive loss with hard negatives (refer to paper section 2.2).
Compute Infrastructure
- Nvidia A100 SXM4 80GB.
- torch 2.0.0, transformers 4.29.2.
Citation
BibTeX:
@article{zhang2023language,
title={Language Models are Universal Embedders},
author={Zhang, Xin and Li, Zehan and Zhang, Yanzhao and Long, Dingkun and Xie, Pengjun and Zhang, Meishan and Zhang, Min},
journal={arXiv preprint arXiv:2310.08232},
year={2023}
}