diff --git "a/README.md" "b/README.md" new file mode 100644--- /dev/null +++ "b/README.md" @@ -0,0 +1,7008 @@ +--- +model-index: +- name: ternary-weight-embedding + results: + - dataset: + config: default + name: MTEB AFQMC (default) + revision: b44c3b011063adb25877c13823db83bb193913c4 + split: validation + type: C-MTEB/AFQMC + metrics: + - type: cosine_pearson + value: 54.178784977880845 + - type: cosine_spearman + value: 58.24983603546464 + - type: euclidean_pearson + value: 56.12063302915049 + - type: euclidean_spearman + value: 57.55891180444177 + - type: main_score + value: 58.24983603546464 + - type: manhattan_pearson + value: 56.14231021933658 + - type: manhattan_spearman + value: 57.57073312857721 + task: + type: STS + - dataset: + config: default + name: MTEB ATEC (default) + revision: 0f319b1142f28d00e055a6770f3f726ae9b7d865 + split: test + type: C-MTEB/ATEC + metrics: + - type: cosine_pearson + value: 54.15496422617569 + - type: cosine_spearman + value: 57.14812321292837 + - type: euclidean_pearson + value: 60.12883177451991 + - type: euclidean_spearman + value: 56.79823039497166 + - type: main_score + value: 57.14812321292837 + - type: manhattan_pearson + value: 60.12467197534511 + - type: manhattan_spearman + value: 56.79279245167105 + task: + type: STS + - dataset: + config: default + name: MTEB ATEC (default) + revision: 0f319b1142f28d00e055a6770f3f726ae9b7d865 + split: validation + type: C-MTEB/ATEC + metrics: + - type: cosine_pearson + value: 54.24370907266313 + - type: cosine_spearman + value: 57.79884664537865 + - type: euclidean_pearson + value: 60.05093414384327 + - type: euclidean_spearman + value: 57.47352051181076 + - type: main_score + value: 57.79884664537865 + - type: manhattan_pearson + value: 60.03579846842669 + - type: manhattan_spearman + value: 57.46213161281322 + task: + type: STS + - dataset: + config: default + name: MTEB BQ (default) + revision: e3dda5e115e487b39ec7e618c0c6a29137052a55 + split: test + type: C-MTEB/BQ + metrics: + - type: cosine_pearson + value: 72.11546348999653 + - type: cosine_spearman + value: 74.28360304728436 + - type: euclidean_pearson + value: 72.68784320632835 + - type: euclidean_spearman + value: 73.8988695267803 + - type: main_score + value: 74.28360304728436 + - type: manhattan_pearson + value: 72.67994924531479 + - type: manhattan_spearman + value: 73.89602896299826 + task: + type: STS + - dataset: + config: default + name: MTEB BQ (default) + revision: e3dda5e115e487b39ec7e618c0c6a29137052a55 + split: validation + type: C-MTEB/BQ + metrics: + - type: cosine_pearson + value: 74.80134578588282 + - type: cosine_spearman + value: 76.88519187053075 + - type: euclidean_pearson + value: 75.45155138775218 + - type: euclidean_spearman + value: 76.44577057706553 + - type: main_score + value: 76.88519187053075 + - type: manhattan_pearson + value: 75.44260208073607 + - type: manhattan_spearman + value: 76.43697175914832 + task: + type: STS + - dataset: + config: default + name: MTEB CLSClusteringP2P.v2 (default) + revision: 4b6227591c6c1a73bc76b1055f3b7f3588e72476 + split: test + type: C-MTEB/CLSClusteringP2P + metrics: + - type: main_score + value: 54.68251196895576 + - type: v_measure + value: 54.68251196895576 + - type: v_measure_std + value: 1.8827992663564201 + task: + type: Clustering + - dataset: + config: default + name: MTEB CLSClusteringS2S.v2 (default) + revision: e458b3f5414b62b7f9f83499ac1f5497ae2e869f + split: test + type: C-MTEB/CLSClusteringS2S + metrics: + - type: main_score + value: 48.5426747048895 + - type: v_measure + value: 48.5426747048895 + - type: v_measure_std + value: 1.8768930779321003 + task: + type: Clustering + - dataset: + config: default + name: MTEB CMedQAv1-reranking (default) + revision: 8d7f1e942507dac42dc58017c1a001c3717da7df + split: test + type: C-MTEB/CMedQAv1-reranking + metrics: + - type: main_score + value: 88.74015900697846 + - type: map + value: 88.74015900697846 + - type: mrr + value: 90.6736507936508 + - type: nAUC_map_diff1 + value: 57.22083174189404 + - type: nAUC_map_max + value: 55.77168783483574 + - type: nAUC_map_std + value: 13.021507960473016 + - type: nAUC_mrr_diff1 + value: 62.952041755231626 + - type: nAUC_mrr_max + value: 64.05253602193952 + - type: nAUC_mrr_std + value: 23.213961052744555 + task: + type: Reranking + - dataset: + config: default + name: MTEB CMedQAv2-reranking (default) + revision: 23d186750531a14a0357ca22cd92d712fd512ea0 + split: test + type: C-MTEB/CMedQAv2-reranking + metrics: + - type: main_score + value: 88.38113834645159 + - type: map + value: 88.38113834645159 + - type: mrr + value: 90.52234126984126 + - type: nAUC_map_diff1 + value: 59.755818934870234 + - type: nAUC_map_max + value: 62.11664250496217 + - type: nAUC_map_std + value: 5.676411532480005 + - type: nAUC_mrr_diff1 + value: 67.07181219755715 + - type: nAUC_mrr_max + value: 75.28581323486274 + - type: nAUC_mrr_std + value: 17.997453037644533 + task: + type: Reranking + - dataset: + config: default + name: MTEB CmedqaRetrieval (default) + revision: cd540c506dae1cf9e9a59c3e06f42030d54e7301 + split: dev + type: C-MTEB/CmedqaRetrieval + metrics: + - type: main_score + value: 45.113 + - type: map_at_1 + value: 25.974999999999998 + - type: map_at_10 + value: 38.567 + - type: map_at_100 + value: 40.407 + - type: map_at_1000 + value: 40.52 + - type: map_at_20 + value: 39.564 + - type: map_at_3 + value: 34.410000000000004 + - type: map_at_5 + value: 36.689 + - type: mrr_at_1 + value: 39.65991497874469 + - type: mrr_at_10 + value: 47.631104204622524 + - type: mrr_at_100 + value: 48.59652256415256 + - type: mrr_at_1000 + value: 48.64137820297102 + - type: mrr_at_20 + value: 48.193915296968996 + - type: mrr_at_3 + value: 45.24047678586305 + - type: mrr_at_5 + value: 46.595815620571805 + - type: nauc_map_at_1000_diff1 + value: 47.78866266244895 + - type: nauc_map_at_1000_max + value: 41.6785411481451 + - type: nauc_map_at_1000_std + value: -13.089914559064795 + - type: nauc_map_at_100_diff1 + value: 47.74420073204668 + - type: nauc_map_at_100_max + value: 41.63744109954874 + - type: nauc_map_at_100_std + value: -13.115455589619943 + - type: nauc_map_at_10_diff1 + value: 47.60785675353292 + - type: nauc_map_at_10_max + value: 40.81808736337173 + - type: nauc_map_at_10_std + value: -14.010361731187555 + - type: nauc_map_at_1_diff1 + value: 52.623450404334235 + - type: nauc_map_at_1_max + value: 32.48612623845417 + - type: nauc_map_at_1_std + value: -14.301701958402347 + - type: nauc_map_at_20_diff1 + value: 47.623803967273446 + - type: nauc_map_at_20_max + value: 41.19261840063425 + - type: nauc_map_at_20_std + value: -13.695407747959843 + - type: nauc_map_at_3_diff1 + value: 48.343168244250215 + - type: nauc_map_at_3_max + value: 37.915780566548726 + - type: nauc_map_at_3_std + value: -14.422861700351183 + - type: nauc_map_at_5_diff1 + value: 47.94643138191748 + - type: nauc_map_at_5_max + value: 39.59735520213133 + - type: nauc_map_at_5_std + value: -14.23393731727717 + - type: nauc_mrr_at_1000_diff1 + value: 53.86065646960032 + - type: nauc_mrr_at_1000_max + value: 49.296644570372415 + - type: nauc_mrr_at_1000_std + value: -8.960076833549836 + - type: nauc_mrr_at_100_diff1 + value: 53.83949971297237 + - type: nauc_mrr_at_100_max + value: 49.283394247434124 + - type: nauc_mrr_at_100_std + value: -8.938461521660448 + - type: nauc_mrr_at_10_diff1 + value: 53.771434307862776 + - type: nauc_mrr_at_10_max + value: 49.232816072077185 + - type: nauc_mrr_at_10_std + value: -9.187019034523127 + - type: nauc_mrr_at_1_diff1 + value: 59.00104676791166 + - type: nauc_mrr_at_1_max + value: 51.260502472764436 + - type: nauc_mrr_at_1_std + value: -9.593234514978924 + - type: nauc_mrr_at_20_diff1 + value: 53.78230872607412 + - type: nauc_mrr_at_20_max + value: 49.21546854521065 + - type: nauc_mrr_at_20_std + value: -9.163123771380729 + - type: nauc_mrr_at_3_diff1 + value: 54.55765283386248 + - type: nauc_mrr_at_3_max + value: 49.60566476180881 + - type: nauc_mrr_at_3_std + value: -9.148690932678257 + - type: nauc_mrr_at_5_diff1 + value: 53.97605291263774 + - type: nauc_mrr_at_5_max + value: 49.40659705825686 + - type: nauc_mrr_at_5_std + value: -9.161744382503885 + - type: nauc_ndcg_at_1000_diff1 + value: 48.31934043697359 + - type: nauc_ndcg_at_1000_max + value: 44.71283713093145 + - type: nauc_ndcg_at_1000_std + value: -9.913668236760554 + - type: nauc_ndcg_at_100_diff1 + value: 47.38673504958784 + - type: nauc_ndcg_at_100_max + value: 44.25750824625488 + - type: nauc_ndcg_at_100_std + value: -9.48225516354158 + - type: nauc_ndcg_at_10_diff1 + value: 46.90328760292496 + - type: nauc_ndcg_at_10_max + value: 42.41986408299954 + - type: nauc_ndcg_at_10_std + value: -12.909824341781825 + - type: nauc_ndcg_at_1_diff1 + value: 59.00104676791166 + - type: nauc_ndcg_at_1_max + value: 51.260502472764436 + - type: nauc_ndcg_at_1_std + value: -9.593234514978924 + - type: nauc_ndcg_at_20_diff1 + value: 46.77258181931362 + - type: nauc_ndcg_at_20_max + value: 42.68119586804985 + - type: nauc_ndcg_at_20_std + value: -12.423629313301388 + - type: nauc_ndcg_at_3_diff1 + value: 48.58977327180224 + - type: nauc_ndcg_at_3_max + value: 43.61141480978654 + - type: nauc_ndcg_at_3_std + value: -11.641555503967986 + - type: nauc_ndcg_at_5_diff1 + value: 47.67463895870823 + - type: nauc_ndcg_at_5_max + value: 42.49711123538069 + - type: nauc_ndcg_at_5_std + value: -12.452049013694403 + - type: nauc_precision_at_1000_diff1 + value: 3.4806441777617567 + - type: nauc_precision_at_1000_max + value: 27.678000825789933 + - type: nauc_precision_at_1000_std + value: 17.698677421147963 + - type: nauc_precision_at_100_diff1 + value: 7.875973729853397 + - type: nauc_precision_at_100_max + value: 34.08812146111534 + - type: nauc_precision_at_100_std + value: 16.524389567188365 + - type: nauc_precision_at_10_diff1 + value: 21.272214137016185 + - type: nauc_precision_at_10_max + value: 44.00343281584008 + - type: nauc_precision_at_10_std + value: 0.9945485334597843 + - type: nauc_precision_at_1_diff1 + value: 59.00104676791166 + - type: nauc_precision_at_1_max + value: 51.260502472764436 + - type: nauc_precision_at_1_std + value: -9.593234514978924 + - type: nauc_precision_at_20_diff1 + value: 16.396372302736623 + - type: nauc_precision_at_20_max + value: 40.20456764221775 + - type: nauc_precision_at_20_std + value: 4.91733314934679 + - type: nauc_precision_at_3_diff1 + value: 34.03262918273604 + - type: nauc_precision_at_3_max + value: 48.260349668115424 + - type: nauc_precision_at_3_std + value: -3.812770245046173 + - type: nauc_precision_at_5_diff1 + value: 28.088076219897022 + - type: nauc_precision_at_5_max + value: 46.962465223407946 + - type: nauc_precision_at_5_std + value: -1.663412582138589 + - type: nauc_recall_at_1000_diff1 + value: 29.003325559730303 + - type: nauc_recall_at_1000_max + value: 51.11704447020827 + - type: nauc_recall_at_1000_std + value: 52.358021602871176 + - type: nauc_recall_at_100_diff1 + value: 26.944312318666174 + - type: nauc_recall_at_100_max + value: 35.1360457491317 + - type: nauc_recall_at_100_std + value: 8.995107524344007 + - type: nauc_recall_at_10_diff1 + value: 34.767375834324746 + - type: nauc_recall_at_10_max + value: 33.15737890896935 + - type: nauc_recall_at_10_std + value: -13.876362538877132 + - type: nauc_recall_at_1_diff1 + value: 52.623450404334235 + - type: nauc_recall_at_1_max + value: 32.48612623845417 + - type: nauc_recall_at_1_std + value: -14.301701958402347 + - type: nauc_recall_at_20_diff1 + value: 31.796685961276754 + - type: nauc_recall_at_20_max + value: 31.645635578789094 + - type: nauc_recall_at_20_std + value: -12.896907066742678 + - type: nauc_recall_at_3_diff1 + value: 40.9513328277203 + - type: nauc_recall_at_3_max + value: 33.2576092320736 + - type: nauc_recall_at_3_std + value: -13.677012060621971 + - type: nauc_recall_at_5_diff1 + value: 38.17560293253476 + - type: nauc_recall_at_5_max + value: 33.66607334519376 + - type: nauc_recall_at_5_std + value: -13.710743011608292 + - type: ndcg_at_1 + value: 39.660000000000004 + - type: ndcg_at_10 + value: 45.113 + - type: ndcg_at_100 + value: 52.257 + - type: ndcg_at_1000 + value: 54.25 + - type: ndcg_at_20 + value: 47.857 + - type: ndcg_at_3 + value: 40.022999999999996 + - type: ndcg_at_5 + value: 42.076 + - type: precision_at_1 + value: 39.660000000000004 + - type: precision_at_10 + value: 10.025 + - type: precision_at_100 + value: 1.5789999999999997 + - type: precision_at_1000 + value: 0.183 + - type: precision_at_20 + value: 5.915 + - type: precision_at_3 + value: 22.639 + - type: precision_at_5 + value: 16.359 + - type: recall_at_1 + value: 25.974999999999998 + - type: recall_at_10 + value: 55.332 + - type: recall_at_100 + value: 84.899 + - type: recall_at_1000 + value: 98.312 + - type: recall_at_20 + value: 64.702 + - type: recall_at_3 + value: 39.800999999999995 + - type: recall_at_5 + value: 46.378 + task: + type: Retrieval + - dataset: + config: default + name: MTEB Cmnli (default) + revision: 41bc36f332156f7adc9e38f53777c959b2ae9766 + split: validation + type: C-MTEB/CMNLI + metrics: + - type: cosine_accuracy + value: 85.13529765484064 + - type: cosine_accuracy_threshold + value: 86.13102436065674 + - type: cosine_ap + value: 92.19355954725854 + - type: cosine_f1 + value: 86.10154427285858 + - type: cosine_f1_threshold + value: 84.71119403839111 + - type: cosine_precision + value: 82.02794242167654 + - type: cosine_recall + value: 90.6008884732289 + - type: dot_accuracy + value: 74.91280817799158 + - type: dot_accuracy_threshold + value: 63842.00439453125 + - type: dot_ap + value: 83.44759720338214 + - type: dot_f1 + value: 77.27227076428642 + - type: dot_f1_threshold + value: 60020.81298828125 + - type: dot_precision + value: 67.72887323943661 + - type: dot_recall + value: 89.94622398877719 + - type: euclidean_accuracy + value: 84.47384245339748 + - type: euclidean_accuracy_threshold + value: 1469.5334434509277 + - type: euclidean_ap + value: 91.3556506449566 + - type: euclidean_f1 + value: 85.43989402803841 + - type: euclidean_f1_threshold + value: 1538.3913040161133 + - type: euclidean_precision + value: 80.92848180677541 + - type: euclidean_recall + value: 90.48398410100538 + - type: main_score + value: 85.13529765484064 + - type: manhattan_accuracy + value: 84.41371016235718 + - type: manhattan_accuracy_threshold + value: 49906.90002441406 + - type: manhattan_ap + value: 91.3493016502162 + - type: manhattan_f1 + value: 85.49062844542446 + - type: manhattan_f1_threshold + value: 51897.528076171875 + - type: manhattan_precision + value: 80.88879616106823 + - type: manhattan_recall + value: 90.6476502221183 + - type: max_accuracy + value: 85.13529765484064 + - type: max_ap + value: 92.19355954725854 + - type: max_f1 + value: 86.10154427285858 + - type: max_precision + value: 82.02794242167654 + - type: max_recall + value: 90.6476502221183 + - type: similarity_accuracy + value: 85.13529765484064 + - type: similarity_accuracy_threshold + value: 86.13102436065674 + - type: similarity_ap + value: 92.17394345753924 + - type: similarity_f1 + value: 86.10154427285858 + - type: similarity_f1_threshold + value: 84.71119403839111 + - type: similarity_precision + value: 82.02794242167654 + - type: similarity_recall + value: 90.6008884732289 + task: + type: PairClassification + - dataset: + config: default + name: MTEB CovidRetrieval (default) + revision: 1271c7809071a13532e05f25fb53511ffce77117 + split: dev + type: C-MTEB/CovidRetrieval + metrics: + - type: main_score + value: 82.819 + - type: map_at_1 + value: 69.494 + - type: map_at_10 + value: 78.789 + - type: map_at_100 + value: 78.988 + - type: map_at_1000 + value: 78.993 + - type: map_at_20 + value: 78.917 + - type: map_at_3 + value: 77.011 + - type: map_at_5 + value: 78.17699999999999 + - type: mrr_at_1 + value: 69.6522655426765 + - type: mrr_at_10 + value: 78.76729054811248 + - type: mrr_at_100 + value: 78.95793499668491 + - type: mrr_at_1000 + value: 78.96216519091867 + - type: mrr_at_20 + value: 78.88679240958568 + - type: mrr_at_3 + value: 77.04601334738325 + - type: mrr_at_5 + value: 78.22620302072359 + - type: nauc_map_at_1000_diff1 + value: 80.05214569065625 + - type: nauc_map_at_1000_max + value: 36.21145092927336 + - type: nauc_map_at_1000_std + value: -36.16531861028991 + - type: nauc_map_at_100_diff1 + value: 80.05020798715684 + - type: nauc_map_at_100_max + value: 36.22261208348821 + - type: nauc_map_at_100_std + value: -36.14332748625607 + - type: nauc_map_at_10_diff1 + value: 79.93946377542417 + - type: nauc_map_at_10_max + value: 36.20456984006017 + - type: nauc_map_at_10_std + value: -36.398588779081294 + - type: nauc_map_at_1_diff1 + value: 82.0602604309234 + - type: nauc_map_at_1_max + value: 35.19249651707143 + - type: nauc_map_at_1_std + value: -34.21839088932163 + - type: nauc_map_at_20_diff1 + value: 80.02697105208125 + - type: nauc_map_at_20_max + value: 36.18004342047142 + - type: nauc_map_at_20_std + value: -36.17886432477489 + - type: nauc_map_at_3_diff1 + value: 80.16978634467844 + - type: nauc_map_at_3_max + value: 35.83327596692132 + - type: nauc_map_at_3_std + value: -38.37503788050143 + - type: nauc_map_at_5_diff1 + value: 79.81622895096287 + - type: nauc_map_at_5_max + value: 36.08883221963992 + - type: nauc_map_at_5_std + value: -37.25482423232233 + - type: nauc_mrr_at_1000_diff1 + value: 80.2097372109321 + - type: nauc_mrr_at_1000_max + value: 36.42174288518451 + - type: nauc_mrr_at_1000_std + value: -35.85189499872029 + - type: nauc_mrr_at_100_diff1 + value: 80.20776581779481 + - type: nauc_mrr_at_100_max + value: 36.432844978272236 + - type: nauc_mrr_at_100_std + value: -35.82999739661053 + - type: nauc_mrr_at_10_diff1 + value: 80.09236275670942 + - type: nauc_mrr_at_10_max + value: 36.380671611318604 + - type: nauc_mrr_at_10_std + value: -36.15989339661995 + - type: nauc_mrr_at_1_diff1 + value: 81.87621839783066 + - type: nauc_mrr_at_1_max + value: 35.88505228929411 + - type: nauc_mrr_at_1_std + value: -32.92079357076708 + - type: nauc_mrr_at_20_diff1 + value: 80.18395501536016 + - type: nauc_mrr_at_20_max + value: 36.38952241095837 + - type: nauc_mrr_at_20_std + value: -35.866686376779114 + - type: nauc_mrr_at_3_diff1 + value: 80.23254549571189 + - type: nauc_mrr_at_3_max + value: 36.05935731319566 + - type: nauc_mrr_at_3_std + value: -37.866428822755864 + - type: nauc_mrr_at_5_diff1 + value: 79.88253909007965 + - type: nauc_mrr_at_5_max + value: 36.457475361699146 + - type: nauc_mrr_at_5_std + value: -36.49596735742553 + - type: nauc_ndcg_at_1000_diff1 + value: 79.7618117697378 + - type: nauc_ndcg_at_1000_max + value: 36.629676985491514 + - type: nauc_ndcg_at_1000_std + value: -35.86456326352086 + - type: nauc_ndcg_at_100_diff1 + value: 79.65891920529748 + - type: nauc_ndcg_at_100_max + value: 36.97516050762293 + - type: nauc_ndcg_at_100_std + value: -35.2001931520788 + - type: nauc_ndcg_at_10_diff1 + value: 79.17817241151019 + - type: nauc_ndcg_at_10_max + value: 36.86254607903196 + - type: nauc_ndcg_at_10_std + value: -36.235782359106615 + - type: nauc_ndcg_at_1_diff1 + value: 81.6552730132908 + - type: nauc_ndcg_at_1_max + value: 35.6139365961136 + - type: nauc_ndcg_at_1_std + value: -33.34605824156571 + - type: nauc_ndcg_at_20_diff1 + value: 79.53353610120962 + - type: nauc_ndcg_at_20_max + value: 36.83633652729889 + - type: nauc_ndcg_at_20_std + value: -35.223937945126934 + - type: nauc_ndcg_at_3_diff1 + value: 79.66142161189495 + - type: nauc_ndcg_at_3_max + value: 36.201048838264995 + - type: nauc_ndcg_at_3_std + value: -40.303435032913235 + - type: nauc_ndcg_at_5_diff1 + value: 78.84195807786423 + - type: nauc_ndcg_at_5_max + value: 36.82613815363756 + - type: nauc_ndcg_at_5_std + value: -38.073937317858494 + - type: nauc_precision_at_1000_diff1 + value: -52.61992538101203 + - type: nauc_precision_at_1000_max + value: 2.876218848436758 + - type: nauc_precision_at_1000_std + value: 54.38129782759948 + - type: nauc_precision_at_100_diff1 + value: -15.670131055377192 + - type: nauc_precision_at_100_max + value: 27.850990893649424 + - type: nauc_precision_at_100_std + value: 53.31709331697566 + - type: nauc_precision_at_10_diff1 + value: 38.84116900921712 + - type: nauc_precision_at_10_max + value: 34.79084147327868 + - type: nauc_precision_at_10_std + value: -6.829876002907609 + - type: nauc_precision_at_1_diff1 + value: 81.6552730132908 + - type: nauc_precision_at_1_max + value: 35.6139365961136 + - type: nauc_precision_at_1_std + value: -33.34605824156571 + - type: nauc_precision_at_20_diff1 + value: 25.845188821845383 + - type: nauc_precision_at_20_max + value: 31.18865106000545 + - type: nauc_precision_at_20_std + value: 14.759241268473561 + - type: nauc_precision_at_3_diff1 + value: 70.8201683277703 + - type: nauc_precision_at_3_max + value: 36.6701970406286 + - type: nauc_precision_at_3_std + value: -42.35040269825674 + - type: nauc_precision_at_5_diff1 + value: 59.02227125330898 + - type: nauc_precision_at_5_max + value: 38.69410968250484 + - type: nauc_precision_at_5_std + value: -30.30854393285351 + - type: nauc_recall_at_1000_diff1 + value: .nan + - type: nauc_recall_at_1000_max + value: .nan + - type: nauc_recall_at_1000_std + value: .nan + - type: nauc_recall_at_100_diff1 + value: 64.60431422737145 + - type: nauc_recall_at_100_max + value: 82.10937063790381 + - type: nauc_recall_at_100_std + value: 51.005079723478254 + - type: nauc_recall_at_10_diff1 + value: 70.9120103945018 + - type: nauc_recall_at_10_max + value: 44.17151180123427 + - type: nauc_recall_at_10_std + value: -30.974453318671753 + - type: nauc_recall_at_1_diff1 + value: 82.0602604309234 + - type: nauc_recall_at_1_max + value: 35.19249651707143 + - type: nauc_recall_at_1_std + value: -34.21839088932163 + - type: nauc_recall_at_20_diff1 + value: 72.87073930480337 + - type: nauc_recall_at_20_max + value: 48.156799565062514 + - type: nauc_recall_at_20_std + value: -8.986967985015431 + - type: nauc_recall_at_3_diff1 + value: 77.48100030181683 + - type: nauc_recall_at_3_max + value: 37.72633639333108 + - type: nauc_recall_at_3_std + value: -48.48712826839765 + - type: nauc_recall_at_5_diff1 + value: 72.79074903006723 + - type: nauc_recall_at_5_max + value: 41.41902376036034 + - type: nauc_recall_at_5_std + value: -42.76726369093494 + - type: ndcg_at_1 + value: 69.758 + - type: ndcg_at_10 + value: 82.819 + - type: ndcg_at_100 + value: 83.705 + - type: ndcg_at_1000 + value: 83.814 + - type: ndcg_at_20 + value: 83.26599999999999 + - type: ndcg_at_3 + value: 79.296 + - type: ndcg_at_5 + value: 81.406 + - type: precision_at_1 + value: 69.758 + - type: precision_at_10 + value: 9.621 + - type: precision_at_100 + value: 1.002 + - type: precision_at_1000 + value: 0.101 + - type: precision_at_20 + value: 4.9 + - type: precision_at_3 + value: 28.767 + - type: precision_at_5 + value: 18.335 + - type: recall_at_1 + value: 69.494 + - type: recall_at_10 + value: 95.205 + - type: recall_at_100 + value: 99.157 + - type: recall_at_1000 + value: 100.0 + - type: recall_at_20 + value: 96.944 + - type: recall_at_3 + value: 85.827 + - type: recall_at_5 + value: 90.938 + task: + type: Retrieval + - dataset: + config: default + name: MTEB DuRetrieval (default) + revision: a1a333e290fe30b10f3f56498e3a0d911a693ced + split: dev + type: C-MTEB/DuRetrieval + metrics: + - type: main_score + value: 87.612 + - type: map_at_1 + value: 26.430999999999997 + - type: map_at_10 + value: 80.56400000000001 + - type: map_at_100 + value: 83.41199999999999 + - type: map_at_1000 + value: 83.447 + - type: map_at_20 + value: 82.76 + - type: map_at_3 + value: 56.21000000000001 + - type: map_at_5 + value: 70.743 + - type: mrr_at_1 + value: 91.0 + - type: mrr_at_10 + value: 93.58273809523806 + - type: mrr_at_100 + value: 93.66391788982004 + - type: mrr_at_1000 + value: 93.66580315421686 + - type: mrr_at_20 + value: 93.63874828087282 + - type: mrr_at_3 + value: 93.31666666666662 + - type: mrr_at_5 + value: 93.46916666666664 + - type: nauc_map_at_1000_diff1 + value: 6.40109926511269 + - type: nauc_map_at_1000_max + value: 45.156518205790604 + - type: nauc_map_at_1000_std + value: 18.593211884073153 + - type: nauc_map_at_100_diff1 + value: 6.448711557202101 + - type: nauc_map_at_100_max + value: 45.18486159645747 + - type: nauc_map_at_100_std + value: 18.616651795860722 + - type: nauc_map_at_10_diff1 + value: 9.84371320616581 + - type: nauc_map_at_10_max + value: 40.01693268761593 + - type: nauc_map_at_10_std + value: 6.908031365560747 + - type: nauc_map_at_1_diff1 + value: 44.953791181056026 + - type: nauc_map_at_1_max + value: -11.387509242343764 + - type: nauc_map_at_1_std + value: -29.012460300786657 + - type: nauc_map_at_20_diff1 + value: 7.38580064647179 + - type: nauc_map_at_20_max + value: 44.371332746028116 + - type: nauc_map_at_20_std + value: 15.833956559472313 + - type: nauc_map_at_3_diff1 + value: 29.427952587601474 + - type: nauc_map_at_3_max + value: 3.8473270447259185 + - type: nauc_map_at_3_std + value: -23.647521458336257 + - type: nauc_map_at_5_diff1 + value: 18.713364613154774 + - type: nauc_map_at_5_max + value: 19.953262401680806 + - type: nauc_map_at_5_std + value: -11.913720371911865 + - type: nauc_mrr_at_1000_diff1 + value: 38.040195739817776 + - type: nauc_mrr_at_1000_max + value: 75.65224176632694 + - type: nauc_mrr_at_1000_std + value: 39.40257100387963 + - type: nauc_mrr_at_100_diff1 + value: 38.03778676125757 + - type: nauc_mrr_at_100_max + value: 75.65948630024249 + - type: nauc_mrr_at_100_std + value: 39.41815341603381 + - type: nauc_mrr_at_10_diff1 + value: 37.97394065068133 + - type: nauc_mrr_at_10_max + value: 75.81910017199222 + - type: nauc_mrr_at_10_std + value: 39.61696874072677 + - type: nauc_mrr_at_1_diff1 + value: 38.83675692499215 + - type: nauc_mrr_at_1_max + value: 69.99014420583036 + - type: nauc_mrr_at_1_std + value: 31.8093163191202 + - type: nauc_mrr_at_20_diff1 + value: 38.073680057072046 + - type: nauc_mrr_at_20_max + value: 75.67883974175976 + - type: nauc_mrr_at_20_std + value: 39.40528722492238 + - type: nauc_mrr_at_3_diff1 + value: 38.28768415096703 + - type: nauc_mrr_at_3_max + value: 76.28320654945267 + - type: nauc_mrr_at_3_std + value: 40.02109572008348 + - type: nauc_mrr_at_5_diff1 + value: 38.216868985695534 + - type: nauc_mrr_at_5_max + value: 76.08316007275711 + - type: nauc_mrr_at_5_std + value: 39.972457019046104 + - type: nauc_ndcg_at_1000_diff1 + value: 10.58897864933073 + - type: nauc_ndcg_at_1000_max + value: 55.5870887191566 + - type: nauc_ndcg_at_1000_std + value: 30.15493957024938 + - type: nauc_ndcg_at_100_diff1 + value: 10.868069465772917 + - type: nauc_ndcg_at_100_max + value: 55.83351050391704 + - type: nauc_ndcg_at_100_std + value: 31.072236695619615 + - type: nauc_ndcg_at_10_diff1 + value: 9.27127335978071 + - type: nauc_ndcg_at_10_max + value: 49.307727646759396 + - type: nauc_ndcg_at_10_std + value: 21.552518764523654 + - type: nauc_ndcg_at_1_diff1 + value: 38.83675692499215 + - type: nauc_ndcg_at_1_max + value: 69.99014420583036 + - type: nauc_ndcg_at_1_std + value: 31.8093163191202 + - type: nauc_ndcg_at_20_diff1 + value: 10.99181537977003 + - type: nauc_ndcg_at_20_max + value: 53.47605191227965 + - type: nauc_ndcg_at_20_std + value: 26.382045731517312 + - type: nauc_ndcg_at_3_diff1 + value: 6.5534137252176565 + - type: nauc_ndcg_at_3_max + value: 51.11114538632076 + - type: nauc_ndcg_at_3_std + value: 22.189612114146588 + - type: nauc_ndcg_at_5_diff1 + value: 7.719135315891541 + - type: nauc_ndcg_at_5_max + value: 44.360770591119504 + - type: nauc_ndcg_at_5_std + value: 18.743172873708048 + - type: nauc_precision_at_1000_diff1 + value: -31.034822128380522 + - type: nauc_precision_at_1000_max + value: 20.332934368138066 + - type: nauc_precision_at_1000_std + value: 44.338803511915614 + - type: nauc_precision_at_100_diff1 + value: -30.811245735858193 + - type: nauc_precision_at_100_max + value: 22.198372768725562 + - type: nauc_precision_at_100_std + value: 46.00454797611091 + - type: nauc_precision_at_10_diff1 + value: -31.274194785864424 + - type: nauc_precision_at_10_max + value: 32.71187220917922 + - type: nauc_precision_at_10_std + value: 42.875968635394166 + - type: nauc_precision_at_1_diff1 + value: 38.83675692499215 + - type: nauc_precision_at_1_max + value: 69.99014420583036 + - type: nauc_precision_at_1_std + value: 31.8093163191202 + - type: nauc_precision_at_20_diff1 + value: -30.67860376845899 + - type: nauc_precision_at_20_max + value: 26.74661379366211 + - type: nauc_precision_at_20_std + value: 45.59437091314149 + - type: nauc_precision_at_3_diff1 + value: -27.743818205448356 + - type: nauc_precision_at_3_max + value: 49.61474868721054 + - type: nauc_precision_at_3_std + value: 37.18101217122168 + - type: nauc_precision_at_5_diff1 + value: -32.80762892713753 + - type: nauc_precision_at_5_max + value: 40.44209817083512 + - type: nauc_precision_at_5_std + value: 40.96210133419465 + - type: nauc_recall_at_1000_diff1 + value: 6.1482636348882735 + - type: nauc_recall_at_1000_max + value: 73.7437459367813 + - type: nauc_recall_at_1000_std + value: 74.5949419078317 + - type: nauc_recall_at_100_diff1 + value: 11.893958942544847 + - type: nauc_recall_at_100_max + value: 65.83194149698096 + - type: nauc_recall_at_100_std + value: 64.9416138646916 + - type: nauc_recall_at_10_diff1 + value: 10.763264138044141 + - type: nauc_recall_at_10_max + value: 37.42063013254842 + - type: nauc_recall_at_10_std + value: 5.538711799036518 + - type: nauc_recall_at_1_diff1 + value: 44.953791181056026 + - type: nauc_recall_at_1_max + value: -11.387509242343764 + - type: nauc_recall_at_1_std + value: -29.012460300786657 + - type: nauc_recall_at_20_diff1 + value: 10.112135571654077 + - type: nauc_recall_at_20_max + value: 48.8731227722893 + - type: nauc_recall_at_20_std + value: 27.875300064543712 + - type: nauc_recall_at_3_diff1 + value: 29.035019905697286 + - type: nauc_recall_at_3_max + value: -0.8253741490097173 + - type: nauc_recall_at_3_std + value: -26.374385040717073 + - type: nauc_recall_at_5_diff1 + value: 18.90308787689296 + - type: nauc_recall_at_5_max + value: 12.373785172914031 + - type: nauc_recall_at_5_std + value: -16.43244911732844 + - type: ndcg_at_1 + value: 91.0 + - type: ndcg_at_10 + value: 87.612 + - type: ndcg_at_100 + value: 90.36699999999999 + - type: ndcg_at_1000 + value: 90.712 + - type: ndcg_at_20 + value: 89.184 + - type: ndcg_at_3 + value: 86.875 + - type: ndcg_at_5 + value: 85.724 + - type: precision_at_1 + value: 91.0 + - type: precision_at_10 + value: 41.805 + - type: precision_at_100 + value: 4.805000000000001 + - type: precision_at_1000 + value: 0.48900000000000005 + - type: precision_at_20 + value: 22.793 + - type: precision_at_3 + value: 77.85 + - type: precision_at_5 + value: 65.73 + - type: recall_at_1 + value: 26.430999999999997 + - type: recall_at_10 + value: 88.35499999999999 + - type: recall_at_100 + value: 97.366 + - type: recall_at_1000 + value: 99.235 + - type: recall_at_20 + value: 93.525 + - type: recall_at_3 + value: 58.226 + - type: recall_at_5 + value: 74.919 + task: + type: Retrieval + - dataset: + config: default + name: MTEB EcomRetrieval (default) + revision: 687de13dc7294d6fd9be10c6945f9e8fec8166b9 + split: dev + type: C-MTEB/EcomRetrieval + metrics: + - type: main_score + value: 67.161 + - type: map_at_1 + value: 52.300000000000004 + - type: map_at_10 + value: 62.166999999999994 + - type: map_at_100 + value: 62.778999999999996 + - type: map_at_1000 + value: 62.792 + - type: map_at_20 + value: 62.602000000000004 + - type: map_at_3 + value: 59.516999999999996 + - type: map_at_5 + value: 61.212 + - type: mrr_at_1 + value: 52.300000000000004 + - type: mrr_at_10 + value: 62.16682539682541 + - type: mrr_at_100 + value: 62.77902948698046 + - type: mrr_at_1000 + value: 62.792087634956715 + - type: mrr_at_20 + value: 62.60165070567552 + - type: mrr_at_3 + value: 59.516666666666666 + - type: mrr_at_5 + value: 61.21166666666669 + - type: nauc_map_at_1000_diff1 + value: 65.16885595814446 + - type: nauc_map_at_1000_max + value: 29.496662567956776 + - type: nauc_map_at_1000_std + value: -12.7407402615998 + - type: nauc_map_at_100_diff1 + value: 65.16617941092908 + - type: nauc_map_at_100_max + value: 29.502786614031272 + - type: nauc_map_at_100_std + value: -12.730774193607482 + - type: nauc_map_at_10_diff1 + value: 65.06166340128496 + - type: nauc_map_at_10_max + value: 29.354270783663335 + - type: nauc_map_at_10_std + value: -13.094010608834502 + - type: nauc_map_at_1_diff1 + value: 67.95208901030078 + - type: nauc_map_at_1_max + value: 27.603263383543116 + - type: nauc_map_at_1_std + value: -13.626768587136143 + - type: nauc_map_at_20_diff1 + value: 65.14110896910226 + - type: nauc_map_at_20_max + value: 29.517296694609605 + - type: nauc_map_at_20_std + value: -12.764608204706748 + - type: nauc_map_at_3_diff1 + value: 65.35395305214332 + - type: nauc_map_at_3_max + value: 28.576381656915512 + - type: nauc_map_at_3_std + value: -14.869794329585687 + - type: nauc_map_at_5_diff1 + value: 65.08475092137822 + - type: nauc_map_at_5_max + value: 29.10210463237133 + - type: nauc_map_at_5_std + value: -13.358219612661818 + - type: nauc_mrr_at_1000_diff1 + value: 65.16885595814446 + - type: nauc_mrr_at_1000_max + value: 29.496662567956776 + - type: nauc_mrr_at_1000_std + value: -12.7407402615998 + - type: nauc_mrr_at_100_diff1 + value: 65.16617941092908 + - type: nauc_mrr_at_100_max + value: 29.502786614031272 + - type: nauc_mrr_at_100_std + value: -12.730774193607482 + - type: nauc_mrr_at_10_diff1 + value: 65.06166340128496 + - type: nauc_mrr_at_10_max + value: 29.354270783663335 + - type: nauc_mrr_at_10_std + value: -13.094010608834502 + - type: nauc_mrr_at_1_diff1 + value: 67.95208901030078 + - type: nauc_mrr_at_1_max + value: 27.603263383543116 + - type: nauc_mrr_at_1_std + value: -13.626768587136143 + - type: nauc_mrr_at_20_diff1 + value: 65.14110896910226 + - type: nauc_mrr_at_20_max + value: 29.517296694609605 + - type: nauc_mrr_at_20_std + value: -12.764608204706748 + - type: nauc_mrr_at_3_diff1 + value: 65.35395305214332 + - type: nauc_mrr_at_3_max + value: 28.576381656915512 + - type: nauc_mrr_at_3_std + value: -14.869794329585687 + - type: nauc_mrr_at_5_diff1 + value: 65.08475092137822 + - type: nauc_mrr_at_5_max + value: 29.10210463237133 + - type: nauc_mrr_at_5_std + value: -13.358219612661818 + - type: nauc_ndcg_at_1000_diff1 + value: 64.33394397420086 + - type: nauc_ndcg_at_1000_max + value: 30.864795296588355 + - type: nauc_ndcg_at_1000_std + value: -10.533294101922223 + - type: nauc_ndcg_at_100_diff1 + value: 64.23470966841133 + - type: nauc_ndcg_at_100_max + value: 31.15266740134302 + - type: nauc_ndcg_at_100_std + value: -10.011060738308094 + - type: nauc_ndcg_at_10_diff1 + value: 63.857719978347525 + - type: nauc_ndcg_at_10_max + value: 30.617530021869715 + - type: nauc_ndcg_at_10_std + value: -11.653834629145123 + - type: nauc_ndcg_at_1_diff1 + value: 67.95208901030078 + - type: nauc_ndcg_at_1_max + value: 27.603263383543116 + - type: nauc_ndcg_at_1_std + value: -13.626768587136143 + - type: nauc_ndcg_at_20_diff1 + value: 64.1331572787928 + - type: nauc_ndcg_at_20_max + value: 31.25635677077675 + - type: nauc_ndcg_at_20_std + value: -10.276309085700548 + - type: nauc_ndcg_at_3_diff1 + value: 64.48544321575491 + - type: nauc_ndcg_at_3_max + value: 28.830444576799763 + - type: nauc_ndcg_at_3_std + value: -15.353961858062192 + - type: nauc_ndcg_at_5_diff1 + value: 63.939873109118736 + - type: nauc_ndcg_at_5_max + value: 29.829976475635444 + - type: nauc_ndcg_at_5_std + value: -12.501228923097408 + - type: nauc_precision_at_1000_diff1 + value: 37.252308330738565 + - type: nauc_precision_at_1000_max + value: 83.39039319431379 + - type: nauc_precision_at_1000_std + value: 88.40906733063514 + - type: nauc_precision_at_100_diff1 + value: 49.082892416225164 + - type: nauc_precision_at_100_max + value: 64.97665732959823 + - type: nauc_precision_at_100_std + value: 52.973337483141194 + - type: nauc_precision_at_10_diff1 + value: 57.595510418530985 + - type: nauc_precision_at_10_max + value: 37.97668604684725 + - type: nauc_precision_at_10_std + value: -2.7592219444461974 + - type: nauc_precision_at_1_diff1 + value: 67.95208901030078 + - type: nauc_precision_at_1_max + value: 27.603263383543116 + - type: nauc_precision_at_1_std + value: -13.626768587136143 + - type: nauc_precision_at_20_diff1 + value: 56.98568570908991 + - type: nauc_precision_at_20_max + value: 46.84055109587015 + - type: nauc_precision_at_20_std + value: 13.665255154616707 + - type: nauc_precision_at_3_diff1 + value: 61.609690960979556 + - type: nauc_precision_at_3_max + value: 29.644530943175635 + - type: nauc_precision_at_3_std + value: -16.983269477487482 + - type: nauc_precision_at_5_diff1 + value: 59.36708969559955 + - type: nauc_precision_at_5_max + value: 32.83509607326578 + - type: nauc_precision_at_5_std + value: -8.585997498785966 + - type: nauc_recall_at_1000_diff1 + value: 37.25230833073967 + - type: nauc_recall_at_1000_max + value: 83.39039319431508 + - type: nauc_recall_at_1000_std + value: 88.40906733063599 + - type: nauc_recall_at_100_diff1 + value: 49.08289241622588 + - type: nauc_recall_at_100_max + value: 64.97665732959842 + - type: nauc_recall_at_100_std + value: 52.973337483141805 + - type: nauc_recall_at_10_diff1 + value: 57.59551041853113 + - type: nauc_recall_at_10_max + value: 37.97668604684728 + - type: nauc_recall_at_10_std + value: -2.7592219444460553 + - type: nauc_recall_at_1_diff1 + value: 67.95208901030078 + - type: nauc_recall_at_1_max + value: 27.603263383543116 + - type: nauc_recall_at_1_std + value: -13.626768587136143 + - type: nauc_recall_at_20_diff1 + value: 56.98568570908995 + - type: nauc_recall_at_20_max + value: 46.84055109587016 + - type: nauc_recall_at_20_std + value: 13.665255154616771 + - type: nauc_recall_at_3_diff1 + value: 61.60969096097951 + - type: nauc_recall_at_3_max + value: 29.644530943175585 + - type: nauc_recall_at_3_std + value: -16.983269477487518 + - type: nauc_recall_at_5_diff1 + value: 59.36708969559964 + - type: nauc_recall_at_5_max + value: 32.83509607326576 + - type: nauc_recall_at_5_std + value: -8.585997498785925 + - type: ndcg_at_1 + value: 52.300000000000004 + - type: ndcg_at_10 + value: 67.161 + - type: ndcg_at_100 + value: 69.947 + - type: ndcg_at_1000 + value: 70.28999999999999 + - type: ndcg_at_20 + value: 68.734 + - type: ndcg_at_3 + value: 61.827 + - type: ndcg_at_5 + value: 64.857 + - type: precision_at_1 + value: 52.300000000000004 + - type: precision_at_10 + value: 8.290000000000001 + - type: precision_at_100 + value: 0.955 + - type: precision_at_1000 + value: 0.098 + - type: precision_at_20 + value: 4.455 + - type: precision_at_3 + value: 22.833000000000002 + - type: precision_at_5 + value: 15.160000000000002 + - type: recall_at_1 + value: 52.300000000000004 + - type: recall_at_10 + value: 82.89999999999999 + - type: recall_at_100 + value: 95.5 + - type: recall_at_1000 + value: 98.2 + - type: recall_at_20 + value: 89.1 + - type: recall_at_3 + value: 68.5 + - type: recall_at_5 + value: 75.8 + task: + type: Retrieval + - dataset: + config: default + name: MTEB IFlyTek (default) + revision: 421605374b29664c5fc098418fe20ada9bd55f8a + split: validation + type: C-MTEB/IFlyTek-classification + metrics: + - type: accuracy + value: 46.20238553289727 + - type: f1 + value: 38.90517287997349 + - type: f1_weighted + value: 47.72465908202853 + - type: main_score + value: 46.20238553289727 + task: + type: Classification + - dataset: + config: default + name: MTEB JDReview (default) + revision: b7c64bd89eb87f8ded463478346f76731f07bf8b + split: test + type: C-MTEB/JDReview-classification + metrics: + - type: accuracy + value: 86.86679174484053 + - type: ap + value: 55.4190333000728 + - type: ap_weighted + value: 55.4190333000728 + - type: f1 + value: 81.51249564649538 + - type: f1_weighted + value: 87.76253254123183 + - type: main_score + value: 86.86679174484053 + task: + type: Classification + - dataset: + config: default + name: MTEB LCQMC (default) + revision: 17f9b096f80380fce5ed12a9be8be7784b337daf + split: test + type: C-MTEB/LCQMC + metrics: + - type: cosine_pearson + value: 73.38528996257187 + - type: cosine_spearman + value: 79.5838761369882 + - type: euclidean_pearson + value: 79.46435441303319 + - type: euclidean_spearman + value: 79.43632868036107 + - type: main_score + value: 79.5838761369882 + - type: manhattan_pearson + value: 79.4611643611669 + - type: manhattan_spearman + value: 79.43277700673374 + task: + type: STS + - dataset: + config: default + name: MTEB MMarcoReranking (default) + revision: 8e0c766dbe9e16e1d221116a3f36795fbade07f6 + split: dev + type: C-MTEB/Mmarco-reranking + metrics: + - type: main_score + value: 35.03765306725339 + - type: map + value: 35.03765306725339 + - type: mrr + value: 34.97817460317461 + - type: nAUC_map_diff1 + value: 19.818725573084713 + - type: nAUC_map_max + value: 1.0198000256681778 + - type: nAUC_map_std + value: -17.624562992793454 + - type: nAUC_mrr_diff1 + value: 20.024047042722497 + - type: nAUC_mrr_max + value: 0.7457000534749312 + - type: nAUC_mrr_std + value: -17.90188283839233 + task: + type: Reranking + - dataset: + config: default + name: MTEB MMarcoRetrieval (default) + revision: 539bbde593d947e2a124ba72651aafc09eb33fc2 + split: dev + type: C-MTEB/MMarcoRetrieval + metrics: + - type: main_score + value: 78.377 + - type: map_at_1 + value: 64.496 + - type: map_at_10 + value: 74.316 + - type: map_at_100 + value: 74.67399999999999 + - type: map_at_1000 + value: 74.682 + - type: map_at_20 + value: 74.553 + - type: map_at_3 + value: 72.27799999999999 + - type: map_at_5 + value: 73.601 + - type: mrr_at_1 + value: 66.71919770773638 + - type: mrr_at_10 + value: 74.91401168872505 + - type: mrr_at_100 + value: 75.22834586029683 + - type: mrr_at_1000 + value: 75.23576223918033 + - type: mrr_at_20 + value: 75.1182142204938 + - type: mrr_at_3 + value: 73.12320916905448 + - type: mrr_at_5 + value: 74.29512893982785 + - type: nauc_map_at_1000_diff1 + value: 76.44188987176878 + - type: nauc_map_at_1000_max + value: 41.96583590277497 + - type: nauc_map_at_1000_std + value: -11.893947475776763 + - type: nauc_map_at_100_diff1 + value: 76.43873298246776 + - type: nauc_map_at_100_max + value: 41.97615263718138 + - type: nauc_map_at_100_std + value: -11.876568525374811 + - type: nauc_map_at_10_diff1 + value: 76.28512812077162 + - type: nauc_map_at_10_max + value: 42.111303574942454 + - type: nauc_map_at_10_std + value: -11.92840846101945 + - type: nauc_map_at_1_diff1 + value: 78.56803879890037 + - type: nauc_map_at_1_max + value: 35.78389741877027 + - type: nauc_map_at_1_std + value: -16.393221992175597 + - type: nauc_map_at_20_diff1 + value: 76.38231793080924 + - type: nauc_map_at_20_max + value: 42.05893602716135 + - type: nauc_map_at_20_std + value: -11.826119522321708 + - type: nauc_map_at_3_diff1 + value: 76.19646582467323 + - type: nauc_map_at_3_max + value: 40.85705983970697 + - type: nauc_map_at_3_std + value: -13.612451556499103 + - type: nauc_map_at_5_diff1 + value: 76.27491857077754 + - type: nauc_map_at_5_max + value: 41.85172393299732 + - type: nauc_map_at_5_std + value: -12.431459301392097 + - type: nauc_mrr_at_1000_diff1 + value: 76.92019548266093 + - type: nauc_mrr_at_1000_max + value: 42.48811285027811 + - type: nauc_mrr_at_1000_std + value: -11.317598684822896 + - type: nauc_mrr_at_100_diff1 + value: 76.91765336636446 + - type: nauc_mrr_at_100_max + value: 42.49675674263938 + - type: nauc_mrr_at_100_std + value: -11.300949737082554 + - type: nauc_mrr_at_10_diff1 + value: 76.79353999065357 + - type: nauc_mrr_at_10_max + value: 42.69909380024054 + - type: nauc_mrr_at_10_std + value: -11.270497173030472 + - type: nauc_mrr_at_1_diff1 + value: 79.7117803522604 + - type: nauc_mrr_at_1_max + value: 39.57572084455486 + - type: nauc_mrr_at_1_std + value: -15.17966940645922 + - type: nauc_mrr_at_20_diff1 + value: 76.86280745329786 + - type: nauc_mrr_at_20_max + value: 42.58380455810906 + - type: nauc_mrr_at_20_std + value: -11.229046706936792 + - type: nauc_mrr_at_3_diff1 + value: 76.67262469325809 + - type: nauc_mrr_at_3_max + value: 41.88964824896713 + - type: nauc_mrr_at_3_std + value: -12.50874971662575 + - type: nauc_mrr_at_5_diff1 + value: 76.7796936981817 + - type: nauc_mrr_at_5_max + value: 42.526924542005716 + - type: nauc_mrr_at_5_std + value: -11.587665730732638 + - type: nauc_ndcg_at_1000_diff1 + value: 75.97798533105214 + - type: nauc_ndcg_at_1000_max + value: 43.27769851746166 + - type: nauc_ndcg_at_1000_std + value: -9.42928648077032 + - type: nauc_ndcg_at_100_diff1 + value: 75.88448755710286 + - type: nauc_ndcg_at_100_max + value: 43.60463027736502 + - type: nauc_ndcg_at_100_std + value: -8.788723172592231 + - type: nauc_ndcg_at_10_diff1 + value: 75.15087803753813 + - type: nauc_ndcg_at_10_max + value: 44.53805050283006 + - type: nauc_ndcg_at_10_std + value: -8.665523258131715 + - type: nauc_ndcg_at_1_diff1 + value: 79.7117803522604 + - type: nauc_ndcg_at_1_max + value: 39.57572084455486 + - type: nauc_ndcg_at_1_std + value: -15.17966940645922 + - type: nauc_ndcg_at_20_diff1 + value: 75.46054281314423 + - type: nauc_ndcg_at_20_max + value: 44.27762687558626 + - type: nauc_ndcg_at_20_std + value: -8.193908784135246 + - type: nauc_ndcg_at_3_diff1 + value: 75.11224084416658 + - type: nauc_ndcg_at_3_max + value: 42.17726543068363 + - type: nauc_ndcg_at_3_std + value: -12.105416322156225 + - type: nauc_ndcg_at_5_diff1 + value: 75.185399674847 + - type: nauc_ndcg_at_5_max + value: 43.84929152627948 + - type: nauc_ndcg_at_5_std + value: -10.009405007276152 + - type: nauc_precision_at_1000_diff1 + value: -17.09549485267675 + - type: nauc_precision_at_1000_max + value: 14.73979745312348 + - type: nauc_precision_at_1000_std + value: 24.533810348714972 + - type: nauc_precision_at_100_diff1 + value: -6.236032711925149 + - type: nauc_precision_at_100_max + value: 23.110915607656956 + - type: nauc_precision_at_100_std + value: 28.206485316450536 + - type: nauc_precision_at_10_diff1 + value: 23.03478697515758 + - type: nauc_precision_at_10_max + value: 39.203686218447196 + - type: nauc_precision_at_10_std + value: 17.452696042464027 + - type: nauc_precision_at_1_diff1 + value: 79.7117803522604 + - type: nauc_precision_at_1_max + value: 39.57572084455486 + - type: nauc_precision_at_1_std + value: -15.17966940645922 + - type: nauc_precision_at_20_diff1 + value: 12.09320340381941 + - type: nauc_precision_at_20_max + value: 34.16449285853928 + - type: nauc_precision_at_20_std + value: 24.122439998396754 + - type: nauc_precision_at_3_diff1 + value: 47.52910178931779 + - type: nauc_precision_at_3_max + value: 40.82046536615527 + - type: nauc_precision_at_3_std + value: -0.8258153652756941 + - type: nauc_precision_at_5_diff1 + value: 36.2587080465967 + - type: nauc_precision_at_5_max + value: 41.54352007897181 + - type: nauc_precision_at_5_std + value: 8.545492145033677 + - type: nauc_recall_at_1000_diff1 + value: 62.82235268536862 + - type: nauc_recall_at_1000_max + value: 67.88265991328028 + - type: nauc_recall_at_1000_std + value: 78.62591611987277 + - type: nauc_recall_at_100_diff1 + value: 65.84511669611754 + - type: nauc_recall_at_100_max + value: 68.3187288143991 + - type: nauc_recall_at_100_std + value: 55.49540536103681 + - type: nauc_recall_at_10_diff1 + value: 66.07985276674994 + - type: nauc_recall_at_10_max + value: 59.09892833450687 + - type: nauc_recall_at_10_std + value: 12.173481401831848 + - type: nauc_recall_at_1_diff1 + value: 78.56803879890037 + - type: nauc_recall_at_1_max + value: 35.78389741877027 + - type: nauc_recall_at_1_std + value: -16.393221992175597 + - type: nauc_recall_at_20_diff1 + value: 65.60243952156914 + - type: nauc_recall_at_20_max + value: 63.1161484484425 + - type: nauc_recall_at_20_std + value: 26.62753581692267 + - type: nauc_recall_at_3_diff1 + value: 70.07831205118282 + - type: nauc_recall_at_3_max + value: 45.268817233496385 + - type: nauc_recall_at_3_std + value: -8.031502210695518 + - type: nauc_recall_at_5_diff1 + value: 68.75996804285091 + - type: nauc_recall_at_5_max + value: 51.89436269116051 + - type: nauc_recall_at_5_std + value: 0.2288841275558609 + - type: ndcg_at_1 + value: 66.719 + - type: ndcg_at_10 + value: 78.377 + - type: ndcg_at_100 + value: 79.964 + - type: ndcg_at_1000 + value: 80.185 + - type: ndcg_at_20 + value: 79.183 + - type: ndcg_at_3 + value: 74.533 + - type: ndcg_at_5 + value: 76.782 + - type: precision_at_1 + value: 66.719 + - type: precision_at_10 + value: 9.596 + - type: precision_at_100 + value: 1.038 + - type: precision_at_1000 + value: 0.106 + - type: precision_at_20 + value: 4.971 + - type: precision_at_3 + value: 28.247 + - type: precision_at_5 + value: 18.146 + - type: recall_at_1 + value: 64.496 + - type: recall_at_10 + value: 90.275 + - type: recall_at_100 + value: 97.41300000000001 + - type: recall_at_1000 + value: 99.128 + - type: recall_at_20 + value: 93.378 + - type: recall_at_3 + value: 80.19 + - type: recall_at_5 + value: 85.531 + task: + type: Retrieval + - dataset: + config: ur + name: MTEB MassiveIntentClassification (ur) + revision: 4672e20407010da34463acc759c162ca9734bca6 + split: test + type: mteb/amazon_massive_intent + metrics: + - type: accuracy + value: 9.731002017484869 + - type: f1 + value: 9.069748597280716 + - type: f1_weighted + value: 10.419815856589645 + - type: main_score + value: 9.731002017484869 + task: + type: Classification + - dataset: + config: id + name: MTEB MassiveIntentClassification (id) + revision: 4672e20407010da34463acc759c162ca9734bca6 + split: test + type: mteb/amazon_massive_intent + metrics: + - type: accuracy + value: 43.6079354404842 + - type: f1 + value: 43.25295087060459 + - type: f1_weighted + value: 43.98296829713196 + - type: main_score + value: 43.6079354404842 + task: + type: Classification + - dataset: + config: fa + name: MTEB MassiveIntentClassification (fa) + revision: 4672e20407010da34463acc759c162ca9734bca6 + split: test + type: mteb/amazon_massive_intent + metrics: + - type: accuracy + value: 12.814391392064561 + - type: f1 + value: 11.570863357381153 + - type: f1_weighted + value: 13.101944439649085 + - type: main_score + value: 12.814391392064561 + task: + type: Classification + - dataset: + config: mn + name: MTEB MassiveIntentClassification (mn) + revision: 4672e20407010da34463acc759c162ca9734bca6 + split: test + type: mteb/amazon_massive_intent + metrics: + - type: accuracy + value: 23.97108271687962 + - type: f1 + value: 24.053292531614012 + - type: f1_weighted + value: 23.851839245854936 + - type: main_score + value: 23.97108271687962 + task: + type: Classification + - dataset: + config: en + name: MTEB MassiveIntentClassification (en) + revision: 4672e20407010da34463acc759c162ca9734bca6 + split: test + type: mteb/amazon_massive_intent + metrics: + - type: accuracy + value: 62.81102891728313 + - type: f1 + value: 59.6018863604308 + - type: f1_weighted + value: 62.99690622554801 + - type: main_score + value: 62.81102891728313 + task: + type: Classification + - dataset: + config: fi + name: MTEB MassiveIntentClassification (fi) + revision: 4672e20407010da34463acc759c162ca9734bca6 + split: test + type: mteb/amazon_massive_intent + metrics: + - type: accuracy + value: 41.570275722932074 + - type: f1 + value: 39.46488297245669 + - type: f1_weighted + value: 42.502239816443456 + - type: main_score + value: 41.570275722932074 + task: + type: Classification + - dataset: + config: cy + name: MTEB MassiveIntentClassification (cy) + revision: 4672e20407010da34463acc759c162ca9734bca6 + split: test + type: mteb/amazon_massive_intent + metrics: + - type: accuracy + value: 39.8453261600538 + - type: f1 + value: 38.317101603964545 + - type: f1_weighted + value: 40.72395492044339 + - type: main_score + value: 39.8453261600538 + task: + type: Classification + - dataset: + config: af + name: MTEB MassiveIntentClassification (af) + revision: 4672e20407010da34463acc759c162ca9734bca6 + split: test + type: mteb/amazon_massive_intent + metrics: + - type: accuracy + value: 38.671822461331544 + - type: f1 + value: 37.58007811317987 + - type: f1_weighted + value: 39.10181248428257 + - type: main_score + value: 38.671822461331544 + task: + type: Classification + - dataset: + config: el + name: MTEB MassiveIntentClassification (el) + revision: 4672e20407010da34463acc759c162ca9734bca6 + split: test + type: mteb/amazon_massive_intent + metrics: + - type: accuracy + value: 30.067249495628783 + - type: f1 + value: 27.556355974187912 + - type: f1_weighted + value: 30.963659893012967 + - type: main_score + value: 30.067249495628783 + task: + type: Classification + - dataset: + config: km + name: MTEB MassiveIntentClassification (km) + revision: 4672e20407010da34463acc759c162ca9734bca6 + split: test + type: mteb/amazon_massive_intent + metrics: + - type: accuracy + value: 4.673839946200404 + - type: f1 + value: 2.5494965639802367 + - type: f1_weighted + value: 2.8145341830526283 + - type: main_score + value: 4.673839946200404 + task: + type: Classification + - dataset: + config: it + name: MTEB MassiveIntentClassification (it) + revision: 4672e20407010da34463acc759c162ca9734bca6 + split: test + type: mteb/amazon_massive_intent + metrics: + - type: accuracy + value: 46.96032279757902 + - type: f1 + value: 46.65651797980574 + - type: f1_weighted + value: 46.975519849473244 + - type: main_score + value: 46.96032279757902 + task: + type: Classification + - dataset: + config: sw + name: MTEB MassiveIntentClassification (sw) + revision: 4672e20407010da34463acc759c162ca9734bca6 + split: test + type: mteb/amazon_massive_intent + metrics: + - type: accuracy + value: 40.292535305985204 + - type: f1 + value: 39.17628651352315 + - type: f1_weighted + value: 40.839902493979025 + - type: main_score + value: 40.292535305985204 + task: + type: Classification + - dataset: + config: te + name: MTEB MassiveIntentClassification (te) + revision: 4672e20407010da34463acc759c162ca9734bca6 + split: test + type: mteb/amazon_massive_intent + metrics: + - type: accuracy + value: 2.8513786146603897 + - type: f1 + value: 1.89175192605774 + - type: f1_weighted + value: 2.5364563808332528 + - type: main_score + value: 2.8513786146603897 + task: + type: Classification + - dataset: + config: is + name: MTEB MassiveIntentClassification (is) + revision: 4672e20407010da34463acc759c162ca9734bca6 + split: test + type: mteb/amazon_massive_intent + metrics: + - type: accuracy + value: 34.451916610625425 + - type: f1 + value: 33.53074927467555 + - type: f1_weighted + value: 35.258319919642446 + - type: main_score + value: 34.451916610625425 + task: + type: Classification + - dataset: + config: lv + name: MTEB MassiveIntentClassification (lv) + revision: 4672e20407010da34463acc759c162ca9734bca6 + split: test + type: mteb/amazon_massive_intent + metrics: + - type: accuracy + value: 43.197713517148614 + - type: f1 + value: 42.60125486508949 + - type: f1_weighted + value: 43.64379570892065 + - type: main_score + value: 43.197713517148614 + task: + type: Classification + - dataset: + config: zh-CN + name: MTEB MassiveIntentClassification (zh-CN) + revision: 4672e20407010da34463acc759c162ca9734bca6 + split: test + type: mteb/amazon_massive_intent + metrics: + - type: accuracy + value: 76.01882985877604 + - type: f1 + value: 72.3776579790032 + - type: f1_weighted + value: 76.50838946568814 + - type: main_score + value: 76.01882985877604 + task: + type: Classification + - dataset: + config: az + name: MTEB MassiveIntentClassification (az) + revision: 4672e20407010da34463acc759c162ca9734bca6 + split: test + type: mteb/amazon_massive_intent + metrics: + - type: accuracy + value: 32.75386684599866 + - type: f1 + value: 31.456766708589896 + - type: f1_weighted + value: 34.0366091215231 + - type: main_score + value: 32.75386684599866 + task: + type: Classification + - dataset: + config: nl + name: MTEB MassiveIntentClassification (nl) + revision: 4672e20407010da34463acc759c162ca9734bca6 + split: test + type: mteb/amazon_massive_intent + metrics: + - type: accuracy + value: 42.57565568258238 + - type: f1 + value: 40.287327106843634 + - type: f1_weighted + value: 43.24716333400812 + - type: main_score + value: 42.57565568258238 + task: + type: Classification + - dataset: + config: he + name: MTEB MassiveIntentClassification (he) + revision: 4672e20407010da34463acc759c162ca9734bca6 + split: test + type: mteb/amazon_massive_intent + metrics: + - type: accuracy + value: 2.589105581708137 + - type: f1 + value: 0.9290630374280033 + - type: f1_weighted + value: 1.4161878562981063 + - type: main_score + value: 2.589105581708137 + task: + type: Classification + - dataset: + config: ko + name: MTEB MassiveIntentClassification (ko) + revision: 4672e20407010da34463acc759c162ca9734bca6 + split: test + type: mteb/amazon_massive_intent + metrics: + - type: accuracy + value: 25.131136516476126 + - type: f1 + value: 25.27815061140736 + - type: f1_weighted + value: 25.192290174357346 + - type: main_score + value: 25.131136516476126 + task: + type: Classification + - dataset: + config: pt + name: MTEB MassiveIntentClassification (pt) + revision: 4672e20407010da34463acc759c162ca9734bca6 + split: test + type: mteb/amazon_massive_intent + metrics: + - type: accuracy + value: 46.946872898453265 + - type: f1 + value: 45.91259775330914 + - type: f1_weighted + value: 47.398746034653406 + - type: main_score + value: 46.946872898453265 + task: + type: Classification + - dataset: + config: sl + name: MTEB MassiveIntentClassification (sl) + revision: 4672e20407010da34463acc759c162ca9734bca6 + split: test + type: mteb/amazon_massive_intent + metrics: + - type: accuracy + value: 42.017484868863484 + - type: f1 + value: 40.32799351790967 + - type: f1_weighted + value: 42.5095745023745 + - type: main_score + value: 42.017484868863484 + task: + type: Classification + - dataset: + config: sq + name: MTEB MassiveIntentClassification (sq) + revision: 4672e20407010da34463acc759c162ca9734bca6 + split: test + type: mteb/amazon_massive_intent + metrics: + - type: accuracy + value: 44.16274377942165 + - type: f1 + value: 42.59183982340306 + - type: f1_weighted + value: 45.008212136093455 + - type: main_score + value: 44.16274377942165 + task: + type: Classification + - dataset: + config: fr + name: MTEB MassiveIntentClassification (fr) + revision: 4672e20407010da34463acc759c162ca9734bca6 + split: test + type: mteb/amazon_massive_intent + metrics: + - type: accuracy + value: 43.62474781439139 + - type: f1 + value: 43.380784365848676 + - type: f1_weighted + value: 44.048384678055065 + - type: main_score + value: 43.62474781439139 + task: + type: Classification + - dataset: + config: ar + name: MTEB MassiveIntentClassification (ar) + revision: 4672e20407010da34463acc759c162ca9734bca6 + split: test + type: mteb/amazon_massive_intent + metrics: + - type: accuracy + value: 14.862138533960998 + - type: f1 + value: 13.123956008770515 + - type: f1_weighted + value: 15.175944552935434 + - type: main_score + value: 14.862138533960998 + task: + type: Classification + - dataset: + config: pl + name: MTEB MassiveIntentClassification (pl) + revision: 4672e20407010da34463acc759c162ca9734bca6 + split: test + type: mteb/amazon_massive_intent + metrics: + - type: accuracy + value: 42.59246805648957 + - type: f1 + value: 39.235549164523064 + - type: f1_weighted + value: 43.725796046895724 + - type: main_score + value: 42.59246805648957 + task: + type: Classification + - dataset: + config: es + name: MTEB MassiveIntentClassification (es) + revision: 4672e20407010da34463acc759c162ca9734bca6 + split: test + type: mteb/amazon_massive_intent + metrics: + - type: accuracy + value: 43.35911230665771 + - type: f1 + value: 43.58305920517007 + - type: f1_weighted + value: 43.45163273620528 + - type: main_score + value: 43.35911230665771 + task: + type: Classification + - dataset: + config: bn + name: MTEB MassiveIntentClassification (bn) + revision: 4672e20407010da34463acc759c162ca9734bca6 + split: test + type: mteb/amazon_massive_intent + metrics: + - type: accuracy + value: 3.5743106926698043 + - type: f1 + value: 1.9170031165168782 + - type: f1_weighted + value: 2.876028356151793 + - type: main_score + value: 3.5743106926698043 + task: + type: Classification + - dataset: + config: jv + name: MTEB MassiveIntentClassification (jv) + revision: 4672e20407010da34463acc759c162ca9734bca6 + split: test + type: mteb/amazon_massive_intent + metrics: + - type: accuracy + value: 39.3207800941493 + - type: f1 + value: 37.84945159101641 + - type: f1_weighted + value: 39.76328887450172 + - type: main_score + value: 39.3207800941493 + task: + type: Classification + - dataset: + config: sv + name: MTEB MassiveIntentClassification (sv) + revision: 4672e20407010da34463acc759c162ca9734bca6 + split: test + type: mteb/amazon_massive_intent + metrics: + - type: accuracy + value: 43.17417619367855 + - type: f1 + value: 41.51452421502344 + - type: f1_weighted + value: 43.575740588663436 + - type: main_score + value: 43.17417619367855 + task: + type: Classification + - dataset: + config: my + name: MTEB MassiveIntentClassification (my) + revision: 4672e20407010da34463acc759c162ca9734bca6 + split: test + type: mteb/amazon_massive_intent + metrics: + - type: accuracy + value: 3.940820443846671 + - type: f1 + value: 1.9024958528487437 + - type: f1_weighted + value: 2.457367277110865 + - type: main_score + value: 3.940820443846671 + task: + type: Classification + - dataset: + config: kn + name: MTEB MassiveIntentClassification (kn) + revision: 4672e20407010da34463acc759c162ca9734bca6 + split: test + type: mteb/amazon_massive_intent + metrics: + - type: accuracy + value: 3.702084734364493 + - type: f1 + value: 2.4577796241626655 + - type: f1_weighted + value: 2.718182564884992 + - type: main_score + value: 3.702084734364493 + task: + type: Classification + - dataset: + config: ta + name: MTEB MassiveIntentClassification (ta) + revision: 4672e20407010da34463acc759c162ca9734bca6 + split: test + type: mteb/amazon_massive_intent + metrics: + - type: accuracy + value: 3.338937457969065 + - type: f1 + value: 1.1915267419026427 + - type: f1_weighted + value: 2.050367350794539 + - type: main_score + value: 3.338937457969065 + task: + type: Classification + - dataset: + config: th + name: MTEB MassiveIntentClassification (th) + revision: 4672e20407010da34463acc759c162ca9734bca6 + split: test + type: mteb/amazon_massive_intent + metrics: + - type: accuracy + value: 7.108271687962341 + - type: f1 + value: 6.499099733722208 + - type: f1_weighted + value: 6.067679182650555 + - type: main_score + value: 7.108271687962341 + task: + type: Classification + - dataset: + config: am + name: MTEB MassiveIntentClassification (am) + revision: 4672e20407010da34463acc759c162ca9734bca6 + split: test + type: mteb/amazon_massive_intent + metrics: + - type: accuracy + value: 3.1069266980497647 + - type: f1 + value: 1.7074940472292803 + - type: f1_weighted + value: 1.91368359274973 + - type: main_score + value: 3.1069266980497647 + task: + type: Classification + - dataset: + config: ja + name: MTEB MassiveIntentClassification (ja) + revision: 4672e20407010da34463acc759c162ca9734bca6 + split: test + type: mteb/amazon_massive_intent + metrics: + - type: accuracy + value: 51.04909213180902 + - type: f1 + value: 49.6308815392496 + - type: f1_weighted + value: 51.14123088715286 + - type: main_score + value: 51.04909213180902 + task: + type: Classification + - dataset: + config: nb + name: MTEB MassiveIntentClassification (nb) + revision: 4672e20407010da34463acc759c162ca9734bca6 + split: test + type: mteb/amazon_massive_intent + metrics: + - type: accuracy + value: 40.154673839946206 + - type: f1 + value: 39.60433103070425 + - type: f1_weighted + value: 40.82227593141525 + - type: main_score + value: 40.154673839946206 + task: + type: Classification + - dataset: + config: de + name: MTEB MassiveIntentClassification (de) + revision: 4672e20407010da34463acc759c162ca9734bca6 + split: test + type: mteb/amazon_massive_intent + metrics: + - type: accuracy + value: 42.10827168796234 + - type: f1 + value: 39.80603638685329 + - type: f1_weighted + value: 42.64856677255116 + - type: main_score + value: 42.10827168796234 + task: + type: Classification + - dataset: + config: hu + name: MTEB MassiveIntentClassification (hu) + revision: 4672e20407010da34463acc759c162ca9734bca6 + split: test + type: mteb/amazon_massive_intent + metrics: + - type: accuracy + value: 41.4862138533961 + - type: f1 + value: 39.857700066184535 + - type: f1_weighted + value: 41.81361990540168 + - type: main_score + value: 41.4862138533961 + task: + type: Classification + - dataset: + config: ml + name: MTEB MassiveIntentClassification (ml) + revision: 4672e20407010da34463acc759c162ca9734bca6 + split: test + type: mteb/amazon_massive_intent + metrics: + - type: accuracy + value: 2.8513786146603897 + - type: f1 + value: 1.0396796629394838 + - type: f1_weighted + value: 1.679462787118112 + - type: main_score + value: 2.8513786146603897 + task: + type: Classification + - dataset: + config: tr + name: MTEB MassiveIntentClassification (tr) + revision: 4672e20407010da34463acc759c162ca9734bca6 + split: test + type: mteb/amazon_massive_intent + metrics: + - type: accuracy + value: 33.046402151983855 + - type: f1 + value: 32.071046557110236 + - type: f1_weighted + value: 34.06498014118775 + - type: main_score + value: 33.046402151983855 + task: + type: Classification + - dataset: + config: vi + name: MTEB MassiveIntentClassification (vi) + revision: 4672e20407010da34463acc759c162ca9734bca6 + split: test + type: mteb/amazon_massive_intent + metrics: + - type: accuracy + value: 39.81842636180228 + - type: f1 + value: 38.62179673846196 + - type: f1_weighted + value: 40.75093127009189 + - type: main_score + value: 39.81842636180228 + task: + type: Classification + - dataset: + config: hy + name: MTEB MassiveIntentClassification (hy) + revision: 4672e20407010da34463acc759c162ca9734bca6 + split: test + type: mteb/amazon_massive_intent + metrics: + - type: accuracy + value: 3.533960995292535 + - type: f1 + value: 1.52098804123016 + - type: f1_weighted + value: 2.4587393644122364 + - type: main_score + value: 3.533960995292535 + task: + type: Classification + - dataset: + config: hi + name: MTEB MassiveIntentClassification (hi) + revision: 4672e20407010da34463acc759c162ca9734bca6 + split: test + type: mteb/amazon_massive_intent + metrics: + - type: accuracy + value: 3.4868863483523875 + - type: f1 + value: 2.463127466064235 + - type: f1_weighted + value: 2.814989313283245 + - type: main_score + value: 3.4868863483523875 + task: + type: Classification + - dataset: + config: ms + name: MTEB MassiveIntentClassification (ms) + revision: 4672e20407010da34463acc759c162ca9734bca6 + split: test + type: mteb/amazon_massive_intent + metrics: + - type: accuracy + value: 40.534633490248815 + - type: f1 + value: 39.582513269336324 + - type: f1_weighted + value: 41.20588357370901 + - type: main_score + value: 40.534633490248815 + task: + type: Classification + - dataset: + config: zh-TW + name: MTEB MassiveIntentClassification (zh-TW) + revision: 4672e20407010da34463acc759c162ca9734bca6 + split: test + type: mteb/amazon_massive_intent + metrics: + - type: accuracy + value: 70.70275722932078 + - type: f1 + value: 69.33141153617231 + - type: f1_weighted + value: 71.00260120362651 + - type: main_score + value: 70.70275722932078 + task: + type: Classification + - dataset: + config: ro + name: MTEB MassiveIntentClassification (ro) + revision: 4672e20407010da34463acc759c162ca9734bca6 + split: test + type: mteb/amazon_massive_intent + metrics: + - type: accuracy + value: 43.67854741089442 + - type: f1 + value: 42.5674825712523 + - type: f1_weighted + value: 44.03833175704378 + - type: main_score + value: 43.67854741089442 + task: + type: Classification + - dataset: + config: ru + name: MTEB MassiveIntentClassification (ru) + revision: 4672e20407010da34463acc759c162ca9734bca6 + split: test + type: mteb/amazon_massive_intent + metrics: + - type: accuracy + value: 25.793544048419637 + - type: f1 + value: 25.149981562366037 + - type: f1_weighted + value: 26.24170939636344 + - type: main_score + value: 25.793544048419637 + task: + type: Classification + - dataset: + config: ka + name: MTEB MassiveIntentClassification (ka) + revision: 4672e20407010da34463acc759c162ca9734bca6 + split: test + type: mteb/amazon_massive_intent + metrics: + - type: accuracy + value: 2.7336919973100207 + - type: f1 + value: 1.580154047083068 + - type: f1_weighted + value: 1.4503728058594556 + - type: main_score + value: 2.7336919973100207 + task: + type: Classification + - dataset: + config: da + name: MTEB MassiveIntentClassification (da) + revision: 4672e20407010da34463acc759c162ca9734bca6 + split: test + type: mteb/amazon_massive_intent + metrics: + - type: accuracy + value: 44.9831876260928 + - type: f1 + value: 42.431462813005005 + - type: f1_weighted + value: 45.83858201062737 + - type: main_score + value: 44.9831876260928 + task: + type: Classification + - dataset: + config: tl + name: MTEB MassiveIntentClassification (tl) + revision: 4672e20407010da34463acc759c162ca9734bca6 + split: test + type: mteb/amazon_massive_intent + metrics: + - type: accuracy + value: 41.62071284465366 + - type: f1 + value: 39.31648308450773 + - type: f1_weighted + value: 42.262157671126744 + - type: main_score + value: 41.62071284465366 + task: + type: Classification + - dataset: + config: ur + name: MTEB MassiveIntentClassification (ur) + revision: 4672e20407010da34463acc759c162ca9734bca6 + split: validation + type: mteb/amazon_massive_intent + metrics: + - type: accuracy + value: 9.380226266601081 + - type: f1 + value: 8.990917585770982 + - type: f1_weighted + value: 10.555950181353865 + - type: main_score + value: 9.380226266601081 + task: + type: Classification + - dataset: + config: id + name: MTEB MassiveIntentClassification (id) + revision: 4672e20407010da34463acc759c162ca9734bca6 + split: validation + type: mteb/amazon_massive_intent + metrics: + - type: accuracy + value: 43.30054107230693 + - type: f1 + value: 42.639087356331615 + - type: f1_weighted + value: 43.69592952307332 + - type: main_score + value: 43.30054107230693 + task: + type: Classification + - dataset: + config: fa + name: MTEB MassiveIntentClassification (fa) + revision: 4672e20407010da34463acc759c162ca9734bca6 + split: validation + type: mteb/amazon_massive_intent + metrics: + - type: accuracy + value: 12.700442695523856 + - type: f1 + value: 11.392101752675561 + - type: f1_weighted + value: 13.02878956397257 + - type: main_score + value: 12.700442695523856 + task: + type: Classification + - dataset: + config: mn + name: MTEB MassiveIntentClassification (mn) + revision: 4672e20407010da34463acc759c162ca9734bca6 + split: validation + type: mteb/amazon_massive_intent + metrics: + - type: accuracy + value: 24.176094441711754 + - type: f1 + value: 24.441961354848978 + - type: f1_weighted + value: 23.943329604222363 + - type: main_score + value: 24.176094441711754 + task: + type: Classification + - dataset: + config: en + name: MTEB MassiveIntentClassification (en) + revision: 4672e20407010da34463acc759c162ca9734bca6 + split: validation + type: mteb/amazon_massive_intent + metrics: + - type: accuracy + value: 63.42351205115592 + - type: f1 + value: 60.1409077218653 + - type: f1_weighted + value: 63.23590096985862 + - type: main_score + value: 63.42351205115592 + task: + type: Classification + - dataset: + config: fi + name: MTEB MassiveIntentClassification (fi) + revision: 4672e20407010da34463acc759c162ca9734bca6 + split: validation + type: mteb/amazon_massive_intent + metrics: + - type: accuracy + value: 41.44121987211018 + - type: f1 + value: 39.88179246641776 + - type: f1_weighted + value: 41.9912976137676 + - type: main_score + value: 41.44121987211018 + task: + type: Classification + - dataset: + config: cy + name: MTEB MassiveIntentClassification (cy) + revision: 4672e20407010da34463acc759c162ca9734bca6 + split: validation + type: mteb/amazon_massive_intent + metrics: + - type: accuracy + value: 38.88342351205115 + - type: f1 + value: 38.016148507410676 + - type: f1_weighted + value: 39.67623313324099 + - type: main_score + value: 38.88342351205115 + task: + type: Classification + - dataset: + config: af + name: MTEB MassiveIntentClassification (af) + revision: 4672e20407010da34463acc759c162ca9734bca6 + split: validation + type: mteb/amazon_massive_intent + metrics: + - type: accuracy + value: 38.93753074274471 + - type: f1 + value: 38.21819811332998 + - type: f1_weighted + value: 39.30475730087287 + - type: main_score + value: 38.93753074274471 + task: + type: Classification + - dataset: + config: el + name: MTEB MassiveIntentClassification (el) + revision: 4672e20407010da34463acc759c162ca9734bca6 + split: validation + type: mteb/amazon_massive_intent + metrics: + - type: accuracy + value: 29.80324643384162 + - type: f1 + value: 27.995696304231167 + - type: f1_weighted + value: 30.733913113361595 + - type: main_score + value: 29.80324643384162 + task: + type: Classification + - dataset: + config: km + name: MTEB MassiveIntentClassification (km) + revision: 4672e20407010da34463acc759c162ca9734bca6 + split: validation + type: mteb/amazon_massive_intent + metrics: + - type: accuracy + value: 4.240039350713232 + - type: f1 + value: 2.0292251621717106 + - type: f1_weighted + value: 2.276215235518406 + - type: main_score + value: 4.240039350713232 + task: + type: Classification + - dataset: + config: it + name: MTEB MassiveIntentClassification (it) + revision: 4672e20407010da34463acc759c162ca9734bca6 + split: validation + type: mteb/amazon_massive_intent + metrics: + - type: accuracy + value: 46.36989670437776 + - type: f1 + value: 46.86880494144779 + - type: f1_weighted + value: 46.363803235010955 + - type: main_score + value: 46.36989670437776 + task: + type: Classification + - dataset: + config: sw + name: MTEB MassiveIntentClassification (sw) + revision: 4672e20407010da34463acc759c162ca9734bca6 + split: validation + type: mteb/amazon_massive_intent + metrics: + - type: accuracy + value: 40.40826364977865 + - type: f1 + value: 38.863056374899564 + - type: f1_weighted + value: 41.12559214755415 + - type: main_score + value: 40.40826364977865 + task: + type: Classification + - dataset: + config: te + name: MTEB MassiveIntentClassification (te) + revision: 4672e20407010da34463acc759c162ca9734bca6 + split: validation + type: mteb/amazon_massive_intent + metrics: + - type: accuracy + value: 3.0545991146089517 + - type: f1 + value: 1.5592797896550983 + - type: f1_weighted + value: 2.4262056804247747 + - type: main_score + value: 3.0545991146089517 + task: + type: Classification + - dataset: + config: is + name: MTEB MassiveIntentClassification (is) + revision: 4672e20407010da34463acc759c162ca9734bca6 + split: validation + type: mteb/amazon_massive_intent + metrics: + - type: accuracy + value: 33.571077225774715 + - type: f1 + value: 33.55240297777297 + - type: f1_weighted + value: 34.24069482813271 + - type: main_score + value: 33.571077225774715 + task: + type: Classification + - dataset: + config: lv + name: MTEB MassiveIntentClassification (lv) + revision: 4672e20407010da34463acc759c162ca9734bca6 + split: validation + type: mteb/amazon_massive_intent + metrics: + - type: accuracy + value: 41.1952779144122 + - type: f1 + value: 40.80253122677525 + - type: f1_weighted + value: 41.393629514985236 + - type: main_score + value: 41.1952779144122 + task: + type: Classification + - dataset: + config: zh-CN + name: MTEB MassiveIntentClassification (zh-CN) + revision: 4672e20407010da34463acc759c162ca9734bca6 + split: validation + type: mteb/amazon_massive_intent + metrics: + - type: accuracy + value: 76.91096901131333 + - type: f1 + value: 74.15538363364416 + - type: f1_weighted + value: 76.88270352222386 + - type: main_score + value: 76.91096901131333 + task: + type: Classification + - dataset: + config: az + name: MTEB MassiveIntentClassification (az) + revision: 4672e20407010da34463acc759c162ca9734bca6 + split: validation + type: mteb/amazon_massive_intent + metrics: + - type: accuracy + value: 32.52336448598131 + - type: f1 + value: 31.168524555924627 + - type: f1_weighted + value: 33.63759944502999 + - type: main_score + value: 32.52336448598131 + task: + type: Classification + - dataset: + config: nl + name: MTEB MassiveIntentClassification (nl) + revision: 4672e20407010da34463acc759c162ca9734bca6 + split: validation + type: mteb/amazon_massive_intent + metrics: + - type: accuracy + value: 40.944417117560256 + - type: f1 + value: 40.09456965856331 + - type: f1_weighted + value: 41.32329455781106 + - type: main_score + value: 40.944417117560256 + task: + type: Classification + - dataset: + config: he + name: MTEB MassiveIntentClassification (he) + revision: 4672e20407010da34463acc759c162ca9734bca6 + split: validation + type: mteb/amazon_massive_intent + metrics: + - type: accuracy + value: 2.857845548450566 + - type: f1 + value: 0.9974462765950423 + - type: f1_weighted + value: 1.4711644445043064 + - type: main_score + value: 2.857845548450566 + task: + type: Classification + - dataset: + config: ko + name: MTEB MassiveIntentClassification (ko) + revision: 4672e20407010da34463acc759c162ca9734bca6 + split: validation + type: mteb/amazon_massive_intent + metrics: + - type: accuracy + value: 24.97786522380718 + - type: f1 + value: 25.402064400694012 + - type: f1_weighted + value: 24.952618227938256 + - type: main_score + value: 24.97786522380718 + task: + type: Classification + - dataset: + config: pt + name: MTEB MassiveIntentClassification (pt) + revision: 4672e20407010da34463acc759c162ca9734bca6 + split: validation + type: mteb/amazon_massive_intent + metrics: + - type: accuracy + value: 46.43384161337924 + - type: f1 + value: 45.85033610327831 + - type: f1_weighted + value: 46.896780452483746 + - type: main_score + value: 46.43384161337924 + task: + type: Classification + - dataset: + config: sl + name: MTEB MassiveIntentClassification (sl) + revision: 4672e20407010da34463acc759c162ca9734bca6 + split: validation + type: mteb/amazon_massive_intent + metrics: + - type: accuracy + value: 41.0231185440236 + - type: f1 + value: 39.514332596107806 + - type: f1_weighted + value: 41.48978371983357 + - type: main_score + value: 41.0231185440236 + task: + type: Classification + - dataset: + config: sq + name: MTEB MassiveIntentClassification (sq) + revision: 4672e20407010da34463acc759c162ca9734bca6 + split: validation + type: mteb/amazon_massive_intent + metrics: + - type: accuracy + value: 44.402361042793906 + - type: f1 + value: 43.502050301829684 + - type: f1_weighted + value: 45.11202804620879 + - type: main_score + value: 44.402361042793906 + task: + type: Classification + - dataset: + config: fr + name: MTEB MassiveIntentClassification (fr) + revision: 4672e20407010da34463acc759c162ca9734bca6 + split: validation + type: mteb/amazon_massive_intent + metrics: + - type: accuracy + value: 43.871126414166255 + - type: f1 + value: 43.69727486044771 + - type: f1_weighted + value: 44.23366783802563 + - type: main_score + value: 43.871126414166255 + task: + type: Classification + - dataset: + config: ar + name: MTEB MassiveIntentClassification (ar) + revision: 4672e20407010da34463acc759c162ca9734bca6 + split: validation + type: mteb/amazon_massive_intent + metrics: + - type: accuracy + value: 14.490900147565174 + - type: f1 + value: 12.97222226579436 + - type: f1_weighted + value: 15.100344259005311 + - type: main_score + value: 14.490900147565174 + task: + type: Classification + - dataset: + config: pl + name: MTEB MassiveIntentClassification (pl) + revision: 4672e20407010da34463acc759c162ca9734bca6 + split: validation + type: mteb/amazon_massive_intent + metrics: + - type: accuracy + value: 42.434825381210025 + - type: f1 + value: 39.31393032873764 + - type: f1_weighted + value: 43.40696936550348 + - type: main_score + value: 42.434825381210025 + task: + type: Classification + - dataset: + config: es + name: MTEB MassiveIntentClassification (es) + revision: 4672e20407010da34463acc759c162ca9734bca6 + split: validation + type: mteb/amazon_massive_intent + metrics: + - type: accuracy + value: 43.531726512543045 + - type: f1 + value: 43.87114148531123 + - type: f1_weighted + value: 43.48862520255502 + - type: main_score + value: 43.531726512543045 + task: + type: Classification + - dataset: + config: bn + name: MTEB MassiveIntentClassification (bn) + revision: 4672e20407010da34463acc759c162ca9734bca6 + split: validation + type: mteb/amazon_massive_intent + metrics: + - type: accuracy + value: 3.3202164289227745 + - type: f1 + value: 2.1671008894013344 + - type: f1_weighted + value: 2.565626865416203 + - type: main_score + value: 3.3202164289227745 + task: + type: Classification + - dataset: + config: jv + name: MTEB MassiveIntentClassification (jv) + revision: 4672e20407010da34463acc759c162ca9734bca6 + split: validation + type: mteb/amazon_massive_intent + metrics: + - type: accuracy + value: 39.66060009837678 + - type: f1 + value: 38.09804291983762 + - type: f1_weighted + value: 40.20764763707242 + - type: main_score + value: 39.66060009837678 + task: + type: Classification + - dataset: + config: sv + name: MTEB MassiveIntentClassification (sv) + revision: 4672e20407010da34463acc759c162ca9734bca6 + split: validation + type: mteb/amazon_massive_intent + metrics: + - type: accuracy + value: 42.34628627643876 + - type: f1 + value: 41.351901052262484 + - type: f1_weighted + value: 42.74617846152216 + - type: main_score + value: 42.34628627643876 + task: + type: Classification + - dataset: + config: my + name: MTEB MassiveIntentClassification (my) + revision: 4672e20407010da34463acc759c162ca9734bca6 + split: validation + type: mteb/amazon_massive_intent + metrics: + - type: accuracy + value: 4.520413182488932 + - type: f1 + value: 2.492518890768375 + - type: f1_weighted + value: 2.8083959445441664 + - type: main_score + value: 4.520413182488932 + task: + type: Classification + - dataset: + config: kn + name: MTEB MassiveIntentClassification (kn) + revision: 4672e20407010da34463acc759c162ca9734bca6 + split: validation + type: mteb/amazon_massive_intent + metrics: + - type: accuracy + value: 3.8416133792424993 + - type: f1 + value: 2.249913702782843 + - type: f1_weighted + value: 2.8461656650048237 + - type: main_score + value: 3.8416133792424993 + task: + type: Classification + - dataset: + config: ta + name: MTEB MassiveIntentClassification (ta) + revision: 4672e20407010da34463acc759c162ca9734bca6 + split: validation + type: mteb/amazon_massive_intent + metrics: + - type: accuracy + value: 3.207083128381702 + - type: f1 + value: 1.1165194517745034 + - type: f1_weighted + value: 2.03797254022123 + - type: main_score + value: 3.207083128381702 + task: + type: Classification + - dataset: + config: th + name: MTEB MassiveIntentClassification (th) + revision: 4672e20407010da34463acc759c162ca9734bca6 + split: validation + type: mteb/amazon_massive_intent + metrics: + - type: accuracy + value: 7.19626168224299 + - type: f1 + value: 6.231436588286175 + - type: f1_weighted + value: 6.082174205390991 + - type: main_score + value: 7.19626168224299 + task: + type: Classification + - dataset: + config: am + name: MTEB MassiveIntentClassification (am) + revision: 4672e20407010da34463acc759c162ca9734bca6 + split: validation + type: mteb/amazon_massive_intent + metrics: + - type: accuracy + value: 3.216920806689622 + - type: f1 + value: 1.5706785264445453 + - type: f1_weighted + value: 1.8826764274890193 + - type: main_score + value: 3.216920806689622 + task: + type: Classification + - dataset: + config: ja + name: MTEB MassiveIntentClassification (ja) + revision: 4672e20407010da34463acc759c162ca9734bca6 + split: validation + type: mteb/amazon_massive_intent + metrics: + - type: accuracy + value: 50.90506640432858 + - type: f1 + value: 49.286571027358264 + - type: f1_weighted + value: 51.19695384060615 + - type: main_score + value: 50.90506640432858 + task: + type: Classification + - dataset: + config: nb + name: MTEB MassiveIntentClassification (nb) + revision: 4672e20407010da34463acc759c162ca9734bca6 + split: validation + type: mteb/amazon_massive_intent + metrics: + - type: accuracy + value: 40.39842597147073 + - type: f1 + value: 39.932402097909794 + - type: f1_weighted + value: 40.80662785740894 + - type: main_score + value: 40.39842597147073 + task: + type: Classification + - dataset: + config: de + name: MTEB MassiveIntentClassification (de) + revision: 4672e20407010da34463acc759c162ca9734bca6 + split: validation + type: mteb/amazon_massive_intent + metrics: + - type: accuracy + value: 41.03787506148549 + - type: f1 + value: 38.80554261518457 + - type: f1_weighted + value: 41.656448662726575 + - type: main_score + value: 41.03787506148549 + task: + type: Classification + - dataset: + config: hu + name: MTEB MassiveIntentClassification (hu) + revision: 4672e20407010da34463acc759c162ca9734bca6 + split: validation + type: mteb/amazon_massive_intent + metrics: + - type: accuracy + value: 39.685194294146584 + - type: f1 + value: 38.336045001667216 + - type: f1_weighted + value: 40.09559058068552 + - type: main_score + value: 39.685194294146584 + task: + type: Classification + - dataset: + config: ml + name: MTEB MassiveIntentClassification (ml) + revision: 4672e20407010da34463acc759c162ca9734bca6 + split: validation + type: mteb/amazon_massive_intent + metrics: + - type: accuracy + value: 2.9611411706837187 + - type: f1 + value: 1.069101410866621 + - type: f1_weighted + value: 1.505398164168353 + - type: main_score + value: 2.9611411706837187 + task: + type: Classification + - dataset: + config: tr + name: MTEB MassiveIntentClassification (tr) + revision: 4672e20407010da34463acc759c162ca9734bca6 + split: validation + type: mteb/amazon_massive_intent + metrics: + - type: accuracy + value: 33.295622233152976 + - type: f1 + value: 32.679303624594866 + - type: f1_weighted + value: 34.08394987338718 + - type: main_score + value: 33.295622233152976 + task: + type: Classification + - dataset: + config: vi + name: MTEB MassiveIntentClassification (vi) + revision: 4672e20407010da34463acc759c162ca9734bca6 + split: validation + type: mteb/amazon_massive_intent + metrics: + - type: accuracy + value: 39.621249385145106 + - type: f1 + value: 37.53378817519671 + - type: f1_weighted + value: 40.291091761567074 + - type: main_score + value: 39.621249385145106 + task: + type: Classification + - dataset: + config: hy + name: MTEB MassiveIntentClassification (hy) + revision: 4672e20407010da34463acc759c162ca9734bca6 + split: validation + type: mteb/amazon_massive_intent + metrics: + - type: accuracy + value: 3.2857845548450557 + - type: f1 + value: 1.5889657377830335 + - type: f1_weighted + value: 2.151424756030313 + - type: main_score + value: 3.2857845548450557 + task: + type: Classification + - dataset: + config: hi + name: MTEB MassiveIntentClassification (hi) + revision: 4672e20407010da34463acc759c162ca9734bca6 + split: validation + type: mteb/amazon_massive_intent + metrics: + - type: accuracy + value: 3.4825381210034436 + - type: f1 + value: 2.757982434342728 + - type: f1_weighted + value: 2.758836281053027 + - type: main_score + value: 3.4825381210034436 + task: + type: Classification + - dataset: + config: ms + name: MTEB MassiveIntentClassification (ms) + revision: 4672e20407010da34463acc759c162ca9734bca6 + split: validation + type: mteb/amazon_massive_intent + metrics: + - type: accuracy + value: 40.11805213969503 + - type: f1 + value: 39.5525331645839 + - type: f1_weighted + value: 40.68556481585343 + - type: main_score + value: 40.11805213969503 + task: + type: Classification + - dataset: + config: zh-TW + name: MTEB MassiveIntentClassification (zh-TW) + revision: 4672e20407010da34463acc759c162ca9734bca6 + split: validation + type: mteb/amazon_massive_intent + metrics: + - type: accuracy + value: 70.70339399901624 + - type: f1 + value: 69.56397933117364 + - type: f1_weighted + value: 70.636837176592 + - type: main_score + value: 70.70339399901624 + task: + type: Classification + - dataset: + config: ro + name: MTEB MassiveIntentClassification (ro) + revision: 4672e20407010da34463acc759c162ca9734bca6 + split: validation + type: mteb/amazon_massive_intent + metrics: + - type: accuracy + value: 42.636497786522376 + - type: f1 + value: 41.835721061508046 + - type: f1_weighted + value: 43.09527195252628 + - type: main_score + value: 42.636497786522376 + task: + type: Classification + - dataset: + config: ru + name: MTEB MassiveIntentClassification (ru) + revision: 4672e20407010da34463acc759c162ca9734bca6 + split: validation + type: mteb/amazon_massive_intent + metrics: + - type: accuracy + value: 25.528775209050664 + - type: f1 + value: 24.590196538466525 + - type: f1_weighted + value: 25.7679465276038 + - type: main_score + value: 25.528775209050664 + task: + type: Classification + - dataset: + config: ka + name: MTEB MassiveIntentClassification (ka) + revision: 4672e20407010da34463acc759c162ca9734bca6 + split: validation + type: mteb/amazon_massive_intent + metrics: + - type: accuracy + value: 2.4053123462862764 + - type: f1 + value: 1.0039182333347725 + - type: f1_weighted + value: 1.1504849663162338 + - type: main_score + value: 2.4053123462862764 + task: + type: Classification + - dataset: + config: da + name: MTEB MassiveIntentClassification (da) + revision: 4672e20407010da34463acc759c162ca9734bca6 + split: validation + type: mteb/amazon_massive_intent + metrics: + - type: accuracy + value: 43.94490900147566 + - type: f1 + value: 42.06122028437199 + - type: f1_weighted + value: 44.65360841137625 + - type: main_score + value: 43.94490900147566 + task: + type: Classification + - dataset: + config: tl + name: MTEB MassiveIntentClassification (tl) + revision: 4672e20407010da34463acc759c162ca9734bca6 + split: validation + type: mteb/amazon_massive_intent + metrics: + - type: accuracy + value: 40.57550418101328 + - type: f1 + value: 38.55684696220655 + - type: f1_weighted + value: 41.32491799752021 + - type: main_score + value: 40.57550418101328 + task: + type: Classification + - dataset: + config: is + name: MTEB MassiveScenarioClassification (is) + revision: fad2c6e8459f9e1c45d9315f4953d921437d70f8 + split: test + type: mteb/amazon_massive_scenario + metrics: + - type: accuracy + value: 36.267652992602564 + - type: f1 + value: 34.39028052609626 + - type: f1_weighted + value: 37.125977056069445 + - type: main_score + value: 36.267652992602564 + task: + type: Classification + - dataset: + config: el + name: MTEB MassiveScenarioClassification (el) + revision: fad2c6e8459f9e1c45d9315f4953d921437d70f8 + split: test + type: mteb/amazon_massive_scenario + metrics: + - type: accuracy + value: 31.146603900470748 + - type: f1 + value: 28.930820125296385 + - type: f1_weighted + value: 32.25487660975725 + - type: main_score + value: 31.146603900470748 + task: + type: Classification + - dataset: + config: az + name: MTEB MassiveScenarioClassification (az) + revision: fad2c6e8459f9e1c45d9315f4953d921437d70f8 + split: test + type: mteb/amazon_massive_scenario + metrics: + - type: accuracy + value: 32.59919300605246 + - type: f1 + value: 31.37679773609919 + - type: f1_weighted + value: 33.440069864146615 + - type: main_score + value: 32.59919300605246 + task: + type: Classification + - dataset: + config: it + name: MTEB MassiveScenarioClassification (it) + revision: fad2c6e8459f9e1c45d9315f4953d921437d70f8 + split: test + type: mteb/amazon_massive_scenario + metrics: + - type: accuracy + value: 46.99058507061197 + - type: f1 + value: 44.84473711276094 + - type: f1_weighted + value: 47.46949378039814 + - type: main_score + value: 46.99058507061197 + task: + type: Classification + - dataset: + config: hy + name: MTEB MassiveScenarioClassification (hy) + revision: fad2c6e8459f9e1c45d9315f4953d921437d70f8 + split: test + type: mteb/amazon_massive_scenario + metrics: + - type: accuracy + value: 8.611297915265634 + - type: f1 + value: 5.503073257951686 + - type: f1_weighted + value: 6.564944192746709 + - type: main_score + value: 8.611297915265634 + task: + type: Classification + - dataset: + config: fr + name: MTEB MassiveScenarioClassification (fr) + revision: fad2c6e8459f9e1c45d9315f4953d921437d70f8 + split: test + type: mteb/amazon_massive_scenario + metrics: + - type: accuracy + value: 46.41560188298588 + - type: f1 + value: 44.668711805596985 + - type: f1_weighted + value: 46.9398356896547 + - type: main_score + value: 46.41560188298588 + task: + type: Classification + - dataset: + config: te + name: MTEB MassiveScenarioClassification (te) + revision: fad2c6e8459f9e1c45d9315f4953d921437d70f8 + split: test + type: mteb/amazon_massive_scenario + metrics: + - type: accuracy + value: 7.3537323470074 + - type: f1 + value: 4.683418420407853 + - type: f1_weighted + value: 5.5457976920254515 + - type: main_score + value: 7.3537323470074 + task: + type: Classification + - dataset: + config: tr + name: MTEB MassiveScenarioClassification (tr) + revision: fad2c6e8459f9e1c45d9315f4953d921437d70f8 + split: test + type: mteb/amazon_massive_scenario + metrics: + - type: accuracy + value: 33.843308675184936 + - type: f1 + value: 34.20551221429157 + - type: f1_weighted + value: 34.45819700010564 + - type: main_score + value: 33.843308675184936 + task: + type: Classification + - dataset: + config: am + name: MTEB MassiveScenarioClassification (am) + revision: fad2c6e8459f9e1c45d9315f4953d921437d70f8 + split: test + type: mteb/amazon_massive_scenario + metrics: + - type: accuracy + value: 7.330195023537326 + - type: f1 + value: 4.686351443642142 + - type: f1_weighted + value: 5.028446849221205 + - type: main_score + value: 7.330195023537326 + task: + type: Classification + - dataset: + config: nl + name: MTEB MassiveScenarioClassification (nl) + revision: fad2c6e8459f9e1c45d9315f4953d921437d70f8 + split: test + type: mteb/amazon_massive_scenario + metrics: + - type: accuracy + value: 45.09414929388031 + - type: f1 + value: 41.76347483758408 + - type: f1_weighted + value: 46.25240119606559 + - type: main_score + value: 45.09414929388031 + task: + type: Classification + - dataset: + config: da + name: MTEB MassiveScenarioClassification (da) + revision: fad2c6e8459f9e1c45d9315f4953d921437d70f8 + split: test + type: mteb/amazon_massive_scenario + metrics: + - type: accuracy + value: 46.20040349697378 + - type: f1 + value: 43.19389996393894 + - type: f1_weighted + value: 47.33543539294865 + - type: main_score + value: 46.20040349697378 + task: + type: Classification + - dataset: + config: nb + name: MTEB MassiveScenarioClassification (nb) + revision: fad2c6e8459f9e1c45d9315f4953d921437d70f8 + split: test + type: mteb/amazon_massive_scenario + metrics: + - type: accuracy + value: 39.95965030262273 + - type: f1 + value: 38.05830932392177 + - type: f1_weighted + value: 41.02697348660353 + - type: main_score + value: 39.95965030262273 + task: + type: Classification + - dataset: + config: id + name: MTEB MassiveScenarioClassification (id) + revision: fad2c6e8459f9e1c45d9315f4953d921437d70f8 + split: test + type: mteb/amazon_massive_scenario + metrics: + - type: accuracy + value: 43.409549428379286 + - type: f1 + value: 41.24992542864968 + - type: f1_weighted + value: 44.215059864156004 + - type: main_score + value: 43.409549428379286 + task: + type: Classification + - dataset: + config: ru + name: MTEB MassiveScenarioClassification (ru) + revision: fad2c6e8459f9e1c45d9315f4953d921437d70f8 + split: test + type: mteb/amazon_massive_scenario + metrics: + - type: accuracy + value: 26.186953597848017 + - type: f1 + value: 25.217093636667503 + - type: f1_weighted + value: 26.708313663056355 + - type: main_score + value: 26.186953597848017 + task: + type: Classification + - dataset: + config: tl + name: MTEB MassiveScenarioClassification (tl) + revision: fad2c6e8459f9e1c45d9315f4953d921437d70f8 + split: test + type: mteb/amazon_massive_scenario + metrics: + - type: accuracy + value: 41.355077336919976 + - type: f1 + value: 39.36825529822422 + - type: f1_weighted + value: 42.55595439584406 + - type: main_score + value: 41.355077336919976 + task: + type: Classification + - dataset: + config: ka + name: MTEB MassiveScenarioClassification (ka) + revision: fad2c6e8459f9e1c45d9315f4953d921437d70f8 + split: test + type: mteb/amazon_massive_scenario + metrics: + - type: accuracy + value: 6.940147948890384 + - type: f1 + value: 4.163887735005802 + - type: f1_weighted + value: 4.523934747194612 + - type: main_score + value: 6.940147948890384 + task: + type: Classification + - dataset: + config: en + name: MTEB MassiveScenarioClassification (en) + revision: fad2c6e8459f9e1c45d9315f4953d921437d70f8 + split: test + type: mteb/amazon_massive_scenario + metrics: + - type: accuracy + value: 69.54270342972427 + - type: f1 + value: 67.2450361370488 + - type: f1_weighted + value: 69.95238321926432 + - type: main_score + value: 69.54270342972427 + task: + type: Classification + - dataset: + config: ms + name: MTEB MassiveScenarioClassification (ms) + revision: fad2c6e8459f9e1c45d9315f4953d921437d70f8 + split: test + type: mteb/amazon_massive_scenario + metrics: + - type: accuracy + value: 42.72696704774715 + - type: f1 + value: 40.6887727140475 + - type: f1_weighted + value: 43.8995830956876 + - type: main_score + value: 42.72696704774715 + task: + type: Classification + - dataset: + config: my + name: MTEB MassiveScenarioClassification (my) + revision: fad2c6e8459f9e1c45d9315f4953d921437d70f8 + split: test + type: mteb/amazon_massive_scenario + metrics: + - type: accuracy + value: 10.242098184263618 + - type: f1 + value: 6.484592667006297 + - type: f1_weighted + value: 7.305871266507618 + - type: main_score + value: 10.242098184263618 + task: + type: Classification + - dataset: + config: ml + name: MTEB MassiveScenarioClassification (ml) + revision: fad2c6e8459f9e1c45d9315f4953d921437d70f8 + split: test + type: mteb/amazon_massive_scenario + metrics: + - type: accuracy + value: 6.546738399462004 + - type: f1 + value: 3.918317398893017 + - type: f1_weighted + value: 4.158485825583739 + - type: main_score + value: 6.546738399462004 + task: + type: Classification + - dataset: + config: sv + name: MTEB MassiveScenarioClassification (sv) + revision: fad2c6e8459f9e1c45d9315f4953d921437d70f8 + split: test + type: mteb/amazon_massive_scenario + metrics: + - type: accuracy + value: 42.888365837256224 + - type: f1 + value: 40.10274962871748 + - type: f1_weighted + value: 43.37964867963827 + - type: main_score + value: 42.888365837256224 + task: + type: Classification + - dataset: + config: hu + name: MTEB MassiveScenarioClassification (hu) + revision: fad2c6e8459f9e1c45d9315f4953d921437d70f8 + split: test + type: mteb/amazon_massive_scenario + metrics: + - type: accuracy + value: 42.55884330867518 + - type: f1 + value: 40.38029399273437 + - type: f1_weighted + value: 43.398374319784985 + - type: main_score + value: 42.55884330867518 + task: + type: Classification + - dataset: + config: lv + name: MTEB MassiveScenarioClassification (lv) + revision: fad2c6e8459f9e1c45d9315f4953d921437d70f8 + split: test + type: mteb/amazon_massive_scenario + metrics: + - type: accuracy + value: 41.11970410221923 + - type: f1 + value: 38.79855099321214 + - type: f1_weighted + value: 41.871902773554794 + - type: main_score + value: 41.11970410221923 + task: + type: Classification + - dataset: + config: vi + name: MTEB MassiveScenarioClassification (vi) + revision: fad2c6e8459f9e1c45d9315f4953d921437d70f8 + split: test + type: mteb/amazon_massive_scenario + metrics: + - type: accuracy + value: 38.9004707464694 + - type: f1 + value: 38.49724424945353 + - type: f1_weighted + value: 39.6441441466175 + - type: main_score + value: 38.9004707464694 + task: + type: Classification + - dataset: + config: pl + name: MTEB MassiveScenarioClassification (pl) + revision: fad2c6e8459f9e1c45d9315f4953d921437d70f8 + split: test + type: mteb/amazon_massive_scenario + metrics: + - type: accuracy + value: 40.76664425016812 + - type: f1 + value: 38.92958333060731 + - type: f1_weighted + value: 41.85931316638676 + - type: main_score + value: 40.76664425016812 + task: + type: Classification + - dataset: + config: sl + name: MTEB MassiveScenarioClassification (sl) + revision: fad2c6e8459f9e1c45d9315f4953d921437d70f8 + split: test + type: mteb/amazon_massive_scenario + metrics: + - type: accuracy + value: 41.51647612642905 + - type: f1 + value: 39.840898472140985 + - type: f1_weighted + value: 42.10983379134603 + - type: main_score + value: 41.51647612642905 + task: + type: Classification + - dataset: + config: hi + name: MTEB MassiveScenarioClassification (hi) + revision: fad2c6e8459f9e1c45d9315f4953d921437d70f8 + split: test + type: mteb/amazon_massive_scenario + metrics: + - type: accuracy + value: 8.463349024882314 + - type: f1 + value: 6.687885838127879 + - type: f1_weighted + value: 6.665248063087772 + - type: main_score + value: 8.463349024882314 + task: + type: Classification + - dataset: + config: th + name: MTEB MassiveScenarioClassification (th) + revision: fad2c6e8459f9e1c45d9315f4953d921437d70f8 + split: test + type: mteb/amazon_massive_scenario + metrics: + - type: accuracy + value: 13.3591123066577 + - type: f1 + value: 12.221561665549492 + - type: f1_weighted + value: 11.984542743854307 + - type: main_score + value: 13.3591123066577 + task: + type: Classification + - dataset: + config: sw + name: MTEB MassiveScenarioClassification (sw) + revision: fad2c6e8459f9e1c45d9315f4953d921437d70f8 + split: test + type: mteb/amazon_massive_scenario + metrics: + - type: accuracy + value: 42.663080026899806 + - type: f1 + value: 40.728981841584826 + - type: f1_weighted + value: 43.23457106933713 + - type: main_score + value: 42.663080026899806 + task: + type: Classification + - dataset: + config: fa + name: MTEB MassiveScenarioClassification (fa) + revision: fad2c6e8459f9e1c45d9315f4953d921437d70f8 + split: test + type: mteb/amazon_massive_scenario + metrics: + - type: accuracy + value: 15.494283792871554 + - type: f1 + value: 15.006092058084446 + - type: f1_weighted + value: 15.27733810236863 + - type: main_score + value: 15.494283792871554 + task: + type: Classification + - dataset: + config: af + name: MTEB MassiveScenarioClassification (af) + revision: fad2c6e8459f9e1c45d9315f4953d921437d70f8 + split: test + type: mteb/amazon_massive_scenario + metrics: + - type: accuracy + value: 40.44720914593141 + - type: f1 + value: 37.751297424074835 + - type: f1_weighted + value: 41.61089365114751 + - type: main_score + value: 40.44720914593141 + task: + type: Classification + - dataset: + config: jv + name: MTEB MassiveScenarioClassification (jv) + revision: fad2c6e8459f9e1c45d9315f4953d921437d70f8 + split: test + type: mteb/amazon_massive_scenario + metrics: + - type: accuracy + value: 41.87626092804304 + - type: f1 + value: 40.023561485491356 + - type: f1_weighted + value: 42.18306723227648 + - type: main_score + value: 41.87626092804304 + task: + type: Classification + - dataset: + config: ko + name: MTEB MassiveScenarioClassification (ko) + revision: fad2c6e8459f9e1c45d9315f4953d921437d70f8 + split: test + type: mteb/amazon_massive_scenario + metrics: + - type: accuracy + value: 25.783456624075317 + - type: f1 + value: 24.81316657558517 + - type: f1_weighted + value: 26.453345232329223 + - type: main_score + value: 25.783456624075317 + task: + type: Classification + - dataset: + config: bn + name: MTEB MassiveScenarioClassification (bn) + revision: fad2c6e8459f9e1c45d9315f4953d921437d70f8 + split: test + type: mteb/amazon_massive_scenario + metrics: + - type: accuracy + value: 7.572293207800943 + - type: f1 + value: 5.122318304085453 + - type: f1_weighted + value: 5.918371344281876 + - type: main_score + value: 7.572293207800943 + task: + type: Classification + - dataset: + config: kn + name: MTEB MassiveScenarioClassification (kn) + revision: fad2c6e8459f9e1c45d9315f4953d921437d70f8 + split: test + type: mteb/amazon_massive_scenario + metrics: + - type: accuracy + value: 7.915265635507733 + - type: f1 + value: 5.835330719935933 + - type: f1_weighted + value: 6.015782878022624 + - type: main_score + value: 7.915265635507733 + task: + type: Classification + - dataset: + config: cy + name: MTEB MassiveScenarioClassification (cy) + revision: fad2c6e8459f9e1c45d9315f4953d921437d70f8 + split: test + type: mteb/amazon_massive_scenario + metrics: + - type: accuracy + value: 41.788836583725626 + - type: f1 + value: 38.88966219885448 + - type: f1_weighted + value: 42.8721422217295 + - type: main_score + value: 41.788836583725626 + task: + type: Classification + - dataset: + config: zh-CN + name: MTEB MassiveScenarioClassification (zh-CN) + revision: fad2c6e8459f9e1c45d9315f4953d921437d70f8 + split: test + type: mteb/amazon_massive_scenario + metrics: + - type: accuracy + value: 82.76395427034296 + - type: f1 + value: 82.41363804132969 + - type: f1_weighted + value: 83.00514475868802 + - type: main_score + value: 82.76395427034296 + task: + type: Classification + - dataset: + config: km + name: MTEB MassiveScenarioClassification (km) + revision: fad2c6e8459f9e1c45d9315f4953d921437d70f8 + split: test + type: mteb/amazon_massive_scenario + metrics: + - type: accuracy + value: 9.640215198386013 + - type: f1 + value: 5.2564140077599655 + - type: f1_weighted + value: 7.164883387181156 + - type: main_score + value: 9.640215198386013 + task: + type: Classification + - dataset: + config: pt + name: MTEB MassiveScenarioClassification (pt) + revision: fad2c6e8459f9e1c45d9315f4953d921437d70f8 + split: test + type: mteb/amazon_massive_scenario + metrics: + - type: accuracy + value: 46.64761264290519 + - type: f1 + value: 45.13961173395828 + - type: f1_weighted + value: 47.36627661666301 + - type: main_score + value: 46.64761264290519 + task: + type: Classification + - dataset: + config: sq + name: MTEB MassiveScenarioClassification (sq) + revision: fad2c6e8459f9e1c45d9315f4953d921437d70f8 + split: test + type: mteb/amazon_massive_scenario + metrics: + - type: accuracy + value: 44.061869535978474 + - type: f1 + value: 42.22172641814783 + - type: f1_weighted + value: 45.208537616148384 + - type: main_score + value: 44.061869535978474 + task: + type: Classification + - dataset: + config: ta + name: MTEB MassiveScenarioClassification (ta) + revision: fad2c6e8459f9e1c45d9315f4953d921437d70f8 + split: test + type: mteb/amazon_massive_scenario + metrics: + - type: accuracy + value: 7.528581035642231 + - type: f1 + value: 4.055954812609527 + - type: f1_weighted + value: 5.346908730375917 + - type: main_score + value: 7.528581035642231 + task: + type: Classification + - dataset: + config: es + name: MTEB MassiveScenarioClassification (es) + revision: fad2c6e8459f9e1c45d9315f4953d921437d70f8 + split: test + type: mteb/amazon_massive_scenario + metrics: + - type: accuracy + value: 44.70410221923336 + - type: f1 + value: 43.287928645786955 + - type: f1_weighted + value: 45.12367939775564 + - type: main_score + value: 44.70410221923336 + task: + type: Classification + - dataset: + config: mn + name: MTEB MassiveScenarioClassification (mn) + revision: fad2c6e8459f9e1c45d9315f4953d921437d70f8 + split: test + type: mteb/amazon_massive_scenario + metrics: + - type: accuracy + value: 25.363147276395427 + - type: f1 + value: 24.336469645147293 + - type: f1_weighted + value: 25.552654529491054 + - type: main_score + value: 25.363147276395427 + task: + type: Classification + - dataset: + config: zh-TW + name: MTEB MassiveScenarioClassification (zh-TW) + revision: fad2c6e8459f9e1c45d9315f4953d921437d70f8 + split: test + type: mteb/amazon_massive_scenario + metrics: + - type: accuracy + value: 78.58776059179557 + - type: f1 + value: 78.7265027141245 + - type: f1_weighted + value: 78.79726292257465 + - type: main_score + value: 78.58776059179557 + task: + type: Classification + - dataset: + config: de + name: MTEB MassiveScenarioClassification (de) + revision: fad2c6e8459f9e1c45d9315f4953d921437d70f8 + split: test + type: mteb/amazon_massive_scenario + metrics: + - type: accuracy + value: 44.546065904505724 + - type: f1 + value: 41.38065337271016 + - type: f1_weighted + value: 45.261964994908595 + - type: main_score + value: 44.546065904505724 + task: + type: Classification + - dataset: + config: ar + name: MTEB MassiveScenarioClassification (ar) + revision: fad2c6e8459f9e1c45d9315f4953d921437d70f8 + split: test + type: mteb/amazon_massive_scenario + metrics: + - type: accuracy + value: 18.80295897780767 + - type: f1 + value: 17.010531525930894 + - type: f1_weighted + value: 19.333227207926747 + - type: main_score + value: 18.80295897780767 + task: + type: Classification + - dataset: + config: ro + name: MTEB MassiveScenarioClassification (ro) + revision: fad2c6e8459f9e1c45d9315f4953d921437d70f8 + split: test + type: mteb/amazon_massive_scenario + metrics: + - type: accuracy + value: 45.268997982515124 + - type: f1 + value: 43.011099614102136 + - type: f1_weighted + value: 46.00264255471957 + - type: main_score + value: 45.268997982515124 + task: + type: Classification + - dataset: + config: fi + name: MTEB MassiveScenarioClassification (fi) + revision: fad2c6e8459f9e1c45d9315f4953d921437d70f8 + split: test + type: mteb/amazon_massive_scenario + metrics: + - type: accuracy + value: 39.179556153328846 + - type: f1 + value: 36.72504541129902 + - type: f1_weighted + value: 40.271736849507 + - type: main_score + value: 39.179556153328846 + task: + type: Classification + - dataset: + config: ja + name: MTEB MassiveScenarioClassification (ja) + revision: fad2c6e8459f9e1c45d9315f4953d921437d70f8 + split: test + type: mteb/amazon_massive_scenario + metrics: + - type: accuracy + value: 53.26832548755884 + - type: f1 + value: 53.061278209142884 + - type: f1_weighted + value: 53.17435324984633 + - type: main_score + value: 53.26832548755884 + task: + type: Classification + - dataset: + config: ur + name: MTEB MassiveScenarioClassification (ur) + revision: fad2c6e8459f9e1c45d9315f4953d921437d70f8 + split: test + type: mteb/amazon_massive_scenario + metrics: + - type: accuracy + value: 14.579690652320107 + - type: f1 + value: 14.206144389074717 + - type: f1_weighted + value: 14.748527351475005 + - type: main_score + value: 14.579690652320107 + task: + type: Classification + - dataset: + config: he + name: MTEB MassiveScenarioClassification (he) + revision: fad2c6e8459f9e1c45d9315f4953d921437d70f8 + split: test + type: mteb/amazon_massive_scenario + metrics: + - type: accuracy + value: 7.874915938130464 + - type: f1 + value: 4.118787513270843 + - type: f1_weighted + value: 5.817041463583794 + - type: main_score + value: 7.874915938130464 + task: + type: Classification + - dataset: + config: is + name: MTEB MassiveScenarioClassification (is) + revision: fad2c6e8459f9e1c45d9315f4953d921437d70f8 + split: validation + type: mteb/amazon_massive_scenario + metrics: + - type: accuracy + value: 35.99606492867683 + - type: f1 + value: 35.20584439264268 + - type: f1_weighted + value: 36.67345255137978 + - type: main_score + value: 35.99606492867683 + task: + type: Classification + - dataset: + config: el + name: MTEB MassiveScenarioClassification (el) + revision: fad2c6e8459f9e1c45d9315f4953d921437d70f8 + split: validation + type: mteb/amazon_massive_scenario + metrics: + - type: accuracy + value: 29.98524348253812 + - type: f1 + value: 28.765943623633927 + - type: f1_weighted + value: 30.75476241135753 + - type: main_score + value: 29.98524348253812 + task: + type: Classification + - dataset: + config: az + name: MTEB MassiveScenarioClassification (az) + revision: fad2c6e8459f9e1c45d9315f4953d921437d70f8 + split: validation + type: mteb/amazon_massive_scenario + metrics: + - type: accuracy + value: 31.908509591736355 + - type: f1 + value: 31.68840089466658 + - type: f1_weighted + value: 32.614271948745056 + - type: main_score + value: 31.908509591736355 + task: + type: Classification + - dataset: + config: it + name: MTEB MassiveScenarioClassification (it) + revision: fad2c6e8459f9e1c45d9315f4953d921437d70f8 + split: validation + type: mteb/amazon_massive_scenario + metrics: + - type: accuracy + value: 45.60255779636006 + - type: f1 + value: 44.91130039504065 + - type: f1_weighted + value: 45.901808454025705 + - type: main_score + value: 45.60255779636006 + task: + type: Classification + - dataset: + config: hy + name: MTEB MassiveScenarioClassification (hy) + revision: fad2c6e8459f9e1c45d9315f4953d921437d70f8 + split: validation + type: mteb/amazon_massive_scenario + metrics: + - type: accuracy + value: 8.480078701426464 + - type: f1 + value: 5.33881465273221 + - type: f1_weighted + value: 6.299851933533558 + - type: main_score + value: 8.480078701426464 + task: + type: Classification + - dataset: + config: fr + name: MTEB MassiveScenarioClassification (fr) + revision: fad2c6e8459f9e1c45d9315f4953d921437d70f8 + split: validation + type: mteb/amazon_massive_scenario + metrics: + - type: accuracy + value: 45.61239547466798 + - type: f1 + value: 44.796398022321725 + - type: f1_weighted + value: 46.050421712295055 + - type: main_score + value: 45.61239547466798 + task: + type: Classification + - dataset: + config: te + name: MTEB MassiveScenarioClassification (te) + revision: fad2c6e8459f9e1c45d9315f4953d921437d70f8 + split: validation + type: mteb/amazon_massive_scenario + metrics: + - type: accuracy + value: 7.333989178553861 + - type: f1 + value: 4.666258475314794 + - type: f1_weighted + value: 5.362484645743675 + - type: main_score + value: 7.333989178553861 + task: + type: Classification + - dataset: + config: tr + name: MTEB MassiveScenarioClassification (tr) + revision: fad2c6e8459f9e1c45d9315f4953d921437d70f8 + split: validation + type: mteb/amazon_massive_scenario + metrics: + - type: accuracy + value: 33.561239547466805 + - type: f1 + value: 34.0795945686752 + - type: f1_weighted + value: 33.96219218340987 + - type: main_score + value: 33.561239547466805 + task: + type: Classification + - dataset: + config: am + name: MTEB MassiveScenarioClassification (am) + revision: fad2c6e8459f9e1c45d9315f4953d921437d70f8 + split: validation + type: mteb/amazon_massive_scenario + metrics: + - type: accuracy + value: 7.604525332021645 + - type: f1 + value: 4.536275730289927 + - type: f1_weighted + value: 5.166653156052664 + - type: main_score + value: 7.604525332021645 + task: + type: Classification + - dataset: + config: nl + name: MTEB MassiveScenarioClassification (nl) + revision: fad2c6e8459f9e1c45d9315f4953d921437d70f8 + split: validation + type: mteb/amazon_massive_scenario + metrics: + - type: accuracy + value: 42.749631087063456 + - type: f1 + value: 40.97034342282828 + - type: f1_weighted + value: 43.80143496413575 + - type: main_score + value: 42.749631087063456 + task: + type: Classification + - dataset: + config: da + name: MTEB MassiveScenarioClassification (da) + revision: fad2c6e8459f9e1c45d9315f4953d921437d70f8 + split: validation + type: mteb/amazon_massive_scenario + metrics: + - type: accuracy + value: 44.00393507132317 + - type: f1 + value: 42.542091357589484 + - type: f1_weighted + value: 45.08395452421817 + - type: main_score + value: 44.00393507132317 + task: + type: Classification + - dataset: + config: nb + name: MTEB MassiveScenarioClassification (nb) + revision: fad2c6e8459f9e1c45d9315f4953d921437d70f8 + split: validation + type: mteb/amazon_massive_scenario + metrics: + - type: accuracy + value: 39.527791441219875 + - type: f1 + value: 38.877669437208 + - type: f1_weighted + value: 40.440029463255314 + - type: main_score + value: 39.527791441219875 + task: + type: Classification + - dataset: + config: id + name: MTEB MassiveScenarioClassification (id) + revision: fad2c6e8459f9e1c45d9315f4953d921437d70f8 + split: validation + type: mteb/amazon_massive_scenario + metrics: + - type: accuracy + value: 42.882439744220356 + - type: f1 + value: 41.74248841008145 + - type: f1_weighted + value: 43.49099459057078 + - type: main_score + value: 42.882439744220356 + task: + type: Classification + - dataset: + config: ru + name: MTEB MassiveScenarioClassification (ru) + revision: fad2c6e8459f9e1c45d9315f4953d921437d70f8 + split: validation + type: mteb/amazon_massive_scenario + metrics: + - type: accuracy + value: 25.976389572061 + - type: f1 + value: 25.526996864828078 + - type: f1_weighted + value: 26.323888911031062 + - type: main_score + value: 25.976389572061 + task: + type: Classification + - dataset: + config: tl + name: MTEB MassiveScenarioClassification (tl) + revision: fad2c6e8459f9e1c45d9315f4953d921437d70f8 + split: validation + type: mteb/amazon_massive_scenario + metrics: + - type: accuracy + value: 40.23610427939007 + - type: f1 + value: 39.5110278526761 + - type: f1_weighted + value: 41.1833957480028 + - type: main_score + value: 40.23610427939007 + task: + type: Classification + - dataset: + config: ka + name: MTEB MassiveScenarioClassification (ka) + revision: fad2c6e8459f9e1c45d9315f4953d921437d70f8 + split: validation + type: mteb/amazon_massive_scenario + metrics: + - type: accuracy + value: 6.714215445154944 + - type: f1 + value: 3.9946938970745207 + - type: f1_weighted + value: 4.251073566584421 + - type: main_score + value: 6.714215445154944 + task: + type: Classification + - dataset: + config: en + name: MTEB MassiveScenarioClassification (en) + revision: fad2c6e8459f9e1c45d9315f4953d921437d70f8 + split: validation + type: mteb/amazon_massive_scenario + metrics: + - type: accuracy + value: 68.92769306443678 + - type: f1 + value: 66.90869048636226 + - type: f1_weighted + value: 69.3297020719043 + - type: main_score + value: 68.92769306443678 + task: + type: Classification + - dataset: + config: ms + name: MTEB MassiveScenarioClassification (ms) + revision: fad2c6e8459f9e1c45d9315f4953d921437d70f8 + split: validation + type: mteb/amazon_massive_scenario + metrics: + - type: accuracy + value: 42.18396458435809 + - type: f1 + value: 41.2623335611438 + - type: f1_weighted + value: 43.127995422526844 + - type: main_score + value: 42.18396458435809 + task: + type: Classification + - dataset: + config: my + name: MTEB MassiveScenarioClassification (my) + revision: fad2c6e8459f9e1c45d9315f4953d921437d70f8 + split: validation + type: mteb/amazon_massive_scenario + metrics: + - type: accuracy + value: 11.082144613871126 + - type: f1 + value: 7.425778448776913 + - type: f1_weighted + value: 7.789515570254026 + - type: main_score + value: 11.082144613871126 + task: + type: Classification + - dataset: + config: ml + name: MTEB MassiveScenarioClassification (ml) + revision: fad2c6e8459f9e1c45d9315f4953d921437d70f8 + split: validation + type: mteb/amazon_massive_scenario + metrics: + - type: accuracy + value: 6.5027053615346775 + - type: f1 + value: 3.7466133746195633 + - type: f1_weighted + value: 3.973214419588327 + - type: main_score + value: 6.5027053615346775 + task: + type: Classification + - dataset: + config: sv + name: MTEB MassiveScenarioClassification (sv) + revision: fad2c6e8459f9e1c45d9315f4953d921437d70f8 + split: validation + type: mteb/amazon_massive_scenario + metrics: + - type: accuracy + value: 41.81013280865716 + - type: f1 + value: 40.48354280118566 + - type: f1_weighted + value: 42.13905635564818 + - type: main_score + value: 41.81013280865716 + task: + type: Classification + - dataset: + config: hu + name: MTEB MassiveScenarioClassification (hu) + revision: fad2c6e8459f9e1c45d9315f4953d921437d70f8 + split: validation + type: mteb/amazon_massive_scenario + metrics: + - type: accuracy + value: 39.621249385145106 + - type: f1 + value: 38.77569615351878 + - type: f1_weighted + value: 40.408850998703294 + - type: main_score + value: 39.621249385145106 + task: + type: Classification + - dataset: + config: lv + name: MTEB MassiveScenarioClassification (lv) + revision: fad2c6e8459f9e1c45d9315f4953d921437d70f8 + split: validation + type: mteb/amazon_massive_scenario + metrics: + - type: accuracy + value: 39.51795376291195 + - type: f1 + value: 38.33840469203745 + - type: f1_weighted + value: 40.14217235774783 + - type: main_score + value: 39.51795376291195 + task: + type: Classification + - dataset: + config: vi + name: MTEB MassiveScenarioClassification (vi) + revision: fad2c6e8459f9e1c45d9315f4953d921437d70f8 + split: validation + type: mteb/amazon_massive_scenario + metrics: + - type: accuracy + value: 39.11460895228726 + - type: f1 + value: 38.90884303194236 + - type: f1_weighted + value: 39.61233724806451 + - type: main_score + value: 39.11460895228726 + task: + type: Classification + - dataset: + config: pl + name: MTEB MassiveScenarioClassification (pl) + revision: fad2c6e8459f9e1c45d9315f4953d921437d70f8 + split: validation + type: mteb/amazon_massive_scenario + metrics: + - type: accuracy + value: 39.47860304968028 + - type: f1 + value: 38.456455709925585 + - type: f1_weighted + value: 40.42759387176258 + - type: main_score + value: 39.47860304968028 + task: + type: Classification + - dataset: + config: sl + name: MTEB MassiveScenarioClassification (sl) + revision: fad2c6e8459f9e1c45d9315f4953d921437d70f8 + split: validation + type: mteb/amazon_massive_scenario + metrics: + - type: accuracy + value: 40.78209542547959 + - type: f1 + value: 39.80360672965863 + - type: f1_weighted + value: 41.3388040455552 + - type: main_score + value: 40.78209542547959 + task: + type: Classification + - dataset: + config: hi + name: MTEB MassiveScenarioClassification (hi) + revision: fad2c6e8459f9e1c45d9315f4953d921437d70f8 + split: validation + type: mteb/amazon_massive_scenario + metrics: + - type: accuracy + value: 8.52926709296606 + - type: f1 + value: 6.39881808638407 + - type: f1_weighted + value: 6.616654119148606 + - type: main_score + value: 8.52926709296606 + task: + type: Classification + - dataset: + config: th + name: MTEB MassiveScenarioClassification (th) + revision: fad2c6e8459f9e1c45d9315f4953d921437d70f8 + split: validation + type: mteb/amazon_massive_scenario + metrics: + - type: accuracy + value: 13.753074274471224 + - type: f1 + value: 12.326492805202069 + - type: f1_weighted + value: 12.151551505564898 + - type: main_score + value: 13.753074274471224 + task: + type: Classification + - dataset: + config: sw + name: MTEB MassiveScenarioClassification (sw) + revision: fad2c6e8459f9e1c45d9315f4953d921437d70f8 + split: validation + type: mteb/amazon_massive_scenario + metrics: + - type: accuracy + value: 41.72651254303984 + - type: f1 + value: 40.35106230585296 + - type: f1_weighted + value: 42.504608904683664 + - type: main_score + value: 41.72651254303984 + task: + type: Classification + - dataset: + config: fa + name: MTEB MassiveScenarioClassification (fa) + revision: fad2c6e8459f9e1c45d9315f4953d921437d70f8 + split: validation + type: mteb/amazon_massive_scenario + metrics: + - type: accuracy + value: 15.31234628627644 + - type: f1 + value: 15.057814723718712 + - type: f1_weighted + value: 14.825818910096011 + - type: main_score + value: 15.31234628627644 + task: + type: Classification + - dataset: + config: af + name: MTEB MassiveScenarioClassification (af) + revision: fad2c6e8459f9e1c45d9315f4953d921437d70f8 + split: validation + type: mteb/amazon_massive_scenario + metrics: + - type: accuracy + value: 40.9640924741761 + - type: f1 + value: 39.415962035771635 + - type: f1_weighted + value: 41.880993077133425 + - type: main_score + value: 40.9640924741761 + task: + type: Classification + - dataset: + config: jv + name: MTEB MassiveScenarioClassification (jv) + revision: fad2c6e8459f9e1c45d9315f4953d921437d70f8 + split: validation + type: mteb/amazon_massive_scenario + metrics: + - type: accuracy + value: 40.850959173635026 + - type: f1 + value: 40.083644344440664 + - type: f1_weighted + value: 41.0320978528804 + - type: main_score + value: 40.850959173635026 + task: + type: Classification + - dataset: + config: ko + name: MTEB MassiveScenarioClassification (ko) + revision: fad2c6e8459f9e1c45d9315f4953d921437d70f8 + split: validation + type: mteb/amazon_massive_scenario + metrics: + - type: accuracy + value: 25.336940482046238 + - type: f1 + value: 24.385419734710172 + - type: f1_weighted + value: 25.75871028873885 + - type: main_score + value: 25.336940482046238 + task: + type: Classification + - dataset: + config: bn + name: MTEB MassiveScenarioClassification (bn) + revision: fad2c6e8459f9e1c45d9315f4953d921437d70f8 + split: validation + type: mteb/amazon_massive_scenario + metrics: + - type: accuracy + value: 7.496310870634529 + - type: f1 + value: 4.99273826850258 + - type: f1_weighted + value: 5.802198772868682 + - type: main_score + value: 7.496310870634529 + task: + type: Classification + - dataset: + config: kn + name: MTEB MassiveScenarioClassification (kn) + revision: fad2c6e8459f9e1c45d9315f4953d921437d70f8 + split: validation + type: mteb/amazon_massive_scenario + metrics: + - type: accuracy + value: 8.53418593212002 + - type: f1 + value: 6.456521102839466 + - type: f1_weighted + value: 6.396377755145225 + - type: main_score + value: 8.53418593212002 + task: + type: Classification + - dataset: + config: cy + name: MTEB MassiveScenarioClassification (cy) + revision: fad2c6e8459f9e1c45d9315f4953d921437d70f8 + split: validation + type: mteb/amazon_massive_scenario + metrics: + - type: accuracy + value: 40.20167240531235 + - type: f1 + value: 38.24662692870761 + - type: f1_weighted + value: 41.01478059789378 + - type: main_score + value: 40.20167240531235 + task: + type: Classification + - dataset: + config: zh-CN + name: MTEB MassiveScenarioClassification (zh-CN) + revision: fad2c6e8459f9e1c45d9315f4953d921437d70f8 + split: validation + type: mteb/amazon_massive_scenario + metrics: + - type: accuracy + value: 82.36104279390064 + - type: f1 + value: 81.67294237294372 + - type: f1_weighted + value: 82.57942325045582 + - type: main_score + value: 82.36104279390064 + task: + type: Classification + - dataset: + config: km + name: MTEB MassiveScenarioClassification (km) + revision: fad2c6e8459f9e1c45d9315f4953d921437d70f8 + split: validation + type: mteb/amazon_massive_scenario + metrics: + - type: accuracy + value: 9.24741760944417 + - type: f1 + value: 5.058356544468532 + - type: f1_weighted + value: 6.903761892494975 + - type: main_score + value: 9.24741760944417 + task: + type: Classification + - dataset: + config: pt + name: MTEB MassiveScenarioClassification (pt) + revision: fad2c6e8459f9e1c45d9315f4953d921437d70f8 + split: validation + type: mteb/amazon_massive_scenario + metrics: + - type: accuracy + value: 44.92867683226759 + - type: f1 + value: 44.483430371315194 + - type: f1_weighted + value: 45.52549319038087 + - type: main_score + value: 44.92867683226759 + task: + type: Classification + - dataset: + config: sq + name: MTEB MassiveScenarioClassification (sq) + revision: fad2c6e8459f9e1c45d9315f4953d921437d70f8 + split: validation + type: mteb/amazon_massive_scenario + metrics: + - type: accuracy + value: 43.059517953762914 + - type: f1 + value: 42.65977402358004 + - type: f1_weighted + value: 43.97266713539029 + - type: main_score + value: 43.059517953762914 + task: + type: Classification + - dataset: + config: ta + name: MTEB MassiveScenarioClassification (ta) + revision: fad2c6e8459f9e1c45d9315f4953d921437d70f8 + split: validation + type: mteb/amazon_massive_scenario + metrics: + - type: accuracy + value: 7.530742744712249 + - type: f1 + value: 4.0701142041780844 + - type: f1_weighted + value: 5.372399660680421 + - type: main_score + value: 7.530742744712249 + task: + type: Classification + - dataset: + config: es + name: MTEB MassiveScenarioClassification (es) + revision: fad2c6e8459f9e1c45d9315f4953d921437d70f8 + split: validation + type: mteb/amazon_massive_scenario + metrics: + - type: accuracy + value: 42.75454992621741 + - type: f1 + value: 42.45294276737515 + - type: f1_weighted + value: 42.97139601656561 + - type: main_score + value: 42.75454992621741 + task: + type: Classification + - dataset: + config: mn + name: MTEB MassiveScenarioClassification (mn) + revision: fad2c6e8459f9e1c45d9315f4953d921437d70f8 + split: validation + type: mteb/amazon_massive_scenario + metrics: + - type: accuracy + value: 24.84997540580423 + - type: f1 + value: 24.120781457641122 + - type: f1_weighted + value: 24.975936906386913 + - type: main_score + value: 24.84997540580423 + task: + type: Classification + - dataset: + config: zh-TW + name: MTEB MassiveScenarioClassification (zh-TW) + revision: fad2c6e8459f9e1c45d9315f4953d921437d70f8 + split: validation + type: mteb/amazon_massive_scenario + metrics: + - type: accuracy + value: 79.05558288243975 + - type: f1 + value: 78.9835268881931 + - type: f1_weighted + value: 79.28165747769849 + - type: main_score + value: 79.05558288243975 + task: + type: Classification + - dataset: + config: de + name: MTEB MassiveScenarioClassification (de) + revision: fad2c6e8459f9e1c45d9315f4953d921437d70f8 + split: validation + type: mteb/amazon_massive_scenario + metrics: + - type: accuracy + value: 42.58730939498278 + - type: f1 + value: 40.70852558786616 + - type: f1_weighted + value: 43.251635086069804 + - type: main_score + value: 42.58730939498278 + task: + type: Classification + - dataset: + config: ar + name: MTEB MassiveScenarioClassification (ar) + revision: fad2c6e8459f9e1c45d9315f4953d921437d70f8 + split: validation + type: mteb/amazon_massive_scenario + metrics: + - type: accuracy + value: 18.017707820954254 + - type: f1 + value: 16.687990492609256 + - type: f1_weighted + value: 18.46136988933461 + - type: main_score + value: 18.017707820954254 + task: + type: Classification + - dataset: + config: ro + name: MTEB MassiveScenarioClassification (ro) + revision: fad2c6e8459f9e1c45d9315f4953d921437d70f8 + split: validation + type: mteb/amazon_massive_scenario + metrics: + - type: accuracy + value: 43.78750614854894 + - type: f1 + value: 43.35116606491994 + - type: f1_weighted + value: 44.31377810377738 + - type: main_score + value: 43.78750614854894 + task: + type: Classification + - dataset: + config: fi + name: MTEB MassiveScenarioClassification (fi) + revision: fad2c6e8459f9e1c45d9315f4953d921437d70f8 + split: validation + type: mteb/amazon_massive_scenario + metrics: + - type: accuracy + value: 38.25381210034432 + - type: f1 + value: 36.84507866986097 + - type: f1_weighted + value: 39.15201046891089 + - type: main_score + value: 38.25381210034432 + task: + type: Classification + - dataset: + config: ja + name: MTEB MassiveScenarioClassification (ja) + revision: fad2c6e8459f9e1c45d9315f4953d921437d70f8 + split: validation + type: mteb/amazon_massive_scenario + metrics: + - type: accuracy + value: 53.29562223315298 + - type: f1 + value: 53.13848950509313 + - type: f1_weighted + value: 53.26552991076052 + - type: main_score + value: 53.29562223315298 + task: + type: Classification + - dataset: + config: ur + name: MTEB MassiveScenarioClassification (ur) + revision: fad2c6e8459f9e1c45d9315f4953d921437d70f8 + split: validation + type: mteb/amazon_massive_scenario + metrics: + - type: accuracy + value: 14.594195769798329 + - type: f1 + value: 14.397328781753599 + - type: f1_weighted + value: 14.660528748357581 + - type: main_score + value: 14.594195769798329 + task: + type: Classification + - dataset: + config: he + name: MTEB MassiveScenarioClassification (he) + revision: fad2c6e8459f9e1c45d9315f4953d921437d70f8 + split: validation + type: mteb/amazon_massive_scenario + metrics: + - type: accuracy + value: 7.879980324643385 + - type: f1 + value: 4.038729452806328 + - type: f1_weighted + value: 5.6615870271046065 + - type: main_score + value: 7.879980324643385 + task: + type: Classification + - dataset: + config: default + name: MTEB MedicalRetrieval (default) + revision: 2039188fb5800a9803ba5048df7b76e6fb151fc6 + split: dev + type: C-MTEB/MedicalRetrieval + metrics: + - type: main_score + value: 63.67699999999999 + - type: map_at_1 + value: 53.800000000000004 + - type: map_at_10 + value: 60.431999999999995 + - type: map_at_100 + value: 61.029999999999994 + - type: map_at_1000 + value: 61.071 + - type: map_at_20 + value: 60.777 + - type: map_at_3 + value: 58.75 + - type: map_at_5 + value: 59.81999999999999 + - type: mrr_at_1 + value: 53.800000000000004 + - type: mrr_at_10 + value: 60.431507936507934 + - type: mrr_at_100 + value: 61.02980943818268 + - type: mrr_at_1000 + value: 61.07101618078645 + - type: mrr_at_20 + value: 60.77656156107394 + - type: mrr_at_3 + value: 58.750000000000014 + - type: mrr_at_5 + value: 59.81999999999999 + - type: nauc_map_at_1000_diff1 + value: 78.00629416107189 + - type: nauc_map_at_1000_max + value: 52.07420225671461 + - type: nauc_map_at_1000_std + value: 7.235435225393745 + - type: nauc_map_at_100_diff1 + value: 77.9857887699591 + - type: nauc_map_at_100_max + value: 52.055541763889416 + - type: nauc_map_at_100_std + value: 7.236124232996337 + - type: nauc_map_at_10_diff1 + value: 77.91480569228007 + - type: nauc_map_at_10_max + value: 52.04826888831761 + - type: nauc_map_at_10_std + value: 7.006421025182576 + - type: nauc_map_at_1_diff1 + value: 82.20582404755503 + - type: nauc_map_at_1_max + value: 54.4721454376076 + - type: nauc_map_at_1_std + value: 4.496959883774145 + - type: nauc_map_at_20_diff1 + value: 77.95162276147624 + - type: nauc_map_at_20_max + value: 52.023771967662434 + - type: nauc_map_at_20_std + value: 7.196031931418907 + - type: nauc_map_at_3_diff1 + value: 79.03105924086732 + - type: nauc_map_at_3_max + value: 53.07038869379984 + - type: nauc_map_at_3_std + value: 6.793908288837009 + - type: nauc_map_at_5_diff1 + value: 78.21124356618412 + - type: nauc_map_at_5_max + value: 52.395049895165236 + - type: nauc_map_at_5_std + value: 6.899017664263203 + - type: nauc_mrr_at_1000_diff1 + value: 78.00601481881016 + - type: nauc_mrr_at_1000_max + value: 51.9762321296784 + - type: nauc_mrr_at_1000_std + value: 7.350305802489671 + - type: nauc_mrr_at_100_diff1 + value: 77.98550974774857 + - type: nauc_mrr_at_100_max + value: 51.957702695554374 + - type: nauc_mrr_at_100_std + value: 7.3508410873853 + - type: nauc_mrr_at_10_diff1 + value: 77.91480569228007 + - type: nauc_mrr_at_10_max + value: 51.951920429485554 + - type: nauc_mrr_at_10_std + value: 7.118827560486609 + - type: nauc_mrr_at_1_diff1 + value: 82.20582404755503 + - type: nauc_mrr_at_1_max + value: 54.29747391710586 + - type: nauc_mrr_at_1_std + value: 4.700743324359466 + - type: nauc_mrr_at_20_diff1 + value: 77.95162276147624 + - type: nauc_mrr_at_20_max + value: 51.92663048569973 + - type: nauc_mrr_at_20_std + value: 7.30936366037528 + - type: nauc_mrr_at_3_diff1 + value: 79.03105924086732 + - type: nauc_mrr_at_3_max + value: 52.97714066514463 + - type: nauc_mrr_at_3_std + value: 6.902697655601455 + - type: nauc_mrr_at_5_diff1 + value: 78.21124356618412 + - type: nauc_mrr_at_5_max + value: 52.300015316154855 + - type: nauc_mrr_at_5_std + value: 7.0098913397753355 + - type: nauc_ndcg_at_1000_diff1 + value: 76.68028623834495 + - type: nauc_ndcg_at_1000_max + value: 51.004333240246936 + - type: nauc_ndcg_at_1000_std + value: 8.698515324279041 + - type: nauc_ndcg_at_100_diff1 + value: 76.05356190504821 + - type: nauc_ndcg_at_100_max + value: 50.35277313216198 + - type: nauc_ndcg_at_100_std + value: 9.249854230791188 + - type: nauc_ndcg_at_10_diff1 + value: 75.6902794635337 + - type: nauc_ndcg_at_10_max + value: 50.299670065322886 + - type: nauc_ndcg_at_10_std + value: 8.187845234344188 + - type: nauc_ndcg_at_1_diff1 + value: 82.20582404755503 + - type: nauc_ndcg_at_1_max + value: 54.4721454376076 + - type: nauc_ndcg_at_1_std + value: 4.496959883774145 + - type: nauc_ndcg_at_20_diff1 + value: 75.79556771472184 + - type: nauc_ndcg_at_20_max + value: 50.21494338026765 + - type: nauc_ndcg_at_20_std + value: 8.945573827175862 + - type: nauc_ndcg_at_3_diff1 + value: 78.05339525168148 + - type: nauc_ndcg_at_3_max + value: 52.459350220474875 + - type: nauc_ndcg_at_3_std + value: 7.638500728186775 + - type: nauc_ndcg_at_5_diff1 + value: 76.45369105380743 + - type: nauc_ndcg_at_5_max + value: 51.19789390012578 + - type: nauc_ndcg_at_5_std + value: 7.86689928703537 + - type: nauc_precision_at_1000_diff1 + value: 59.591503267973344 + - type: nauc_precision_at_1000_max + value: 39.813258636787445 + - type: nauc_precision_at_1000_std + value: 43.9635854341733 + - type: nauc_precision_at_100_diff1 + value: 61.11499109543613 + - type: nauc_precision_at_100_max + value: 35.64662848762093 + - type: nauc_precision_at_100_std + value: 28.753011837832272 + - type: nauc_precision_at_10_diff1 + value: 66.60448251399139 + - type: nauc_precision_at_10_max + value: 42.93455303203327 + - type: nauc_precision_at_10_std + value: 12.939019781028152 + - type: nauc_precision_at_1_diff1 + value: 82.20582404755503 + - type: nauc_precision_at_1_max + value: 54.4721454376076 + - type: nauc_precision_at_1_std + value: 4.496959883774145 + - type: nauc_precision_at_20_diff1 + value: 65.68865023744912 + - type: nauc_precision_at_20_max + value: 41.36253272803797 + - type: nauc_precision_at_20_std + value: 17.955772857309167 + - type: nauc_precision_at_3_diff1 + value: 75.01623067257978 + - type: nauc_precision_at_3_max + value: 50.493541645806374 + - type: nauc_precision_at_3_std + value: 10.313986802282976 + - type: nauc_precision_at_5_diff1 + value: 70.23831710178575 + - type: nauc_precision_at_5_max + value: 46.894174569451266 + - type: nauc_precision_at_5_std + value: 11.210506044454311 + - type: nauc_recall_at_1000_diff1 + value: 59.59150326797431 + - type: nauc_recall_at_1000_max + value: 39.81325863678807 + - type: nauc_recall_at_1000_std + value: 43.96358543417411 + - type: nauc_recall_at_100_diff1 + value: 61.11499109543585 + - type: nauc_recall_at_100_max + value: 35.64662848762088 + - type: nauc_recall_at_100_std + value: 28.753011837832233 + - type: nauc_recall_at_10_diff1 + value: 66.60448251399139 + - type: nauc_recall_at_10_max + value: 42.934553032033186 + - type: nauc_recall_at_10_std + value: 12.939019781028199 + - type: nauc_recall_at_1_diff1 + value: 82.20582404755503 + - type: nauc_recall_at_1_max + value: 54.4721454376076 + - type: nauc_recall_at_1_std + value: 4.496959883774145 + - type: nauc_recall_at_20_diff1 + value: 65.68865023744912 + - type: nauc_recall_at_20_max + value: 41.36253272803804 + - type: nauc_recall_at_20_std + value: 17.955772857309242 + - type: nauc_recall_at_3_diff1 + value: 75.01623067257968 + - type: nauc_recall_at_3_max + value: 50.49354164580636 + - type: nauc_recall_at_3_std + value: 10.313986802282942 + - type: nauc_recall_at_5_diff1 + value: 70.23831710178574 + - type: nauc_recall_at_5_max + value: 46.894174569451344 + - type: nauc_recall_at_5_std + value: 11.210506044454428 + - type: ndcg_at_1 + value: 53.800000000000004 + - type: ndcg_at_10 + value: 63.67699999999999 + - type: ndcg_at_100 + value: 66.81899999999999 + - type: ndcg_at_1000 + value: 67.93299999999999 + - type: ndcg_at_20 + value: 64.90599999999999 + - type: ndcg_at_3 + value: 60.260999999999996 + - type: ndcg_at_5 + value: 62.193 + - type: precision_at_1 + value: 53.800000000000004 + - type: precision_at_10 + value: 7.39 + - type: precision_at_100 + value: 0.8920000000000001 + - type: precision_at_1000 + value: 0.098 + - type: precision_at_20 + value: 3.9350000000000005 + - type: precision_at_3 + value: 21.532999999999998 + - type: precision_at_5 + value: 13.86 + - type: recall_at_1 + value: 53.800000000000004 + - type: recall_at_10 + value: 73.9 + - type: recall_at_100 + value: 89.2 + - type: recall_at_1000 + value: 98.0 + - type: recall_at_20 + value: 78.7 + - type: recall_at_3 + value: 64.60000000000001 + - type: recall_at_5 + value: 69.3 + task: + type: Retrieval + - dataset: + config: default + name: MTEB MultilingualSentiment (default) + revision: 46958b007a63fdbf239b7672c25d0bea67b5ea1a + split: test + type: C-MTEB/MultilingualSentiment-classification + metrics: + - type: accuracy + value: 75.08333333333333 + - type: f1 + value: 74.89816720884292 + - type: f1_weighted + value: 74.89816720884292 + - type: main_score + value: 75.08333333333333 + task: + type: Classification + - dataset: + config: default + name: MTEB MultilingualSentiment (default) + revision: 46958b007a63fdbf239b7672c25d0bea67b5ea1a + split: validation + type: C-MTEB/MultilingualSentiment-classification + metrics: + - type: accuracy + value: 74.05999999999999 + - type: f1 + value: 73.86707495511577 + - type: f1_weighted + value: 73.86707495511577 + - type: main_score + value: 74.05999999999999 + task: + type: Classification + - dataset: + config: default + name: MTEB Ocnli (default) + revision: 66e76a618a34d6d565d5538088562851e6daa7ec + split: validation + type: C-MTEB/OCNLI + metrics: + - type: cosine_accuracy + value: 83.21602598808879 + - type: cosine_accuracy_threshold + value: 84.68668460845947 + - type: cosine_ap + value: 88.4718001921049 + - type: cosine_f1 + value: 84.60774577954318 + - type: cosine_f1_threshold + value: 84.68668460845947 + - type: cosine_precision + value: 79.85004686035614 + - type: cosine_recall + value: 89.96832101372756 + - type: dot_accuracy + value: 74.12019491066594 + - type: dot_accuracy_threshold + value: 61163.34228515625 + - type: dot_ap + value: 82.21843913674356 + - type: dot_f1 + value: 77.42230347349177 + - type: dot_f1_threshold + value: 58876.74560546875 + - type: dot_precision + value: 68.25141015310233 + - type: dot_recall + value: 89.44033790918691 + - type: euclidean_accuracy + value: 81.64591229020033 + - type: euclidean_accuracy_threshold + value: 1478.951644897461 + - type: euclidean_ap + value: 86.55778250928002 + - type: euclidean_f1 + value: 83.37326305701964 + - type: euclidean_f1_threshold + value: 1558.5298538208008 + - type: euclidean_precision + value: 76.31578947368422 + - type: euclidean_recall + value: 91.86906019007391 + - type: main_score + value: 83.21602598808879 + - type: manhattan_accuracy + value: 81.48348673524634 + - type: manhattan_accuracy_threshold + value: 51344.219970703125 + - type: manhattan_ap + value: 86.53822978018471 + - type: manhattan_f1 + value: 83.26139088729016 + - type: manhattan_f1_threshold + value: 52366.3330078125 + - type: manhattan_precision + value: 76.27416520210897 + - type: manhattan_recall + value: 91.65786694825766 + - type: max_accuracy + value: 83.21602598808879 + - type: max_ap + value: 88.4718001921049 + - type: max_f1 + value: 84.60774577954318 + - type: max_precision + value: 79.85004686035614 + - type: max_recall + value: 91.86906019007391 + - type: similarity_accuracy + value: 83.21602598808879 + - type: similarity_accuracy_threshold + value: 84.686678647995 + - type: similarity_ap + value: 88.4718001921049 + - type: similarity_f1 + value: 84.60774577954318 + - type: similarity_f1_threshold + value: 84.686678647995 + - type: similarity_precision + value: 79.85004686035614 + - type: similarity_recall + value: 89.96832101372756 + task: + type: PairClassification + - dataset: + config: default + name: MTEB OnlineShopping (default) + revision: e610f2ebd179a8fda30ae534c3878750a96db120 + split: test + type: C-MTEB/OnlineShopping-classification + metrics: + - type: accuracy + value: 93.11 + - type: ap + value: 90.79657759750567 + - type: ap_weighted + value: 90.79657759750567 + - type: f1 + value: 93.09242477250346 + - type: f1_weighted + value: 93.10745591896395 + - type: main_score + value: 93.11 + task: + type: Classification + - dataset: + config: default + name: MTEB PAWSX (default) + revision: 9c6a90e430ac22b5779fb019a23e820b11a8b5e1 + split: test + type: C-MTEB/PAWSX + metrics: + - type: cosine_pearson + value: 40.85294247983506 + - type: cosine_spearman + value: 46.26470443016128 + - type: euclidean_pearson + value: 45.71637535133479 + - type: euclidean_spearman + value: 45.97506398282988 + - type: main_score + value: 46.26470443016128 + - type: manhattan_pearson + value: 45.729052797800186 + - type: manhattan_spearman + value: 45.985167052566766 + task: + type: STS + - dataset: + config: default + name: MTEB QBQTC (default) + revision: 790b0510dc52b1553e8c49f3d2afb48c0e5c48b7 + split: test + type: C-MTEB/QBQTC + metrics: + - type: cosine_pearson + value: 39.61472967042291 + - type: cosine_spearman + value: 41.89107113069583 + - type: euclidean_pearson + value: 33.90266552269117 + - type: euclidean_spearman + value: 38.34637073395628 + - type: main_score + value: 41.89107113069583 + - type: manhattan_pearson + value: 33.91591969583774 + - type: manhattan_spearman + value: 38.3574043430055 + task: + type: STS + - dataset: + config: pl-en + name: MTEB STS22.v2 (pl-en) + revision: d31f33a128469b20e357535c39b82fb3c3f6f2bd + split: test + type: mteb/sts22-crosslingual-sts + metrics: + - type: cosine_pearson + value: 43.59038612153097 + - type: cosine_spearman + value: 39.81437610772274 + - type: euclidean_pearson + value: 40.891702223169865 + - type: euclidean_spearman + value: 37.26643328502484 + - type: main_score + value: 39.81437610772274 + - type: manhattan_pearson + value: 41.986843698898646 + - type: manhattan_spearman + value: 39.32140456159206 + - type: pearson + value: 43.59041415960545 + - type: spearman + value: 39.81437610772274 + task: + type: STS + - dataset: + config: ar + name: MTEB STS22.v2 (ar) + revision: d31f33a128469b20e357535c39b82fb3c3f6f2bd + split: test + type: mteb/sts22-crosslingual-sts + metrics: + - type: cosine_pearson + value: 33.5428416005511 + - type: cosine_spearman + value: 30.37853891189232 + - type: euclidean_pearson + value: 34.60992317007351 + - type: euclidean_spearman + value: 30.63949361401844 + - type: main_score + value: 30.37853891189232 + - type: manhattan_pearson + value: 34.5473591619201 + - type: manhattan_spearman + value: 30.55163856425125 + - type: pearson + value: 33.54285209728335 + - type: spearman + value: 30.37853891189232 + task: + type: STS + - dataset: + config: en + name: MTEB STS22.v2 (en) + revision: d31f33a128469b20e357535c39b82fb3c3f6f2bd + split: test + type: mteb/sts22-crosslingual-sts + metrics: + - type: cosine_pearson + value: 55.197301705940106 + - type: cosine_spearman + value: 60.735304152341016 + - type: euclidean_pearson + value: 58.01165360549454 + - type: euclidean_spearman + value: 59.382742411350954 + - type: main_score + value: 60.735304152341016 + - type: manhattan_pearson + value: 57.83223965160013 + - type: manhattan_spearman + value: 59.21305162189526 + - type: pearson + value: 55.197304101079325 + - type: spearman + value: 60.735304152341016 + task: + type: STS + - dataset: + config: es + name: MTEB STS22.v2 (es) + revision: d31f33a128469b20e357535c39b82fb3c3f6f2bd + split: test + type: mteb/sts22-crosslingual-sts + metrics: + - type: cosine_pearson + value: 46.72328050183308 + - type: cosine_spearman + value: 53.40518194954016 + - type: euclidean_pearson + value: 49.16877625976702 + - type: euclidean_spearman + value: 53.00258608772506 + - type: main_score + value: 53.40518194954016 + - type: manhattan_pearson + value: 49.01410667940025 + - type: manhattan_spearman + value: 53.062750442834506 + - type: pearson + value: 46.7232634869361 + - type: spearman + value: 53.40518194954016 + task: + type: STS + - dataset: + config: tr + name: MTEB STS22.v2 (tr) + revision: d31f33a128469b20e357535c39b82fb3c3f6f2bd + split: test + type: mteb/sts22-crosslingual-sts + metrics: + - type: cosine_pearson + value: 32.0893349414057 + - type: cosine_spearman + value: 39.88878359477052 + - type: euclidean_pearson + value: 36.428586715032374 + - type: euclidean_spearman + value: 40.255813965229805 + - type: main_score + value: 39.88878359477052 + - type: manhattan_pearson + value: 36.445628185483436 + - type: manhattan_spearman + value: 40.164916492530764 + - type: pearson + value: 32.08931820630885 + - type: spearman + value: 39.88878359477052 + task: + type: STS + - dataset: + config: ru + name: MTEB STS22.v2 (ru) + revision: d31f33a128469b20e357535c39b82fb3c3f6f2bd + split: test + type: mteb/sts22-crosslingual-sts + metrics: + - type: cosine_pearson + value: 25.70940596089177 + - type: cosine_spearman + value: 24.851779346905516 + - type: euclidean_pearson + value: 26.662807591660304 + - type: euclidean_spearman + value: 25.1585901925807 + - type: main_score + value: 24.851779346905516 + - type: manhattan_pearson + value: 26.78879644774409 + - type: manhattan_spearman + value: 25.30589435887287 + - type: pearson + value: 25.70945811392969 + - type: spearman + value: 24.851779346905516 + task: + type: STS + - dataset: + config: pl + name: MTEB STS22.v2 (pl) + revision: d31f33a128469b20e357535c39b82fb3c3f6f2bd + split: test + type: mteb/sts22-crosslingual-sts + metrics: + - type: cosine_pearson + value: 9.765885880324584 + - type: cosine_spearman + value: 16.56405546386883 + - type: euclidean_pearson + value: 1.266714500111099 + - type: euclidean_spearman + value: 15.873350818474066 + - type: main_score + value: 16.56405546386883 + - type: manhattan_pearson + value: 1.5996037603404067 + - type: manhattan_spearman + value: 15.984627455042919 + - type: pearson + value: 9.765881253247253 + - type: spearman + value: 16.37790963582666 + task: + type: STS + - dataset: + config: fr-pl + name: MTEB STS22.v2 (fr-pl) + revision: d31f33a128469b20e357535c39b82fb3c3f6f2bd + split: test + type: mteb/sts22-crosslingual-sts + metrics: + - type: cosine_pearson + value: 72.48621899222191 + - type: cosine_spearman + value: 84.51542547285167 + - type: euclidean_pearson + value: 79.91610128578839 + - type: euclidean_spearman + value: 84.51542547285167 + - type: main_score + value: 84.51542547285167 + - type: manhattan_pearson + value: 78.02332046266785 + - type: manhattan_spearman + value: 84.51542547285167 + - type: pearson + value: 72.48643781257383 + - type: spearman + value: 84.51542547285167 + task: + type: STS + - dataset: + config: it + name: MTEB STS22.v2 (it) + revision: d31f33a128469b20e357535c39b82fb3c3f6f2bd + split: test + type: mteb/sts22-crosslingual-sts + metrics: + - type: cosine_pearson + value: 42.88595613594089 + - type: cosine_spearman + value: 50.00902534757776 + - type: euclidean_pearson + value: 46.575636873141114 + - type: euclidean_spearman + value: 50.233303173324586 + - type: main_score + value: 50.00902534757776 + - type: manhattan_pearson + value: 46.74155972808209 + - type: manhattan_spearman + value: 50.463048251775234 + - type: pearson + value: 42.88596484422846 + - type: spearman + value: 50.00902534757776 + task: + type: STS + - dataset: + config: es-it + name: MTEB STS22.v2 (es-it) + revision: d31f33a128469b20e357535c39b82fb3c3f6f2bd + split: test + type: mteb/sts22-crosslingual-sts + metrics: + - type: cosine_pearson + value: 34.232754484342784 + - type: cosine_spearman + value: 35.46570858579162 + - type: euclidean_pearson + value: 35.3961638859008 + - type: euclidean_spearman + value: 34.24608020799141 + - type: main_score + value: 35.46570858579162 + - type: manhattan_pearson + value: 35.350258241347966 + - type: manhattan_spearman + value: 34.26016474055034 + - type: pearson + value: 34.23272177114215 + - type: spearman + value: 35.46570858579162 + task: + type: STS + - dataset: + config: de-pl + name: MTEB STS22.v2 (de-pl) + revision: d31f33a128469b20e357535c39b82fb3c3f6f2bd + split: test + type: mteb/sts22-crosslingual-sts + metrics: + - type: cosine_pearson + value: 12.399198871280888 + - type: cosine_spearman + value: 7.26740842591211 + - type: euclidean_pearson + value: 4.516712707783254 + - type: euclidean_spearman + value: 6.466761734921793 + - type: main_score + value: 7.26740842591211 + - type: manhattan_pearson + value: 4.192789002097003 + - type: manhattan_spearman + value: 6.805496873417696 + - type: pearson + value: 12.39933152364361 + - type: spearman + value: 7.26740842591211 + task: + type: STS + - dataset: + config: de-en + name: MTEB STS22.v2 (de-en) + revision: d31f33a128469b20e357535c39b82fb3c3f6f2bd + split: test + type: mteb/sts22-crosslingual-sts + metrics: + - type: cosine_pearson + value: 36.68189817510892 + - type: cosine_spearman + value: 42.40215334784587 + - type: euclidean_pearson + value: 37.445161615466496 + - type: euclidean_spearman + value: 41.662219127863445 + - type: main_score + value: 42.40215334784587 + - type: manhattan_pearson + value: 37.428330543761284 + - type: manhattan_spearman + value: 41.15139706997196 + - type: pearson + value: 36.68189189374063 + - type: spearman + value: 42.40215334784587 + task: + type: STS + - dataset: + config: de + name: MTEB STS22.v2 (de) + revision: d31f33a128469b20e357535c39b82fb3c3f6f2bd + split: test + type: mteb/sts22-crosslingual-sts + metrics: + - type: cosine_pearson + value: 14.610716018581904 + - type: cosine_spearman + value: 20.259656537989873 + - type: euclidean_pearson + value: 13.39968577970463 + - type: euclidean_spearman + value: 19.999373521862104 + - type: main_score + value: 20.259656537989873 + - type: manhattan_pearson + value: 13.403453329892018 + - type: manhattan_spearman + value: 20.035611306496893 + - type: pearson + value: 14.610694852334966 + - type: spearman + value: 20.265485348016593 + task: + type: STS + - dataset: + config: zh + name: MTEB STS22.v2 (zh) + revision: d31f33a128469b20e357535c39b82fb3c3f6f2bd + split: test + type: mteb/sts22-crosslingual-sts + metrics: + - type: cosine_pearson + value: 73.90500579546007 + - type: cosine_spearman + value: 73.22277115520708 + - type: euclidean_pearson + value: 72.26966368799292 + - type: euclidean_spearman + value: 72.94244978403874 + - type: main_score + value: 73.22277115520708 + - type: manhattan_pearson + value: 72.16998184519838 + - type: manhattan_spearman + value: 72.90099748660055 + - type: pearson + value: 73.90500331010179 + - type: spearman + value: 73.22277115520708 + task: + type: STS + - dataset: + config: fr + name: MTEB STS22.v2 (fr) + revision: d31f33a128469b20e357535c39b82fb3c3f6f2bd + split: test + type: mteb/sts22-crosslingual-sts + metrics: + - type: cosine_pearson + value: 44.47331432475519 + - type: cosine_spearman + value: 53.744106729883455 + - type: euclidean_pearson + value: 50.300639054319085 + - type: euclidean_spearman + value: 53.34089424965791 + - type: main_score + value: 53.744106729883455 + - type: manhattan_pearson + value: 50.179266864566344 + - type: manhattan_spearman + value: 53.30577964927791 + - type: pearson + value: 44.473302379385146 + - type: spearman + value: 53.744106729883455 + task: + type: STS + - dataset: + config: zh-en + name: MTEB STS22.v2 (zh-en) + revision: d31f33a128469b20e357535c39b82fb3c3f6f2bd + split: test + type: mteb/sts22-crosslingual-sts + metrics: + - type: cosine_pearson + value: 59.792988838605446 + - type: cosine_spearman + value: 65.17573588512339 + - type: euclidean_pearson + value: 63.23291280267431 + - type: euclidean_spearman + value: 65.05898841612739 + - type: main_score + value: 65.17573588512339 + - type: manhattan_pearson + value: 63.14148457672645 + - type: manhattan_spearman + value: 64.86625240506501 + - type: pearson + value: 59.79298501419738 + - type: spearman + value: 65.17573588512339 + task: + type: STS + - dataset: + config: es-en + name: MTEB STS22.v2 (es-en) + revision: d31f33a128469b20e357535c39b82fb3c3f6f2bd + split: test + type: mteb/sts22-crosslingual-sts + metrics: + - type: cosine_pearson + value: 44.85365072983659 + - type: cosine_spearman + value: 48.95861510766423 + - type: euclidean_pearson + value: 44.12298881847811 + - type: euclidean_spearman + value: 47.73736578240132 + - type: main_score + value: 48.95861510766423 + - type: manhattan_pearson + value: 43.96292551331725 + - type: manhattan_spearman + value: 47.58103280136259 + - type: pearson + value: 44.853649047286616 + - type: spearman + value: 48.95861510766423 + task: + type: STS + - dataset: + config: de-fr + name: MTEB STS22.v2 (de-fr) + revision: d31f33a128469b20e357535c39b82fb3c3f6f2bd + split: test + type: mteb/sts22-crosslingual-sts + metrics: + - type: cosine_pearson + value: 22.860434001017367 + - type: cosine_spearman + value: 32.320901954587185 + - type: euclidean_pearson + value: 26.992993163046002 + - type: euclidean_spearman + value: 32.24246077908339 + - type: main_score + value: 32.320901954587185 + - type: manhattan_pearson + value: 26.552931446274357 + - type: manhattan_spearman + value: 33.0347166516718 + - type: pearson + value: 22.860425946102264 + - type: spearman + value: 32.320901954587185 + task: + type: STS + - dataset: + config: default + name: MTEB STSB (default) + revision: 0cde68302b3541bb8b3c340dc0644b0b745b3dc0 + split: test + type: C-MTEB/STSB + metrics: + - type: cosine_pearson + value: 79.54312520554313 + - type: cosine_spearman + value: 81.46931930954155 + - type: euclidean_pearson + value: 80.05389266606164 + - type: euclidean_spearman + value: 80.50696783023315 + - type: main_score + value: 81.46931930954155 + - type: manhattan_pearson + value: 80.04970274541007 + - type: manhattan_spearman + value: 80.50864360601699 + task: + type: STS + - dataset: + config: default + name: MTEB STSB (default) + revision: 0cde68302b3541bb8b3c340dc0644b0b745b3dc0 + split: validation + type: C-MTEB/STSB + metrics: + - type: cosine_pearson + value: 83.48618236826155 + - type: cosine_spearman + value: 85.3132611044822 + - type: euclidean_pearson + value: 82.93001035409826 + - type: euclidean_spearman + value: 83.10105109108397 + - type: main_score + value: 85.3132611044822 + - type: manhattan_pearson + value: 82.88533878745255 + - type: manhattan_spearman + value: 83.04890918186679 + task: + type: STS + - dataset: + config: default + name: MTEB T2Reranking (default) + revision: 76631901a18387f85eaa53e5450019b87ad58ef9 + split: dev + type: C-MTEB/T2Reranking + metrics: + - type: main_score + value: 68.38732925233572 + - type: map + value: 68.38732925233572 + - type: mrr + value: 79.11475293118828 + - type: nAUC_map_diff1 + value: -11.253110368505121 + - type: nAUC_map_max + value: 38.51256776553827 + - type: nAUC_map_std + value: -1.9662536573348381 + - type: nAUC_mrr_diff1 + value: -10.630308274175526 + - type: nAUC_mrr_max + value: 32.327491874763126 + - type: nAUC_mrr_std + value: -1.7352995694761828 + task: + type: Reranking + - dataset: + config: default + name: MTEB T2Retrieval (default) + revision: 8731a845f1bf500a4f111cf1070785c793d10e64 + split: dev + type: C-MTEB/T2Retrieval + metrics: + - type: main_score + value: 82.36099999999999 + - type: map_at_1 + value: 26.255 + - type: map_at_10 + value: 74.149 + - type: map_at_100 + value: 77.973 + - type: map_at_1000 + value: 78.044 + - type: map_at_20 + value: 77.015 + - type: map_at_3 + value: 51.812000000000005 + - type: map_at_5 + value: 63.827999999999996 + - type: mrr_at_1 + value: 87.51972645975802 + - type: mrr_at_10 + value: 90.78435813509476 + - type: mrr_at_100 + value: 90.89422964077919 + - type: mrr_at_1000 + value: 90.89770903884691 + - type: mrr_at_20 + value: 90.85502538725395 + - type: mrr_at_3 + value: 90.25001461219234 + - type: mrr_at_5 + value: 90.60859781401618 + - type: nauc_map_at_1000_diff1 + value: 16.021743379862592 + - type: nauc_map_at_1000_max + value: 40.378025084637784 + - type: nauc_map_at_1000_std + value: 18.32476707192497 + - type: nauc_map_at_100_diff1 + value: 16.02545540308598 + - type: nauc_map_at_100_max + value: 40.265093943279986 + - type: nauc_map_at_100_std + value: 18.292534126250082 + - type: nauc_map_at_10_diff1 + value: 19.617650276071352 + - type: nauc_map_at_10_max + value: 28.371544730138975 + - type: nauc_map_at_10_std + value: 5.6371012355859955 + - type: nauc_map_at_1_diff1 + value: 47.33700373223127 + - type: nauc_map_at_1_max + value: -25.333610801205936 + - type: nauc_map_at_1_std + value: -29.566785599172384 + - type: nauc_map_at_20_diff1 + value: 16.614949908223352 + - type: nauc_map_at_20_max + value: 37.6796912032617 + - type: nauc_map_at_20_std + value: 15.427406349207578 + - type: nauc_map_at_3_diff1 + value: 35.44856017189094 + - type: nauc_map_at_3_max + value: -13.397520313065106 + - type: nauc_map_at_3_std + value: -25.733491121718828 + - type: nauc_map_at_5_diff1 + value: 29.102390100969412 + - type: nauc_map_at_5_max + value: 1.9850020715468955 + - type: nauc_map_at_5_std + value: -15.925386927929594 + - type: nauc_mrr_at_1000_diff1 + value: 44.97733258856923 + - type: nauc_mrr_at_1000_max + value: 70.12348371687067 + - type: nauc_mrr_at_1000_std + value: 36.125835138518404 + - type: nauc_mrr_at_100_diff1 + value: 44.978054094169536 + - type: nauc_mrr_at_100_max + value: 70.12938961836127 + - type: nauc_mrr_at_100_std + value: 36.13929863715235 + - type: nauc_mrr_at_10_diff1 + value: 44.95657675119858 + - type: nauc_mrr_at_10_max + value: 70.23157682911028 + - type: nauc_mrr_at_10_std + value: 36.186025142743325 + - type: nauc_mrr_at_1_diff1 + value: 45.53039470469623 + - type: nauc_mrr_at_1_max + value: 67.19215396968853 + - type: nauc_mrr_at_1_std + value: 31.73945035410656 + - type: nauc_mrr_at_20_diff1 + value: 44.99711921466835 + - type: nauc_mrr_at_20_max + value: 70.16747553191094 + - type: nauc_mrr_at_20_std + value: 36.19983013748518 + - type: nauc_mrr_at_3_diff1 + value: 44.92504582719811 + - type: nauc_mrr_at_3_max + value: 70.17206471786271 + - type: nauc_mrr_at_3_std + value: 35.86538182968299 + - type: nauc_mrr_at_5_diff1 + value: 44.88240202293795 + - type: nauc_mrr_at_5_max + value: 70.25570239429001 + - type: nauc_mrr_at_5_std + value: 36.12414563662779 + - type: nauc_ndcg_at_1000_diff1 + value: 20.085183872723004 + - type: nauc_ndcg_at_1000_max + value: 52.54091309895082 + - type: nauc_ndcg_at_1000_std + value: 29.182787889425697 + - type: nauc_ndcg_at_100_diff1 + value: 19.70087837544669 + - type: nauc_ndcg_at_100_max + value: 51.41291578231909 + - type: nauc_ndcg_at_100_std + value: 29.391058739202926 + - type: nauc_ndcg_at_10_diff1 + value: 19.61155631423253 + - type: nauc_ndcg_at_10_max + value: 41.69125197676835 + - type: nauc_ndcg_at_10_std + value: 18.485185354348886 + - type: nauc_ndcg_at_1_diff1 + value: 45.53039470469623 + - type: nauc_ndcg_at_1_max + value: 67.19215396968853 + - type: nauc_ndcg_at_1_std + value: 31.73945035410656 + - type: nauc_ndcg_at_20_diff1 + value: 19.884673845091573 + - type: nauc_ndcg_at_20_max + value: 45.77756113362076 + - type: nauc_ndcg_at_20_std + value: 23.65628062983877 + - type: nauc_ndcg_at_3_diff1 + value: 16.396492503030576 + - type: nauc_ndcg_at_3_max + value: 57.36632449538558 + - type: nauc_ndcg_at_3_std + value: 28.110468173360108 + - type: nauc_ndcg_at_5_diff1 + value: 15.990139806075096 + - type: nauc_ndcg_at_5_max + value: 49.87917678198262 + - type: nauc_ndcg_at_5_std + value: 23.77870723184885 + - type: nauc_precision_at_1000_diff1 + value: -29.64669603533363 + - type: nauc_precision_at_1000_max + value: 51.500364584604654 + - type: nauc_precision_at_1000_std + value: 50.16891173815387 + - type: nauc_precision_at_100_diff1 + value: -29.466626146744606 + - type: nauc_precision_at_100_max + value: 52.73740649971535 + - type: nauc_precision_at_100_std + value: 51.846137080440656 + - type: nauc_precision_at_10_diff1 + value: -27.896447661643325 + - type: nauc_precision_at_10_max + value: 55.722284162663684 + - type: nauc_precision_at_10_std + value: 47.727742997832934 + - type: nauc_precision_at_1_diff1 + value: 45.53039470469623 + - type: nauc_precision_at_1_max + value: 67.19215396968853 + - type: nauc_precision_at_1_std + value: 31.73945035410656 + - type: nauc_precision_at_20_diff1 + value: -28.744448426184704 + - type: nauc_precision_at_20_max + value: 54.491376214009925 + - type: nauc_precision_at_20_std + value: 50.901916245689236 + - type: nauc_precision_at_3_diff1 + value: -19.456295360120368 + - type: nauc_precision_at_3_max + value: 62.40625587462132 + - type: nauc_precision_at_3_std + value: 40.87948253365838 + - type: nauc_precision_at_5_diff1 + value: -25.9612067834916 + - type: nauc_precision_at_5_max + value: 59.139116299790494 + - type: nauc_precision_at_5_std + value: 44.29380766917937 + - type: nauc_recall_at_1000_diff1 + value: 5.392030054530998 + - type: nauc_recall_at_1000_max + value: 53.602628435557854 + - type: nauc_recall_at_1000_std + value: 64.0692631857673 + - type: nauc_recall_at_100_diff1 + value: 8.764716008636329 + - type: nauc_recall_at_100_max + value: 42.289617268904074 + - type: nauc_recall_at_100_std + value: 46.008944199137 + - type: nauc_recall_at_10_diff1 + value: 17.70428741242271 + - type: nauc_recall_at_10_max + value: 16.82055621929404 + - type: nauc_recall_at_10_std + value: 0.9443257188294529 + - type: nauc_recall_at_1_diff1 + value: 47.33700373223127 + - type: nauc_recall_at_1_max + value: -25.333610801205936 + - type: nauc_recall_at_1_std + value: -29.566785599172384 + - type: nauc_recall_at_20_diff1 + value: 13.37892799655899 + - type: nauc_recall_at_20_max + value: 30.092126871647647 + - type: nauc_recall_at_20_std + value: 20.023192649193746 + - type: nauc_recall_at_3_diff1 + value: 33.201540040096866 + - type: nauc_recall_at_3_max + value: -18.541554905715042 + - type: nauc_recall_at_3_std + value: -28.22641082741867 + - type: nauc_recall_at_5_diff1 + value: 27.11037328467852 + - type: nauc_recall_at_5_max + value: -7.787633276369825 + - type: nauc_recall_at_5_std + value: -21.13477331887689 + - type: ndcg_at_1 + value: 87.52 + - type: ndcg_at_10 + value: 82.36099999999999 + - type: ndcg_at_100 + value: 86.447 + - type: ndcg_at_1000 + value: 87.14 + - type: ndcg_at_20 + value: 84.358 + - type: ndcg_at_3 + value: 83.48400000000001 + - type: ndcg_at_5 + value: 82.15299999999999 + - type: precision_at_1 + value: 87.52 + - type: precision_at_10 + value: 41.223 + - type: precision_at_100 + value: 4.993 + - type: precision_at_1000 + value: 0.516 + - type: precision_at_20 + value: 23.044 + - type: precision_at_3 + value: 73.156 + - type: precision_at_5 + value: 61.45 + - type: recall_at_1 + value: 26.255 + - type: recall_at_10 + value: 81.81 + - type: recall_at_100 + value: 94.973 + - type: recall_at_1000 + value: 98.488 + - type: recall_at_20 + value: 88.306 + - type: recall_at_3 + value: 53.913999999999994 + - type: recall_at_5 + value: 67.95400000000001 + task: + type: Retrieval + - dataset: + config: default + name: MTEB TNews (default) + revision: 317f262bf1e6126357bbe89e875451e4b0938fe4 + split: validation + type: C-MTEB/TNews-classification + metrics: + - type: accuracy + value: 50.16399999999999 + - type: f1 + value: 48.68459760388684 + - type: f1_weighted + value: 50.19811019404876 + - type: main_score + value: 50.16399999999999 + task: + type: Classification + - dataset: + config: default + name: MTEB ThuNewsClusteringP2P.v2 (default) + revision: 5798586b105c0434e4f0fe5e767abe619442cf93 + split: test + type: C-MTEB/ThuNewsClusteringP2P + metrics: + - type: main_score + value: 77.21421760551837 + - type: v_measure + value: 77.21421760551837 + - type: v_measure_std + value: 2.0725968692052548 + task: + type: Clustering + - dataset: + config: default + name: MTEB ThuNewsClusteringS2S.v2 (default) + revision: 8a8b2caeda43f39e13c4bc5bea0f8a667896e10d + split: test + type: C-MTEB/ThuNewsClusteringS2S + metrics: + - type: main_score + value: 67.40189244236278 + - type: v_measure + value: 67.40189244236278 + - type: v_measure_std + value: 2.1667814876460776 + task: + type: Clustering + - dataset: + config: default + name: MTEB VideoRetrieval (default) + revision: 58c2597a5943a2ba48f4668c3b90d796283c5639 + split: dev + type: C-MTEB/VideoRetrieval + metrics: + - type: main_score + value: 74.18 + - type: map_at_1 + value: 58.3 + - type: map_at_10 + value: 69.202 + - type: map_at_100 + value: 69.606 + - type: map_at_1000 + value: 69.612 + - type: map_at_20 + value: 69.465 + - type: map_at_3 + value: 67.133 + - type: map_at_5 + value: 68.28800000000001 + - type: mrr_at_1 + value: 58.3 + - type: mrr_at_10 + value: 69.201626984127 + - type: mrr_at_100 + value: 69.60581565029914 + - type: mrr_at_1000 + value: 69.61192749722464 + - type: mrr_at_20 + value: 69.46514532809812 + - type: mrr_at_3 + value: 67.13333333333334 + - type: mrr_at_5 + value: 68.28833333333336 + - type: nauc_map_at_1000_diff1 + value: 59.34478083408511 + - type: nauc_map_at_1000_max + value: 13.389346875878058 + - type: nauc_map_at_1000_std + value: -36.85885637076865 + - type: nauc_map_at_100_diff1 + value: 59.34807111517757 + - type: nauc_map_at_100_max + value: 13.403390284190857 + - type: nauc_map_at_100_std + value: -36.84487623810107 + - type: nauc_map_at_10_diff1 + value: 59.27462774183162 + - type: nauc_map_at_10_max + value: 13.127942709744026 + - type: nauc_map_at_10_std + value: -37.37951203716279 + - type: nauc_map_at_1_diff1 + value: 62.06374419623817 + - type: nauc_map_at_1_max + value: 12.216836842404124 + - type: nauc_map_at_1_std + value: -32.376437563970576 + - type: nauc_map_at_20_diff1 + value: 59.358042547082235 + - type: nauc_map_at_20_max + value: 13.425901648847269 + - type: nauc_map_at_20_std + value: -36.950481605718885 + - type: nauc_map_at_3_diff1 + value: 58.504179878593376 + - type: nauc_map_at_3_max + value: 13.717981132556927 + - type: nauc_map_at_3_std + value: -37.61571792082351 + - type: nauc_map_at_5_diff1 + value: 58.850653009668854 + - type: nauc_map_at_5_max + value: 12.895882710197466 + - type: nauc_map_at_5_std + value: -38.15921238754432 + - type: nauc_mrr_at_1000_diff1 + value: 59.34478083408511 + - type: nauc_mrr_at_1000_max + value: 13.389346875878058 + - type: nauc_mrr_at_1000_std + value: -36.85885637076865 + - type: nauc_mrr_at_100_diff1 + value: 59.34807111517757 + - type: nauc_mrr_at_100_max + value: 13.403390284190857 + - type: nauc_mrr_at_100_std + value: -36.84487623810107 + - type: nauc_mrr_at_10_diff1 + value: 59.27462774183162 + - type: nauc_mrr_at_10_max + value: 13.127942709744026 + - type: nauc_mrr_at_10_std + value: -37.37951203716279 + - type: nauc_mrr_at_1_diff1 + value: 62.06374419623817 + - type: nauc_mrr_at_1_max + value: 12.216836842404124 + - type: nauc_mrr_at_1_std + value: -32.376437563970576 + - type: nauc_mrr_at_20_diff1 + value: 59.358042547082235 + - type: nauc_mrr_at_20_max + value: 13.425901648847269 + - type: nauc_mrr_at_20_std + value: -36.950481605718885 + - type: nauc_mrr_at_3_diff1 + value: 58.504179878593376 + - type: nauc_mrr_at_3_max + value: 13.717981132556927 + - type: nauc_mrr_at_3_std + value: -37.61571792082351 + - type: nauc_mrr_at_5_diff1 + value: 58.850653009668854 + - type: nauc_mrr_at_5_max + value: 12.895882710197466 + - type: nauc_mrr_at_5_std + value: -38.15921238754432 + - type: nauc_ndcg_at_1000_diff1 + value: 59.13898931852901 + - type: nauc_ndcg_at_1000_max + value: 13.869168802179367 + - type: nauc_ndcg_at_1000_std + value: -36.57313702589791 + - type: nauc_ndcg_at_100_diff1 + value: 59.25286696443839 + - type: nauc_ndcg_at_100_max + value: 14.329031676206943 + - type: nauc_ndcg_at_100_std + value: -36.068844762404865 + - type: nauc_ndcg_at_10_diff1 + value: 58.952556848610506 + - type: nauc_ndcg_at_10_max + value: 13.058643821489968 + - type: nauc_ndcg_at_10_std + value: -38.637915329187656 + - type: nauc_ndcg_at_1_diff1 + value: 62.06374419623817 + - type: nauc_ndcg_at_1_max + value: 12.216836842404124 + - type: nauc_ndcg_at_1_std + value: -32.376437563970576 + - type: nauc_ndcg_at_20_diff1 + value: 59.265051921114534 + - type: nauc_ndcg_at_20_max + value: 14.424145412434584 + - type: nauc_ndcg_at_20_std + value: -36.67703874681748 + - type: nauc_ndcg_at_3_diff1 + value: 57.2145236748128 + - type: nauc_ndcg_at_3_max + value: 14.163756082395205 + - type: nauc_ndcg_at_3_std + value: -39.40414075321231 + - type: nauc_ndcg_at_5_diff1 + value: 57.809804336288416 + - type: nauc_ndcg_at_5_max + value: 12.527159951596808 + - type: nauc_ndcg_at_5_std + value: -40.552009922696094 + - type: nauc_precision_at_1000_diff1 + value: 56.125116713349286 + - type: nauc_precision_at_1000_max + value: 48.571428571425365 + - type: nauc_precision_at_1000_std + value: 59.68253968253704 + - type: nauc_precision_at_100_diff1 + value: 68.61646619432061 + - type: nauc_precision_at_100_max + value: 65.6533201515904 + - type: nauc_precision_at_100_std + value: 38.515406162464885 + - type: nauc_precision_at_10_diff1 + value: 58.1383709608438 + - type: nauc_precision_at_10_max + value: 12.102681699471365 + - type: nauc_precision_at_10_std + value: -45.96425614006378 + - type: nauc_precision_at_1_diff1 + value: 62.06374419623817 + - type: nauc_precision_at_1_max + value: 12.216836842404124 + - type: nauc_precision_at_1_std + value: -32.376437563970576 + - type: nauc_precision_at_20_diff1 + value: 61.35243841126161 + - type: nauc_precision_at_20_max + value: 29.105077928606924 + - type: nauc_precision_at_20_std + value: -24.708037060978505 + - type: nauc_precision_at_3_diff1 + value: 52.33167085174655 + - type: nauc_precision_at_3_max + value: 15.808744545343355 + - type: nauc_precision_at_3_std + value: -46.12751133667663 + - type: nauc_precision_at_5_diff1 + value: 53.2268623802632 + - type: nauc_precision_at_5_max + value: 10.356224013966232 + - type: nauc_precision_at_5_std + value: -51.58816300277973 + - type: nauc_recall_at_1000_diff1 + value: 56.125116713353194 + - type: nauc_recall_at_1000_max + value: 48.57142857143081 + - type: nauc_recall_at_1000_std + value: 59.682539682538724 + - type: nauc_recall_at_100_diff1 + value: 68.61646619432132 + - type: nauc_recall_at_100_max + value: 65.65332015158995 + - type: nauc_recall_at_100_std + value: 38.515406162465176 + - type: nauc_recall_at_10_diff1 + value: 58.13837096084369 + - type: nauc_recall_at_10_max + value: 12.10268169947128 + - type: nauc_recall_at_10_std + value: -45.964256140063505 + - type: nauc_recall_at_1_diff1 + value: 62.06374419623817 + - type: nauc_recall_at_1_max + value: 12.216836842404124 + - type: nauc_recall_at_1_std + value: -32.376437563970576 + - type: nauc_recall_at_20_diff1 + value: 61.35243841126176 + - type: nauc_recall_at_20_max + value: 29.10507792860751 + - type: nauc_recall_at_20_std + value: -24.708037060978192 + - type: nauc_recall_at_3_diff1 + value: 52.33167085174654 + - type: nauc_recall_at_3_max + value: 15.808744545343204 + - type: nauc_recall_at_3_std + value: -46.12751133667685 + - type: nauc_recall_at_5_diff1 + value: 53.2268623802632 + - type: nauc_recall_at_5_max + value: 10.356224013966395 + - type: nauc_recall_at_5_std + value: -51.58816300277942 + - type: ndcg_at_1 + value: 58.3 + - type: ndcg_at_10 + value: 74.18 + - type: ndcg_at_100 + value: 76.067 + - type: ndcg_at_1000 + value: 76.223 + - type: ndcg_at_20 + value: 75.14 + - type: ndcg_at_3 + value: 69.907 + - type: ndcg_at_5 + value: 71.956 + - type: precision_at_1 + value: 58.3 + - type: precision_at_10 + value: 8.97 + - type: precision_at_100 + value: 0.983 + - type: precision_at_1000 + value: 0.1 + - type: precision_at_20 + value: 4.675 + - type: precision_at_3 + value: 25.967000000000002 + - type: precision_at_5 + value: 16.56 + - type: recall_at_1 + value: 58.3 + - type: recall_at_10 + value: 89.7 + - type: recall_at_100 + value: 98.3 + - type: recall_at_1000 + value: 99.5 + - type: recall_at_20 + value: 93.5 + - type: recall_at_3 + value: 77.9 + - type: recall_at_5 + value: 82.8 + task: + type: Retrieval + - dataset: + config: default + name: MTEB Waimai (default) + revision: 339287def212450dcaa9df8c22bf93e9980c7023 + split: test + type: C-MTEB/waimai-classification + metrics: + - type: accuracy + value: 86.19 + - type: ap + value: 69.16016629543033 + - type: ap_weighted + value: 69.16016629543033 + - type: f1 + value: 84.65946646369608 + - type: f1_weighted + value: 86.35113290203732 + - type: main_score + value: 86.19 + task: + type: Classification +tags: +- mteb +--- +# ternary-weight-embedding + +基于xiaobu-embedding-v2[1]，在nli-zh[2]和t2ranking[3]数据集文本上微调得到的三元权重text embedding模型 + +## Reference +1. https://huggingface.co/lier007/xiaobu-embedding-v2 +2. https://huggingface.co/datasets/shibing624/nli-zh-all/tree/main/sampled_data +3. https://huggingface.co/datasets/sentence-transformers/t2ranking/tree/main/triplet \ No newline at end of file