{ "STSBenchmark": { "train": { "pearson": [ 0.73125161607048, 0.0 ], "spearman": [ 0.7150765613453124, 0.0 ], "nsamples": 5749 }, "dev": { "pearson": [ 0.7443000568532008, 6.40691279840157e-265 ], "spearman": [ 0.759890982808152, 1.6901750172343795e-282 ], "nsamples": 1500 }, "test": { "pearson": [ 0.7216210553290073, 3.539646733532031e-222 ], "spearman": [ 0.7304939228930519, 2.4834600870262812e-230 ], "nsamples": 1379 }, "all": { "pearson": { "all": 0.7322719424112719, "mean": 0.7323909094175627, "wmean": 0.7319808833295888 }, "spearman": { "all": 0.7295849518142924, "mean": 0.7351538223488387, "wmean": 0.7253317970625808 } } }, "SICKRelatedness": { "train": { "pearson": [ 0.7596859741218331, 0.0 ], "spearman": [ 0.7018231072465838, 0.0 ], "nsamples": 4500 }, "dev": { "pearson": [ 0.777647576791967, 1.9353259230163057e-102 ], "spearman": [ 0.7337127308506983, 1.2717377929862408e-85 ], "nsamples": 500 }, "test": { "pearson": [ 0.7566490759566877, 0.0 ], "spearman": [ 0.6936198984184592, 0.0 ], "nsamples": 4927 }, "all": { "pearson": { "all": 0.7589921709943909, "mean": 0.7646608756234959, "wmean": 0.7590833755598704 }, "spearman": { "all": 0.6992260214830822, "mean": 0.7097185788385804, "wmean": 0.6993578712141356 } } }, "MR": { "devacc": 81.81, "acc": 81.41, "ndev": 10662, "ntest": 10662 }, "CR": { "devacc": 87.87, "acc": 87.23, "ndev": 3775, "ntest": 3775 }, "SUBJ": { "devacc": 95.31, "acc": 95.0, "ndev": 10000, "ntest": 10000 }, "MPQA": { "devacc": 89.07, "acc": 89.01, "ndev": 10606, "ntest": 10606 }, "SST2": { "devacc": 84.4, "acc": 86.27, "ndev": 872, "ntest": 1821 }, "TREC": { "devacc": 80.43, "acc": 87.6, "ndev": 5452, "ntest": 500 }, "MRPC": { "devacc": 70.39, "acc": 67.07, "f1": 74.82, "ndev": 4076, "ntest": 1725 }, "STS12": { "MSRpar": { "pearson": [ 0.44512999156378674, 8.846234223022077e-38 ], "spearman": [ 0.4509834995822136, 7.501496598491026e-39 ], "nsamples": 750 }, "MSRvid": { "pearson": [ 0.8158138179672533, 4.485078256906633e-180 ], "spearman": [ 0.8238805942224864, 1.2395324040896922e-186 ], "nsamples": 750 }, "SMTeuroparl": { "pearson": [ 0.5025777230618875, 9.478558829456811e-31 ], "spearman": [ 0.6028335206096967, 9.378736061336642e-47 ], "nsamples": 459 }, "surprise.OnWN": { "pearson": [ 0.7075050481901065, 7.017744961203796e-115 ], "spearman": [ 0.6550259208484078, 4.101580378925782e-93 ], "nsamples": 750 }, "surprise.SMTnews": { "pearson": [ 0.6149709443544354, 7.125268059032375e-43 ], "spearman": [ 0.6179122177365513, 2.2209746905791215e-43 ], "nsamples": 399 }, "all": { "pearson": { "all": 0.5765393545095476, "mean": 0.617199505027494, "wmean": 0.6281831483184317 }, "spearman": { "all": 0.5812709943392725, "mean": 0.6301271505998711, "wmean": 0.6340621209223184 } } }, "STS13": { "FNWN": { "pearson": [ 0.5404400970895422, 1.0024601828505787e-15 ], "spearman": [ 0.5700597864027528, 1.1134900500394056e-17 ], "nsamples": 189 }, "headlines": { "pearson": [ 0.7696057345254828, 7.053396385656569e-148 ], "spearman": [ 0.7771493600170609, 1.3424669074971778e-152 ], "nsamples": 750 }, "OnWN": { "pearson": [ 0.770711585062639, 1.6777951910420866e-111 ], "spearman": [ 0.7682009295399416, 2.3759249362431374e-110 ], "nsamples": 561 }, "all": { "pearson": { "all": 0.7077401152320973, "mean": 0.6935858055592213, "wmean": 0.7411444523094507 }, "spearman": { "all": 0.7340343966035406, "mean": 0.705136691986585, "wmean": 0.7477093607432154 } } }, "STS14": { "deft-forum": { "pearson": [ 0.5050347882312676, 1.6792481128963397e-30 ], "spearman": [ 0.5006511549237668, 6.351186067873574e-30 ], "nsamples": 450 }, "deft-news": { "pearson": [ 0.7505543283782676, 1.480995559109095e-55 ], "spearman": [ 0.7172316918319939, 1.1485152196025994e-48 ], "nsamples": 300 }, "headlines": { "pearson": [ 0.7381665393126864, 5.392499213485816e-130 ], "spearman": [ 0.6987173657276973, 6.630592581022267e-111 ], "nsamples": 750 }, "images": { "pearson": [ 0.8184796221575482, 3.315905577355896e-182 ], "spearman": [ 0.7800399098279688, 1.8607695353393358e-154 ], "nsamples": 750 }, "OnWN": { "pearson": [ 0.8151737890413434, 1.4397424024788037e-179 ], "spearman": [ 0.8163228192853189, 1.7682187396229704e-180 ], "nsamples": 750 }, "tweet-news": { "pearson": [ 0.6920222067407532, 5.678306228875661e-108 ], "spearman": [ 0.6388729139230673, 2.906642337192504e-87 ], "nsamples": 750 }, "all": { "pearson": { "all": 0.6768936191011258, "mean": 0.719905212310311, "wmean": 0.7334169523084798 }, "spearman": { "all": 0.6371047443411619, "mean": 0.6919726425866354, "wmean": 0.704247275690222 } } }, "STS15": { "answers-forums": { "pearson": [ 0.6715484368711376, 1.6630650795173534e-50 ], "spearman": [ 0.6791768037991774, 4.803053709588311e-52 ], "nsamples": 375 }, "answers-students": { "pearson": [ 0.6752231441950838, 5.897966571800042e-101 ], "spearman": [ 0.6804903297262569, 4.1825635033521844e-103 ], "nsamples": 750 }, "belief": { "pearson": [ 0.7171089394500281, 1.8875842940680164e-60 ], "spearman": [ 0.7340838023228742, 1.1234078020923912e-64 ], "nsamples": 375 }, "headlines": { "pearson": [ 0.7957787507836831, 4.2857031841813906e-165 ], "spearman": [ 0.7959923201180171, 3.028810020400643e-165 ], "nsamples": 750 }, "images": { "pearson": [ 0.847852285173722, 2.736037311458429e-208 ], "spearman": [ 0.8592562196355571, 5.9692336633848975e-220 ], "nsamples": 750 }, "all": { "pearson": { "all": 0.7528553221562304, "mean": 0.7415023112947309, "wmean": 0.753295717078268 }, "spearman": { "all": 0.7672378740860561, "mean": 0.7497998951203765, "wmean": 0.7605922931352141 } } }, "STS16": { "answer-answer": { "pearson": [ 0.6932025545095016, 1.0441476948753323e-37 ], "spearman": [ 0.716984850240451, 2.2505666854089843e-41 ], "nsamples": 254 }, "headlines": { "pearson": [ 0.7647140202671684, 4.751336314867901e-49 ], "spearman": [ 0.7735811184492498, 7.591019148208835e-51 ], "nsamples": 249 }, "plagiarism": { "pearson": [ 0.8094434845489986, 1.2478658754392033e-54 ], "spearman": [ 0.8231197837843675, 5.9498794720031536e-58 ], "nsamples": 230 }, "postediting": { "pearson": [ 0.7946011230206756, 2.306149833128848e-54 ], "spearman": [ 0.8178716303963535, 5.115171930640644e-60 ], "nsamples": 244 }, "question-question": { "pearson": [ 0.7803307090745555, 4.4645254931736274e-44 ], "spearman": [ 0.7927716723925775, 2.1646221593031153e-46 ], "nsamples": 209 }, "all": { "pearson": { "all": 0.7636230650001257, "mean": 0.7684583782841801, "wmean": 0.7669738900099788 }, "spearman": { "all": 0.7833717936510003, "mean": 0.7848658110525999, "wmean": 0.7835610101788375 } } }, "eval_senteval-STS12": 0.5812709943392725, "eval_senteval-STS13": 0.7340343966035406, "eval_senteval-STS14": 0.6371047443411619, "eval_senteval-STS15": 0.7672378740860561, "eval_senteval-STS16": 0.7833717936510003, "eval_senteval-STSBenchmark": 0.7295849518142924, "eval_senteval-SICKRelatedness": 0.6992260214830822, "eval_senteval-avg_sts_7": 0.7045472537597723, "eval_senteval-MR": 81.81, "eval_senteval-CR": 87.87, "eval_senteval-SUBJ": 95.31, "eval_senteval-MPQA": 89.07, "eval_senteval-SST2": 84.4, "eval_senteval-TREC": 80.43, "eval_senteval-MRPC": 70.39, "eval_senteval-avg_transfer": 84.18285714285716 }