{ "STSBenchmark": { "train": { "pearson": [ 0.7868313677903059, 0.0 ], "spearman": [ 0.7533061602723176, 0.0 ], "nsamples": 5749 }, "dev": { "pearson": [ 0.8010234738701788, 0.0 ], "spearman": [ 0.8010173901569266, 0.0 ], "nsamples": 1500 }, "test": { "pearson": [ 0.7766524771374954, 1.1662374252806932e-278 ], "spearman": [ 0.7638460069862814, 3.8220116833205965e-264 ], "nsamples": 1379 }, "all": { "pearson": { "all": 0.7882517084559206, "mean": 0.7881691062659933, "wmean": 0.7876718254757005 }, "spearman": { "all": 0.7668425138024548, "mean": 0.7727231858051752, "wmean": 0.7632854478761041 } } }, "SICKRelatedness": { "train": { "pearson": [ 0.7853064744370001, 0.0 ], "spearman": [ 0.7045889188784015, 0.0 ], "nsamples": 4500 }, "dev": { "pearson": [ 0.7952406911697713, 2.6811654216760936e-110 ], "spearman": [ 0.7309801454276812, 1.0951577254305346e-84 ], "nsamples": 500 }, "test": { "pearson": [ 0.7794843403240526, 0.0 ], "spearman": [ 0.6977349592492605, 0.0 ], "nsamples": 4927 }, "all": { "pearson": { "all": 0.7828650051750712, "mean": 0.7866771686436079, "wmean": 0.7829171779317007 }, "spearman": { "all": 0.7025121676824595, "mean": 0.7111013411851145, "wmean": 0.702516404944873 } } }, "MR": { "devacc": 81.09, "acc": 80.59, "ndev": 10662, "ntest": 10662 }, "CR": { "devacc": 87.01, "acc": 86.6, "ndev": 3775, "ntest": 3775 }, "SUBJ": { "devacc": 95.34, "acc": 95.02, "ndev": 10000, "ntest": 10000 }, "MPQA": { "devacc": 88.89, "acc": 88.99, "ndev": 10606, "ntest": 10606 }, "SST2": { "devacc": 84.06, "acc": 84.95, "ndev": 872, "ntest": 1821 }, "TREC": { "devacc": 82.7, "acc": 87.2, "ndev": 5452, "ntest": 500 }, "MRPC": { "devacc": 70.8, "acc": 68.81, "f1": 76.97, "ndev": 4076, "ntest": 1725 }, "STS12": { "MSRpar": { "pearson": [ 0.4506656429961876, 8.587667122234213e-39 ], "spearman": [ 0.4537783738578545, 2.270328303306969e-39 ], "nsamples": 750 }, "MSRvid": { "pearson": [ 0.8735316429807825, 4.363488536726283e-236 ], "spearman": [ 0.8745116790315722, 2.8887254873277706e-237 ], "nsamples": 750 }, "SMTeuroparl": { "pearson": [ 0.5276646768590786, 2.9080130640693665e-34 ], "spearman": [ 0.6225349369600102, 1.3236704498055218e-50 ], "nsamples": 459 }, "surprise.OnWN": { "pearson": [ 0.7340552729415586, 7.588168789192366e-128 ], "spearman": [ 0.6862387036955726, 1.6734444180335097e-105 ], "nsamples": 750 }, "surprise.SMTnews": { "pearson": [ 0.6554833920285361, 2.382715639542081e-50 ], "spearman": [ 0.6338572026855626, 3.2084519789805647e-46 ], "nsamples": 399 }, "all": { "pearson": { "all": 0.667994262204242, "mean": 0.6482801255612287, "wmean": 0.6587597745452379 }, "spearman": { "all": 0.6198695741812278, "mean": 0.6541841792461144, "wmean": 0.659443091175976 } } }, "STS13": { "FNWN": { "pearson": [ 0.570828218970704, 9.847987645161228e-18 ], "spearman": [ 0.6046232001507675, 3.1840503190651897e-20 ], "nsamples": 189 }, "headlines": { "pearson": [ 0.7995020825200164, 9.500980524179015e-168 ], "spearman": [ 0.8125460785135767, 1.649528631120915e-177 ], "nsamples": 750 }, "OnWN": { "pearson": [ 0.7844322816916485, 4.612357080156043e-118 ], "spearman": [ 0.7801367000550998, 5.871872377147612e-116 ], "nsamples": 561 }, "all": { "pearson": { "all": 0.7407620228952401, "mean": 0.718254194394123, "wmean": 0.7650530702029934 }, "spearman": { "all": 0.7458274512781141, "mean": 0.7324353262398148, "wmean": 0.7742266882963925 } } }, "STS14": { "deft-forum": { "pearson": [ 0.5631321216962222, 5.210192672663357e-39 ], "spearman": [ 0.5511772152789268, 4.031968923211026e-37 ], "nsamples": 450 }, "deft-news": { "pearson": [ 0.7692532638001923, 6.387272025432673e-60 ], "spearman": [ 0.7225821241233733, 1.0522226454927607e-49 ], "nsamples": 300 }, "headlines": { "pearson": [ 0.7685532605008147, 3.1092413652073723e-147 ], "spearman": [ 0.72597520728351, 9.721513948491652e-124 ], "nsamples": 750 }, "images": { "pearson": [ 0.8353848860067018, 1.3687751285353585e-196 ], "spearman": [ 0.8008852796417836, 9.492757773828998e-169 ], "nsamples": 750 }, "OnWN": { "pearson": [ 0.8314850837955547, 3.9440310563877586e-193 ], "spearman": [ 0.8316458764161619, 2.851406716141642e-193 ], "nsamples": 750 }, "tweet-news": { "pearson": [ 0.7778758524787571, 4.607989885863323e-153 ], "spearman": [ 0.6958906739489547, 1.1738361486418922e-109 ], "nsamples": 750 }, "all": { "pearson": { "all": 0.73992310716163, "mean": 0.7576140780463737, "wmean": 0.7717759322639277 }, "spearman": { "all": 0.6864467990669687, "mean": 0.7213593961154516, "wmean": 0.734827243221423 } } }, "STS15": { "answers-forums": { "pearson": [ 0.7391296029963142, 5.385805927099778e-66 ], "spearman": [ 0.7409312029744309, 1.789609179402688e-66 ], "nsamples": 375 }, "answers-students": { "pearson": [ 0.7026326622651042, 1.1706959560110963e-112 ], "spearman": [ 0.7107886172788801, 2.1014876744572312e-116 ], "nsamples": 750 }, "belief": { "pearson": [ 0.8006281964159404, 5.439878843301103e-85 ], "spearman": [ 0.8172868949869829, 2.543177483321865e-91 ], "nsamples": 375 }, "headlines": { "pearson": [ 0.8172091326355518, 3.472486751097505e-181 ], "spearman": [ 0.8158723780481846, 4.030205865269612e-180 ], "nsamples": 750 }, "images": { "pearson": [ 0.8784869743544939, 3.750940760878824e-242 ], "spearman": [ 0.8806887757856461, 6.211464354884257e-245 ], "nsamples": 750 }, "all": { "pearson": { "all": 0.7913051772787048, "mean": 0.787617313733481, "wmean": 0.7920519172403193 }, "spearman": { "all": 0.7986117512115241, "mean": 0.7931135738148248, "wmean": 0.7966147050233544 } } }, "STS16": { "answer-answer": { "pearson": [ 0.7177168935513755, 1.7118144003107402e-41 ], "spearman": [ 0.7132893030185777, 8.841119832138394e-41 ], "nsamples": 254 }, "headlines": { "pearson": [ 0.7910112149047425, 1.2553390356155912e-54 ], "spearman": [ 0.8059772680734066, 3.561704789876049e-58 ], "nsamples": 249 }, "plagiarism": { "pearson": [ 0.8146246368802861, 7.416724178775043e-56 ], "spearman": [ 0.8287733749128491, 2.0728091350119785e-59 ], "nsamples": 230 }, "postediting": { "pearson": [ 0.8358664674126319, 5.627683437146896e-65 ], "spearman": [ 0.8514410852739017, 8.855209165489321e-70 ], "nsamples": 244 }, "question-question": { "pearson": [ 0.7757563825820808, 2.9043591729136265e-43 ], "spearman": [ 0.7882765418102572, 1.5472955880308416e-45 ], "nsamples": 209 }, "all": { "pearson": { "all": 0.7727412422624585, "mean": 0.7869951190662233, "wmean": 0.7864334333593027 }, "spearman": { "all": 0.7823935956761426, "mean": 0.7975515146177985, "wmean": 0.7967817209039866 } } }, "eval_senteval-STS12": 0.6198695741812278, "eval_senteval-STS13": 0.7458274512781141, "eval_senteval-STS14": 0.6864467990669687, "eval_senteval-STS15": 0.7986117512115241, "eval_senteval-STS16": 0.7823935956761426, "eval_senteval-STSBenchmark": 0.7668425138024548, "eval_senteval-SICKRelatedness": 0.7025121676824595, "eval_senteval-avg_sts_7": 0.7289291218426988, "eval_senteval-MR": 81.09, "eval_senteval-CR": 87.01, "eval_senteval-SUBJ": 95.34, "eval_senteval-MPQA": 88.89, "eval_senteval-SST2": 84.06, "eval_senteval-TREC": 82.7, "eval_senteval-MRPC": 70.8, "eval_senteval-avg_transfer": 84.27 }