| { | |
| "STSBenchmark": { | |
| "train": { | |
| "pearson": [ | |
| 0.7681082386326152, | |
| 0.0 | |
| ], | |
| "spearman": [ | |
| 0.7398280920285996, | |
| 0.0 | |
| ], | |
| "nsamples": 5749 | |
| }, | |
| "dev": { | |
| "pearson": [ | |
| 0.7871260331908001, | |
| 1.11150867e-316 | |
| ], | |
| "spearman": [ | |
| 0.792474398503786, | |
| 0.0 | |
| ], | |
| "nsamples": 1500 | |
| }, | |
| "test": { | |
| "pearson": [ | |
| 0.7651683167764847, | |
| 1.338574321582192e-265 | |
| ], | |
| "spearman": [ | |
| 0.7613108217890912, | |
| 2.22088758431869e-261 | |
| ], | |
| "nsamples": 1379 | |
| }, | |
| "all": { | |
| "pearson": { | |
| "all": 0.7717571144826049, | |
| "mean": 0.7734675295333, | |
| "wmean": 0.7709446479508435 | |
| }, | |
| "spearman": { | |
| "all": 0.7568905857914078, | |
| "mean": 0.7645377707738256, | |
| "wmean": 0.7524143396007481 | |
| } | |
| } | |
| }, | |
| "SICKRelatedness": { | |
| "train": { | |
| "pearson": [ | |
| 0.7686547151919503, | |
| 0.0 | |
| ], | |
| "spearman": [ | |
| 0.686670691209585, | |
| 0.0 | |
| ], | |
| "nsamples": 4500 | |
| }, | |
| "dev": { | |
| "pearson": [ | |
| 0.7902400028051256, | |
| 5.478669559071821e-108 | |
| ], | |
| "spearman": [ | |
| 0.7241117459531325, | |
| 2.1871273476832277e-82 | |
| ], | |
| "nsamples": 500 | |
| }, | |
| "test": { | |
| "pearson": [ | |
| 0.7603205521880406, | |
| 0.0 | |
| ], | |
| "spearman": [ | |
| 0.677217475345889, | |
| 0.0 | |
| ], | |
| "nsamples": 4927 | |
| }, | |
| "all": { | |
| "pearson": { | |
| "all": 0.7655429200754945, | |
| "mean": 0.7730717567283723, | |
| "wmean": 0.7656054780292953 | |
| }, | |
| "spearman": { | |
| "all": 0.6838845953897259, | |
| "mean": 0.6959999708362021, | |
| "wmean": 0.6838646604662934 | |
| } | |
| } | |
| }, | |
| "MR": { | |
| "devacc": 80.71, | |
| "acc": 80.16, | |
| "ndev": 10662, | |
| "ntest": 10662 | |
| }, | |
| "CR": { | |
| "devacc": 87.05, | |
| "acc": 86.01, | |
| "ndev": 3775, | |
| "ntest": 3775 | |
| }, | |
| "SUBJ": { | |
| "devacc": 95.04, | |
| "acc": 94.71, | |
| "ndev": 10000, | |
| "ntest": 10000 | |
| }, | |
| "MPQA": { | |
| "devacc": 89.21, | |
| "acc": 89.07, | |
| "ndev": 10606, | |
| "ntest": 10606 | |
| }, | |
| "SST2": { | |
| "devacc": 84.06, | |
| "acc": 85.12, | |
| "ndev": 872, | |
| "ntest": 1821 | |
| }, | |
| "TREC": { | |
| "devacc": 80.61, | |
| "acc": 86.6, | |
| "ndev": 5452, | |
| "ntest": 500 | |
| }, | |
| "MRPC": { | |
| "devacc": 70.58, | |
| "acc": 65.51, | |
| "f1": 72.42, | |
| "ndev": 4076, | |
| "ntest": 1725 | |
| }, | |
| "STS12": { | |
| "MSRpar": { | |
| "pearson": [ | |
| 0.4590340421521655, | |
| 2.3274572346703796e-40 | |
| ], | |
| "spearman": [ | |
| 0.45385589375614366, | |
| 2.1959530521625813e-39 | |
| ], | |
| "nsamples": 750 | |
| }, | |
| "MSRvid": { | |
| "pearson": [ | |
| 0.8528310967725129, | |
| 2.9347357918307595e-213 | |
| ], | |
| "spearman": [ | |
| 0.854220777715675, | |
| 1.1158473252287983e-214 | |
| ], | |
| "nsamples": 750 | |
| }, | |
| "SMTeuroparl": { | |
| "pearson": [ | |
| 0.4943065992119092, | |
| 1.1819222066299774e-29 | |
| ], | |
| "spearman": [ | |
| 0.6119562407045409, | |
| 1.670765662223567e-48 | |
| ], | |
| "nsamples": 459 | |
| }, | |
| "surprise.OnWN": { | |
| "pearson": [ | |
| 0.7373027197906938, | |
| 1.5365117817869742e-129 | |
| ], | |
| "spearman": [ | |
| 0.6745662466490533, | |
| 1.0854552176152885e-100 | |
| ], | |
| "nsamples": 750 | |
| }, | |
| "surprise.SMTnews": { | |
| "pearson": [ | |
| 0.6071798135532868, | |
| 1.4735857118756226e-41 | |
| ], | |
| "spearman": [ | |
| 0.6088785137011317, | |
| 7.667348467856033e-42 | |
| ], | |
| "nsamples": 399 | |
| }, | |
| "all": { | |
| "pearson": { | |
| "all": 0.6623608523114471, | |
| "mean": 0.6301308542961137, | |
| "wmean": 0.645439951313564 | |
| }, | |
| "spearman": { | |
| "all": 0.6274520092064783, | |
| "mean": 0.6406955345053089, | |
| "wmean": 0.6469796106952348 | |
| } | |
| } | |
| }, | |
| "STS13": { | |
| "FNWN": { | |
| "pearson": [ | |
| 0.6104407298501328, | |
| 1.1065428255812799e-20 | |
| ], | |
| "spearman": [ | |
| 0.6416446744357271, | |
| 2.5814733335245812e-23 | |
| ], | |
| "nsamples": 189 | |
| }, | |
| "headlines": { | |
| "pearson": [ | |
| 0.7850124142146208, | |
| 1.0139977037812637e-157 | |
| ], | |
| "spearman": [ | |
| 0.793315510943691, | |
| 2.2793688891679397e-163 | |
| ], | |
| "nsamples": 750 | |
| }, | |
| "OnWN": { | |
| "pearson": [ | |
| 0.7766707271490412, | |
| 2.7086108094896696e-114 | |
| ], | |
| "spearman": [ | |
| 0.764681188348543, | |
| 9.231297843026176e-109 | |
| ], | |
| "nsamples": 561 | |
| }, | |
| "all": { | |
| "pearson": { | |
| "all": 0.741621137420334, | |
| "mean": 0.7240412904045982, | |
| "wmean": 0.7598965910221684 | |
| }, | |
| "spearman": { | |
| "all": 0.7537361964068123, | |
| "mean": 0.7332137912426537, | |
| "wmean": 0.7634957488931022 | |
| } | |
| } | |
| }, | |
| "STS14": { | |
| "deft-forum": { | |
| "pearson": [ | |
| 0.49892982440741956, | |
| 1.0652261555598016e-29 | |
| ], | |
| "spearman": [ | |
| 0.4865545802805047, | |
| 4.028109290574198e-28 | |
| ], | |
| "nsamples": 450 | |
| }, | |
| "deft-news": { | |
| "pearson": [ | |
| 0.7661538564791797, | |
| 3.605720461784356e-59 | |
| ], | |
| "spearman": [ | |
| 0.7240898336096998, | |
| 5.311081186778723e-50 | |
| ], | |
| "nsamples": 300 | |
| }, | |
| "headlines": { | |
| "pearson": [ | |
| 0.7510342724250237, | |
| 5.45925815985743e-137 | |
| ], | |
| "spearman": [ | |
| 0.7127027118216717, | |
| 2.657235544077791e-117 | |
| ], | |
| "nsamples": 750 | |
| }, | |
| "images": { | |
| "pearson": [ | |
| 0.8457064177556566, | |
| 3.3443785854353087e-206 | |
| ], | |
| "spearman": [ | |
| 0.8049272327834235, | |
| 1.0191671573943109e-171 | |
| ], | |
| "nsamples": 750 | |
| }, | |
| "OnWN": { | |
| "pearson": [ | |
| 0.8241300010910765, | |
| 7.676405298271845e-187 | |
| ], | |
| "spearman": [ | |
| 0.8223616193261, | |
| 2.2573313609528353e-185 | |
| ], | |
| "nsamples": 750 | |
| }, | |
| "tweet-news": { | |
| "pearson": [ | |
| 0.7181193562444603, | |
| 6.957548479833527e-120 | |
| ], | |
| "spearman": [ | |
| 0.6469747801580121, | |
| 3.7447154677630655e-90 | |
| ], | |
| "nsamples": 750 | |
| }, | |
| "all": { | |
| "pearson": { | |
| "all": 0.7207630295977413, | |
| "mean": 0.7340122880671359, | |
| "wmean": 0.7489618969504681 | |
| }, | |
| "spearman": { | |
| "all": 0.6776154802540348, | |
| "mean": 0.6996017929965687, | |
| "wmean": 0.713707005140278 | |
| } | |
| } | |
| }, | |
| "STS15": { | |
| "answers-forums": { | |
| "pearson": [ | |
| 0.7104221785245369, | |
| 7.180350548638676e-59 | |
| ], | |
| "spearman": [ | |
| 0.7157297042374495, | |
| 4.032601594392119e-60 | |
| ], | |
| "nsamples": 375 | |
| }, | |
| "answers-students": { | |
| "pearson": [ | |
| 0.6841750672458253, | |
| 1.2325511825986507e-104 | |
| ], | |
| "spearman": [ | |
| 0.6893747601179209, | |
| 7.795024410550969e-107 | |
| ], | |
| "nsamples": 750 | |
| }, | |
| "belief": { | |
| "pearson": [ | |
| 0.749669412292463, | |
| 7.484565874638222e-69 | |
| ], | |
| "spearman": [ | |
| 0.7689611847810851, | |
| 1.80218707214869e-74 | |
| ], | |
| "nsamples": 375 | |
| }, | |
| "headlines": { | |
| "pearson": [ | |
| 0.8103746195832492, | |
| 7.843957651748014e-176 | |
| ], | |
| "spearman": [ | |
| 0.8082136911038486, | |
| 3.484989273824295e-174 | |
| ], | |
| "nsamples": 750 | |
| }, | |
| "images": { | |
| "pearson": [ | |
| 0.8764187325324918, | |
| 1.372705232924295e-239 | |
| ], | |
| "spearman": [ | |
| 0.8823684545353031, | |
| 4.309886303233611e-247 | |
| ], | |
| "nsamples": 750 | |
| }, | |
| "all": { | |
| "pearson": { | |
| "all": 0.784979164135427, | |
| "mean": 0.7662120020357133, | |
| "wmean": 0.7752535536925166 | |
| }, | |
| "spearman": { | |
| "all": 0.7953840242507239, | |
| "mean": 0.7729295589551215, | |
| "wmean": 0.780575587566585 | |
| } | |
| } | |
| }, | |
| "STS16": { | |
| "answer-answer": { | |
| "pearson": [ | |
| 0.7062392803273457, | |
| 1.1334345573551127e-39 | |
| ], | |
| "spearman": [ | |
| 0.7160237031906911, | |
| 3.2192023033831215e-41 | |
| ], | |
| "nsamples": 254 | |
| }, | |
| "headlines": { | |
| "pearson": [ | |
| 0.7684075132132632, | |
| 8.674627222788605e-50 | |
| ], | |
| "spearman": [ | |
| 0.7792317750235211, | |
| 4.923195475618921e-52 | |
| ], | |
| "nsamples": 249 | |
| }, | |
| "plagiarism": { | |
| "pearson": [ | |
| 0.814329742038296, | |
| 8.73007637868767e-56 | |
| ], | |
| "spearman": [ | |
| 0.8315423069613309, | |
| 3.8270415984455366e-60 | |
| ], | |
| "nsamples": 230 | |
| }, | |
| "postediting": { | |
| "pearson": [ | |
| 0.805735873745958, | |
| 5.663717248324878e-57 | |
| ], | |
| "spearman": [ | |
| 0.8273031164960505, | |
| 1.5191782859949922e-62 | |
| ], | |
| "nsamples": 244 | |
| }, | |
| "question-question": { | |
| "pearson": [ | |
| 0.7827391505554354, | |
| 1.635608527236305e-44 | |
| ], | |
| "spearman": [ | |
| 0.7928031349595721, | |
| 2.1346612440747243e-46 | |
| ], | |
| "nsamples": 209 | |
| }, | |
| "all": { | |
| "pearson": { | |
| "all": 0.7742750560870418, | |
| "mean": 0.7754903119760597, | |
| "wmean": 0.7742041520422902 | |
| }, | |
| "spearman": { | |
| "all": 0.7904588398710303, | |
| "mean": 0.7893808073262332, | |
| "wmean": 0.7881208084519268 | |
| } | |
| } | |
| }, | |
| "eval_senteval-STS12": 0.6274520092064783, | |
| "eval_senteval-STS13": 0.7537361964068123, | |
| "eval_senteval-STS14": 0.6776154802540348, | |
| "eval_senteval-STS15": 0.7953840242507239, | |
| "eval_senteval-STS16": 0.7904588398710303, | |
| "eval_senteval-STSBenchmark": 0.7568905857914078, | |
| "eval_senteval-SICKRelatedness": 0.6838845953897259, | |
| "eval_senteval-avg_sts_7": 0.7264888187386019, | |
| "eval_senteval-MR": 80.71, | |
| "eval_senteval-CR": 87.05, | |
| "eval_senteval-SUBJ": 95.04, | |
| "eval_senteval-MPQA": 89.21, | |
| "eval_senteval-SST2": 84.06, | |
| "eval_senteval-TREC": 80.61, | |
| "eval_senteval-MRPC": 70.58, | |
| "eval_senteval-avg_transfer": 83.89428571428572 | |
| } | |