| { | |
| "metadata": { | |
| "timestamp": "2025-04-07T13:06:44.552168", | |
| "vidore_benchmark_version": "5.0.1.dev6+g9e0da63" | |
| }, | |
| "metrics": { | |
| "./data_dir/eval_vidore/syntheticDocQA_artificial_intelligence_test": { | |
| "ndcg_at_1": 0.9, | |
| "ndcg_at_3": 0.94786, | |
| "ndcg_at_5": 0.95603, | |
| "ndcg_at_10": 0.95603, | |
| "ndcg_at_20": 0.95603, | |
| "ndcg_at_50": 0.95603, | |
| "ndcg_at_100": 0.95603, | |
| "map_at_1": 0.9, | |
| "map_at_3": 0.93667, | |
| "map_at_5": 0.94117, | |
| "map_at_10": 0.94117, | |
| "map_at_20": 0.94117, | |
| "map_at_50": 0.94117, | |
| "map_at_100": 0.94117, | |
| "recall_at_1": 0.9, | |
| "recall_at_3": 0.98, | |
| "recall_at_5": 1.0, | |
| "recall_at_10": 1.0, | |
| "recall_at_20": 1.0, | |
| "recall_at_50": 1.0, | |
| "recall_at_100": 1.0, | |
| "precision_at_1": 0.9, | |
| "precision_at_3": 0.32667, | |
| "precision_at_5": 0.2, | |
| "precision_at_10": 0.1, | |
| "precision_at_20": 0.05, | |
| "precision_at_50": 0.02, | |
| "precision_at_100": 0.01, | |
| "mrr_at_1": 0.9, | |
| "mrr_at_3": 0.9366666666666668, | |
| "mrr_at_5": 0.9411666666666666, | |
| "mrr_at_10": 0.9411666666666666, | |
| "mrr_at_20": 0.9411666666666666, | |
| "mrr_at_50": 0.9411666666666666, | |
| "mrr_at_100": 0.9411666666666666, | |
| "naucs_at_1_max": 0.2919701213818866, | |
| "naucs_at_1_std": -0.11045751633986804, | |
| "naucs_at_1_diff1": 0.8921568627450981, | |
| "naucs_at_3_max": 1.0, | |
| "naucs_at_3_std": 0.5401493930905577, | |
| "naucs_at_3_diff1": 0.7957516339869218, | |
| "naucs_at_5_max": 1.0, | |
| "naucs_at_5_std": 1.0, | |
| "naucs_at_5_diff1": 1.0, | |
| "naucs_at_10_max": 1.0, | |
| "naucs_at_10_std": 1.0, | |
| "naucs_at_10_diff1": 1.0, | |
| "naucs_at_20_max": 1.0, | |
| "naucs_at_20_std": 1.0, | |
| "naucs_at_20_diff1": 1.0, | |
| "naucs_at_50_max": null, | |
| "naucs_at_50_std": null, | |
| "naucs_at_50_diff1": null, | |
| "naucs_at_100_max": null, | |
| "naucs_at_100_std": null, | |
| "naucs_at_100_diff1": null | |
| }, | |
| "./data_dir/eval_vidore/syntheticDocQA_energy_test": { | |
| "ndcg_at_1": 0.83, | |
| "ndcg_at_3": 0.88547, | |
| "ndcg_at_5": 0.89365, | |
| "ndcg_at_10": 0.89969, | |
| "ndcg_at_20": 0.90488, | |
| "ndcg_at_50": 0.909, | |
| "ndcg_at_100": 0.909, | |
| "map_at_1": 0.83, | |
| "map_at_3": 0.87333, | |
| "map_at_5": 0.87783, | |
| "map_at_10": 0.88008, | |
| "map_at_20": 0.88157, | |
| "map_at_50": 0.88229, | |
| "map_at_100": 0.88229, | |
| "recall_at_1": 0.83, | |
| "recall_at_3": 0.92, | |
| "recall_at_5": 0.94, | |
| "recall_at_10": 0.96, | |
| "recall_at_20": 0.98, | |
| "recall_at_50": 1.0, | |
| "recall_at_100": 1.0, | |
| "precision_at_1": 0.83, | |
| "precision_at_3": 0.30667, | |
| "precision_at_5": 0.188, | |
| "precision_at_10": 0.096, | |
| "precision_at_20": 0.049, | |
| "precision_at_50": 0.02, | |
| "precision_at_100": 0.01, | |
| "mrr_at_1": 0.83, | |
| "mrr_at_3": 0.875, | |
| "mrr_at_5": 0.88, | |
| "mrr_at_10": 0.88225, | |
| "mrr_at_20": 0.8837335164835164, | |
| "mrr_at_50": 0.8844540133779265, | |
| "mrr_at_100": 0.8844540133779265, | |
| "naucs_at_1_max": 0.4281941138909857, | |
| "naucs_at_1_std": -0.6699077030243235, | |
| "naucs_at_1_diff1": 0.9181517385499506, | |
| "naucs_at_3_max": 0.510504201680672, | |
| "naucs_at_3_std": -0.565943043884219, | |
| "naucs_at_3_diff1": 0.932598039215688, | |
| "naucs_at_5_max": 0.6495487083722373, | |
| "naucs_at_5_std": -0.37068160597572447, | |
| "naucs_at_5_diff1": 0.9564270152505476, | |
| "naucs_at_10_max": 0.5764472455648917, | |
| "naucs_at_10_std": -0.6027077497665663, | |
| "naucs_at_10_diff1": 0.9346405228758139, | |
| "naucs_at_20_max": 0.15289449112978798, | |
| "naucs_at_20_std": -1.445845004668519, | |
| "naucs_at_20_diff1": 0.9346405228758136, | |
| "naucs_at_50_max": null, | |
| "naucs_at_50_std": null, | |
| "naucs_at_50_diff1": null, | |
| "naucs_at_100_max": null, | |
| "naucs_at_100_std": null, | |
| "naucs_at_100_diff1": null | |
| }, | |
| "./data_dir/eval_vidore/tatdqa_test": { | |
| "ndcg_at_1": 0.53645, | |
| "ndcg_at_3": 0.6649, | |
| "ndcg_at_5": 0.69656, | |
| "ndcg_at_10": 0.72095, | |
| "ndcg_at_20": 0.73253, | |
| "ndcg_at_50": 0.73934, | |
| "ndcg_at_100": 0.74094, | |
| "map_at_1": 0.53645, | |
| "map_at_3": 0.63366, | |
| "map_at_5": 0.65131, | |
| "map_at_10": 0.66162, | |
| "map_at_20": 0.66491, | |
| "map_at_50": 0.66608, | |
| "map_at_100": 0.66622, | |
| "recall_at_1": 0.53645, | |
| "recall_at_3": 0.75516, | |
| "recall_at_5": 0.83171, | |
| "recall_at_10": 0.90583, | |
| "recall_at_20": 0.95079, | |
| "recall_at_50": 0.9842, | |
| "recall_at_100": 0.99392, | |
| "precision_at_1": 0.53645, | |
| "precision_at_3": 0.25172, | |
| "precision_at_5": 0.16634, | |
| "precision_at_10": 0.09058, | |
| "precision_at_20": 0.04754, | |
| "precision_at_50": 0.01968, | |
| "precision_at_100": 0.00994, | |
| "mrr_at_1": 0.5352369380315918, | |
| "mrr_at_3": 0.6331510733090321, | |
| "mrr_at_5": 0.6503746456055083, | |
| "mrr_at_10": 0.6608750892013346, | |
| "mrr_at_20": 0.6641810998655294, | |
| "mrr_at_50": 0.6652992427218853, | |
| "mrr_at_100": 0.6654468968145465, | |
| "naucs_at_1_max": 0.37583584198144765, | |
| "naucs_at_1_std": -0.21052650239689327, | |
| "naucs_at_1_diff1": 0.6527557283121106, | |
| "naucs_at_3_max": 0.48310279218105395, | |
| "naucs_at_3_std": -0.17172161973082617, | |
| "naucs_at_3_diff1": 0.5692455177556047, | |
| "naucs_at_5_max": 0.523277724900881, | |
| "naucs_at_5_std": -0.107036305579362, | |
| "naucs_at_5_diff1": 0.5361020176422691, | |
| "naucs_at_10_max": 0.6288561537478294, | |
| "naucs_at_10_std": 0.021546705702201473, | |
| "naucs_at_10_diff1": 0.5134827945107916, | |
| "naucs_at_20_max": 0.7441492098609671, | |
| "naucs_at_20_std": 0.27041988401083406, | |
| "naucs_at_20_diff1": 0.48750569426670487, | |
| "naucs_at_50_max": 0.8858777041752212, | |
| "naucs_at_50_std": 0.6361789684070575, | |
| "naucs_at_50_diff1": 0.45001197824202754, | |
| "naucs_at_100_max": 0.9329946469504776, | |
| "naucs_at_100_std": 0.8095136618572032, | |
| "naucs_at_100_diff1": 0.5208594283118784 | |
| }, | |
| "./data_dir/eval_vidore/shiftproject_test": { | |
| "ndcg_at_1": 0.62, | |
| "ndcg_at_3": 0.72702, | |
| "ndcg_at_5": 0.75972, | |
| "ndcg_at_10": 0.7826, | |
| "ndcg_at_20": 0.78789, | |
| "ndcg_at_50": 0.79402, | |
| "ndcg_at_100": 0.79402, | |
| "map_at_1": 0.62, | |
| "map_at_3": 0.70167, | |
| "map_at_5": 0.71967, | |
| "map_at_10": 0.72923, | |
| "map_at_20": 0.73081, | |
| "map_at_50": 0.73186, | |
| "map_at_100": 0.73186, | |
| "recall_at_1": 0.62, | |
| "recall_at_3": 0.8, | |
| "recall_at_5": 0.88, | |
| "recall_at_10": 0.95, | |
| "recall_at_20": 0.97, | |
| "recall_at_50": 1.0, | |
| "recall_at_100": 1.0, | |
| "precision_at_1": 0.62, | |
| "precision_at_3": 0.26667, | |
| "precision_at_5": 0.176, | |
| "precision_at_10": 0.095, | |
| "precision_at_20": 0.0485, | |
| "precision_at_50": 0.02, | |
| "precision_at_100": 0.01, | |
| "mrr_at_1": 0.63, | |
| "mrr_at_3": 0.7116666666666667, | |
| "mrr_at_5": 0.7291666666666667, | |
| "mrr_at_10": 0.7389563492063492, | |
| "mrr_at_20": 0.7396230158730158, | |
| "mrr_at_50": 0.740700271950272, | |
| "mrr_at_100": 0.740700271950272, | |
| "naucs_at_1_max": 0.3217241485401346, | |
| "naucs_at_1_std": -0.38008799187767306, | |
| "naucs_at_1_diff1": 0.6488785650555333, | |
| "naucs_at_3_max": 0.3389610389610386, | |
| "naucs_at_3_std": -0.13676323676323673, | |
| "naucs_at_3_diff1": 0.5323676323676315, | |
| "naucs_at_5_max": 0.2474944320712715, | |
| "naucs_at_5_std": -0.1048759147311472, | |
| "naucs_at_5_diff1": 0.4449968183264394, | |
| "naucs_at_10_max": -0.04463118580765084, | |
| "naucs_at_10_std": -0.35536881419233807, | |
| "naucs_at_10_diff1": 0.30046685340803403, | |
| "naucs_at_20_max": 0.3862433862433828, | |
| "naucs_at_20_std": -0.4780578898225885, | |
| "naucs_at_20_diff1": 0.6150015561780299, | |
| "naucs_at_50_max": null, | |
| "naucs_at_50_std": null, | |
| "naucs_at_50_diff1": null, | |
| "naucs_at_100_max": null, | |
| "naucs_at_100_std": null, | |
| "naucs_at_100_diff1": null | |
| }, | |
| "./data_dir/eval_vidore/syntheticDocQA_healthcare_industry_test": { | |
| "ndcg_at_1": 0.9, | |
| "ndcg_at_3": 0.95917, | |
| "ndcg_at_5": 0.95917, | |
| "ndcg_at_10": 0.95917, | |
| "ndcg_at_20": 0.95917, | |
| "ndcg_at_50": 0.95917, | |
| "ndcg_at_100": 0.95917, | |
| "map_at_1": 0.9, | |
| "map_at_3": 0.945, | |
| "map_at_5": 0.945, | |
| "map_at_10": 0.945, | |
| "map_at_20": 0.945, | |
| "map_at_50": 0.945, | |
| "map_at_100": 0.945, | |
| "recall_at_1": 0.9, | |
| "recall_at_3": 1.0, | |
| "recall_at_5": 1.0, | |
| "recall_at_10": 1.0, | |
| "recall_at_20": 1.0, | |
| "recall_at_50": 1.0, | |
| "recall_at_100": 1.0, | |
| "precision_at_1": 0.9, | |
| "precision_at_3": 0.33333, | |
| "precision_at_5": 0.2, | |
| "precision_at_10": 0.1, | |
| "precision_at_20": 0.05, | |
| "precision_at_50": 0.02, | |
| "precision_at_100": 0.01, | |
| "mrr_at_1": 0.9, | |
| "mrr_at_3": 0.9466666666666668, | |
| "mrr_at_5": 0.9466666666666668, | |
| "mrr_at_10": 0.9466666666666668, | |
| "mrr_at_20": 0.9466666666666668, | |
| "mrr_at_50": 0.9466666666666668, | |
| "mrr_at_100": 0.9466666666666668, | |
| "naucs_at_1_max": 0.41400560224089555, | |
| "naucs_at_1_std": -0.37707749766573445, | |
| "naucs_at_1_diff1": 0.8197945845004672, | |
| "naucs_at_3_max": 1.0, | |
| "naucs_at_3_std": 1.0, | |
| "naucs_at_3_diff1": 1.0, | |
| "naucs_at_5_max": 1.0, | |
| "naucs_at_5_std": 1.0, | |
| "naucs_at_5_diff1": 1.0, | |
| "naucs_at_10_max": 1.0, | |
| "naucs_at_10_std": 1.0, | |
| "naucs_at_10_diff1": 1.0, | |
| "naucs_at_20_max": 1.0, | |
| "naucs_at_20_std": 1.0, | |
| "naucs_at_20_diff1": 1.0, | |
| "naucs_at_50_max": null, | |
| "naucs_at_50_std": null, | |
| "naucs_at_50_diff1": null, | |
| "naucs_at_100_max": null, | |
| "naucs_at_100_std": null, | |
| "naucs_at_100_diff1": null | |
| }, | |
| "./data_dir/eval_vidore/syntheticDocQA_government_reports_test": { | |
| "ndcg_at_1": 0.84, | |
| "ndcg_at_3": 0.90917, | |
| "ndcg_at_5": 0.91734, | |
| "ndcg_at_10": 0.92383, | |
| "ndcg_at_20": 0.92383, | |
| "ndcg_at_50": 0.92383, | |
| "ndcg_at_100": 0.92383, | |
| "map_at_1": 0.84, | |
| "map_at_3": 0.89167, | |
| "map_at_5": 0.89617, | |
| "map_at_10": 0.89885, | |
| "map_at_20": 0.89885, | |
| "map_at_50": 0.89885, | |
| "map_at_100": 0.89885, | |
| "recall_at_1": 0.84, | |
| "recall_at_3": 0.96, | |
| "recall_at_5": 0.98, | |
| "recall_at_10": 1.0, | |
| "recall_at_20": 1.0, | |
| "recall_at_50": 1.0, | |
| "recall_at_100": 1.0, | |
| "precision_at_1": 0.84, | |
| "precision_at_3": 0.32, | |
| "precision_at_5": 0.196, | |
| "precision_at_10": 0.1, | |
| "precision_at_20": 0.05, | |
| "precision_at_50": 0.02, | |
| "precision_at_100": 0.01, | |
| "mrr_at_1": 0.84, | |
| "mrr_at_3": 0.895, | |
| "mrr_at_5": 0.897, | |
| "mrr_at_10": 0.8999166666666666, | |
| "mrr_at_20": 0.8999166666666666, | |
| "mrr_at_50": 0.8999166666666666, | |
| "mrr_at_100": 0.8999166666666666, | |
| "naucs_at_1_max": 0.4499570446735392, | |
| "naucs_at_1_std": -0.2825540009818354, | |
| "naucs_at_1_diff1": 0.9005891016200291, | |
| "naucs_at_3_max": 0.777077497665732, | |
| "naucs_at_3_std": -1.019374416433236, | |
| "naucs_at_3_diff1": 0.8231792717086835, | |
| "naucs_at_5_max": 0.7770774976657324, | |
| "naucs_at_5_std": -1.445845004668527, | |
| "naucs_at_5_diff1": 0.7117180205415541, | |
| "naucs_at_10_max": 1.0, | |
| "naucs_at_10_std": 1.0, | |
| "naucs_at_10_diff1": 1.0, | |
| "naucs_at_20_max": 1.0, | |
| "naucs_at_20_std": 1.0, | |
| "naucs_at_20_diff1": 1.0, | |
| "naucs_at_50_max": null, | |
| "naucs_at_50_std": null, | |
| "naucs_at_50_diff1": null, | |
| "naucs_at_100_max": null, | |
| "naucs_at_100_std": null, | |
| "naucs_at_100_diff1": null | |
| }, | |
| "./data_dir/eval_vidore/docvqa_test_subsampled": { | |
| "ndcg_at_1": 0.43902, | |
| "ndcg_at_3": 0.51628, | |
| "ndcg_at_5": 0.5494, | |
| "ndcg_at_10": 0.57156, | |
| "ndcg_at_20": 0.58989, | |
| "ndcg_at_50": 0.60436, | |
| "ndcg_at_100": 0.61126, | |
| "map_at_1": 0.43902, | |
| "map_at_3": 0.49704, | |
| "map_at_5": 0.51556, | |
| "map_at_10": 0.52466, | |
| "map_at_20": 0.5299, | |
| "map_at_50": 0.53221, | |
| "map_at_100": 0.53284, | |
| "recall_at_1": 0.43902, | |
| "recall_at_3": 0.57206, | |
| "recall_at_5": 0.65188, | |
| "recall_at_10": 0.72062, | |
| "recall_at_20": 0.79157, | |
| "recall_at_50": 0.86475, | |
| "recall_at_100": 0.90687, | |
| "precision_at_1": 0.43902, | |
| "precision_at_3": 0.19069, | |
| "precision_at_5": 0.13038, | |
| "precision_at_10": 0.07206, | |
| "precision_at_20": 0.03958, | |
| "precision_at_50": 0.01729, | |
| "precision_at_100": 0.00907, | |
| "mrr_at_1": 0.44124168514412415, | |
| "mrr_at_3": 0.49667405764966743, | |
| "mrr_at_5": 0.516740576496674, | |
| "mrr_at_10": 0.5254285010382571, | |
| "mrr_at_20": 0.5306318984396768, | |
| "mrr_at_50": 0.5329498110334698, | |
| "mrr_at_100": 0.5335789553671992, | |
| "naucs_at_1_max": 0.6393285163665859, | |
| "naucs_at_1_std": -0.09880708103870717, | |
| "naucs_at_1_diff1": 0.7461246750006423, | |
| "naucs_at_3_max": 0.6920529884413887, | |
| "naucs_at_3_std": -0.04398858453959152, | |
| "naucs_at_3_diff1": 0.6344590291599227, | |
| "naucs_at_5_max": 0.6666549043335187, | |
| "naucs_at_5_std": -0.0865393648944786, | |
| "naucs_at_5_diff1": 0.5734951941149218, | |
| "naucs_at_10_max": 0.676010942842476, | |
| "naucs_at_10_std": -0.01997923324099346, | |
| "naucs_at_10_diff1": 0.5276488513249722, | |
| "naucs_at_20_max": 0.6902135293866369, | |
| "naucs_at_20_std": 0.0005736118755368109, | |
| "naucs_at_20_diff1": 0.520763324089702, | |
| "naucs_at_50_max": 0.809712009909262, | |
| "naucs_at_50_std": 0.2647772773491593, | |
| "naucs_at_50_diff1": 0.605506427700935, | |
| "naucs_at_100_max": 0.8480803026871803, | |
| "naucs_at_100_std": 0.34052949798730764, | |
| "naucs_at_100_diff1": 0.5613196121068196 | |
| }, | |
| "./data_dir/eval_vidore/infovqa_test_subsampled": { | |
| "ndcg_at_1": 0.79352, | |
| "ndcg_at_3": 0.85394, | |
| "ndcg_at_5": 0.86822, | |
| "ndcg_at_10": 0.87733, | |
| "ndcg_at_20": 0.8804, | |
| "ndcg_at_50": 0.88243, | |
| "ndcg_at_100": 0.88377, | |
| "map_at_1": 0.79352, | |
| "map_at_3": 0.83974, | |
| "map_at_5": 0.84774, | |
| "map_at_10": 0.85147, | |
| "map_at_20": 0.85231, | |
| "map_at_50": 0.85264, | |
| "map_at_100": 0.85277, | |
| "recall_at_1": 0.79352, | |
| "recall_at_3": 0.89474, | |
| "recall_at_5": 0.92915, | |
| "recall_at_10": 0.95749, | |
| "recall_at_20": 0.96964, | |
| "recall_at_50": 0.97976, | |
| "recall_at_100": 0.98785, | |
| "precision_at_1": 0.79352, | |
| "precision_at_3": 0.29825, | |
| "precision_at_5": 0.18583, | |
| "precision_at_10": 0.09575, | |
| "precision_at_20": 0.04848, | |
| "precision_at_50": 0.0196, | |
| "precision_at_100": 0.00988, | |
| "mrr_at_1": 0.7955465587044535, | |
| "mrr_at_3": 0.8424426450742241, | |
| "mrr_at_5": 0.8493252361673415, | |
| "mrr_at_10": 0.8528524837735364, | |
| "mrr_at_20": 0.8538822353257338, | |
| "mrr_at_50": 0.8542210105461472, | |
| "mrr_at_100": 0.8543424815607966, | |
| "naucs_at_1_max": 0.3962601081011797, | |
| "naucs_at_1_std": -0.2933243060801231, | |
| "naucs_at_1_diff1": 0.8734072776909251, | |
| "naucs_at_3_max": 0.4464123243127931, | |
| "naucs_at_3_std": -0.29538023225170024, | |
| "naucs_at_3_diff1": 0.7986216770271082, | |
| "naucs_at_5_max": 0.5416702750875175, | |
| "naucs_at_5_std": -0.06292498235695163, | |
| "naucs_at_5_diff1": 0.7465945205297018, | |
| "naucs_at_10_max": 0.7821168538276996, | |
| "naucs_at_10_std": 0.3726058691221081, | |
| "naucs_at_10_diff1": 0.767352315750433, | |
| "naucs_at_20_max": 0.7718044313624357, | |
| "naucs_at_20_std": 0.3394190587630231, | |
| "naucs_at_20_diff1": 0.7102224184431164, | |
| "naucs_at_50_max": 0.9069257991918063, | |
| "naucs_at_50_std": 0.6657094096862151, | |
| "naucs_at_50_diff1": 0.6953082100394825, | |
| "naucs_at_100_max": 0.8884101500079317, | |
| "naucs_at_100_std": 0.6842824400749656, | |
| "naucs_at_100_diff1": 0.6345205180300497 | |
| }, | |
| "./data_dir/eval_vidore/arxivqa_test_subsampled": { | |
| "ndcg_at_1": 0.782, | |
| "ndcg_at_3": 0.83112, | |
| "ndcg_at_5": 0.84179, | |
| "ndcg_at_10": 0.85908, | |
| "ndcg_at_20": 0.86664, | |
| "ndcg_at_50": 0.87036, | |
| "ndcg_at_100": 0.87102, | |
| "map_at_1": 0.782, | |
| "map_at_3": 0.81967, | |
| "map_at_5": 0.82557, | |
| "map_at_10": 0.83261, | |
| "map_at_20": 0.83467, | |
| "map_at_50": 0.83533, | |
| "map_at_100": 0.83539, | |
| "recall_at_1": 0.782, | |
| "recall_at_3": 0.864, | |
| "recall_at_5": 0.89, | |
| "recall_at_10": 0.944, | |
| "recall_at_20": 0.974, | |
| "recall_at_50": 0.992, | |
| "recall_at_100": 0.996, | |
| "precision_at_1": 0.782, | |
| "precision_at_3": 0.288, | |
| "precision_at_5": 0.178, | |
| "precision_at_10": 0.0944, | |
| "precision_at_20": 0.0487, | |
| "precision_at_50": 0.01984, | |
| "precision_at_100": 0.00996, | |
| "mrr_at_1": 0.78, | |
| "mrr_at_3": 0.8186666666666667, | |
| "mrr_at_5": 0.8250666666666667, | |
| "mrr_at_10": 0.8316119047619047, | |
| "mrr_at_20": 0.8337184350736982, | |
| "mrr_at_50": 0.8344780727568765, | |
| "mrr_at_100": 0.8345425722408806, | |
| "naucs_at_1_max": 0.7047304586616104, | |
| "naucs_at_1_std": -0.2845373920597051, | |
| "naucs_at_1_diff1": 0.8899044007416296, | |
| "naucs_at_3_max": 0.7526739489105085, | |
| "naucs_at_3_std": -0.20816265574329929, | |
| "naucs_at_3_diff1": 0.8587642942481646, | |
| "naucs_at_5_max": 0.7448243278068887, | |
| "naucs_at_5_std": -0.20444119920969092, | |
| "naucs_at_5_diff1": 0.8403917189244926, | |
| "naucs_at_10_max": 0.7687741763372007, | |
| "naucs_at_10_std": -0.17371948779512258, | |
| "naucs_at_10_diff1": 0.8275977057489662, | |
| "naucs_at_20_max": 0.80683042447748, | |
| "naucs_at_20_std": -0.15934065934066605, | |
| "naucs_at_20_diff1": 0.8283415930474759, | |
| "naucs_at_50_max": 0.9305555555555584, | |
| "naucs_at_50_std": 0.23214285714284258, | |
| "naucs_at_50_diff1": 0.8231792717086877, | |
| "naucs_at_100_max": 1.0, | |
| "naucs_at_100_std": 0.8611111111111168, | |
| "naucs_at_100_diff1": 0.7770774976657331 | |
| }, | |
| "./data_dir/eval_vidore/tabfquad_test_subsampled": { | |
| "ndcg_at_1": 0.77143, | |
| "ndcg_at_3": 0.84422, | |
| "ndcg_at_5": 0.84883, | |
| "ndcg_at_10": 0.85655, | |
| "ndcg_at_20": 0.86634, | |
| "ndcg_at_50": 0.87064, | |
| "ndcg_at_100": 0.87183, | |
| "map_at_1": 0.77143, | |
| "map_at_3": 0.82619, | |
| "map_at_5": 0.82887, | |
| "map_at_10": 0.83185, | |
| "map_at_20": 0.83446, | |
| "map_at_50": 0.83517, | |
| "map_at_100": 0.83529, | |
| "recall_at_1": 0.77143, | |
| "recall_at_3": 0.89643, | |
| "recall_at_5": 0.90714, | |
| "recall_at_10": 0.93214, | |
| "recall_at_20": 0.97143, | |
| "recall_at_50": 0.99286, | |
| "recall_at_100": 1.0, | |
| "precision_at_1": 0.77143, | |
| "precision_at_3": 0.29881, | |
| "precision_at_5": 0.18143, | |
| "precision_at_10": 0.09321, | |
| "precision_at_20": 0.04857, | |
| "precision_at_50": 0.01986, | |
| "precision_at_100": 0.01, | |
| "mrr_at_1": 0.775, | |
| "mrr_at_3": 0.8273809523809523, | |
| "mrr_at_5": 0.8300595238095237, | |
| "mrr_at_10": 0.832954931972789, | |
| "mrr_at_20": 0.835542627890317, | |
| "mrr_at_50": 0.8362529551132558, | |
| "mrr_at_100": 0.8363674239777247, | |
| "naucs_at_1_max": 0.8231877771351453, | |
| "naucs_at_1_std": 0.41470663839084937, | |
| "naucs_at_1_diff1": 0.8718752008225689, | |
| "naucs_at_3_max": 0.8601183738154516, | |
| "naucs_at_3_std": 0.6783369449205982, | |
| "naucs_at_3_diff1": 0.816375044471037, | |
| "naucs_at_5_max": 0.8618293471234652, | |
| "naucs_at_5_std": 0.6871902607196722, | |
| "naucs_at_5_diff1": 0.8011384040795805, | |
| "naucs_at_10_max": 0.8662096417514373, | |
| "naucs_at_10_std": 0.711828591085555, | |
| "naucs_at_10_diff1": 0.788564548626468, | |
| "naucs_at_20_max": 0.827847805788984, | |
| "naucs_at_20_std": 0.6514355742296987, | |
| "naucs_at_20_diff1": 0.8584850606909447, | |
| "naucs_at_50_max": 0.9346405228758071, | |
| "naucs_at_50_std": 0.5613912231559166, | |
| "naucs_at_50_diff1": 0.9346405228758071, | |
| "naucs_at_100_max": 1.0, | |
| "naucs_at_100_std": 1.0, | |
| "naucs_at_100_diff1": 1.0 | |
| } | |
| } | |
| } |