{ "metadata": { "timestamp": "2025-04-07T13:06:44.552168", "vidore_benchmark_version": "5.0.1.dev6+g9e0da63" }, "metrics": { "./data_dir/eval_vidore/syntheticDocQA_artificial_intelligence_test": { "ndcg_at_1": 0.9, "ndcg_at_3": 0.94786, "ndcg_at_5": 0.95603, "ndcg_at_10": 0.95603, "ndcg_at_20": 0.95603, "ndcg_at_50": 0.95603, "ndcg_at_100": 0.95603, "map_at_1": 0.9, "map_at_3": 0.93667, "map_at_5": 0.94117, "map_at_10": 0.94117, "map_at_20": 0.94117, "map_at_50": 0.94117, "map_at_100": 0.94117, "recall_at_1": 0.9, "recall_at_3": 0.98, "recall_at_5": 1.0, "recall_at_10": 1.0, "recall_at_20": 1.0, "recall_at_50": 1.0, "recall_at_100": 1.0, "precision_at_1": 0.9, "precision_at_3": 0.32667, "precision_at_5": 0.2, "precision_at_10": 0.1, "precision_at_20": 0.05, "precision_at_50": 0.02, "precision_at_100": 0.01, "mrr_at_1": 0.9, "mrr_at_3": 0.9366666666666668, "mrr_at_5": 0.9411666666666666, "mrr_at_10": 0.9411666666666666, "mrr_at_20": 0.9411666666666666, "mrr_at_50": 0.9411666666666666, "mrr_at_100": 0.9411666666666666, "naucs_at_1_max": 0.2919701213818866, "naucs_at_1_std": -0.11045751633986804, "naucs_at_1_diff1": 0.8921568627450981, "naucs_at_3_max": 1.0, "naucs_at_3_std": 0.5401493930905577, "naucs_at_3_diff1": 0.7957516339869218, "naucs_at_5_max": 1.0, "naucs_at_5_std": 1.0, "naucs_at_5_diff1": 1.0, "naucs_at_10_max": 1.0, "naucs_at_10_std": 1.0, "naucs_at_10_diff1": 1.0, "naucs_at_20_max": 1.0, "naucs_at_20_std": 1.0, "naucs_at_20_diff1": 1.0, "naucs_at_50_max": null, "naucs_at_50_std": null, "naucs_at_50_diff1": null, "naucs_at_100_max": null, "naucs_at_100_std": null, "naucs_at_100_diff1": null }, "./data_dir/eval_vidore/syntheticDocQA_energy_test": { "ndcg_at_1": 0.83, "ndcg_at_3": 0.88547, "ndcg_at_5": 0.89365, "ndcg_at_10": 0.89969, "ndcg_at_20": 0.90488, "ndcg_at_50": 0.909, "ndcg_at_100": 0.909, "map_at_1": 0.83, "map_at_3": 0.87333, "map_at_5": 0.87783, "map_at_10": 0.88008, "map_at_20": 0.88157, "map_at_50": 0.88229, "map_at_100": 0.88229, "recall_at_1": 0.83, "recall_at_3": 0.92, "recall_at_5": 0.94, "recall_at_10": 0.96, "recall_at_20": 0.98, "recall_at_50": 1.0, "recall_at_100": 1.0, "precision_at_1": 0.83, "precision_at_3": 0.30667, "precision_at_5": 0.188, "precision_at_10": 0.096, "precision_at_20": 0.049, "precision_at_50": 0.02, "precision_at_100": 0.01, "mrr_at_1": 0.83, "mrr_at_3": 0.875, "mrr_at_5": 0.88, "mrr_at_10": 0.88225, "mrr_at_20": 0.8837335164835164, "mrr_at_50": 0.8844540133779265, "mrr_at_100": 0.8844540133779265, "naucs_at_1_max": 0.4281941138909857, "naucs_at_1_std": -0.6699077030243235, "naucs_at_1_diff1": 0.9181517385499506, "naucs_at_3_max": 0.510504201680672, "naucs_at_3_std": -0.565943043884219, "naucs_at_3_diff1": 0.932598039215688, "naucs_at_5_max": 0.6495487083722373, "naucs_at_5_std": -0.37068160597572447, "naucs_at_5_diff1": 0.9564270152505476, "naucs_at_10_max": 0.5764472455648917, "naucs_at_10_std": -0.6027077497665663, "naucs_at_10_diff1": 0.9346405228758139, "naucs_at_20_max": 0.15289449112978798, "naucs_at_20_std": -1.445845004668519, "naucs_at_20_diff1": 0.9346405228758136, "naucs_at_50_max": null, "naucs_at_50_std": null, "naucs_at_50_diff1": null, "naucs_at_100_max": null, "naucs_at_100_std": null, "naucs_at_100_diff1": null }, "./data_dir/eval_vidore/tatdqa_test": { "ndcg_at_1": 0.53645, "ndcg_at_3": 0.6649, "ndcg_at_5": 0.69656, "ndcg_at_10": 0.72095, "ndcg_at_20": 0.73253, "ndcg_at_50": 0.73934, "ndcg_at_100": 0.74094, "map_at_1": 0.53645, "map_at_3": 0.63366, "map_at_5": 0.65131, "map_at_10": 0.66162, "map_at_20": 0.66491, "map_at_50": 0.66608, "map_at_100": 0.66622, "recall_at_1": 0.53645, "recall_at_3": 0.75516, "recall_at_5": 0.83171, "recall_at_10": 0.90583, "recall_at_20": 0.95079, "recall_at_50": 0.9842, "recall_at_100": 0.99392, "precision_at_1": 0.53645, "precision_at_3": 0.25172, "precision_at_5": 0.16634, "precision_at_10": 0.09058, "precision_at_20": 0.04754, "precision_at_50": 0.01968, "precision_at_100": 0.00994, "mrr_at_1": 0.5352369380315918, "mrr_at_3": 0.6331510733090321, "mrr_at_5": 0.6503746456055083, "mrr_at_10": 0.6608750892013346, "mrr_at_20": 0.6641810998655294, "mrr_at_50": 0.6652992427218853, "mrr_at_100": 0.6654468968145465, "naucs_at_1_max": 0.37583584198144765, "naucs_at_1_std": -0.21052650239689327, "naucs_at_1_diff1": 0.6527557283121106, "naucs_at_3_max": 0.48310279218105395, "naucs_at_3_std": -0.17172161973082617, "naucs_at_3_diff1": 0.5692455177556047, "naucs_at_5_max": 0.523277724900881, "naucs_at_5_std": -0.107036305579362, "naucs_at_5_diff1": 0.5361020176422691, "naucs_at_10_max": 0.6288561537478294, "naucs_at_10_std": 0.021546705702201473, "naucs_at_10_diff1": 0.5134827945107916, "naucs_at_20_max": 0.7441492098609671, "naucs_at_20_std": 0.27041988401083406, "naucs_at_20_diff1": 0.48750569426670487, "naucs_at_50_max": 0.8858777041752212, "naucs_at_50_std": 0.6361789684070575, "naucs_at_50_diff1": 0.45001197824202754, "naucs_at_100_max": 0.9329946469504776, "naucs_at_100_std": 0.8095136618572032, "naucs_at_100_diff1": 0.5208594283118784 }, "./data_dir/eval_vidore/shiftproject_test": { "ndcg_at_1": 0.62, "ndcg_at_3": 0.72702, "ndcg_at_5": 0.75972, "ndcg_at_10": 0.7826, "ndcg_at_20": 0.78789, "ndcg_at_50": 0.79402, "ndcg_at_100": 0.79402, "map_at_1": 0.62, "map_at_3": 0.70167, "map_at_5": 0.71967, "map_at_10": 0.72923, "map_at_20": 0.73081, "map_at_50": 0.73186, "map_at_100": 0.73186, "recall_at_1": 0.62, "recall_at_3": 0.8, "recall_at_5": 0.88, "recall_at_10": 0.95, "recall_at_20": 0.97, "recall_at_50": 1.0, "recall_at_100": 1.0, "precision_at_1": 0.62, "precision_at_3": 0.26667, "precision_at_5": 0.176, "precision_at_10": 0.095, "precision_at_20": 0.0485, "precision_at_50": 0.02, "precision_at_100": 0.01, "mrr_at_1": 0.63, "mrr_at_3": 0.7116666666666667, "mrr_at_5": 0.7291666666666667, "mrr_at_10": 0.7389563492063492, "mrr_at_20": 0.7396230158730158, "mrr_at_50": 0.740700271950272, "mrr_at_100": 0.740700271950272, "naucs_at_1_max": 0.3217241485401346, "naucs_at_1_std": -0.38008799187767306, "naucs_at_1_diff1": 0.6488785650555333, "naucs_at_3_max": 0.3389610389610386, "naucs_at_3_std": -0.13676323676323673, "naucs_at_3_diff1": 0.5323676323676315, "naucs_at_5_max": 0.2474944320712715, "naucs_at_5_std": -0.1048759147311472, "naucs_at_5_diff1": 0.4449968183264394, "naucs_at_10_max": -0.04463118580765084, "naucs_at_10_std": -0.35536881419233807, "naucs_at_10_diff1": 0.30046685340803403, "naucs_at_20_max": 0.3862433862433828, "naucs_at_20_std": -0.4780578898225885, "naucs_at_20_diff1": 0.6150015561780299, "naucs_at_50_max": null, "naucs_at_50_std": null, "naucs_at_50_diff1": null, "naucs_at_100_max": null, "naucs_at_100_std": null, "naucs_at_100_diff1": null }, "./data_dir/eval_vidore/syntheticDocQA_healthcare_industry_test": { "ndcg_at_1": 0.9, "ndcg_at_3": 0.95917, "ndcg_at_5": 0.95917, "ndcg_at_10": 0.95917, "ndcg_at_20": 0.95917, "ndcg_at_50": 0.95917, "ndcg_at_100": 0.95917, "map_at_1": 0.9, "map_at_3": 0.945, "map_at_5": 0.945, "map_at_10": 0.945, "map_at_20": 0.945, "map_at_50": 0.945, "map_at_100": 0.945, "recall_at_1": 0.9, "recall_at_3": 1.0, "recall_at_5": 1.0, "recall_at_10": 1.0, "recall_at_20": 1.0, "recall_at_50": 1.0, "recall_at_100": 1.0, "precision_at_1": 0.9, "precision_at_3": 0.33333, "precision_at_5": 0.2, "precision_at_10": 0.1, "precision_at_20": 0.05, "precision_at_50": 0.02, "precision_at_100": 0.01, "mrr_at_1": 0.9, "mrr_at_3": 0.9466666666666668, "mrr_at_5": 0.9466666666666668, "mrr_at_10": 0.9466666666666668, "mrr_at_20": 0.9466666666666668, "mrr_at_50": 0.9466666666666668, "mrr_at_100": 0.9466666666666668, "naucs_at_1_max": 0.41400560224089555, "naucs_at_1_std": -0.37707749766573445, "naucs_at_1_diff1": 0.8197945845004672, "naucs_at_3_max": 1.0, "naucs_at_3_std": 1.0, "naucs_at_3_diff1": 1.0, "naucs_at_5_max": 1.0, "naucs_at_5_std": 1.0, "naucs_at_5_diff1": 1.0, "naucs_at_10_max": 1.0, "naucs_at_10_std": 1.0, "naucs_at_10_diff1": 1.0, "naucs_at_20_max": 1.0, "naucs_at_20_std": 1.0, "naucs_at_20_diff1": 1.0, "naucs_at_50_max": null, "naucs_at_50_std": null, "naucs_at_50_diff1": null, "naucs_at_100_max": null, "naucs_at_100_std": null, "naucs_at_100_diff1": null }, "./data_dir/eval_vidore/syntheticDocQA_government_reports_test": { "ndcg_at_1": 0.84, "ndcg_at_3": 0.90917, "ndcg_at_5": 0.91734, "ndcg_at_10": 0.92383, "ndcg_at_20": 0.92383, "ndcg_at_50": 0.92383, "ndcg_at_100": 0.92383, "map_at_1": 0.84, "map_at_3": 0.89167, "map_at_5": 0.89617, "map_at_10": 0.89885, "map_at_20": 0.89885, "map_at_50": 0.89885, "map_at_100": 0.89885, "recall_at_1": 0.84, "recall_at_3": 0.96, "recall_at_5": 0.98, "recall_at_10": 1.0, "recall_at_20": 1.0, "recall_at_50": 1.0, "recall_at_100": 1.0, "precision_at_1": 0.84, "precision_at_3": 0.32, "precision_at_5": 0.196, "precision_at_10": 0.1, "precision_at_20": 0.05, "precision_at_50": 0.02, "precision_at_100": 0.01, "mrr_at_1": 0.84, "mrr_at_3": 0.895, "mrr_at_5": 0.897, "mrr_at_10": 0.8999166666666666, "mrr_at_20": 0.8999166666666666, "mrr_at_50": 0.8999166666666666, "mrr_at_100": 0.8999166666666666, "naucs_at_1_max": 0.4499570446735392, "naucs_at_1_std": -0.2825540009818354, "naucs_at_1_diff1": 0.9005891016200291, "naucs_at_3_max": 0.777077497665732, "naucs_at_3_std": -1.019374416433236, "naucs_at_3_diff1": 0.8231792717086835, "naucs_at_5_max": 0.7770774976657324, "naucs_at_5_std": -1.445845004668527, "naucs_at_5_diff1": 0.7117180205415541, "naucs_at_10_max": 1.0, "naucs_at_10_std": 1.0, "naucs_at_10_diff1": 1.0, "naucs_at_20_max": 1.0, "naucs_at_20_std": 1.0, "naucs_at_20_diff1": 1.0, "naucs_at_50_max": null, "naucs_at_50_std": null, "naucs_at_50_diff1": null, "naucs_at_100_max": null, "naucs_at_100_std": null, "naucs_at_100_diff1": null }, "./data_dir/eval_vidore/docvqa_test_subsampled": { "ndcg_at_1": 0.43902, "ndcg_at_3": 0.51628, "ndcg_at_5": 0.5494, "ndcg_at_10": 0.57156, "ndcg_at_20": 0.58989, "ndcg_at_50": 0.60436, "ndcg_at_100": 0.61126, "map_at_1": 0.43902, "map_at_3": 0.49704, "map_at_5": 0.51556, "map_at_10": 0.52466, "map_at_20": 0.5299, "map_at_50": 0.53221, "map_at_100": 0.53284, "recall_at_1": 0.43902, "recall_at_3": 0.57206, "recall_at_5": 0.65188, "recall_at_10": 0.72062, "recall_at_20": 0.79157, "recall_at_50": 0.86475, "recall_at_100": 0.90687, "precision_at_1": 0.43902, "precision_at_3": 0.19069, "precision_at_5": 0.13038, "precision_at_10": 0.07206, "precision_at_20": 0.03958, "precision_at_50": 0.01729, "precision_at_100": 0.00907, "mrr_at_1": 0.44124168514412415, "mrr_at_3": 0.49667405764966743, "mrr_at_5": 0.516740576496674, "mrr_at_10": 0.5254285010382571, "mrr_at_20": 0.5306318984396768, "mrr_at_50": 0.5329498110334698, "mrr_at_100": 0.5335789553671992, "naucs_at_1_max": 0.6393285163665859, "naucs_at_1_std": -0.09880708103870717, "naucs_at_1_diff1": 0.7461246750006423, "naucs_at_3_max": 0.6920529884413887, "naucs_at_3_std": -0.04398858453959152, "naucs_at_3_diff1": 0.6344590291599227, "naucs_at_5_max": 0.6666549043335187, "naucs_at_5_std": -0.0865393648944786, "naucs_at_5_diff1": 0.5734951941149218, "naucs_at_10_max": 0.676010942842476, "naucs_at_10_std": -0.01997923324099346, "naucs_at_10_diff1": 0.5276488513249722, "naucs_at_20_max": 0.6902135293866369, "naucs_at_20_std": 0.0005736118755368109, "naucs_at_20_diff1": 0.520763324089702, "naucs_at_50_max": 0.809712009909262, "naucs_at_50_std": 0.2647772773491593, "naucs_at_50_diff1": 0.605506427700935, "naucs_at_100_max": 0.8480803026871803, "naucs_at_100_std": 0.34052949798730764, "naucs_at_100_diff1": 0.5613196121068196 }, "./data_dir/eval_vidore/infovqa_test_subsampled": { "ndcg_at_1": 0.79352, "ndcg_at_3": 0.85394, "ndcg_at_5": 0.86822, "ndcg_at_10": 0.87733, "ndcg_at_20": 0.8804, "ndcg_at_50": 0.88243, "ndcg_at_100": 0.88377, "map_at_1": 0.79352, "map_at_3": 0.83974, "map_at_5": 0.84774, "map_at_10": 0.85147, "map_at_20": 0.85231, "map_at_50": 0.85264, "map_at_100": 0.85277, "recall_at_1": 0.79352, "recall_at_3": 0.89474, "recall_at_5": 0.92915, "recall_at_10": 0.95749, "recall_at_20": 0.96964, "recall_at_50": 0.97976, "recall_at_100": 0.98785, "precision_at_1": 0.79352, "precision_at_3": 0.29825, "precision_at_5": 0.18583, "precision_at_10": 0.09575, "precision_at_20": 0.04848, "precision_at_50": 0.0196, "precision_at_100": 0.00988, "mrr_at_1": 0.7955465587044535, "mrr_at_3": 0.8424426450742241, "mrr_at_5": 0.8493252361673415, "mrr_at_10": 0.8528524837735364, "mrr_at_20": 0.8538822353257338, "mrr_at_50": 0.8542210105461472, "mrr_at_100": 0.8543424815607966, "naucs_at_1_max": 0.3962601081011797, "naucs_at_1_std": -0.2933243060801231, "naucs_at_1_diff1": 0.8734072776909251, "naucs_at_3_max": 0.4464123243127931, "naucs_at_3_std": -0.29538023225170024, "naucs_at_3_diff1": 0.7986216770271082, "naucs_at_5_max": 0.5416702750875175, "naucs_at_5_std": -0.06292498235695163, "naucs_at_5_diff1": 0.7465945205297018, "naucs_at_10_max": 0.7821168538276996, "naucs_at_10_std": 0.3726058691221081, "naucs_at_10_diff1": 0.767352315750433, "naucs_at_20_max": 0.7718044313624357, "naucs_at_20_std": 0.3394190587630231, "naucs_at_20_diff1": 0.7102224184431164, "naucs_at_50_max": 0.9069257991918063, "naucs_at_50_std": 0.6657094096862151, "naucs_at_50_diff1": 0.6953082100394825, "naucs_at_100_max": 0.8884101500079317, "naucs_at_100_std": 0.6842824400749656, "naucs_at_100_diff1": 0.6345205180300497 }, "./data_dir/eval_vidore/arxivqa_test_subsampled": { "ndcg_at_1": 0.782, "ndcg_at_3": 0.83112, "ndcg_at_5": 0.84179, "ndcg_at_10": 0.85908, "ndcg_at_20": 0.86664, "ndcg_at_50": 0.87036, "ndcg_at_100": 0.87102, "map_at_1": 0.782, "map_at_3": 0.81967, "map_at_5": 0.82557, "map_at_10": 0.83261, "map_at_20": 0.83467, "map_at_50": 0.83533, "map_at_100": 0.83539, "recall_at_1": 0.782, "recall_at_3": 0.864, "recall_at_5": 0.89, "recall_at_10": 0.944, "recall_at_20": 0.974, "recall_at_50": 0.992, "recall_at_100": 0.996, "precision_at_1": 0.782, "precision_at_3": 0.288, "precision_at_5": 0.178, "precision_at_10": 0.0944, "precision_at_20": 0.0487, "precision_at_50": 0.01984, "precision_at_100": 0.00996, "mrr_at_1": 0.78, "mrr_at_3": 0.8186666666666667, "mrr_at_5": 0.8250666666666667, "mrr_at_10": 0.8316119047619047, "mrr_at_20": 0.8337184350736982, "mrr_at_50": 0.8344780727568765, "mrr_at_100": 0.8345425722408806, "naucs_at_1_max": 0.7047304586616104, "naucs_at_1_std": -0.2845373920597051, "naucs_at_1_diff1": 0.8899044007416296, "naucs_at_3_max": 0.7526739489105085, "naucs_at_3_std": -0.20816265574329929, "naucs_at_3_diff1": 0.8587642942481646, "naucs_at_5_max": 0.7448243278068887, "naucs_at_5_std": -0.20444119920969092, "naucs_at_5_diff1": 0.8403917189244926, "naucs_at_10_max": 0.7687741763372007, "naucs_at_10_std": -0.17371948779512258, "naucs_at_10_diff1": 0.8275977057489662, "naucs_at_20_max": 0.80683042447748, "naucs_at_20_std": -0.15934065934066605, "naucs_at_20_diff1": 0.8283415930474759, "naucs_at_50_max": 0.9305555555555584, "naucs_at_50_std": 0.23214285714284258, "naucs_at_50_diff1": 0.8231792717086877, "naucs_at_100_max": 1.0, "naucs_at_100_std": 0.8611111111111168, "naucs_at_100_diff1": 0.7770774976657331 }, "./data_dir/eval_vidore/tabfquad_test_subsampled": { "ndcg_at_1": 0.77143, "ndcg_at_3": 0.84422, "ndcg_at_5": 0.84883, "ndcg_at_10": 0.85655, "ndcg_at_20": 0.86634, "ndcg_at_50": 0.87064, "ndcg_at_100": 0.87183, "map_at_1": 0.77143, "map_at_3": 0.82619, "map_at_5": 0.82887, "map_at_10": 0.83185, "map_at_20": 0.83446, "map_at_50": 0.83517, "map_at_100": 0.83529, "recall_at_1": 0.77143, "recall_at_3": 0.89643, "recall_at_5": 0.90714, "recall_at_10": 0.93214, "recall_at_20": 0.97143, "recall_at_50": 0.99286, "recall_at_100": 1.0, "precision_at_1": 0.77143, "precision_at_3": 0.29881, "precision_at_5": 0.18143, "precision_at_10": 0.09321, "precision_at_20": 0.04857, "precision_at_50": 0.01986, "precision_at_100": 0.01, "mrr_at_1": 0.775, "mrr_at_3": 0.8273809523809523, "mrr_at_5": 0.8300595238095237, "mrr_at_10": 0.832954931972789, "mrr_at_20": 0.835542627890317, "mrr_at_50": 0.8362529551132558, "mrr_at_100": 0.8363674239777247, "naucs_at_1_max": 0.8231877771351453, "naucs_at_1_std": 0.41470663839084937, "naucs_at_1_diff1": 0.8718752008225689, "naucs_at_3_max": 0.8601183738154516, "naucs_at_3_std": 0.6783369449205982, "naucs_at_3_diff1": 0.816375044471037, "naucs_at_5_max": 0.8618293471234652, "naucs_at_5_std": 0.6871902607196722, "naucs_at_5_diff1": 0.8011384040795805, "naucs_at_10_max": 0.8662096417514373, "naucs_at_10_std": 0.711828591085555, "naucs_at_10_diff1": 0.788564548626468, "naucs_at_20_max": 0.827847805788984, "naucs_at_20_std": 0.6514355742296987, "naucs_at_20_diff1": 0.8584850606909447, "naucs_at_50_max": 0.9346405228758071, "naucs_at_50_std": 0.5613912231559166, "naucs_at_50_diff1": 0.9346405228758071, "naucs_at_100_max": 1.0, "naucs_at_100_std": 1.0, "naucs_at_100_diff1": 1.0 } } }