| {"vidore/arxivqa_test_subsampled": {"ndcg_at_1": 0.104, "ndcg_at_3": 0.15607, "ndcg_at_5": 0.17061, "ndcg_at_10": 0.19166, "ndcg_at_20": 0.21091, "ndcg_at_100": 0.26441, "ndcg_at_1000": 0.30779, "map_at_1": 0.104, "map_at_3": 0.143, "map_at_5": 0.1509, "map_at_10": 0.15942, "map_at_20": 0.16471, "map_at_100": 0.17157, "map_at_1000": 0.1733, "recall_at_1": 0.104, "recall_at_3": 0.194, "recall_at_5": 0.23, "recall_at_10": 0.296, "recall_at_20": 0.372, "recall_at_100": 0.668, "recall_at_1000": 1.0, "precision_at_1": 0.104, "precision_at_3": 0.06467, "precision_at_5": 0.046, "precision_at_10": 0.0296, "precision_at_20": 0.0186, "precision_at_100": 0.00668, "precision_at_1000": 0.001, "mrr_at_1": 0.104, "mrr_at_3": 0.14300000000000002, "mrr_at_5": 0.15090000000000006, "mrr_at_10": 0.15941984126984132, "mrr_at_20": 0.16471309947362595, "mrr_at_100": 0.17156615415816578, "mrr_at_1000": 0.17330341202710475, "naucs_at_1_max": 0.3455228874343908, "naucs_at_1_std": 0.0313722004976893, "naucs_at_1_diff1": 0.3992126889860104, "naucs_at_3_max": 0.17806211730769864, "naucs_at_3_std": 0.008084710832652355, "naucs_at_3_diff1": 0.23588235488704887, "naucs_at_5_max": 0.18027790912280198, "naucs_at_5_std": 0.009069416268818652, "naucs_at_5_diff1": 0.2412104757401508, "naucs_at_10_max": 0.07915438096377954, "naucs_at_10_std": -0.032888509286359134, "naucs_at_10_diff1": 0.18613519894678446, "naucs_at_20_max": 0.07282224692247101, "naucs_at_20_std": -0.0012628436943915612, "naucs_at_20_diff1": 0.20050774665894372, "naucs_at_100_max": 0.0651476013515794, "naucs_at_100_std": 0.10893903730331674, "naucs_at_100_diff1": 0.16289092270234953, "naucs_at_1000_max": 1.0, "naucs_at_1000_std": 1.0, "naucs_at_1000_diff1": 1.0}, "vidore/docvqa_test_subsampled": {"ndcg_at_1": 0.06652, "ndcg_at_3": 0.09165, "ndcg_at_5": 0.1071, "ndcg_at_10": 0.1226, "ndcg_at_20": 0.14041, "ndcg_at_100": 0.19011, "ndcg_at_1000": 0.24944, "map_at_1": 0.06652, "map_at_3": 0.08574, "map_at_5": 0.09427, "map_at_10": 0.10052, "map_at_20": 0.10533, "map_at_100": 0.11196, "map_at_1000": 0.11422, "recall_at_1": 0.06652, "recall_at_3": 0.10865, "recall_at_5": 0.14634, "recall_at_10": 0.19512, "recall_at_20": 0.26608, "recall_at_100": 0.5388, "recall_at_1000": 1.0, "precision_at_1": 0.06652, "precision_at_3": 0.03622, "precision_at_5": 0.02927, "precision_at_10": 0.01951, "precision_at_20": 0.0133, "precision_at_100": 0.00539, "precision_at_1000": 0.001, "mrr_at_1": 0.06651884700665188, "mrr_at_3": 0.08573540280857353, "mrr_at_5": 0.0942719881744272, "mrr_at_10": 0.10051824868898042, "mrr_at_20": 0.1053324007317713, "mrr_at_100": 0.11195972492820888, "mrr_at_1000": 0.11421878717895428, "naucs_at_1_max": 0.2968790744740374, "naucs_at_1_std": 0.20407932979988064, "naucs_at_1_diff1": 0.35619410819797476, "naucs_at_3_max": 0.13168858012185652, "naucs_at_3_std": 0.08313372825156064, "naucs_at_3_diff1": 0.20941908370986612, "naucs_at_5_max": 0.15747089449710808, "naucs_at_5_std": 0.15702872903692497, "naucs_at_5_diff1": 0.1830782839974118, "naucs_at_10_max": 0.15185227885290847, "naucs_at_10_std": 0.12627087674362714, "naucs_at_10_diff1": 0.15672004897619332, "naucs_at_20_max": 0.15486608299711213, "naucs_at_20_std": 0.1552990171450711, "naucs_at_20_diff1": 0.12093985230099419, "naucs_at_100_max": 0.2274325743383367, "naucs_at_100_std": 0.25548194762526616, "naucs_at_100_diff1": 0.11145011777954891, "naucs_at_1000_max": 1.0, "naucs_at_1000_std": 1.0, "naucs_at_1000_diff1": 1.0}, "vidore/infovqa_test_subsampled": {"ndcg_at_1": 0.22065, "ndcg_at_3": 0.28123, "ndcg_at_5": 0.30118, "ndcg_at_10": 0.3224, "ndcg_at_20": 0.33989, "ndcg_at_100": 0.37807, "ndcg_at_1000": 0.415, "map_at_1": 0.22065, "map_at_3": 0.26586, "map_at_5": 0.27689, "map_at_10": 0.28543, "map_at_20": 0.29027, "map_at_100": 0.29551, "map_at_1000": 0.29699, "recall_at_1": 0.22065, "recall_at_3": 0.32591, "recall_at_5": 0.37449, "recall_at_10": 0.4413, "recall_at_20": 0.51012, "recall_at_100": 0.7166, "recall_at_1000": 1.0, "precision_at_1": 0.22065, "precision_at_3": 0.10864, "precision_at_5": 0.0749, "precision_at_10": 0.04413, "precision_at_20": 0.02551, "precision_at_100": 0.00717, "precision_at_1000": 0.001, "mrr_at_1": 0.22064777327935223, "mrr_at_3": 0.2658569500674763, "mrr_at_5": 0.27688933873144395, "mrr_at_10": 0.2854331341173446, "mrr_at_20": 0.29026860693117384, "mrr_at_100": 0.2955104296079053, "mrr_at_1000": 0.29699279984591476, "naucs_at_1_max": 0.29007159959082884, "naucs_at_1_std": 0.002342360000887197, "naucs_at_1_diff1": 0.528063954032928, "naucs_at_3_max": 0.2632895122204285, "naucs_at_3_std": 0.07845473554398108, "naucs_at_3_diff1": 0.42839433396202464, "naucs_at_5_max": 0.2002187302309899, "naucs_at_5_std": 0.02673459887598484, "naucs_at_5_diff1": 0.408936214821079, "naucs_at_10_max": 0.2170202936459891, "naucs_at_10_std": 0.12400951776088111, "naucs_at_10_diff1": 0.3725293363805678, "naucs_at_20_max": 0.1645683660698112, "naucs_at_20_std": 0.11672017017680857, "naucs_at_20_diff1": 0.30351590830206027, "naucs_at_100_max": 0.05523284185881574, "naucs_at_100_std": 0.08599440115133186, "naucs_at_100_diff1": 0.17532007886878212, "naucs_at_1000_max": 1.0, "naucs_at_1000_std": 1.0, "naucs_at_1000_diff1": 1.0}, "vidore/tabfquad_test_subsampled": {"ndcg_at_1": 0.09643, "ndcg_at_3": 0.13644, "ndcg_at_5": 0.16303, "ndcg_at_10": 0.19044, "ndcg_at_20": 0.22141, "ndcg_at_100": 0.32553, "ndcg_at_1000": 0.32553, "map_at_1": 0.09643, "map_at_3": 0.12679, "map_at_5": 0.14161, "map_at_10": 0.15275, "map_at_20": 0.16093, "map_at_100": 0.17506, "map_at_1000": 0.17506, "recall_at_1": 0.09643, "recall_at_3": 0.16429, "recall_at_5": 0.22857, "recall_at_10": 0.31429, "recall_at_20": 0.43929, "recall_at_100": 1.0, "recall_at_1000": 1.0, "precision_at_1": 0.09643, "precision_at_3": 0.05476, "precision_at_5": 0.04571, "precision_at_10": 0.03143, "precision_at_20": 0.02196, "precision_at_100": 0.01, "precision_at_1000": 0.001, "mrr_at_1": 0.09642857142857143, "mrr_at_3": 0.12678571428571425, "mrr_at_5": 0.14160714285714282, "mrr_at_10": 0.1527508503401361, "mrr_at_20": 0.16092678853224532, "mrr_at_100": 0.17505586920245503, "mrr_at_1000": 0.17505586920245503, "naucs_at_1_max": 0.2916356120005167, "naucs_at_1_std": 0.10791123831102362, "naucs_at_1_diff1": 0.35776962903337994, "naucs_at_3_max": 0.171959236901897, "naucs_at_3_std": 0.1265914682513008, "naucs_at_3_diff1": 0.1504550893373308, "naucs_at_5_max": 0.1531831687494538, "naucs_at_5_std": 0.08789216114655254, "naucs_at_5_diff1": 0.07764026042121833, "naucs_at_10_max": 0.09738567102388196, "naucs_at_10_std": 0.08809943650093938, "naucs_at_10_diff1": -0.000579790700348686, "naucs_at_20_max": 0.1146492957024569, "naucs_at_20_std": 0.04177791445918426, "naucs_at_20_diff1": -0.004468550074320491, "naucs_at_100_max": 1.0, "naucs_at_100_std": 1.0, "naucs_at_100_diff1": 1.0, "naucs_at_1000_max": NaN, "naucs_at_1000_std": NaN, "naucs_at_1000_diff1": NaN}, "vidore/tatdqa_test": {"ndcg_at_1": 0.0164, "ndcg_at_3": 0.02342, "ndcg_at_5": 0.02716, "ndcg_at_10": 0.03445, "ndcg_at_20": 0.04779, "ndcg_at_100": 0.09992, "ndcg_at_1000": 0.1784, "map_at_1": 0.0164, "map_at_3": 0.02147, "map_at_5": 0.02353, "map_at_10": 0.02656, "map_at_20": 0.03021, "map_at_100": 0.03626, "map_at_1000": 0.03964, "recall_at_1": 0.0164, "recall_at_3": 0.02916, "recall_at_5": 0.03827, "recall_at_10": 0.06075, "recall_at_20": 0.11361, "recall_at_100": 0.41252, "recall_at_1000": 1.0, "precision_at_1": 0.0164, "precision_at_3": 0.00972, "precision_at_5": 0.00765, "precision_at_10": 0.00608, "precision_at_20": 0.00568, "precision_at_100": 0.00413, "precision_at_1000": 0.001, "mrr_at_1": 0.016403402187120292, "mrr_at_3": 0.021466180639935196, "mrr_at_5": 0.02353179424868368, "mrr_at_10": 0.02656078227159639, "mrr_at_20": 0.03020524948206848, "mrr_at_100": 0.03625292972104715, "mrr_at_1000": 0.03963899996428956, "naucs_at_1_max": 0.10715893998625696, "naucs_at_1_std": 0.13642739774834864, "naucs_at_1_diff1": 0.33174290667854495, "naucs_at_3_max": 0.08104871200452107, "naucs_at_3_std": 0.07802716059658023, "naucs_at_3_diff1": 0.2571873236779454, "naucs_at_5_max": 0.05155575988754007, "naucs_at_5_std": 0.05710975769459344, "naucs_at_5_diff1": 0.18456304089795544, "naucs_at_10_max": -0.005643291991770268, "naucs_at_10_std": 0.03182878276062131, "naucs_at_10_diff1": 0.11299561077175131, "naucs_at_20_max": -0.002393108499603467, "naucs_at_20_std": 0.010264003771035111, "naucs_at_20_diff1": 0.12401042544304355, "naucs_at_100_max": 0.0054277626389831355, "naucs_at_100_std": -0.019957379131073743, "naucs_at_100_diff1": 0.04529642272216635, "naucs_at_1000_max": NaN, "naucs_at_1000_std": NaN, "naucs_at_1000_diff1": NaN}, "vidore/shiftproject_test": {"ndcg_at_1": 0.0, "ndcg_at_3": 0.00631, "ndcg_at_5": 0.01062, "ndcg_at_10": 0.0171, "ndcg_at_20": 0.02445, "ndcg_at_100": 0.03843, "ndcg_at_1000": 0.13788, "map_at_1": 0.0, "map_at_3": 0.005, "map_at_5": 0.0075, "map_at_10": 0.01018, "map_at_20": 0.01209, "map_at_100": 0.01369, "map_at_1000": 0.01633, "recall_at_1": 0.0, "recall_at_3": 0.01, "recall_at_5": 0.02, "recall_at_10": 0.04, "recall_at_20": 0.07, "recall_at_100": 0.15, "recall_at_1000": 1.0, "precision_at_1": 0.0, "precision_at_3": 0.00333, "precision_at_5": 0.004, "precision_at_10": 0.004, "precision_at_20": 0.0035, "precision_at_100": 0.0015, "precision_at_1000": 0.001, "mrr_at_1": 0.0, "mrr_at_3": 0.005, "mrr_at_5": 0.0075, "mrr_at_10": 0.010178571428571427, "mrr_at_20": 0.01208766233766234, "mrr_at_100": 0.013687564536460119, "mrr_at_1000": 0.016328696481406103, "naucs_at_1_max": NaN, "naucs_at_1_std": NaN, "naucs_at_1_diff1": NaN, "naucs_at_3_max": -0.07056613898577946, "naucs_at_3_std": 0.3238529648510868, "naucs_at_3_diff1": 0.3238529648510868, "naucs_at_5_max": 0.013952240407834753, "naucs_at_5_std": 0.05902870941776234, "naucs_at_5_diff1": 0.6619264824255433, "naucs_at_10_max": -0.1881540112691173, "naucs_at_10_std": -0.1318084250067078, "naucs_at_10_diff1": 0.47719345317950107, "naucs_at_20_max": -0.212043389934455, "naucs_at_20_std": -0.10835984514546351, "naucs_at_20_diff1": 0.34616121737130623, "naucs_at_100_max": -0.08914423949198684, "naucs_at_100_std": -0.010563451264993568, "naucs_at_100_diff1": 0.05154722306219119, "naucs_at_1000_max": NaN, "naucs_at_1000_std": NaN, "naucs_at_1000_diff1": NaN}, "vidore/syntheticDocQA_artificial_intelligence_test": {"ndcg_at_1": 0.09, "ndcg_at_3": 0.11262, "ndcg_at_5": 0.12853, "ndcg_at_10": 0.14174, "ndcg_at_20": 0.15692, "ndcg_at_100": 0.1956, "ndcg_at_1000": 0.2576, "map_at_1": 0.09, "map_at_3": 0.10667, "map_at_5": 0.11517, "map_at_10": 0.12076, "map_at_20": 0.12493, "map_at_100": 0.12955, "map_at_1000": 0.13149, "recall_at_1": 0.09, "recall_at_3": 0.13, "recall_at_5": 0.17, "recall_at_10": 0.21, "recall_at_20": 0.27, "recall_at_100": 0.49, "recall_at_1000": 1.0, "precision_at_1": 0.09, "precision_at_3": 0.04333, "precision_at_5": 0.034, "precision_at_10": 0.021, "precision_at_20": 0.0135, "precision_at_100": 0.0049, "precision_at_1000": 0.001, "mrr_at_1": 0.09, "mrr_at_3": 0.10666666666666667, "mrr_at_5": 0.11516666666666668, "mrr_at_10": 0.12076190476190476, "mrr_at_20": 0.12492597680097679, "mrr_at_100": 0.12955099848359827, "mrr_at_1000": 0.1314941304077516, "naucs_at_1_max": 0.41558596428464945, "naucs_at_1_std": 0.22421965834898483, "naucs_at_1_diff1": 0.4170467757803417, "naucs_at_3_max": 0.36108437241163177, "naucs_at_3_std": 0.11365315305231583, "naucs_at_3_diff1": 0.45696313044256914, "naucs_at_5_max": 0.3349896404407853, "naucs_at_5_std": 0.032490511377179385, "naucs_at_5_diff1": 0.3495480298501989, "naucs_at_10_max": 0.22472959918919347, "naucs_at_10_std": -0.058466752181418044, "naucs_at_10_diff1": 0.3424390707397026, "naucs_at_20_max": 0.16154512778985333, "naucs_at_20_std": -0.17657588543664504, "naucs_at_20_diff1": 0.2285300259983802, "naucs_at_100_max": 0.1720287334191972, "naucs_at_100_std": -0.047747400847131666, "naucs_at_100_diff1": 0.112052368117058, "naucs_at_1000_max": NaN, "naucs_at_1000_std": NaN, "naucs_at_1000_diff1": NaN}, "vidore/syntheticDocQA_energy_test": {"ndcg_at_1": 0.09, "ndcg_at_3": 0.105, "ndcg_at_5": 0.10887, "ndcg_at_10": 0.12789, "ndcg_at_20": 0.14326, "ndcg_at_100": 0.19347, "ndcg_at_1000": 0.25255, "map_at_1": 0.09, "map_at_3": 0.1, "map_at_5": 0.102, "map_at_10": 0.10963, "map_at_20": 0.11397, "map_at_100": 0.12093, "map_at_1000": 0.12279, "recall_at_1": 0.09, "recall_at_3": 0.12, "recall_at_5": 0.13, "recall_at_10": 0.19, "recall_at_20": 0.25, "recall_at_100": 0.52, "recall_at_1000": 1.0, "precision_at_1": 0.09, "precision_at_3": 0.04, "precision_at_5": 0.026, "precision_at_10": 0.019, "precision_at_20": 0.0125, "precision_at_100": 0.0052, "precision_at_1000": 0.001, "mrr_at_1": 0.09, "mrr_at_3": 0.1, "mrr_at_5": 0.102, "mrr_at_10": 0.10963492063492064, "mrr_at_20": 0.11397087123402912, "mrr_at_100": 0.12092788952355515, "mrr_at_1000": 0.12278871629861259, "naucs_at_1_max": 0.3907521688578838, "naucs_at_1_std": -0.11111111111111105, "naucs_at_1_diff1": 0.611126017350863, "naucs_at_3_max": 0.24808075063974971, "naucs_at_3_std": -0.148540422708748, "naucs_at_3_diff1": 0.4600511799829398, "naucs_at_5_max": 0.19269772335519678, "naucs_at_5_std": -0.18201965480960808, "naucs_at_5_diff1": 0.48987038570884917, "naucs_at_10_max": 0.11068888403140906, "naucs_at_10_std": -0.22213441393573566, "naucs_at_10_diff1": 0.2639853378844179, "naucs_at_20_max": 0.036984185362265345, "naucs_at_20_std": -0.22074292019124697, "naucs_at_20_diff1": 0.17866862817212192, "naucs_at_100_max": 0.09329941781380936, "naucs_at_100_std": -0.12035475270689369, "naucs_at_100_diff1": 0.23539093530660016, "naucs_at_1000_max": NaN, "naucs_at_1000_std": NaN, "naucs_at_1000_diff1": NaN}, "vidore/syntheticDocQA_government_reports_test": {"ndcg_at_1": 0.07, "ndcg_at_3": 0.09762, "ndcg_at_5": 0.11353, "ndcg_at_10": 0.12332, "ndcg_at_20": 0.14116, "ndcg_at_100": 0.17628, "ndcg_at_1000": 0.24123, "map_at_1": 0.07, "map_at_3": 0.09, "map_at_5": 0.0985, "map_at_10": 0.1026, "map_at_20": 0.10758, "map_at_100": 0.1118, "map_at_1000": 0.1137, "recall_at_1": 0.07, "recall_at_3": 0.12, "recall_at_5": 0.16, "recall_at_10": 0.19, "recall_at_20": 0.26, "recall_at_100": 0.46, "recall_at_1000": 1.0, "precision_at_1": 0.07, "precision_at_3": 0.04, "precision_at_5": 0.032, "precision_at_10": 0.019, "precision_at_20": 0.013, "precision_at_100": 0.0046, "precision_at_1000": 0.001, "mrr_at_1": 0.07, "mrr_at_3": 0.09, "mrr_at_5": 0.09850000000000002, "mrr_at_10": 0.10259523809523807, "mrr_at_20": 0.10757875457875457, "mrr_at_100": 0.11179717837857363, "mrr_at_1000": 0.11370405831853495, "naucs_at_1_max": 0.3913526773736056, "naucs_at_1_std": 0.385028172793131, "naucs_at_1_diff1": 0.29866993752156057, "naucs_at_3_max": 0.2637901620699458, "naucs_at_3_std": 0.056037342431996796, "naucs_at_3_diff1": 0.18457966069566847, "naucs_at_5_max": 0.25026885850361014, "naucs_at_5_std": 0.11635811952680884, "naucs_at_5_diff1": 0.07126670763558139, "naucs_at_10_max": 0.24328686967195182, "naucs_at_10_std": 0.09920468111580019, "naucs_at_10_diff1": 0.0634405528559177, "naucs_at_20_max": 0.23374577202162394, "naucs_at_20_std": 0.15308895377410295, "naucs_at_20_diff1": 0.12922436471914642, "naucs_at_100_max": 0.22699797640009467, "naucs_at_100_std": 0.08420278047883087, "naucs_at_100_diff1": 0.18607920948201076, "naucs_at_1000_max": NaN, "naucs_at_1000_std": NaN, "naucs_at_1000_diff1": NaN}, "vidore/syntheticDocQA_healthcare_industry_test": {"ndcg_at_1": 0.13, "ndcg_at_3": 0.15262, "ndcg_at_5": 0.15693, "ndcg_at_10": 0.16931, "ndcg_at_20": 0.18145, "ndcg_at_100": 0.22302, "ndcg_at_1000": 0.28535, "map_at_1": 0.13, "map_at_3": 0.14667, "map_at_5": 0.14917, "map_at_10": 0.15396, "map_at_20": 0.15702, "map_at_100": 0.16229, "map_at_1000": 0.16443, "recall_at_1": 0.13, "recall_at_3": 0.17, "recall_at_5": 0.18, "recall_at_10": 0.22, "recall_at_20": 0.27, "recall_at_100": 0.5, "recall_at_1000": 1.0, "precision_at_1": 0.13, "precision_at_3": 0.05667, "precision_at_5": 0.036, "precision_at_10": 0.022, "precision_at_20": 0.0135, "precision_at_100": 0.005, "precision_at_1000": 0.001, "mrr_at_1": 0.13, "mrr_at_3": 0.14666666666666667, "mrr_at_5": 0.14916666666666667, "mrr_at_10": 0.1539563492063492, "mrr_at_20": 0.15701785714285715, "mrr_at_100": 0.1622867575475181, "mrr_at_1000": 0.16442812357794354, "naucs_at_1_max": 0.5167334512323432, "naucs_at_1_std": 0.025698999350809295, "naucs_at_1_diff1": 0.7677687985493946, "naucs_at_3_max": 0.4470745704908415, "naucs_at_3_std": 0.04103485579126886, "naucs_at_3_diff1": 0.5958488421129837, "naucs_at_5_max": 0.4489001157772864, "naucs_at_5_std": 0.08253517173630816, "naucs_at_5_diff1": 0.5429253061081288, "naucs_at_10_max": 0.3272523852441185, "naucs_at_10_std": 0.14177828883985452, "naucs_at_10_diff1": 0.3927028622929422, "naucs_at_20_max": 0.26201537172001294, "naucs_at_20_std": 0.11153731406895946, "naucs_at_20_diff1": 0.3159584594183749, "naucs_at_100_max": 0.2998001332445033, "naucs_at_100_std": 0.11499000666222502, "naucs_at_100_diff1": 0.2516322451698865, "naucs_at_1000_max": NaN, "naucs_at_1000_std": NaN, "naucs_at_1000_diff1": NaN}} |