5-shot,FineWeb-1.5T,Ours-Base,Ours-Upsampling2,All-Upsampling1
time: 20 min,Llama-8x8B-seq8192,Llama-8x8B-seq8192,Llama-8x8B-seq8192,Llama-8x8B-seq8192
5k,,0.2579,0.2482,0.2456
10k,0.2594,0.2612,0.2628,0.2525
15k,,,0.2334,0.2503
20k,0.2495,0.2467,0.2449,0.254
25k,,0.2431,0.2571,0.2534
30k,,,0.2678,0.2557
35k,0.2426,0.2591,0.2562,0.2494
40k,0.2467,0.2485,0.2408,0.2686
45k,0.2418,0.2296,0.2712,0.2503
50k,0.2382,0.2441,0.2558,0.2322
55k,0.2408,0.2536,0.244,0.2747
60k,0.2718,0.2539,0.2339,0.2432
65k,0.2637,0.2423,0.2342,0.2478
70k,0.2534,0.2359,0.2673,0.2478
75k,0.2529,0.2372,0.2579,0.2478
80k,0.2504,0.2344,0.2535,0.2718
85k,0.2547,0.2496,0.2418,0.2465
90k,0.2595,0.2464,0.2359,0.2475
95k,0.2621,0.2469,0.2534,0.2424
100k,0.255,,0.2461,0.2497
105k,0.2659,,0.2729,0.2468
110k,0.2551,0.2629,0.2604,0.2522
115k,0.2624,0.2324,0.259,0.2584
120k,0.2626,0.2663,0.2629,0.2748
125k,0.2712,0.2733,0.2768,0.257
130k,0.2404,0.2635,0.2676,0.2812
135k,0.2641,,0.2735,0.2882
140k,0.2553,,0.2765,0.3019
145k,0.2492,,0.2708,0.309
150k,0.2595,,,0.3199
155k,0.2681,,0.2463,0.3116
160k,0.2605,,0.2821,0.324
165k,0.2725,,0.2816,0.3478
170k,0.2514,,0.2893,0.3423
175k,0.2535,,0.3317,0.3156
180k,0.2561,,0.2624,0.2893
185k,0.2523,,0.3026,0.3876
190k,0.2653,,,0.3131
195k,0.2681,,,0.3473
200k,0.2515,,,0.3257
205k,0.2619,,,0.3836
210k,0.2687,,,0.3063
215k,0.2653,,,0.3947
220k,0.2631,,,0.3621
225k,0.2737,,,0.4151
230k,0.2833,,,0.3825
235k,0.2703,,,0.3897
240k,0.2572,,,
245k,0.27,,,
250k,0.2639,,,
255k,0.268,,,
260k,0.2897,,,
265k,0.2815,,,
270k,0.2693,,,
275k,0.2789,,,
280k,0.3052,,,
285k,0.285,,,
290k,,,,
300k,,,,
305k,,,,
310k,,,,
315k,,,,
320k,,,,
325k,,,,
330k,,,,
335k,,,,