5-shot,Slim-Pajama 600B (bsz=4K x 1024),,,FineWeb-1.5T,Ours-Base,Ours-Upsampling1,Ours-Upsampling2,Ours-Code-Upsampling2,All-Upsampling1,All-Upsampling1,All-Upsampling1,All-Upsampling1,DCLM-Base
time: 5 min,Llama-8x8B-baseline,Llama-8x8B-seq8192,Llama-8x8B-mup,Llama-8x8B-seq8192,Llama-8x8B-seq8192,Llama-8x8B-seq8192,Llama-8x8B-seq8192,Llama-8x8B-seq8192,Llama-8x8B-seq8192,Llama-1x8B-seq8192,Llama_extend-1x8B-seq8192,Jais-1x8B-seq8192,Llama-1x8B-seq8192
5k,0.2335,0.2308,,0.2251,,0.2157,0.2221,0.2231,0.2211,0.2251,0.2191,0.2271,0.2238
10k,0.2489,0.2519,,0.2379,0.2211,0.2332,0.2415,0.2342,0.2399,0.2285,0.2342,0.2402,0.2224
15k,0.2626,0.2469,,0.2526,,0.2389,0.2322,0.2479,0.2580,0.2375,0.2271,0.2355,0.2375
20k,0.2737,0.2606,,0.2469,0.2399,,0.2419,0.2526,0.2663,0.2469,0.2499,0.2439,0.2322
25k,0.2700,0.2653,,0.2523,0.2395,0.2600,0.2526,0.2616,0.2559,0.2369,0.2476,0.2462,0.2355
30k,0.2687,0.2556,,0.2402,,0.2452,0.2533,0.2606,0.2503,0.2456,0.2452,0.2446,0.2372
35k,0.2765,0.2533,,0.2683,0.2596,0.2590,0.2509,0.2630,0.2737,0.2392,0.2405,0.2536,0.2402
40k,0.2667,0.2683,,0.2496,0.2496,0.2593,0.2529,0.2697,0.2663,0.2379,0.2486,0.2526,0.2422
45k,0.2750,0.2620,,0.2616,0.2586,0.2563,0.2503,0.2683,0.2673,0.2479,0.2496,0.2513,0.2472
50k,0.2861,0.2697,,0.2693,0.2553,0.2596,0.2553,0.2700,0.2771,0.2442,0.2425,0.2546,0.2395
55k,0.2848,0.2693,,0.2640,0.2630,0.2566,0.2479,0.2630,0.2757,0.2526,0.2506,0.2586,0.2509
60k,0.2945,0.2784,,0.2727,0.2596,0.2633,0.2590,0.2690,0.2714,0.2519,0.2563,0.2553,0.2479
65k,0.3008,0.2767,,0.2680,0.2623,0.2704,0.2610,0.2492,0.2727,0.2529,0.2559,0.2647,0.2462
70k,0.2891,0.2824,,0.2730,0.2596,0.2710,0.2700,0.2677,0.2807,0.2469,0.2459,0.2626,0.2576
75k,0.2982,0.2938,,0.2784,0.2647,0.2630,0.2697,0.2777,0.2620,0.2626,0.2499,0.2583,0.2549
80k,0.2948,0.2801,,0.2737,0.2727,0.2643,0.2553,0.2657,0.2704,0.2509,0.2590,0.2549,0.2563
85k,0.2992,0.2938,,0.2754,0.2620,0.2704,0.2677,0.2600,0.2771,0.2496,0.2385,0.2620,0.2529
90k,0.3002,0.2888,,0.2764,0.2714,0.2737,0.2573,0.2693,0.2918,0.2616,0.2492,0.2566,0.2516
95k,0.3025,0.2817,,0.2616,0.2690,0.2737,0.2523,0.2690,0.2791,0.2492,0.2576,0.2576,0.2549
100k,0.2951,0.2894,,0.2616,,0.2817,0.2660,0.2757,0.2861,0.2546,0.2479,0.2667,0.2559
105k,0.3052,0.2928,,0.2653,,0.2710,0.2707,0.2771,0.2868,0.2529,0.2482,0.2640,0.2633
110k,0.3052,0.2985,,0.2600,0.2764,0.2781,0.2600,0.2764,0.2824,0.2536,,0.2727,0.2606
115k,0.3025,0.2985,,0.2690,0.2791,0.2720,0.2704,0.2744,0.2918,0.2623,,0.2807,0.2496
120k,0.3042,0.2985,,0.2750,0.2647,0.2650,0.2814,0.2754,0.2955,0.2677,,0.2626,0.2586
125k,0.3149,0.3018,,0.2683,0.2707,0.2647,0.2757,0.2760,0.2804,0.2509,,0.2704,0.2496
130k,0.3179,0.2978,,0.2781,0.2747,0.2653,0.2760,0.2774,0.2767,0.2593,,,0.2513
135k,0.3226,0.2945,,0.2747,,0.2717,0.2673,0.2784,0.2884,0.2606,,,0.2533
140k,,0.3018,,0.2771,,0.2757,0.2794,0.2787,0.2821,0.2459,,,0.2596
145k,,,,0.2724,,0.2650,0.2720,0.2888,0.2801,0.2543,,,0.2633
150k,,,,0.2720,,0.2814,,0.2864,0.2901,0.2590,,,0.2543
155k,,,,,,0.2784,0.2720,0.2874,0.2938,0.2580,,,0.2566
160k,,,,0.2817,,0.2834,0.2653,0.2807,0.2814,0.2563,,,0.2549
165k,,,,0.2834,,0.2821,0.2804,,0.2955,0.2559,,,0.2536
170k,,,,0.2854,,0.2824,0.2804,,0.3119,0.2536,,,0.2626
175k,,,,0.2804,,0.2915,0.2750,,0.2988,0.2489,,,0.2657
180k,,,,0.2767,,0.2901,0.2958,,0.3099,0.2623,,,0.2643
185k,,,,0.2767,,0.2948,0.2804,,0.3055,0.2570,,,0.2643
190k,,,,0.2787,,0.2925,,,0.3065,0.2573,,,0.2760
195k,,,,0.2858,,0.2898,,,0.3119,0.2640,,,0.2657
200k,,,,0.2771,,0.3028,,,0.3112,0.2610,,,0.2687
205k,,,,0.2851,,0.2921,,,0.3002,0.2680,,,0.2667
210k,,,,0.2838,,0.2817,,,0.3022,0.2650,,,0.2714
215k,,,,0.2838,,0.2851,,,0.3069,0.2653,,,0.2600
220k,,,,0.2938,,0.2814,,,0.3002,0.2549,,,
225k,,,,0.2935,,0.2898,,,0.3049,0.2633,,,
230k,,,,0.2888,,,,,0.3132,0.2653,,,
235k,,,,0.3055,,,,,0.2951,0.2717,,,
240k,,,,0.2995,,,,,,0.2667,,,
245k,,,,0.2928,,,,,,0.2610,,,
250k,,,,0.3092,,,,,,0.2650,,,
255k,,,,0.3152,,,,,,0.2643,,,
260k,,,,0.2951,,,,,,0.2616,,,
265k,,,,0.3045,,,,,,0.2610,,,
270k,,,,0.3018,,,,,,,,,
275k,,,,0.3065,,,,,,,,,
280k,,,,0.3015,,,,,,,,,
285k,,,,0.2965,,,,,,0.2586,,,
290k,,,,,,,,,,0.2623,,,
300k,,,,,,,,,,0.2603,,,
305k,,,,,,,,,,0.2630,,,
310k,,,,,,,,,,0.2710,,,
315k,,,,,,,,,,0.2677,,,
320k,,,,,,,,,,0.2650,,,
325k,,,,,,,,,,,,,
330k,,,,,,,,,,,,,
335k,,,,,,,,,,,,,