0-shot,Slim-Pajama 600B (bsz=4K x 1024),,,FineWeb-1.5T,Ours-Base,Ours-Upsampling1,Ours-Upsampling2,Ours-Code-Upsampling2,All-Upsampling1,All-Upsampling1,All-Upsampling1,All-Upsampling1,DCLM-Base
time: 3 min,Llama-8x8B-baseline,Llama-8x8B-seq8192,Llama-8x8B-mup,Llama-8x8B-seq8192,Llama-8x8B-seq8192,Llama-8x8B-seq8192,Llama-8x8B-seq8192,Llama-8x8B-seq8192,Llama-8x8B-seq8192,Llama-1x8B-seq8192,Llama_extend-1x8B-seq8192,Jais-1x8B-seq8192,Llama-1x8B-seq8192
5k,0.2679,0.2946,,0.2152,,0.2781,0.2482,0.2160,0.2687,0.2639,0.2215,0.2584,0.2396
10k,0.2749,0.2506,,0.2380,0.2372,0.2624,0.2616,0.2734,0.2718,0.2647,0.2435,0.2789,0.2435
15k,0.2388,0.2773,,0.2270,,0.2749,0.2797,0.2545,0.2639,0.2726,0.2773,0.2647,0.2412
20k,0.2412,0.2773,,0.2419,,,0.2317,0.2789,0.2757,0.2372,0.2506,0.2742,0.2577
25k,0.2302,0.2624,,0.2184,,0.2561,0.2569,0.2804,0.2474,0.2616,0.2498,0.2624,0.2419
30k,0.2545,0.2553,,0.2679,0.2522,0.2239,0.2097,0.2742,0.2608,0.2592,0.2694,0.2655,0.2765
35k,0.2757,0.2577,,0.2647,0.2655,0.2671,0.2467,0.2907,0.2694,0.2797,0.2498,0.2749,0.2514
40k,0.2710,0.2608,,0.2671,0.2396,0.2490,0.2569,0.2734,0.2482,0.2647,0.2742,0.2608,0.2671
45k,0.2765,0.2506,,0.2742,0.2734,0.2600,0.2255,0.2859,0.2333,0.2671,0.2435,0.2474,0.2749
50k,0.2742,0.2624,,0.2749,0.2537,0.2482,0.2372,0.2412,0.2655,0.2789,0.2412,0.2702,0.2577
55k,0.2679,0.2545,,0.2797,0.2561,0.2632,0.2294,0.2718,0.2537,0.2647,0.2647,0.2757,0.2632
60k,0.2773,0.2427,,0.2294,0.2325,0.2789,,0.2419,0.2639,0.2679,0.2506,0.2687,0.2419
65k,0.2404,0.2687,,0.2663,0.2757,0.2310,0.2749,0.2836,0.2726,0.2734,0.2537,0.2608,0.2459
70k,0.2600,0.2592,,0.2592,0.2757,0.2797,0.2632,0.2569,0.2435,0.2773,0.2765,0.2702,0.2584
75k,0.2679,0.2584,,0.2490,0.2679,0.2789,0.2616,0.2710,0.2765,0.2742,0.2710,0.2687,0.2789
80k,0.2632,0.2702,,0.2797,0.2419,0.2757,0.2522,0.2616,0.2789,0.2655,0.2694,0.2435,0.2757
85k,0.2734,0.2451,,0.2655,0.2844,0.2608,0.2687,0.2742,0.2553,0.2663,0.2749,0.2639,0.2773
90k,0.2797,0.2506,,0.2310,0.2364,0.2679,0.2624,,0.2679,0.2608,0.2561,0.2765,0.2820
95k,0.2529,0.2545,,0.2742,0.2820,0.2797,0.2647,0.2757,0.2749,0.2663,0.2105,0.2655,0.2749
100k,0.2694,0.2459,,0.2679,,0.2168,0.2702,0.2459,0.2663,0.2655,0.2537,0.2655,0.2781
105k,0.2537,0.2529,,0.2655,,0.2773,0.2632,0.2592,0.2726,0.2687,0.2671,0.2749,0.2812
110k,0.2663,0.2419,,0.2718,0.2474,0.2584,0.2537,0.2569,0.2537,0.2349,,0.2537,0.2765
115k,0.2459,0.2419,,0.2655,0.2718,0.2773,0.2247,0.2852,0.2867,0.2490,,0.2561,0.2364
120k,0.2624,0.2561,,0.2930,0.2537,0.2671,,0.2718,0.2844,0.2545,,0.2608,0.2443
125k,0.2451,0.2742,,0.2624,0.2364,0.2451,0.2145,0.2985,0.2883,0.2726,,0.2498,0.2867
130k,0.2655,0.2797,,0.2828,0.2412,0.2836,0.2891,0.2930,0.2922,0.2522,,,0.2765
135k,0.2749,0.2655,,,,0.2443,0.2765,0.2883,0.2702,0.2679,,,0.2679
140k,,0.2781,,0.2529,,0.2427,0.2545,0.2962,0.2930,0.2569,,,0.2820
145k,,,,0.2490,,0.2427,0.2718,0.3048,0.3024,0.2639,,,0.2632
150k,,,,,,0.2694,,0.2482,0.3244,0.2655,,,0.3150
155k,,,,0.2608,,0.2789,0.2624,0.3134,,0.2490,,,0.3009
160k,,,,0.2529,,0.2765,0.2726,0.3079,0.2852,0.2577,,,0.2757
165k,,,,0.2388,,0.2592,0.2742,,0.2561,0.2380,,,0.3009
170k,,,,0.2435,,0.2852,0.2506,,0.3056,0.2380,,,0.2836
175k,,,,0.2632,,0.2757,0.2647,,0.3126,0.2671,,,0.2993
180k,,,,0.2608,,0.2592,0.2899,,0.3166,0.2396,,,0.3071
185k,,,,0.2710,,0.2859,0.2561,,0.3268,0.2537,,,0.2490
190k,,,,0.2812,,0.2914,,,0.3040,0.2577,,,0.2828
195k,,,,0.2482,,0.2797,,,0.3472,0.2694,,,0.2883
200k,,,,0.2639,,0.2584,,,0.3339,0.2639,,,0.3126
205k,,,,0.2514,,0.3158,,,0.3409,,,,0.2710
210k,,,,0.2742,,0.3016,,,0.3378,0.2624,,,0.2962
215k,,,,0.2592,,0.2859,,,0.3362,,,,0.2859
220k,,,,0.2262,,0.3001,,,0.3559,0.2781,,,
225k,,,,0.2490,,0.3134,,,0.3213,0.2608,,,
230k,,,,0.2357,,,,,0.3472,0.2828,,,
235k,,,,0.2514,,,,,0.3614,0.2639,,,
240k,,,,0.2624,,,,,,0.2867,,,
245k,,,,0.2482,,,,,,0.2718,,,
250k,,,,0.2592,,,,,,0.2624,,,
255k,,,,0.2537,,,,,,0.2781,,,
260k,,,,0.2639,,,,,,0.2679,,,
265k,,,,0.2844,,,,,,0.2616,,,
270k,,,,0.2624,,,,,,,,,
275k,,,,0.2757,,,,,,,,,
280k,,,,0.2852,,,,,,0.2592,,,
285k,,,,0.2726,,,,,,0.2781,,,
290k,,,,,,,,,,0.2671,,,
300k,,,,,,,,,,0.2742,,,
305k,,,,,,,,,,0.2624,,,
310k,,,,,,,,,,0.2718,,,
315k,,,,,,,,,,0.2694,,,
320k,,,,,,,,,,0.2749,,,
325k,,,,,,,,,,,,,
330k,,,,,,,,,,,,,
335k,,,,,,,,,,,,,