| 0-shot,Slim-Pajama 600B (bsz=4K x 1024),,,FineWeb-1.5T,Ours-Base,Ours-Upsampling1,Ours-Upsampling2,Ours-Code-Upsampling2,All-Upsampling1,All-Upsampling1,All-Upsampling1,All-Upsampling1,DCLM-Base | |
| time: 3 min,Llama-8x8B-baseline,Llama-8x8B-seq8192,Llama-8x8B-mup,Llama-8x8B-seq8192,Llama-8x8B-seq8192,Llama-8x8B-seq8192,Llama-8x8B-seq8192,Llama-8x8B-seq8192,Llama-8x8B-seq8192,Llama-1x8B-seq8192,Llama_extend-1x8B-seq8192,Jais-1x8B-seq8192,Llama-1x8B-seq8192 | |
| 5k,0.2679,0.2946,,0.2152,,0.2781,0.2482,0.2160,0.2687,0.2639,0.2215,0.2584,0.2396 | |
| 10k,0.2749,0.2506,,0.2380,0.2372,0.2624,0.2616,0.2734,0.2718,0.2647,0.2435,0.2789,0.2435 | |
| 15k,0.2388,0.2773,,0.2270,,0.2749,0.2797,0.2545,0.2639,0.2726,0.2773,0.2647,0.2412 | |
| 20k,0.2412,0.2773,,0.2419,,,0.2317,0.2789,0.2757,0.2372,0.2506,0.2742,0.2577 | |
| 25k,0.2302,0.2624,,0.2184,,0.2561,0.2569,0.2804,0.2474,0.2616,0.2498,0.2624,0.2419 | |
| 30k,0.2545,0.2553,,0.2679,0.2522,0.2239,0.2097,0.2742,0.2608,0.2592,0.2694,0.2655,0.2765 | |
| 35k,0.2757,0.2577,,0.2647,0.2655,0.2671,0.2467,0.2907,0.2694,0.2797,0.2498,0.2749,0.2514 | |
| 40k,0.2710,0.2608,,0.2671,0.2396,0.2490,0.2569,0.2734,0.2482,0.2647,0.2742,0.2608,0.2671 | |
| 45k,0.2765,0.2506,,0.2742,0.2734,0.2600,0.2255,0.2859,0.2333,0.2671,0.2435,0.2474,0.2749 | |
| 50k,0.2742,0.2624,,0.2749,0.2537,0.2482,0.2372,0.2412,0.2655,0.2789,0.2412,0.2702,0.2577 | |
| 55k,0.2679,0.2545,,0.2797,0.2561,0.2632,0.2294,0.2718,0.2537,0.2647,0.2647,0.2757,0.2632 | |
| 60k,0.2773,0.2427,,0.2294,0.2325,0.2789,,0.2419,0.2639,0.2679,0.2506,0.2687,0.2419 | |
| 65k,0.2404,0.2687,,0.2663,0.2757,0.2310,0.2749,0.2836,0.2726,0.2734,0.2537,0.2608,0.2459 | |
| 70k,0.2600,0.2592,,0.2592,0.2757,0.2797,0.2632,0.2569,0.2435,0.2773,0.2765,0.2702,0.2584 | |
| 75k,0.2679,0.2584,,0.2490,0.2679,0.2789,0.2616,0.2710,0.2765,0.2742,0.2710,0.2687,0.2789 | |
| 80k,0.2632,0.2702,,0.2797,0.2419,0.2757,0.2522,0.2616,0.2789,0.2655,0.2694,0.2435,0.2757 | |
| 85k,0.2734,0.2451,,0.2655,0.2844,0.2608,0.2687,0.2742,0.2553,0.2663,0.2749,0.2639,0.2773 | |
| 90k,0.2797,0.2506,,0.2310,0.2364,0.2679,0.2624,,0.2679,0.2608,0.2561,0.2765,0.2820 | |
| 95k,0.2529,0.2545,,0.2742,0.2820,0.2797,0.2647,0.2757,0.2749,0.2663,0.2105,0.2655,0.2749 | |
| 100k,0.2694,0.2459,,0.2679,,0.2168,0.2702,0.2459,0.2663,0.2655,0.2537,0.2655,0.2781 | |
| 105k,0.2537,0.2529,,0.2655,,0.2773,0.2632,0.2592,0.2726,0.2687,0.2671,0.2749,0.2812 | |
| 110k,0.2663,0.2419,,0.2718,0.2474,0.2584,0.2537,0.2569,0.2537,0.2349,,0.2537,0.2765 | |
| 115k,0.2459,0.2419,,0.2655,0.2718,0.2773,0.2247,0.2852,0.2867,0.2490,,0.2561,0.2364 | |
| 120k,0.2624,0.2561,,0.2930,0.2537,0.2671,,0.2718,0.2844,0.2545,,0.2608,0.2443 | |
| 125k,0.2451,0.2742,,0.2624,0.2364,0.2451,0.2145,0.2985,0.2883,0.2726,,0.2498,0.2867 | |
| 130k,0.2655,0.2797,,0.2828,0.2412,0.2836,0.2891,0.2930,0.2922,0.2522,,,0.2765 | |
| 135k,0.2749,0.2655,,,,0.2443,0.2765,0.2883,0.2702,0.2679,,,0.2679 | |
| 140k,,0.2781,,0.2529,,0.2427,0.2545,0.2962,0.2930,0.2569,,,0.2820 | |
| 145k,,,,0.2490,,0.2427,0.2718,0.3048,0.3024,0.2639,,,0.2632 | |
| 150k,,,,,,0.2694,,0.2482,0.3244,0.2655,,,0.3150 | |
| 155k,,,,0.2608,,0.2789,0.2624,0.3134,,0.2490,,,0.3009 | |
| 160k,,,,0.2529,,0.2765,0.2726,0.3079,0.2852,0.2577,,,0.2757 | |
| 165k,,,,0.2388,,0.2592,0.2742,,0.2561,0.2380,,,0.3009 | |
| 170k,,,,0.2435,,0.2852,0.2506,,0.3056,0.2380,,,0.2836 | |
| 175k,,,,0.2632,,0.2757,0.2647,,0.3126,0.2671,,,0.2993 | |
| 180k,,,,0.2608,,0.2592,0.2899,,0.3166,0.2396,,,0.3071 | |
| 185k,,,,0.2710,,0.2859,0.2561,,0.3268,0.2537,,,0.2490 | |
| 190k,,,,0.2812,,0.2914,,,0.3040,0.2577,,,0.2828 | |
| 195k,,,,0.2482,,0.2797,,,0.3472,0.2694,,,0.2883 | |
| 200k,,,,0.2639,,0.2584,,,0.3339,0.2639,,,0.3126 | |
| 205k,,,,0.2514,,0.3158,,,0.3409,,,,0.2710 | |
| 210k,,,,0.2742,,0.3016,,,0.3378,0.2624,,,0.2962 | |
| 215k,,,,0.2592,,0.2859,,,0.3362,,,,0.2859 | |
| 220k,,,,0.2262,,0.3001,,,0.3559,0.2781,,, | |
| 225k,,,,0.2490,,0.3134,,,0.3213,0.2608,,, | |
| 230k,,,,0.2357,,,,,0.3472,0.2828,,, | |
| 235k,,,,0.2514,,,,,0.3614,0.2639,,, | |
| 240k,,,,0.2624,,,,,,0.2867,,, | |
| 245k,,,,0.2482,,,,,,0.2718,,, | |
| 250k,,,,0.2592,,,,,,0.2624,,, | |
| 255k,,,,0.2537,,,,,,0.2781,,, | |
| 260k,,,,0.2639,,,,,,0.2679,,, | |
| 265k,,,,0.2844,,,,,,0.2616,,, | |
| 270k,,,,0.2624,,,,,,,,, | |
| 275k,,,,0.2757,,,,,,,,, | |
| 280k,,,,0.2852,,,,,,0.2592,,, | |
| 285k,,,,0.2726,,,,,,0.2781,,, | |
| 290k,,,,,,,,,,0.2671,,, | |
| 300k,,,,,,,,,,0.2742,,, | |
| 305k,,,,,,,,,,0.2624,,, | |
| 310k,,,,,,,,,,0.2718,,, | |
| 315k,,,,,,,,,,0.2694,,, | |
| 320k,,,,,,,,,,0.2749,,, | |
| 325k,,,,,,,,,,,,, | |
| 330k,,,,,,,,,,,,, | |
| 335k,,,,,,,,,,,,, |