| 5-shot,Slim-Pajama 600B (bsz=4K x 1024),,,FineWeb-1.5T,Ours-Base,Ours-Upsampling1,Ours-Upsampling2,Ours-Code-Upsampling2,All-Upsampling1,All-Upsampling1,All-Upsampling1,All-Upsampling1,DCLM-Base | |
| time: 5 min,Llama-8x8B-baseline,Llama-8x8B-seq8192,Llama-8x8B-mup,Llama-8x8B-seq8192,Llama-8x8B-seq8192,Llama-8x8B-seq8192,Llama-8x8B-seq8192,Llama-8x8B-seq8192,Llama-8x8B-seq8192,Llama-1x8B-seq8192,Llama_extend-1x8B-seq8192,Jais-1x8B-seq8192,Llama-1x8B-seq8192 | |
| 5k,0.2335,0.2308,,0.2251,,0.2157,0.2221,0.2231,0.2211,0.2251,0.2191,0.2271,0.2238 | |
| 10k,0.2489,0.2519,,0.2379,0.2211,0.2332,0.2415,0.2342,0.2399,0.2285,0.2342,0.2402,0.2224 | |
| 15k,0.2626,0.2469,,0.2526,,0.2389,0.2322,0.2479,0.2580,0.2375,0.2271,0.2355,0.2375 | |
| 20k,0.2737,0.2606,,0.2469,0.2399,,0.2419,0.2526,0.2663,0.2469,0.2499,0.2439,0.2322 | |
| 25k,0.2700,0.2653,,0.2523,0.2395,0.2600,0.2526,0.2616,0.2559,0.2369,0.2476,0.2462,0.2355 | |
| 30k,0.2687,0.2556,,0.2402,,0.2452,0.2533,0.2606,0.2503,0.2456,0.2452,0.2446,0.2372 | |
| 35k,0.2765,0.2533,,0.2683,0.2596,0.2590,0.2509,0.2630,0.2737,0.2392,0.2405,0.2536,0.2402 | |
| 40k,0.2667,0.2683,,0.2496,0.2496,0.2593,0.2529,0.2697,0.2663,0.2379,0.2486,0.2526,0.2422 | |
| 45k,0.2750,0.2620,,0.2616,0.2586,0.2563,0.2503,0.2683,0.2673,0.2479,0.2496,0.2513,0.2472 | |
| 50k,0.2861,0.2697,,0.2693,0.2553,0.2596,0.2553,0.2700,0.2771,0.2442,0.2425,0.2546,0.2395 | |
| 55k,0.2848,0.2693,,0.2640,0.2630,0.2566,0.2479,0.2630,0.2757,0.2526,0.2506,0.2586,0.2509 | |
| 60k,0.2945,0.2784,,0.2727,0.2596,0.2633,0.2590,0.2690,0.2714,0.2519,0.2563,0.2553,0.2479 | |
| 65k,0.3008,0.2767,,0.2680,0.2623,0.2704,0.2610,0.2492,0.2727,0.2529,0.2559,0.2647,0.2462 | |
| 70k,0.2891,0.2824,,0.2730,0.2596,0.2710,0.2700,0.2677,0.2807,0.2469,0.2459,0.2626,0.2576 | |
| 75k,0.2982,0.2938,,0.2784,0.2647,0.2630,0.2697,0.2777,0.2620,0.2626,0.2499,0.2583,0.2549 | |
| 80k,0.2948,0.2801,,0.2737,0.2727,0.2643,0.2553,0.2657,0.2704,0.2509,0.2590,0.2549,0.2563 | |
| 85k,0.2992,0.2938,,0.2754,0.2620,0.2704,0.2677,0.2600,0.2771,0.2496,0.2385,0.2620,0.2529 | |
| 90k,0.3002,0.2888,,0.2764,0.2714,0.2737,0.2573,0.2693,0.2918,0.2616,0.2492,0.2566,0.2516 | |
| 95k,0.3025,0.2817,,0.2616,0.2690,0.2737,0.2523,0.2690,0.2791,0.2492,0.2576,0.2576,0.2549 | |
| 100k,0.2951,0.2894,,0.2616,,0.2817,0.2660,0.2757,0.2861,0.2546,0.2479,0.2667,0.2559 | |
| 105k,0.3052,0.2928,,0.2653,,0.2710,0.2707,0.2771,0.2868,0.2529,0.2482,0.2640,0.2633 | |
| 110k,0.3052,0.2985,,0.2600,0.2764,0.2781,0.2600,0.2764,0.2824,0.2536,,0.2727,0.2606 | |
| 115k,0.3025,0.2985,,0.2690,0.2791,0.2720,0.2704,0.2744,0.2918,0.2623,,0.2807,0.2496 | |
| 120k,0.3042,0.2985,,0.2750,0.2647,0.2650,0.2814,0.2754,0.2955,0.2677,,0.2626,0.2586 | |
| 125k,0.3149,0.3018,,0.2683,0.2707,0.2647,0.2757,0.2760,0.2804,0.2509,,0.2704,0.2496 | |
| 130k,0.3179,0.2978,,0.2781,0.2747,0.2653,0.2760,0.2774,0.2767,0.2593,,,0.2513 | |
| 135k,0.3226,0.2945,,0.2747,,0.2717,0.2673,0.2784,0.2884,0.2606,,,0.2533 | |
| 140k,,0.3018,,0.2771,,0.2757,0.2794,0.2787,0.2821,0.2459,,,0.2596 | |
| 145k,,,,0.2724,,0.2650,0.2720,0.2888,0.2801,0.2543,,,0.2633 | |
| 150k,,,,0.2720,,0.2814,,0.2864,0.2901,0.2590,,,0.2543 | |
| 155k,,,,,,0.2784,0.2720,0.2874,0.2938,0.2580,,,0.2566 | |
| 160k,,,,0.2817,,0.2834,0.2653,0.2807,0.2814,0.2563,,,0.2549 | |
| 165k,,,,0.2834,,0.2821,0.2804,,0.2955,0.2559,,,0.2536 | |
| 170k,,,,0.2854,,0.2824,0.2804,,0.3119,0.2536,,,0.2626 | |
| 175k,,,,0.2804,,0.2915,0.2750,,0.2988,0.2489,,,0.2657 | |
| 180k,,,,0.2767,,0.2901,0.2958,,0.3099,0.2623,,,0.2643 | |
| 185k,,,,0.2767,,0.2948,0.2804,,0.3055,0.2570,,,0.2643 | |
| 190k,,,,0.2787,,0.2925,,,0.3065,0.2573,,,0.2760 | |
| 195k,,,,0.2858,,0.2898,,,0.3119,0.2640,,,0.2657 | |
| 200k,,,,0.2771,,0.3028,,,0.3112,0.2610,,,0.2687 | |
| 205k,,,,0.2851,,0.2921,,,0.3002,0.2680,,,0.2667 | |
| 210k,,,,0.2838,,0.2817,,,0.3022,0.2650,,,0.2714 | |
| 215k,,,,0.2838,,0.2851,,,0.3069,0.2653,,,0.2600 | |
| 220k,,,,0.2938,,0.2814,,,0.3002,0.2549,,, | |
| 225k,,,,0.2935,,0.2898,,,0.3049,0.2633,,, | |
| 230k,,,,0.2888,,,,,0.3132,0.2653,,, | |
| 235k,,,,0.3055,,,,,0.2951,0.2717,,, | |
| 240k,,,,0.2995,,,,,,0.2667,,, | |
| 245k,,,,0.2928,,,,,,0.2610,,, | |
| 250k,,,,0.3092,,,,,,0.2650,,, | |
| 255k,,,,0.3152,,,,,,0.2643,,, | |
| 260k,,,,0.2951,,,,,,0.2616,,, | |
| 265k,,,,0.3045,,,,,,0.2610,,, | |
| 270k,,,,0.3018,,,,,,,,, | |
| 275k,,,,0.3065,,,,,,,,, | |
| 280k,,,,0.3015,,,,,,,,, | |
| 285k,,,,0.2965,,,,,,0.2586,,, | |
| 290k,,,,,,,,,,0.2623,,, | |
| 300k,,,,,,,,,,0.2603,,, | |
| 305k,,,,,,,,,,0.2630,,, | |
| 310k,,,,,,,,,,0.2710,,, | |
| 315k,,,,,,,,,,0.2677,,, | |
| 320k,,,,,,,,,,0.2650,,, | |
| 325k,,,,,,,,,,,,, | |
| 330k,,,,,,,,,,,,, | |
| 335k,,,,,,,,,,,,, |