| 0-shot,FineWeb-1.5T,Ours-Base,Ours-Upsampling2,All-Upsampling1 | |
| time: 3 min,Llama-8x8B-seq8192,Llama-8x8B-seq8192,Llama-8x8B-seq8192,Llama-8x8B-seq8192 | |
| 5k,0.2152,,0.2482,0.2687 | |
| 10k,0.238,0.2372,0.2616,0.2718 | |
| 15k,0.227,,0.2797,0.2639 | |
| 20k,0.2419,,0.2317,0.2757 | |
| 25k,0.2184,,0.2569,0.2474 | |
| 30k,0.2679,0.2522,0.2097,0.2608 | |
| 35k,0.2647,0.2655,0.2467,0.2694 | |
| 40k,0.2671,0.2396,0.2569,0.2482 | |
| 45k,0.2742,0.2734,0.2255,0.2333 | |
| 50k,0.2749,0.2537,0.2372,0.2655 | |
| 55k,0.2797,0.2561,0.2294,0.2537 | |
| 60k,0.2294,0.2325,,0.2639 | |
| 65k,0.2663,0.2757,0.2749,0.2726 | |
| 70k,0.2592,0.2757,0.2632,0.2435 | |
| 75k,0.249,0.2679,0.2616,0.2765 | |
| 80k,0.2797,0.2419,0.2522,0.2789 | |
| 85k,0.2655,0.2844,0.2687,0.2553 | |
| 90k,0.231,0.2364,0.2624,0.2679 | |
| 95k,0.2742,0.282,0.2647,0.2749 | |
| 100k,0.2679,,0.2702,0.2663 | |
| 105k,0.2655,,0.2632,0.2726 | |
| 110k,0.2718,0.2474,0.2537,0.2537 | |
| 115k,0.2655,0.2718,0.2247,0.2867 | |
| 120k,0.293,0.2537,,0.2844 | |
| 125k,0.2624,0.2364,0.2145,0.2883 | |
| 130k,0.2828,0.2412,0.2891,0.2922 | |
| 135k,,,0.2765,0.2702 | |
| 140k,0.2529,,0.2545,0.293 | |
| 145k,0.249,,0.2718,0.3024 | |
| 150k,,,,0.3244 | |
| 155k,0.2608,,0.2624, | |
| 160k,0.2529,,0.2726,0.2852 | |
| 165k,0.2388,,0.2742,0.2561 | |
| 170k,0.2435,,0.2506,0.3056 | |
| 175k,0.2632,,0.2647,0.3126 | |
| 180k,0.2608,,0.2899,0.3166 | |
| 185k,0.271,,0.2561,0.3268 | |
| 190k,0.2812,,,0.304 | |
| 195k,0.2482,,,0.3472 | |
| 200k,0.2639,,,0.3339 | |
| 205k,0.2514,,,0.3409 | |
| 210k,0.2742,,,0.3378 | |
| 215k,0.2592,,,0.3362 | |
| 220k,0.2262,,,0.3559 | |
| 225k,0.249,,,0.3213 | |
| 230k,0.2357,,,0.3472 | |
| 235k,0.2514,,,0.3614 | |
| 240k,0.2624,,, | |
| 245k,0.2482,,, | |
| 250k,0.2592,,, | |
| 255k,0.2537,,, | |
| 260k,0.2639,,, | |
| 265k,0.2844,,, | |
| 270k,0.2624,,, | |
| 275k,0.2757,,, | |
| 280k,0.2852,,, | |
| 285k,0.2726,,, | |
| 290k,,,, | |
| 300k,,,, | |
| 305k,,,, | |
| 310k,,,, | |
| 315k,,,, | |
| 320k,,,, | |
| 325k,,,, | |
| 330k,,,, | |
| 335k,,,, |