| ga,FineWeb-1.5T,Ours-Base,Ours-Upsampling2,All-Upsampling1 | |
| 0-shot: 5 min,Llama-8x8B-seq8192,Llama-8x8B-seq8192,Llama-8x8B-seq8192,Llama-8x8B-seq8192 | |
| 5k,0.5622,0.5254,0.5324,0.5366 | |
| 10k,0.6433,0.5836,0.6046,0.6139 | |
| 15k,0.6716,0.6114,0.6336,0.6388 | |
| 20k,0.6855,0.6271,0.6492,0.6548 | |
| 25k,0.6945,0.6413,0.6665,0.6683 | |
| 30k,0.7059,,0.6746,0.6741 | |
| 35k,0.7158,0.6547,0.6832,0.6864 | |
| 40k,0.7184,0.6642,0.6821,0.6917 | |
| 45k,0.722,0.6698,0.6905,0.6933 | |
| 50k,0.725,0.6689,0.6964,0.7018 | |
| 55k,0.7305,0.6697,0.6959,0.7052 | |
| 60k,0.7236,0.6748,0.6904,0.704 | |
| 65k,0.7355,0.6752,0.7061,0.7074 | |
| 70k,0.7399,0.6773,0.7054,0.7074 | |
| 75k,0.7374,0.6854,0.7065,0.7027 | |
| 80k,0.7422,0.6862,0.7118,0.7139 | |
| 85k,0.7444,0.6887,0.7126,0.7178 | |
| 90k,0.7443,0.6917,0.7148,0.7146 | |
| 95k,0.7376,0.6901,0.7115,0.724 | |
| 100k,0.7457,,0.7117,0.7241 | |
| 105k,0.7476,,0.7132,0.7263 | |
| 110k,0.7486,0.6942,0.7166,0.7284 | |
| 115k,0.7522,0.6957,0.7179,0.7274 | |
| 120k,0.752,0.7022,0.7224,0.7329 | |
| 125k,0.7533,0.7029,0.7221,0.7285 | |
| 130k,0.7573,0.7032,0.7261,0.7337 | |
| 135k,0.758,,0.7198,0.7324 | |
| 140k,0.7596,,0.7245,0.7338 | |
| 145k,0.7573,,0.7247,0.7431 | |
| 150k,0.7614,,,0.7386 | |
| 155k,0.7579,,0.7294,0.7448 | |
| 160k,0.7606,,0.7279,0.7385 | |
| 165k,,,0.7297,0.7493 | |
| 170k,0.7696,,0.7323,0.7499 | |
| 175k,0.7745,,0.7338,0.7502 | |
| 180k,0.7676,,0.7316,0.7457 | |
| 185k,0.7678,,0.7354,0.7519 | |
| 190k,0.7701,,,0.7493 | |
| 195k,0.773,,,0.7579 | |
| 200k,0.7753,,,0.7567 | |
| 205k,0.7744,,,0.756 | |
| 210k,0.7729,,,0.7658 | |
| 215k,0.7804,,,0.7621 | |
| 220k,0.7752,,,0.7678 | |
| 225k,0.7808,,,0.7649 | |
| 230k,0.7786,,,0.7662 | |
| 235k,0.7844,,,0.7676 | |
| 240k,0.7866,,, | |
| 245k,0.7857,,, | |
| 250k,0.7851,,, | |
| 255k,0.7845,,, | |
| 260k,0.7893,,, | |
| 265k,0.7918,,, | |
| 270k,0.7917,,, | |
| 275k,0.7925,,, | |
| 280k,0.7943,,, | |
| 285k,0.7946,,, | |
| 290k,,,, | |
| 300k,,,, | |
| 305k,,,, | |
| 310k,,,, | |
| 315k,,,, | |
| 320k,,,, | |
| 325k,,,, | |
| 330k,,,, | |
| 335k,,,, |