ga,FineWeb-1.5T,Ours-Base,Ours-Upsampling2,All-Upsampling1
0-shot: 5 min,Llama-8x8B-seq8192,Llama-8x8B-seq8192,Llama-8x8B-seq8192,Llama-8x8B-seq8192
5k,0.5622,0.5254,0.5324,0.5366
10k,0.6433,0.5836,0.6046,0.6139
15k,0.6716,0.6114,0.6336,0.6388
20k,0.6855,0.6271,0.6492,0.6548
25k,0.6945,0.6413,0.6665,0.6683
30k,0.7059,,0.6746,0.6741
35k,0.7158,0.6547,0.6832,0.6864
40k,0.7184,0.6642,0.6821,0.6917
45k,0.722,0.6698,0.6905,0.6933
50k,0.725,0.6689,0.6964,0.7018
55k,0.7305,0.6697,0.6959,0.7052
60k,0.7236,0.6748,0.6904,0.704
65k,0.7355,0.6752,0.7061,0.7074
70k,0.7399,0.6773,0.7054,0.7074
75k,0.7374,0.6854,0.7065,0.7027
80k,0.7422,0.6862,0.7118,0.7139
85k,0.7444,0.6887,0.7126,0.7178
90k,0.7443,0.6917,0.7148,0.7146
95k,0.7376,0.6901,0.7115,0.724
100k,0.7457,,0.7117,0.7241
105k,0.7476,,0.7132,0.7263
110k,0.7486,0.6942,0.7166,0.7284
115k,0.7522,0.6957,0.7179,0.7274
120k,0.752,0.7022,0.7224,0.7329
125k,0.7533,0.7029,0.7221,0.7285
130k,0.7573,0.7032,0.7261,0.7337
135k,0.758,,0.7198,0.7324
140k,0.7596,,0.7245,0.7338
145k,0.7573,,0.7247,0.7431
150k,0.7614,,,0.7386
155k,0.7579,,0.7294,0.7448
160k,0.7606,,0.7279,0.7385
165k,,,0.7297,0.7493
170k,0.7696,,0.7323,0.7499
175k,0.7745,,0.7338,0.7502
180k,0.7676,,0.7316,0.7457
185k,0.7678,,0.7354,0.7519
190k,0.7701,,,0.7493
195k,0.773,,,0.7579
200k,0.7753,,,0.7567
205k,0.7744,,,0.756
210k,0.7729,,,0.7658
215k,0.7804,,,0.7621
220k,0.7752,,,0.7678
225k,0.7808,,,0.7649
230k,0.7786,,,0.7662
235k,0.7844,,,0.7676
240k,0.7866,,,
245k,0.7857,,,
250k,0.7851,,,
255k,0.7845,,,
260k,0.7893,,,
265k,0.7918,,,
270k,0.7917,,,
275k,0.7925,,,
280k,0.7943,,,
285k,0.7946,,,
290k,,,,
300k,,,,
305k,,,,
310k,,,,
315k,,,,
320k,,,,
325k,,,,
330k,,,,
335k,,,,