0-shot,Slim-Pajama 600B (bsz=4K x 1024),,,FineWeb-1.5T,Ours-Base,Ours-Upsampling1,Ours-Upsampling2,Ours-Code-Upsampling2,All-Upsampling1,All-Upsampling1,All-Upsampling1,All-Upsampling1,DCLM-Base
hf-time: 4 min,Llama-8x8B-baseline,Llama-8x8B-seq8192,Llama-8x8B-mup,Llama-8x8B-seq8192,Llama-8x8B-seq8192,Llama-8x8B-seq8192,Llama-8x8B-seq8192,Llama-8x8B-seq8192,Llama-8x8B-seq8192,Llama-1x8B-seq8192,Llama_extend-1x8B-seq8192,Jais-1x8B-seq8192,Llama-1x8B-seq8192
5k,0.5761,0.5624,,0.6116,0.5514,0.5945,0.5446,0.5336,0.5902,0.5908,0.5394,0.5865,0.5284
10k,0.6242,0.5853,,0.6131,,0.5358,0.6122,0.6080,0.5471,0.5511,0.6138,0.5902,0.5780
15k,0.6480,0.6291,,0.6061,0.6217,0.5468,0.6205,0.6242,0.6248,0.5917,0.6211,0.5933,0.5713
20k,0.6541,0.6474,,0.5865,0.6187,0.6122,0.6199,0.6116,0.6119,0.5636,0.6239,0.5988,0.5850
25k,0.6670,0.6012,,0.6398,0.6251,0.6162,0.6349,0.6239,0.6291,0.5630,0.6336,0.6232,0.6312
30k,0.6777,0.6523,,0.6379,0.6083,0.6260,0.6437,0.6263,0.6107,0.5835,0.5865,0.6391,0.6425
35k,0.6495,0.6584,,0.6388,,0.6333,0.6346,0.6343,0.6144,0.4933,0.6043,0.6278,0.6480
40k,0.6771,0.6930,,0.6489,0.6410,0.6596,0.6330,0.6214,0.6520,0.5685,0.5768,0.6343,0.6505
45k,0.6624,0.6887,,0.6590,0.6422,0.6223,0.6401,0.6131,0.6153,0.5578,0.6058,0.6336,0.6529
50k,0.6761,0.6951,,0.6575,0.6566,0.6593,0.6557,0.6058,0.6541,0.5972,0.6018,0.6177,0.6563
55k,0.6847,0.6725,,0.6752,0.6321,0.6688,0.6523,0.6520,0.6679,0.5908,0.5343,0.6214,0.6618
60k,0.6920,0.6697,,0.6566,0.6226,0.6642,0.6401,0.6162,0.6361,0.5908,0.5972,0.6226,0.6645
65k,0.6979,0.6905,,0.6865,0.6352,0.6758,0.6688,0.6691,0.6942,0.6315,0.5682,0.6196,0.6352
70k,0.7104,0.6966,,0.6795,0.6456,0.6746,0.6651,0.6624,0.6575,0.5997,0.5324,0.6358,0.6526
75k,0.7269,0.6850,,0.6862,0.6514,,0.6621,0.6774,0.6817,0.6217,0.6009,0.6453,0.6535
80k,0.6997,0.6817,,0.6945,0.6327,0.6664,0.6667,0.6709,0.6703,0.6275,0.5896,0.6502,0.6612
85k,0.7346,0.6939,,0.6853,0.6746,0.6902,0.6602,0.6330,0.6737,0.6272,0.5239,0.6489,0.6703
90k,0.7254,0.6908,,0.6936,0.6612,0.6713,0.6755,0.6835,0.6315,0.6275,0.5428,0.6128,0.6807
95k,0.7165,0.7229,,0.7003,0.6587,,0.6823,0.6404,0.6670,0.6089,0.6138,0.6456,0.6612
100k,0.7153,0.7073,,0.6869,,0.6676,0.6746,0.6618,0.6587,0.6006,0.5584,0.6566,0.6810
105k,0.7333,0.7147,,0.6682,,0.6899,0.6609,0.6853,0.6853,0.6544,0.5740,0.6520,0.6755
110k,0.7376,0.7095,,0.6954,0.6664,0.6703,0.6810,0.6612,0.6798,0.6618,,0.6346,0.6434
115k,0.7168,0.7095,,0.7156,0.6645,0.6746,0.6997,0.6829,0.6813,0.6523,,0.6596,0.6920
120k,0.7370,0.7226,,0.7177,0.6648,0.6752,0.7015,,0.6841,0.6633,,0.6587,0.6890
125k,0.7361,0.7144,,0.7034,0.6636,0.6826,0.6869,0.6657,,0.6593,,0.6593,0.6795
130k,0.7284,0.7269,,0.6939,0.6786,0.6554,0.6988,0.6719,0.6777,0.6260,,,0.7018
135k,0.7483,0.7141,,0.7128,,0.6847,0.7028,0.6838,0.6933,0.6602,,,0.6966
140k,,0.7312,,0.7080,,0.6777,0.6997,0.6957,0.7040,0.6624,,,0.6884
145k,,,,0.7281,,0.6844,0.6908,0.6743,0.6914,0.6657,,,0.7061
150k,,,,0.7297,,0.6795,,0.6807,0.6991,0.6526,,,0.7024
155k,,,,0.7162,,0.7021,0.6976,0.6792,0.6927,0.6587,,,0.7028
160k,,,,0.6902,,0.6810,0.6985,0.6930,0.6893,0.6434,,,0.7098
165k,,,,0.7239,,0.6896,0.7037,,0.7021,0.6581,,,0.7080
170k,,,,0.7471,,0.6780,0.7141,,0.6911,0.6761,,,0.7058
175k,,,,0.7486,,0.6817,0.6942,,0.7095,0.6557,,,0.7021
180k,,,,0.6985,,0.6979,0.7162,,0.7067,0.6468,,,0.6523
185k,,,,0.7187,,0.6887,0.7031,,0.6917,0.6642,,,0.6914
190k,,,,0.7333,,0.6963,,,0.7113,0.6563,,,0.718
195k,,,,0.7269,,0.7021,,,0.7199,0.6817,,,0.7165
200k,,,,0.7135,,0.7080,,,0.707,0.6709,,,0.7015
205k,,,,0.7388,,0.7015,,,0.7168,0.6722,,,0.722
210k,,,,0.7489,,0.7089,,,,0.6765,,,0.6948
215k,,,,0.7538,,0.7183,,,0.7309,0.6869,,,0.6835
220k,,,,0.7474,,0.7171,,,0.7398,0.6893,,,
225k,,,,0.7251,,0.7131,,,0.7061,0.6801,,,
230k,,,,0.7083,,,,,0.7232,0.6765,,,
235k,,,,0.6930,,,,,0.6884,0.6434,,,
240k,,,,0.7541,,,,,,0.6875,,,
245k,,,,0.7541,,,,,,0.6713,,,
250k,,,,0.7498,,,,,,0.6798,,,
255k,,,,0.7749,,,,,,0.6578,,,
260k,,,,0.7615,,,,,,0.6954,,,
265k,,,,0.7486,,,,,,0.6807,,,
270k,,,,0.7226,,,,,,0.6869,,,
275k,,,,0.7269,,,,,,0.6841,,,
280k,,,,0.7517,,,,,,0.6804,,,
285k,,,,0.7150,,,,,,0.7006,,,
290k,,,,,,,,,,0.6826,,,
300k,,,,,,,,,,0.6706,,,
305k,,,,,,,,,,0.7006,,,
310k,,,,,,,,,,0.6777,,,
315k,,,,,,,,,,0.6859,,,
320k,,,,,,,,,,0.6939,,,
325k,,,,,,,,,,,,,
330k,,,,,,,,,,,,,
335k,,,,,,,,,,,,,