5-shot,Slim-Pajama 600B (bsz=4K x 1024),,,FineWeb-1.5T,Ours-Base,Ours-Upsampling1,Ours-Upsampling2,Ours-Code-Upsampling2,All-Upsampling1,All-Upsampling1,All-Upsampling1,All-Upsampling1,DCLM-Base,Comments
time: 76 min,Llama-8x8B-baseline,Llama-8x8B-seq8192,Llama-8x8B-mup,Llama-8x8B-seq8192,Llama-8x8B-seq8192,Llama-8x8B-seq8192,Llama-8x8B-seq8192,Llama-8x8B-seq8192,Llama-8x8B-seq8192,Llama-1x8B-seq8192,Llama_extend-1x8B-seq8192,Jais-1x8B-seq8192,Llama-1x8B-seq8192,1. Takes 25min to load checkpoints.
5k,0.1944,0.1764,,0.1025,0.1232,0.1086,0.1260,0.1647,,0.0841,0.0762,0.0824,0.1066,2. GPU utility is only 20%
10k,0.3372,0.3292,,0.2073,,0.2636,0.1150,0.2659,0.2604,0.1348,0.1343,0.1585,0.1850,
15k,0.4050,0.3909,,0.3005,,0.3250,0.1872,0.3445,0.3244,0.1821,0.1930,0.1968,0.2443,
20k,0.4451,0.4497,,0.3506,0.2795,,0.2719,0.3802,0.3637,0.2086,0.2196,0.2231,0.2772,
25k,0.4899,0.4601,,0.3070,,0.3975,0.4093,0.4105,0.4120,0.2261,0.2375,0.2574,0.3146,
30k,0.5125,0.4824,,0.2461,0.2974,0.0303,0.4195,0.4330,0.4294,0.2352,0.2484,0.2675,0.3328,
35k,0.5249,0.5091,,0.3639,0.3572,0.1983,0.3587,0.4434,0.4428,0.2433,,0.2863,0.3507,
40k,0.5555,0.5166,,0.3537,0.0346,0.4571,0.4434,0.4618,0.4623,0.2708,0.2828,0.3020,0.3606,
45k,0.5664,0.5403,,0.3602,0.2674,0.2654,0.4366,0.4746,0.4792,0.2668,0.3018,0.3065,0.3726,
50k,0.5690,0.5217,,0.2407,0.3689,0.4355,0.4051,0.4885,0.4795,0.2906,0.2952,0.3187,0.3807,
55k,0.5843,0.5680,,0.2081,0.4101,0.4341,0.3230,0.4931,0.4940,0.2940,0.3117,0.3242,0.3984,
60k,0.5916,0.5814,,0.4068,0.4107,0.4861,0.4469,0.4955,0.5130,0.3137,0.3090,0.3422,0.4081,
65k,0.6032,0.5774,,0.3145,0.4477,0.4858,0.4907,0.5039,0.5087,0.3097,0.3184,0.3397,0.4156,
70k,0.6030,0.5920,,0.4102,0.4736,0.5080,0.4920,0.5164,0.5129,0.3236,0.3360,0.3375,0.4242,
75k,0.6216,0.6187,,0.2820,0.4226,0.4777,0.2245,0.5190,0.5042,0.3265,0.3341,0.3483,0.4220,
80k,0.6397,0.6127,,0.0975,0.4217,0.3698,,0.5185,0.5301,0.3352,0.3412,0.3532,0.4306,
85k,0.6416,0.6254,,0.0722,0.4763,0.3700,0.5029,0.5249,0.5350,0.3448,0.3423,0.3530,0.4340,
90k,0.6510,0.6317,,0.3388,0.1472,0.4793,0.0317,0.5337,0.5220,0.3440,0.3559,0.3644,0.4418,
95k,0.6655,0.6479,,0.5283,0.4938,0.5144,0.5180,0.5432,0.5446,0.3331,0.3393,0.3683,0.4454,
100k,0.6723,0.6486,,0.4317,0.1100,0.5121,0.5358,0.5383,0.5514,0.3520,0.3544,0.3698,0.4378,
105k,0.6755,0.6582,,0.1886,,0.5280,0.5153,0.5499,0.5562,0.3626,0.3642,0.3683,0.4525,
110k,0.6798,0.6668,,0.3510,,0.5468,0.5182,0.5541,0.5654,0.3694,,0.3903,0.4566,
115k,0.6796,0.6668,,0.3692,0.4759,0.5347,0.5132,0.5508,0.5577,0.3741,,0.3908,0.4482,
120k,0.6822,0.6688,,0.3690,0.4352,0.5376,0.5483,0.5567,0.5658,0.3881,,0.3950,0.4524,
125k,0.6894,0.6743,,0.3365,0.5206,0.4855,0.5211,0.5617,0.5658,0.3725,,0.3880,0.4592,
130k,0.6914,0.6709,,0.3550,0.0088,0.5238,0.5245,0.5597,0.5609,0.3698,,,0.4594,
135k,0.6915,0.6721,,0.3892,,0.5467,0.3977,0.5541,0.5774,0.3782,,,0.4636,
140k,,0.6773,,0.3930,,0.3110,0.4991,0.5572,0.5675,0.3906,,,0.4741,
145k,,,,0.4538,,0.5720,0.4872,0.5642,0.5639,,,,0.4720,
150k,,,,0.2883,,0.5612,,0.5701,0.5844,0.3899,,,0.4651,
155k,,,,0.4185,,0.5030,0.1586,0.5790,0.5755,0.4044,,,0.4784,
160k,,,,0.2720,,0.5701,0.5630,0.5819,0.5864,0.4049,,,0.4665,
165k,,,,0.4252,,0.5388,0.5642,,0.5853,0.4007,,,0.4793,
170k,,,,0.1507,,0.5951,0.5739,,,0.4150,,,0.4846,
175k,,,,0.3242,,0.5437,0.5640,,0.5979,0.4092,,,0.4908,
180k,,,,0.2653,,0.5580,0.5912,,0.6054,0.4189,,,,
185k,,,,0.2651,,0.5709,0.5852,,0.6064,,,,0.5030,
190k,,,,0.2380,,0.5142,,,0.5996,0.4193,,,0.5115,
195k,,,,0.4048,,0.5964,,,0.6243,0.4265,,,,
200k,,,,0.5058,,0.5684,,,0.6248,0.4256,,,,
205k,,,,0.0945,,0.5878,,,0.6224,0.4190,,,0.5105,
210k,,,,0.1557,,0.6020,,,0.6311,0.4415,,,0.5164,
215k,,,,0.2483,,0.5995,,,0.6293,0.4353,,,0.5163,
220k,,,,0.1725,,0.5924,,,0.6375,,,,,
225k,,,,0.2467,,0.4832,,,0.6340,0.4556,,,,
230k,,,,0.1653,,,,,0.6436,0.4622,,,,
235k,,,,0.1884,,,,,0.6411,0.4608,,,,
240k,,,,0.0719,,,,,,0.4536,,,,
245k,,,,0.3757,,,,,,0.4641,,,,
250k,,,,0.5859,,,,,,,,,,
255k,,,,0.4987,,,,,,0.4741,,,,
260k,,,,0.3940,,,,,,0.4712,,,,
265k,,,,0.3607,,,,,,0.4767,,,,
270k,,,,0.3898,,,,,,0.4795,,,,
275k,,,,0.4123,,,,,,,,,,
280k,,,,0.2413,,,,,,0.4787,,,,
285k,,,,0.3665,,,,,,0.4843,,,,
290k,,,,,,,,,,0.4818,,,,
300k,,,,,,,,,,0.4969,,,,
305k,,,,,,,,,,0.4941,,,,
310k,,,,,,,,,,0.4963,,,,
315k,,,,,,,,,,,,,,
320k,,,,,,,,,,,,,,
325k,,,,,,,,,,,,,,
330k,,,,,,,,,,,,,,
335k,,,,,,,,,,,,,,