| 0-shot,Slim-Pajama 600B (bsz=4K x 1024),,,FineWeb-1.5T,Ours-Base,Ours-Upsampling1,Ours-Upsampling2,Ours-Code-Upsampling2,All-Upsampling1,All-Upsampling1,All-Upsampling1,All-Upsampling1,DCLM-Base | |
| hf-time: 4 min,Llama-8x8B-baseline,Llama-8x8B-seq8192,Llama-8x8B-mup,Llama-8x8B-seq8192,Llama-8x8B-seq8192,Llama-8x8B-seq8192,Llama-8x8B-seq8192,Llama-8x8B-seq8192,Llama-8x8B-seq8192,Llama-1x8B-seq8192,Llama_extend-1x8B-seq8192,Jais-1x8B-seq8192,Llama-1x8B-seq8192 | |
| 5k,0.5761,0.5624,,0.6116,0.5514,0.5945,0.5446,0.5336,0.5902,0.5908,0.5394,0.5865,0.5284 | |
| 10k,0.6242,0.5853,,0.6131,,0.5358,0.6122,0.6080,0.5471,0.5511,0.6138,0.5902,0.5780 | |
| 15k,0.6480,0.6291,,0.6061,0.6217,0.5468,0.6205,0.6242,0.6248,0.5917,0.6211,0.5933,0.5713 | |
| 20k,0.6541,0.6474,,0.5865,0.6187,0.6122,0.6199,0.6116,0.6119,0.5636,0.6239,0.5988,0.5850 | |
| 25k,0.6670,0.6012,,0.6398,0.6251,0.6162,0.6349,0.6239,0.6291,0.5630,0.6336,0.6232,0.6312 | |
| 30k,0.6777,0.6523,,0.6379,0.6083,0.6260,0.6437,0.6263,0.6107,0.5835,0.5865,0.6391,0.6425 | |
| 35k,0.6495,0.6584,,0.6388,,0.6333,0.6346,0.6343,0.6144,0.4933,0.6043,0.6278,0.6480 | |
| 40k,0.6771,0.6930,,0.6489,0.6410,0.6596,0.6330,0.6214,0.6520,0.5685,0.5768,0.6343,0.6505 | |
| 45k,0.6624,0.6887,,0.6590,0.6422,0.6223,0.6401,0.6131,0.6153,0.5578,0.6058,0.6336,0.6529 | |
| 50k,0.6761,0.6951,,0.6575,0.6566,0.6593,0.6557,0.6058,0.6541,0.5972,0.6018,0.6177,0.6563 | |
| 55k,0.6847,0.6725,,0.6752,0.6321,0.6688,0.6523,0.6520,0.6679,0.5908,0.5343,0.6214,0.6618 | |
| 60k,0.6920,0.6697,,0.6566,0.6226,0.6642,0.6401,0.6162,0.6361,0.5908,0.5972,0.6226,0.6645 | |
| 65k,0.6979,0.6905,,0.6865,0.6352,0.6758,0.6688,0.6691,0.6942,0.6315,0.5682,0.6196,0.6352 | |
| 70k,0.7104,0.6966,,0.6795,0.6456,0.6746,0.6651,0.6624,0.6575,0.5997,0.5324,0.6358,0.6526 | |
| 75k,0.7269,0.6850,,0.6862,0.6514,,0.6621,0.6774,0.6817,0.6217,0.6009,0.6453,0.6535 | |
| 80k,0.6997,0.6817,,0.6945,0.6327,0.6664,0.6667,0.6709,0.6703,0.6275,0.5896,0.6502,0.6612 | |
| 85k,0.7346,0.6939,,0.6853,0.6746,0.6902,0.6602,0.6330,0.6737,0.6272,0.5239,0.6489,0.6703 | |
| 90k,0.7254,0.6908,,0.6936,0.6612,0.6713,0.6755,0.6835,0.6315,0.6275,0.5428,0.6128,0.6807 | |
| 95k,0.7165,0.7229,,0.7003,0.6587,,0.6823,0.6404,0.6670,0.6089,0.6138,0.6456,0.6612 | |
| 100k,0.7153,0.7073,,0.6869,,0.6676,0.6746,0.6618,0.6587,0.6006,0.5584,0.6566,0.6810 | |
| 105k,0.7333,0.7147,,0.6682,,0.6899,0.6609,0.6853,0.6853,0.6544,0.5740,0.6520,0.6755 | |
| 110k,0.7376,0.7095,,0.6954,0.6664,0.6703,0.6810,0.6612,0.6798,0.6618,,0.6346,0.6434 | |
| 115k,0.7168,0.7095,,0.7156,0.6645,0.6746,0.6997,0.6829,0.6813,0.6523,,0.6596,0.6920 | |
| 120k,0.7370,0.7226,,0.7177,0.6648,0.6752,0.7015,,0.6841,0.6633,,0.6587,0.6890 | |
| 125k,0.7361,0.7144,,0.7034,0.6636,0.6826,0.6869,0.6657,,0.6593,,0.6593,0.6795 | |
| 130k,0.7284,0.7269,,0.6939,0.6786,0.6554,0.6988,0.6719,0.6777,0.6260,,,0.7018 | |
| 135k,0.7483,0.7141,,0.7128,,0.6847,0.7028,0.6838,0.6933,0.6602,,,0.6966 | |
| 140k,,0.7312,,0.7080,,0.6777,0.6997,0.6957,0.7040,0.6624,,,0.6884 | |
| 145k,,,,0.7281,,0.6844,0.6908,0.6743,0.6914,0.6657,,,0.7061 | |
| 150k,,,,0.7297,,0.6795,,0.6807,0.6991,0.6526,,,0.7024 | |
| 155k,,,,0.7162,,0.7021,0.6976,0.6792,0.6927,0.6587,,,0.7028 | |
| 160k,,,,0.6902,,0.6810,0.6985,0.6930,0.6893,0.6434,,,0.7098 | |
| 165k,,,,0.7239,,0.6896,0.7037,,0.7021,0.6581,,,0.7080 | |
| 170k,,,,0.7471,,0.6780,0.7141,,0.6911,0.6761,,,0.7058 | |
| 175k,,,,0.7486,,0.6817,0.6942,,0.7095,0.6557,,,0.7021 | |
| 180k,,,,0.6985,,0.6979,0.7162,,0.7067,0.6468,,,0.6523 | |
| 185k,,,,0.7187,,0.6887,0.7031,,0.6917,0.6642,,,0.6914 | |
| 190k,,,,0.7333,,0.6963,,,0.7113,0.6563,,,0.718 | |
| 195k,,,,0.7269,,0.7021,,,0.7199,0.6817,,,0.7165 | |
| 200k,,,,0.7135,,0.7080,,,0.707,0.6709,,,0.7015 | |
| 205k,,,,0.7388,,0.7015,,,0.7168,0.6722,,,0.722 | |
| 210k,,,,0.7489,,0.7089,,,,0.6765,,,0.6948 | |
| 215k,,,,0.7538,,0.7183,,,0.7309,0.6869,,,0.6835 | |
| 220k,,,,0.7474,,0.7171,,,0.7398,0.6893,,, | |
| 225k,,,,0.7251,,0.7131,,,0.7061,0.6801,,, | |
| 230k,,,,0.7083,,,,,0.7232,0.6765,,, | |
| 235k,,,,0.6930,,,,,0.6884,0.6434,,, | |
| 240k,,,,0.7541,,,,,,0.6875,,, | |
| 245k,,,,0.7541,,,,,,0.6713,,, | |
| 250k,,,,0.7498,,,,,,0.6798,,, | |
| 255k,,,,0.7749,,,,,,0.6578,,, | |
| 260k,,,,0.7615,,,,,,0.6954,,, | |
| 265k,,,,0.7486,,,,,,0.6807,,, | |
| 270k,,,,0.7226,,,,,,0.6869,,, | |
| 275k,,,,0.7269,,,,,,0.6841,,, | |
| 280k,,,,0.7517,,,,,,0.6804,,, | |
| 285k,,,,0.7150,,,,,,0.7006,,, | |
| 290k,,,,,,,,,,0.6826,,, | |
| 300k,,,,,,,,,,0.6706,,, | |
| 305k,,,,,,,,,,0.7006,,, | |
| 310k,,,,,,,,,,0.6777,,, | |
| 315k,,,,,,,,,,0.6859,,, | |
| 320k,,,,,,,,,,0.6939,,, | |
| 325k,,,,,,,,,,,,, | |
| 330k,,,,,,,,,,,,, | |
| 335k,,,,,,,,,,,,, |