| 5-shot,Slim-Pajama 600B (bsz=4K x 1024),,,FineWeb-1.5T,Ours-Base,Ours-Upsampling1,Ours-Upsampling2,Ours-Code-Upsampling2,All-Upsampling1,All-Upsampling1,All-Upsampling1,All-Upsampling1,DCLM-Base | |
| hf-time: 115 min,Llama-8x8B-baseline,Llama-8x8B-seq8192,Llama-8x8B-mup,Llama-8x8B-seq8192,Llama-8x8B-seq8192,Llama-8x8B-seq8192,Llama-8x8B-seq8192,Llama-8x8B-seq8192,Llama-8x8B-seq8192,Llama-1x8B-seq8192,Llama_extend-1x8B-seq8192,Jais-1x8B-seq8192,Llama-1x8B-seq8192 | |
| 5k,0.0152,0.0099,,,0.0076,0.0015,0.0045,0.0030,,0.0152,0.0106,0.0197,0.0197 | |
| 10k,0.0152,0.0190,,0.0015,,0.0091,0.0000,0.0212,0.0144,0.0159,0.0136,0.0174,0.0243 | |
| 15k,0.0182,0.0167,,0.0053,0.0068,0.0045,0.0083,0.0212,0.0068,0.0174,0.0190,0.0174,0.0136 | |
| 20k,0.0250,0.0212,,,,,0.0030,0.0159,0.0220,0.0167,0.0190,0.0220,0.0174 | |
| 25k,0.0288,0.0114,,,,0.0129,0.0053,0.0258,0.0144,0.0152,0.0144,0.0144,0.0144 | |
| 30k,0.0220,0.0265,,0.0197,0.0038,0.0152,0.0167,0.0227,0.0220,0.0205,0.0129,0.0167,0.0038 | |
| 35k,0.0296,0.0212,,0.0136,0.0045,0.0190,0.0045,0.0227,0.0220,0.0174,0.0174,0.0243,0.0182 | |
| 40k,0.0235,0.0288,,0.0068,0.0121,0.0220,0.0015,0.0243,0.0265,0.0152,0.0212,0.0190,0.0182 | |
| 45k,0.0387,0.0250,,0.0258,0.0038,0.0273,0.0106,0.0296,0.0273,0.0182,0.0152,0.0174,0.0129 | |
| 50k,0.0318,0.0303,,0.0015,0.0243,0.0227,0.0121,0.0190,0.0220,0.0197,0.0205,0.0182,0.0068 | |
| 55k,0.0296,0.0311,,0.0023,0.0235,0.0235,0.0250,0.0326,0.0197,0.0182,0.0174,0.0250,0.0091 | |
| 60k,0.0432,0.0326,,0.0167,0.0212,0.0212,0.0182,0.0349,0.0220,0.0182,0.0099,0.0190,0.0197 | |
| 65k,0.0470,0.0379,,0.0015,0.0159,0.0281,0.0136,0.0296,0.0212,0.0212,0.0129,0.0205,0.0114 | |
| 70k,0.0432,0.0417,,0.0136,0.0197,0.0174,0.0114,0.0341,0.0243,0.0205,0.0136,0.0250,0.0091 | |
| 75k,0.0508,0.0470,,0.0174,0.0121,0.0250,0.0182,0.0356,0.0288,0.0281,0.0174,0.0190,0.0106 | |
| 80k,0.0561,0.0417,,0.0068,0.0000,0.0190,0.0083,0.0318,0.0356,0.0273,0.0167,0.0265,0.0182 | |
| 85k,0.0728,0.0341,,0.0341,0.0190,0.0296,0.0205,0.0265,0.0250,0.0220,0.0129,0.0235,0.0083 | |
| 90k,0.0690,0.0425,,0.0197,0.0190,0.0281,0.0061,0.0417,0.0265,0.0273,0.0167,0.0190,0.0182 | |
| 95k,0.0735,0.0447,,0.0167,0.0250,0.0281,0.0136,0.0349,0.0281,0.0174,0.0106,0.0288,0.0159 | |
| 100k,0.0637,0.0470,,0.0159,,0.0227,0.0045,0.0409,0.0311,0.0265,0.0205,0.0190,0.0190 | |
| 105k,0.0637,0.0447,,0.0341,,0.0303,0.0129,0.0371,0.0311,0.0273,0.0205,0.0311,0.0129 | |
| 110k,0.0872,0.0576,,0.0038,0.0273,0.0129,0.0205,0.0478,0.0296,0.0212,,0.0281,0.0182 | |
| 115k,0.0788,0.0576,,0.0091,0.0167,0.0311,0.0167,0.0508,0.0349,0.0220,,0.0220,0.0174 | |
| 120k,0.0834,0.0455,,0.0227,0.0265,0.0167,0.0212,0.0371,0.0318,0.0167,,0.0220,0.0152 | |
| 125k,0.1001,0.0493,,0.0288,0.0250,0.0205,0.0387,0.0402,0.0318,0.0182,,0.0235,0.0144 | |
| 130k,0.0766,0.0470,,0.0068,0.0258,0.0288,0.0174,,0.0341,0.0243,,,0.0205 | |
| 135k,0.0879,0.0607,,0.0190,,0.0349,0.0258,0.0409,0.0288,0.0212,,,0.0281 | |
| 140k,,0.0569,,0.0379,,0.0356,0.0227,0.0440,0.0341,0.0144,,,0.0144 | |
| 145k,,,,0.0341,,0.0379,0.0015,0.0387,,0.0174,,,0.0273 | |
| 150k,,,,,,0.0281,,0.0470,0.0265,0.0220,,,0.0258 | |
| 155k,,,,0.0318,,0.0303,0.0121,0.0561,0.0523,0.0227,,,0.0243 | |
| 160k,,,,0.0356,,0.0243,0.0061,0.0425,0.0432,0.0220,,,0.0303 | |
| 165k,,,,0.0167,,0.0409,0.0015,,0.0470,0.0281,,, | |
| 170k,,,,0.0334,,0.0281,0.0129,,0.0455,0.0273,,,0.0235 | |
| 175k,,,,0.0371,,0.0326,0.0190,,0.0409,0.0190,,,0.0273 | |
| 180k,,,,0.0425,,0.0364,0.0227,,0.0356,0.0243,,,0.0288 | |
| 185k,,,,0.0341,,0.0318,0.0341,,0.0546,0.0235,,,0.0364 | |
| 190k,,,,0.0296,,0.0364,,,0.0425,0.0220,,,0.0349 | |
| 195k,,,,0.0250,,0.0303,,,0.0493,0.0258,,, | |
| 200k,,,,0.0250,,0.0371,,,0.0493,0.0273,,,0.0205 | |
| 205k,,,,0.0455,,0.0409,,,0.0553,0.0220,,,0.0258 | |
| 210k,,,,0.0462,,0.0371,,,0.0523,0.0281,,, | |
| 215k,,,,0.0349,,0.0265,,,0.0500,0.0235,,,0.0281 | |
| 220k,,,,0.0432,,0.0167,,,0.0462,0.0326,,, | |
| 225k,,,,0.0447,,0.0212,,,,0.0265,,, | |
| 230k,,,,0.0440,,,,,0.0493,0.0273,,, | |
| 235k,,,,0.0402,,,,,0.0508,0.0220,,, | |
| 240k,,,,0.0341,,,,,,0.0281,,, | |
| 245k,,,,0.0462,,,,,,0.0356,,, | |
| 250k,,,,0.0500,,,,,,,,, | |
| 255k,,,,0.0569,,,,,,0.0303,,, | |
| 260k,,,,0.0500,,,,,,0.0334,,, | |
| 265k,,,,0.0455,,,,,,0.0318,,, | |
| 270k,,,,0.0538,,,,,,0.0273,,, | |
| 275k,,,,0.0470,,,,,,,,, | |
| 280k,,,,0.0553,,,,,,0.0364,,, | |
| 285k,,,,0.0531,,,,,,0.0349,,, | |
| 290k,,,,,,,,,,0.0311,,, | |
| 300k,,,,,,,,,,,,, | |
| 305k,,,,,,,,,,0.0311,,, | |
| 310k,,,,,,,,,,0.0273,,, | |
| 315k,,,,,,,,,,,,, | |
| 320k,,,,,,,,,,,,, | |
| 325k,,,,,,,,,,,,, | |
| 330k,,,,,,,,,,,,, | |
| 335k,,,,,,,,,,,,, |