| 5-shot,Slim-Pajama 600B (bsz=4K x 1024),,,FineWeb-1.5T,Ours-Base,Ours-Upsampling1,Ours-Upsampling2,Ours-Code-Upsampling2,All-Upsampling1,All-Upsampling1,All-Upsampling1,All-Upsampling1,DCLM-Base | |
| time: 22 min,Llama-8x8B-baseline,Llama-8x8B-seq8192,Llama-8x8B-mup,Llama-8x8B-seq8192,Llama-8x8B-seq8192,Llama-8x8B-seq8192,Llama-8x8B-seq8192,Llama-8x8B-seq8192,Llama-8x8B-seq8192,Llama-1x8B-seq8192,Llama_extend-1x8B-seq8192,Jais-1x8B-seq8192,Llama-1x8B-seq8192 | |
| 5k,0.0615,0.0537,,0.0341,0.0416,0.0634,0.0565,0.0579,0.0526,0.0219,0.0213,0.0205,0.0274 | |
| 10k,0.1075,0.1053,,0.0715,,0.0906,0.0931,0.0828,0.0767,0.0391,0.0418,0.0529,0.0554 | |
| 15k,0.1382,0.1136,,0.0765,,0.1147,0.1061,0.1152,0.1127,0.0607,0.0560,0.0629,0.0587 | |
| 20k,0.1490,0.1393,,0.0787,,0.1161,0.1183,0.1285,0.1247,0.0529,0.0623,0.0668,0.0709 | |
| 25k,0.1687,0.1416,,0.0892,0.1150,0.1402,0.1352,0.1380,0.1343,0.0584,0.0687,0.0762,0.0828 | |
| 30k,0.1767,0.1557,,0.0911,0.1366,0.1454,0.1271,0.1501,0.1421,0.0723,0.0687,0.0723,0.0839 | |
| 35k,0.1706,0.1756,,0.0970,0.1488,0.1573,0.1485,0.1565,0.1524,0.0803,0.0798,0.0803,0.0778 | |
| 40k,0.1942,0.1759,,0.1028,0.1355,0.1560,0.1488,0.1554,0.1562,0.0759,0.0848,0.0845,0.0886 | |
| 45k,0.1798,0.1820,,0.1078,0.1488,0.1715,0.1620,0.1684,0.1598,0.0881,0.0911,0.0867,0.0848 | |
| 50k,0.1972,0.1809,,0.1050,0.1540,,0.1590,0.1657,0.1698,0.0864,0.0909,0.0909,0.0884 | |
| 55k,0.2158,0.1956,,0.1097,0.1607,0.1659,0.1662,0.1751,0.1704,0.0892,0.0898,0.0745,0.0931 | |
| 60k,0.2039,0.2036,,0.1211,0.1654,0.1734,0.1612,0.1745,0.1801,0.0817,0.0850,0.0922,0.0986 | |
| 65k,0.2244,0.2044,,0.1089,0.1573,0.1765,0.1693,0.1776,0.1823,0.0920,0.0967,0.1025,0.1066 | |
| 70k,0.2233,0.2233,,0.1222,0.1634,0.1845,0.1679,0.1859,0.1767,0.1022,0.0925,0.1039,0.1177 | |
| 75k,0.2305,0.2277,,0.1097,0.1709,0.1825,0.1881,0.1737,0.1762,0.1069,0.0936,0.1116,0.1199 | |
| 80k,0.2457,0.2252,,0.1277,0.1573,0.1900,0.1776,0.1787,0.1964,0.1047,0.0981,0.1033,0.1097 | |
| 85k,0.2501,0.2285,,0.1280,0.1776,0.1914,0.1889,0.1870,0.1889,0.0942,0.0964,0.1144,0.1213 | |
| 90k,0.2504,0.2521,,0.1158,0.1598,0.1911,0.1806,0.1898,0.1773,0.1058,0.0964,0.1186,0.1163 | |
| 95k,0.2579,0.2443,,0.1235,0.1762,0.1911,0.1781,0.1989,0.1917,0.1097,0.0928,0.1213,0.1169 | |
| 100k,0.2526,0.2446,,0.1258,,0.2097,0.1928,0.1903,0.1947,0.1125,0.1025,0.1127,0.1188 | |
| 105k,0.2679,0.2482,,0.1366,,0.2028,0.1814,0.1922,0.2094,0.1199,0.1069,0.1186,0.1269 | |
| 110k,0.2717,0.2562,,0.1377,0.1756,0.2019,0.1859,0.1975,,0.1152,,0.1252,0.1252 | |
| 115k,0.2745,0.2562,,0.1346,0.1831,0.1956,0.1947,0.1903,0.2119,0.1127,,0.1285,0.1111 | |
| 120k,0.2801,0.2612,,0.1402,0.2014,0.2000,,0.2044,0.2119,0.1188,,0.1166,0.1219 | |
| 125k,0.2751,0.2657,,0.1307,0.2030,0.2014,0.1992,0.2053,0.1787,0.1230,,0.1274,0.1418 | |
| 130k,0.2884,0.2673,,0.1368,0.1997,0.2125,0.1994,0.2011,0.2086,0.1127,,,0.1335 | |
| 135k,0.2842,0.2673,,0.1363,,0.2069,0.2014,0.2036,0.2069,0.1255,,,0.1299 | |
| 140k,,0.2679,,0.1435,,0.2039,0.1986,0.2042,0.2058,0.1263,,,0.1299 | |
| 145k,,,,0.1532,,0.2172,0.1953,0.2078,0.2102,0.1274,,,0.1443 | |
| 150k,,,,0.1404,,0.2125,,0.2127,0.2075,0.1263,,,0.1410 | |
| 155k,,,,0.1418,,0.2235,0.1931,0.2066,0.2205,0.1418,,,0.1460 | |
| 160k,,,,0.1346,,0.2183,0.2116,0.2069,0.2208,0.1319,,,0.1413 | |
| 165k,,,,0.1524,,0.2219,0.2139,,0.2213,0.1296,,,0.1424 | |
| 170k,,,,0.1388,,0.2175,,,0.2169,0.1366,,,0.1454 | |
| 175k,,,,0.1438,,0.2235,0.2222,,0.2321,0.1349,,,0.1399 | |
| 180k,,,,0.1471,,0.2260,0.2249,,0.236,0.1465,,,0.1421 | |
| 185k,,,,0.1499,,0.2341,0.2222,,0.2366,0.1449,,,0.1421 | |
| 190k,,,,0.1504,,0.2233,,,0.2274,0.1413,,,0.1471 | |
| 195k,,,,0.1554,,0.2330,,,0.2454,0.1440,,,0.1407 | |
| 200k,,,,0.1565,,0.2238,,,0.2346,0.1407,,,0.1449 | |
| 205k,,,,0.1726,,0.2271,,,0.2316,0.1382,,,0.1501 | |
| 210k,,,,0.1623,,0.2305,,,0.2493,0.1526,,,0.1424 | |
| 215k,,,,0.1576,,0.2299,,,0.2355,0.1518,,,0.1535 | |
| 220k,,,,0.1693,,0.2330,,,0.2427,0.1529,,, | |
| 225k,,,,0.1596,,0.2366,,,0.2440,0.1479,,, | |
| 230k,,,,0.1693,,,,,0.2554,0.1560,,, | |
| 235k,,,,0.1720,,,,,0.2535,0.1540,,, | |
| 240k,,,,0.1712,,,,,,0.1554,,, | |
| 245k,,,,0.1704,,,,,,0.1532,,, | |
| 250k,,,,0.1784,,,,,,0.1551,,, | |
| 255k,,,,0.1740,,,,,,0.1623,,, | |
| 260k,,,,0.1756,,,,,,0.1618,,, | |
| 265k,,,,0.1886,,,,,,0.1604,,, | |
| 270k,,,,0.1820,,,,,,0.1612,,, | |
| 275k,,,,0.1870,,,,,,0.1629,,, | |
| 280k,,,,0.1704,,,,,,0.1645,,, | |
| 285k,,,,0.1903,,,,,,0.1665,,, | |
| 290k,,,,,,,,,,0.1648,,, | |
| 300k,,,,,,,,,,0.1712,,, | |
| 305k,,,,,,,,,,0.1690,,, | |
| 310k,,,,,,,,,,0.1712,,, | |
| 315k,,,,,,,,,,,,, | |
| 320k,,,,,,,,,,,,, | |
| 325k,,,,,,,,,,,,, | |
| 330k,,,,,,,,,,,,, | |
| 335k,,,,,,,,,,,,, |