Spaces:
Running
Running
File size: 4,325 Bytes
9a127b5 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 |
5-shot,Slim-Pajama 600B (bsz=4K x 1024),,,FineWeb-1.5T,Ours-Base,Ours-Upsampling1,Ours-Upsampling2,Ours-Code-Upsampling2,All-Upsampling1,All-Upsampling1,All-Upsampling1,All-Upsampling1,DCLM-Base
time: 22 min,Llama-8x8B-baseline,Llama-8x8B-seq8192,Llama-8x8B-mup,Llama-8x8B-seq8192,Llama-8x8B-seq8192,Llama-8x8B-seq8192,Llama-8x8B-seq8192,Llama-8x8B-seq8192,Llama-8x8B-seq8192,Llama-1x8B-seq8192,Llama_extend-1x8B-seq8192,Jais-1x8B-seq8192,Llama-1x8B-seq8192
5k,0.0615,0.0537,,0.0341,0.0416,0.0634,0.0565,0.0579,0.0526,0.0219,0.0213,0.0205,0.0274
10k,0.1075,0.1053,,0.0715,,0.0906,0.0931,0.0828,0.0767,0.0391,0.0418,0.0529,0.0554
15k,0.1382,0.1136,,0.0765,,0.1147,0.1061,0.1152,0.1127,0.0607,0.0560,0.0629,0.0587
20k,0.1490,0.1393,,0.0787,,0.1161,0.1183,0.1285,0.1247,0.0529,0.0623,0.0668,0.0709
25k,0.1687,0.1416,,0.0892,0.1150,0.1402,0.1352,0.1380,0.1343,0.0584,0.0687,0.0762,0.0828
30k,0.1767,0.1557,,0.0911,0.1366,0.1454,0.1271,0.1501,0.1421,0.0723,0.0687,0.0723,0.0839
35k,0.1706,0.1756,,0.0970,0.1488,0.1573,0.1485,0.1565,0.1524,0.0803,0.0798,0.0803,0.0778
40k,0.1942,0.1759,,0.1028,0.1355,0.1560,0.1488,0.1554,0.1562,0.0759,0.0848,0.0845,0.0886
45k,0.1798,0.1820,,0.1078,0.1488,0.1715,0.1620,0.1684,0.1598,0.0881,0.0911,0.0867,0.0848
50k,0.1972,0.1809,,0.1050,0.1540,,0.1590,0.1657,0.1698,0.0864,0.0909,0.0909,0.0884
55k,0.2158,0.1956,,0.1097,0.1607,0.1659,0.1662,0.1751,0.1704,0.0892,0.0898,0.0745,0.0931
60k,0.2039,0.2036,,0.1211,0.1654,0.1734,0.1612,0.1745,0.1801,0.0817,0.0850,0.0922,0.0986
65k,0.2244,0.2044,,0.1089,0.1573,0.1765,0.1693,0.1776,0.1823,0.0920,0.0967,0.1025,0.1066
70k,0.2233,0.2233,,0.1222,0.1634,0.1845,0.1679,0.1859,0.1767,0.1022,0.0925,0.1039,0.1177
75k,0.2305,0.2277,,0.1097,0.1709,0.1825,0.1881,0.1737,0.1762,0.1069,0.0936,0.1116,0.1199
80k,0.2457,0.2252,,0.1277,0.1573,0.1900,0.1776,0.1787,0.1964,0.1047,0.0981,0.1033,0.1097
85k,0.2501,0.2285,,0.1280,0.1776,0.1914,0.1889,0.1870,0.1889,0.0942,0.0964,0.1144,0.1213
90k,0.2504,0.2521,,0.1158,0.1598,0.1911,0.1806,0.1898,0.1773,0.1058,0.0964,0.1186,0.1163
95k,0.2579,0.2443,,0.1235,0.1762,0.1911,0.1781,0.1989,0.1917,0.1097,0.0928,0.1213,0.1169
100k,0.2526,0.2446,,0.1258,,0.2097,0.1928,0.1903,0.1947,0.1125,0.1025,0.1127,0.1188
105k,0.2679,0.2482,,0.1366,,0.2028,0.1814,0.1922,0.2094,0.1199,0.1069,0.1186,0.1269
110k,0.2717,0.2562,,0.1377,0.1756,0.2019,0.1859,0.1975,,0.1152,,0.1252,0.1252
115k,0.2745,0.2562,,0.1346,0.1831,0.1956,0.1947,0.1903,0.2119,0.1127,,0.1285,0.1111
120k,0.2801,0.2612,,0.1402,0.2014,0.2000,,0.2044,0.2119,0.1188,,0.1166,0.1219
125k,0.2751,0.2657,,0.1307,0.2030,0.2014,0.1992,0.2053,0.1787,0.1230,,0.1274,0.1418
130k,0.2884,0.2673,,0.1368,0.1997,0.2125,0.1994,0.2011,0.2086,0.1127,,,0.1335
135k,0.2842,0.2673,,0.1363,,0.2069,0.2014,0.2036,0.2069,0.1255,,,0.1299
140k,,0.2679,,0.1435,,0.2039,0.1986,0.2042,0.2058,0.1263,,,0.1299
145k,,,,0.1532,,0.2172,0.1953,0.2078,0.2102,0.1274,,,0.1443
150k,,,,0.1404,,0.2125,,0.2127,0.2075,0.1263,,,0.1410
155k,,,,0.1418,,0.2235,0.1931,0.2066,0.2205,0.1418,,,0.1460
160k,,,,0.1346,,0.2183,0.2116,0.2069,0.2208,0.1319,,,0.1413
165k,,,,0.1524,,0.2219,0.2139,,0.2213,0.1296,,,0.1424
170k,,,,0.1388,,0.2175,,,0.2169,0.1366,,,0.1454
175k,,,,0.1438,,0.2235,0.2222,,0.2321,0.1349,,,0.1399
180k,,,,0.1471,,0.2260,0.2249,,0.236,0.1465,,,0.1421
185k,,,,0.1499,,0.2341,0.2222,,0.2366,0.1449,,,0.1421
190k,,,,0.1504,,0.2233,,,0.2274,0.1413,,,0.1471
195k,,,,0.1554,,0.2330,,,0.2454,0.1440,,,0.1407
200k,,,,0.1565,,0.2238,,,0.2346,0.1407,,,0.1449
205k,,,,0.1726,,0.2271,,,0.2316,0.1382,,,0.1501
210k,,,,0.1623,,0.2305,,,0.2493,0.1526,,,0.1424
215k,,,,0.1576,,0.2299,,,0.2355,0.1518,,,0.1535
220k,,,,0.1693,,0.2330,,,0.2427,0.1529,,,
225k,,,,0.1596,,0.2366,,,0.2440,0.1479,,,
230k,,,,0.1693,,,,,0.2554,0.1560,,,
235k,,,,0.1720,,,,,0.2535,0.1540,,,
240k,,,,0.1712,,,,,,0.1554,,,
245k,,,,0.1704,,,,,,0.1532,,,
250k,,,,0.1784,,,,,,0.1551,,,
255k,,,,0.1740,,,,,,0.1623,,,
260k,,,,0.1756,,,,,,0.1618,,,
265k,,,,0.1886,,,,,,0.1604,,,
270k,,,,0.1820,,,,,,0.1612,,,
275k,,,,0.1870,,,,,,0.1629,,,
280k,,,,0.1704,,,,,,0.1645,,,
285k,,,,0.1903,,,,,,0.1665,,,
290k,,,,,,,,,,0.1648,,,
300k,,,,,,,,,,0.1712,,,
305k,,,,,,,,,,0.1690,,,
310k,,,,,,,,,,0.1712,,,
315k,,,,,,,,,,,,,
320k,,,,,,,,,,,,,
325k,,,,,,,,,,,,,
330k,,,,,,,,,,,,,
335k,,,,,,,,,,,,, |