File size: 4,264 Bytes
9a127b5 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 |
5-shot,Slim-Pajama 600B (bsz=4K x 1024),,,FineWeb-1.5T,Ours-Base,Ours-Upsampling1,Ours-Upsampling2,Ours-Code-Upsampling2,All-Upsampling1,All-Upsampling1,All-Upsampling1,All-Upsampling1,DCLM-Base
hf-time: 115 min,Llama-8x8B-baseline,Llama-8x8B-seq8192,Llama-8x8B-mup,Llama-8x8B-seq8192,Llama-8x8B-seq8192,Llama-8x8B-seq8192,Llama-8x8B-seq8192,Llama-8x8B-seq8192,Llama-8x8B-seq8192,Llama-1x8B-seq8192,Llama_extend-1x8B-seq8192,Jais-1x8B-seq8192,Llama-1x8B-seq8192
5k,0.0152,0.0099,,,0.0076,0.0015,0.0045,0.0030,,0.0152,0.0106,0.0197,0.0197
10k,0.0152,0.0190,,0.0015,,0.0091,0.0000,0.0212,0.0144,0.0159,0.0136,0.0174,0.0243
15k,0.0182,0.0167,,0.0053,0.0068,0.0045,0.0083,0.0212,0.0068,0.0174,0.0190,0.0174,0.0136
20k,0.0250,0.0212,,,,,0.0030,0.0159,0.0220,0.0167,0.0190,0.0220,0.0174
25k,0.0288,0.0114,,,,0.0129,0.0053,0.0258,0.0144,0.0152,0.0144,0.0144,0.0144
30k,0.0220,0.0265,,0.0197,0.0038,0.0152,0.0167,0.0227,0.0220,0.0205,0.0129,0.0167,0.0038
35k,0.0296,0.0212,,0.0136,0.0045,0.0190,0.0045,0.0227,0.0220,0.0174,0.0174,0.0243,0.0182
40k,0.0235,0.0288,,0.0068,0.0121,0.0220,0.0015,0.0243,0.0265,0.0152,0.0212,0.0190,0.0182
45k,0.0387,0.0250,,0.0258,0.0038,0.0273,0.0106,0.0296,0.0273,0.0182,0.0152,0.0174,0.0129
50k,0.0318,0.0303,,0.0015,0.0243,0.0227,0.0121,0.0190,0.0220,0.0197,0.0205,0.0182,0.0068
55k,0.0296,0.0311,,0.0023,0.0235,0.0235,0.0250,0.0326,0.0197,0.0182,0.0174,0.0250,0.0091
60k,0.0432,0.0326,,0.0167,0.0212,0.0212,0.0182,0.0349,0.0220,0.0182,0.0099,0.0190,0.0197
65k,0.0470,0.0379,,0.0015,0.0159,0.0281,0.0136,0.0296,0.0212,0.0212,0.0129,0.0205,0.0114
70k,0.0432,0.0417,,0.0136,0.0197,0.0174,0.0114,0.0341,0.0243,0.0205,0.0136,0.0250,0.0091
75k,0.0508,0.0470,,0.0174,0.0121,0.0250,0.0182,0.0356,0.0288,0.0281,0.0174,0.0190,0.0106
80k,0.0561,0.0417,,0.0068,0.0000,0.0190,0.0083,0.0318,0.0356,0.0273,0.0167,0.0265,0.0182
85k,0.0728,0.0341,,0.0341,0.0190,0.0296,0.0205,0.0265,0.0250,0.0220,0.0129,0.0235,0.0083
90k,0.0690,0.0425,,0.0197,0.0190,0.0281,0.0061,0.0417,0.0265,0.0273,0.0167,0.0190,0.0182
95k,0.0735,0.0447,,0.0167,0.0250,0.0281,0.0136,0.0349,0.0281,0.0174,0.0106,0.0288,0.0159
100k,0.0637,0.0470,,0.0159,,0.0227,0.0045,0.0409,0.0311,0.0265,0.0205,0.0190,0.0190
105k,0.0637,0.0447,,0.0341,,0.0303,0.0129,0.0371,0.0311,0.0273,0.0205,0.0311,0.0129
110k,0.0872,0.0576,,0.0038,0.0273,0.0129,0.0205,0.0478,0.0296,0.0212,,0.0281,0.0182
115k,0.0788,0.0576,,0.0091,0.0167,0.0311,0.0167,0.0508,0.0349,0.0220,,0.0220,0.0174
120k,0.0834,0.0455,,0.0227,0.0265,0.0167,0.0212,0.0371,0.0318,0.0167,,0.0220,0.0152
125k,0.1001,0.0493,,0.0288,0.0250,0.0205,0.0387,0.0402,0.0318,0.0182,,0.0235,0.0144
130k,0.0766,0.0470,,0.0068,0.0258,0.0288,0.0174,,0.0341,0.0243,,,0.0205
135k,0.0879,0.0607,,0.0190,,0.0349,0.0258,0.0409,0.0288,0.0212,,,0.0281
140k,,0.0569,,0.0379,,0.0356,0.0227,0.0440,0.0341,0.0144,,,0.0144
145k,,,,0.0341,,0.0379,0.0015,0.0387,,0.0174,,,0.0273
150k,,,,,,0.0281,,0.0470,0.0265,0.0220,,,0.0258
155k,,,,0.0318,,0.0303,0.0121,0.0561,0.0523,0.0227,,,0.0243
160k,,,,0.0356,,0.0243,0.0061,0.0425,0.0432,0.0220,,,0.0303
165k,,,,0.0167,,0.0409,0.0015,,0.0470,0.0281,,,
170k,,,,0.0334,,0.0281,0.0129,,0.0455,0.0273,,,0.0235
175k,,,,0.0371,,0.0326,0.0190,,0.0409,0.0190,,,0.0273
180k,,,,0.0425,,0.0364,0.0227,,0.0356,0.0243,,,0.0288
185k,,,,0.0341,,0.0318,0.0341,,0.0546,0.0235,,,0.0364
190k,,,,0.0296,,0.0364,,,0.0425,0.0220,,,0.0349
195k,,,,0.0250,,0.0303,,,0.0493,0.0258,,,
200k,,,,0.0250,,0.0371,,,0.0493,0.0273,,,0.0205
205k,,,,0.0455,,0.0409,,,0.0553,0.0220,,,0.0258
210k,,,,0.0462,,0.0371,,,0.0523,0.0281,,,
215k,,,,0.0349,,0.0265,,,0.0500,0.0235,,,0.0281
220k,,,,0.0432,,0.0167,,,0.0462,0.0326,,,
225k,,,,0.0447,,0.0212,,,,0.0265,,,
230k,,,,0.0440,,,,,0.0493,0.0273,,,
235k,,,,0.0402,,,,,0.0508,0.0220,,,
240k,,,,0.0341,,,,,,0.0281,,,
245k,,,,0.0462,,,,,,0.0356,,,
250k,,,,0.0500,,,,,,,,,
255k,,,,0.0569,,,,,,0.0303,,,
260k,,,,0.0500,,,,,,0.0334,,,
265k,,,,0.0455,,,,,,0.0318,,,
270k,,,,0.0538,,,,,,0.0273,,,
275k,,,,0.0470,,,,,,,,,
280k,,,,0.0553,,,,,,0.0364,,,
285k,,,,0.0531,,,,,,0.0349,,,
290k,,,,,,,,,,0.0311,,,
300k,,,,,,,,,,,,,
305k,,,,,,,,,,0.0311,,,
310k,,,,,,,,,,0.0273,,,
315k,,,,,,,,,,,,,
320k,,,,,,,,,,,,,
325k,,,,,,,,,,,,,
330k,,,,,,,,,,,,,
335k,,,,,,,,,,,,, |