5-shot,FineWeb-1.5T,Ours-Base,Ours-Upsampling2,All-Upsampling1
time: 76 min,Llama-8x8B-seq8192,Llama-8x8B-seq8192,Llama-8x8B-seq8192,Llama-8x8B-seq8192
5k,0.1025,0.1232,0.126,
10k,0.2073,,0.115,0.2604
15k,0.3005,,0.1872,0.3244
20k,0.3506,0.2795,0.2719,0.3637
25k,0.307,,0.4093,0.412
30k,0.2461,0.2974,0.4195,0.4294
35k,0.3639,0.3572,0.3587,0.4428
40k,0.3537,0.0346,0.4434,0.4623
45k,0.3602,0.2674,0.4366,0.4792
50k,0.2407,0.3689,0.4051,0.4795
55k,0.2081,0.4101,0.323,0.494
60k,0.4068,0.4107,0.4469,0.513
65k,0.3145,0.4477,0.4907,0.5087
70k,0.4102,0.4736,0.492,0.5129
75k,0.282,0.4226,0.2245,0.5042
80k,0.0975,0.4217,,0.5301
85k,0.0722,0.4763,0.5029,0.535
90k,0.3388,0.1472,0.0317,0.522
95k,0.5283,0.4938,0.518,0.5446
100k,0.4317,0.11,0.5358,0.5514
105k,0.1886,,0.5153,0.5562
110k,0.351,,0.5182,0.5654
115k,0.3692,0.4759,0.5132,0.5577
120k,0.369,0.4352,0.5483,0.5658
125k,0.3365,0.5206,0.5211,0.5658
130k,0.355,0.0088,0.5245,0.5609
135k,0.3892,,0.3977,0.5774
140k,0.393,,0.4991,0.5675
145k,0.4538,,0.4872,0.5639
150k,0.2883,,,0.5844
155k,0.4185,,0.1586,0.5755
160k,0.272,,0.563,0.5864
165k,0.4252,,0.5642,0.5853
170k,0.1507,,0.5739,
175k,0.3242,,0.564,0.5979
180k,0.2653,,0.5912,0.6054
185k,0.2651,,0.5852,0.6064
190k,0.238,,,0.5996
195k,0.4048,,,0.6243
200k,0.5058,,,0.6248
205k,0.0945,,,0.6224
210k,0.1557,,,0.6311
215k,0.2483,,,0.6293
220k,0.1725,,,0.6375
225k,0.2467,,,0.634
230k,0.1653,,,0.6436
235k,0.1884,,,0.6411
240k,0.0719,,,
245k,0.3757,,,
250k,0.5859,,,
255k,0.4987,,,
260k,0.394,,,
265k,0.3607,,,
270k,0.3898,,,
275k,0.4123,,,
280k,0.2413,,,
285k,0.3665,,,
290k,,,,
300k,,,,
305k,,,,
310k,,,,
315k,,,,
320k,,,,
325k,,,,
330k,,,,
335k,,,,