5-shot,Slim-Pajama 600B (bsz=4K x 1024),,,FineWeb-1.5T,Ours-Base,Ours-Upsampling1,Ours-Upsampling2,Ours-Code-Upsampling2,All-Upsampling1,All-Upsampling1,All-Upsampling1,All-Upsampling1,DCLM-Base,Comments
time: 20 min,Llama-8x8B-baseline,Llama-8x8B-seq8192,Llama-8x8B-mup,Llama-8x8B-seq8192,Llama-8x8B-seq8192,Llama-8x8B-seq8192,Llama-8x8B-seq8192,Llama-8x8B-seq8192,Llama-8x8B-seq8192,Llama-1x8B-seq8192,Llama_extend-1x8B-seq8192,Jais-1x8B-seq8192,Llama-1x8B-seq8192,"1. Comparing with upsample2-155k, the social science acc of dclm-195k is much higher"
5k,0.2398,0.2671,,,0.2579,0.2418,0.2482,0.2690,0.2456,0.2512,0.2532,0.2428,0.2530,
10k,0.2535,0.2520,,0.2594,0.2612,,0.2628,0.2319,0.2525,0.2462,0.2582,0.2713,0.2529,
15k,0.2527,0.2347,,,,0.2489,0.2334,0.2483,0.2503,0.2549,0.2380,0.2653,0.2507,
20k,0.2530,0.2478,,0.2495,0.2467,0.2677,0.2449,0.2507,0.2540,0.2416,0.2612,0.2482,0.2553,
25k,0.2503,0.2488,,,0.2431,0.2597,0.2571,0.2506,0.2534,0.2505,0.2464,0.2577,0.2552,
30k,0.2297,0.2539,,,,0.2592,0.2678,0.2389,0.2557,0.2468,0.2517,0.2485,0.2556,
35k,0.2356,0.2374,,0.2426,0.2591,0.2550,0.2562,0.2594,0.2494,0.2403,0.2451,0.2547,0.2443,
40k,0.2406,0.2462,,0.2467,0.2485,0.2344,0.2408,0.2555,0.2686,0.2552,0.2500,0.2553,0.2775,
45k,0.2470,0.2428,,0.2418,0.2296,0.2512,0.2712,0.2630,0.2503,0.2368,0.2536,0.2557,0.2393,
50k,0.2421,0.2368,,0.2382,0.2441,0.2727,0.2558,0.2558,0.2322,0.2499,0.2563,0.2305,0.2485,
55k,0.2460,0.2551,,0.2408,0.2536,0.2389,0.2440,0.2444,0.2747,0.2552,0.2516,0.2339,0.2595,
60k,0.2415,0.2397,,0.2718,0.2539,0.2518,0.2339,0.2551,0.2432,0.2517,0.2589,0.2379,0.2589,
65k,0.2490,0.2641,,0.2637,0.2423,0.2589,0.2342,0.2303,0.2478,0.2485,0.2643,0.2485,0.2798,
70k,0.2578,0.2641,,0.2534,0.2359,0.2716,0.2673,0.2307,0.2478,0.2483,0.2426,0.2499,0.2583,
75k,0.2587,0.2599,,0.2529,0.2372,0.2514,0.2579,0.2519,0.2478,0.2742,0.2594,0.2371,0.2653,
80k,0.2493,0.2519,,0.2504,0.2344,0.2582,0.2535,0.2433,0.2718,0.2596,0.2536,0.2553,0.2573,
85k,0.2527,0.2789,,0.2547,0.2496,0.2564,0.2418,0.2572,0.2465,0.2663,0.2552,0.2485,0.2584,
90k,0.2679,0.2668,,0.2595,0.2464,0.2608,0.2359,0.2777,0.2475,0.2543,0.2514,0.2411,0.2499,
95k,0.2551,0.2763,,0.2621,0.2469,0.2505,0.2534,0.2584,0.2424,0.2607,0.2742,0.2385,0.2521,
100k,0.2594,0.2564,,0.2550,,0.2614,0.2461,0.2611,0.2497,0.2675,0.2545,0.2540,0.2574,
105k,0.2787,0.2473,,0.2659,,0.2542,0.2729,0.2666,0.2468,0.2610,0.2726,0.2465,0.2798,
110k,0.3079,0.2458,,0.2551,0.2629,0.2512,0.2604,0.3027,0.2522,0.2673,,0.2410,0.2540,
115k,0.3185,0.2458,,0.2624,0.2324,0.2569,0.2590,0.2863,0.2584,0.2624,,0.2396,0.2771,
120k,0.3139,0.2832,,0.2626,0.2663,0.2718,0.2629,0.3190,0.2748,0.2419,,0.2544,0.2772,
125k,0.2960,0.2928,,0.2712,0.2733,0.2663,0.2768,0.2788,0.2570,0.2616,,0.2466,0.2856,
130k,0.3033,0.2844,,0.2404,0.2635,0.2767,0.2676,0.3191,0.2812,0.2538,,,0.2973,
135k,0.2934,0.2895,,0.2641,,0.2713,0.2735,0.3119,0.2882,0.2661,,,0.3203,
140k,,0.3045,,0.2553,,0.2811,0.2765,0.2866,0.3019,0.2730,,,0.2772,
145k,,,,0.2492,,0.2850,0.2708,0.3107,0.3090,0.2582,,,0.3435,
150k,,,,0.2595,,0.2780,,0.3225,0.3199,0.2541,,,0.3112,
155k,,,,0.2681,,0.2664,0.2463,0.3618,0.3116,0.2594,,,0.3361,
160k,,,,0.2605,,0.2793,0.2821,0.3047,0.3240,0.2688,,,0.3392,
165k,,,,0.2725,,0.2933,0.2816,,0.3478,0.2653,,,0.3485,
170k,,,,0.2514,,0.2656,0.2893,,0.3423,0.2537,,,0.3355,
175k,,,,0.2535,,0.3007,0.3317,,0.3156,0.2621,,,0.3162,
180k,,,,0.2561,,0.2785,0.2624,,0.2893,0.2555,,,0.3398,
185k,,,,0.2523,,0.3131,0.3026,,0.3876,0.2461,,,0.3631,
190k,,,,0.2653,,0.3226,,,0.3131,0.2540,,,0.3930,
195k,,,,0.2681,,0.3136,,,0.3473,0.2550,,,0.3972,
200k,,,,0.2515,,0.2811,,,0.3257,0.2481,,,0.3660,
205k,,,,0.2619,,0.3004,,,0.3836,,,,0.3748,
210k,,,,0.2687,,0.2996,,,0.3063,0.2646,,,0.3668,
215k,,,,0.2653,,0.3329,,,0.3947,0.2626,,,0.3641,
220k,,,,0.2631,,0.3590,,,0.3621,0.2600,,,,
225k,,,,0.2737,,0.3453,,,0.4151,0.2589,,,,
230k,,,,0.2833,,,,,0.3825,0.2587,,,,
235k,,,,0.2703,,,,,0.3897,,,,,
240k,,,,0.2572,,,,,,0.2610,,,,
245k,,,,0.2700,,,,,,0.2612,,,,
250k,,,,0.2639,,,,,,0.2583,,,,
255k,,,,0.2680,,,,,,0.2564,,,,
260k,,,,0.2897,,,,,,0.2631,,,,
265k,,,,0.2815,,,,,,0.2635,,,,
270k,,,,0.2693,,,,,,,,,,
275k,,,,0.2789,,,,,,0.2643,,,,
280k,,,,0.3052,,,,,,0.2687,,,,
285k,,,,0.2850,,,,,,0.2605,,,,
290k,,,,,,,,,,0.2779,,,,
300k,,,,,,,,,,0.2755,,,,
305k,,,,,,,,,,,,,,
310k,,,,,,,,,,0.2614,,,,
315k,,,,,,,,,,0.2646,,,,
320k,,,,,,,,,,0.2745,,,,
325k,,,,,,,,,,,,,,
330k,,,,,,,,,,,,,,
335k,,,,,,,,,,,,,,