Spaces:
Runtime error
Runtime error
| 5-shot,Slim-Pajama 600B (bsz=4K x 1024),,,FineWeb-1.5T,Ours-Base,Ours-Upsampling1,Ours-Upsampling2,Ours-Code-Upsampling2,All-Upsampling1,All-Upsampling1,All-Upsampling1,All-Upsampling1,DCLM-Base,Comments | |
| time: 20 min,Llama-8x8B-baseline,Llama-8x8B-seq8192,Llama-8x8B-mup,Llama-8x8B-seq8192,Llama-8x8B-seq8192,Llama-8x8B-seq8192,Llama-8x8B-seq8192,Llama-8x8B-seq8192,Llama-8x8B-seq8192,Llama-1x8B-seq8192,Llama_extend-1x8B-seq8192,Jais-1x8B-seq8192,Llama-1x8B-seq8192,"1. Comparing with upsample2-155k, the social science acc of dclm-195k is much higher" | |
| 5k,0.2398,0.2671,,,0.2579,0.2418,0.2482,0.2690,0.2456,0.2512,0.2532,0.2428,0.2530, | |
| 10k,0.2535,0.2520,,0.2594,0.2612,,0.2628,0.2319,0.2525,0.2462,0.2582,0.2713,0.2529, | |
| 15k,0.2527,0.2347,,,,0.2489,0.2334,0.2483,0.2503,0.2549,0.2380,0.2653,0.2507, | |
| 20k,0.2530,0.2478,,0.2495,0.2467,0.2677,0.2449,0.2507,0.2540,0.2416,0.2612,0.2482,0.2553, | |
| 25k,0.2503,0.2488,,,0.2431,0.2597,0.2571,0.2506,0.2534,0.2505,0.2464,0.2577,0.2552, | |
| 30k,0.2297,0.2539,,,,0.2592,0.2678,0.2389,0.2557,0.2468,0.2517,0.2485,0.2556, | |
| 35k,0.2356,0.2374,,0.2426,0.2591,0.2550,0.2562,0.2594,0.2494,0.2403,0.2451,0.2547,0.2443, | |
| 40k,0.2406,0.2462,,0.2467,0.2485,0.2344,0.2408,0.2555,0.2686,0.2552,0.2500,0.2553,0.2775, | |
| 45k,0.2470,0.2428,,0.2418,0.2296,0.2512,0.2712,0.2630,0.2503,0.2368,0.2536,0.2557,0.2393, | |
| 50k,0.2421,0.2368,,0.2382,0.2441,0.2727,0.2558,0.2558,0.2322,0.2499,0.2563,0.2305,0.2485, | |
| 55k,0.2460,0.2551,,0.2408,0.2536,0.2389,0.2440,0.2444,0.2747,0.2552,0.2516,0.2339,0.2595, | |
| 60k,0.2415,0.2397,,0.2718,0.2539,0.2518,0.2339,0.2551,0.2432,0.2517,0.2589,0.2379,0.2589, | |
| 65k,0.2490,0.2641,,0.2637,0.2423,0.2589,0.2342,0.2303,0.2478,0.2485,0.2643,0.2485,0.2798, | |
| 70k,0.2578,0.2641,,0.2534,0.2359,0.2716,0.2673,0.2307,0.2478,0.2483,0.2426,0.2499,0.2583, | |
| 75k,0.2587,0.2599,,0.2529,0.2372,0.2514,0.2579,0.2519,0.2478,0.2742,0.2594,0.2371,0.2653, | |
| 80k,0.2493,0.2519,,0.2504,0.2344,0.2582,0.2535,0.2433,0.2718,0.2596,0.2536,0.2553,0.2573, | |
| 85k,0.2527,0.2789,,0.2547,0.2496,0.2564,0.2418,0.2572,0.2465,0.2663,0.2552,0.2485,0.2584, | |
| 90k,0.2679,0.2668,,0.2595,0.2464,0.2608,0.2359,0.2777,0.2475,0.2543,0.2514,0.2411,0.2499, | |
| 95k,0.2551,0.2763,,0.2621,0.2469,0.2505,0.2534,0.2584,0.2424,0.2607,0.2742,0.2385,0.2521, | |
| 100k,0.2594,0.2564,,0.2550,,0.2614,0.2461,0.2611,0.2497,0.2675,0.2545,0.2540,0.2574, | |
| 105k,0.2787,0.2473,,0.2659,,0.2542,0.2729,0.2666,0.2468,0.2610,0.2726,0.2465,0.2798, | |
| 110k,0.3079,0.2458,,0.2551,0.2629,0.2512,0.2604,0.3027,0.2522,0.2673,,0.2410,0.2540, | |
| 115k,0.3185,0.2458,,0.2624,0.2324,0.2569,0.2590,0.2863,0.2584,0.2624,,0.2396,0.2771, | |
| 120k,0.3139,0.2832,,0.2626,0.2663,0.2718,0.2629,0.3190,0.2748,0.2419,,0.2544,0.2772, | |
| 125k,0.2960,0.2928,,0.2712,0.2733,0.2663,0.2768,0.2788,0.2570,0.2616,,0.2466,0.2856, | |
| 130k,0.3033,0.2844,,0.2404,0.2635,0.2767,0.2676,0.3191,0.2812,0.2538,,,0.2973, | |
| 135k,0.2934,0.2895,,0.2641,,0.2713,0.2735,0.3119,0.2882,0.2661,,,0.3203, | |
| 140k,,0.3045,,0.2553,,0.2811,0.2765,0.2866,0.3019,0.2730,,,0.2772, | |
| 145k,,,,0.2492,,0.2850,0.2708,0.3107,0.3090,0.2582,,,0.3435, | |
| 150k,,,,0.2595,,0.2780,,0.3225,0.3199,0.2541,,,0.3112, | |
| 155k,,,,0.2681,,0.2664,0.2463,0.3618,0.3116,0.2594,,,0.3361, | |
| 160k,,,,0.2605,,0.2793,0.2821,0.3047,0.3240,0.2688,,,0.3392, | |
| 165k,,,,0.2725,,0.2933,0.2816,,0.3478,0.2653,,,0.3485, | |
| 170k,,,,0.2514,,0.2656,0.2893,,0.3423,0.2537,,,0.3355, | |
| 175k,,,,0.2535,,0.3007,0.3317,,0.3156,0.2621,,,0.3162, | |
| 180k,,,,0.2561,,0.2785,0.2624,,0.2893,0.2555,,,0.3398, | |
| 185k,,,,0.2523,,0.3131,0.3026,,0.3876,0.2461,,,0.3631, | |
| 190k,,,,0.2653,,0.3226,,,0.3131,0.2540,,,0.3930, | |
| 195k,,,,0.2681,,0.3136,,,0.3473,0.2550,,,0.3972, | |
| 200k,,,,0.2515,,0.2811,,,0.3257,0.2481,,,0.3660, | |
| 205k,,,,0.2619,,0.3004,,,0.3836,,,,0.3748, | |
| 210k,,,,0.2687,,0.2996,,,0.3063,0.2646,,,0.3668, | |
| 215k,,,,0.2653,,0.3329,,,0.3947,0.2626,,,0.3641, | |
| 220k,,,,0.2631,,0.3590,,,0.3621,0.2600,,,, | |
| 225k,,,,0.2737,,0.3453,,,0.4151,0.2589,,,, | |
| 230k,,,,0.2833,,,,,0.3825,0.2587,,,, | |
| 235k,,,,0.2703,,,,,0.3897,,,,, | |
| 240k,,,,0.2572,,,,,,0.2610,,,, | |
| 245k,,,,0.2700,,,,,,0.2612,,,, | |
| 250k,,,,0.2639,,,,,,0.2583,,,, | |
| 255k,,,,0.2680,,,,,,0.2564,,,, | |
| 260k,,,,0.2897,,,,,,0.2631,,,, | |
| 265k,,,,0.2815,,,,,,0.2635,,,, | |
| 270k,,,,0.2693,,,,,,,,,, | |
| 275k,,,,0.2789,,,,,,0.2643,,,, | |
| 280k,,,,0.3052,,,,,,0.2687,,,, | |
| 285k,,,,0.2850,,,,,,0.2605,,,, | |
| 290k,,,,,,,,,,0.2779,,,, | |
| 300k,,,,,,,,,,0.2755,,,, | |
| 305k,,,,,,,,,,,,,, | |
| 310k,,,,,,,,,,0.2614,,,, | |
| 315k,,,,,,,,,,0.2646,,,, | |
| 320k,,,,,,,,,,0.2745,,,, | |
| 325k,,,,,,,,,,,,,, | |
| 330k,,,,,,,,,,,,,, | |
| 335k,,,,,,,,,,,,,, |