diff --git "a/checkpoint-25177/trainer_state.json" "b/checkpoint-25177/trainer_state.json" new file mode 100644--- /dev/null +++ "b/checkpoint-25177/trainer_state.json" @@ -0,0 +1,187422 @@ +{ + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 5.050383286332764, + "global_step": 25177, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.0, + "learning_rate": 9.92063492063492e-07, + "loss": 10.8952, + "theoretical_loss": 20.81281780154715, + "tokens_seen": 65536 + }, + { + "epoch": 0.0, + "learning_rate": 1.984126984126984e-06, + "loss": 10.8925, + "theoretical_loss": 17.566201104328645, + "tokens_seen": 131072 + }, + { + "epoch": 0.0, + "learning_rate": 2.9761904761904763e-06, + "loss": 10.8113, + "theoretical_loss": 15.939477092836569, + "tokens_seen": 196608 + }, + { + "epoch": 0.0, + "learning_rate": 3.968253968253968e-06, + "loss": 10.7096, + "theoretical_loss": 14.89231675598857, + "tokens_seen": 262144 + }, + { + "epoch": 0.0, + "learning_rate": 4.96031746031746e-06, + "loss": 10.516, + "theoretical_loss": 14.136216937762974, + "tokens_seen": 327680 + }, + { + "epoch": 0.0, + "learning_rate": 5.9523809523809525e-06, + "loss": 10.5507, + "theoretical_loss": 13.552561472550224, + "tokens_seen": 393216 + }, + { + "epoch": 0.0, + "learning_rate": 6.944444444444444e-06, + "loss": 10.1736, + "theoretical_loss": 13.08180900140119, + "tokens_seen": 458752 + }, + { + "epoch": 0.0, + "learning_rate": 7.936507936507936e-06, + "loss": 10.0954, + "theoretical_loss": 12.690129625483323, + "tokens_seen": 524288 + }, + { + "epoch": 0.0, + "learning_rate": 8.928571428571428e-06, + "loss": 9.9354, + "theoretical_loss": 12.356592463873625, + "tokens_seen": 589824 + }, + { + "epoch": 0.0, + "learning_rate": 9.92063492063492e-06, + "loss": 9.9176, + "theoretical_loss": 12.067412607035077, + "tokens_seen": 655360 + }, + { + "epoch": 0.0, + "learning_rate": 1.0912698412698412e-05, + "loss": 9.8329, + "theoretical_loss": 11.813066231101676, + "tokens_seen": 720896 + }, + { + "epoch": 0.0, + "learning_rate": 1.1904761904761905e-05, + "loss": 9.5392, + "theoretical_loss": 11.586719208706729, + "tokens_seen": 786432 + }, + { + "epoch": 0.0, + "learning_rate": 1.2896825396825396e-05, + "loss": 9.6937, + "theoretical_loss": 11.383314140186787, + "tokens_seen": 851968 + }, + { + "epoch": 0.0, + "learning_rate": 1.3888888888888888e-05, + "loss": 9.4581, + "theoretical_loss": 11.199011702111871, + "tokens_seen": 917504 + }, + { + "epoch": 0.0, + "learning_rate": 1.4880952380952381e-05, + "loss": 9.4229, + "theoretical_loss": 11.030833917977912, + "tokens_seen": 983040 + }, + { + "epoch": 0.0, + "learning_rate": 1.5873015873015872e-05, + "loss": 9.4341, + "theoretical_loss": 10.87642808645695, + "tokens_seen": 1048576 + }, + { + "epoch": 0.0, + "learning_rate": 1.6865079365079364e-05, + "loss": 9.3235, + "theoretical_loss": 10.733905740062724, + "tokens_seen": 1114112 + }, + { + "epoch": 0.0, + "learning_rate": 1.7857142857142855e-05, + "loss": 9.2909, + "theoretical_loss": 10.60172987623028, + "tokens_seen": 1179648 + }, + { + "epoch": 0.0, + "learning_rate": 1.884920634920635e-05, + "loss": 9.2573, + "theoretical_loss": 10.478634172356642, + "tokens_seen": 1245184 + }, + { + "epoch": 0.0, + "learning_rate": 1.984126984126984e-05, + "loss": 9.159, + "theoretical_loss": 10.36356394376333, + "tokens_seen": 1310720 + }, + { + "epoch": 0.0, + "learning_rate": 2.0833333333333333e-05, + "loss": 9.0708, + "theoretical_loss": 10.255632220896747, + "tokens_seen": 1376256 + }, + { + "epoch": 0.0, + "learning_rate": 2.1825396825396824e-05, + "loss": 9.2882, + "theoretical_loss": 10.15408655327002, + "tokens_seen": 1441792 + }, + { + "epoch": 0.0, + "learning_rate": 2.2817460317460315e-05, + "loss": 8.981, + "theoretical_loss": 10.058283561732598, + "tokens_seen": 1507328 + }, + { + "epoch": 0.0, + "learning_rate": 2.380952380952381e-05, + "loss": 9.1814, + "theoretical_loss": 9.967669178840278, + "tokens_seen": 1572864 + }, + { + "epoch": 0.0, + "objective/train/docs_used": 67480, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 9.154304504394531, + "objective/train/theoretical_loss": 9.881763126393109, + "objective/train/tokens_used": 22098400, + "theoretical_loss": 9.881763126393109, + "tokens_seen": 1638400 + }, + { + "epoch": 0.0, + "learning_rate": 2.48015873015873e-05, + "loss": 9.2744, + "theoretical_loss": 9.881763126393109, + "tokens_seen": 1638400 + }, + { + "epoch": 0.0, + "learning_rate": 2.5793650793650793e-05, + "loss": 9.1924, + "theoretical_loss": 9.80014659154056, + "tokens_seen": 1703936 + }, + { + "epoch": 0.0, + "learning_rate": 2.6785714285714284e-05, + "loss": 9.1481, + "theoretical_loss": 9.722452346907446, + "tokens_seen": 1769472 + }, + { + "epoch": 0.0, + "learning_rate": 2.7777777777777776e-05, + "loss": 8.8718, + "theoretical_loss": 9.648356759081546, + "tokens_seen": 1835008 + }, + { + "epoch": 0.0, + "learning_rate": 2.876984126984127e-05, + "loss": 9.0305, + "theoretical_loss": 9.577573271145639, + "tokens_seen": 1900544 + }, + { + "epoch": 0.0, + "learning_rate": 2.9761904761904762e-05, + "loss": 8.9527, + "theoretical_loss": 9.509847046764852, + "tokens_seen": 1966080 + }, + { + "epoch": 0.0, + "learning_rate": 3.075396825396825e-05, + "loss": 8.8801, + "theoretical_loss": 9.444950537631936, + "tokens_seen": 2031616 + }, + { + "epoch": 0.0, + "learning_rate": 3.1746031746031745e-05, + "loss": 8.7157, + "theoretical_loss": 9.382679790910457, + "tokens_seen": 2097152 + }, + { + "epoch": 0.0, + "learning_rate": 3.273809523809524e-05, + "loss": 8.6865, + "theoretical_loss": 9.32285135423398, + "tokens_seen": 2162688 + }, + { + "epoch": 0.0, + "learning_rate": 3.373015873015873e-05, + "loss": 8.7653, + "theoretical_loss": 9.265299666660276, + "tokens_seen": 2228224 + }, + { + "epoch": 0.0, + "learning_rate": 3.472222222222222e-05, + "loss": 8.6092, + "theoretical_loss": 9.209874847444755, + "tokens_seen": 2293760 + }, + { + "epoch": 0.0, + "learning_rate": 3.571428571428571e-05, + "loss": 8.7534, + "theoretical_loss": 9.156440812508292, + "tokens_seen": 2359296 + }, + { + "epoch": 0.0, + "learning_rate": 3.670634920634921e-05, + "loss": 8.5526, + "theoretical_loss": 9.10487366241335, + "tokens_seen": 2424832 + }, + { + "epoch": 0.0, + "learning_rate": 3.76984126984127e-05, + "loss": 8.4992, + "theoretical_loss": 9.055060296533734, + "tokens_seen": 2490368 + }, + { + "epoch": 0.0, + "learning_rate": 3.8690476190476195e-05, + "loss": 8.5932, + "theoretical_loss": 9.006897216643829, + "tokens_seen": 2555904 + }, + { + "epoch": 0.0, + "learning_rate": 3.968253968253968e-05, + "loss": 8.5997, + "theoretical_loss": 8.960289489909357, + "tokens_seen": 2621440 + }, + { + "epoch": 0.0, + "learning_rate": 4.067460317460318e-05, + "loss": 8.3689, + "theoretical_loss": 8.915149846640611, + "tokens_seen": 2686976 + }, + { + "epoch": 0.0, + "learning_rate": 4.1666666666666665e-05, + "loss": 8.3045, + "theoretical_loss": 8.871397892478225, + "tokens_seen": 2752512 + }, + { + "epoch": 0.0, + "learning_rate": 4.265873015873016e-05, + "loss": 8.3696, + "theoretical_loss": 8.828959418153499, + "tokens_seen": 2818048 + }, + { + "epoch": 0.0, + "learning_rate": 4.365079365079365e-05, + "loss": 8.3294, + "theoretical_loss": 8.787765792778412, + "tokens_seen": 2883584 + }, + { + "epoch": 0.0, + "learning_rate": 4.464285714285714e-05, + "loss": 7.9479, + "theoretical_loss": 8.747753428911455, + "tokens_seen": 2949120 + }, + { + "epoch": 0.0, + "learning_rate": 4.563492063492063e-05, + "loss": 7.9269, + "theoretical_loss": 8.708863309520833, + "tokens_seen": 3014656 + }, + { + "epoch": 0.0, + "learning_rate": 4.6626984126984126e-05, + "loss": 7.9812, + "theoretical_loss": 8.671040568508847, + "tokens_seen": 3080192 + }, + { + "epoch": 0.0, + "learning_rate": 4.761904761904762e-05, + "loss": 8.0261, + "theoretical_loss": 8.634234117735474, + "tokens_seen": 3145728 + }, + { + "epoch": 0.0, + "learning_rate": 4.8611111111111115e-05, + "loss": 8.2961, + "theoretical_loss": 8.598396314536323, + "tokens_seen": 3211264 + }, + { + "epoch": 0.0, + "objective/train/docs_used": 72733, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 8.41878604888916, + "objective/train/theoretical_loss": 8.563482664611069, + "objective/train/tokens_used": 23736800, + "theoretical_loss": 8.563482664611069, + "tokens_seen": 3276800 + }, + { + "epoch": 0.0, + "learning_rate": 4.96031746031746e-05, + "loss": 8.2388, + "theoretical_loss": 8.563482664611069, + "tokens_seen": 3276800 + }, + { + "epoch": 0.0, + "learning_rate": 5.05952380952381e-05, + "loss": 7.8592, + "theoretical_loss": 8.529451555895115, + "tokens_seen": 3342336 + }, + { + "epoch": 0.0, + "learning_rate": 5.1587301587301586e-05, + "loss": 7.7882, + "theoretical_loss": 8.496264019646002, + "tokens_seen": 3407872 + }, + { + "epoch": 0.0, + "learning_rate": 5.257936507936508e-05, + "loss": 8.0092, + "theoretical_loss": 8.463883515497187, + "tokens_seen": 3473408 + }, + { + "epoch": 0.0, + "learning_rate": 5.357142857142857e-05, + "loss": 7.8125, + "theoretical_loss": 8.432275737672779, + "tokens_seen": 3538944 + }, + { + "epoch": 0.0, + "learning_rate": 5.4563492063492063e-05, + "loss": 8.1055, + "theoretical_loss": 8.401408439930716, + "tokens_seen": 3604480 + }, + { + "epoch": 0.0, + "learning_rate": 5.555555555555555e-05, + "loss": 7.6534, + "theoretical_loss": 8.371251277120209, + "tokens_seen": 3670016 + }, + { + "epoch": 0.0, + "learning_rate": 5.6547619047619046e-05, + "loss": 7.8082, + "theoretical_loss": 8.341775661511075, + "tokens_seen": 3735552 + }, + { + "epoch": 0.0, + "learning_rate": 5.753968253968254e-05, + "loss": 7.7956, + "theoretical_loss": 8.31295463228533, + "tokens_seen": 3801088 + }, + { + "epoch": 0.0, + "learning_rate": 5.8531746031746036e-05, + "loss": 7.7599, + "theoretical_loss": 8.284762736781182, + "tokens_seen": 3866624 + }, + { + "epoch": 0.0, + "learning_rate": 5.9523809523809524e-05, + "loss": 7.683, + "theoretical_loss": 8.257175922251864, + "tokens_seen": 3932160 + }, + { + "epoch": 0.0, + "learning_rate": 6.051587301587302e-05, + "loss": 7.5917, + "theoretical_loss": 8.230171437050114, + "tokens_seen": 3997696 + }, + { + "epoch": 0.0, + "learning_rate": 6.15079365079365e-05, + "loss": 7.6448, + "theoretical_loss": 8.20372774027797, + "tokens_seen": 4063232 + }, + { + "epoch": 0.0, + "learning_rate": 6.25e-05, + "loss": 7.5636, + "theoretical_loss": 8.177824419053046, + "tokens_seen": 4128768 + }, + { + "epoch": 0.0, + "learning_rate": 6.349206349206349e-05, + "loss": 7.3527, + "theoretical_loss": 8.152442112639616, + "tokens_seen": 4194304 + }, + { + "epoch": 0.0, + "learning_rate": 6.448412698412699e-05, + "loss": 7.6386, + "theoretical_loss": 8.1275624427775, + "tokens_seen": 4259840 + }, + { + "epoch": 0.0, + "learning_rate": 6.547619047619048e-05, + "loss": 7.4188, + "theoretical_loss": 8.10316794961571, + "tokens_seen": 4325376 + }, + { + "epoch": 0.0, + "learning_rate": 6.646825396825397e-05, + "loss": 7.5008, + "theoretical_loss": 8.07924203272264, + "tokens_seen": 4390912 + }, + { + "epoch": 0.0, + "learning_rate": 6.746031746031745e-05, + "loss": 7.5319, + "theoretical_loss": 8.055768896701416, + "tokens_seen": 4456448 + }, + { + "epoch": 0.0, + "learning_rate": 6.845238095238096e-05, + "loss": 7.048, + "theoretical_loss": 8.032733500989007, + "tokens_seen": 4521984 + }, + { + "epoch": 0.0, + "learning_rate": 6.944444444444444e-05, + "loss": 7.2613, + "theoretical_loss": 8.010121513461836, + "tokens_seen": 4587520 + }, + { + "epoch": 0.0, + "learning_rate": 7.043650793650793e-05, + "loss": 7.2432, + "theoretical_loss": 7.987919267509379, + "tokens_seen": 4653056 + }, + { + "epoch": 0.0, + "learning_rate": 7.142857142857142e-05, + "loss": 7.2, + "theoretical_loss": 7.966113722271801, + "tokens_seen": 4718592 + }, + { + "epoch": 0.0, + "learning_rate": 7.242063492063492e-05, + "loss": 7.2711, + "theoretical_loss": 7.944692425767988, + "tokens_seen": 4784128 + }, + { + "epoch": 0.0, + "learning_rate": 7.341269841269842e-05, + "loss": 7.2298, + "theoretical_loss": 7.9236434806675184, + "tokens_seen": 4849664 + }, + { + "epoch": 0.0, + "objective/train/docs_used": 77606, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 7.1540398597717285, + "objective/train/theoretical_loss": 7.902955512484067, + "objective/train/tokens_used": 25375200, + "theoretical_loss": 7.902955512484067, + "tokens_seen": 4915200 + }, + { + "epoch": 0.0, + "learning_rate": 7.440476190476191e-05, + "loss": 7.0498, + "theoretical_loss": 7.902955512484067, + "tokens_seen": 4915200 + }, + { + "epoch": 0.0, + "learning_rate": 7.53968253968254e-05, + "loss": 6.8919, + "theoretical_loss": 7.882617639989203, + "tokens_seen": 4980736 + }, + { + "epoch": 0.0, + "learning_rate": 7.63888888888889e-05, + "loss": 7.1864, + "theoretical_loss": 7.862619447664628, + "tokens_seen": 5046272 + }, + { + "epoch": 0.0, + "learning_rate": 7.738095238095239e-05, + "loss": 7.153, + "theoretical_loss": 7.842950960027937, + "tokens_seen": 5111808 + }, + { + "epoch": 0.0, + "learning_rate": 7.837301587301588e-05, + "loss": 7.0043, + "theoretical_loss": 7.823602617682313, + "tokens_seen": 5177344 + }, + { + "epoch": 0.0, + "learning_rate": 7.936507936507937e-05, + "loss": 7.067, + "theoretical_loss": 7.804565254954165, + "tokens_seen": 5242880 + }, + { + "epoch": 0.0, + "learning_rate": 8.035714285714287e-05, + "loss": 7.0251, + "theoretical_loss": 7.7858300789950725, + "tokens_seen": 5308416 + }, + { + "epoch": 0.0, + "learning_rate": 8.134920634920635e-05, + "loss": 6.7699, + "theoretical_loss": 7.767388650235364, + "tokens_seen": 5373952 + }, + { + "epoch": 0.0, + "learning_rate": 8.234126984126984e-05, + "loss": 6.8388, + "theoretical_loss": 7.749232864086619, + "tokens_seen": 5439488 + }, + { + "epoch": 0.0, + "learning_rate": 8.333333333333333e-05, + "loss": 6.9893, + "theoretical_loss": 7.731354933799318, + "tokens_seen": 5505024 + }, + { + "epoch": 0.0, + "learning_rate": 8.432539682539683e-05, + "loss": 6.819, + "theoretical_loss": 7.71374737438992, + "tokens_seen": 5570560 + }, + { + "epoch": 0.0, + "learning_rate": 8.531746031746032e-05, + "loss": 6.6608, + "theoretical_loss": 7.696402987558934, + "tokens_seen": 5636096 + }, + { + "epoch": 0.0, + "learning_rate": 8.630952380952381e-05, + "loss": 6.5322, + "theoretical_loss": 7.679314847528181, + "tokens_seen": 5701632 + }, + { + "epoch": 0.0, + "learning_rate": 8.73015873015873e-05, + "loss": 6.7129, + "theoretical_loss": 7.662476287731328, + "tokens_seen": 5767168 + }, + { + "epoch": 0.0, + "learning_rate": 8.82936507936508e-05, + "loss": 6.7745, + "theoretical_loss": 7.645880888297279, + "tokens_seen": 5832704 + }, + { + "epoch": 0.0, + "learning_rate": 8.928571428571429e-05, + "loss": 6.8321, + "theoretical_loss": 7.629522464270861, + "tokens_seen": 5898240 + }, + { + "epoch": 0.0, + "learning_rate": 9.027777777777777e-05, + "loss": 6.6706, + "theoretical_loss": 7.613395054519696, + "tokens_seen": 5963776 + }, + { + "epoch": 0.0, + "learning_rate": 9.126984126984126e-05, + "loss": 6.6371, + "theoretical_loss": 7.59749291128028, + "tokens_seen": 6029312 + }, + { + "epoch": 0.0, + "learning_rate": 9.226190476190476e-05, + "loss": 6.6738, + "theoretical_loss": 7.581810490299888, + "tokens_seen": 6094848 + }, + { + "epoch": 0.0, + "learning_rate": 9.325396825396825e-05, + "loss": 6.6876, + "theoretical_loss": 7.5663424415343705, + "tokens_seen": 6160384 + }, + { + "epoch": 0.0, + "learning_rate": 9.424603174603175e-05, + "loss": 6.5013, + "theoretical_loss": 7.551083600364949, + "tokens_seen": 6225920 + }, + { + "epoch": 0.0, + "learning_rate": 9.523809523809524e-05, + "loss": 6.5264, + "theoretical_loss": 7.536028979299919, + "tokens_seen": 6291456 + }, + { + "epoch": 0.0, + "learning_rate": 9.623015873015874e-05, + "loss": 6.4506, + "theoretical_loss": 7.521173760129762, + "tokens_seen": 6356992 + }, + { + "epoch": 0.0, + "learning_rate": 9.722222222222223e-05, + "loss": 6.4093, + "theoretical_loss": 7.506513286506497, + "tokens_seen": 6422528 + }, + { + "epoch": 0.0, + "learning_rate": 9.821428571428572e-05, + "loss": 6.3955, + "theoretical_loss": 7.492043056920249, + "tokens_seen": 6488064 + }, + { + "epoch": 0.0, + "objective/train/docs_used": 82513, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 6.486067771911621, + "objective/train/theoretical_loss": 7.4777587180480305, + "objective/train/tokens_used": 27013600, + "theoretical_loss": 7.4777587180480305, + "tokens_seen": 6553600 + }, + { + "epoch": 0.0, + "learning_rate": 9.92063492063492e-05, + "loss": 6.6458, + "theoretical_loss": 7.4777587180480305, + "tokens_seen": 6553600 + }, + { + "epoch": 0.0, + "learning_rate": 0.00010019841269841271, + "loss": 6.6237, + "theoretical_loss": 7.463656058451462, + "tokens_seen": 6619136 + }, + { + "epoch": 0.0, + "learning_rate": 0.0001011904761904762, + "loss": 6.5474, + "theoretical_loss": 7.449731002601916, + "tokens_seen": 6684672 + }, + { + "epoch": 0.0, + "learning_rate": 0.00010218253968253968, + "loss": 6.3712, + "theoretical_loss": 7.435979605213019, + "tokens_seen": 6750208 + }, + { + "epoch": 0.0, + "learning_rate": 0.00010317460317460317, + "loss": 6.7222, + "theoretical_loss": 7.422398045861905, + "tokens_seen": 6815744 + }, + { + "epoch": 0.0, + "learning_rate": 0.00010416666666666667, + "loss": 6.7042, + "theoretical_loss": 7.408982623881875, + "tokens_seen": 6881280 + }, + { + "epoch": 0.0, + "learning_rate": 0.00010515873015873016, + "loss": 6.5424, + "theoretical_loss": 7.395729753510345, + "tokens_seen": 6946816 + }, + { + "epoch": 0.0, + "learning_rate": 0.00010615079365079365, + "loss": 6.4443, + "theoretical_loss": 7.3826359592770325, + "tokens_seen": 7012352 + }, + { + "epoch": 0.0, + "learning_rate": 0.00010714285714285714, + "loss": 6.4673, + "theoretical_loss": 7.369697871618373, + "tokens_seen": 7077888 + }, + { + "epoch": 0.0, + "learning_rate": 0.00010813492063492064, + "loss": 6.7002, + "theoretical_loss": 7.3569122227050885, + "tokens_seen": 7143424 + }, + { + "epoch": 0.0, + "learning_rate": 0.00010912698412698413, + "loss": 6.2387, + "theoretical_loss": 7.3442758424706875, + "tokens_seen": 7208960 + }, + { + "epoch": 0.0, + "learning_rate": 0.00011011904761904761, + "loss": 6.3967, + "theoretical_loss": 7.331785654829519, + "tokens_seen": 7274496 + }, + { + "epoch": 0.0, + "learning_rate": 0.0001111111111111111, + "loss": 6.3247, + "theoretical_loss": 7.319438674073677, + "tokens_seen": 7340032 + }, + { + "epoch": 0.0, + "learning_rate": 0.0001121031746031746, + "loss": 6.4478, + "theoretical_loss": 7.307232001438824, + "tokens_seen": 7405568 + }, + { + "epoch": 0.0, + "learning_rate": 0.00011309523809523809, + "loss": 6.3822, + "theoretical_loss": 7.295162821829564, + "tokens_seen": 7471104 + }, + { + "epoch": 0.0, + "learning_rate": 0.00011408730158730158, + "loss": 6.57, + "theoretical_loss": 7.283228400695652, + "tokens_seen": 7536640 + }, + { + "epoch": 0.0, + "learning_rate": 0.00011507936507936508, + "loss": 6.351, + "theoretical_loss": 7.271426081050832, + "tokens_seen": 7602176 + }, + { + "epoch": 0.0, + "learning_rate": 0.00011607142857142858, + "loss": 6.1891, + "theoretical_loss": 7.259753280626623, + "tokens_seen": 7667712 + }, + { + "epoch": 0.0, + "learning_rate": 0.00011706349206349207, + "loss": 6.2692, + "theoretical_loss": 7.24820748915387, + "tokens_seen": 7733248 + }, + { + "epoch": 0.0, + "learning_rate": 0.00011805555555555556, + "loss": 6.3516, + "theoretical_loss": 7.236786265765262, + "tokens_seen": 7798784 + }, + { + "epoch": 0.0, + "learning_rate": 0.00011904761904761905, + "loss": 6.1213, + "theoretical_loss": 7.225487236512497, + "tokens_seen": 7864320 + }, + { + "epoch": 0.0, + "learning_rate": 0.00012003968253968255, + "loss": 6.3916, + "theoretical_loss": 7.21430809199212, + "tokens_seen": 7929856 + }, + { + "epoch": 0.0, + "learning_rate": 0.00012103174603174604, + "loss": 6.4588, + "theoretical_loss": 7.2032465850744005, + "tokens_seen": 7995392 + }, + { + "epoch": 0.0, + "learning_rate": 0.00012202380952380953, + "loss": 6.1372, + "theoretical_loss": 7.192300528730015, + "tokens_seen": 8060928 + }, + { + "epoch": 0.0, + "learning_rate": 0.000123015873015873, + "loss": 6.2326, + "theoretical_loss": 7.1814677939495155, + "tokens_seen": 8126464 + }, + { + "epoch": 0.0, + "objective/train/docs_used": 87567, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 6.155417442321777, + "objective/train/theoretical_loss": 7.1707463077509646, + "objective/train/tokens_used": 28652000, + "theoretical_loss": 7.1707463077509646, + "tokens_seen": 8192000 + }, + { + "epoch": 0.0, + "learning_rate": 0.0001240079365079365, + "loss": 6.3427, + "theoretical_loss": 7.1707463077509646, + "tokens_seen": 8192000 + }, + { + "epoch": 0.0, + "learning_rate": 0.000125, + "loss": 6.1035, + "theoretical_loss": 7.160134051271272, + "tokens_seen": 8257536 + }, + { + "epoch": 0.0, + "learning_rate": 0.0001259920634920635, + "loss": 6.2199, + "theoretical_loss": 7.149629057937138, + "tokens_seen": 8323072 + }, + { + "epoch": 0.0, + "learning_rate": 0.00012698412698412698, + "loss": 6.1004, + "theoretical_loss": 7.139229411711638, + "tokens_seen": 8388608 + }, + { + "epoch": 0.0, + "learning_rate": 0.00012797619047619048, + "loss": 6.084, + "theoretical_loss": 7.128933245412794, + "tokens_seen": 8454144 + }, + { + "epoch": 0.0, + "learning_rate": 0.00012896825396825398, + "loss": 5.9073, + "theoretical_loss": 7.118738739100616, + "tokens_seen": 8519680 + }, + { + "epoch": 0.0, + "learning_rate": 0.00012996031746031748, + "loss": 6.1843, + "theoretical_loss": 7.1086441185293445, + "tokens_seen": 8585216 + }, + { + "epoch": 0.0, + "learning_rate": 0.00013095238095238096, + "loss": 5.9909, + "theoretical_loss": 7.09864765366177, + "tokens_seen": 8650752 + }, + { + "epoch": 0.0, + "learning_rate": 0.00013194444444444446, + "loss": 5.8824, + "theoretical_loss": 7.088747657242693, + "tokens_seen": 8716288 + }, + { + "epoch": 0.0, + "learning_rate": 0.00013293650793650793, + "loss": 6.1145, + "theoretical_loss": 7.078942483428749, + "tokens_seen": 8781824 + }, + { + "epoch": 0.0, + "learning_rate": 0.00013392857142857144, + "loss": 6.1675, + "theoretical_loss": 7.069230526471966, + "tokens_seen": 8847360 + }, + { + "epoch": 0.0, + "learning_rate": 0.0001349206349206349, + "loss": 6.1415, + "theoretical_loss": 7.059610219454568, + "tokens_seen": 8912896 + }, + { + "epoch": 0.0, + "learning_rate": 0.0001359126984126984, + "loss": 6.1579, + "theoretical_loss": 7.0500800330726685, + "tokens_seen": 8978432 + }, + { + "epoch": 0.0, + "learning_rate": 0.0001369047619047619, + "loss": 6.2736, + "theoretical_loss": 7.040638474466625, + "tokens_seen": 9043968 + }, + { + "epoch": 0.0, + "learning_rate": 0.00013789682539682541, + "loss": 6.0392, + "theoretical_loss": 7.031284086095933, + "tokens_seen": 9109504 + }, + { + "epoch": 0.0, + "learning_rate": 0.0001388888888888889, + "loss": 6.2167, + "theoretical_loss": 7.022015444656678, + "tokens_seen": 9175040 + }, + { + "epoch": 0.0, + "learning_rate": 0.0001398809523809524, + "loss": 5.9301, + "theoretical_loss": 7.012831160039609, + "tokens_seen": 9240576 + }, + { + "epoch": 0.0, + "learning_rate": 0.00014087301587301586, + "loss": 5.9091, + "theoretical_loss": 7.003729874327071, + "tokens_seen": 9306112 + }, + { + "epoch": 0.0, + "learning_rate": 0.00014186507936507937, + "loss": 6.1878, + "theoretical_loss": 6.994710260827057, + "tokens_seen": 9371648 + }, + { + "epoch": 0.0, + "learning_rate": 0.00014285714285714284, + "loss": 5.9379, + "theoretical_loss": 6.98577102314278, + "tokens_seen": 9437184 + }, + { + "epoch": 0.0, + "learning_rate": 0.00014384920634920634, + "loss": 5.8658, + "theoretical_loss": 6.976910894276189, + "tokens_seen": 9502720 + }, + { + "epoch": 0.0, + "learning_rate": 0.00014484126984126984, + "loss": 5.9478, + "theoretical_loss": 6.968128635764015, + "tokens_seen": 9568256 + }, + { + "epoch": 0.0, + "learning_rate": 0.00014583333333333335, + "loss": 6.1221, + "theoretical_loss": 6.959423036844894, + "tokens_seen": 9633792 + }, + { + "epoch": 0.0, + "learning_rate": 0.00014682539682539685, + "loss": 5.9375, + "theoretical_loss": 6.950792913656309, + "tokens_seen": 9699328 + }, + { + "epoch": 0.0, + "learning_rate": 0.00014781746031746032, + "loss": 5.9392, + "theoretical_loss": 6.942237108460029, + "tokens_seen": 9764864 + }, + { + "epoch": 0.0, + "objective/train/docs_used": 92738, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 5.9081549644470215, + "objective/train/theoretical_loss": 6.9337544888949, + "objective/train/tokens_used": 30290400, + "theoretical_loss": 6.9337544888949, + "tokens_seen": 9830400 + }, + { + "epoch": 0.0, + "learning_rate": 0.00014880952380952382, + "loss": 5.9356, + "theoretical_loss": 6.9337544888949, + "tokens_seen": 9830400 + }, + { + "epoch": 0.0, + "learning_rate": 0.0001498015873015873, + "loss": 6.1536, + "theoretical_loss": 6.925343947255817, + "tokens_seen": 9895936 + }, + { + "epoch": 0.0, + "learning_rate": 0.0001507936507936508, + "loss": 5.9417, + "theoretical_loss": 6.917004399797798, + "tokens_seen": 9961472 + }, + { + "epoch": 0.0, + "learning_rate": 0.00015178571428571427, + "loss": 6.0151, + "theoretical_loss": 6.908734786064147, + "tokens_seen": 10027008 + }, + { + "epoch": 0.0, + "learning_rate": 0.0001527777777777778, + "loss": 6.0507, + "theoretical_loss": 6.900534068237688, + "tokens_seen": 10092544 + }, + { + "epoch": 0.0, + "learning_rate": 0.00015376984126984128, + "loss": 5.9881, + "theoretical_loss": 6.89240123051416, + "tokens_seen": 10158080 + }, + { + "epoch": 0.0, + "learning_rate": 0.00015476190476190478, + "loss": 6.0754, + "theoretical_loss": 6.884335278496871, + "tokens_seen": 10223616 + }, + { + "epoch": 0.0, + "learning_rate": 0.00015575396825396825, + "loss": 5.7266, + "theoretical_loss": 6.87633523861175, + "tokens_seen": 10289152 + }, + { + "epoch": 0.0, + "learning_rate": 0.00015674603174603175, + "loss": 5.9508, + "theoretical_loss": 6.868400157541997, + "tokens_seen": 10354688 + }, + { + "epoch": 0.0, + "learning_rate": 0.00015773809523809523, + "loss": 5.7484, + "theoretical_loss": 6.860529101681551, + "tokens_seen": 10420224 + }, + { + "epoch": 0.0, + "learning_rate": 0.00015873015873015873, + "loss": 5.9385, + "theoretical_loss": 6.85272115660663, + "tokens_seen": 10485760 + }, + { + "epoch": 0.0, + "learning_rate": 0.0001597222222222222, + "loss": 5.9719, + "theoretical_loss": 6.844975426564642, + "tokens_seen": 10551296 + }, + { + "epoch": 0.0, + "learning_rate": 0.00016071428571428573, + "loss": 5.9549, + "theoretical_loss": 6.8372910339797945, + "tokens_seen": 10616832 + }, + { + "epoch": 0.0, + "learning_rate": 0.0001617063492063492, + "loss": 6.0978, + "theoretical_loss": 6.829667118974749, + "tokens_seen": 10682368 + }, + { + "epoch": 0.0, + "learning_rate": 0.0001626984126984127, + "loss": 5.7867, + "theoretical_loss": 6.8221028389077185, + "tokens_seen": 10747904 + }, + { + "epoch": 0.0, + "learning_rate": 0.00016369047619047618, + "loss": 5.8499, + "theoretical_loss": 6.814597367924395, + "tokens_seen": 10813440 + }, + { + "epoch": 0.0, + "learning_rate": 0.00016468253968253969, + "loss": 5.9166, + "theoretical_loss": 6.807149896524181, + "tokens_seen": 10878976 + }, + { + "epoch": 0.0, + "learning_rate": 0.00016567460317460316, + "loss": 5.7915, + "theoretical_loss": 6.799759631140145, + "tokens_seen": 10944512 + }, + { + "epoch": 0.0, + "learning_rate": 0.00016666666666666666, + "loss": 5.7431, + "theoretical_loss": 6.7924257937322245, + "tokens_seen": 11010048 + }, + { + "epoch": 0.0, + "learning_rate": 0.00016765873015873016, + "loss": 5.8272, + "theoretical_loss": 6.785147621393148, + "tokens_seen": 11075584 + }, + { + "epoch": 0.0, + "learning_rate": 0.00016865079365079366, + "loss": 5.6441, + "theoretical_loss": 6.777924365966638, + "tokens_seen": 11141120 + }, + { + "epoch": 0.0, + "learning_rate": 0.00016964285714285717, + "loss": 5.733, + "theoretical_loss": 6.770755293677423, + "tokens_seen": 11206656 + }, + { + "epoch": 0.0, + "learning_rate": 0.00017063492063492064, + "loss": 5.8183, + "theoretical_loss": 6.763639684772625, + "tokens_seen": 11272192 + }, + { + "epoch": 0.0, + "learning_rate": 0.00017162698412698414, + "loss": 5.7902, + "theoretical_loss": 6.756576833174123, + "tokens_seen": 11337728 + }, + { + "epoch": 0.0, + "learning_rate": 0.00017261904761904762, + "loss": 5.7469, + "theoretical_loss": 6.749566046141486, + "tokens_seen": 11403264 + }, + { + "epoch": 0.0, + "objective/train/docs_used": 97786, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 6.325038433074951, + "objective/train/theoretical_loss": 6.7426066439450905, + "objective/train/tokens_used": 31928800, + "theoretical_loss": 6.7426066439450905, + "tokens_seen": 11468800 + }, + { + "epoch": 0.0, + "learning_rate": 0.00017361111111111112, + "loss": 5.8377, + "theoretical_loss": 6.7426066439450905, + "tokens_seen": 11468800 + }, + { + "epoch": 0.0, + "learning_rate": 0.0001746031746031746, + "loss": 6.0515, + "theoretical_loss": 6.735697959549075, + "tokens_seen": 11534336 + }, + { + "epoch": 0.0, + "learning_rate": 0.0001755952380952381, + "loss": 5.8069, + "theoretical_loss": 6.728839338303761, + "tokens_seen": 11599872 + }, + { + "epoch": 0.0, + "learning_rate": 0.0001765873015873016, + "loss": 5.6908, + "theoretical_loss": 6.722030137647226, + "tokens_seen": 11665408 + }, + { + "epoch": 0.0, + "learning_rate": 0.0001775793650793651, + "loss": 5.5015, + "theoretical_loss": 6.715269726815689, + "tokens_seen": 11730944 + }, + { + "epoch": 0.0, + "learning_rate": 0.00017857142857142857, + "loss": 5.6095, + "theoretical_loss": 6.7085574865624125, + "tokens_seen": 11796480 + }, + { + "epoch": 0.0, + "learning_rate": 0.00017956349206349207, + "loss": 5.588, + "theoretical_loss": 6.701892808884824, + "tokens_seen": 11862016 + }, + { + "epoch": 0.0, + "learning_rate": 0.00018055555555555555, + "loss": 5.7956, + "theoretical_loss": 6.695275096759559, + "tokens_seen": 11927552 + }, + { + "epoch": 0.0, + "learning_rate": 0.00018154761904761905, + "loss": 5.759, + "theoretical_loss": 6.68870376388518, + "tokens_seen": 11993088 + }, + { + "epoch": 0.0, + "learning_rate": 0.00018253968253968252, + "loss": 5.7904, + "theoretical_loss": 6.682178234432274, + "tokens_seen": 12058624 + }, + { + "epoch": 0.0, + "learning_rate": 0.00018353174603174602, + "loss": 5.8681, + "theoretical_loss": 6.675697942800715, + "tokens_seen": 12124160 + }, + { + "epoch": 0.0, + "learning_rate": 0.00018452380952380953, + "loss": 5.6942, + "theoretical_loss": 6.669262333383815, + "tokens_seen": 12189696 + }, + { + "epoch": 0.0, + "learning_rate": 0.00018551587301587303, + "loss": 5.612, + "theoretical_loss": 6.662870860339158, + "tokens_seen": 12255232 + }, + { + "epoch": 0.0, + "learning_rate": 0.0001865079365079365, + "loss": 5.8235, + "theoretical_loss": 6.656522987365879, + "tokens_seen": 12320768 + }, + { + "epoch": 0.0, + "learning_rate": 0.0001875, + "loss": 5.7374, + "theoretical_loss": 6.6502181874881705, + "tokens_seen": 12386304 + }, + { + "epoch": 0.0, + "learning_rate": 0.0001884920634920635, + "loss": 5.624, + "theoretical_loss": 6.643955942844831, + "tokens_seen": 12451840 + }, + { + "epoch": 0.0, + "learning_rate": 0.00018948412698412698, + "loss": 5.7172, + "theoretical_loss": 6.637735744484626, + "tokens_seen": 12517376 + }, + { + "epoch": 0.0, + "learning_rate": 0.00019047619047619048, + "loss": 5.6884, + "theoretical_loss": 6.631557092167304, + "tokens_seen": 12582912 + }, + { + "epoch": 0.0, + "learning_rate": 0.00019146825396825398, + "loss": 5.5524, + "theoretical_loss": 6.625419494170049, + "tokens_seen": 12648448 + }, + { + "epoch": 0.0, + "learning_rate": 0.00019246031746031748, + "loss": 5.9368, + "theoretical_loss": 6.619322467099223, + "tokens_seen": 12713984 + }, + { + "epoch": 0.0, + "learning_rate": 0.00019345238095238096, + "loss": 5.6623, + "theoretical_loss": 6.613265535707211, + "tokens_seen": 12779520 + }, + { + "epoch": 0.0, + "learning_rate": 0.00019444444444444446, + "loss": 5.4595, + "theoretical_loss": 6.607248232714213, + "tokens_seen": 12845056 + }, + { + "epoch": 0.0, + "learning_rate": 0.00019543650793650793, + "loss": 5.2926, + "theoretical_loss": 6.60127009863481, + "tokens_seen": 12910592 + }, + { + "epoch": 0.0, + "learning_rate": 0.00019642857142857144, + "loss": 5.5284, + "theoretical_loss": 6.59533068160918, + "tokens_seen": 12976128 + }, + { + "epoch": 0.0, + "learning_rate": 0.0001974206349206349, + "loss": 5.7083, + "theoretical_loss": 6.589429537238785, + "tokens_seen": 13041664 + }, + { + "epoch": 0.0, + "objective/train/docs_used": 102516, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 5.887484073638916, + "objective/train/theoretical_loss": 6.583566228426414, + "objective/train/tokens_used": 33567200, + "theoretical_loss": 6.583566228426414, + "tokens_seen": 13107200 + }, + { + "epoch": 0.0, + "learning_rate": 0.0001984126984126984, + "loss": 5.7401, + "theoretical_loss": 6.583566228426414, + "tokens_seen": 13107200 + }, + { + "epoch": 0.0, + "learning_rate": 0.00019940476190476191, + "loss": 5.6309, + "theoretical_loss": 6.5777403252204305, + "tokens_seen": 13172736 + }, + { + "epoch": 0.0, + "learning_rate": 0.00020039682539682542, + "loss": 5.5628, + "theoretical_loss": 6.571951404663098, + "tokens_seen": 13238272 + }, + { + "epoch": 0.0, + "learning_rate": 0.0002013888888888889, + "loss": 5.7721, + "theoretical_loss": 6.566199050642863, + "tokens_seen": 13303808 + }, + { + "epoch": 0.0, + "learning_rate": 0.0002023809523809524, + "loss": 5.3532, + "theoretical_loss": 6.560482853750463, + "tokens_seen": 13369344 + }, + { + "epoch": 0.0, + "learning_rate": 0.00020337301587301587, + "loss": 5.6038, + "theoretical_loss": 6.554802411138745, + "tokens_seen": 13434880 + }, + { + "epoch": 0.0, + "learning_rate": 0.00020436507936507937, + "loss": 5.8779, + "theoretical_loss": 6.549157326386091, + "tokens_seen": 13500416 + }, + { + "epoch": 0.0, + "learning_rate": 0.00020535714285714284, + "loss": 5.8416, + "theoretical_loss": 6.54354720936333, + "tokens_seen": 13565952 + }, + { + "epoch": 0.0, + "learning_rate": 0.00020634920634920634, + "loss": 5.3702, + "theoretical_loss": 6.537971676104026, + "tokens_seen": 13631488 + }, + { + "epoch": 0.0, + "learning_rate": 0.00020734126984126985, + "loss": 5.8494, + "theoretical_loss": 6.532430348678068, + "tokens_seen": 13697024 + }, + { + "epoch": 0.0, + "learning_rate": 0.00020833333333333335, + "loss": 5.6039, + "theoretical_loss": 6.5269228550684195, + "tokens_seen": 13762560 + }, + { + "epoch": 0.0, + "learning_rate": 0.00020932539682539685, + "loss": 5.5831, + "theoretical_loss": 6.521448829050978, + "tokens_seen": 13828096 + }, + { + "epoch": 0.0, + "learning_rate": 0.00021031746031746032, + "loss": 5.5982, + "theoretical_loss": 6.516007910077416, + "tokens_seen": 13893632 + }, + { + "epoch": 0.0, + "learning_rate": 0.00021130952380952382, + "loss": 5.4289, + "theoretical_loss": 6.51059974316095, + "tokens_seen": 13959168 + }, + { + "epoch": 0.0, + "learning_rate": 0.0002123015873015873, + "loss": 5.5763, + "theoretical_loss": 6.50522397876491, + "tokens_seen": 14024704 + }, + { + "epoch": 0.0, + "learning_rate": 0.0002132936507936508, + "loss": 5.5622, + "theoretical_loss": 6.499880272694068, + "tokens_seen": 14090240 + }, + { + "epoch": 0.0, + "learning_rate": 0.00021428571428571427, + "loss": 5.6438, + "theoretical_loss": 6.494568285988618, + "tokens_seen": 14155776 + }, + { + "epoch": 0.0, + "learning_rate": 0.0002152777777777778, + "loss": 5.662, + "theoretical_loss": 6.489287684820745, + "tokens_seen": 14221312 + }, + { + "epoch": 0.0, + "learning_rate": 0.00021626984126984128, + "loss": 5.3087, + "theoretical_loss": 6.484038140393699, + "tokens_seen": 14286848 + }, + { + "epoch": 0.0, + "learning_rate": 0.00021726190476190478, + "loss": 5.3336, + "theoretical_loss": 6.4788193288433105, + "tokens_seen": 14352384 + }, + { + "epoch": 0.0, + "learning_rate": 0.00021825396825396825, + "loss": 5.4532, + "theoretical_loss": 6.473630931141869, + "tokens_seen": 14417920 + }, + { + "epoch": 0.0, + "learning_rate": 0.00021924603174603176, + "loss": 5.6727, + "theoretical_loss": 6.468472633004308, + "tokens_seen": 14483456 + }, + { + "epoch": 0.0, + "learning_rate": 0.00022023809523809523, + "loss": 5.3546, + "theoretical_loss": 6.463344124796616, + "tokens_seen": 14548992 + }, + { + "epoch": 0.0, + "learning_rate": 0.00022123015873015873, + "loss": 5.469, + "theoretical_loss": 6.45824510144643, + "tokens_seen": 14614528 + }, + { + "epoch": 0.0, + "learning_rate": 0.0002222222222222222, + "loss": 5.7059, + "theoretical_loss": 6.45317526235573, + "tokens_seen": 14680064 + }, + { + "epoch": 0.0, + "objective/train/docs_used": 107633, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 5.52135705947876, + "objective/train/theoretical_loss": 6.448134311315593, + "objective/train/tokens_used": 35205600, + "theoretical_loss": 6.448134311315593, + "tokens_seen": 14745600 + }, + { + "epoch": 0.0, + "learning_rate": 0.00022321428571428573, + "loss": 5.493, + "theoretical_loss": 6.448134311315593, + "tokens_seen": 14745600 + }, + { + "epoch": 0.0, + "learning_rate": 0.0002242063492063492, + "loss": 5.5841, + "theoretical_loss": 6.443121956422939, + "tokens_seen": 14811136 + }, + { + "epoch": 0.0, + "learning_rate": 0.0002251984126984127, + "loss": 5.329, + "theoretical_loss": 6.438137909999214, + "tokens_seen": 14876672 + }, + { + "epoch": 0.0, + "learning_rate": 0.00022619047619047618, + "loss": 5.5932, + "theoretical_loss": 6.433181888510964, + "tokens_seen": 14942208 + }, + { + "epoch": 0.0, + "learning_rate": 0.00022718253968253969, + "loss": 5.3925, + "theoretical_loss": 6.428253612492239, + "tokens_seen": 15007744 + }, + { + "epoch": 0.0, + "learning_rate": 0.00022817460317460316, + "loss": 5.3742, + "theoretical_loss": 6.4233528064687855, + "tokens_seen": 15073280 + }, + { + "epoch": 0.0, + "learning_rate": 0.00022916666666666666, + "loss": 5.6301, + "theoretical_loss": 6.418479198883969, + "tokens_seen": 15138816 + }, + { + "epoch": 0.0, + "learning_rate": 0.00023015873015873016, + "loss": 5.0119, + "theoretical_loss": 6.413632522026391, + "tokens_seen": 15204352 + }, + { + "epoch": 0.0, + "learning_rate": 0.00023115079365079367, + "loss": 5.4315, + "theoretical_loss": 6.40881251195914, + "tokens_seen": 15269888 + }, + { + "epoch": 0.0, + "learning_rate": 0.00023214285714285717, + "loss": 5.401, + "theoretical_loss": 6.404018908450656, + "tokens_seen": 15335424 + }, + { + "epoch": 0.0, + "learning_rate": 0.00023313492063492064, + "loss": 5.2261, + "theoretical_loss": 6.399251454907132, + "tokens_seen": 15400960 + }, + { + "epoch": 0.0, + "learning_rate": 0.00023412698412698414, + "loss": 5.44, + "theoretical_loss": 6.394509898306452, + "tokens_seen": 15466496 + }, + { + "epoch": 0.0, + "learning_rate": 0.00023511904761904762, + "loss": 5.4848, + "theoretical_loss": 6.389793989133574, + "tokens_seen": 15532032 + }, + { + "epoch": 0.0, + "learning_rate": 0.00023611111111111112, + "loss": 5.5739, + "theoretical_loss": 6.385103481317387, + "tokens_seen": 15597568 + }, + { + "epoch": 0.0, + "learning_rate": 0.0002371031746031746, + "loss": 5.3712, + "theoretical_loss": 6.380438132168923, + "tokens_seen": 15663104 + }, + { + "epoch": 0.0, + "learning_rate": 0.0002380952380952381, + "loss": 5.3527, + "theoretical_loss": 6.375797702320966, + "tokens_seen": 15728640 + }, + { + "epoch": 0.0, + "learning_rate": 0.0002390873015873016, + "loss": 5.4, + "theoretical_loss": 6.371181955668966, + "tokens_seen": 15794176 + }, + { + "epoch": 0.0, + "learning_rate": 0.0002400793650793651, + "loss": 5.3654, + "theoretical_loss": 6.366590659313248, + "tokens_seen": 15859712 + }, + { + "epoch": 0.0, + "learning_rate": 0.00024107142857142857, + "loss": 5.4837, + "theoretical_loss": 6.36202358350248, + "tokens_seen": 15925248 + }, + { + "epoch": 0.0, + "learning_rate": 0.00024206349206349207, + "loss": 5.2794, + "theoretical_loss": 6.357480501578371, + "tokens_seen": 15990784 + }, + { + "epoch": 0.0, + "learning_rate": 0.00024305555555555555, + "loss": 5.2403, + "theoretical_loss": 6.352961189921553, + "tokens_seen": 16056320 + }, + { + "epoch": 0.0, + "learning_rate": 0.00024404761904761905, + "loss": 5.3106, + "theoretical_loss": 6.348465427898629, + "tokens_seen": 16121856 + }, + { + "epoch": 0.0, + "learning_rate": 0.00024503968253968255, + "loss": 5.3799, + "theoretical_loss": 6.343992997810366, + "tokens_seen": 16187392 + }, + { + "epoch": 0.0, + "learning_rate": 0.000246031746031746, + "loss": 5.3256, + "theoretical_loss": 6.33954368484097, + "tokens_seen": 16252928 + }, + { + "epoch": 0.0, + "learning_rate": 0.00024702380952380955, + "loss": 5.5652, + "theoretical_loss": 6.33511727700846, + "tokens_seen": 16318464 + }, + { + "epoch": 0.0, + "objective/train/docs_used": 112492, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 5.672967910766602, + "objective/train/theoretical_loss": 6.330713565116083, + "objective/train/tokens_used": 36844000, + "theoretical_loss": 6.330713565116083, + "tokens_seen": 16384000 + }, + { + "epoch": 0.0, + "learning_rate": 0.000248015873015873, + "loss": 5.5336, + "theoretical_loss": 6.330713565116083, + "tokens_seen": 16384000 + }, + { + "epoch": 0.0, + "learning_rate": 0.0002490079365079365, + "loss": 5.1247, + "theoretical_loss": 6.326332342704751, + "tokens_seen": 16449536 + }, + { + "epoch": 0.01, + "learning_rate": 0.00025, + "loss": 4.9896, + "theoretical_loss": 6.32197340600647, + "tokens_seen": 16515072 + }, + { + "epoch": 0.01, + "learning_rate": 0.0002509920634920635, + "loss": 5.323, + "theoretical_loss": 6.3176365538987636, + "tokens_seen": 16580608 + }, + { + "epoch": 0.01, + "learning_rate": 0.000251984126984127, + "loss": 5.0875, + "theoretical_loss": 6.313321587860021, + "tokens_seen": 16646144 + }, + { + "epoch": 0.01, + "learning_rate": 0.00025297619047619046, + "loss": 5.4551, + "theoretical_loss": 6.309028311925785, + "tokens_seen": 16711680 + }, + { + "epoch": 0.01, + "learning_rate": 0.00025396825396825396, + "loss": 5.3549, + "theoretical_loss": 6.304756532645939, + "tokens_seen": 16777216 + }, + { + "epoch": 0.01, + "learning_rate": 0.00025496031746031746, + "loss": 5.1, + "theoretical_loss": 6.300506059042775, + "tokens_seen": 16842752 + }, + { + "epoch": 0.01, + "learning_rate": 0.00025595238095238096, + "loss": 5.4827, + "theoretical_loss": 6.296276702569918, + "tokens_seen": 16908288 + }, + { + "epoch": 0.01, + "learning_rate": 0.0002569444444444444, + "loss": 5.4362, + "theoretical_loss": 6.292068277072099, + "tokens_seen": 16973824 + }, + { + "epoch": 0.01, + "learning_rate": 0.00025793650793650796, + "loss": 5.3388, + "theoretical_loss": 6.28788059874573, + "tokens_seen": 17039360 + }, + { + "epoch": 0.01, + "learning_rate": 0.00025892857142857146, + "loss": 5.2424, + "theoretical_loss": 6.283713486100297, + "tokens_seen": 17104896 + }, + { + "epoch": 0.01, + "learning_rate": 0.00025992063492063497, + "loss": 5.3369, + "theoretical_loss": 6.279566759920507, + "tokens_seen": 17170432 + }, + { + "epoch": 0.01, + "learning_rate": 0.0002609126984126984, + "loss": 5.3566, + "theoretical_loss": 6.275440243229228, + "tokens_seen": 17235968 + }, + { + "epoch": 0.01, + "learning_rate": 0.0002619047619047619, + "loss": 5.1989, + "theoretical_loss": 6.271333761251142, + "tokens_seen": 17301504 + }, + { + "epoch": 0.01, + "learning_rate": 0.0002628968253968254, + "loss": 5.2989, + "theoretical_loss": 6.267247141377137, + "tokens_seen": 17367040 + }, + { + "epoch": 0.01, + "learning_rate": 0.0002638888888888889, + "loss": 5.1155, + "theoretical_loss": 6.2631802131294085, + "tokens_seen": 17432576 + }, + { + "epoch": 0.01, + "learning_rate": 0.00026488095238095237, + "loss": 5.0634, + "theoretical_loss": 6.259132808127246, + "tokens_seen": 17498112 + }, + { + "epoch": 0.01, + "learning_rate": 0.00026587301587301587, + "loss": 5.2512, + "theoretical_loss": 6.255104760053497, + "tokens_seen": 17563648 + }, + { + "epoch": 0.01, + "learning_rate": 0.00026686507936507937, + "loss": 5.2924, + "theoretical_loss": 6.251095904621689, + "tokens_seen": 17629184 + }, + { + "epoch": 0.01, + "learning_rate": 0.00026785714285714287, + "loss": 5.1212, + "theoretical_loss": 6.247106079543801, + "tokens_seen": 17694720 + }, + { + "epoch": 0.01, + "learning_rate": 0.0002688492063492063, + "loss": 5.2689, + "theoretical_loss": 6.243135124498652, + "tokens_seen": 17760256 + }, + { + "epoch": 0.01, + "learning_rate": 0.0002698412698412698, + "loss": 5.2765, + "theoretical_loss": 6.239182881100916, + "tokens_seen": 17825792 + }, + { + "epoch": 0.01, + "learning_rate": 0.0002708333333333333, + "loss": 5.2352, + "theoretical_loss": 6.235249192870732, + "tokens_seen": 17891328 + }, + { + "epoch": 0.01, + "learning_rate": 0.0002718253968253968, + "loss": 5.5256, + "theoretical_loss": 6.231333905203899, + "tokens_seen": 17956864 + }, + { + "epoch": 0.01, + "objective/train/docs_used": 117713, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 5.518486022949219, + "objective/train/theoretical_loss": 6.227436865342643, + "objective/train/tokens_used": 38482400, + "theoretical_loss": 6.227436865342643, + "tokens_seen": 18022400 + }, + { + "epoch": 0.01, + "learning_rate": 0.0002728174603174603, + "loss": 5.1989, + "theoretical_loss": 6.227436865342643, + "tokens_seen": 18022400 + }, + { + "epoch": 0.01, + "learning_rate": 0.0002738095238095238, + "loss": 5.243, + "theoretical_loss": 6.223557922346955, + "tokens_seen": 18087936 + }, + { + "epoch": 0.01, + "learning_rate": 0.0002748015873015873, + "loss": 5.205, + "theoretical_loss": 6.219696927066456, + "tokens_seen": 18153472 + }, + { + "epoch": 0.01, + "learning_rate": 0.00027579365079365083, + "loss": 5.068, + "theoretical_loss": 6.215853732112821, + "tokens_seen": 18219008 + }, + { + "epoch": 0.01, + "learning_rate": 0.00027678571428571433, + "loss": 5.1643, + "theoretical_loss": 6.212028191832702, + "tokens_seen": 18284544 + }, + { + "epoch": 0.01, + "learning_rate": 0.0002777777777777778, + "loss": 5.1496, + "theoretical_loss": 6.208220162281178, + "tokens_seen": 18350080 + }, + { + "epoch": 0.01, + "learning_rate": 0.0002787698412698413, + "loss": 5.0765, + "theoretical_loss": 6.204429501195701, + "tokens_seen": 18415616 + }, + { + "epoch": 0.01, + "learning_rate": 0.0002797619047619048, + "loss": 4.9861, + "theoretical_loss": 6.20065606797053, + "tokens_seen": 18481152 + }, + { + "epoch": 0.01, + "learning_rate": 0.0002807539682539683, + "loss": 5.2004, + "theoretical_loss": 6.19689972363164, + "tokens_seen": 18546688 + }, + { + "epoch": 0.01, + "learning_rate": 0.00028174603174603173, + "loss": 5.3659, + "theoretical_loss": 6.1931603308120975, + "tokens_seen": 18612224 + }, + { + "epoch": 0.01, + "learning_rate": 0.00028273809523809523, + "loss": 5.1814, + "theoretical_loss": 6.189437753727901, + "tokens_seen": 18677760 + }, + { + "epoch": 0.01, + "learning_rate": 0.00028373015873015873, + "loss": 5.1199, + "theoretical_loss": 6.185731858154261, + "tokens_seen": 18743296 + }, + { + "epoch": 0.01, + "learning_rate": 0.00028472222222222223, + "loss": 5.2977, + "theoretical_loss": 6.182042511402313, + "tokens_seen": 18808832 + }, + { + "epoch": 0.01, + "learning_rate": 0.0002857142857142857, + "loss": 5.1583, + "theoretical_loss": 6.17836958229627, + "tokens_seen": 18874368 + }, + { + "epoch": 0.01, + "learning_rate": 0.0002867063492063492, + "loss": 5.0914, + "theoretical_loss": 6.1747129411509825, + "tokens_seen": 18939904 + }, + { + "epoch": 0.01, + "learning_rate": 0.0002876984126984127, + "loss": 4.8355, + "theoretical_loss": 6.171072459749913, + "tokens_seen": 19005440 + }, + { + "epoch": 0.01, + "learning_rate": 0.0002886904761904762, + "loss": 5.1587, + "theoretical_loss": 6.1674480113235095, + "tokens_seen": 19070976 + }, + { + "epoch": 0.01, + "learning_rate": 0.0002896825396825397, + "loss": 5.0374, + "theoretical_loss": 6.163839470527964, + "tokens_seen": 19136512 + }, + { + "epoch": 0.01, + "learning_rate": 0.0002906746031746032, + "loss": 5.3875, + "theoretical_loss": 6.160246713424372, + "tokens_seen": 19202048 + }, + { + "epoch": 0.01, + "learning_rate": 0.0002916666666666667, + "loss": 5.1875, + "theoretical_loss": 6.156669617458243, + "tokens_seen": 19267584 + }, + { + "epoch": 0.01, + "learning_rate": 0.0002926587301587302, + "loss": 5.3545, + "theoretical_loss": 6.153108061439397, + "tokens_seen": 19333120 + }, + { + "epoch": 0.01, + "learning_rate": 0.0002936507936507937, + "loss": 5.1569, + "theoretical_loss": 6.149561925522211, + "tokens_seen": 19398656 + }, + { + "epoch": 0.01, + "learning_rate": 0.00029464285714285714, + "loss": 5.2313, + "theoretical_loss": 6.146031091186222, + "tokens_seen": 19464192 + }, + { + "epoch": 0.01, + "learning_rate": 0.00029563492063492064, + "loss": 5.3309, + "theoretical_loss": 6.142515441217064, + "tokens_seen": 19529728 + }, + { + "epoch": 0.01, + "learning_rate": 0.00029662698412698414, + "loss": 5.2886, + "theoretical_loss": 6.1390148596877605, + "tokens_seen": 19595264 + }, + { + "epoch": 0.01, + "objective/train/docs_used": 122717, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 5.544928073883057, + "objective/train/theoretical_loss": 6.135529231940326, + "objective/train/tokens_used": 40120800, + "theoretical_loss": 6.135529231940326, + "tokens_seen": 19660800 + }, + { + "epoch": 0.01, + "learning_rate": 0.00029761904761904765, + "loss": 5.3891, + "theoretical_loss": 6.135529231940326, + "tokens_seen": 19660800 + }, + { + "epoch": 0.01, + "learning_rate": 0.0002986111111111111, + "loss": 5.1961, + "theoretical_loss": 6.132058444567705, + "tokens_seen": 19726336 + }, + { + "epoch": 0.01, + "learning_rate": 0.0002996031746031746, + "loss": 5.166, + "theoretical_loss": 6.128602385396022, + "tokens_seen": 19791872 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003005952380952381, + "loss": 5.1897, + "theoretical_loss": 6.125160943467138, + "tokens_seen": 19857408 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003015873015873016, + "loss": 5.1994, + "theoretical_loss": 6.121734009021521, + "tokens_seen": 19922944 + }, + { + "epoch": 0.01, + "learning_rate": 0.00030257936507936505, + "loss": 5.0567, + "theoretical_loss": 6.118321473481398, + "tokens_seen": 19988480 + }, + { + "epoch": 0.01, + "learning_rate": 0.00030357142857142855, + "loss": 5.1657, + "theoretical_loss": 6.114923229434213, + "tokens_seen": 20054016 + }, + { + "epoch": 0.01, + "learning_rate": 0.00030456349206349205, + "loss": 5.1727, + "theoretical_loss": 6.111539170616359, + "tokens_seen": 20119552 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003055555555555556, + "loss": 4.7328, + "theoretical_loss": 6.108169191897195, + "tokens_seen": 20185088 + }, + { + "epoch": 0.01, + "learning_rate": 0.00030654761904761905, + "loss": 5.2345, + "theoretical_loss": 6.104813189263336, + "tokens_seen": 20250624 + }, + { + "epoch": 0.01, + "learning_rate": 0.00030753968253968255, + "loss": 5.1383, + "theoretical_loss": 6.101471059803204, + "tokens_seen": 20316160 + }, + { + "epoch": 0.01, + "learning_rate": 0.00030853174603174605, + "loss": 5.1325, + "theoretical_loss": 6.098142701691856, + "tokens_seen": 20381696 + }, + { + "epoch": 0.01, + "learning_rate": 0.00030952380952380956, + "loss": 4.9321, + "theoretical_loss": 6.094828014176053, + "tokens_seen": 20447232 + }, + { + "epoch": 0.01, + "learning_rate": 0.000310515873015873, + "loss": 5.3554, + "theoretical_loss": 6.091526897559593, + "tokens_seen": 20512768 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003115079365079365, + "loss": 5.1831, + "theoretical_loss": 6.088239253188885, + "tokens_seen": 20578304 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003125, + "loss": 5.2792, + "theoretical_loss": 6.084964983438763, + "tokens_seen": 20643840 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003134920634920635, + "loss": 4.9628, + "theoretical_loss": 6.0817039916985465, + "tokens_seen": 20709376 + }, + { + "epoch": 0.01, + "learning_rate": 0.000314484126984127, + "loss": 5.0786, + "theoretical_loss": 6.078456182358325, + "tokens_seen": 20774912 + }, + { + "epoch": 0.01, + "learning_rate": 0.00031547619047619046, + "loss": 5.3443, + "theoretical_loss": 6.075221460795472, + "tokens_seen": 20840448 + }, + { + "epoch": 0.01, + "learning_rate": 0.00031646825396825396, + "loss": 5.2776, + "theoretical_loss": 6.071999733361386, + "tokens_seen": 20905984 + }, + { + "epoch": 0.01, + "learning_rate": 0.00031746031746031746, + "loss": 4.7741, + "theoretical_loss": 6.068790907368448, + "tokens_seen": 20971520 + }, + { + "epoch": 0.01, + "learning_rate": 0.00031845238095238096, + "loss": 5.0769, + "theoretical_loss": 6.0655948910771915, + "tokens_seen": 21037056 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003194444444444444, + "loss": 5.2176, + "theoretical_loss": 6.062411593683687, + "tokens_seen": 21102592 + }, + { + "epoch": 0.01, + "learning_rate": 0.00032043650793650796, + "loss": 5.2326, + "theoretical_loss": 6.059240925307134, + "tokens_seen": 21168128 + }, + { + "epoch": 0.01, + "learning_rate": 0.00032142857142857147, + "loss": 5.2254, + "theoretical_loss": 6.056082796977648, + "tokens_seen": 21233664 + }, + { + "epoch": 0.01, + "objective/train/docs_used": 127954, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 4.949924468994141, + "objective/train/theoretical_loss": 6.052937120624258, + "objective/train/tokens_used": 41759200, + "theoretical_loss": 6.052937120624258, + "tokens_seen": 21299200 + }, + { + "epoch": 0.01, + "learning_rate": 0.00032242063492063497, + "loss": 5.1471, + "theoretical_loss": 6.052937120624258, + "tokens_seen": 21299200 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003234126984126984, + "loss": 4.9762, + "theoretical_loss": 6.049803809063083, + "tokens_seen": 21364736 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003244047619047619, + "loss": 4.7374, + "theoretical_loss": 6.0466827759857145, + "tokens_seen": 21430272 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003253968253968254, + "loss": 4.8236, + "theoretical_loss": 6.04357393594778, + "tokens_seen": 21495808 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003263888888888889, + "loss": 5.0896, + "theoretical_loss": 6.040477204357686, + "tokens_seen": 21561344 + }, + { + "epoch": 0.01, + "learning_rate": 0.00032738095238095237, + "loss": 4.9351, + "theoretical_loss": 6.037392497465552, + "tokens_seen": 21626880 + }, + { + "epoch": 0.01, + "learning_rate": 0.00032837301587301587, + "loss": 4.9554, + "theoretical_loss": 6.034319732352309, + "tokens_seen": 21692416 + }, + { + "epoch": 0.01, + "learning_rate": 0.00032936507936507937, + "loss": 5.2635, + "theoretical_loss": 6.031258826918979, + "tokens_seen": 21757952 + }, + { + "epoch": 0.01, + "learning_rate": 0.00033035714285714287, + "loss": 5.0873, + "theoretical_loss": 6.0282096998761245, + "tokens_seen": 21823488 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003313492063492063, + "loss": 5.1317, + "theoretical_loss": 6.025172270733464, + "tokens_seen": 21889024 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003323412698412698, + "loss": 4.7562, + "theoretical_loss": 6.0221464597896475, + "tokens_seen": 21954560 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003333333333333333, + "loss": 5.0833, + "theoretical_loss": 6.0191321881221995, + "tokens_seen": 22020096 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003343253968253968, + "loss": 4.9952, + "theoretical_loss": 6.016129377577614, + "tokens_seen": 22085632 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003353174603174603, + "loss": 5.141, + "theoretical_loss": 6.01313795076161, + "tokens_seen": 22151168 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003363095238095238, + "loss": 4.8368, + "theoretical_loss": 6.010157831029533, + "tokens_seen": 22216704 + }, + { + "epoch": 0.01, + "learning_rate": 0.00033730158730158733, + "loss": 5.0532, + "theoretical_loss": 6.007188942476907, + "tokens_seen": 22282240 + }, + { + "epoch": 0.01, + "learning_rate": 0.00033829365079365083, + "loss": 5.0954, + "theoretical_loss": 6.0042312099301425, + "tokens_seen": 22347776 + }, + { + "epoch": 0.01, + "learning_rate": 0.00033928571428571433, + "loss": 5.0574, + "theoretical_loss": 6.001284558937368, + "tokens_seen": 22413312 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003402777777777778, + "loss": 4.8983, + "theoretical_loss": 5.998348915759426, + "tokens_seen": 22478848 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003412698412698413, + "loss": 4.9248, + "theoretical_loss": 5.995424207360987, + "tokens_seen": 22544384 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003422619047619048, + "loss": 4.8788, + "theoretical_loss": 5.992510361401818, + "tokens_seen": 22609920 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003432539682539683, + "loss": 5.0268, + "theoretical_loss": 5.989607306228168, + "tokens_seen": 22675456 + }, + { + "epoch": 0.01, + "learning_rate": 0.00034424603174603173, + "loss": 4.9311, + "theoretical_loss": 5.986714970864292, + "tokens_seen": 22740992 + }, + { + "epoch": 0.01, + "learning_rate": 0.00034523809523809523, + "loss": 4.9288, + "theoretical_loss": 5.983833285004112, + "tokens_seen": 22806528 + }, + { + "epoch": 0.01, + "learning_rate": 0.00034623015873015873, + "loss": 5.1659, + "theoretical_loss": 5.980962179002983, + "tokens_seen": 22872064 + }, + { + "epoch": 0.01, + "objective/train/docs_used": 130738, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 5.168276786804199, + "objective/train/theoretical_loss": 5.978101583869607, + "objective/train/tokens_used": 43397600, + "theoretical_loss": 5.978101583869607, + "tokens_seen": 22937600 + }, + { + "epoch": 0.01, + "learning_rate": 0.00034722222222222224, + "loss": 4.6862, + "theoretical_loss": 5.978101583869607, + "tokens_seen": 22937600 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003482142857142857, + "loss": 4.9681, + "theoretical_loss": 5.975251431258057, + "tokens_seen": 23003136 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003492063492063492, + "loss": 5.004, + "theoretical_loss": 5.972411653459913, + "tokens_seen": 23068672 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003501984126984127, + "loss": 5.0517, + "theoretical_loss": 5.9695821833965335, + "tokens_seen": 23134208 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003511904761904762, + "loss": 5.2897, + "theoretical_loss": 5.966762954611432, + "tokens_seen": 23199744 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003521825396825397, + "loss": 5.0365, + "theoretical_loss": 5.963953901262764, + "tokens_seen": 23265280 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003531746031746032, + "loss": 4.9465, + "theoretical_loss": 5.961154958115937, + "tokens_seen": 23330816 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003541666666666667, + "loss": 5.1461, + "theoretical_loss": 5.958366060536315, + "tokens_seen": 23396352 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003551587301587302, + "loss": 4.8917, + "theoretical_loss": 5.955587144482044, + "tokens_seen": 23461888 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003561507936507937, + "loss": 4.9521, + "theoretical_loss": 5.952818146496978, + "tokens_seen": 23527424 + }, + { + "epoch": 0.01, + "learning_rate": 0.00035714285714285714, + "loss": 4.8648, + "theoretical_loss": 5.950059003703704, + "tokens_seen": 23592960 + }, + { + "epoch": 0.01, + "learning_rate": 0.00035813492063492064, + "loss": 5.081, + "theoretical_loss": 5.94730965379668, + "tokens_seen": 23658496 + }, + { + "epoch": 0.01, + "learning_rate": 0.00035912698412698415, + "loss": 4.9531, + "theoretical_loss": 5.944570035035458, + "tokens_seen": 23724032 + }, + { + "epoch": 0.01, + "learning_rate": 0.00036011904761904765, + "loss": 4.8974, + "theoretical_loss": 5.941840086238027, + "tokens_seen": 23789568 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003611111111111111, + "loss": 4.9338, + "theoretical_loss": 5.939119746774228, + "tokens_seen": 23855104 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003621031746031746, + "loss": 4.9457, + "theoretical_loss": 5.936408956559284, + "tokens_seen": 23920640 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003630952380952381, + "loss": 4.8877, + "theoretical_loss": 5.933707656047414, + "tokens_seen": 23986176 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003640873015873016, + "loss": 5.2136, + "theoretical_loss": 5.93101578622554, + "tokens_seen": 24051712 + }, + { + "epoch": 0.01, + "learning_rate": 0.00036507936507936505, + "loss": 5.1442, + "theoretical_loss": 5.928333288607086, + "tokens_seen": 24117248 + }, + { + "epoch": 0.01, + "learning_rate": 0.00036607142857142855, + "loss": 4.939, + "theoretical_loss": 5.925660105225867, + "tokens_seen": 24182784 + }, + { + "epoch": 0.01, + "learning_rate": 0.00036706349206349205, + "loss": 4.9773, + "theoretical_loss": 5.92299617863006, + "tokens_seen": 24248320 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003680555555555556, + "loss": 4.8856, + "theoretical_loss": 5.920341451876267, + "tokens_seen": 24313856 + }, + { + "epoch": 0.01, + "learning_rate": 0.00036904761904761905, + "loss": 4.8102, + "theoretical_loss": 5.9176958685236585, + "tokens_seen": 24379392 + }, + { + "epoch": 0.01, + "learning_rate": 0.00037003968253968255, + "loss": 4.9364, + "theoretical_loss": 5.9150593726282015, + "tokens_seen": 24444928 + }, + { + "epoch": 0.01, + "learning_rate": 0.00037103174603174606, + "loss": 4.7276, + "theoretical_loss": 5.912431908736972, + "tokens_seen": 24510464 + }, + { + "epoch": 0.01, + "objective/train/docs_used": 131522, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 5.190422534942627, + "objective/train/theoretical_loss": 5.909813421882534, + "objective/train/tokens_used": 45036000, + "theoretical_loss": 5.909813421882534, + "tokens_seen": 24576000 + }, + { + "epoch": 0.01, + "learning_rate": 0.00037202380952380956, + "loss": 4.8887, + "theoretical_loss": 5.909813421882534, + "tokens_seen": 24576000 + }, + { + "epoch": 0.01, + "learning_rate": 0.000373015873015873, + "loss": 4.9041, + "theoretical_loss": 5.907203857577422, + "tokens_seen": 24641536 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003740079365079365, + "loss": 4.8034, + "theoretical_loss": 5.9046031618086765, + "tokens_seen": 24707072 + }, + { + "epoch": 0.01, + "learning_rate": 0.000375, + "loss": 5.0023, + "theoretical_loss": 5.902011281032472, + "tokens_seen": 24772608 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003759920634920635, + "loss": 5.0488, + "theoretical_loss": 5.899428162168808, + "tokens_seen": 24838144 + }, + { + "epoch": 0.01, + "learning_rate": 0.000376984126984127, + "loss": 5.0453, + "theoretical_loss": 5.896853752596286, + "tokens_seen": 24903680 + }, + { + "epoch": 0.01, + "learning_rate": 0.00037797619047619046, + "loss": 4.813, + "theoretical_loss": 5.894288000146949, + "tokens_seen": 24969216 + }, + { + "epoch": 0.01, + "learning_rate": 0.00037896825396825396, + "loss": 4.8831, + "theoretical_loss": 5.891730853101199, + "tokens_seen": 25034752 + }, + { + "epoch": 0.01, + "learning_rate": 0.00037996031746031746, + "loss": 5.263, + "theoretical_loss": 5.88918226018278, + "tokens_seen": 25100288 + }, + { + "epoch": 0.01, + "learning_rate": 0.00038095238095238096, + "loss": 5.1195, + "theoretical_loss": 5.8866421705538325, + "tokens_seen": 25165824 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003819444444444444, + "loss": 4.9757, + "theoretical_loss": 5.8841105338100155, + "tokens_seen": 25231360 + }, + { + "epoch": 0.01, + "learning_rate": 0.00038293650793650797, + "loss": 5.1901, + "theoretical_loss": 5.881587299975694, + "tokens_seen": 25296896 + }, + { + "epoch": 0.01, + "learning_rate": 0.00038392857142857147, + "loss": 4.8656, + "theoretical_loss": 5.8790724194991935, + "tokens_seen": 25362432 + }, + { + "epoch": 0.01, + "learning_rate": 0.00038492063492063497, + "loss": 5.0726, + "theoretical_loss": 5.876565843248124, + "tokens_seen": 25427968 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003859126984126984, + "loss": 5.1417, + "theoretical_loss": 5.8740675225047525, + "tokens_seen": 25493504 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003869047619047619, + "loss": 5.1355, + "theoretical_loss": 5.871577408961457, + "tokens_seen": 25559040 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003878968253968254, + "loss": 5.0327, + "theoretical_loss": 5.869095454716231, + "tokens_seen": 25624576 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003888888888888889, + "loss": 5.139, + "theoretical_loss": 5.866621612268246, + "tokens_seen": 25690112 + }, + { + "epoch": 0.01, + "learning_rate": 0.00038988095238095237, + "loss": 4.9834, + "theoretical_loss": 5.864155834513486, + "tokens_seen": 25755648 + }, + { + "epoch": 0.01, + "learning_rate": 0.00039087301587301587, + "loss": 5.187, + "theoretical_loss": 5.8616980747404295, + "tokens_seen": 25821184 + }, + { + "epoch": 0.01, + "learning_rate": 0.00039186507936507937, + "loss": 5.0414, + "theoretical_loss": 5.859248286625787, + "tokens_seen": 25886720 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003928571428571429, + "loss": 5.0615, + "theoretical_loss": 5.856806424230314, + "tokens_seen": 25952256 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003938492063492063, + "loss": 4.9987, + "theoretical_loss": 5.854372441994654, + "tokens_seen": 26017792 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003948412698412698, + "loss": 5.0551, + "theoretical_loss": 5.851946294735258, + "tokens_seen": 26083328 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003958333333333333, + "loss": 4.8932, + "theoretical_loss": 5.849527937640345, + "tokens_seen": 26148864 + }, + { + "epoch": 0.01, + "objective/train/docs_used": 132646, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 4.532196998596191, + "objective/train/theoretical_loss": 5.8471173262659235, + "objective/train/tokens_used": 46674400, + "theoretical_loss": 5.8471173262659235, + "tokens_seen": 26214400 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003968253968253968, + "loss": 4.9406, + "theoretical_loss": 5.8471173262659235, + "tokens_seen": 26214400 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003978174603174603, + "loss": 5.0843, + "theoretical_loss": 5.84471441653186, + "tokens_seen": 26279936 + }, + { + "epoch": 0.01, + "learning_rate": 0.00039880952380952383, + "loss": 5.0651, + "theoretical_loss": 5.842319164718004, + "tokens_seen": 26345472 + }, + { + "epoch": 0.01, + "learning_rate": 0.00039980158730158733, + "loss": 5.2335, + "theoretical_loss": 5.83993152746036, + "tokens_seen": 26411008 + }, + { + "epoch": 0.01, + "learning_rate": 0.00040079365079365083, + "loss": 4.9606, + "theoretical_loss": 5.83755146174731, + "tokens_seen": 26476544 + }, + { + "epoch": 0.01, + "learning_rate": 0.00040178571428571433, + "loss": 4.8502, + "theoretical_loss": 5.835178924915889, + "tokens_seen": 26542080 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004027777777777778, + "loss": 5.1473, + "theoretical_loss": 5.832813874648102, + "tokens_seen": 26607616 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004037698412698413, + "loss": 4.9566, + "theoretical_loss": 5.8304562689673, + "tokens_seen": 26673152 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004047619047619048, + "loss": 4.7435, + "theoretical_loss": 5.828106066234588, + "tokens_seen": 26738688 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004057539682539683, + "loss": 5.1525, + "theoretical_loss": 5.825763225145295, + "tokens_seen": 26804224 + }, + { + "epoch": 0.01, + "learning_rate": 0.00040674603174603173, + "loss": 5.0371, + "theoretical_loss": 5.823427704725473, + "tokens_seen": 26869760 + }, + { + "epoch": 0.01, + "learning_rate": 0.00040773809523809523, + "loss": 5.0707, + "theoretical_loss": 5.82109946432846, + "tokens_seen": 26935296 + }, + { + "epoch": 0.01, + "learning_rate": 0.00040873015873015874, + "loss": 4.9852, + "theoretical_loss": 5.818778463631473, + "tokens_seen": 27000832 + }, + { + "epoch": 0.01, + "learning_rate": 0.00040972222222222224, + "loss": 5.0129, + "theoretical_loss": 5.816464662632243, + "tokens_seen": 27066368 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004107142857142857, + "loss": 4.8632, + "theoretical_loss": 5.8141580216457065, + "tokens_seen": 27131904 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004117063492063492, + "loss": 5.2237, + "theoretical_loss": 5.811858501300729, + "tokens_seen": 27197440 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004126984126984127, + "loss": 4.7749, + "theoretical_loss": 5.809566062536868, + "tokens_seen": 27262976 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004136904761904762, + "loss": 4.9421, + "theoretical_loss": 5.807280666601191, + "tokens_seen": 27328512 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004146825396825397, + "loss": 5.3685, + "theoretical_loss": 5.805002275045111, + "tokens_seen": 27394048 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004156746031746032, + "loss": 4.9844, + "theoretical_loss": 5.8027308497212875, + "tokens_seen": 27459584 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004166666666666667, + "loss": 5.0175, + "theoretical_loss": 5.800466352780546, + "tokens_seen": 27525120 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004176587301587302, + "loss": 5.2404, + "theoretical_loss": 5.798208746668847, + "tokens_seen": 27590656 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004186507936507937, + "loss": 4.9851, + "theoretical_loss": 5.795957994124291, + "tokens_seen": 27656192 + }, + { + "epoch": 0.01, + "learning_rate": 0.00041964285714285714, + "loss": 5.0305, + "theoretical_loss": 5.7937140581741575, + "tokens_seen": 27721728 + }, + { + "epoch": 0.01, + "learning_rate": 0.00042063492063492065, + "loss": 4.947, + "theoretical_loss": 5.791476902131985, + "tokens_seen": 27787264 + }, + { + "epoch": 0.01, + "objective/train/docs_used": 133227, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 4.931251525878906, + "objective/train/theoretical_loss": 5.789246489594688, + "objective/train/tokens_used": 48312800, + "theoretical_loss": 5.789246489594688, + "tokens_seen": 27852800 + }, + { + "epoch": 0.01, + "learning_rate": 0.00042162698412698415, + "loss": 5.0547, + "theoretical_loss": 5.789246489594688, + "tokens_seen": 27852800 + }, + { + "epoch": 0.01, + "learning_rate": 0.00042261904761904765, + "loss": 5.0383, + "theoretical_loss": 5.787022784439701, + "tokens_seen": 27918336 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004236111111111111, + "loss": 5.0555, + "theoretical_loss": 5.784805750822171, + "tokens_seen": 27983872 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004246031746031746, + "loss": 4.825, + "theoretical_loss": 5.782595353172176, + "tokens_seen": 28049408 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004255952380952381, + "loss": 5.0085, + "theoretical_loss": 5.780391556191977, + "tokens_seen": 28114944 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004265873015873016, + "loss": 4.7223, + "theoretical_loss": 5.778194324853311, + "tokens_seen": 28180480 + }, + { + "epoch": 0.01, + "learning_rate": 0.00042757936507936505, + "loss": 4.9563, + "theoretical_loss": 5.776003624394711, + "tokens_seen": 28246016 + }, + { + "epoch": 0.01, + "learning_rate": 0.00042857142857142855, + "loss": 4.8565, + "theoretical_loss": 5.773819420318858, + "tokens_seen": 28311552 + }, + { + "epoch": 0.01, + "learning_rate": 0.00042956349206349205, + "loss": 4.653, + "theoretical_loss": 5.771641678389971, + "tokens_seen": 28377088 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004305555555555556, + "loss": 4.9409, + "theoretical_loss": 5.769470364631225, + "tokens_seen": 28442624 + }, + { + "epoch": 0.01, + "learning_rate": 0.00043154761904761905, + "loss": 5.1167, + "theoretical_loss": 5.767305445322201, + "tokens_seen": 28508160 + }, + { + "epoch": 0.01, + "learning_rate": 0.00043253968253968256, + "loss": 4.933, + "theoretical_loss": 5.765146886996363, + "tokens_seen": 28573696 + }, + { + "epoch": 0.01, + "learning_rate": 0.00043353174603174606, + "loss": 4.8283, + "theoretical_loss": 5.762994656438579, + "tokens_seen": 28639232 + }, + { + "epoch": 0.01, + "learning_rate": 0.00043452380952380956, + "loss": 5.0357, + "theoretical_loss": 5.760848720682651, + "tokens_seen": 28704768 + }, + { + "epoch": 0.01, + "learning_rate": 0.000435515873015873, + "loss": 5.0796, + "theoretical_loss": 5.758709047008894, + "tokens_seen": 28770304 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004365079365079365, + "loss": 4.8349, + "theoretical_loss": 5.756575602941732, + "tokens_seen": 28835840 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004375, + "loss": 5.0019, + "theoretical_loss": 5.75444835624733, + "tokens_seen": 28901376 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004384920634920635, + "loss": 4.8212, + "theoretical_loss": 5.752327274931249, + "tokens_seen": 28966912 + }, + { + "epoch": 0.01, + "learning_rate": 0.000439484126984127, + "loss": 4.9358, + "theoretical_loss": 5.750212327236129, + "tokens_seen": 29032448 + }, + { + "epoch": 0.01, + "learning_rate": 0.00044047619047619046, + "loss": 4.9691, + "theoretical_loss": 5.7481034816394105, + "tokens_seen": 29097984 + }, + { + "epoch": 0.01, + "learning_rate": 0.00044146825396825396, + "loss": 4.9269, + "theoretical_loss": 5.7460007068510635, + "tokens_seen": 29163520 + }, + { + "epoch": 0.01, + "learning_rate": 0.00044246031746031746, + "loss": 5.0363, + "theoretical_loss": 5.74390397181136, + "tokens_seen": 29229056 + }, + { + "epoch": 0.01, + "learning_rate": 0.00044345238095238096, + "loss": 4.6776, + "theoretical_loss": 5.741813245688668, + "tokens_seen": 29294592 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004444444444444444, + "loss": 4.8552, + "theoretical_loss": 5.739728497877267, + "tokens_seen": 29360128 + }, + { + "epoch": 0.01, + "learning_rate": 0.00044543650793650797, + "loss": 5.2274, + "theoretical_loss": 5.737649697995197, + "tokens_seen": 29425664 + }, + { + "epoch": 0.01, + "objective/train/docs_used": 134422, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 5.431413650512695, + "objective/train/theoretical_loss": 5.7355768158821245, + "objective/train/tokens_used": 49951200, + "theoretical_loss": 5.7355768158821245, + "tokens_seen": 29491200 + }, + { + "epoch": 0.01, + "learning_rate": 0.00044642857142857147, + "loss": 5.112, + "theoretical_loss": 5.7355768158821245, + "tokens_seen": 29491200 + }, + { + "epoch": 0.01, + "learning_rate": 0.00044742063492063497, + "loss": 5.1022, + "theoretical_loss": 5.73350982159724, + "tokens_seen": 29556736 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004484126984126984, + "loss": 5.0531, + "theoretical_loss": 5.731448685417178, + "tokens_seen": 29622272 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004494047619047619, + "loss": 4.8186, + "theoretical_loss": 5.729393377833956, + "tokens_seen": 29687808 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004503968253968254, + "loss": 4.9572, + "theoretical_loss": 5.7273438695529535, + "tokens_seen": 29753344 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004513888888888889, + "loss": 4.9607, + "theoretical_loss": 5.725300131490888, + "tokens_seen": 29818880 + }, + { + "epoch": 0.01, + "learning_rate": 0.00045238095238095237, + "loss": 4.8473, + "theoretical_loss": 5.7232621347738455, + "tokens_seen": 29884416 + }, + { + "epoch": 0.01, + "learning_rate": 0.00045337301587301587, + "loss": 5.0085, + "theoretical_loss": 5.721229850735305, + "tokens_seen": 29949952 + }, + { + "epoch": 0.01, + "learning_rate": 0.00045436507936507937, + "loss": 4.938, + "theoretical_loss": 5.719203250914208, + "tokens_seen": 30015488 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004553571428571429, + "loss": 4.8577, + "theoretical_loss": 5.717182307053037, + "tokens_seen": 30081024 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004563492063492063, + "loss": 5.0517, + "theoretical_loss": 5.715166991095922, + "tokens_seen": 30146560 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004573412698412698, + "loss": 4.8027, + "theoretical_loss": 5.713157275186761, + "tokens_seen": 30212096 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004583333333333333, + "loss": 5.0271, + "theoretical_loss": 5.71115313166738, + "tokens_seen": 30277632 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004593253968253968, + "loss": 5.0041, + "theoretical_loss": 5.709154533075688, + "tokens_seen": 30343168 + }, + { + "epoch": 0.01, + "learning_rate": 0.00046031746031746033, + "loss": 4.9309, + "theoretical_loss": 5.707161452143879, + "tokens_seen": 30408704 + }, + { + "epoch": 0.01, + "learning_rate": 0.00046130952380952383, + "loss": 4.9085, + "theoretical_loss": 5.7051738617966326, + "tokens_seen": 30474240 + }, + { + "epoch": 0.01, + "learning_rate": 0.00046230158730158733, + "loss": 4.6103, + "theoretical_loss": 5.7031917351493515, + "tokens_seen": 30539776 + }, + { + "epoch": 0.01, + "learning_rate": 0.00046329365079365083, + "loss": 5.0814, + "theoretical_loss": 5.701215045506411, + "tokens_seen": 30605312 + }, + { + "epoch": 0.01, + "learning_rate": 0.00046428571428571433, + "loss": 5.0333, + "theoretical_loss": 5.699243766359421, + "tokens_seen": 30670848 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004652777777777778, + "loss": 5.0393, + "theoretical_loss": 5.697277871385534, + "tokens_seen": 30736384 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004662698412698413, + "loss": 4.813, + "theoretical_loss": 5.695317334445736, + "tokens_seen": 30801920 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004672619047619048, + "loss": 4.8031, + "theoretical_loss": 5.693362129583184, + "tokens_seen": 30867456 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004682539682539683, + "loss": 4.6799, + "theoretical_loss": 5.691412231021549, + "tokens_seen": 30932992 + }, + { + "epoch": 0.01, + "learning_rate": 0.00046924603174603173, + "loss": 4.9719, + "theoretical_loss": 5.689467613163388, + "tokens_seen": 30998528 + }, + { + "epoch": 0.01, + "learning_rate": 0.00047023809523809523, + "loss": 4.7362, + "theoretical_loss": 5.687528250588518, + "tokens_seen": 31064064 + }, + { + "epoch": 0.01, + "objective/train/docs_used": 135224, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 4.8863983154296875, + "objective/train/theoretical_loss": 5.6855941180524265, + "objective/train/tokens_used": 51589600, + "theoretical_loss": 5.6855941180524265, + "tokens_seen": 31129600 + }, + { + "epoch": 0.01, + "learning_rate": 0.00047123015873015874, + "loss": 4.893, + "theoretical_loss": 5.6855941180524265, + "tokens_seen": 31129600 + }, + { + "epoch": 0.01, + "learning_rate": 0.00047222222222222224, + "loss": 4.9991, + "theoretical_loss": 5.683665190484683, + "tokens_seen": 31195136 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004732142857142857, + "loss": 5.1718, + "theoretical_loss": 5.681741442987381, + "tokens_seen": 31260672 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004742063492063492, + "loss": 4.9702, + "theoretical_loss": 5.679822850833591, + "tokens_seen": 31326208 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004751984126984127, + "loss": 4.9426, + "theoretical_loss": 5.677909389465831, + "tokens_seen": 31391744 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004761904761904762, + "loss": 4.5549, + "theoretical_loss": 5.676001034494554, + "tokens_seen": 31457280 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004771825396825397, + "loss": 4.8956, + "theoretical_loss": 5.674097761696653, + "tokens_seen": 31522816 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004781746031746032, + "loss": 4.7593, + "theoretical_loss": 5.672199547013983, + "tokens_seen": 31588352 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004791666666666667, + "loss": 4.7632, + "theoretical_loss": 5.670306366551898, + "tokens_seen": 31653888 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004801587301587302, + "loss": 4.8197, + "theoretical_loss": 5.6684181965778, + "tokens_seen": 31719424 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004811507936507937, + "loss": 5.0023, + "theoretical_loss": 5.666535013519715, + "tokens_seen": 31784960 + }, + { + "epoch": 0.01, + "learning_rate": 0.00048214285714285715, + "loss": 4.265, + "theoretical_loss": 5.6646567939648715, + "tokens_seen": 31850496 + }, + { + "epoch": 0.01, + "learning_rate": 0.00048313492063492065, + "loss": 4.6841, + "theoretical_loss": 5.6627835146583045, + "tokens_seen": 31916032 + }, + { + "epoch": 0.01, + "learning_rate": 0.00048412698412698415, + "loss": 4.9755, + "theoretical_loss": 5.660915152501465, + "tokens_seen": 31981568 + }, + { + "epoch": 0.01, + "learning_rate": 0.00048511904761904765, + "loss": 4.838, + "theoretical_loss": 5.659051684550857, + "tokens_seen": 32047104 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004861111111111111, + "loss": 4.8988, + "theoretical_loss": 5.657193088016677, + "tokens_seen": 32112640 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004871031746031746, + "loss": 4.8451, + "theoretical_loss": 5.655339340261474, + "tokens_seen": 32178176 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004880952380952381, + "loss": 4.9273, + "theoretical_loss": 5.653490418798825, + "tokens_seen": 32243712 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004890873015873016, + "loss": 4.6617, + "theoretical_loss": 5.651646301292022, + "tokens_seen": 32309248 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004900793650793651, + "loss": 4.6989, + "theoretical_loss": 5.649806965552774, + "tokens_seen": 32374784 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004910714285714286, + "loss": 4.6449, + "theoretical_loss": 5.6479723895399205, + "tokens_seen": 32440320 + }, + { + "epoch": 0.01, + "learning_rate": 0.000492063492063492, + "loss": 4.9494, + "theoretical_loss": 5.6461425513581665, + "tokens_seen": 32505856 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004930555555555556, + "loss": 4.6846, + "theoretical_loss": 5.6443174292568195, + "tokens_seen": 32571392 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004940476190476191, + "loss": 4.6929, + "theoretical_loss": 5.6424970016285485, + "tokens_seen": 32636928 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004950396825396826, + "loss": 4.6288, + "theoretical_loss": 5.640681247008156, + "tokens_seen": 32702464 + }, + { + "epoch": 0.01, + "objective/train/docs_used": 136821, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 4.2028489112854, + "objective/train/theoretical_loss": 5.638870144071353, + "objective/train/tokens_used": 53228000, + "theoretical_loss": 5.638870144071353, + "tokens_seen": 32768000 + }, + { + "epoch": 0.01, + "learning_rate": 0.000496031746031746, + "loss": 4.6949, + "theoretical_loss": 5.638870144071353, + "tokens_seen": 32768000 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004970238095238095, + "loss": 4.5009, + "theoretical_loss": 5.637063671633564, + "tokens_seen": 32833536 + }, + { + "epoch": 0.01, + "learning_rate": 0.000498015873015873, + "loss": 4.7564, + "theoretical_loss": 5.635261808648728, + "tokens_seen": 32899072 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004990079365079365, + "loss": 5.0523, + "theoretical_loss": 5.6334645342081195, + "tokens_seen": 32964608 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005, + "loss": 4.6206, + "theoretical_loss": 5.631671827539186, + "tokens_seen": 33030144 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004999899699097292, + "loss": 4.7802, + "theoretical_loss": 5.629883668004389, + "tokens_seen": 33095680 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004999799398194584, + "loss": 4.7891, + "theoretical_loss": 5.628100035100061, + "tokens_seen": 33161216 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004999699097291876, + "loss": 4.8953, + "theoretical_loss": 5.626320908455279, + "tokens_seen": 33226752 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004999598796389167, + "loss": 4.5953, + "theoretical_loss": 5.6245462678307385, + "tokens_seen": 33292288 + }, + { + "epoch": 0.01, + "learning_rate": 0.000499949849548646, + "loss": 4.67, + "theoretical_loss": 5.622776093117652, + "tokens_seen": 33357824 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004999398194583751, + "loss": 4.8824, + "theoretical_loss": 5.621010364336651, + "tokens_seen": 33423360 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004999297893681044, + "loss": 4.7663, + "theoretical_loss": 5.619249061636698, + "tokens_seen": 33488896 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004999197592778335, + "loss": 4.7263, + "theoretical_loss": 5.61749216529402, + "tokens_seen": 33554432 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004999097291875627, + "loss": 4.9792, + "theoretical_loss": 5.615739655711037, + "tokens_seen": 33619968 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004998996990972919, + "loss": 4.9381, + "theoretical_loss": 5.61399151341532, + "tokens_seen": 33685504 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004998896690070211, + "loss": 4.9476, + "theoretical_loss": 5.6122477190585425, + "tokens_seen": 33751040 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004998796389167503, + "loss": 4.5086, + "theoretical_loss": 5.610508253415453, + "tokens_seen": 33816576 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004998696088264795, + "loss": 4.5371, + "theoretical_loss": 5.6087730973828585, + "tokens_seen": 33882112 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004998595787362087, + "loss": 4.5089, + "theoretical_loss": 5.6070422319786095, + "tokens_seen": 33947648 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004998495486459378, + "loss": 4.8347, + "theoretical_loss": 5.605315638340606, + "tokens_seen": 34013184 + }, + { + "epoch": 0.01, + "learning_rate": 0.000499839518555667, + "loss": 4.7779, + "theoretical_loss": 5.603593297725807, + "tokens_seen": 34078720 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004998294884653962, + "loss": 4.7224, + "theoretical_loss": 5.601875191509249, + "tokens_seen": 34144256 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004998194583751254, + "loss": 4.8306, + "theoretical_loss": 5.600161301183084, + "tokens_seen": 34209792 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004998094282848546, + "loss": 5.0491, + "theoretical_loss": 5.598451608355614, + "tokens_seen": 34275328 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004997993981945837, + "loss": 4.6312, + "theoretical_loss": 5.596746094750342, + "tokens_seen": 34340864 + }, + { + "epoch": 0.01, + "objective/train/docs_used": 137555, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 4.4468183517456055, + "objective/train/theoretical_loss": 5.595044742205037, + "objective/train/tokens_used": 54866400, + "theoretical_loss": 5.595044742205037, + "tokens_seen": 34406400 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004997893681043129, + "loss": 4.5163, + "theoretical_loss": 5.595044742205037, + "tokens_seen": 34406400 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004997793380140421, + "loss": 4.9292, + "theoretical_loss": 5.5933475326707995, + "tokens_seen": 34471936 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004997693079237714, + "loss": 4.8149, + "theoretical_loss": 5.591654448211143, + "tokens_seen": 34537472 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004997592778335005, + "loss": 4.954, + "theoretical_loss": 5.589965471001077, + "tokens_seen": 34603008 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004997492477432298, + "loss": 4.9477, + "theoretical_loss": 5.5882805833262115, + "tokens_seen": 34668544 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004997392176529588, + "loss": 4.7965, + "theoretical_loss": 5.586599767581859, + "tokens_seen": 34734080 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004997291875626881, + "loss": 5.0125, + "theoretical_loss": 5.584923006272151, + "tokens_seen": 34799616 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004997191574724173, + "loss": 4.7564, + "theoretical_loss": 5.583250282009159, + "tokens_seen": 34865152 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004997091273821465, + "loss": 4.7508, + "theoretical_loss": 5.581581577512031, + "tokens_seen": 34930688 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004996990972918757, + "loss": 4.7624, + "theoretical_loss": 5.579916875606134, + "tokens_seen": 34996224 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004996890672016048, + "loss": 4.6123, + "theoretical_loss": 5.578256159222196, + "tokens_seen": 35061760 + }, + { + "epoch": 0.01, + "learning_rate": 0.000499679037111334, + "loss": 4.8336, + "theoretical_loss": 5.576599411395472, + "tokens_seen": 35127296 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004996690070210632, + "loss": 4.9514, + "theoretical_loss": 5.574946615264906, + "tokens_seen": 35192832 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004996589769307924, + "loss": 4.7192, + "theoretical_loss": 5.5732977540723105, + "tokens_seen": 35258368 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004996489468405216, + "loss": 4.7461, + "theoretical_loss": 5.571652811161542, + "tokens_seen": 35323904 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004996389167502507, + "loss": 4.6751, + "theoretical_loss": 5.570011769977693, + "tokens_seen": 35389440 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004996288866599799, + "loss": 4.6333, + "theoretical_loss": 5.568374614066299, + "tokens_seen": 35454976 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004996188565697091, + "loss": 4.6241, + "theoretical_loss": 5.566741327072535, + "tokens_seen": 35520512 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004996088264794383, + "loss": 4.8779, + "theoretical_loss": 5.565111892740433, + "tokens_seen": 35586048 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004995987963891675, + "loss": 4.4276, + "theoretical_loss": 5.563486294912105, + "tokens_seen": 35651584 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004995887662988968, + "loss": 4.6749, + "theoretical_loss": 5.56186451752697, + "tokens_seen": 35717120 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004995787362086258, + "loss": 4.665, + "theoretical_loss": 5.560246544620993, + "tokens_seen": 35782656 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004995687061183551, + "loss": 4.7076, + "theoretical_loss": 5.558632360325929, + "tokens_seen": 35848192 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004995586760280842, + "loss": 4.7311, + "theoretical_loss": 5.557021948868571, + "tokens_seen": 35913728 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004995486459378135, + "loss": 4.6598, + "theoretical_loss": 5.555415294570011, + "tokens_seen": 35979264 + }, + { + "epoch": 0.01, + "objective/train/docs_used": 138886, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 4.407133102416992, + "objective/train/theoretical_loss": 5.553812381844907, + "objective/train/tokens_used": 56504800, + "theoretical_loss": 5.553812381844907, + "tokens_seen": 36044800 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004995386158475427, + "loss": 4.5711, + "theoretical_loss": 5.553812381844907, + "tokens_seen": 36044800 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004995285857572718, + "loss": 4.7261, + "theoretical_loss": 5.552213195200755, + "tokens_seen": 36110336 + }, + { + "epoch": 0.01, + "learning_rate": 0.000499518555667001, + "loss": 4.7503, + "theoretical_loss": 5.550617719237167, + "tokens_seen": 36175872 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004995085255767302, + "loss": 4.7123, + "theoretical_loss": 5.549025938645155, + "tokens_seen": 36241408 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004994984954864594, + "loss": 4.8397, + "theoretical_loss": 5.547437838206435, + "tokens_seen": 36306944 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004994884653961886, + "loss": 4.8057, + "theoretical_loss": 5.545853402792717, + "tokens_seen": 36372480 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004994784353059178, + "loss": 4.711, + "theoretical_loss": 5.544272617365014, + "tokens_seen": 36438016 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004994684052156469, + "loss": 4.758, + "theoretical_loss": 5.542695466972956, + "tokens_seen": 36503552 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004994583751253761, + "loss": 4.9674, + "theoretical_loss": 5.541121936754111, + "tokens_seen": 36569088 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004994483450351053, + "loss": 4.8971, + "theoretical_loss": 5.539552011933312, + "tokens_seen": 36634624 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004994383149448345, + "loss": 4.6474, + "theoretical_loss": 5.537985677821986, + "tokens_seen": 36700160 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004994282848545637, + "loss": 4.8243, + "theoretical_loss": 5.536422919817495, + "tokens_seen": 36765696 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004994182547642928, + "loss": 4.7595, + "theoretical_loss": 5.5348637234024824, + "tokens_seen": 36831232 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004994082246740221, + "loss": 4.8208, + "theoretical_loss": 5.53330807414422, + "tokens_seen": 36896768 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004993981945837512, + "loss": 4.5145, + "theoretical_loss": 5.5317559576939725, + "tokens_seen": 36962304 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004993881644934805, + "loss": 4.5815, + "theoretical_loss": 5.530207359786353, + "tokens_seen": 37027840 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004993781344032096, + "loss": 4.7233, + "theoretical_loss": 5.5286622662386975, + "tokens_seen": 37093376 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004993681043129389, + "loss": 4.7717, + "theoretical_loss": 5.52712066295044, + "tokens_seen": 37158912 + }, + { + "epoch": 0.01, + "learning_rate": 0.000499358074222668, + "loss": 4.3869, + "theoretical_loss": 5.525582535902489, + "tokens_seen": 37224448 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004993480441323972, + "loss": 4.5429, + "theoretical_loss": 5.524047871156618, + "tokens_seen": 37289984 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004993380140421264, + "loss": 4.8437, + "theoretical_loss": 5.52251665485486, + "tokens_seen": 37355520 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004993279839518556, + "loss": 4.7926, + "theoretical_loss": 5.520988873218897, + "tokens_seen": 37421056 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004993179538615848, + "loss": 4.8248, + "theoretical_loss": 5.519464512549478, + "tokens_seen": 37486592 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004993079237713139, + "loss": 4.7691, + "theoretical_loss": 5.5179435592258095, + "tokens_seen": 37552128 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004992978936810431, + "loss": 4.6871, + "theoretical_loss": 5.516425999704987, + "tokens_seen": 37617664 + }, + { + "epoch": 0.01, + "objective/train/docs_used": 139553, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 4.058813571929932, + "objective/train/theoretical_loss": 5.514911820521407, + "objective/train/tokens_used": 58143200, + "theoretical_loss": 5.514911820521407, + "tokens_seen": 37683200 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004992878635907723, + "loss": 4.5181, + "theoretical_loss": 5.514911820521407, + "tokens_seen": 37683200 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004992778335005015, + "loss": 4.4909, + "theoretical_loss": 5.5134010082861895, + "tokens_seen": 37748736 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004992678034102307, + "loss": 4.5355, + "theoretical_loss": 5.511893549686616, + "tokens_seen": 37814272 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004992577733199598, + "loss": 4.7593, + "theoretical_loss": 5.51038943148556, + "tokens_seen": 37879808 + }, + { + "epoch": 0.01, + "learning_rate": 0.000499247743229689, + "loss": 4.6539, + "theoretical_loss": 5.508888640520928, + "tokens_seen": 37945344 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004992377131394183, + "loss": 4.5281, + "theoretical_loss": 5.50739116370511, + "tokens_seen": 38010880 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004992276830491475, + "loss": 4.4719, + "theoretical_loss": 5.505896988024423, + "tokens_seen": 38076416 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004992176529588767, + "loss": 4.8288, + "theoretical_loss": 5.5044061005385725, + "tokens_seen": 38141952 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004992076228686059, + "loss": 4.376, + "theoretical_loss": 5.502918488380116, + "tokens_seen": 38207488 + }, + { + "epoch": 0.01, + "learning_rate": 0.000499197592778335, + "loss": 4.5187, + "theoretical_loss": 5.501434138753918, + "tokens_seen": 38273024 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004991875626880642, + "loss": 4.5366, + "theoretical_loss": 5.499953038936635, + "tokens_seen": 38338560 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004991775325977934, + "loss": 4.5867, + "theoretical_loss": 5.498475176276176, + "tokens_seen": 38404096 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004991675025075226, + "loss": 4.5793, + "theoretical_loss": 5.497000538191195, + "tokens_seen": 38469632 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004991574724172518, + "loss": 4.4392, + "theoretical_loss": 5.495529112170568, + "tokens_seen": 38535168 + }, + { + "epoch": 0.01, + "learning_rate": 0.000499147442326981, + "loss": 4.6516, + "theoretical_loss": 5.494060885772887, + "tokens_seen": 38600704 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004991374122367101, + "loss": 4.5034, + "theoretical_loss": 5.492595846625951, + "tokens_seen": 38666240 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004991273821464393, + "loss": 4.8256, + "theoretical_loss": 5.491133982426266, + "tokens_seen": 38731776 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004991173520561685, + "loss": 5.0062, + "theoretical_loss": 5.489675280938547, + "tokens_seen": 38797312 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004991073219658977, + "loss": 4.6335, + "theoretical_loss": 5.488219729995227, + "tokens_seen": 38862848 + }, + { + "epoch": 0.01, + "learning_rate": 0.000499097291875627, + "loss": 4.2841, + "theoretical_loss": 5.486767317495966, + "tokens_seen": 38928384 + }, + { + "epoch": 0.01, + "learning_rate": 0.000499087261785356, + "loss": 4.8828, + "theoretical_loss": 5.48531803140717, + "tokens_seen": 38993920 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004990772316950853, + "loss": 4.6358, + "theoretical_loss": 5.483871859761511, + "tokens_seen": 39059456 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004990672016048144, + "loss": 4.7315, + "theoretical_loss": 5.482428790657449, + "tokens_seen": 39124992 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004990571715145437, + "loss": 4.8301, + "theoretical_loss": 5.480988812258763, + "tokens_seen": 39190528 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004990471414242729, + "loss": 4.6629, + "theoretical_loss": 5.479551912794086, + "tokens_seen": 39256064 + }, + { + "epoch": 0.01, + "objective/train/docs_used": 141085, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 4.329990386962891, + "objective/train/theoretical_loss": 5.478118080556438, + "objective/train/tokens_used": 59781600, + "theoretical_loss": 5.478118080556438, + "tokens_seen": 39321600 + }, + { + "epoch": 0.01, + "learning_rate": 0.000499037111334002, + "loss": 4.4817, + "theoretical_loss": 5.478118080556438, + "tokens_seen": 39321600 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004990270812437312, + "loss": 4.7964, + "theoretical_loss": 5.476687303902768, + "tokens_seen": 39387136 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004990170511534604, + "loss": 4.6476, + "theoretical_loss": 5.475259571253502, + "tokens_seen": 39452672 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004990070210631896, + "loss": 4.5687, + "theoretical_loss": 5.473834871092089, + "tokens_seen": 39518208 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004989969909729188, + "loss": 4.6718, + "theoretical_loss": 5.4724131919645576, + "tokens_seen": 39583744 + }, + { + "epoch": 0.01, + "learning_rate": 0.000498986960882648, + "loss": 4.5519, + "theoretical_loss": 5.470994522479069, + "tokens_seen": 39649280 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004989769307923771, + "loss": 4.8962, + "theoretical_loss": 5.4695788513054815, + "tokens_seen": 39714816 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004989669007021063, + "loss": 4.7691, + "theoretical_loss": 5.468166167174912, + "tokens_seen": 39780352 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004989568706118355, + "loss": 4.8426, + "theoretical_loss": 5.466756458879306, + "tokens_seen": 39845888 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004989468405215647, + "loss": 4.3942, + "theoretical_loss": 5.465349715271013, + "tokens_seen": 39911424 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004989368104312939, + "loss": 4.5906, + "theoretical_loss": 5.463945925262355, + "tokens_seen": 39976960 + }, + { + "epoch": 0.01, + "learning_rate": 0.000498926780341023, + "loss": 4.7871, + "theoretical_loss": 5.462545077825214, + "tokens_seen": 40042496 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004989167502507523, + "loss": 4.3864, + "theoretical_loss": 5.461147161990611, + "tokens_seen": 40108032 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004989067201604814, + "loss": 4.5497, + "theoretical_loss": 5.459752166848292, + "tokens_seen": 40173568 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004988966900702107, + "loss": 4.45, + "theoretical_loss": 5.458360081546321, + "tokens_seen": 40239104 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004988866599799398, + "loss": 4.5501, + "theoretical_loss": 5.456970895290674, + "tokens_seen": 40304640 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004988766298896691, + "loss": 4.6729, + "theoretical_loss": 5.455584597344835, + "tokens_seen": 40370176 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004988665997993982, + "loss": 4.7367, + "theoretical_loss": 5.454201177029395, + "tokens_seen": 40435712 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004988565697091274, + "loss": 4.5589, + "theoretical_loss": 5.452820623721662, + "tokens_seen": 40501248 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004988465396188566, + "loss": 4.6359, + "theoretical_loss": 5.45144292685526, + "tokens_seen": 40566784 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004988365095285858, + "loss": 4.5998, + "theoretical_loss": 5.450068075919752, + "tokens_seen": 40632320 + }, + { + "epoch": 0.01, + "learning_rate": 0.000498826479438315, + "loss": 4.6231, + "theoretical_loss": 5.44869606046024, + "tokens_seen": 40697856 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004988164493480441, + "loss": 4.5891, + "theoretical_loss": 5.447326870076996, + "tokens_seen": 40763392 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004988064192577733, + "loss": 4.7368, + "theoretical_loss": 5.445960494425072, + "tokens_seen": 40828928 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004987963891675025, + "loss": 4.4854, + "theoretical_loss": 5.444596923213931, + "tokens_seen": 40894464 + }, + { + "epoch": 0.01, + "objective/train/docs_used": 141824, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 4.5050435066223145, + "objective/train/theoretical_loss": 5.443236146207074, + "objective/train/tokens_used": 61420000, + "theoretical_loss": 5.443236146207074, + "tokens_seen": 40960000 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004987863590772317, + "loss": 4.6512, + "theoretical_loss": 5.443236146207074, + "tokens_seen": 40960000 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004987763289869609, + "loss": 4.5578, + "theoretical_loss": 5.441878153221662, + "tokens_seen": 41025536 + }, + { + "epoch": 0.01, + "learning_rate": 0.00049876629889669, + "loss": 4.7668, + "theoretical_loss": 5.440522934128164, + "tokens_seen": 41091072 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004987562688064192, + "loss": 4.523, + "theoretical_loss": 5.439170478849976, + "tokens_seen": 41156608 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004987462387161484, + "loss": 4.6254, + "theoretical_loss": 5.437820777363078, + "tokens_seen": 41222144 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004987362086258777, + "loss": 4.6625, + "theoretical_loss": 5.4364738196956655, + "tokens_seen": 41287680 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004987261785356068, + "loss": 4.6117, + "theoretical_loss": 5.435129595927794, + "tokens_seen": 41353216 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004987161484453361, + "loss": 4.7588, + "theoretical_loss": 5.433788096191039, + "tokens_seen": 41418752 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004987061183550651, + "loss": 4.6583, + "theoretical_loss": 5.432449310668134, + "tokens_seen": 41484288 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004986960882647944, + "loss": 4.5594, + "theoretical_loss": 5.4311132295926345, + "tokens_seen": 41549824 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004986860581745236, + "loss": 4.682, + "theoretical_loss": 5.42977984324857, + "tokens_seen": 41615360 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004986760280842528, + "loss": 4.5311, + "theoretical_loss": 5.428449141970107, + "tokens_seen": 41680896 + }, + { + "epoch": 0.01, + "learning_rate": 0.000498665997993982, + "loss": 4.6285, + "theoretical_loss": 5.427121116141212, + "tokens_seen": 41746432 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004986559679037111, + "loss": 4.4254, + "theoretical_loss": 5.42579575619531, + "tokens_seen": 41811968 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004986459378134403, + "loss": 4.3521, + "theoretical_loss": 5.424473052614967, + "tokens_seen": 41877504 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004986359077231695, + "loss": 4.2651, + "theoretical_loss": 5.423152995931552, + "tokens_seen": 41943040 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004986258776328987, + "loss": 4.4635, + "theoretical_loss": 5.421835576724906, + "tokens_seen": 42008576 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004986158475426279, + "loss": 4.4402, + "theoretical_loss": 5.420520785623031, + "tokens_seen": 42074112 + }, + { + "epoch": 0.01, + "learning_rate": 0.000498605817452357, + "loss": 4.4318, + "theoretical_loss": 5.4192086133017625, + "tokens_seen": 42139648 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004985957873620862, + "loss": 4.6869, + "theoretical_loss": 5.417899050484451, + "tokens_seen": 42205184 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004985857572718154, + "loss": 4.5541, + "theoretical_loss": 5.416592087941646, + "tokens_seen": 42270720 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004985757271815446, + "loss": 4.6914, + "theoretical_loss": 5.415287716490787, + "tokens_seen": 42336256 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004985656970912738, + "loss": 4.4525, + "theoretical_loss": 5.413985926995892, + "tokens_seen": 42401792 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004985556670010031, + "loss": 4.5715, + "theoretical_loss": 5.412686710367245, + "tokens_seen": 42467328 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004985456369107321, + "loss": 4.4437, + "theoretical_loss": 5.411390057561097, + "tokens_seen": 42532864 + }, + { + "epoch": 0.01, + "objective/train/docs_used": 143445, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 4.786459922790527, + "objective/train/theoretical_loss": 5.410095959579362, + "objective/train/tokens_used": 63058400, + "theoretical_loss": 5.410095959579362, + "tokens_seen": 42598400 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004985356068204614, + "loss": 4.5962, + "theoretical_loss": 5.410095959579362, + "tokens_seen": 42598400 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004985255767301905, + "loss": 4.64, + "theoretical_loss": 5.408804407469308, + "tokens_seen": 42663936 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004985155466399198, + "loss": 4.639, + "theoretical_loss": 5.407515392323276, + "tokens_seen": 42729472 + }, + { + "epoch": 0.01, + "learning_rate": 0.000498505516549649, + "loss": 4.5622, + "theoretical_loss": 5.406228905278368, + "tokens_seen": 42795008 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004984954864593782, + "loss": 4.7461, + "theoretical_loss": 5.404944937516161, + "tokens_seen": 42860544 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004984854563691073, + "loss": 4.6478, + "theoretical_loss": 5.403663480262418, + "tokens_seen": 42926080 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004984754262788365, + "loss": 4.8657, + "theoretical_loss": 5.402384524786797, + "tokens_seen": 42991616 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004984653961885657, + "loss": 4.6469, + "theoretical_loss": 5.401108062402562, + "tokens_seen": 43057152 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004984553660982949, + "loss": 4.3658, + "theoretical_loss": 5.399834084466306, + "tokens_seen": 43122688 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004984453360080241, + "loss": 4.6816, + "theoretical_loss": 5.398562582377666, + "tokens_seen": 43188224 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004984353059177532, + "loss": 4.3855, + "theoretical_loss": 5.397293547579041, + "tokens_seen": 43253760 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004984252758274825, + "loss": 4.568, + "theoretical_loss": 5.396026971555319, + "tokens_seen": 43319296 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004984152457372116, + "loss": 4.4991, + "theoretical_loss": 5.394762845833601, + "tokens_seen": 43384832 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004984052156469409, + "loss": 4.583, + "theoretical_loss": 5.393501161982926, + "tokens_seen": 43450368 + }, + { + "epoch": 0.01, + "learning_rate": 0.00049839518555667, + "loss": 4.2179, + "theoretical_loss": 5.392241911614005, + "tokens_seen": 43515904 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004983851554663993, + "loss": 4.5224, + "theoretical_loss": 5.390985086378949, + "tokens_seen": 43581440 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004983751253761284, + "loss": 4.7938, + "theoretical_loss": 5.389730677971002, + "tokens_seen": 43646976 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004983650952858576, + "loss": 4.4534, + "theoretical_loss": 5.388478678124285, + "tokens_seen": 43712512 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004983550651955868, + "loss": 4.2336, + "theoretical_loss": 5.387229078613521, + "tokens_seen": 43778048 + }, + { + "epoch": 0.01, + "learning_rate": 0.000498345035105316, + "loss": 4.6225, + "theoretical_loss": 5.385981871253785, + "tokens_seen": 43843584 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004983350050150452, + "loss": 4.4481, + "theoretical_loss": 5.384737047900243, + "tokens_seen": 43909120 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004983249749247743, + "loss": 4.5426, + "theoretical_loss": 5.3834946004478965, + "tokens_seen": 43974656 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004983149448345035, + "loss": 4.722, + "theoretical_loss": 5.382254520831328, + "tokens_seen": 44040192 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004983049147442327, + "loss": 4.4346, + "theoretical_loss": 5.381016801024449, + "tokens_seen": 44105728 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004982948846539619, + "loss": 4.5081, + "theoretical_loss": 5.379781433040252, + "tokens_seen": 44171264 + }, + { + "epoch": 0.01, + "objective/train/docs_used": 144147, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 4.279332637786865, + "objective/train/theoretical_loss": 5.378548408930558, + "objective/train/tokens_used": 64696800, + "theoretical_loss": 5.378548408930558, + "tokens_seen": 44236800 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004982848545636911, + "loss": 4.6137, + "theoretical_loss": 5.378548408930558, + "tokens_seen": 44236800 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004982748244734202, + "loss": 4.4353, + "theoretical_loss": 5.377317720785777, + "tokens_seen": 44302336 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004982647943831494, + "loss": 4.4919, + "theoretical_loss": 5.37608936073466, + "tokens_seen": 44367872 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004982547642928786, + "loss": 4.3873, + "theoretical_loss": 5.374863320944057, + "tokens_seen": 44433408 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004982447342026079, + "loss": 4.5235, + "theoretical_loss": 5.373639593618675, + "tokens_seen": 44498944 + }, + { + "epoch": 0.01, + "learning_rate": 0.000498234704112337, + "loss": 4.5877, + "theoretical_loss": 5.372418171000847, + "tokens_seen": 44564480 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004982246740220663, + "loss": 4.3387, + "theoretical_loss": 5.371199045370283, + "tokens_seen": 44630016 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004982146439317953, + "loss": 4.4968, + "theoretical_loss": 5.369982209043851, + "tokens_seen": 44695552 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004982046138415246, + "loss": 4.5509, + "theoretical_loss": 5.368767654375327, + "tokens_seen": 44761088 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004981945837512538, + "loss": 4.6583, + "theoretical_loss": 5.367555373755179, + "tokens_seen": 44826624 + }, + { + "epoch": 0.01, + "learning_rate": 0.000498184553660983, + "loss": 4.2686, + "theoretical_loss": 5.366345359610327, + "tokens_seen": 44892160 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004981745235707122, + "loss": 4.5189, + "theoretical_loss": 5.365137604403923, + "tokens_seen": 44957696 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004981644934804413, + "loss": 4.7356, + "theoretical_loss": 5.363932100635117, + "tokens_seen": 45023232 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004981544633901705, + "loss": 4.5647, + "theoretical_loss": 5.362728840838843, + "tokens_seen": 45088768 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004981444332998997, + "loss": 4.4656, + "theoretical_loss": 5.361527817585586, + "tokens_seen": 45154304 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004981344032096289, + "loss": 4.6026, + "theoretical_loss": 5.360329023481169, + "tokens_seen": 45219840 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004981243731193581, + "loss": 4.4202, + "theoretical_loss": 5.359132451166534, + "tokens_seen": 45285376 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004981143430290873, + "loss": 4.5494, + "theoretical_loss": 5.357938093317518, + "tokens_seen": 45350912 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004981043129388164, + "loss": 4.3413, + "theoretical_loss": 5.356745942644645, + "tokens_seen": 45416448 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004980942828485456, + "loss": 4.2947, + "theoretical_loss": 5.355555991892905, + "tokens_seen": 45481984 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004980842527582748, + "loss": 4.7378, + "theoretical_loss": 5.35436823384155, + "tokens_seen": 45547520 + }, + { + "epoch": 0.01, + "learning_rate": 0.000498074222668004, + "loss": 4.5112, + "theoretical_loss": 5.353182661303873, + "tokens_seen": 45613056 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004980641925777333, + "loss": 4.6471, + "theoretical_loss": 5.35199926712701, + "tokens_seen": 45678592 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004980541624874623, + "loss": 4.4652, + "theoretical_loss": 5.350818044191721, + "tokens_seen": 45744128 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004980441323971916, + "loss": 4.3502, + "theoretical_loss": 5.349638985412193, + "tokens_seen": 45809664 + }, + { + "epoch": 0.01, + "objective/train/docs_used": 144842, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 4.692098617553711, + "objective/train/theoretical_loss": 5.348462083735834, + "objective/train/tokens_used": 66335200, + "theoretical_loss": 5.348462083735834, + "tokens_seen": 45875200 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004980341023069207, + "loss": 4.4348, + "theoretical_loss": 5.348462083735834, + "tokens_seen": 45875200 + }, + { + "epoch": 0.01, + "learning_rate": 0.00049802407221665, + "loss": 4.45, + "theoretical_loss": 5.347287332143064, + "tokens_seen": 45940736 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004980140421263792, + "loss": 4.4526, + "theoretical_loss": 5.346114723647119, + "tokens_seen": 46006272 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004980040120361084, + "loss": 4.5448, + "theoretical_loss": 5.344944251293852, + "tokens_seen": 46071808 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004979939819458375, + "loss": 4.3139, + "theoretical_loss": 5.343775908161532, + "tokens_seen": 46137344 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004979839518555667, + "loss": 4.5183, + "theoretical_loss": 5.342609687360644, + "tokens_seen": 46202880 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004979739217652959, + "loss": 4.2806, + "theoretical_loss": 5.341445582033705, + "tokens_seen": 46268416 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004979638916750251, + "loss": 4.6082, + "theoretical_loss": 5.3402835853550545, + "tokens_seen": 46333952 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004979538615847543, + "loss": 4.286, + "theoretical_loss": 5.339123690530673, + "tokens_seen": 46399488 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004979438314944834, + "loss": 4.4594, + "theoretical_loss": 5.337965890797989, + "tokens_seen": 46465024 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004979338014042126, + "loss": 4.4121, + "theoretical_loss": 5.336810179425685, + "tokens_seen": 46530560 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004979237713139418, + "loss": 4.3507, + "theoretical_loss": 5.335656549713516, + "tokens_seen": 46596096 + }, + { + "epoch": 0.01, + "learning_rate": 0.000497913741223671, + "loss": 4.4211, + "theoretical_loss": 5.334504994992115, + "tokens_seen": 46661632 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004979037111334002, + "loss": 4.548, + "theoretical_loss": 5.333355508622814, + "tokens_seen": 46727168 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004978936810431293, + "loss": 4.4084, + "theoretical_loss": 5.332208083997459, + "tokens_seen": 46792704 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004978836509528586, + "loss": 4.3348, + "theoretical_loss": 5.33106271453822, + "tokens_seen": 46858240 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004978736208625877, + "loss": 4.3399, + "theoretical_loss": 5.329919393697422, + "tokens_seen": 46923776 + }, + { + "epoch": 0.01, + "learning_rate": 0.000497863590772317, + "loss": 4.2743, + "theoretical_loss": 5.328778114957351, + "tokens_seen": 46989312 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004978535606820461, + "loss": 4.3191, + "theoretical_loss": 5.327638871830089, + "tokens_seen": 47054848 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004978435305917754, + "loss": 4.3063, + "theoretical_loss": 5.326501657857326, + "tokens_seen": 47120384 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004978335005015045, + "loss": 4.7146, + "theoretical_loss": 5.32536646661019, + "tokens_seen": 47185920 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004978234704112337, + "loss": 4.5358, + "theoretical_loss": 5.324233291689069, + "tokens_seen": 47251456 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004978134403209629, + "loss": 4.3223, + "theoretical_loss": 5.323102126723439, + "tokens_seen": 47316992 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004978034102306921, + "loss": 4.3505, + "theoretical_loss": 5.321972965371691, + "tokens_seen": 47382528 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004977933801404213, + "loss": 4.515, + "theoretical_loss": 5.320845801320959, + "tokens_seen": 47448064 + }, + { + "epoch": 0.01, + "objective/train/docs_used": 146139, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 4.262459754943848, + "objective/train/theoretical_loss": 5.319720628286955, + "objective/train/tokens_used": 67973600, + "theoretical_loss": 5.319720628286955, + "tokens_seen": 47513600 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004977833500501504, + "loss": 4.315, + "theoretical_loss": 5.319720628286955, + "tokens_seen": 47513600 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004977733199598796, + "loss": 4.3331, + "theoretical_loss": 5.318597440013795, + "tokens_seen": 47579136 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004977632898696088, + "loss": 4.458, + "theoretical_loss": 5.317476230273831, + "tokens_seen": 47644672 + }, + { + "epoch": 0.01, + "learning_rate": 0.000497753259779338, + "loss": 4.3781, + "theoretical_loss": 5.316356992867491, + "tokens_seen": 47710208 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004977432296890672, + "loss": 4.3273, + "theoretical_loss": 5.31523972162311, + "tokens_seen": 47775744 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004977331995987965, + "loss": 4.613, + "theoretical_loss": 5.314124410396767, + "tokens_seen": 47841280 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004977231695085255, + "loss": 4.3982, + "theoretical_loss": 5.31301105307212, + "tokens_seen": 47906816 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004977131394182548, + "loss": 4.3032, + "theoretical_loss": 5.311899643560251, + "tokens_seen": 47972352 + }, + { + "epoch": 0.01, + "learning_rate": 0.000497703109327984, + "loss": 4.449, + "theoretical_loss": 5.310790175799497, + "tokens_seen": 48037888 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004976930792377132, + "loss": 4.4828, + "theoretical_loss": 5.3096826437553, + "tokens_seen": 48103424 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004976830491474424, + "loss": 3.9312, + "theoretical_loss": 5.308577041420046, + "tokens_seen": 48168960 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004976730190571715, + "loss": 4.3421, + "theoretical_loss": 5.3074733628129005, + "tokens_seen": 48234496 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004976629889669007, + "loss": 4.5473, + "theoretical_loss": 5.3063716019796665, + "tokens_seen": 48300032 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004976529588766299, + "loss": 4.5363, + "theoretical_loss": 5.305271752992619, + "tokens_seen": 48365568 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004976429287863591, + "loss": 4.5261, + "theoretical_loss": 5.304173809950358, + "tokens_seen": 48431104 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004976328986960883, + "loss": 4.2883, + "theoretical_loss": 5.303077766977653, + "tokens_seen": 48496640 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004976228686058175, + "loss": 4.7128, + "theoretical_loss": 5.3019836182252895, + "tokens_seen": 48562176 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004976128385155466, + "loss": 4.4607, + "theoretical_loss": 5.300891357869929, + "tokens_seen": 48627712 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004976028084252758, + "loss": 4.5213, + "theoretical_loss": 5.299800980113945, + "tokens_seen": 48693248 + }, + { + "epoch": 0.01, + "learning_rate": 0.000497592778335005, + "loss": 4.3922, + "theoretical_loss": 5.298712479185288, + "tokens_seen": 48758784 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004975827482447342, + "loss": 4.3163, + "theoretical_loss": 5.297625849337331, + "tokens_seen": 48824320 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004975727181544635, + "loss": 4.2912, + "theoretical_loss": 5.296541084848727, + "tokens_seen": 48889856 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004975626880641925, + "loss": 4.47, + "theoretical_loss": 5.295458180023262, + "tokens_seen": 48955392 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004975526579739218, + "loss": 4.3223, + "theoretical_loss": 5.294377129189715, + "tokens_seen": 49020928 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004975426278836509, + "loss": 4.2547, + "theoretical_loss": 5.293297926701706, + "tokens_seen": 49086464 + }, + { + "epoch": 0.01, + "objective/train/docs_used": 146831, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 4.401537895202637, + "objective/train/theoretical_loss": 5.292220566937567, + "objective/train/tokens_used": 69612000, + "theoretical_loss": 5.292220566937567, + "tokens_seen": 49152000 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004975325977933802, + "loss": 4.4595, + "theoretical_loss": 5.292220566937567, + "tokens_seen": 49152000 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004975225677031094, + "loss": 4.2178, + "theoretical_loss": 5.29114504430019, + "tokens_seen": 49217536 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004975125376128386, + "loss": 4.2229, + "theoretical_loss": 5.290071353216895, + "tokens_seen": 49283072 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004975025075225677, + "loss": 4.3304, + "theoretical_loss": 5.288999488139284, + "tokens_seen": 49348608 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004974924774322969, + "loss": 4.2719, + "theoretical_loss": 5.28792944354311, + "tokens_seen": 49414144 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004974824473420261, + "loss": 4.4284, + "theoretical_loss": 5.286861213928137, + "tokens_seen": 49479680 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004974724172517553, + "loss": 4.5986, + "theoretical_loss": 5.285794793817999, + "tokens_seen": 49545216 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004974623871614845, + "loss": 4.2011, + "theoretical_loss": 5.284730177760077, + "tokens_seen": 49610752 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004974523570712136, + "loss": 4.3674, + "theoretical_loss": 5.283667360325351, + "tokens_seen": 49676288 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004974423269809428, + "loss": 4.5679, + "theoretical_loss": 5.2826063361082785, + "tokens_seen": 49741824 + }, + { + "epoch": 0.02, + "learning_rate": 0.000497432296890672, + "loss": 4.6443, + "theoretical_loss": 5.281547099726654, + "tokens_seen": 49807360 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004974222668004012, + "loss": 4.2155, + "theoretical_loss": 5.280489645821483, + "tokens_seen": 49872896 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004974122367101304, + "loss": 4.2667, + "theoretical_loss": 5.279433969056848, + "tokens_seen": 49938432 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004974022066198595, + "loss": 4.4801, + "theoretical_loss": 5.278380064119782, + "tokens_seen": 50003968 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004973921765295888, + "loss": 4.2357, + "theoretical_loss": 5.277327925720137, + "tokens_seen": 50069504 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004973821464393179, + "loss": 4.7003, + "theoretical_loss": 5.276277548590457, + "tokens_seen": 50135040 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004973721163490472, + "loss": 4.1875, + "theoretical_loss": 5.275228927485855, + "tokens_seen": 50200576 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004973620862587763, + "loss": 4.3904, + "theoretical_loss": 5.2741820571838804, + "tokens_seen": 50266112 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004973520561685056, + "loss": 4.3429, + "theoretical_loss": 5.273136932484399, + "tokens_seen": 50331648 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004973420260782347, + "loss": 4.6097, + "theoretical_loss": 5.272093548209467, + "tokens_seen": 50397184 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004973319959879639, + "loss": 4.1553, + "theoretical_loss": 5.271051899203207, + "tokens_seen": 50462720 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004973219658976931, + "loss": 4.3946, + "theoretical_loss": 5.270011980331685, + "tokens_seen": 50528256 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004973119358074223, + "loss": 4.1563, + "theoretical_loss": 5.268973786482794, + "tokens_seen": 50593792 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004973019057171515, + "loss": 4.4526, + "theoretical_loss": 5.267937312566123, + "tokens_seen": 50659328 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004972918756268806, + "loss": 4.3346, + "theoretical_loss": 5.266902553512847, + "tokens_seen": 50724864 + }, + { + "epoch": 0.02, + "objective/train/docs_used": 148451, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.5394937992095947, + "objective/train/theoretical_loss": 5.265869504275602, + "objective/train/tokens_used": 71250400, + "theoretical_loss": 5.265869504275602, + "tokens_seen": 50790400 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004972818455366098, + "loss": 4.2104, + "theoretical_loss": 5.265869504275602, + "tokens_seen": 50790400 + }, + { + "epoch": 0.02, + "learning_rate": 0.000497271815446339, + "loss": 4.6599, + "theoretical_loss": 5.264838159828369, + "tokens_seen": 50855936 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004972617853560682, + "loss": 4.5403, + "theoretical_loss": 5.263808515166355, + "tokens_seen": 50921472 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004972517552657974, + "loss": 3.8993, + "theoretical_loss": 5.262780565305875, + "tokens_seen": 50987008 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004972417251755266, + "loss": 4.4009, + "theoretical_loss": 5.261754305284241, + "tokens_seen": 51052544 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004972316950852557, + "loss": 4.4019, + "theoretical_loss": 5.260729730159641, + "tokens_seen": 51118080 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004972216649949849, + "loss": 4.191, + "theoretical_loss": 5.259706835011027, + "tokens_seen": 51183616 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004972116349047142, + "loss": 4.3533, + "theoretical_loss": 5.2586856149380035, + "tokens_seen": 51249152 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004972016048144433, + "loss": 4.1959, + "theoretical_loss": 5.257666065060709, + "tokens_seen": 51314688 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004971915747241726, + "loss": 4.5251, + "theoretical_loss": 5.256648180519708, + "tokens_seen": 51380224 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004971815446339017, + "loss": 4.348, + "theoretical_loss": 5.255631956475881, + "tokens_seen": 51445760 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004971715145436309, + "loss": 4.4193, + "theoretical_loss": 5.25461738811031, + "tokens_seen": 51511296 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004971614844533601, + "loss": 4.2312, + "theoretical_loss": 5.25360447062417, + "tokens_seen": 51576832 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004971514543630893, + "loss": 4.6115, + "theoretical_loss": 5.252593199238619, + "tokens_seen": 51642368 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004971414242728185, + "loss": 4.2891, + "theoretical_loss": 5.2515835691946915, + "tokens_seen": 51707904 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004971313941825477, + "loss": 4.0652, + "theoretical_loss": 5.2505755757531904, + "tokens_seen": 51773440 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004971213640922768, + "loss": 4.5308, + "theoretical_loss": 5.24956921419458, + "tokens_seen": 51838976 + }, + { + "epoch": 0.02, + "learning_rate": 0.000497111334002006, + "loss": 4.2823, + "theoretical_loss": 5.248564479818876, + "tokens_seen": 51904512 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004971013039117352, + "loss": 4.3883, + "theoretical_loss": 5.247561367945544, + "tokens_seen": 51970048 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004970912738214644, + "loss": 4.3249, + "theoretical_loss": 5.246559873913396, + "tokens_seen": 52035584 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004970812437311936, + "loss": 4.3358, + "theoretical_loss": 5.245559993080484, + "tokens_seen": 52101120 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004970712136409227, + "loss": 4.0621, + "theoretical_loss": 5.24456172082399, + "tokens_seen": 52166656 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004970611835506519, + "loss": 4.5677, + "theoretical_loss": 5.243565052540136, + "tokens_seen": 52232192 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004970511534603811, + "loss": 4.4829, + "theoretical_loss": 5.242569983644074, + "tokens_seen": 52297728 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004970411233701103, + "loss": 4.1669, + "theoretical_loss": 5.241576509569784, + "tokens_seen": 52363264 + }, + { + "epoch": 0.02, + "objective/train/docs_used": 149172, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 4.183032989501953, + "objective/train/theoretical_loss": 5.240584625769978, + "objective/train/tokens_used": 72888800, + "theoretical_loss": 5.240584625769978, + "tokens_seen": 52428800 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004970310932798396, + "loss": 4.3346, + "theoretical_loss": 5.240584625769978, + "tokens_seen": 52428800 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004970210631895686, + "loss": 4.3921, + "theoretical_loss": 5.239594327715992, + "tokens_seen": 52494336 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004970110330992979, + "loss": 4.4333, + "theoretical_loss": 5.238605610897698, + "tokens_seen": 52559872 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004970010030090271, + "loss": 4.2883, + "theoretical_loss": 5.237618470823394, + "tokens_seen": 52625408 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004969909729187563, + "loss": 4.1672, + "theoretical_loss": 5.2366329030197125, + "tokens_seen": 52690944 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004969809428284855, + "loss": 4.335, + "theoretical_loss": 5.235648903031521, + "tokens_seen": 52756480 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004969709127382147, + "loss": 4.1939, + "theoretical_loss": 5.2346664664218245, + "tokens_seen": 52822016 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004969608826479438, + "loss": 4.2423, + "theoretical_loss": 5.233685588771669, + "tokens_seen": 52887552 + }, + { + "epoch": 0.02, + "learning_rate": 0.000496950852557673, + "loss": 4.1699, + "theoretical_loss": 5.232706265680049, + "tokens_seen": 52953088 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004969408224674022, + "loss": 4.3159, + "theoretical_loss": 5.231728492763811, + "tokens_seen": 53018624 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004969307923771314, + "loss": 3.9781, + "theoretical_loss": 5.230752265657554, + "tokens_seen": 53084160 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004969207622868606, + "loss": 4.0656, + "theoretical_loss": 5.229777580013545, + "tokens_seen": 53149696 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004969107321965897, + "loss": 4.6733, + "theoretical_loss": 5.228804431501619, + "tokens_seen": 53215232 + }, + { + "epoch": 0.02, + "learning_rate": 0.000496900702106319, + "loss": 4.4798, + "theoretical_loss": 5.227832815809087, + "tokens_seen": 53280768 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004968906720160481, + "loss": 4.4063, + "theoretical_loss": 5.226862728640651, + "tokens_seen": 53346304 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004968806419257774, + "loss": 4.3503, + "theoretical_loss": 5.2258941657183, + "tokens_seen": 53411840 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004968706118355065, + "loss": 4.346, + "theoretical_loss": 5.2249271227812315, + "tokens_seen": 53477376 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004968605817452358, + "loss": 4.3345, + "theoretical_loss": 5.223961595585755, + "tokens_seen": 53542912 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004968505516549649, + "loss": 4.2838, + "theoretical_loss": 5.222997579905204, + "tokens_seen": 53608448 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004968405215646941, + "loss": 3.9395, + "theoretical_loss": 5.222035071529845, + "tokens_seen": 53673984 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004968304914744233, + "loss": 4.2813, + "theoretical_loss": 5.2210740662667945, + "tokens_seen": 53739520 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004968204613841525, + "loss": 4.2565, + "theoretical_loss": 5.220114559939923, + "tokens_seen": 53805056 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004968104312938817, + "loss": 4.2532, + "theoretical_loss": 5.219156548389775, + "tokens_seen": 53870592 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004968004012036108, + "loss": 4.1641, + "theoretical_loss": 5.218200027473481, + "tokens_seen": 53936128 + }, + { + "epoch": 0.02, + "learning_rate": 0.00049679037111334, + "loss": 4.2325, + "theoretical_loss": 5.217244993064664, + "tokens_seen": 54001664 + }, + { + "epoch": 0.02, + "objective/train/docs_used": 150441, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 4.059125900268555, + "objective/train/theoretical_loss": 5.216291441053366, + "objective/train/tokens_used": 74527200, + "theoretical_loss": 5.216291441053366, + "tokens_seen": 54067200 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004967803410230692, + "loss": 4.3793, + "theoretical_loss": 5.216291441053366, + "tokens_seen": 54067200 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004967703109327984, + "loss": 4.3139, + "theoretical_loss": 5.215339367345955, + "tokens_seen": 54132736 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004967602808425276, + "loss": 4.4466, + "theoretical_loss": 5.214388767865036, + "tokens_seen": 54198272 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004967502507522568, + "loss": 4.2178, + "theoretical_loss": 5.2134396385493815, + "tokens_seen": 54263808 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004967402206619859, + "loss": 4.3888, + "theoretical_loss": 5.212491975353835, + "tokens_seen": 54329344 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004967301905717151, + "loss": 4.4649, + "theoretical_loss": 5.211545774249233, + "tokens_seen": 54394880 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004967201604814444, + "loss": 4.5714, + "theoretical_loss": 5.210601031222324, + "tokens_seen": 54460416 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004967101303911735, + "loss": 4.3355, + "theoretical_loss": 5.209657742275683, + "tokens_seen": 54525952 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004967001003009028, + "loss": 4.2998, + "theoretical_loss": 5.208715903427631, + "tokens_seen": 54591488 + }, + { + "epoch": 0.02, + "learning_rate": 0.000496690070210632, + "loss": 4.4595, + "theoretical_loss": 5.207775510712159, + "tokens_seen": 54657024 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004966800401203611, + "loss": 4.4266, + "theoretical_loss": 5.2068365601788384, + "tokens_seen": 54722560 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004966700100300903, + "loss": 4.3538, + "theoretical_loss": 5.205899047892753, + "tokens_seen": 54788096 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004966599799398195, + "loss": 4.2137, + "theoretical_loss": 5.2049629699344075, + "tokens_seen": 54853632 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004966499498495487, + "loss": 4.4532, + "theoretical_loss": 5.204028322399658, + "tokens_seen": 54919168 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004966399197592779, + "loss": 4.365, + "theoretical_loss": 5.203095101399628, + "tokens_seen": 54984704 + }, + { + "epoch": 0.02, + "learning_rate": 0.000496629889669007, + "loss": 4.4775, + "theoretical_loss": 5.202163303060633, + "tokens_seen": 55050240 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004966198595787362, + "loss": 4.6992, + "theoretical_loss": 5.201232923524104, + "tokens_seen": 55115776 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004966098294884654, + "loss": 4.293, + "theoretical_loss": 5.20030395894651, + "tokens_seen": 55181312 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004965997993981946, + "loss": 4.3133, + "theoretical_loss": 5.199376405499277, + "tokens_seen": 55246848 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004965897693079238, + "loss": 4.4571, + "theoretical_loss": 5.198450259368721, + "tokens_seen": 55312384 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004965797392176529, + "loss": 4.3994, + "theoretical_loss": 5.197525516755965, + "tokens_seen": 55377920 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004965697091273821, + "loss": 3.9741, + "theoretical_loss": 5.196602173876867, + "tokens_seen": 55443456 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004965596790371113, + "loss": 4.0186, + "theoretical_loss": 5.195680226961947, + "tokens_seen": 55508992 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004965496489468405, + "loss": 4.2808, + "theoretical_loss": 5.194759672256309, + "tokens_seen": 55574528 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004965396188565698, + "loss": 4.4242, + "theoretical_loss": 5.19384050601957, + "tokens_seen": 55640064 + }, + { + "epoch": 0.02, + "objective/train/docs_used": 151052, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.9295265674591064, + "objective/train/theoretical_loss": 5.192922724525789, + "objective/train/tokens_used": 76165600, + "theoretical_loss": 5.192922724525789, + "tokens_seen": 55705600 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004965295887662988, + "loss": 4.1666, + "theoretical_loss": 5.192922724525789, + "tokens_seen": 55705600 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004965195586760281, + "loss": 4.4752, + "theoretical_loss": 5.19200632406339, + "tokens_seen": 55771136 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004965095285857573, + "loss": 4.162, + "theoretical_loss": 5.19109130093509, + "tokens_seen": 55836672 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004964994984954865, + "loss": 4.268, + "theoretical_loss": 5.190177651457833, + "tokens_seen": 55902208 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004964894684052157, + "loss": 4.2668, + "theoretical_loss": 5.189265371962712, + "tokens_seen": 55967744 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004964794383149449, + "loss": 4.1351, + "theoretical_loss": 5.188354458794902, + "tokens_seen": 56033280 + }, + { + "epoch": 0.02, + "learning_rate": 0.000496469408224674, + "loss": 4.3194, + "theoretical_loss": 5.187444908313586, + "tokens_seen": 56098816 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004964593781344032, + "loss": 4.3667, + "theoretical_loss": 5.186536716891892, + "tokens_seen": 56164352 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004964493480441324, + "loss": 4.3239, + "theoretical_loss": 5.185629880916814, + "tokens_seen": 56229888 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004964393179538616, + "loss": 4.1802, + "theoretical_loss": 5.18472439678915, + "tokens_seen": 56295424 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004964292878635908, + "loss": 4.1018, + "theoretical_loss": 5.18382026092343, + "tokens_seen": 56360960 + }, + { + "epoch": 0.02, + "learning_rate": 0.00049641925777332, + "loss": 4.2962, + "theoretical_loss": 5.182917469747851, + "tokens_seen": 56426496 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004964092276830491, + "loss": 3.9407, + "theoretical_loss": 5.182016019704204, + "tokens_seen": 56492032 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004963991975927783, + "loss": 4.468, + "theoretical_loss": 5.1811159072478095, + "tokens_seen": 56557568 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004963891675025075, + "loss": 4.2165, + "theoretical_loss": 5.180217128847451, + "tokens_seen": 56623104 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004963791374122367, + "loss": 4.3633, + "theoretical_loss": 5.17931968098531, + "tokens_seen": 56688640 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004963691073219659, + "loss": 4.2837, + "theoretical_loss": 5.178423560156894, + "tokens_seen": 56754176 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004963590772316951, + "loss": 4.5775, + "theoretical_loss": 5.177528762870973, + "tokens_seen": 56819712 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004963490471414242, + "loss": 4.2386, + "theoretical_loss": 5.176635285649521, + "tokens_seen": 56885248 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004963390170511535, + "loss": 4.4926, + "theoretical_loss": 5.175743125027638, + "tokens_seen": 56950784 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004963289869608827, + "loss": 4.2936, + "theoretical_loss": 5.174852277553498, + "tokens_seen": 57016320 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004963189568706119, + "loss": 4.3052, + "theoretical_loss": 5.173962739788276, + "tokens_seen": 57081856 + }, + { + "epoch": 0.02, + "learning_rate": 0.000496308926780341, + "loss": 4.2385, + "theoretical_loss": 5.17307450830609, + "tokens_seen": 57147392 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004962988966900702, + "loss": 4.0804, + "theoretical_loss": 5.172187579693933, + "tokens_seen": 57212928 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004962888665997994, + "loss": 4.2481, + "theoretical_loss": 5.1713019505516105, + "tokens_seen": 57278464 + }, + { + "epoch": 0.02, + "objective/train/docs_used": 152615, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 4.15744161605835, + "objective/train/theoretical_loss": 5.170417617491682, + "objective/train/tokens_used": 77804000, + "theoretical_loss": 5.170417617491682, + "tokens_seen": 57344000 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004962788365095286, + "loss": 4.1901, + "theoretical_loss": 5.170417617491682, + "tokens_seen": 57344000 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004962688064192578, + "loss": 4.4107, + "theoretical_loss": 5.169534577139395, + "tokens_seen": 57409536 + }, + { + "epoch": 0.02, + "learning_rate": 0.000496258776328987, + "loss": 4.4229, + "theoretical_loss": 5.168652826132623, + "tokens_seen": 57475072 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004962487462387161, + "loss": 4.2645, + "theoretical_loss": 5.167772361121805, + "tokens_seen": 57540608 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004962387161484453, + "loss": 4.0974, + "theoretical_loss": 5.166893178769884, + "tokens_seen": 57606144 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004962286860581746, + "loss": 4.5159, + "theoretical_loss": 5.1660152757522475, + "tokens_seen": 57671680 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004962186559679037, + "loss": 4.3628, + "theoretical_loss": 5.165138648756665, + "tokens_seen": 57737216 + }, + { + "epoch": 0.02, + "learning_rate": 0.000496208625877633, + "loss": 3.9702, + "theoretical_loss": 5.164263294483226, + "tokens_seen": 57802752 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004961985957873621, + "loss": 4.1025, + "theoretical_loss": 5.163389209644287, + "tokens_seen": 57868288 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004961885656970913, + "loss": 4.0294, + "theoretical_loss": 5.162516390964408, + "tokens_seen": 57933824 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004961785356068205, + "loss": 4.155, + "theoretical_loss": 5.1616448351802875, + "tokens_seen": 57999360 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004961685055165497, + "loss": 4.1206, + "theoretical_loss": 5.160774539040716, + "tokens_seen": 58064896 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004961584754262789, + "loss": 4.2619, + "theoretical_loss": 5.159905499306511, + "tokens_seen": 58130432 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004961484453360081, + "loss": 4.2084, + "theoretical_loss": 5.159037712750455, + "tokens_seen": 58195968 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004961384152457372, + "loss": 4.3638, + "theoretical_loss": 5.158171176157245, + "tokens_seen": 58261504 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004961283851554664, + "loss": 4.4218, + "theoretical_loss": 5.157305886323435, + "tokens_seen": 58327040 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004961183550651956, + "loss": 4.4587, + "theoretical_loss": 5.156441840057371, + "tokens_seen": 58392576 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004961083249749248, + "loss": 4.1511, + "theoretical_loss": 5.155579034179144, + "tokens_seen": 58458112 + }, + { + "epoch": 0.02, + "learning_rate": 0.000496098294884654, + "loss": 3.9374, + "theoretical_loss": 5.15471746552053, + "tokens_seen": 58523648 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004960882647943831, + "loss": 4.1491, + "theoretical_loss": 5.153857130924929, + "tokens_seen": 58589184 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004960782347041123, + "loss": 4.3473, + "theoretical_loss": 5.1529980272473175, + "tokens_seen": 58654720 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004960682046138415, + "loss": 4.2407, + "theoretical_loss": 5.152140151354191, + "tokens_seen": 58720256 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004960581745235707, + "loss": 4.2574, + "theoretical_loss": 5.151283500123505, + "tokens_seen": 58785792 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004960481444333, + "loss": 4.196, + "theoretical_loss": 5.150428070444621, + "tokens_seen": 58851328 + }, + { + "epoch": 0.02, + "learning_rate": 0.000496038114343029, + "loss": 4.1719, + "theoretical_loss": 5.149573859218261, + "tokens_seen": 58916864 + }, + { + "epoch": 0.02, + "objective/train/docs_used": 153245, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 4.41132926940918, + "objective/train/theoretical_loss": 5.1487208633564405, + "objective/train/tokens_used": 79442400, + "theoretical_loss": 5.1487208633564405, + "tokens_seen": 58982400 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004960280842527583, + "loss": 4.2262, + "theoretical_loss": 5.1487208633564405, + "tokens_seen": 58982400 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004960180541624875, + "loss": 4.3165, + "theoretical_loss": 5.147869079782423, + "tokens_seen": 59047936 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004960080240722167, + "loss": 4.1407, + "theoretical_loss": 5.147018505430666, + "tokens_seen": 59113472 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004959979939819459, + "loss": 4.2968, + "theoretical_loss": 5.146169137246765, + "tokens_seen": 59179008 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004959879638916751, + "loss": 4.3721, + "theoretical_loss": 5.145320972187402, + "tokens_seen": 59244544 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004959779338014042, + "loss": 3.9537, + "theoretical_loss": 5.144474007220293, + "tokens_seen": 59310080 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004959679037111334, + "loss": 4.0455, + "theoretical_loss": 5.143628239324139, + "tokens_seen": 59375616 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004959578736208626, + "loss": 4.1954, + "theoretical_loss": 5.142783665488567, + "tokens_seen": 59441152 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004959478435305918, + "loss": 4.4969, + "theoretical_loss": 5.1419402827140885, + "tokens_seen": 59506688 + }, + { + "epoch": 0.02, + "learning_rate": 0.000495937813440321, + "loss": 4.0657, + "theoretical_loss": 5.141098088012036, + "tokens_seen": 59572224 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004959277833500501, + "loss": 4.0912, + "theoretical_loss": 5.140257078404524, + "tokens_seen": 59637760 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004959177532597793, + "loss": 4.2994, + "theoretical_loss": 5.13941725092439, + "tokens_seen": 59703296 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004959077231695085, + "loss": 4.0762, + "theoretical_loss": 5.138578602615146, + "tokens_seen": 59768832 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004958976930792377, + "loss": 4.3081, + "theoretical_loss": 5.137741130530934, + "tokens_seen": 59834368 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004958876629889669, + "loss": 4.1723, + "theoretical_loss": 5.1369048317364685, + "tokens_seen": 59899904 + }, + { + "epoch": 0.02, + "learning_rate": 0.000495877632898696, + "loss": 4.1926, + "theoretical_loss": 5.13606970330699, + "tokens_seen": 59965440 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004958676028084253, + "loss": 4.3296, + "theoretical_loss": 5.135235742328217, + "tokens_seen": 60030976 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004958575727181544, + "loss": 4.2648, + "theoretical_loss": 5.134402945896297, + "tokens_seen": 60096512 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004958475426278837, + "loss": 4.0622, + "theoretical_loss": 5.133571311117755, + "tokens_seen": 60162048 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004958375125376129, + "loss": 4.1772, + "theoretical_loss": 5.132740835109448, + "tokens_seen": 60227584 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004958274824473421, + "loss": 3.9454, + "theoretical_loss": 5.131911514998518, + "tokens_seen": 60293120 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004958174523570712, + "loss": 4.3073, + "theoretical_loss": 5.131083347922338, + "tokens_seen": 60358656 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004958074222668004, + "loss": 4.0057, + "theoretical_loss": 5.130256331028474, + "tokens_seen": 60424192 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004957973921765296, + "loss": 4.3214, + "theoretical_loss": 5.129430461474628, + "tokens_seen": 60489728 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004957873620862588, + "loss": 4.1685, + "theoretical_loss": 5.128605736428597, + "tokens_seen": 60555264 + }, + { + "epoch": 0.02, + "objective/train/docs_used": 154602, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 4.488969802856445, + "objective/train/theoretical_loss": 5.127782153068225, + "objective/train/tokens_used": 81080800, + "theoretical_loss": 5.127782153068225, + "tokens_seen": 60620800 + }, + { + "epoch": 0.02, + "learning_rate": 0.000495777331995988, + "loss": 4.1876, + "theoretical_loss": 5.127782153068225, + "tokens_seen": 60620800 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004957673019057172, + "loss": 4.1921, + "theoretical_loss": 5.126959708581356, + "tokens_seen": 60686336 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004957572718154463, + "loss": 4.2615, + "theoretical_loss": 5.1261384001657895, + "tokens_seen": 60751872 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004957472417251755, + "loss": 4.156, + "theoretical_loss": 5.125318225029231, + "tokens_seen": 60817408 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004957372116349047, + "loss": 4.1728, + "theoretical_loss": 5.124499180389249, + "tokens_seen": 60882944 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004957271815446339, + "loss": 4.2523, + "theoretical_loss": 5.12368126347323, + "tokens_seen": 60948480 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004957171514543631, + "loss": 4.1074, + "theoretical_loss": 5.122864471518334, + "tokens_seen": 61014016 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004957071213640923, + "loss": 4.2443, + "theoretical_loss": 5.122048801771443, + "tokens_seen": 61079552 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004956970912738214, + "loss": 3.8773, + "theoretical_loss": 5.121234251489128, + "tokens_seen": 61145088 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004956870611835507, + "loss": 4.0361, + "theoretical_loss": 5.120420817937591, + "tokens_seen": 61210624 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004956770310932798, + "loss": 3.9077, + "theoretical_loss": 5.119608498392633, + "tokens_seen": 61276160 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004956670010030091, + "loss": 3.9712, + "theoretical_loss": 5.118797290139605, + "tokens_seen": 61341696 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004956569709127383, + "loss": 4.1847, + "theoretical_loss": 5.117987190473361, + "tokens_seen": 61407232 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004956469408224674, + "loss": 3.9935, + "theoretical_loss": 5.1171781966982195, + "tokens_seen": 61472768 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004956369107321966, + "loss": 3.932, + "theoretical_loss": 5.116370306127921, + "tokens_seen": 61538304 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004956268806419258, + "loss": 4.3194, + "theoretical_loss": 5.11556351608558, + "tokens_seen": 61603840 + }, + { + "epoch": 0.02, + "learning_rate": 0.000495616850551655, + "loss": 4.322, + "theoretical_loss": 5.114757823903647, + "tokens_seen": 61669376 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004956068204613842, + "loss": 4.0708, + "theoretical_loss": 5.113953226923864, + "tokens_seen": 61734912 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004955967903711133, + "loss": 4.3149, + "theoretical_loss": 5.113149722497221, + "tokens_seen": 61800448 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004955867602808425, + "loss": 4.3968, + "theoretical_loss": 5.112347307983919, + "tokens_seen": 61865984 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004955767301905717, + "loss": 4.1143, + "theoretical_loss": 5.111545980753322, + "tokens_seen": 61931520 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004955667001003009, + "loss": 3.9537, + "theoretical_loss": 5.110745738183919, + "tokens_seen": 61997056 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004955566700100301, + "loss": 4.1058, + "theoretical_loss": 5.109946577663284, + "tokens_seen": 62062592 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004955466399197592, + "loss": 4.3783, + "theoretical_loss": 5.109148496588032, + "tokens_seen": 62128128 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004955366098294884, + "loss": 4.1335, + "theoretical_loss": 5.108351492363779, + "tokens_seen": 62193664 + }, + { + "epoch": 0.02, + "objective/train/docs_used": 155165, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.9781312942504883, + "objective/train/theoretical_loss": 5.107555562405102, + "objective/train/tokens_used": 82719200, + "theoretical_loss": 5.107555562405102, + "tokens_seen": 62259200 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004955265797392177, + "loss": 4.1683, + "theoretical_loss": 5.107555562405102, + "tokens_seen": 62259200 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004955165496489468, + "loss": 4.3202, + "theoretical_loss": 5.106760704135499, + "tokens_seen": 62324736 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004955065195586761, + "loss": 4.1865, + "theoretical_loss": 5.105966914987349, + "tokens_seen": 62390272 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004954964894684052, + "loss": 4.0606, + "theoretical_loss": 5.1051741924018685, + "tokens_seen": 62455808 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004954864593781344, + "loss": 4.1826, + "theoretical_loss": 5.10438253382908, + "tokens_seen": 62521344 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004954764292878636, + "loss": 4.123, + "theoretical_loss": 5.103591936727762, + "tokens_seen": 62586880 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004954663991975928, + "loss": 3.9976, + "theoretical_loss": 5.102802398565418, + "tokens_seen": 62652416 + }, + { + "epoch": 0.02, + "learning_rate": 0.000495456369107322, + "loss": 4.1302, + "theoretical_loss": 5.102013916818235, + "tokens_seen": 62717952 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004954463390170512, + "loss": 4.1543, + "theoretical_loss": 5.101226488971042, + "tokens_seen": 62783488 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004954363089267803, + "loss": 3.8957, + "theoretical_loss": 5.100440112517276, + "tokens_seen": 62849024 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004954262788365095, + "loss": 4.1473, + "theoretical_loss": 5.09965478495894, + "tokens_seen": 62914560 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004954162487462387, + "loss": 4.3325, + "theoretical_loss": 5.098870503806567, + "tokens_seen": 62980096 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004954062186559679, + "loss": 4.268, + "theoretical_loss": 5.09808726657918, + "tokens_seen": 63045632 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004953961885656971, + "loss": 4.096, + "theoretical_loss": 5.097305070804255, + "tokens_seen": 63111168 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004953861584754263, + "loss": 4.0601, + "theoretical_loss": 5.096523914017688, + "tokens_seen": 63176704 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004953761283851555, + "loss": 3.6987, + "theoretical_loss": 5.095743793763747, + "tokens_seen": 63242240 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004953660982948846, + "loss": 4.1201, + "theoretical_loss": 5.094964707595047, + "tokens_seen": 63307776 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004953560682046139, + "loss": 3.6626, + "theoretical_loss": 5.094186653072505, + "tokens_seen": 63373312 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004953460381143431, + "loss": 4.0223, + "theoretical_loss": 5.093409627765306, + "tokens_seen": 63438848 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004953360080240723, + "loss": 4.2156, + "theoretical_loss": 5.092633629250866, + "tokens_seen": 63504384 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004953259779338014, + "loss": 3.9654, + "theoretical_loss": 5.091858655114796, + "tokens_seen": 63569920 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004953159478435306, + "loss": 4.0676, + "theoretical_loss": 5.091084702950868, + "tokens_seen": 63635456 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004953059177532598, + "loss": 4.0842, + "theoretical_loss": 5.090311770360971, + "tokens_seen": 63700992 + }, + { + "epoch": 0.02, + "learning_rate": 0.000495295887662989, + "loss": 4.2643, + "theoretical_loss": 5.089539854955088, + "tokens_seen": 63766528 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004952858575727182, + "loss": 3.9339, + "theoretical_loss": 5.088768954351249, + "tokens_seen": 63832064 + }, + { + "epoch": 0.02, + "objective/train/docs_used": 156647, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.6908023357391357, + "objective/train/theoretical_loss": 5.087999066175502, + "objective/train/tokens_used": 84357600, + "theoretical_loss": 5.087999066175502, + "tokens_seen": 63897600 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004952758274824474, + "loss": 3.8197, + "theoretical_loss": 5.087999066175502, + "tokens_seen": 63897600 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004952657973921765, + "loss": 4.0257, + "theoretical_loss": 5.0872301880618735, + "tokens_seen": 63963136 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004952557673019057, + "loss": 4.062, + "theoretical_loss": 5.086462317652341, + "tokens_seen": 64028672 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004952457372116349, + "loss": 4.1515, + "theoretical_loss": 5.085695452596788, + "tokens_seen": 64094208 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004952357071213641, + "loss": 4.1296, + "theoretical_loss": 5.084929590552976, + "tokens_seen": 64159744 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004952256770310933, + "loss": 3.8944, + "theoretical_loss": 5.0841647291865115, + "tokens_seen": 64225280 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004952156469408225, + "loss": 4.2949, + "theoretical_loss": 5.083400866170806, + "tokens_seen": 64290816 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004952056168505516, + "loss": 3.9266, + "theoretical_loss": 5.082637999187046, + "tokens_seen": 64356352 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004951955867602809, + "loss": 4.0743, + "theoretical_loss": 5.081876125924159, + "tokens_seen": 64421888 + }, + { + "epoch": 0.02, + "learning_rate": 0.00049518555667001, + "loss": 3.9946, + "theoretical_loss": 5.0811152440787755, + "tokens_seen": 64487424 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004951755265797393, + "loss": 4.1916, + "theoretical_loss": 5.0803553513552036, + "tokens_seen": 64552960 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004951654964894685, + "loss": 4.2992, + "theoretical_loss": 5.079596445465386, + "tokens_seen": 64618496 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004951554663991976, + "loss": 3.9929, + "theoretical_loss": 5.078838524128878, + "tokens_seen": 64684032 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004951454363089268, + "loss": 3.7549, + "theoretical_loss": 5.078081585072802, + "tokens_seen": 64749568 + }, + { + "epoch": 0.02, + "learning_rate": 0.000495135406218656, + "loss": 4.0781, + "theoretical_loss": 5.077325626031826, + "tokens_seen": 64815104 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004951253761283852, + "loss": 3.9946, + "theoretical_loss": 5.076570644748123, + "tokens_seen": 64880640 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004951153460381144, + "loss": 3.966, + "theoretical_loss": 5.075816638971341, + "tokens_seen": 64946176 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004951053159478435, + "loss": 4.3961, + "theoretical_loss": 5.075063606458576, + "tokens_seen": 65011712 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004950952858575727, + "loss": 4.1845, + "theoretical_loss": 5.074311544974331, + "tokens_seen": 65077248 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004950852557673019, + "loss": 4.0004, + "theoretical_loss": 5.07356045229049, + "tokens_seen": 65142784 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004950752256770311, + "loss": 3.7156, + "theoretical_loss": 5.072810326186285, + "tokens_seen": 65208320 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004950651955867603, + "loss": 4.0024, + "theoretical_loss": 5.072061164448261, + "tokens_seen": 65273856 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004950551654964894, + "loss": 4.0715, + "theoretical_loss": 5.071312964870252, + "tokens_seen": 65339392 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004950451354062186, + "loss": 4.07, + "theoretical_loss": 5.070565725253344, + "tokens_seen": 65404928 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004950351053159479, + "loss": 4.1316, + "theoretical_loss": 5.069819443405842, + "tokens_seen": 65470464 + }, + { + "epoch": 0.02, + "objective/train/docs_used": 157489, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 4.333700180053711, + "objective/train/theoretical_loss": 5.069074117143246, + "objective/train/tokens_used": 85996000, + "theoretical_loss": 5.069074117143246, + "tokens_seen": 65536000 + }, + { + "epoch": 0.02, + "learning_rate": 0.000495025075225677, + "loss": 3.9793, + "theoretical_loss": 5.069074117143246, + "tokens_seen": 65536000 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004950150451354063, + "loss": 3.8486, + "theoretical_loss": 5.068329744288216, + "tokens_seen": 65601536 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004950050150451354, + "loss": 4.252, + "theoretical_loss": 5.067586322670541, + "tokens_seen": 65667072 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004949949849548646, + "loss": 4.079, + "theoretical_loss": 5.0668438501271105, + "tokens_seen": 65732608 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004949849548645938, + "loss": 4.09, + "theoretical_loss": 5.066102324501883, + "tokens_seen": 65798144 + }, + { + "epoch": 0.02, + "learning_rate": 0.000494974924774323, + "loss": 4.1249, + "theoretical_loss": 5.065361743645855, + "tokens_seen": 65863680 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004949648946840522, + "loss": 3.8237, + "theoretical_loss": 5.064622105417033, + "tokens_seen": 65929216 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004949548645937814, + "loss": 4.0333, + "theoretical_loss": 5.063883407680405, + "tokens_seen": 65994752 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004949448345035105, + "loss": 4.0507, + "theoretical_loss": 5.063145648307904, + "tokens_seen": 66060288 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004949348044132397, + "loss": 3.5886, + "theoretical_loss": 5.062408825178388, + "tokens_seen": 66125824 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004949247743229689, + "loss": 3.9934, + "theoretical_loss": 5.061672936177604, + "tokens_seen": 66191360 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004949147442326981, + "loss": 3.8668, + "theoretical_loss": 5.06093797919816, + "tokens_seen": 66256896 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004949047141424273, + "loss": 4.1734, + "theoretical_loss": 5.060203952139497, + "tokens_seen": 66322432 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004948946840521565, + "loss": 3.8566, + "theoretical_loss": 5.059470852907861, + "tokens_seen": 66387968 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004948846539618856, + "loss": 3.9639, + "theoretical_loss": 5.0587386794162725, + "tokens_seen": 66453504 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004948746238716148, + "loss": 3.9601, + "theoretical_loss": 5.058007429584498, + "tokens_seen": 66519040 + }, + { + "epoch": 0.02, + "learning_rate": 0.000494864593781344, + "loss": 3.7703, + "theoretical_loss": 5.057277101339023, + "tokens_seen": 66584576 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004948545636910733, + "loss": 4.193, + "theoretical_loss": 5.056547692613021, + "tokens_seen": 66650112 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004948445336008024, + "loss": 3.8359, + "theoretical_loss": 5.055819201346331, + "tokens_seen": 66715648 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004948345035105316, + "loss": 4.019, + "theoretical_loss": 5.055091625485421, + "tokens_seen": 66781184 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004948244734202607, + "loss": 3.6005, + "theoretical_loss": 5.054364962983367, + "tokens_seen": 66846720 + }, + { + "epoch": 0.02, + "learning_rate": 0.00049481444332999, + "loss": 4.065, + "theoretical_loss": 5.053639211799824, + "tokens_seen": 66912256 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004948044132397192, + "loss": 4.0457, + "theoretical_loss": 5.052914369900997, + "tokens_seen": 66977792 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004947943831494484, + "loss": 3.8273, + "theoretical_loss": 5.052190435259614, + "tokens_seen": 67043328 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004947843530591776, + "loss": 3.9551, + "theoretical_loss": 5.051467405854897, + "tokens_seen": 67108864 + }, + { + "epoch": 0.02, + "objective/train/docs_used": 158845, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.314136505126953, + "objective/train/theoretical_loss": 5.05074527967254, + "objective/train/tokens_used": 87634400, + "theoretical_loss": 5.05074527967254, + "tokens_seen": 67174400 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004947743229689067, + "loss": 3.8036, + "theoretical_loss": 5.05074527967254, + "tokens_seen": 67174400 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004947642928786359, + "loss": 4.2425, + "theoretical_loss": 5.050024054704677, + "tokens_seen": 67239936 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004947542627883651, + "loss": 3.6902, + "theoretical_loss": 5.049303728949859, + "tokens_seen": 67305472 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004947442326980943, + "loss": 4.0812, + "theoretical_loss": 5.048584300413019, + "tokens_seen": 67371008 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004947342026078235, + "loss": 3.9309, + "theoretical_loss": 5.04786576710546, + "tokens_seen": 67436544 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004947241725175527, + "loss": 4.0231, + "theoretical_loss": 5.0471481270448155, + "tokens_seen": 67502080 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004947141424272818, + "loss": 3.7707, + "theoretical_loss": 5.046431378255027, + "tokens_seen": 67567616 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004947041123370111, + "loss": 4.1525, + "theoretical_loss": 5.045715518766322, + "tokens_seen": 67633152 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004946940822467402, + "loss": 3.7804, + "theoretical_loss": 5.0450005466151815, + "tokens_seen": 67698688 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004946840521564695, + "loss": 3.9759, + "theoretical_loss": 5.044286459844319, + "tokens_seen": 67764224 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004946740220661987, + "loss": 4.0479, + "theoretical_loss": 5.043573256502652, + "tokens_seen": 67829760 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004946639919759278, + "loss": 4.1937, + "theoretical_loss": 5.0428609346452795, + "tokens_seen": 67895296 + }, + { + "epoch": 0.02, + "learning_rate": 0.000494653961885657, + "loss": 4.1032, + "theoretical_loss": 5.042149492333452, + "tokens_seen": 67960832 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004946439317953862, + "loss": 3.7257, + "theoretical_loss": 5.041438927634549, + "tokens_seen": 68026368 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004946339017051154, + "loss": 4.3465, + "theoretical_loss": 5.040729238622053, + "tokens_seen": 68091904 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004946238716148446, + "loss": 3.7946, + "theoretical_loss": 5.040020423375525, + "tokens_seen": 68157440 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004946138415245737, + "loss": 3.8581, + "theoretical_loss": 5.039312479980579, + "tokens_seen": 68222976 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004946038114343029, + "loss": 3.9283, + "theoretical_loss": 5.038605406528857, + "tokens_seen": 68288512 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004945937813440321, + "loss": 3.9674, + "theoretical_loss": 5.037899201118005, + "tokens_seen": 68354048 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004945837512537613, + "loss": 3.9114, + "theoretical_loss": 5.037193861851646, + "tokens_seen": 68419584 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004945737211634905, + "loss": 4.0903, + "theoretical_loss": 5.03648938683936, + "tokens_seen": 68485120 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004945636910732196, + "loss": 4.0617, + "theoretical_loss": 5.035785774196654, + "tokens_seen": 68550656 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004945536609829488, + "loss": 3.9602, + "theoretical_loss": 5.035083022044944, + "tokens_seen": 68616192 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004945436308926781, + "loss": 3.7551, + "theoretical_loss": 5.034381128511525, + "tokens_seen": 68681728 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004945336008024072, + "loss": 3.728, + "theoretical_loss": 5.0336800917295506, + "tokens_seen": 68747264 + }, + { + "epoch": 0.02, + "objective/train/docs_used": 159466, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.931746244430542, + "objective/train/theoretical_loss": 5.032979909838007, + "objective/train/tokens_used": 89272800, + "theoretical_loss": 5.032979909838007, + "tokens_seen": 68812800 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004945235707121365, + "loss": 4.1672, + "theoretical_loss": 5.032979909838007, + "tokens_seen": 68812800 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004945135406218656, + "loss": 3.7009, + "theoretical_loss": 5.032280580981691, + "tokens_seen": 68878336 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004945035105315948, + "loss": 3.9125, + "theoretical_loss": 5.031582103311187, + "tokens_seen": 68943872 + }, + { + "epoch": 0.02, + "learning_rate": 0.000494493480441324, + "loss": 3.8562, + "theoretical_loss": 5.030884474982842, + "tokens_seen": 69009408 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004944834503510532, + "loss": 4.0789, + "theoretical_loss": 5.030187694158739, + "tokens_seen": 69074944 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004944734202607824, + "loss": 3.6415, + "theoretical_loss": 5.02949175900668, + "tokens_seen": 69140480 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004944633901705116, + "loss": 3.9927, + "theoretical_loss": 5.028796667700159, + "tokens_seen": 69206016 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004944533600802407, + "loss": 3.6965, + "theoretical_loss": 5.0281024184183405, + "tokens_seen": 69271552 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004944433299899699, + "loss": 3.9956, + "theoretical_loss": 5.0274090093460355, + "tokens_seen": 69337088 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004944332998996991, + "loss": 4.0111, + "theoretical_loss": 5.026716438673677, + "tokens_seen": 69402624 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004944232698094283, + "loss": 3.8318, + "theoretical_loss": 5.0260247045973045, + "tokens_seen": 69468160 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004944132397191575, + "loss": 4.0343, + "theoretical_loss": 5.02533380531853, + "tokens_seen": 69533696 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004944032096288867, + "loss": 3.6072, + "theoretical_loss": 5.024643739044526, + "tokens_seen": 69599232 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004943931795386158, + "loss": 4.0637, + "theoretical_loss": 5.023954503987998, + "tokens_seen": 69664768 + }, + { + "epoch": 0.02, + "learning_rate": 0.000494383149448345, + "loss": 3.9123, + "theoretical_loss": 5.023266098367161, + "tokens_seen": 69730304 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004943731193580742, + "loss": 3.7934, + "theoretical_loss": 5.022578520405721, + "tokens_seen": 69795840 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004943630892678035, + "loss": 3.8608, + "theoretical_loss": 5.0218917683328534, + "tokens_seen": 69861376 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004943530591775326, + "loss": 3.8826, + "theoretical_loss": 5.021205840383175, + "tokens_seen": 69926912 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004943430290872618, + "loss": 3.7947, + "theoretical_loss": 5.020520734796728, + "tokens_seen": 69992448 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004943329989969909, + "loss": 3.8164, + "theoretical_loss": 5.019836449818957, + "tokens_seen": 70057984 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004943229689067202, + "loss": 3.6969, + "theoretical_loss": 5.019152983700687, + "tokens_seen": 70123520 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004943129388164494, + "loss": 3.8535, + "theoretical_loss": 5.018470334698101, + "tokens_seen": 70189056 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004943029087261786, + "loss": 3.9834, + "theoretical_loss": 5.01778850107272, + "tokens_seen": 70254592 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004942928786359078, + "loss": 3.7347, + "theoretical_loss": 5.017107481091379, + "tokens_seen": 70320128 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004942828485456369, + "loss": 3.6688, + "theoretical_loss": 5.016427273026212, + "tokens_seen": 70385664 + }, + { + "epoch": 0.02, + "objective/train/docs_used": 160845, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.9590885639190674, + "objective/train/theoretical_loss": 5.015747875154622, + "objective/train/tokens_used": 90911200, + "theoretical_loss": 5.015747875154622, + "tokens_seen": 70451200 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004942728184553661, + "loss": 4.0648, + "theoretical_loss": 5.015747875154622, + "tokens_seen": 70451200 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004942627883650953, + "loss": 3.7767, + "theoretical_loss": 5.015069285759269, + "tokens_seen": 70516736 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004942527582748245, + "loss": 4.1511, + "theoretical_loss": 5.01439150312804, + "tokens_seen": 70582272 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004942427281845537, + "loss": 3.8699, + "theoretical_loss": 5.0137145255540405, + "tokens_seen": 70647808 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004942326980942828, + "loss": 3.8057, + "theoretical_loss": 5.013038351335559, + "tokens_seen": 70713344 + }, + { + "epoch": 0.02, + "learning_rate": 0.000494222668004012, + "loss": 3.8407, + "theoretical_loss": 5.012362978776057, + "tokens_seen": 70778880 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004942126379137412, + "loss": 3.7493, + "theoretical_loss": 5.011688406184147, + "tokens_seen": 70844416 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004942026078234704, + "loss": 3.9297, + "theoretical_loss": 5.011014631873566, + "tokens_seen": 70909952 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004941925777331996, + "loss": 4.0194, + "theoretical_loss": 5.010341654163167, + "tokens_seen": 70975488 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004941825476429289, + "loss": 3.9314, + "theoretical_loss": 5.009669471376882, + "tokens_seen": 71041024 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004941725175526579, + "loss": 3.553, + "theoretical_loss": 5.008998081843721, + "tokens_seen": 71106560 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004941624874623872, + "loss": 4.1107, + "theoretical_loss": 5.008327483897736, + "tokens_seen": 71172096 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004941524573721163, + "loss": 3.8063, + "theoretical_loss": 5.00765767587801, + "tokens_seen": 71237632 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004941424272818456, + "loss": 3.7438, + "theoretical_loss": 5.006988656128635, + "tokens_seen": 71303168 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004941323971915748, + "loss": 3.6967, + "theoretical_loss": 5.006320422998691, + "tokens_seen": 71368704 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004941223671013039, + "loss": 3.9589, + "theoretical_loss": 5.00565297484223, + "tokens_seen": 71434240 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004941123370110331, + "loss": 3.9641, + "theoretical_loss": 5.004986310018252, + "tokens_seen": 71499776 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004941023069207623, + "loss": 4.093, + "theoretical_loss": 5.004320426890686, + "tokens_seen": 71565312 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004940922768304915, + "loss": 3.8581, + "theoretical_loss": 5.003655323828376, + "tokens_seen": 71630848 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004940822467402207, + "loss": 3.9614, + "theoretical_loss": 5.002990999205057, + "tokens_seen": 71696384 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004940722166499498, + "loss": 3.9838, + "theoretical_loss": 5.002327451399335, + "tokens_seen": 71761920 + }, + { + "epoch": 0.02, + "learning_rate": 0.000494062186559679, + "loss": 4.1628, + "theoretical_loss": 5.001664678794671, + "tokens_seen": 71827456 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004940521564694082, + "loss": 3.9386, + "theoretical_loss": 5.001002679779363, + "tokens_seen": 71892992 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004940421263791374, + "loss": 3.9657, + "theoretical_loss": 5.0003414527465235, + "tokens_seen": 71958528 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004940320962888666, + "loss": 4.0775, + "theoretical_loss": 4.99968099609406, + "tokens_seen": 72024064 + }, + { + "epoch": 0.02, + "objective/train/docs_used": 161502, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 4.11187744140625, + "objective/train/theoretical_loss": 4.999021308224664, + "objective/train/tokens_used": 92549600, + "theoretical_loss": 4.999021308224664, + "tokens_seen": 72089600 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004940220661985958, + "loss": 3.9657, + "theoretical_loss": 4.999021308224664, + "tokens_seen": 72089600 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004940120361083249, + "loss": 3.9299, + "theoretical_loss": 4.998362387545782, + "tokens_seen": 72155136 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004940020060180542, + "loss": 3.833, + "theoretical_loss": 4.997704232469606, + "tokens_seen": 72220672 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004939919759277834, + "loss": 3.6457, + "theoretical_loss": 4.997046841413049, + "tokens_seen": 72286208 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004939819458375126, + "loss": 4.0112, + "theoretical_loss": 4.996390212797728, + "tokens_seen": 72351744 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004939719157472418, + "loss": 3.6914, + "theoretical_loss": 4.995734345049949, + "tokens_seen": 72417280 + }, + { + "epoch": 0.02, + "learning_rate": 0.000493961885656971, + "loss": 3.8694, + "theoretical_loss": 4.995079236600686, + "tokens_seen": 72482816 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004939518555667001, + "loss": 3.9413, + "theoretical_loss": 4.994424885885564, + "tokens_seen": 72548352 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004939418254764293, + "loss": 4.0597, + "theoretical_loss": 4.993771291344839, + "tokens_seen": 72613888 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004939317953861585, + "loss": 4.0831, + "theoretical_loss": 4.993118451423381, + "tokens_seen": 72679424 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004939217652958877, + "loss": 3.8775, + "theoretical_loss": 4.992466364570659, + "tokens_seen": 72744960 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004939117352056169, + "loss": 4.0318, + "theoretical_loss": 4.991815029240721, + "tokens_seen": 72810496 + }, + { + "epoch": 0.02, + "learning_rate": 0.000493901705115346, + "loss": 3.9009, + "theoretical_loss": 4.991164443892175, + "tokens_seen": 72876032 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004938916750250752, + "loss": 3.8903, + "theoretical_loss": 4.990514606988173, + "tokens_seen": 72941568 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004938816449348044, + "loss": 3.8758, + "theoretical_loss": 4.989865516996396, + "tokens_seen": 73007104 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004938716148445337, + "loss": 3.6876, + "theoretical_loss": 4.98921717238903, + "tokens_seen": 73072640 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004938615847542628, + "loss": 3.9345, + "theoretical_loss": 4.988569571642756, + "tokens_seen": 73138176 + }, + { + "epoch": 0.02, + "learning_rate": 0.000493851554663992, + "loss": 3.6821, + "theoretical_loss": 4.98792271323873, + "tokens_seen": 73203712 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004938415245737211, + "loss": 3.9708, + "theoretical_loss": 4.9872765956625615, + "tokens_seen": 73269248 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004938314944834504, + "loss": 3.8872, + "theoretical_loss": 4.9866312174043035, + "tokens_seen": 73334784 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004938214643931796, + "loss": 3.799, + "theoretical_loss": 4.9859865769584335, + "tokens_seen": 73400320 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004938114343029088, + "loss": 3.9449, + "theoretical_loss": 4.9853426728238315, + "tokens_seen": 73465856 + }, + { + "epoch": 0.02, + "learning_rate": 0.000493801404212638, + "loss": 3.5261, + "theoretical_loss": 4.984699503503771, + "tokens_seen": 73531392 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004937913741223671, + "loss": 3.9521, + "theoretical_loss": 4.984057067505898, + "tokens_seen": 73596928 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004937813440320963, + "loss": 3.977, + "theoretical_loss": 4.9834153633422105, + "tokens_seen": 73662464 + }, + { + "epoch": 0.02, + "objective/train/docs_used": 162491, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.735835313796997, + "objective/train/theoretical_loss": 4.982774389529053, + "objective/train/tokens_used": 94188000, + "theoretical_loss": 4.982774389529053, + "tokens_seen": 73728000 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004937713139418255, + "loss": 3.751, + "theoretical_loss": 4.982774389529053, + "tokens_seen": 73728000 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004937612838515547, + "loss": 3.8968, + "theoretical_loss": 4.9821341445870875, + "tokens_seen": 73793536 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004937512537612839, + "loss": 3.5854, + "theoretical_loss": 4.981494627041286, + "tokens_seen": 73859072 + }, + { + "epoch": 0.02, + "learning_rate": 0.000493741223671013, + "loss": 3.5525, + "theoretical_loss": 4.98085583542091, + "tokens_seen": 73924608 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004937311935807422, + "loss": 3.9853, + "theoretical_loss": 4.980217768259496, + "tokens_seen": 73990144 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004937211634904714, + "loss": 3.9015, + "theoretical_loss": 4.979580424094836, + "tokens_seen": 74055680 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004937111334002006, + "loss": 4.0376, + "theoretical_loss": 4.978943801468967, + "tokens_seen": 74121216 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004937011033099298, + "loss": 3.5555, + "theoretical_loss": 4.978307898928149, + "tokens_seen": 74186752 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004936910732196591, + "loss": 3.9641, + "theoretical_loss": 4.977672715022855, + "tokens_seen": 74252288 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004936810431293881, + "loss": 3.8699, + "theoretical_loss": 4.97703824830775, + "tokens_seen": 74317824 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004936710130391174, + "loss": 3.9669, + "theoretical_loss": 4.976404497341676, + "tokens_seen": 74383360 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004936609829488465, + "loss": 3.6727, + "theoretical_loss": 4.975771460687641, + "tokens_seen": 74448896 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004936509528585758, + "loss": 3.4212, + "theoretical_loss": 4.975139136912794, + "tokens_seen": 74514432 + }, + { + "epoch": 0.02, + "learning_rate": 0.000493640922768305, + "loss": 3.8349, + "theoretical_loss": 4.974507524588424, + "tokens_seen": 74579968 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004936308926780341, + "loss": 3.9229, + "theoretical_loss": 4.973876622289927, + "tokens_seen": 74645504 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004936208625877633, + "loss": 3.4796, + "theoretical_loss": 4.973246428596802, + "tokens_seen": 74711040 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004936108324974925, + "loss": 3.7328, + "theoretical_loss": 4.972616942092634, + "tokens_seen": 74776576 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004936008024072217, + "loss": 4.1417, + "theoretical_loss": 4.971988161365077, + "tokens_seen": 74842112 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004935907723169509, + "loss": 3.5812, + "theoretical_loss": 4.9713600850058395, + "tokens_seen": 74907648 + }, + { + "epoch": 0.02, + "learning_rate": 0.00049358074222668, + "loss": 3.661, + "theoretical_loss": 4.970732711610667, + "tokens_seen": 74973184 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004935707121364092, + "loss": 3.9342, + "theoretical_loss": 4.97010603977933, + "tokens_seen": 75038720 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004935606820461384, + "loss": 3.7614, + "theoretical_loss": 4.96948006811561, + "tokens_seen": 75104256 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004935506519558676, + "loss": 3.6541, + "theoretical_loss": 4.968854795227281, + "tokens_seen": 75169792 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004935406218655968, + "loss": 3.8975, + "theoretical_loss": 4.968230219726093, + "tokens_seen": 75235328 + }, + { + "epoch": 0.02, + "learning_rate": 0.000493530591775326, + "loss": 3.8834, + "theoretical_loss": 4.967606340227765, + "tokens_seen": 75300864 + }, + { + "epoch": 0.02, + "objective/train/docs_used": 163075, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.548557996749878, + "objective/train/theoretical_loss": 4.966983155351962, + "objective/train/tokens_used": 95826400, + "theoretical_loss": 4.966983155351962, + "tokens_seen": 75366400 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004935205616850551, + "loss": 3.6221, + "theoretical_loss": 4.966983155351962, + "tokens_seen": 75366400 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004935105315947844, + "loss": 4.1199, + "theoretical_loss": 4.966360663722287, + "tokens_seen": 75431936 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004935005015045135, + "loss": 3.9567, + "theoretical_loss": 4.96573886396626, + "tokens_seen": 75497472 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004934904714142428, + "loss": 4.2001, + "theoretical_loss": 4.965117754715307, + "tokens_seen": 75563008 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004934804413239719, + "loss": 3.8199, + "theoretical_loss": 4.964497334604748, + "tokens_seen": 75628544 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004934704112337011, + "loss": 3.8548, + "theoretical_loss": 4.963877602273776, + "tokens_seen": 75694080 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004934603811434303, + "loss": 4.0167, + "theoretical_loss": 4.963258556365449, + "tokens_seen": 75759616 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004934503510531595, + "loss": 3.5764, + "theoretical_loss": 4.962640195526673, + "tokens_seen": 75825152 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004934403209628887, + "loss": 3.9003, + "theoretical_loss": 4.962022518408183, + "tokens_seen": 75890688 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004934302908726179, + "loss": 3.9603, + "theoretical_loss": 4.96140552366454, + "tokens_seen": 75956224 + }, + { + "epoch": 0.02, + "learning_rate": 0.000493420260782347, + "loss": 3.6928, + "theoretical_loss": 4.9607892099541075, + "tokens_seen": 76021760 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004934102306920762, + "loss": 3.9968, + "theoretical_loss": 4.9601735759390415, + "tokens_seen": 76087296 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004934002006018054, + "loss": 3.5467, + "theoretical_loss": 4.959558620285274, + "tokens_seen": 76152832 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004933901705115346, + "loss": 3.4065, + "theoretical_loss": 4.958944341662502, + "tokens_seen": 76218368 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004933801404212638, + "loss": 4.0583, + "theoretical_loss": 4.958330738744172, + "tokens_seen": 76283904 + }, + { + "epoch": 0.02, + "learning_rate": 0.000493370110330993, + "loss": 3.7728, + "theoretical_loss": 4.957717810207466, + "tokens_seen": 76349440 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004933600802407221, + "loss": 3.7919, + "theoretical_loss": 4.957105554733289, + "tokens_seen": 76414976 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004933500501504513, + "loss": 3.9052, + "theoretical_loss": 4.956493971006253, + "tokens_seen": 76480512 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004933400200601805, + "loss": 3.7208, + "theoretical_loss": 4.955883057714669, + "tokens_seen": 76546048 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004933299899699098, + "loss": 3.7532, + "theoretical_loss": 4.955272813550524, + "tokens_seen": 76611584 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004933199598796389, + "loss": 3.8077, + "theoretical_loss": 4.954663237209477, + "tokens_seen": 76677120 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004933099297893682, + "loss": 3.8766, + "theoretical_loss": 4.954054327390841, + "tokens_seen": 76742656 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004932998996990972, + "loss": 3.9885, + "theoretical_loss": 4.9534460827975675, + "tokens_seen": 76808192 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004932898696088265, + "loss": 3.7149, + "theoretical_loss": 4.952838502136241, + "tokens_seen": 76873728 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004932798395185557, + "loss": 3.8799, + "theoretical_loss": 4.952231584117056, + "tokens_seen": 76939264 + }, + { + "epoch": 0.02, + "objective/train/docs_used": 164442, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.9509365558624268, + "objective/train/theoretical_loss": 4.951625327453812, + "objective/train/tokens_used": 97464800, + "theoretical_loss": 4.951625327453812, + "tokens_seen": 77004800 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004932698094282849, + "loss": 3.6636, + "theoretical_loss": 4.951625327453812, + "tokens_seen": 77004800 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004932597793380141, + "loss": 3.8467, + "theoretical_loss": 4.951019730863894, + "tokens_seen": 77070336 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004932497492477432, + "loss": 3.7791, + "theoretical_loss": 4.950414793068266, + "tokens_seen": 77135872 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004932397191574724, + "loss": 3.6553, + "theoretical_loss": 4.94981051279145, + "tokens_seen": 77201408 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004932296890672016, + "loss": 4.0068, + "theoretical_loss": 4.94920688876152, + "tokens_seen": 77266944 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004932196589769308, + "loss": 3.4971, + "theoretical_loss": 4.948603919710088, + "tokens_seen": 77332480 + }, + { + "epoch": 0.02, + "learning_rate": 0.00049320962888666, + "loss": 3.5554, + "theoretical_loss": 4.948001604372287, + "tokens_seen": 77398016 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004931995987963893, + "loss": 3.8427, + "theoretical_loss": 4.947399941486762, + "tokens_seen": 77463552 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004931895687061183, + "loss": 3.7901, + "theoretical_loss": 4.946798929795658, + "tokens_seen": 77529088 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004931795386158476, + "loss": 3.6006, + "theoretical_loss": 4.946198568044602, + "tokens_seen": 77594624 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004931695085255767, + "loss": 3.7489, + "theoretical_loss": 4.945598854982698, + "tokens_seen": 77660160 + }, + { + "epoch": 0.02, + "learning_rate": 0.000493159478435306, + "loss": 3.6273, + "theoretical_loss": 4.944999789362508, + "tokens_seen": 77725696 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004931494483450352, + "loss": 3.9755, + "theoretical_loss": 4.944401369940043, + "tokens_seen": 77791232 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004931394182547643, + "loss": 3.8917, + "theoretical_loss": 4.9438035954747495, + "tokens_seen": 77856768 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004931293881644935, + "loss": 3.9238, + "theoretical_loss": 4.9432064647294975, + "tokens_seen": 77922304 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004931193580742227, + "loss": 3.8091, + "theoretical_loss": 4.942609976470566, + "tokens_seen": 77987840 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004931093279839519, + "loss": 3.6847, + "theoretical_loss": 4.942014129467637, + "tokens_seen": 78053376 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004930992978936811, + "loss": 3.7321, + "theoretical_loss": 4.941418922493774, + "tokens_seen": 78118912 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004930892678034102, + "loss": 3.9275, + "theoretical_loss": 4.940824354325419, + "tokens_seen": 78184448 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004930792377131394, + "loss": 3.5937, + "theoretical_loss": 4.940230423742372, + "tokens_seen": 78249984 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004930692076228686, + "loss": 3.9973, + "theoretical_loss": 4.939637129527789, + "tokens_seen": 78315520 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004930591775325978, + "loss": 3.6964, + "theoretical_loss": 4.939044470468156, + "tokens_seen": 78381056 + }, + { + "epoch": 0.02, + "learning_rate": 0.000493049147442327, + "loss": 3.8794, + "theoretical_loss": 4.938452445353294, + "tokens_seen": 78446592 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004930391173520562, + "loss": 3.6823, + "theoretical_loss": 4.937861052976332, + "tokens_seen": 78512128 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004930290872617853, + "loss": 4.0123, + "theoretical_loss": 4.937270292133704, + "tokens_seen": 78577664 + }, + { + "epoch": 0.02, + "objective/train/docs_used": 165209, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.9666755199432373, + "objective/train/theoretical_loss": 4.9366801616251355, + "objective/train/tokens_used": 99103200, + "theoretical_loss": 4.9366801616251355, + "tokens_seen": 78643200 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004930190571715146, + "loss": 3.7987, + "theoretical_loss": 4.9366801616251355, + "tokens_seen": 78643200 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004930090270812437, + "loss": 3.7133, + "theoretical_loss": 4.93609066025363, + "tokens_seen": 78708736 + }, + { + "epoch": 0.02, + "learning_rate": 0.000492998996990973, + "loss": 3.8611, + "theoretical_loss": 4.935501786825457, + "tokens_seen": 78774272 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004929889669007021, + "loss": 3.7493, + "theoretical_loss": 4.934913540150143, + "tokens_seen": 78839808 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004929789368104313, + "loss": 3.8265, + "theoretical_loss": 4.934325919040461, + "tokens_seen": 78905344 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004929689067201605, + "loss": 3.7146, + "theoretical_loss": 4.933738922312413, + "tokens_seen": 78970880 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004929588766298897, + "loss": 3.8796, + "theoretical_loss": 4.933152548785222, + "tokens_seen": 79036416 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004929488465396189, + "loss": 3.6061, + "theoretical_loss": 4.932566797281324, + "tokens_seen": 79101952 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004929388164493481, + "loss": 3.8318, + "theoretical_loss": 4.931981666626351, + "tokens_seen": 79167488 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004929287863590773, + "loss": 3.8867, + "theoretical_loss": 4.931397155649121, + "tokens_seen": 79233024 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004929187562688064, + "loss": 3.6513, + "theoretical_loss": 4.930813263181631, + "tokens_seen": 79298560 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004929087261785356, + "loss": 4.1337, + "theoretical_loss": 4.93022998805904, + "tokens_seen": 79364096 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004928986960882648, + "loss": 3.7265, + "theoretical_loss": 4.929647329119659, + "tokens_seen": 79429632 + }, + { + "epoch": 0.02, + "learning_rate": 0.000492888665997994, + "loss": 3.6387, + "theoretical_loss": 4.9290652852049455, + "tokens_seen": 79495168 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004928786359077232, + "loss": 3.569, + "theoretical_loss": 4.928483855159485, + "tokens_seen": 79560704 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004928686058174523, + "loss": 4.0239, + "theoretical_loss": 4.927903037830983, + "tokens_seen": 79626240 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004928585757271815, + "loss": 3.9572, + "theoretical_loss": 4.9273228320702565, + "tokens_seen": 79691776 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004928485456369107, + "loss": 3.7791, + "theoretical_loss": 4.926743236731218, + "tokens_seen": 79757312 + }, + { + "epoch": 0.02, + "learning_rate": 0.00049283851554664, + "loss": 3.8708, + "theoretical_loss": 4.926164250670868, + "tokens_seen": 79822848 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004928284854563691, + "loss": 3.6422, + "theoretical_loss": 4.925585872749284, + "tokens_seen": 79888384 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004928184553660984, + "loss": 3.9702, + "theoretical_loss": 4.925008101829608, + "tokens_seen": 79953920 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004928084252758274, + "loss": 3.745, + "theoretical_loss": 4.9244309367780374, + "tokens_seen": 80019456 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004927983951855567, + "loss": 3.6731, + "theoretical_loss": 4.923854376463816, + "tokens_seen": 80084992 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004927883650952859, + "loss": 3.8668, + "theoretical_loss": 4.923278419759217, + "tokens_seen": 80150528 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004927783350050151, + "loss": 3.8823, + "theoretical_loss": 4.92270306553954, + "tokens_seen": 80216064 + }, + { + "epoch": 0.02, + "objective/train/docs_used": 165995, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.573359489440918, + "objective/train/theoretical_loss": 4.922128312683096, + "objective/train/tokens_used": 100741600, + "theoretical_loss": 4.922128312683096, + "tokens_seen": 80281600 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004927683049147443, + "loss": 3.6868, + "theoretical_loss": 4.922128312683096, + "tokens_seen": 80281600 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004927582748244734, + "loss": 3.8374, + "theoretical_loss": 4.921554160071194, + "tokens_seen": 80347136 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004927482447342026, + "loss": 3.6288, + "theoretical_loss": 4.920980606588142, + "tokens_seen": 80412672 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004927382146439318, + "loss": 3.7754, + "theoretical_loss": 4.920407651121222, + "tokens_seen": 80478208 + }, + { + "epoch": 0.02, + "learning_rate": 0.000492728184553661, + "loss": 3.869, + "theoretical_loss": 4.919835292560689, + "tokens_seen": 80543744 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004927181544633902, + "loss": 3.8912, + "theoretical_loss": 4.919263529799759, + "tokens_seen": 80609280 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004927081243731193, + "loss": 3.8343, + "theoretical_loss": 4.918692361734598, + "tokens_seen": 80674816 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004926980942828485, + "loss": 3.8784, + "theoretical_loss": 4.91812178726431, + "tokens_seen": 80740352 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004926880641925777, + "loss": 3.6742, + "theoretical_loss": 4.917551805290929, + "tokens_seen": 80805888 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004926780341023069, + "loss": 3.7316, + "theoretical_loss": 4.916982414719408, + "tokens_seen": 80871424 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004926680040120361, + "loss": 3.9762, + "theoretical_loss": 4.9164136144576105, + "tokens_seen": 80936960 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004926579739217654, + "loss": 3.5511, + "theoretical_loss": 4.915845403416299, + "tokens_seen": 81002496 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004926479438314944, + "loss": 4.2042, + "theoretical_loss": 4.915277780509124, + "tokens_seen": 81068032 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004926379137412237, + "loss": 3.7369, + "theoretical_loss": 4.914710744652614, + "tokens_seen": 81133568 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004926278836509528, + "loss": 3.7712, + "theoretical_loss": 4.914144294766169, + "tokens_seen": 81199104 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004926178535606821, + "loss": 3.7433, + "theoretical_loss": 4.913578429772047, + "tokens_seen": 81264640 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004926078234704113, + "loss": 3.6981, + "theoretical_loss": 4.913013148595355, + "tokens_seen": 81330176 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004925977933801404, + "loss": 3.6646, + "theoretical_loss": 4.912448450164041, + "tokens_seen": 81395712 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004925877632898696, + "loss": 3.775, + "theoretical_loss": 4.91188433340888, + "tokens_seen": 81461248 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004925777331995988, + "loss": 3.7487, + "theoretical_loss": 4.911320797263471, + "tokens_seen": 81526784 + }, + { + "epoch": 0.02, + "learning_rate": 0.000492567703109328, + "loss": 3.6504, + "theoretical_loss": 4.910757840664219, + "tokens_seen": 81592320 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004925576730190572, + "loss": 3.6375, + "theoretical_loss": 4.910195462550334, + "tokens_seen": 81657856 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004925476429287864, + "loss": 3.6929, + "theoretical_loss": 4.909633661863811, + "tokens_seen": 81723392 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004925376128385155, + "loss": 3.7826, + "theoretical_loss": 4.909072437549434, + "tokens_seen": 81788928 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004925275827482447, + "loss": 3.561, + "theoretical_loss": 4.908511788554753, + "tokens_seen": 81854464 + }, + { + "epoch": 0.02, + "objective/train/docs_used": 167243, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 4.2206902503967285, + "objective/train/theoretical_loss": 4.907951713830082, + "objective/train/tokens_used": 102380000, + "theoretical_loss": 4.907951713830082, + "tokens_seen": 81920000 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004925175526579739, + "loss": 4.1068, + "theoretical_loss": 4.907951713830082, + "tokens_seen": 81920000 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004925075225677031, + "loss": 3.8568, + "theoretical_loss": 4.907392212328489, + "tokens_seen": 81985536 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004924974924774323, + "loss": 3.5104, + "theoretical_loss": 4.906833283005785, + "tokens_seen": 82051072 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004924874623871615, + "loss": 3.5485, + "theoretical_loss": 4.906274924820515, + "tokens_seen": 82116608 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004924774322968907, + "loss": 4.0778, + "theoretical_loss": 4.90571713673395, + "tokens_seen": 82182144 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004924674022066199, + "loss": 3.9291, + "theoretical_loss": 4.905159917710073, + "tokens_seen": 82247680 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004924573721163491, + "loss": 3.6175, + "theoretical_loss": 4.904603266715578, + "tokens_seen": 82313216 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004924473420260783, + "loss": 3.8755, + "theoretical_loss": 4.904047182719854, + "tokens_seen": 82378752 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004924373119358075, + "loss": 3.7729, + "theoretical_loss": 4.903491664694977, + "tokens_seen": 82444288 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004924272818455366, + "loss": 3.8916, + "theoretical_loss": 4.902936711615702, + "tokens_seen": 82509824 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004924172517552658, + "loss": 3.7129, + "theoretical_loss": 4.902382322459456, + "tokens_seen": 82575360 + }, + { + "epoch": 0.03, + "learning_rate": 0.000492407221664995, + "loss": 3.5962, + "theoretical_loss": 4.901828496206322, + "tokens_seen": 82640896 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004923971915747242, + "loss": 3.7346, + "theoretical_loss": 4.90127523183904, + "tokens_seen": 82706432 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004923871614844534, + "loss": 3.8739, + "theoretical_loss": 4.900722528342988, + "tokens_seen": 82771968 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004923771313941825, + "loss": 3.7026, + "theoretical_loss": 4.900170384706181, + "tokens_seen": 82837504 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004923671013039117, + "loss": 3.6768, + "theoretical_loss": 4.899618799919256, + "tokens_seen": 82903040 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004923570712136409, + "loss": 3.5168, + "theoretical_loss": 4.899067772975469, + "tokens_seen": 82968576 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004923470411233702, + "loss": 3.357, + "theoretical_loss": 4.898517302870679, + "tokens_seen": 83034112 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004923370110330993, + "loss": 3.6077, + "theoretical_loss": 4.897967388603346, + "tokens_seen": 83099648 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004923269809428286, + "loss": 3.879, + "theoretical_loss": 4.897418029174519, + "tokens_seen": 83165184 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004923169508525576, + "loss": 3.5125, + "theoretical_loss": 4.896869223587828, + "tokens_seen": 83230720 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004923069207622869, + "loss": 3.7582, + "theoretical_loss": 4.896320970849472, + "tokens_seen": 83296256 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004922968906720161, + "loss": 3.9325, + "theoretical_loss": 4.895773269968219, + "tokens_seen": 83361792 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004922868605817453, + "loss": 3.6416, + "theoretical_loss": 4.895226119955386, + "tokens_seen": 83427328 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004922768304914745, + "loss": 3.7782, + "theoretical_loss": 4.894679519824841, + "tokens_seen": 83492864 + }, + { + "epoch": 0.03, + "objective/train/docs_used": 167845, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.8555917739868164, + "objective/train/theoretical_loss": 4.894133468592984, + "objective/train/tokens_used": 104018400, + "theoretical_loss": 4.894133468592984, + "tokens_seen": 83558400 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004922668004012036, + "loss": 3.9438, + "theoretical_loss": 4.894133468592984, + "tokens_seen": 83558400 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004922567703109328, + "loss": 3.5061, + "theoretical_loss": 4.8935879652787495, + "tokens_seen": 83623936 + }, + { + "epoch": 0.03, + "learning_rate": 0.000492246740220662, + "loss": 3.6214, + "theoretical_loss": 4.893043008903591, + "tokens_seen": 83689472 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004922367101303912, + "loss": 3.9424, + "theoretical_loss": 4.892498598491473, + "tokens_seen": 83755008 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004922266800401204, + "loss": 3.4731, + "theoretical_loss": 4.891954733068863, + "tokens_seen": 83820544 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004922166499498495, + "loss": 3.7306, + "theoretical_loss": 4.891411411664727, + "tokens_seen": 83886080 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004922066198595787, + "loss": 3.8514, + "theoretical_loss": 4.890868633310515, + "tokens_seen": 83951616 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004921965897693079, + "loss": 3.7332, + "theoretical_loss": 4.890326397040158, + "tokens_seen": 84017152 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004921865596790371, + "loss": 3.6843, + "theoretical_loss": 4.889784701890056, + "tokens_seen": 84082688 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004921765295887663, + "loss": 3.5876, + "theoretical_loss": 4.8892435468990705, + "tokens_seen": 84148224 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004921664994984956, + "loss": 3.6961, + "theoretical_loss": 4.88870293110852, + "tokens_seen": 84213760 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004921564694082246, + "loss": 3.6685, + "theoretical_loss": 4.888162853562166, + "tokens_seen": 84279296 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004921464393179539, + "loss": 3.4346, + "theoretical_loss": 4.88762331330621, + "tokens_seen": 84344832 + }, + { + "epoch": 0.03, + "learning_rate": 0.000492136409227683, + "loss": 3.5563, + "theoretical_loss": 4.88708430938928, + "tokens_seen": 84410368 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004921263791374123, + "loss": 3.7265, + "theoretical_loss": 4.8865458408624285, + "tokens_seen": 84475904 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004921163490471415, + "loss": 3.5372, + "theoretical_loss": 4.8860079067791204, + "tokens_seen": 84541440 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004921063189568706, + "loss": 3.4829, + "theoretical_loss": 4.885470506195227, + "tokens_seen": 84606976 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004920962888665998, + "loss": 3.533, + "theoretical_loss": 4.884933638169014, + "tokens_seen": 84672512 + }, + { + "epoch": 0.03, + "learning_rate": 0.000492086258776329, + "loss": 3.9744, + "theoretical_loss": 4.88439730176114, + "tokens_seen": 84738048 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004920762286860582, + "loss": 3.674, + "theoretical_loss": 4.883861496034644, + "tokens_seen": 84803584 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004920661985957874, + "loss": 3.9751, + "theoretical_loss": 4.88332622005494, + "tokens_seen": 84869120 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004920561685055166, + "loss": 3.6296, + "theoretical_loss": 4.8827914728898065, + "tokens_seen": 84934656 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004920461384152457, + "loss": 3.6182, + "theoretical_loss": 4.88225725360938, + "tokens_seen": 85000192 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004920361083249749, + "loss": 3.7477, + "theoretical_loss": 4.881723561286149, + "tokens_seen": 85065728 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004920260782347041, + "loss": 3.6688, + "theoretical_loss": 4.881190394994943, + "tokens_seen": 85131264 + }, + { + "epoch": 0.03, + "objective/train/docs_used": 169231, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 4.038014888763428, + "objective/train/theoretical_loss": 4.880657753812926, + "objective/train/tokens_used": 105656800, + "theoretical_loss": 4.880657753812926, + "tokens_seen": 85196800 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004920160481444333, + "loss": 3.8814, + "theoretical_loss": 4.880657753812926, + "tokens_seen": 85196800 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004920060180541625, + "loss": 3.6396, + "theoretical_loss": 4.880125636819594, + "tokens_seen": 85262336 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004919959879638916, + "loss": 3.7983, + "theoretical_loss": 4.879594043096755, + "tokens_seen": 85327872 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004919859578736209, + "loss": 3.8557, + "theoretical_loss": 4.879062971728534, + "tokens_seen": 85393408 + }, + { + "epoch": 0.03, + "learning_rate": 0.00049197592778335, + "loss": 3.5785, + "theoretical_loss": 4.87853242180136, + "tokens_seen": 85458944 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004919658976930793, + "loss": 3.4825, + "theoretical_loss": 4.878002392403959, + "tokens_seen": 85524480 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004919558676028084, + "loss": 3.5934, + "theoretical_loss": 4.877472882627343, + "tokens_seen": 85590016 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004919458375125377, + "loss": 3.7652, + "theoretical_loss": 4.8769438915648085, + "tokens_seen": 85655552 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004919358074222668, + "loss": 3.5099, + "theoretical_loss": 4.876415418311928, + "tokens_seen": 85721088 + }, + { + "epoch": 0.03, + "learning_rate": 0.000491925777331996, + "loss": 3.4595, + "theoretical_loss": 4.875887461966537, + "tokens_seen": 85786624 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004919157472417252, + "loss": 3.7013, + "theoretical_loss": 4.875360021628733, + "tokens_seen": 85852160 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004919057171514544, + "loss": 3.8989, + "theoretical_loss": 4.874833096400865, + "tokens_seen": 85917696 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004918956870611836, + "loss": 3.8334, + "theoretical_loss": 4.874306685387525, + "tokens_seen": 85983232 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004918856569709127, + "loss": 3.4766, + "theoretical_loss": 4.873780787695547, + "tokens_seen": 86048768 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004918756268806419, + "loss": 3.6543, + "theoretical_loss": 4.87325540243399, + "tokens_seen": 86114304 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004918655967903711, + "loss": 3.5469, + "theoretical_loss": 4.872730528714139, + "tokens_seen": 86179840 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004918555667001003, + "loss": 3.7906, + "theoretical_loss": 4.872206165649493, + "tokens_seen": 86245376 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004918455366098295, + "loss": 3.7537, + "theoretical_loss": 4.871682312355761, + "tokens_seen": 86310912 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004918355065195586, + "loss": 3.8324, + "theoretical_loss": 4.871158967950852, + "tokens_seen": 86376448 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004918254764292878, + "loss": 3.5791, + "theoretical_loss": 4.870636131554869, + "tokens_seen": 86441984 + }, + { + "epoch": 0.03, + "learning_rate": 0.000491815446339017, + "loss": 3.6966, + "theoretical_loss": 4.8701138022901045, + "tokens_seen": 86507520 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004918054162487463, + "loss": 3.89, + "theoretical_loss": 4.869591979281028, + "tokens_seen": 86573056 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004917953861584754, + "loss": 3.9098, + "theoretical_loss": 4.8690706616542805, + "tokens_seen": 86638592 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004917853560682047, + "loss": 3.6221, + "theoretical_loss": 4.868549848538675, + "tokens_seen": 86704128 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004917753259779337, + "loss": 3.7127, + "theoretical_loss": 4.868029539065176, + "tokens_seen": 86769664 + }, + { + "epoch": 0.03, + "objective/train/docs_used": 169899, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.6036012172698975, + "objective/train/theoretical_loss": 4.867509732366907, + "objective/train/tokens_used": 107295200, + "theoretical_loss": 4.867509732366907, + "tokens_seen": 86835200 + }, + { + "epoch": 0.03, + "learning_rate": 0.000491765295887663, + "loss": 3.6755, + "theoretical_loss": 4.867509732366907, + "tokens_seen": 86835200 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004917552657973922, + "loss": 3.8447, + "theoretical_loss": 4.866990427579129, + "tokens_seen": 86900736 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004917452357071214, + "loss": 3.8378, + "theoretical_loss": 4.866471623839248, + "tokens_seen": 86966272 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004917352056168506, + "loss": 3.7024, + "theoretical_loss": 4.8659533202867955, + "tokens_seen": 87031808 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004917251755265797, + "loss": 3.7791, + "theoretical_loss": 4.86543551606343, + "tokens_seen": 87097344 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004917151454363089, + "loss": 3.7572, + "theoretical_loss": 4.864918210312927, + "tokens_seen": 87162880 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004917051153460381, + "loss": 3.4891, + "theoretical_loss": 4.864401402181173, + "tokens_seen": 87228416 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004916950852557673, + "loss": 3.7334, + "theoretical_loss": 4.863885090816158, + "tokens_seen": 87293952 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004916850551654965, + "loss": 3.818, + "theoretical_loss": 4.863369275367968, + "tokens_seen": 87359488 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004916750250752258, + "loss": 3.6889, + "theoretical_loss": 4.862853954988781, + "tokens_seen": 87425024 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004916649949849548, + "loss": 3.6571, + "theoretical_loss": 4.862339128832857, + "tokens_seen": 87490560 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004916549648946841, + "loss": 3.6037, + "theoretical_loss": 4.861824796056533, + "tokens_seen": 87556096 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004916449348044132, + "loss": 4.1055, + "theoretical_loss": 4.861310955818219, + "tokens_seen": 87621632 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004916349047141425, + "loss": 3.9186, + "theoretical_loss": 4.860797607278385, + "tokens_seen": 87687168 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004916248746238717, + "loss": 3.6732, + "theoretical_loss": 4.86028474959956, + "tokens_seen": 87752704 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004916148445336008, + "loss": 3.6389, + "theoretical_loss": 4.859772381946323, + "tokens_seen": 87818240 + }, + { + "epoch": 0.03, + "learning_rate": 0.00049160481444333, + "loss": 3.4738, + "theoretical_loss": 4.859260503485298, + "tokens_seen": 87883776 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004915947843530592, + "loss": 3.6106, + "theoretical_loss": 4.858749113385144, + "tokens_seen": 87949312 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004915847542627884, + "loss": 3.8513, + "theoretical_loss": 4.858238210816554, + "tokens_seen": 88014848 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004915747241725176, + "loss": 3.5283, + "theoretical_loss": 4.8577277949522415, + "tokens_seen": 88080384 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004915646940822468, + "loss": 3.6318, + "theoretical_loss": 4.857217864966943, + "tokens_seen": 88145920 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004915546639919759, + "loss": 3.9574, + "theoretical_loss": 4.856708420037402, + "tokens_seen": 88211456 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004915446339017051, + "loss": 3.6005, + "theoretical_loss": 4.8561994593423705, + "tokens_seen": 88276992 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004915346038114343, + "loss": 3.6561, + "theoretical_loss": 4.8556909820625975, + "tokens_seen": 88342528 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004915245737211635, + "loss": 3.5624, + "theoretical_loss": 4.855182987380823, + "tokens_seen": 88408064 + }, + { + "epoch": 0.03, + "objective/train/docs_used": 170786, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.6651480197906494, + "objective/train/theoretical_loss": 4.854675474481779, + "objective/train/tokens_used": 108933600, + "theoretical_loss": 4.854675474481779, + "tokens_seen": 88473600 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004915145436308927, + "loss": 3.5245, + "theoretical_loss": 4.854675474481779, + "tokens_seen": 88473600 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004915045135406218, + "loss": 3.9545, + "theoretical_loss": 4.8541684425521705, + "tokens_seen": 88539136 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004914944834503511, + "loss": 3.7957, + "theoretical_loss": 4.85366189078068, + "tokens_seen": 88604672 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004914844533600802, + "loss": 3.9079, + "theoretical_loss": 4.853155818357957, + "tokens_seen": 88670208 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004914744232698095, + "loss": 3.9184, + "theoretical_loss": 4.852650224476609, + "tokens_seen": 88735744 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004914643931795386, + "loss": 3.731, + "theoretical_loss": 4.852145108331205, + "tokens_seen": 88801280 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004914543630892679, + "loss": 3.7566, + "theoretical_loss": 4.851640469118255, + "tokens_seen": 88866816 + }, + { + "epoch": 0.03, + "learning_rate": 0.000491444332998997, + "loss": 3.6482, + "theoretical_loss": 4.851136306036219, + "tokens_seen": 88932352 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004914343029087262, + "loss": 3.7222, + "theoretical_loss": 4.850632618285486, + "tokens_seen": 88997888 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004914242728184554, + "loss": 3.6696, + "theoretical_loss": 4.850129405068383, + "tokens_seen": 89063424 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004914142427281846, + "loss": 3.8089, + "theoretical_loss": 4.849626665589156, + "tokens_seen": 89128960 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004914042126379138, + "loss": 3.5295, + "theoretical_loss": 4.849124399053969, + "tokens_seen": 89194496 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004913941825476429, + "loss": 3.6736, + "theoretical_loss": 4.8486226046709024, + "tokens_seen": 89260032 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004913841524573721, + "loss": 3.5729, + "theoretical_loss": 4.8481212816499415, + "tokens_seen": 89325568 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004913741223671013, + "loss": 3.7998, + "theoretical_loss": 4.847620429202967, + "tokens_seen": 89391104 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004913640922768305, + "loss": 3.4795, + "theoretical_loss": 4.847120046543763, + "tokens_seen": 89456640 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004913540621865597, + "loss": 3.8715, + "theoretical_loss": 4.846620132887992, + "tokens_seen": 89522176 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004913440320962888, + "loss": 3.7386, + "theoretical_loss": 4.8461206874532055, + "tokens_seen": 89587712 + }, + { + "epoch": 0.03, + "learning_rate": 0.000491334002006018, + "loss": 3.6659, + "theoretical_loss": 4.845621709458831, + "tokens_seen": 89653248 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004913239719157472, + "loss": 3.4551, + "theoretical_loss": 4.845123198126162, + "tokens_seen": 89718784 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004913139418254765, + "loss": 3.6237, + "theoretical_loss": 4.844625152678364, + "tokens_seen": 89784320 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004913039117352056, + "loss": 3.8513, + "theoretical_loss": 4.844127572340455, + "tokens_seen": 89849856 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004912938816449349, + "loss": 3.6724, + "theoretical_loss": 4.84363045633931, + "tokens_seen": 89915392 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004912838515546639, + "loss": 3.8743, + "theoretical_loss": 4.843133803903651, + "tokens_seen": 89980928 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004912738214643932, + "loss": 3.6873, + "theoretical_loss": 4.84263761426404, + "tokens_seen": 90046464 + }, + { + "epoch": 0.03, + "objective/train/docs_used": 171435, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.2105295658111572, + "objective/train/theoretical_loss": 4.842141886652876, + "objective/train/tokens_used": 110572000, + "theoretical_loss": 4.842141886652876, + "tokens_seen": 90112000 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004912637913741224, + "loss": 3.2592, + "theoretical_loss": 4.842141886652876, + "tokens_seen": 90112000 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004912537612838516, + "loss": 3.6091, + "theoretical_loss": 4.841646620304388, + "tokens_seen": 90177536 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004912437311935808, + "loss": 3.7829, + "theoretical_loss": 4.841151814454632, + "tokens_seen": 90243072 + }, + { + "epoch": 0.03, + "learning_rate": 0.00049123370110331, + "loss": 3.8082, + "theoretical_loss": 4.840657468341476, + "tokens_seen": 90308608 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004912236710130391, + "loss": 3.5862, + "theoretical_loss": 4.84016358120461, + "tokens_seen": 90374144 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004912136409227683, + "loss": 3.523, + "theoretical_loss": 4.839670152285526, + "tokens_seen": 90439680 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004912036108324975, + "loss": 3.3392, + "theoretical_loss": 4.8391771808275195, + "tokens_seen": 90505216 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004911935807422267, + "loss": 3.6086, + "theoretical_loss": 4.838684666075682, + "tokens_seen": 90570752 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004911835506519559, + "loss": 3.7421, + "theoretical_loss": 4.838192607276896, + "tokens_seen": 90636288 + }, + { + "epoch": 0.03, + "learning_rate": 0.000491173520561685, + "loss": 3.549, + "theoretical_loss": 4.837701003679829, + "tokens_seen": 90701824 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004911634904714142, + "loss": 3.6988, + "theoretical_loss": 4.8372098545349305, + "tokens_seen": 90767360 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004911534603811434, + "loss": 3.5152, + "theoretical_loss": 4.836719159094422, + "tokens_seen": 90832896 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004911434302908726, + "loss": 3.5065, + "theoretical_loss": 4.836228916612292, + "tokens_seen": 90898432 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004911334002006019, + "loss": 3.8109, + "theoretical_loss": 4.835739126344298, + "tokens_seen": 90963968 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004911233701103309, + "loss": 3.9112, + "theoretical_loss": 4.8352497875479505, + "tokens_seen": 91029504 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004911133400200602, + "loss": 3.8912, + "theoretical_loss": 4.834760899482514, + "tokens_seen": 91095040 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004911033099297893, + "loss": 3.9841, + "theoretical_loss": 4.834272461409001, + "tokens_seen": 91160576 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004910932798395186, + "loss": 3.6817, + "theoretical_loss": 4.833784472590165, + "tokens_seen": 91226112 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004910832497492478, + "loss": 3.795, + "theoretical_loss": 4.833296932290495, + "tokens_seen": 91291648 + }, + { + "epoch": 0.03, + "learning_rate": 0.000491073219658977, + "loss": 3.8488, + "theoretical_loss": 4.832809839776213, + "tokens_seen": 91357184 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004910631895687061, + "loss": 4.0801, + "theoretical_loss": 4.832323194315265, + "tokens_seen": 91422720 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004910531594784353, + "loss": 3.8667, + "theoretical_loss": 4.831836995177319, + "tokens_seen": 91488256 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004910431293881645, + "loss": 3.849, + "theoretical_loss": 4.831351241633756, + "tokens_seen": 91553792 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004910330992978937, + "loss": 3.8149, + "theoretical_loss": 4.8308659329576695, + "tokens_seen": 91619328 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004910230692076229, + "loss": 3.7799, + "theoretical_loss": 4.830381068423856, + "tokens_seen": 91684864 + }, + { + "epoch": 0.03, + "objective/train/docs_used": 171740, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.2981529235839844, + "objective/train/theoretical_loss": 4.8298966473088125, + "objective/train/tokens_used": 112210400, + "theoretical_loss": 4.8298966473088125, + "tokens_seen": 91750400 + }, + { + "epoch": 0.03, + "learning_rate": 0.000491013039117352, + "loss": 3.4557, + "theoretical_loss": 4.8298966473088125, + "tokens_seen": 91750400 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004910030090270812, + "loss": 4.0207, + "theoretical_loss": 4.829412668890729, + "tokens_seen": 91815936 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004909929789368104, + "loss": 3.8245, + "theoretical_loss": 4.8289291324494865, + "tokens_seen": 91881472 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004909829488465397, + "loss": 3.7635, + "theoretical_loss": 4.828446037266647, + "tokens_seen": 91947008 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004909729187562688, + "loss": 3.6794, + "theoretical_loss": 4.827963382625454, + "tokens_seen": 92012544 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004909628886659981, + "loss": 3.7506, + "theoretical_loss": 4.827481167810825, + "tokens_seen": 92078080 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004909528585757272, + "loss": 4.0619, + "theoretical_loss": 4.826999392109344, + "tokens_seen": 92143616 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004909428284854564, + "loss": 3.8418, + "theoretical_loss": 4.826518054809259, + "tokens_seen": 92209152 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004909327983951856, + "loss": 3.809, + "theoretical_loss": 4.826037155200478, + "tokens_seen": 92274688 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004909227683049148, + "loss": 3.9797, + "theoretical_loss": 4.825556692574562, + "tokens_seen": 92340224 + }, + { + "epoch": 0.03, + "learning_rate": 0.000490912738214644, + "loss": 3.8061, + "theoretical_loss": 4.825076666224717, + "tokens_seen": 92405760 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004909027081243731, + "loss": 3.7777, + "theoretical_loss": 4.824597075445799, + "tokens_seen": 92471296 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004908926780341023, + "loss": 3.8168, + "theoretical_loss": 4.824117919534297, + "tokens_seen": 92536832 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004908826479438315, + "loss": 3.8435, + "theoretical_loss": 4.823639197788334, + "tokens_seen": 92602368 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004908726178535607, + "loss": 3.5213, + "theoretical_loss": 4.823160909507665, + "tokens_seen": 92667904 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004908625877632899, + "loss": 3.6417, + "theoretical_loss": 4.822683053993664, + "tokens_seen": 92733440 + }, + { + "epoch": 0.03, + "learning_rate": 0.000490852557673019, + "loss": 3.4858, + "theoretical_loss": 4.822205630549329, + "tokens_seen": 92798976 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004908425275827482, + "loss": 3.7943, + "theoretical_loss": 4.821728638479267, + "tokens_seen": 92864512 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004908324974924774, + "loss": 3.7895, + "theoretical_loss": 4.821252077089696, + "tokens_seen": 92930048 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004908224674022067, + "loss": 3.753, + "theoretical_loss": 4.820775945688437, + "tokens_seen": 92995584 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004908124373119358, + "loss": 3.9838, + "theoretical_loss": 4.820300243584913, + "tokens_seen": 93061120 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004908024072216651, + "loss": 3.8889, + "theoretical_loss": 4.819824970090138, + "tokens_seen": 93126656 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004907923771313941, + "loss": 3.8958, + "theoretical_loss": 4.819350124516717, + "tokens_seen": 93192192 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004907823470411234, + "loss": 3.6089, + "theoretical_loss": 4.818875706178841, + "tokens_seen": 93257728 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004907723169508526, + "loss": 3.9225, + "theoretical_loss": 4.818401714392279, + "tokens_seen": 93323264 + }, + { + "epoch": 0.03, + "objective/train/docs_used": 171740, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.6480627059936523, + "objective/train/theoretical_loss": 4.817928148474378, + "objective/train/tokens_used": 113848800, + "theoretical_loss": 4.817928148474378, + "tokens_seen": 93388800 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004907622868605818, + "loss": 3.544, + "theoretical_loss": 4.817928148474378, + "tokens_seen": 93388800 + }, + { + "epoch": 0.03, + "learning_rate": 0.000490752256770311, + "loss": 3.8504, + "theoretical_loss": 4.817455007744052, + "tokens_seen": 93454336 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004907422266800401, + "loss": 3.6831, + "theoretical_loss": 4.816982291521785, + "tokens_seen": 93519872 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004907321965897693, + "loss": 3.7159, + "theoretical_loss": 4.816509999129618, + "tokens_seen": 93585408 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004907221664994985, + "loss": 3.8475, + "theoretical_loss": 4.816038129891151, + "tokens_seen": 93650944 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004907121364092277, + "loss": 3.867, + "theoretical_loss": 4.815566683131536, + "tokens_seen": 93716480 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004907021063189569, + "loss": 3.7402, + "theoretical_loss": 4.815095658177472, + "tokens_seen": 93782016 + }, + { + "epoch": 0.03, + "learning_rate": 0.000490692076228686, + "loss": 3.6066, + "theoretical_loss": 4.814625054357199, + "tokens_seen": 93847552 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004906820461384152, + "loss": 3.7494, + "theoretical_loss": 4.814154871000497, + "tokens_seen": 93913088 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004906720160481444, + "loss": 3.996, + "theoretical_loss": 4.813685107438679, + "tokens_seen": 93978624 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004906619859578736, + "loss": 3.7818, + "theoretical_loss": 4.813215763004585, + "tokens_seen": 94044160 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004906519558676028, + "loss": 3.8077, + "theoretical_loss": 4.812746837032582, + "tokens_seen": 94109696 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004906419257773321, + "loss": 3.8845, + "theoretical_loss": 4.812278328858554, + "tokens_seen": 94175232 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004906318956870611, + "loss": 3.8249, + "theoretical_loss": 4.811810237819904, + "tokens_seen": 94240768 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004906218655967904, + "loss": 3.8511, + "theoretical_loss": 4.81134256325554, + "tokens_seen": 94306304 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004906118355065195, + "loss": 3.5025, + "theoretical_loss": 4.810875304505881, + "tokens_seen": 94371840 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004906018054162488, + "loss": 3.9814, + "theoretical_loss": 4.810408460912846, + "tokens_seen": 94437376 + }, + { + "epoch": 0.03, + "learning_rate": 0.000490591775325978, + "loss": 3.7851, + "theoretical_loss": 4.809942031819853, + "tokens_seen": 94502912 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004905817452357072, + "loss": 3.7676, + "theoretical_loss": 4.809476016571809, + "tokens_seen": 94568448 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004905717151454363, + "loss": 3.8481, + "theoretical_loss": 4.809010414515113, + "tokens_seen": 94633984 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004905616850551655, + "loss": 3.8722, + "theoretical_loss": 4.808545224997644, + "tokens_seen": 94699520 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004905516549648947, + "loss": 4.0819, + "theoretical_loss": 4.808080447368766, + "tokens_seen": 94765056 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004905416248746239, + "loss": 3.7927, + "theoretical_loss": 4.807616080979315, + "tokens_seen": 94830592 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004905315947843531, + "loss": 3.8727, + "theoretical_loss": 4.807152125181597, + "tokens_seen": 94896128 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004905215646940822, + "loss": 3.7942, + "theoretical_loss": 4.806688579329387, + "tokens_seen": 94961664 + }, + { + "epoch": 0.03, + "objective/train/docs_used": 173310, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.714308500289917, + "objective/train/theoretical_loss": 4.8062254427779205, + "objective/train/tokens_used": 115487200, + "theoretical_loss": 4.8062254427779205, + "tokens_seen": 95027200 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004905115346038114, + "loss": 3.8544, + "theoretical_loss": 4.8062254427779205, + "tokens_seen": 95027200 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004905015045135406, + "loss": 3.7483, + "theoretical_loss": 4.80576271488389, + "tokens_seen": 95092736 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004904914744232698, + "loss": 3.8588, + "theoretical_loss": 4.805300395005444, + "tokens_seen": 95158272 + }, + { + "epoch": 0.03, + "learning_rate": 0.000490481444332999, + "loss": 4.0297, + "theoretical_loss": 4.804838482502181, + "tokens_seen": 95223808 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004904714142427281, + "loss": 3.9291, + "theoretical_loss": 4.8043769767351385, + "tokens_seen": 95289344 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004904613841524574, + "loss": 3.7934, + "theoretical_loss": 4.8039158770668005, + "tokens_seen": 95354880 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004904513540621865, + "loss": 3.911, + "theoretical_loss": 4.803455182861087, + "tokens_seen": 95420416 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004904413239719158, + "loss": 3.6903, + "theoretical_loss": 4.802994893483348, + "tokens_seen": 95485952 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004904312938816449, + "loss": 3.3024, + "theoretical_loss": 4.802535008300364, + "tokens_seen": 95551488 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004904212637913742, + "loss": 3.9319, + "theoretical_loss": 4.802075526680335, + "tokens_seen": 95617024 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004904112337011033, + "loss": 3.905, + "theoretical_loss": 4.801616447992888, + "tokens_seen": 95682560 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004904012036108325, + "loss": 3.6539, + "theoretical_loss": 4.801157771609061, + "tokens_seen": 95748096 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004903911735205617, + "loss": 3.9673, + "theoretical_loss": 4.8006994969013, + "tokens_seen": 95813632 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004903811434302909, + "loss": 3.9648, + "theoretical_loss": 4.800241623243467, + "tokens_seen": 95879168 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004903711133400201, + "loss": 3.6807, + "theoretical_loss": 4.799784150010819, + "tokens_seen": 95944704 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004903610832497492, + "loss": 3.5081, + "theoretical_loss": 4.799327076580017, + "tokens_seen": 96010240 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004903510531594784, + "loss": 3.9447, + "theoretical_loss": 4.798870402329115, + "tokens_seen": 96075776 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004903410230692076, + "loss": 3.8515, + "theoretical_loss": 4.798414126637558, + "tokens_seen": 96141312 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004903309929789368, + "loss": 3.6779, + "theoretical_loss": 4.797958248886179, + "tokens_seen": 96206848 + }, + { + "epoch": 0.03, + "learning_rate": 0.000490320962888666, + "loss": 3.7298, + "theoretical_loss": 4.797502768457193, + "tokens_seen": 96272384 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004903109327983952, + "loss": 3.9204, + "theoretical_loss": 4.797047684734192, + "tokens_seen": 96337920 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004903009027081243, + "loss": 3.7826, + "theoretical_loss": 4.796592997102147, + "tokens_seen": 96403456 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004902908726178535, + "loss": 3.9417, + "theoretical_loss": 4.796138704947397, + "tokens_seen": 96468992 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004902808425275828, + "loss": 3.7212, + "theoretical_loss": 4.795684807657649, + "tokens_seen": 96534528 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004902708124373119, + "loss": 3.97, + "theoretical_loss": 4.795231304621968, + "tokens_seen": 96600064 + }, + { + "epoch": 0.03, + "objective/train/docs_used": 173936, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.69027042388916, + "objective/train/theoretical_loss": 4.794778195230787, + "objective/train/tokens_used": 117125600, + "theoretical_loss": 4.794778195230787, + "tokens_seen": 96665600 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004902607823470412, + "loss": 3.7908, + "theoretical_loss": 4.794778195230787, + "tokens_seen": 96665600 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004902507522567703, + "loss": 3.6508, + "theoretical_loss": 4.794325478875885, + "tokens_seen": 96731136 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004902407221664995, + "loss": 3.8929, + "theoretical_loss": 4.793873154950399, + "tokens_seen": 96796672 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004902306920762287, + "loss": 3.9375, + "theoretical_loss": 4.793421222848808, + "tokens_seen": 96862208 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004902206619859579, + "loss": 3.9788, + "theoretical_loss": 4.7929696819669365, + "tokens_seen": 96927744 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004902106318956871, + "loss": 3.9239, + "theoretical_loss": 4.792518531701948, + "tokens_seen": 96993280 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004902006018054163, + "loss": 3.8721, + "theoretical_loss": 4.792067771452341, + "tokens_seen": 97058816 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004901905717151454, + "loss": 3.6114, + "theoretical_loss": 4.791617400617948, + "tokens_seen": 97124352 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004901805416248746, + "loss": 3.7371, + "theoretical_loss": 4.791167418599925, + "tokens_seen": 97189888 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004901705115346038, + "loss": 3.74, + "theoretical_loss": 4.790717824800755, + "tokens_seen": 97255424 + }, + { + "epoch": 0.03, + "learning_rate": 0.000490160481444333, + "loss": 3.9032, + "theoretical_loss": 4.790268618624239, + "tokens_seen": 97320960 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004901504513540623, + "loss": 3.6149, + "theoretical_loss": 4.789819799475499, + "tokens_seen": 97386496 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004901404212637913, + "loss": 3.8858, + "theoretical_loss": 4.789371366760961, + "tokens_seen": 97452032 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004901303911735206, + "loss": 3.722, + "theoretical_loss": 4.788923319888369, + "tokens_seen": 97517568 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004901203610832497, + "loss": 3.6514, + "theoretical_loss": 4.788475658266766, + "tokens_seen": 97583104 + }, + { + "epoch": 0.03, + "learning_rate": 0.000490110330992979, + "loss": 3.6426, + "theoretical_loss": 4.788028381306497, + "tokens_seen": 97648640 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004901003009027082, + "loss": 3.6375, + "theoretical_loss": 4.787581488419207, + "tokens_seen": 97714176 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004900902708124374, + "loss": 3.7361, + "theoretical_loss": 4.787134979017832, + "tokens_seen": 97779712 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004900802407221665, + "loss": 3.676, + "theoretical_loss": 4.786688852516599, + "tokens_seen": 97845248 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004900702106318957, + "loss": 3.9366, + "theoretical_loss": 4.786243108331024, + "tokens_seen": 97910784 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004900601805416249, + "loss": 3.7287, + "theoretical_loss": 4.7857977458779, + "tokens_seen": 97976320 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004900501504513541, + "loss": 3.5148, + "theoretical_loss": 4.785352764575304, + "tokens_seen": 98041856 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004900401203610833, + "loss": 3.8189, + "theoretical_loss": 4.784908163842585, + "tokens_seen": 98107392 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004900300902708124, + "loss": 3.741, + "theoretical_loss": 4.784463943100367, + "tokens_seen": 98172928 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004900200601805416, + "loss": 3.9671, + "theoretical_loss": 4.7840201017705395, + "tokens_seen": 98238464 + }, + { + "epoch": 0.03, + "objective/train/docs_used": 175137, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.979011058807373, + "objective/train/theoretical_loss": 4.783576639276257, + "objective/train/tokens_used": 118764000, + "theoretical_loss": 4.783576639276257, + "tokens_seen": 98304000 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004900100300902708, + "loss": 3.7197, + "theoretical_loss": 4.783576639276257, + "tokens_seen": 98304000 + }, + { + "epoch": 0.03, + "learning_rate": 0.00049, + "loss": 3.9945, + "theoretical_loss": 4.783133555041934, + "tokens_seen": 98369536 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004899899699097292, + "loss": 3.8699, + "theoretical_loss": 4.782690848493245, + "tokens_seen": 98435072 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004899799398194583, + "loss": 3.7009, + "theoretical_loss": 4.7822485190571165, + "tokens_seen": 98500608 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004899699097291876, + "loss": 3.9709, + "theoretical_loss": 4.781806566161723, + "tokens_seen": 98566144 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004899598796389167, + "loss": 3.8659, + "theoretical_loss": 4.781364989236488, + "tokens_seen": 98631680 + }, + { + "epoch": 0.03, + "learning_rate": 0.000489949849548646, + "loss": 3.6496, + "theoretical_loss": 4.78092378771208, + "tokens_seen": 98697216 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004899398194583751, + "loss": 3.7254, + "theoretical_loss": 4.780482961020402, + "tokens_seen": 98762752 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004899297893681044, + "loss": 3.7948, + "theoretical_loss": 4.780042508594596, + "tokens_seen": 98828288 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004899197592778335, + "loss": 3.9568, + "theoretical_loss": 4.779602429869035, + "tokens_seen": 98893824 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004899097291875627, + "loss": 3.7481, + "theoretical_loss": 4.779162724279324, + "tokens_seen": 98959360 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004898996990972919, + "loss": 3.755, + "theoretical_loss": 4.7787233912622895, + "tokens_seen": 99024896 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004898896690070211, + "loss": 3.7307, + "theoretical_loss": 4.778284430255981, + "tokens_seen": 99090432 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004898796389167503, + "loss": 3.4225, + "theoretical_loss": 4.77784584069967, + "tokens_seen": 99155968 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004898696088264794, + "loss": 3.8225, + "theoretical_loss": 4.777407622033838, + "tokens_seen": 99221504 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004898595787362086, + "loss": 3.5734, + "theoretical_loss": 4.776969773700181, + "tokens_seen": 99287040 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004898495486459378, + "loss": 3.6576, + "theoretical_loss": 4.776532295141601, + "tokens_seen": 99352576 + }, + { + "epoch": 0.03, + "learning_rate": 0.000489839518555667, + "loss": 3.916, + "theoretical_loss": 4.776095185802211, + "tokens_seen": 99418112 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004898294884653962, + "loss": 3.4804, + "theoretical_loss": 4.775658445127318, + "tokens_seen": 99483648 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004898194583751254, + "loss": 3.6648, + "theoretical_loss": 4.775222072563429, + "tokens_seen": 99549184 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004898094282848545, + "loss": 4.0798, + "theoretical_loss": 4.7747860675582485, + "tokens_seen": 99614720 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004897993981945837, + "loss": 3.7901, + "theoretical_loss": 4.77435042956067, + "tokens_seen": 99680256 + }, + { + "epoch": 0.03, + "learning_rate": 0.000489789368104313, + "loss": 3.6989, + "theoretical_loss": 4.773915158020776, + "tokens_seen": 99745792 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004897793380140421, + "loss": 3.6796, + "theoretical_loss": 4.773480252389831, + "tokens_seen": 99811328 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004897693079237714, + "loss": 3.6931, + "theoretical_loss": 4.773045712120284, + "tokens_seen": 99876864 + }, + { + "epoch": 0.03, + "objective/train/docs_used": 175797, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.162777900695801, + "objective/train/theoretical_loss": 4.77261153666576, + "objective/train/tokens_used": 120402400, + "theoretical_loss": 4.77261153666576, + "tokens_seen": 99942400 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004897592778335005, + "loss": 3.6811, + "theoretical_loss": 4.77261153666576, + "tokens_seen": 99942400 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004897492477432297, + "loss": 3.8359, + "theoretical_loss": 4.772177725481062, + "tokens_seen": 100007936 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004897392176529589, + "loss": 4.1084, + "theoretical_loss": 4.77174427802216, + "tokens_seen": 100073472 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004897291875626881, + "loss": 3.5353, + "theoretical_loss": 4.771311193746191, + "tokens_seen": 100139008 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004897191574724173, + "loss": 3.6427, + "theoretical_loss": 4.770878472111465, + "tokens_seen": 100204544 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004897091273821465, + "loss": 3.7959, + "theoretical_loss": 4.770446112577445, + "tokens_seen": 100270080 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004896990972918756, + "loss": 3.7708, + "theoretical_loss": 4.770014114604756, + "tokens_seen": 100335616 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004896890672016048, + "loss": 3.698, + "theoretical_loss": 4.769582477655177, + "tokens_seen": 100401152 + }, + { + "epoch": 0.03, + "learning_rate": 0.000489679037111334, + "loss": 3.8029, + "theoretical_loss": 4.769151201191641, + "tokens_seen": 100466688 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004896690070210632, + "loss": 3.488, + "theoretical_loss": 4.768720284678228, + "tokens_seen": 100532224 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004896589769307924, + "loss": 3.7797, + "theoretical_loss": 4.768289727580161, + "tokens_seen": 100597760 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004896489468405215, + "loss": 3.4447, + "theoretical_loss": 4.767859529363809, + "tokens_seen": 100663296 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004896389167502507, + "loss": 3.7995, + "theoretical_loss": 4.767429689496682, + "tokens_seen": 100728832 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004896288866599799, + "loss": 3.8211, + "theoretical_loss": 4.767000207447417, + "tokens_seen": 100794368 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004896188565697091, + "loss": 3.5972, + "theoretical_loss": 4.766571082685794, + "tokens_seen": 100859904 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004896088264794384, + "loss": 3.8521, + "theoretical_loss": 4.766142314682716, + "tokens_seen": 100925440 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004895987963891674, + "loss": 3.7264, + "theoretical_loss": 4.765713902910214, + "tokens_seen": 100990976 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004895887662988967, + "loss": 3.702, + "theoretical_loss": 4.765285846841444, + "tokens_seen": 101056512 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004895787362086259, + "loss": 3.8745, + "theoretical_loss": 4.76485814595068, + "tokens_seen": 101122048 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004895687061183551, + "loss": 3.8407, + "theoretical_loss": 4.764430799713314, + "tokens_seen": 101187584 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004895586760280843, + "loss": 3.5995, + "theoretical_loss": 4.764003807605853, + "tokens_seen": 101253120 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004895486459378135, + "loss": 3.6566, + "theoretical_loss": 4.763577169105912, + "tokens_seen": 101318656 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004895386158475426, + "loss": 3.7968, + "theoretical_loss": 4.763150883692218, + "tokens_seen": 101384192 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004895285857572718, + "loss": 3.6979, + "theoretical_loss": 4.762724950844598, + "tokens_seen": 101449728 + }, + { + "epoch": 0.03, + "learning_rate": 0.000489518555667001, + "loss": 3.7237, + "theoretical_loss": 4.762299370043984, + "tokens_seen": 101515264 + }, + { + "epoch": 0.03, + "objective/train/docs_used": 179449, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.7288663387298584, + "objective/train/theoretical_loss": 4.761874140772408, + "objective/train/tokens_used": 122040800, + "theoretical_loss": 4.761874140772408, + "tokens_seen": 101580800 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004895085255767302, + "loss": 3.8993, + "theoretical_loss": 4.761874140772408, + "tokens_seen": 101580800 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004894984954864594, + "loss": 3.523, + "theoretical_loss": 4.761449262512993, + "tokens_seen": 101646336 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004894884653961885, + "loss": 3.4939, + "theoretical_loss": 4.761024734749958, + "tokens_seen": 101711872 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004894784353059178, + "loss": 3.8311, + "theoretical_loss": 4.76060055696861, + "tokens_seen": 101777408 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004894684052156469, + "loss": 3.8815, + "theoretical_loss": 4.760176728655345, + "tokens_seen": 101842944 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004894583751253762, + "loss": 3.8344, + "theoretical_loss": 4.75975324929764, + "tokens_seen": 101908480 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004894483450351053, + "loss": 3.7128, + "theoretical_loss": 4.759330118384053, + "tokens_seen": 101974016 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004894383149448346, + "loss": 3.7163, + "theoretical_loss": 4.758907335404221, + "tokens_seen": 102039552 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004894282848545637, + "loss": 3.5908, + "theoretical_loss": 4.758484899848854, + "tokens_seen": 102105088 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004894182547642929, + "loss": 3.8424, + "theoretical_loss": 4.7580628112097365, + "tokens_seen": 102170624 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004894082246740221, + "loss": 3.6798, + "theoretical_loss": 4.7576410689797175, + "tokens_seen": 102236160 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004893981945837513, + "loss": 3.6937, + "theoretical_loss": 4.757219672652717, + "tokens_seen": 102301696 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004893881644934805, + "loss": 3.6042, + "theoretical_loss": 4.756798621723712, + "tokens_seen": 102367232 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004893781344032096, + "loss": 3.8252, + "theoretical_loss": 4.756377915688748, + "tokens_seen": 102432768 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004893681043129388, + "loss": 3.6281, + "theoretical_loss": 4.755957554044917, + "tokens_seen": 102498304 + }, + { + "epoch": 0.03, + "learning_rate": 0.000489358074222668, + "loss": 3.634, + "theoretical_loss": 4.755537536290373, + "tokens_seen": 102563840 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004893480441323972, + "loss": 3.5845, + "theoretical_loss": 4.755117861924321, + "tokens_seen": 102629376 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004893380140421264, + "loss": 3.7616, + "theoretical_loss": 4.754698530447009, + "tokens_seen": 102694912 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004893279839518556, + "loss": 3.8346, + "theoretical_loss": 4.754279541359738, + "tokens_seen": 102760448 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004893179538615847, + "loss": 3.4694, + "theoretical_loss": 4.753860894164845, + "tokens_seen": 102825984 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004893079237713139, + "loss": 3.8897, + "theoretical_loss": 4.75344258836571, + "tokens_seen": 102891520 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004892978936810432, + "loss": 3.5267, + "theoretical_loss": 4.753024623466752, + "tokens_seen": 102957056 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004892878635907723, + "loss": 3.4275, + "theoretical_loss": 4.752606998973421, + "tokens_seen": 103022592 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004892778335005016, + "loss": 3.5392, + "theoretical_loss": 4.752189714392202, + "tokens_seen": 103088128 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004892678034102307, + "loss": 3.6427, + "theoretical_loss": 4.7517727692306035, + "tokens_seen": 103153664 + }, + { + "epoch": 0.03, + "objective/train/docs_used": 184726, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9697039127349854, + "objective/train/theoretical_loss": 4.751356162997164, + "objective/train/tokens_used": 123679200, + "theoretical_loss": 4.751356162997164, + "tokens_seen": 103219200 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004892577733199599, + "loss": 3.525, + "theoretical_loss": 4.751356162997164, + "tokens_seen": 103219200 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004892477432296891, + "loss": 3.6341, + "theoretical_loss": 4.750939895201443, + "tokens_seen": 103284736 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004892377131394183, + "loss": 3.6469, + "theoretical_loss": 4.750523965354024, + "tokens_seen": 103350272 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004892276830491475, + "loss": 3.4156, + "theoretical_loss": 4.750108372966501, + "tokens_seen": 103415808 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004892176529588767, + "loss": 3.5315, + "theoretical_loss": 4.749693117551491, + "tokens_seen": 103481344 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004892076228686058, + "loss": 3.6321, + "theoretical_loss": 4.749278198622617, + "tokens_seen": 103546880 + }, + { + "epoch": 0.03, + "learning_rate": 0.000489197592778335, + "loss": 3.54, + "theoretical_loss": 4.748863615694514, + "tokens_seen": 103612416 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004891875626880642, + "loss": 3.6321, + "theoretical_loss": 4.748449368282822, + "tokens_seen": 103677952 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004891775325977934, + "loss": 3.7374, + "theoretical_loss": 4.748035455904185, + "tokens_seen": 103743488 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004891675025075226, + "loss": 3.7389, + "theoretical_loss": 4.747621878076252, + "tokens_seen": 103809024 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004891574724172517, + "loss": 3.8047, + "theoretical_loss": 4.747208634317664, + "tokens_seen": 103874560 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004891474423269809, + "loss": 3.6744, + "theoretical_loss": 4.746795724148061, + "tokens_seen": 103940096 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004891374122367101, + "loss": 3.9085, + "theoretical_loss": 4.746383147088078, + "tokens_seen": 104005632 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004891273821464393, + "loss": 3.471, + "theoretical_loss": 4.745970902659338, + "tokens_seen": 104071168 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004891173520561686, + "loss": 3.8325, + "theoretical_loss": 4.745558990384451, + "tokens_seen": 104136704 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004891073219658976, + "loss": 3.523, + "theoretical_loss": 4.7451474097870125, + "tokens_seen": 104202240 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004890972918756269, + "loss": 3.5334, + "theoretical_loss": 4.744736160391602, + "tokens_seen": 104267776 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004890872617853561, + "loss": 3.6145, + "theoretical_loss": 4.744325241723777, + "tokens_seen": 104333312 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004890772316950853, + "loss": 3.7004, + "theoretical_loss": 4.743914653310073, + "tokens_seen": 104398848 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004890672016048145, + "loss": 3.674, + "theoretical_loss": 4.743504394678, + "tokens_seen": 104464384 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004890571715145437, + "loss": 3.8081, + "theoretical_loss": 4.743094465356039, + "tokens_seen": 104529920 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004890471414242728, + "loss": 3.7858, + "theoretical_loss": 4.742684864873641, + "tokens_seen": 104595456 + }, + { + "epoch": 0.03, + "learning_rate": 0.000489037111334002, + "loss": 3.5326, + "theoretical_loss": 4.742275592761223, + "tokens_seen": 104660992 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004890270812437312, + "loss": 3.5429, + "theoretical_loss": 4.741866648550168, + "tokens_seen": 104726528 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004890170511534604, + "loss": 3.4406, + "theoretical_loss": 4.741458031772817, + "tokens_seen": 104792064 + }, + { + "epoch": 0.03, + "objective/train/docs_used": 187033, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 4.107706069946289, + "objective/train/theoretical_loss": 4.741049741962473, + "objective/train/tokens_used": 125317600, + "theoretical_loss": 4.741049741962473, + "tokens_seen": 104857600 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004890070210631896, + "loss": 3.9333, + "theoretical_loss": 4.741049741962473, + "tokens_seen": 104857600 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004889969909729187, + "loss": 3.5559, + "theoretical_loss": 4.740641778653395, + "tokens_seen": 104923136 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004889869608826479, + "loss": 3.4574, + "theoretical_loss": 4.740234141380794, + "tokens_seen": 104988672 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004889769307923771, + "loss": 3.7356, + "theoretical_loss": 4.739826829680833, + "tokens_seen": 105054208 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004889669007021063, + "loss": 3.472, + "theoretical_loss": 4.739419843090626, + "tokens_seen": 105119744 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004889568706118355, + "loss": 3.7092, + "theoretical_loss": 4.739013181148229, + "tokens_seen": 105185280 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004889468405215647, + "loss": 3.8201, + "theoretical_loss": 4.738606843392644, + "tokens_seen": 105250816 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004889368104312939, + "loss": 3.4478, + "theoretical_loss": 4.738200829363815, + "tokens_seen": 105316352 + }, + { + "epoch": 0.03, + "learning_rate": 0.000488926780341023, + "loss": 3.9938, + "theoretical_loss": 4.737795138602624, + "tokens_seen": 105381888 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004889167502507523, + "loss": 3.7457, + "theoretical_loss": 4.737389770650887, + "tokens_seen": 105447424 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004889067201604815, + "loss": 3.6661, + "theoretical_loss": 4.736984725051357, + "tokens_seen": 105512960 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004888966900702107, + "loss": 3.6371, + "theoretical_loss": 4.736580001347717, + "tokens_seen": 105578496 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004888866599799398, + "loss": 3.7063, + "theoretical_loss": 4.736175599084576, + "tokens_seen": 105644032 + }, + { + "epoch": 0.03, + "learning_rate": 0.000488876629889669, + "loss": 3.7422, + "theoretical_loss": 4.735771517807473, + "tokens_seen": 105709568 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004888665997993982, + "loss": 3.6926, + "theoretical_loss": 4.735367757062869, + "tokens_seen": 105775104 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004888565697091274, + "loss": 3.616, + "theoretical_loss": 4.734964316398148, + "tokens_seen": 105840640 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004888465396188566, + "loss": 3.5101, + "theoretical_loss": 4.734561195361609, + "tokens_seen": 105906176 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004888365095285858, + "loss": 3.5185, + "theoretical_loss": 4.734158393502471, + "tokens_seen": 105971712 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004888264794383149, + "loss": 3.8055, + "theoretical_loss": 4.733755910370867, + "tokens_seen": 106037248 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004888164493480441, + "loss": 3.6915, + "theoretical_loss": 4.73335374551784, + "tokens_seen": 106102784 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004888064192577733, + "loss": 3.5202, + "theoretical_loss": 4.732951898495341, + "tokens_seen": 106168320 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004887963891675025, + "loss": 3.675, + "theoretical_loss": 4.7325503688562325, + "tokens_seen": 106233856 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004887863590772317, + "loss": 3.4758, + "theoretical_loss": 4.732149156154276, + "tokens_seen": 106299392 + }, + { + "epoch": 0.03, + "learning_rate": 0.000488776328986961, + "loss": 3.7876, + "theoretical_loss": 4.731748259944139, + "tokens_seen": 106364928 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048876629889669, + "loss": 3.649, + "theoretical_loss": 4.731347679781386, + "tokens_seen": 106430464 + }, + { + "epoch": 0.03, + "objective/train/docs_used": 192056, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.8580915927886963, + "objective/train/theoretical_loss": 4.730947415222481, + "objective/train/tokens_used": 126956000, + "theoretical_loss": 4.730947415222481, + "tokens_seen": 106496000 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004887562688064193, + "loss": 3.6611, + "theoretical_loss": 4.730947415222481, + "tokens_seen": 106496000 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004887462387161484, + "loss": 3.8264, + "theoretical_loss": 4.730547465824781, + "tokens_seen": 106561536 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004887362086258777, + "loss": 3.863, + "theoretical_loss": 4.730147831146537, + "tokens_seen": 106627072 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004887261785356069, + "loss": 3.5344, + "theoretical_loss": 4.72974851074689, + "tokens_seen": 106692608 + }, + { + "epoch": 0.03, + "learning_rate": 0.000488716148445336, + "loss": 3.4504, + "theoretical_loss": 4.729349504185867, + "tokens_seen": 106758144 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004887061183550652, + "loss": 3.642, + "theoretical_loss": 4.728950811024383, + "tokens_seen": 106823680 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004886960882647944, + "loss": 3.6794, + "theoretical_loss": 4.7285524308242355, + "tokens_seen": 106889216 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004886860581745236, + "loss": 3.2107, + "theoretical_loss": 4.728154363148102, + "tokens_seen": 106954752 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004886760280842528, + "loss": 3.6065, + "theoretical_loss": 4.72775660755954, + "tokens_seen": 107020288 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004886659979939819, + "loss": 3.9955, + "theoretical_loss": 4.72735916362298, + "tokens_seen": 107085824 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004886559679037111, + "loss": 3.6974, + "theoretical_loss": 4.7269620309037315, + "tokens_seen": 107151360 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004886459378134403, + "loss": 3.728, + "theoretical_loss": 4.726565208967973, + "tokens_seen": 107216896 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004886359077231695, + "loss": 3.4342, + "theoretical_loss": 4.726168697382751, + "tokens_seen": 107282432 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004886258776328988, + "loss": 3.2861, + "theoretical_loss": 4.725772495715983, + "tokens_seen": 107347968 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004886158475426278, + "loss": 3.7293, + "theoretical_loss": 4.725376603536446, + "tokens_seen": 107413504 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004886058174523571, + "loss": 3.964, + "theoretical_loss": 4.724981020413787, + "tokens_seen": 107479040 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004885957873620863, + "loss": 3.6411, + "theoretical_loss": 4.724585745918505, + "tokens_seen": 107544576 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004885857572718155, + "loss": 3.6381, + "theoretical_loss": 4.7241907796219635, + "tokens_seen": 107610112 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004885757271815447, + "loss": 3.6016, + "theoretical_loss": 4.723796121096381, + "tokens_seen": 107675648 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004885656970912739, + "loss": 3.3598, + "theoretical_loss": 4.723401769914824, + "tokens_seen": 107741184 + }, + { + "epoch": 0.03, + "learning_rate": 0.000488555667001003, + "loss": 3.731, + "theoretical_loss": 4.723007725651219, + "tokens_seen": 107806720 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004885456369107322, + "loss": 3.6714, + "theoretical_loss": 4.722613987880335, + "tokens_seen": 107872256 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004885356068204614, + "loss": 3.6898, + "theoretical_loss": 4.722220556177792, + "tokens_seen": 107937792 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004885255767301906, + "loss": 3.609, + "theoretical_loss": 4.721827430120053, + "tokens_seen": 108003328 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004885155466399198, + "loss": 3.4883, + "theoretical_loss": 4.721434609284424, + "tokens_seen": 108068864 + }, + { + "epoch": 0.03, + "objective/train/docs_used": 197231, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.762702465057373, + "objective/train/theoretical_loss": 4.721042093249051, + "objective/train/tokens_used": 128594400, + "theoretical_loss": 4.721042093249051, + "tokens_seen": 108134400 + }, + { + "epoch": 0.03, + "learning_rate": 0.000488505516549649, + "loss": 3.6925, + "theoretical_loss": 4.721042093249051, + "tokens_seen": 108134400 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004884954864593781, + "loss": 3.4986, + "theoretical_loss": 4.720649881592919, + "tokens_seen": 108199936 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004884854563691073, + "loss": 3.4754, + "theoretical_loss": 4.7202579738958494, + "tokens_seen": 108265472 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004884754262788365, + "loss": 3.3368, + "theoretical_loss": 4.7198663697384955, + "tokens_seen": 108331008 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004884653961885657, + "loss": 3.6466, + "theoretical_loss": 4.719475068702346, + "tokens_seen": 108396544 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004884553660982949, + "loss": 3.5481, + "theoretical_loss": 4.719084070369714, + "tokens_seen": 108462080 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004884453360080241, + "loss": 3.5884, + "theoretical_loss": 4.718693374323747, + "tokens_seen": 108527616 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004884353059177532, + "loss": 3.8041, + "theoretical_loss": 4.718302980148412, + "tokens_seen": 108593152 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004884252758274825, + "loss": 3.8141, + "theoretical_loss": 4.717912887428501, + "tokens_seen": 108658688 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004884152457372117, + "loss": 3.5719, + "theoretical_loss": 4.717523095749626, + "tokens_seen": 108724224 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004884052156469409, + "loss": 3.7324, + "theoretical_loss": 4.717133604698222, + "tokens_seen": 108789760 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048839518555667, + "loss": 3.6457, + "theoretical_loss": 4.7167444138615355, + "tokens_seen": 108855296 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004883851554663992, + "loss": 3.4354, + "theoretical_loss": 4.716355522827633, + "tokens_seen": 108920832 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004883751253761284, + "loss": 3.6252, + "theoretical_loss": 4.715966931185388, + "tokens_seen": 108986368 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004883650952858576, + "loss": 3.591, + "theoretical_loss": 4.715578638524491, + "tokens_seen": 109051904 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004883550651955868, + "loss": 3.6158, + "theoretical_loss": 4.715190644435435, + "tokens_seen": 109117440 + }, + { + "epoch": 0.03, + "learning_rate": 0.000488345035105316, + "loss": 3.6187, + "theoretical_loss": 4.714802948509522, + "tokens_seen": 109182976 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004883350050150451, + "loss": 3.7455, + "theoretical_loss": 4.71441555033886, + "tokens_seen": 109248512 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004883249749247743, + "loss": 3.712, + "theoretical_loss": 4.714028449516356, + "tokens_seen": 109314048 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004883149448345035, + "loss": 3.7868, + "theoretical_loss": 4.713641645635718, + "tokens_seen": 109379584 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004883049147442327, + "loss": 3.4678, + "theoretical_loss": 4.713255138291454, + "tokens_seen": 109445120 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004882948846539619, + "loss": 3.6966, + "theoretical_loss": 4.712868927078868, + "tokens_seen": 109510656 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004882848545636911, + "loss": 3.7838, + "theoretical_loss": 4.712483011594056, + "tokens_seen": 109576192 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004882748244734203, + "loss": 3.3161, + "theoretical_loss": 4.7120973914339075, + "tokens_seen": 109641728 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048826479438314946, + "loss": 3.8165, + "theoretical_loss": 4.7117120661961005, + "tokens_seen": 109707264 + }, + { + "epoch": 0.03, + "objective/train/docs_used": 198319, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.8059911727905273, + "objective/train/theoretical_loss": 4.711327035479103, + "objective/train/tokens_used": 130232800, + "theoretical_loss": 4.711327035479103, + "tokens_seen": 109772800 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048825476429287864, + "loss": 3.6663, + "theoretical_loss": 4.711327035479103, + "tokens_seen": 109772800 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004882447342026078, + "loss": 3.4004, + "theoretical_loss": 4.710942298882169, + "tokens_seen": 109838336 + }, + { + "epoch": 0.03, + "learning_rate": 0.000488234704112337, + "loss": 3.5996, + "theoretical_loss": 4.710557856005335, + "tokens_seen": 109903872 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048822467402206624, + "loss": 3.7299, + "theoretical_loss": 4.710173706449419, + "tokens_seen": 109969408 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048821464393179536, + "loss": 3.6217, + "theoretical_loss": 4.709789849816021, + "tokens_seen": 110034944 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004882046138415246, + "loss": 3.8175, + "theoretical_loss": 4.7094062857075185, + "tokens_seen": 110100480 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004881945837512537, + "loss": 3.5329, + "theoretical_loss": 4.709023013727063, + "tokens_seen": 110166016 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048818455366098296, + "loss": 3.5599, + "theoretical_loss": 4.708640033478584, + "tokens_seen": 110231552 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048817452357071214, + "loss": 3.4719, + "theoretical_loss": 4.708257344566778, + "tokens_seen": 110297088 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004881644934804413, + "loss": 3.627, + "theoretical_loss": 4.7078749465971175, + "tokens_seen": 110362624 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004881544633901705, + "loss": 3.7143, + "theoretical_loss": 4.707492839175837, + "tokens_seen": 110428160 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048814443329989974, + "loss": 3.5303, + "theoretical_loss": 4.707111021909941, + "tokens_seen": 110493696 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048813440320962887, + "loss": 3.5631, + "theoretical_loss": 4.706729494407197, + "tokens_seen": 110559232 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004881243731193581, + "loss": 3.4962, + "theoretical_loss": 4.706348256276138, + "tokens_seen": 110624768 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048811434302908723, + "loss": 3.6254, + "theoretical_loss": 4.705967307126051, + "tokens_seen": 110690304 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048810431293881646, + "loss": 3.5333, + "theoretical_loss": 4.705586646566987, + "tokens_seen": 110755840 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048809428284854564, + "loss": 3.386, + "theoretical_loss": 4.705206274209751, + "tokens_seen": 110821376 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004880842527582748, + "loss": 3.6736, + "theoretical_loss": 4.704826189665905, + "tokens_seen": 110886912 + }, + { + "epoch": 0.03, + "learning_rate": 0.000488074222668004, + "loss": 3.4569, + "theoretical_loss": 4.704446392547759, + "tokens_seen": 110952448 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004880641925777332, + "loss": 3.5925, + "theoretical_loss": 4.7040668824683785, + "tokens_seen": 111017984 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048805416248746237, + "loss": 3.6843, + "theoretical_loss": 4.7036876590415755, + "tokens_seen": 111083520 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004880441323971916, + "loss": 3.8082, + "theoretical_loss": 4.7033087218819105, + "tokens_seen": 111149056 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048803410230692073, + "loss": 3.5422, + "theoretical_loss": 4.7029300706046895, + "tokens_seen": 111214592 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048802407221664997, + "loss": 3.7771, + "theoretical_loss": 4.702551704825957, + "tokens_seen": 111280128 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048801404212637915, + "loss": 3.6929, + "theoretical_loss": 4.702173624162507, + "tokens_seen": 111345664 + }, + { + "epoch": 0.03, + "objective/train/docs_used": 199018, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.2992162704467773, + "objective/train/theoretical_loss": 4.701795828231866, + "objective/train/tokens_used": 131871200, + "theoretical_loss": 4.701795828231866, + "tokens_seen": 111411200 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048800401203610833, + "loss": 3.3269, + "theoretical_loss": 4.701795828231866, + "tokens_seen": 111411200 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004879939819458375, + "loss": 3.4583, + "theoretical_loss": 4.701418316652299, + "tokens_seen": 111476736 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004879839518555667, + "loss": 3.5179, + "theoretical_loss": 4.701041089042813, + "tokens_seen": 111542272 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004879739217652959, + "loss": 3.5889, + "theoretical_loss": 4.700664145023142, + "tokens_seen": 111607808 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004879638916750251, + "loss": 3.6855, + "theoretical_loss": 4.700287484213753, + "tokens_seen": 111673344 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004879538615847543, + "loss": 3.6633, + "theoretical_loss": 4.699911106235849, + "tokens_seen": 111738880 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048794383149448347, + "loss": 3.3769, + "theoretical_loss": 4.6995350107113545, + "tokens_seen": 111804416 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048793380140421265, + "loss": 3.4899, + "theoretical_loss": 4.699159197262922, + "tokens_seen": 111869952 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048792377131394183, + "loss": 3.4922, + "theoretical_loss": 4.698783665513934, + "tokens_seen": 111935488 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048791374122367107, + "loss": 3.6508, + "theoretical_loss": 4.698408415088491, + "tokens_seen": 112001024 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004879037111334002, + "loss": 3.644, + "theoretical_loss": 4.698033445611415, + "tokens_seen": 112066560 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048789368104312943, + "loss": 3.6637, + "theoretical_loss": 4.6976587567082495, + "tokens_seen": 112132096 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048788365095285856, + "loss": 3.3232, + "theoretical_loss": 4.697284348005253, + "tokens_seen": 112197632 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004878736208625878, + "loss": 3.6476, + "theoretical_loss": 4.696910219129402, + "tokens_seen": 112263168 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048786359077231697, + "loss": 3.7732, + "theoretical_loss": 4.696536369708386, + "tokens_seen": 112328704 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048785356068204615, + "loss": 3.5794, + "theoretical_loss": 4.696162799370606, + "tokens_seen": 112394240 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048784353059177533, + "loss": 3.5167, + "theoretical_loss": 4.695789507745176, + "tokens_seen": 112459776 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048783350050150457, + "loss": 3.508, + "theoretical_loss": 4.695416494461917, + "tokens_seen": 112525312 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004878234704112337, + "loss": 3.6201, + "theoretical_loss": 4.695043759151353, + "tokens_seen": 112590848 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048781344032096293, + "loss": 3.5772, + "theoretical_loss": 4.694671301444722, + "tokens_seen": 112656384 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048780341023069206, + "loss": 3.511, + "theoretical_loss": 4.694299120973957, + "tokens_seen": 112721920 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004877933801404213, + "loss": 3.4497, + "theoretical_loss": 4.693927217371698, + "tokens_seen": 112787456 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004877833500501505, + "loss": 3.6642, + "theoretical_loss": 4.693555590271282, + "tokens_seen": 112852992 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048777331995987966, + "loss": 3.1849, + "theoretical_loss": 4.693184239306744, + "tokens_seen": 112918528 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048776328986960884, + "loss": 3.7865, + "theoretical_loss": 4.692813164112819, + "tokens_seen": 112984064 + }, + { + "epoch": 0.03, + "objective/train/docs_used": 200274, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.979872226715088, + "objective/train/theoretical_loss": 4.692442364324931, + "objective/train/tokens_used": 133509600, + "theoretical_loss": 4.692442364324931, + "tokens_seen": 113049600 + }, + { + "epoch": 0.03, + "learning_rate": 0.000487753259779338, + "loss": 3.2723, + "theoretical_loss": 4.692442364324931, + "tokens_seen": 113049600 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004877432296890672, + "loss": 3.5444, + "theoretical_loss": 4.692071839579201, + "tokens_seen": 113115136 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048773319959879644, + "loss": 3.4821, + "theoretical_loss": 4.6917015895124425, + "tokens_seen": 113180672 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048772316950852556, + "loss": 3.8326, + "theoretical_loss": 4.691331613762153, + "tokens_seen": 113246208 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004877131394182548, + "loss": 3.4582, + "theoretical_loss": 4.690961911966523, + "tokens_seen": 113311744 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004877031093279839, + "loss": 3.3548, + "theoretical_loss": 4.690592483764427, + "tokens_seen": 113377280 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048769307923771316, + "loss": 3.4375, + "theoretical_loss": 4.690223328795424, + "tokens_seen": 113442816 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048768304914744234, + "loss": 3.5515, + "theoretical_loss": 4.689854446699757, + "tokens_seen": 113508352 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004876730190571715, + "loss": 3.5217, + "theoretical_loss": 4.689485837118347, + "tokens_seen": 113573888 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004876629889669007, + "loss": 3.5703, + "theoretical_loss": 4.689117499692798, + "tokens_seen": 113639424 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048765295887662994, + "loss": 3.568, + "theoretical_loss": 4.688749434065389, + "tokens_seen": 113704960 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048764292878635907, + "loss": 3.5214, + "theoretical_loss": 4.688381639879076, + "tokens_seen": 113770496 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004876328986960883, + "loss": 3.7132, + "theoretical_loss": 4.68801411677749, + "tokens_seen": 113836032 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048762286860581743, + "loss": 3.6747, + "theoretical_loss": 4.687646864404934, + "tokens_seen": 113901568 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048761283851554666, + "loss": 3.7008, + "theoretical_loss": 4.687279882406381, + "tokens_seen": 113967104 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048760280842527584, + "loss": 3.6326, + "theoretical_loss": 4.686913170427477, + "tokens_seen": 114032640 + }, + { + "epoch": 0.03, + "learning_rate": 0.000487592778335005, + "loss": 3.5939, + "theoretical_loss": 4.68654672811453, + "tokens_seen": 114098176 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004875827482447342, + "loss": 3.5751, + "theoretical_loss": 4.68618055511452, + "tokens_seen": 114163712 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004875727181544634, + "loss": 3.4383, + "theoretical_loss": 4.685814651075088, + "tokens_seen": 114229248 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048756268806419257, + "loss": 3.484, + "theoretical_loss": 4.685449015644537, + "tokens_seen": 114294784 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004875526579739218, + "loss": 3.5631, + "theoretical_loss": 4.685083648471835, + "tokens_seen": 114360320 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048754262788365093, + "loss": 3.592, + "theoretical_loss": 4.684718549206607, + "tokens_seen": 114425856 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048753259779338017, + "loss": 3.7434, + "theoretical_loss": 4.6843537174991345, + "tokens_seen": 114491392 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048752256770310935, + "loss": 3.5416, + "theoretical_loss": 4.6839891530003595, + "tokens_seen": 114556928 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048751253761283853, + "loss": 3.8109, + "theoretical_loss": 4.683624855361876, + "tokens_seen": 114622464 + }, + { + "epoch": 0.03, + "objective/train/docs_used": 200959, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 4.169064998626709, + "objective/train/theoretical_loss": 4.68326082423593, + "objective/train/tokens_used": 135148000, + "theoretical_loss": 4.68326082423593, + "tokens_seen": 114688000 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004875025075225677, + "loss": 3.5971, + "theoretical_loss": 4.68326082423593, + "tokens_seen": 114688000 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004874924774322969, + "loss": 3.6054, + "theoretical_loss": 4.682897059275422, + "tokens_seen": 114753536 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048748244734202607, + "loss": 3.5122, + "theoretical_loss": 4.682533560133901, + "tokens_seen": 114819072 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004874724172517553, + "loss": 3.5253, + "theoretical_loss": 4.682170326465565, + "tokens_seen": 114884608 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048746238716148443, + "loss": 3.4639, + "theoretical_loss": 4.681807357925257, + "tokens_seen": 114950144 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048745235707121367, + "loss": 3.4482, + "theoretical_loss": 4.681444654168468, + "tokens_seen": 115015680 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004874423269809428, + "loss": 3.4827, + "theoretical_loss": 4.68108221485133, + "tokens_seen": 115081216 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048743229689067203, + "loss": 3.5963, + "theoretical_loss": 4.680720039630617, + "tokens_seen": 115146752 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004874222668004012, + "loss": 3.604, + "theoretical_loss": 4.680358128163747, + "tokens_seen": 115212288 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004874122367101304, + "loss": 3.575, + "theoretical_loss": 4.679996480108773, + "tokens_seen": 115277824 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004874022066198596, + "loss": 3.6315, + "theoretical_loss": 4.6796350951243895, + "tokens_seen": 115343360 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048739217652958876, + "loss": 3.5988, + "theoretical_loss": 4.679273972869922, + "tokens_seen": 115408896 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048738214643931794, + "loss": 3.544, + "theoretical_loss": 4.678913113005333, + "tokens_seen": 115474432 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048737211634904717, + "loss": 3.434, + "theoretical_loss": 4.6785525151912175, + "tokens_seen": 115539968 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004873620862587763, + "loss": 3.7679, + "theoretical_loss": 4.678192179088802, + "tokens_seen": 115605504 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048735205616850553, + "loss": 3.3392, + "theoretical_loss": 4.6778321043599425, + "tokens_seen": 115671040 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004873420260782347, + "loss": 3.6271, + "theoretical_loss": 4.677472290667122, + "tokens_seen": 115736576 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004873319959879639, + "loss": 3.5469, + "theoretical_loss": 4.677112737673453, + "tokens_seen": 115802112 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004873219658976931, + "loss": 3.7895, + "theoretical_loss": 4.676753445042669, + "tokens_seen": 115867648 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048731193580742226, + "loss": 3.395, + "theoretical_loss": 4.676394412439132, + "tokens_seen": 115933184 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048730190571715144, + "loss": 3.4239, + "theoretical_loss": 4.6760356395278215, + "tokens_seen": 115998720 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004872918756268807, + "loss": 3.8464, + "theoretical_loss": 4.675677125974339, + "tokens_seen": 116064256 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004872818455366098, + "loss": 3.6943, + "theoretical_loss": 4.675318871444908, + "tokens_seen": 116129792 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048727181544633904, + "loss": 3.2368, + "theoretical_loss": 4.674960875606366, + "tokens_seen": 116195328 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048726178535606816, + "loss": 3.5752, + "theoretical_loss": 4.674603138126168, + "tokens_seen": 116260864 + }, + { + "epoch": 0.04, + "objective/train/docs_used": 202277, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.6310184001922607, + "objective/train/theoretical_loss": 4.674245658672382, + "objective/train/tokens_used": 136786400, + "theoretical_loss": 4.674245658672382, + "tokens_seen": 116326400 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004872517552657974, + "loss": 3.3857, + "theoretical_loss": 4.674245658672382, + "tokens_seen": 116326400 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004872417251755266, + "loss": 3.6198, + "theoretical_loss": 4.673888436913694, + "tokens_seen": 116391936 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048723169508525576, + "loss": 3.5595, + "theoretical_loss": 4.673531472519397, + "tokens_seen": 116457472 + }, + { + "epoch": 0.04, + "learning_rate": 0.000487221664994985, + "loss": 3.6411, + "theoretical_loss": 4.673174765159393, + "tokens_seen": 116523008 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004872116349047141, + "loss": 3.5595, + "theoretical_loss": 4.672818314504198, + "tokens_seen": 116588544 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048720160481444336, + "loss": 3.5692, + "theoretical_loss": 4.6724621202249335, + "tokens_seen": 116654080 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048719157472417254, + "loss": 3.3932, + "theoretical_loss": 4.672106181993324, + "tokens_seen": 116719616 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004871815446339017, + "loss": 3.5326, + "theoretical_loss": 4.6717504994817, + "tokens_seen": 116785152 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004871715145436309, + "loss": 3.2663, + "theoretical_loss": 4.671395072362996, + "tokens_seen": 116850688 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048716148445336014, + "loss": 3.565, + "theoretical_loss": 4.671039900310747, + "tokens_seen": 116916224 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048715145436308927, + "loss": 3.582, + "theoretical_loss": 4.670684982999088, + "tokens_seen": 116981760 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004871414242728185, + "loss": 3.6837, + "theoretical_loss": 4.670330320102753, + "tokens_seen": 117047296 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048713139418254763, + "loss": 3.6938, + "theoretical_loss": 4.669975911297072, + "tokens_seen": 117112832 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048712136409227686, + "loss": 3.5719, + "theoretical_loss": 4.669621756257971, + "tokens_seen": 117178368 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048711133400200604, + "loss": 3.6868, + "theoretical_loss": 4.669267854661973, + "tokens_seen": 117243904 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004871013039117352, + "loss": 3.3895, + "theoretical_loss": 4.668914206186189, + "tokens_seen": 117309440 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004870912738214644, + "loss": 3.4743, + "theoretical_loss": 4.6685608105083265, + "tokens_seen": 117374976 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004870812437311936, + "loss": 3.4699, + "theoretical_loss": 4.66820766730668, + "tokens_seen": 117440512 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048707121364092277, + "loss": 3.727, + "theoretical_loss": 4.667854776260132, + "tokens_seen": 117506048 + }, + { + "epoch": 0.04, + "learning_rate": 0.000487061183550652, + "loss": 3.5116, + "theoretical_loss": 4.667502137048155, + "tokens_seen": 117571584 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048705115346038113, + "loss": 3.4195, + "theoretical_loss": 4.667149749350805, + "tokens_seen": 117637120 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048704112337011037, + "loss": 3.6593, + "theoretical_loss": 4.666797612848723, + "tokens_seen": 117702656 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048703109327983955, + "loss": 3.3363, + "theoretical_loss": 4.666445727223134, + "tokens_seen": 117768192 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048702106318956873, + "loss": 3.5724, + "theoretical_loss": 4.666094092155843, + "tokens_seen": 117833728 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004870110330992979, + "loss": 3.4272, + "theoretical_loss": 4.665742707329238, + "tokens_seen": 117899264 + }, + { + "epoch": 0.04, + "objective/train/docs_used": 203088, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.953324317932129, + "objective/train/theoretical_loss": 4.665391572426282, + "objective/train/tokens_used": 138424800, + "theoretical_loss": 4.665391572426282, + "tokens_seen": 117964800 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004870010030090271, + "loss": 3.5257, + "theoretical_loss": 4.665391572426282, + "tokens_seen": 117964800 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048699097291875627, + "loss": 3.6506, + "theoretical_loss": 4.665040687130518, + "tokens_seen": 118030336 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004869809428284855, + "loss": 3.5996, + "theoretical_loss": 4.664690051126065, + "tokens_seen": 118095872 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048697091273821463, + "loss": 3.5075, + "theoretical_loss": 4.664339664097617, + "tokens_seen": 118161408 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048696088264794387, + "loss": 3.6508, + "theoretical_loss": 4.66398952573044, + "tokens_seen": 118226944 + }, + { + "epoch": 0.04, + "learning_rate": 0.000486950852557673, + "loss": 3.7334, + "theoretical_loss": 4.663639635710373, + "tokens_seen": 118292480 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048694082246740223, + "loss": 3.6102, + "theoretical_loss": 4.663289993723826, + "tokens_seen": 118358016 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004869307923771314, + "loss": 3.5511, + "theoretical_loss": 4.662940599457777, + "tokens_seen": 118423552 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004869207622868606, + "loss": 3.5001, + "theoretical_loss": 4.662591452599774, + "tokens_seen": 118489088 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004869107321965898, + "loss": 3.485, + "theoretical_loss": 4.662242552837929, + "tokens_seen": 118554624 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048690070210631896, + "loss": 3.6105, + "theoretical_loss": 4.661893899860923, + "tokens_seen": 118620160 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048689067201604814, + "loss": 3.7023, + "theoretical_loss": 4.6615454933579965, + "tokens_seen": 118685696 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048688064192577737, + "loss": 3.4614, + "theoretical_loss": 4.661197333018957, + "tokens_seen": 118751232 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004868706118355065, + "loss": 3.5773, + "theoretical_loss": 4.66084941853417, + "tokens_seen": 118816768 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048686058174523573, + "loss": 3.6587, + "theoretical_loss": 4.6605017495945615, + "tokens_seen": 118882304 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004868505516549649, + "loss": 3.7256, + "theoretical_loss": 4.660154325891618, + "tokens_seen": 118947840 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004868405215646941, + "loss": 3.6683, + "theoretical_loss": 4.659807147117382, + "tokens_seen": 119013376 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004868304914744233, + "loss": 3.5433, + "theoretical_loss": 4.6594602129644525, + "tokens_seen": 119078912 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048682046138415246, + "loss": 3.6163, + "theoretical_loss": 4.659113523125981, + "tokens_seen": 119144448 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048681043129388164, + "loss": 3.6002, + "theoretical_loss": 4.6587670772956775, + "tokens_seen": 119209984 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004868004012036109, + "loss": 3.744, + "theoretical_loss": 4.658420875167799, + "tokens_seen": 119275520 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048679037111334, + "loss": 3.5674, + "theoretical_loss": 4.658074916437155, + "tokens_seen": 119341056 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048678034102306924, + "loss": 3.705, + "theoretical_loss": 4.657729200799105, + "tokens_seen": 119406592 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048677031093279836, + "loss": 3.5421, + "theoretical_loss": 4.657383727949558, + "tokens_seen": 119472128 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004867602808425276, + "loss": 3.2356, + "theoretical_loss": 4.657038497584967, + "tokens_seen": 119537664 + }, + { + "epoch": 0.04, + "objective/train/docs_used": 204120, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.542322874069214, + "objective/train/theoretical_loss": 4.656693509402331, + "objective/train/tokens_used": 140063200, + "theoretical_loss": 4.656693509402331, + "tokens_seen": 119603200 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004867502507522568, + "loss": 3.5035, + "theoretical_loss": 4.656693509402331, + "tokens_seen": 119603200 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048674022066198596, + "loss": 3.6295, + "theoretical_loss": 4.6563487630991975, + "tokens_seen": 119668736 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048673019057171514, + "loss": 3.6794, + "theoretical_loss": 4.656004258373651, + "tokens_seen": 119734272 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004867201604814443, + "loss": 3.4914, + "theoretical_loss": 4.655659994924323, + "tokens_seen": 119799808 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004867101303911735, + "loss": 3.4807, + "theoretical_loss": 4.655315972450383, + "tokens_seen": 119865344 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048670010030090274, + "loss": 3.6322, + "theoretical_loss": 4.65497219065154, + "tokens_seen": 119930880 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048669007021063187, + "loss": 3.5918, + "theoretical_loss": 4.654628649228041, + "tokens_seen": 119996416 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004866800401203611, + "loss": 3.6249, + "theoretical_loss": 4.654285347880672, + "tokens_seen": 120061952 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004866700100300903, + "loss": 3.6893, + "theoretical_loss": 4.653942286310749, + "tokens_seen": 120127488 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048665997993981947, + "loss": 3.7484, + "theoretical_loss": 4.653599464220129, + "tokens_seen": 120193024 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048664994984954865, + "loss": 3.6358, + "theoretical_loss": 4.653256881311198, + "tokens_seen": 120258560 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048663991975927783, + "loss": 3.5567, + "theoretical_loss": 4.6529145372868745, + "tokens_seen": 120324096 + }, + { + "epoch": 0.04, + "learning_rate": 0.000486629889669007, + "loss": 3.519, + "theoretical_loss": 4.652572431850608, + "tokens_seen": 120389632 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048661985957873624, + "loss": 3.4446, + "theoretical_loss": 4.652230564706377, + "tokens_seen": 120455168 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048660982948846537, + "loss": 3.4514, + "theoretical_loss": 4.651888935558688, + "tokens_seen": 120520704 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004865997993981946, + "loss": 3.3068, + "theoretical_loss": 4.651547544112575, + "tokens_seen": 120586240 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048658976930792373, + "loss": 3.7197, + "theoretical_loss": 4.651206390073597, + "tokens_seen": 120651776 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048657973921765297, + "loss": 3.4481, + "theoretical_loss": 4.650865473147837, + "tokens_seen": 120717312 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048656970912738215, + "loss": 3.3836, + "theoretical_loss": 4.650524793041903, + "tokens_seen": 120782848 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048655967903711133, + "loss": 3.259, + "theoretical_loss": 4.650184349462922, + "tokens_seen": 120848384 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004865496489468405, + "loss": 3.4753, + "theoretical_loss": 4.649844142118544, + "tokens_seen": 120913920 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048653961885656975, + "loss": 3.6915, + "theoretical_loss": 4.6495041707169396, + "tokens_seen": 120979456 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004865295887662989, + "loss": 3.562, + "theoretical_loss": 4.649164434966794, + "tokens_seen": 121044992 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004865195586760281, + "loss": 3.4441, + "theoretical_loss": 4.648824934577313, + "tokens_seen": 121110528 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048650952858575724, + "loss": 3.7728, + "theoretical_loss": 4.648485669258216, + "tokens_seen": 121176064 + }, + { + "epoch": 0.04, + "objective/train/docs_used": 204902, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.444392681121826, + "objective/train/theoretical_loss": 4.648146638719739, + "objective/train/tokens_used": 141701600, + "theoretical_loss": 4.648146638719739, + "tokens_seen": 121241600 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048649949849548647, + "loss": 3.7584, + "theoretical_loss": 4.648146638719739, + "tokens_seen": 121241600 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048648946840521565, + "loss": 3.3618, + "theoretical_loss": 4.647807842672631, + "tokens_seen": 121307136 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048647943831494483, + "loss": 3.6952, + "theoretical_loss": 4.647469280828153, + "tokens_seen": 121372672 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048646940822467407, + "loss": 3.4908, + "theoretical_loss": 4.647130952898077, + "tokens_seen": 121438208 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004864593781344032, + "loss": 3.9747, + "theoretical_loss": 4.646792858594686, + "tokens_seen": 121503744 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048644934804413243, + "loss": 3.3985, + "theoretical_loss": 4.64645499763077, + "tokens_seen": 121569280 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004864393179538616, + "loss": 3.4271, + "theoretical_loss": 4.646117369719629, + "tokens_seen": 121634816 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004864292878635908, + "loss": 3.5924, + "theoretical_loss": 4.645779974575069, + "tokens_seen": 121700352 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048641925777332, + "loss": 3.4433, + "theoretical_loss": 4.6454428119113995, + "tokens_seen": 121765888 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048640922768304916, + "loss": 3.6515, + "theoretical_loss": 4.6451058814434365, + "tokens_seen": 121831424 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048639919759277834, + "loss": 3.8542, + "theoretical_loss": 4.644769182886495, + "tokens_seen": 121896960 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048638916750250757, + "loss": 3.6482, + "theoretical_loss": 4.644432715956399, + "tokens_seen": 121962496 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004863791374122367, + "loss": 3.4727, + "theoretical_loss": 4.644096480369466, + "tokens_seen": 122028032 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048636910732196593, + "loss": 3.4828, + "theoretical_loss": 4.643760475842518, + "tokens_seen": 122093568 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004863590772316951, + "loss": 3.3981, + "theoretical_loss": 4.6434247020928705, + "tokens_seen": 122159104 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004863490471414243, + "loss": 3.4389, + "theoretical_loss": 4.643089158838341, + "tokens_seen": 122224640 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004863390170511535, + "loss": 3.7861, + "theoretical_loss": 4.642753845797243, + "tokens_seen": 122290176 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048632898696088266, + "loss": 3.4705, + "theoretical_loss": 4.642418762688379, + "tokens_seen": 122355712 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048631895687061184, + "loss": 3.5749, + "theoretical_loss": 4.642083909231053, + "tokens_seen": 122421248 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004863089267803411, + "loss": 3.6578, + "theoretical_loss": 4.641749285145057, + "tokens_seen": 122486784 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004862988966900702, + "loss": 3.407, + "theoretical_loss": 4.641414890150675, + "tokens_seen": 122552320 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048628886659979944, + "loss": 3.5225, + "theoretical_loss": 4.641080723968684, + "tokens_seen": 122617856 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048627883650952857, + "loss": 3.4672, + "theoretical_loss": 4.6407467863203475, + "tokens_seen": 122683392 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004862688064192578, + "loss": 3.336, + "theoretical_loss": 4.640413076927418, + "tokens_seen": 122748928 + }, + { + "epoch": 0.04, + "learning_rate": 0.000486258776328987, + "loss": 3.2779, + "theoretical_loss": 4.6400795955121374, + "tokens_seen": 122814464 + }, + { + "epoch": 0.04, + "objective/train/docs_used": 206332, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 4.079704761505127, + "objective/train/theoretical_loss": 4.639746341797229, + "objective/train/tokens_used": 143340000, + "theoretical_loss": 4.639746341797229, + "tokens_seen": 122880000 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048624874623871616, + "loss": 3.7945, + "theoretical_loss": 4.639746341797229, + "tokens_seen": 122880000 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048623871614844534, + "loss": 3.7026, + "theoretical_loss": 4.639413315505905, + "tokens_seen": 122945536 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004862286860581745, + "loss": 3.7991, + "theoretical_loss": 4.639080516361861, + "tokens_seen": 123011072 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004862186559679037, + "loss": 3.3407, + "theoretical_loss": 4.638747944089273, + "tokens_seen": 123076608 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048620862587763294, + "loss": 3.6945, + "theoretical_loss": 4.638415598412799, + "tokens_seen": 123142144 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048619859578736207, + "loss": 3.3499, + "theoretical_loss": 4.638083479057579, + "tokens_seen": 123207680 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004861885656970913, + "loss": 3.4411, + "theoretical_loss": 4.637751585749234, + "tokens_seen": 123273216 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004861785356068205, + "loss": 3.419, + "theoretical_loss": 4.6374199182138565, + "tokens_seen": 123338752 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048616850551654967, + "loss": 3.1327, + "theoretical_loss": 4.637088476178025, + "tokens_seen": 123404288 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048615847542627885, + "loss": 3.3689, + "theoretical_loss": 4.636757259368787, + "tokens_seen": 123469824 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048614844533600803, + "loss": 3.2499, + "theoretical_loss": 4.636426267513668, + "tokens_seen": 123535360 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004861384152457372, + "loss": 3.443, + "theoretical_loss": 4.636095500340669, + "tokens_seen": 123600896 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048612838515546644, + "loss": 3.4505, + "theoretical_loss": 4.635764957578261, + "tokens_seen": 123666432 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048611835506519557, + "loss": 3.4719, + "theoretical_loss": 4.635434638955388, + "tokens_seen": 123731968 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004861083249749248, + "loss": 3.6203, + "theoretical_loss": 4.635104544201465, + "tokens_seen": 123797504 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048609829488465393, + "loss": 3.1321, + "theoretical_loss": 4.634774673046376, + "tokens_seen": 123863040 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048608826479438317, + "loss": 3.3834, + "theoretical_loss": 4.634445025220475, + "tokens_seen": 123928576 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048607823470411235, + "loss": 3.5649, + "theoretical_loss": 4.634115600454582, + "tokens_seen": 123994112 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048606820461384153, + "loss": 3.3337, + "theoretical_loss": 4.633786398479983, + "tokens_seen": 124059648 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004860581745235707, + "loss": 3.3326, + "theoretical_loss": 4.6334574190284314, + "tokens_seen": 124125184 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048604814443329995, + "loss": 3.2998, + "theoretical_loss": 4.633128661832145, + "tokens_seen": 124190720 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004860381143430291, + "loss": 3.581, + "theoretical_loss": 4.632800126623803, + "tokens_seen": 124256256 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004860280842527583, + "loss": 3.6333, + "theoretical_loss": 4.632471813136547, + "tokens_seen": 124321792 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048601805416248744, + "loss": 3.447, + "theoretical_loss": 4.632143721103983, + "tokens_seen": 124387328 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048600802407221667, + "loss": 3.2293, + "theoretical_loss": 4.631815850260173, + "tokens_seen": 124452864 + }, + { + "epoch": 0.04, + "objective/train/docs_used": 207169, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.6358470916748047, + "objective/train/theoretical_loss": 4.631488200339643, + "objective/train/tokens_used": 144978400, + "theoretical_loss": 4.631488200339643, + "tokens_seen": 124518400 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048599799398194585, + "loss": 3.4651, + "theoretical_loss": 4.631488200339643, + "tokens_seen": 124518400 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048598796389167503, + "loss": 3.4181, + "theoretical_loss": 4.63116077107737, + "tokens_seen": 124583936 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004859779338014042, + "loss": 3.6055, + "theoretical_loss": 4.630833562208797, + "tokens_seen": 124649472 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004859679037111334, + "loss": 3.4885, + "theoretical_loss": 4.630506573469815, + "tokens_seen": 124715008 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004859578736208626, + "loss": 3.5052, + "theoretical_loss": 4.630179804596775, + "tokens_seen": 124780544 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004859478435305918, + "loss": 3.5164, + "theoretical_loss": 4.629853255326481, + "tokens_seen": 124846080 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048593781344032094, + "loss": 3.3836, + "theoretical_loss": 4.629526925396189, + "tokens_seen": 124911616 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004859277833500502, + "loss": 3.4527, + "theoretical_loss": 4.6292008145436085, + "tokens_seen": 124977152 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004859177532597793, + "loss": 3.37, + "theoretical_loss": 4.628874922506897, + "tokens_seen": 125042688 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048590772316950854, + "loss": 3.294, + "theoretical_loss": 4.628549249024666, + "tokens_seen": 125108224 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004858976930792377, + "loss": 3.3212, + "theoretical_loss": 4.628223793835975, + "tokens_seen": 125173760 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004858876629889669, + "loss": 3.3518, + "theoretical_loss": 4.627898556680327, + "tokens_seen": 125239296 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004858776328986961, + "loss": 3.2403, + "theoretical_loss": 4.627573537297678, + "tokens_seen": 125304832 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004858676028084253, + "loss": 3.4935, + "theoretical_loss": 4.627248735428427, + "tokens_seen": 125370368 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048585757271815444, + "loss": 3.4322, + "theoretical_loss": 4.6269241508134185, + "tokens_seen": 125435904 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004858475426278837, + "loss": 3.5357, + "theoretical_loss": 4.6265997831939405, + "tokens_seen": 125501440 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004858375125376128, + "loss": 3.4814, + "theoretical_loss": 4.6262756323117245, + "tokens_seen": 125566976 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048582748244734204, + "loss": 3.4993, + "theoretical_loss": 4.625951697908944, + "tokens_seen": 125632512 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004858174523570712, + "loss": 3.4275, + "theoretical_loss": 4.625627979728212, + "tokens_seen": 125698048 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004858074222668004, + "loss": 3.38, + "theoretical_loss": 4.625304477512584, + "tokens_seen": 125763584 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004857973921765296, + "loss": 3.1327, + "theoretical_loss": 4.624981191005554, + "tokens_seen": 125829120 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048578736208625877, + "loss": 3.4219, + "theoretical_loss": 4.624658119951052, + "tokens_seen": 125894656 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048577733199598795, + "loss": 3.4453, + "theoretical_loss": 4.624335264093447, + "tokens_seen": 125960192 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004857673019057172, + "loss": 3.5913, + "theoretical_loss": 4.624012623177544, + "tokens_seen": 126025728 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004857572718154463, + "loss": 3.3552, + "theoretical_loss": 4.623690196948582, + "tokens_seen": 126091264 + }, + { + "epoch": 0.04, + "objective/train/docs_used": 208560, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.118776321411133, + "objective/train/theoretical_loss": 4.623367985152234, + "objective/train/tokens_used": 146616800, + "theoretical_loss": 4.623367985152234, + "tokens_seen": 126156800 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048574724172517554, + "loss": 3.3034, + "theoretical_loss": 4.623367985152234, + "tokens_seen": 126156800 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048573721163490467, + "loss": 3.4021, + "theoretical_loss": 4.623045987534609, + "tokens_seen": 126222336 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004857271815446339, + "loss": 3.4968, + "theoretical_loss": 4.622724203842246, + "tokens_seen": 126287872 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048571715145436314, + "loss": 3.6168, + "theoretical_loss": 4.622402633822114, + "tokens_seen": 126353408 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048570712136409227, + "loss": 3.6483, + "theoretical_loss": 4.622081277221616, + "tokens_seen": 126418944 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004856970912738215, + "loss": 3.1841, + "theoretical_loss": 4.62176013378858, + "tokens_seen": 126484480 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004856870611835507, + "loss": 3.3364, + "theoretical_loss": 4.621439203271267, + "tokens_seen": 126550016 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048567703109327987, + "loss": 3.5219, + "theoretical_loss": 4.621118485418362, + "tokens_seen": 126615552 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048566700100300905, + "loss": 3.5676, + "theoretical_loss": 4.620797979978978, + "tokens_seen": 126681088 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048565697091273823, + "loss": 3.0699, + "theoretical_loss": 4.620477686702651, + "tokens_seen": 126746624 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004856469408224674, + "loss": 3.4134, + "theoretical_loss": 4.620157605339347, + "tokens_seen": 126812160 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048563691073219664, + "loss": 3.3575, + "theoretical_loss": 4.619837735639452, + "tokens_seen": 126877696 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048562688064192577, + "loss": 3.1724, + "theoretical_loss": 4.619518077353776, + "tokens_seen": 126943232 + }, + { + "epoch": 0.04, + "learning_rate": 0.000485616850551655, + "loss": 3.5963, + "theoretical_loss": 4.619198630233547, + "tokens_seen": 127008768 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048560682046138413, + "loss": 3.2372, + "theoretical_loss": 4.6188793940304205, + "tokens_seen": 127074304 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048559679037111337, + "loss": 3.5238, + "theoretical_loss": 4.618560368496466, + "tokens_seen": 127139840 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048558676028084255, + "loss": 3.4135, + "theoretical_loss": 4.618241553384175, + "tokens_seen": 127205376 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048557673019057173, + "loss": 3.5748, + "theoretical_loss": 4.617922948446459, + "tokens_seen": 127270912 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004855667001003009, + "loss": 3.2217, + "theoretical_loss": 4.617604553436642, + "tokens_seen": 127336448 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048555667001003015, + "loss": 3.4823, + "theoretical_loss": 4.617286368108466, + "tokens_seen": 127401984 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004855466399197593, + "loss": 3.5026, + "theoretical_loss": 4.6169683922160925, + "tokens_seen": 127467520 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004855366098294885, + "loss": 3.3712, + "theoretical_loss": 4.616650625514091, + "tokens_seen": 127533056 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048552657973921764, + "loss": 3.3915, + "theoretical_loss": 4.616333067757449, + "tokens_seen": 127598592 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048551654964894687, + "loss": 3.5052, + "theoretical_loss": 4.616015718701563, + "tokens_seen": 127664128 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048550651955867605, + "loss": 3.4679, + "theoretical_loss": 4.615698578102245, + "tokens_seen": 127729664 + }, + { + "epoch": 0.04, + "objective/train/docs_used": 209446, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.4149038791656494, + "objective/train/theoretical_loss": 4.615381645715717, + "objective/train/tokens_used": 148255200, + "theoretical_loss": 4.615381645715717, + "tokens_seen": 127795200 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048549648946840523, + "loss": 3.3464, + "theoretical_loss": 4.615381645715717, + "tokens_seen": 127795200 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004854864593781344, + "loss": 3.448, + "theoretical_loss": 4.615064921298608, + "tokens_seen": 127860736 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004854764292878636, + "loss": 3.4669, + "theoretical_loss": 4.61474840460796, + "tokens_seen": 127926272 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004854663991975928, + "loss": 3.2942, + "theoretical_loss": 4.614432095401219, + "tokens_seen": 127991808 + }, + { + "epoch": 0.04, + "learning_rate": 0.000485456369107322, + "loss": 3.5544, + "theoretical_loss": 4.614115993436242, + "tokens_seen": 128057344 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048544633901705114, + "loss": 3.6377, + "theoretical_loss": 4.613800098471291, + "tokens_seen": 128122880 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004854363089267804, + "loss": 3.4673, + "theoretical_loss": 4.613484410265032, + "tokens_seen": 128188416 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004854262788365095, + "loss": 3.3257, + "theoretical_loss": 4.613168928576538, + "tokens_seen": 128253952 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048541624874623874, + "loss": 3.3088, + "theoretical_loss": 4.612853653165283, + "tokens_seen": 128319488 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004854062186559679, + "loss": 3.3764, + "theoretical_loss": 4.612538583791146, + "tokens_seen": 128385024 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004853961885656971, + "loss": 3.347, + "theoretical_loss": 4.612223720214407, + "tokens_seen": 128450560 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004853861584754263, + "loss": 3.5082, + "theoretical_loss": 4.611909062195749, + "tokens_seen": 128516096 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004853761283851555, + "loss": 3.5355, + "theoretical_loss": 4.61159460949625, + "tokens_seen": 128581632 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048536609829488464, + "loss": 3.5775, + "theoretical_loss": 4.611280361877393, + "tokens_seen": 128647168 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004853560682046139, + "loss": 3.4158, + "theoretical_loss": 4.610966319101056, + "tokens_seen": 128712704 + }, + { + "epoch": 0.04, + "learning_rate": 0.000485346038114343, + "loss": 3.4332, + "theoretical_loss": 4.610652480929515, + "tokens_seen": 128778240 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048533600802407224, + "loss": 3.5417, + "theoretical_loss": 4.610338847125445, + "tokens_seen": 128843776 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004853259779338014, + "loss": 3.2011, + "theoretical_loss": 4.610025417451913, + "tokens_seen": 128909312 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004853159478435306, + "loss": 3.3289, + "theoretical_loss": 4.6097121916723856, + "tokens_seen": 128974848 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004853059177532598, + "loss": 3.4422, + "theoretical_loss": 4.609399169550718, + "tokens_seen": 129040384 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048529588766298897, + "loss": 3.4443, + "theoretical_loss": 4.609086350851165, + "tokens_seen": 129105920 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048528585757271815, + "loss": 3.5788, + "theoretical_loss": 4.6087737353383655, + "tokens_seen": 129171456 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004852758274824474, + "loss": 3.2671, + "theoretical_loss": 4.6084613227773605, + "tokens_seen": 129236992 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004852657973921765, + "loss": 3.4653, + "theoretical_loss": 4.608149112933571, + "tokens_seen": 129302528 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048525576730190574, + "loss": 3.2752, + "theoretical_loss": 4.607837105572816, + "tokens_seen": 129368064 + }, + { + "epoch": 0.04, + "objective/train/docs_used": 210153, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.262286424636841, + "objective/train/theoretical_loss": 4.607525300461299, + "objective/train/tokens_used": 149893600, + "theoretical_loss": 4.607525300461299, + "tokens_seen": 129433600 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048524573721163487, + "loss": 3.4158, + "theoretical_loss": 4.607525300461299, + "tokens_seen": 129433600 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004852357071213641, + "loss": 3.84, + "theoretical_loss": 4.607213697365613, + "tokens_seen": 129499136 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004852256770310933, + "loss": 3.2357, + "theoretical_loss": 4.606902296052739, + "tokens_seen": 129564672 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048521564694082247, + "loss": 3.1954, + "theoretical_loss": 4.6065910962900425, + "tokens_seen": 129630208 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048520561685055165, + "loss": 3.2395, + "theoretical_loss": 4.606280097845277, + "tokens_seen": 129695744 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004851955867602809, + "loss": 3.6586, + "theoretical_loss": 4.60596930048658, + "tokens_seen": 129761280 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048518555667001, + "loss": 3.8546, + "theoretical_loss": 4.605658703982471, + "tokens_seen": 129826816 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048517552657973925, + "loss": 3.6046, + "theoretical_loss": 4.6053483081018545, + "tokens_seen": 129892352 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004851654964894684, + "loss": 3.6551, + "theoretical_loss": 4.605038112614018, + "tokens_seen": 129957888 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004851554663991976, + "loss": 3.7827, + "theoretical_loss": 4.604728117288631, + "tokens_seen": 130023424 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004851454363089268, + "loss": 3.4682, + "theoretical_loss": 4.604418321895739, + "tokens_seen": 130088960 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048513540621865597, + "loss": 3.4146, + "theoretical_loss": 4.604108726205774, + "tokens_seen": 130154496 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048512537612838515, + "loss": 3.6829, + "theoretical_loss": 4.60379932998954, + "tokens_seen": 130220032 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048511534603811433, + "loss": 3.2773, + "theoretical_loss": 4.6034901330182265, + "tokens_seen": 130285568 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004851053159478435, + "loss": 3.441, + "theoretical_loss": 4.603181135063394, + "tokens_seen": 130351104 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048509528585757275, + "loss": 3.4939, + "theoretical_loss": 4.6028723358969845, + "tokens_seen": 130416640 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004850852557673019, + "loss": 3.6256, + "theoretical_loss": 4.602563735291312, + "tokens_seen": 130482176 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004850752256770311, + "loss": 3.3061, + "theoretical_loss": 4.602255333019068, + "tokens_seen": 130547712 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048506519558676024, + "loss": 3.4777, + "theoretical_loss": 4.6019471288533165, + "tokens_seen": 130613248 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004850551654964895, + "loss": 3.5478, + "theoretical_loss": 4.601639122567497, + "tokens_seen": 130678784 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048504513540621866, + "loss": 3.2939, + "theoretical_loss": 4.601331313935418, + "tokens_seen": 130744320 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048503510531594784, + "loss": 3.3667, + "theoretical_loss": 4.601023702731264, + "tokens_seen": 130809856 + }, + { + "epoch": 0.04, + "learning_rate": 0.000485025075225677, + "loss": 3.7578, + "theoretical_loss": 4.600716288729587, + "tokens_seen": 130875392 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048501504513540625, + "loss": 3.3572, + "theoretical_loss": 4.600409071705312, + "tokens_seen": 130940928 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004850050150451354, + "loss": 3.5029, + "theoretical_loss": 4.60010205143373, + "tokens_seen": 131006464 + }, + { + "epoch": 0.04, + "objective/train/docs_used": 211647, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.9844844341278076, + "objective/train/theoretical_loss": 4.599795227690505, + "objective/train/tokens_used": 151532000, + "theoretical_loss": 4.599795227690505, + "tokens_seen": 131072000 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004849949849548646, + "loss": 3.4689, + "theoretical_loss": 4.599795227690505, + "tokens_seen": 131072000 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004849849548645938, + "loss": 3.5353, + "theoretical_loss": 4.5994886002516635, + "tokens_seen": 131137536 + }, + { + "epoch": 0.04, + "learning_rate": 0.000484974924774323, + "loss": 3.7795, + "theoretical_loss": 4.599182168893604, + "tokens_seen": 131203072 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004849648946840522, + "loss": 3.42, + "theoretical_loss": 4.598875933393089, + "tokens_seen": 131268608 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048495486459378134, + "loss": 3.4034, + "theoretical_loss": 4.5985698935272445, + "tokens_seen": 131334144 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004849448345035106, + "loss": 3.5382, + "theoretical_loss": 4.598264049073565, + "tokens_seen": 131399680 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004849348044132397, + "loss": 3.336, + "theoretical_loss": 4.597958399809908, + "tokens_seen": 131465216 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048492477432296894, + "loss": 3.3876, + "theoretical_loss": 4.59765294551449, + "tokens_seen": 131530752 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004849147442326981, + "loss": 3.418, + "theoretical_loss": 4.597347685965897, + "tokens_seen": 131596288 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004849047141424273, + "loss": 3.4876, + "theoretical_loss": 4.597042620943069, + "tokens_seen": 131661824 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004848946840521565, + "loss": 3.7115, + "theoretical_loss": 4.596737750225311, + "tokens_seen": 131727360 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004848846539618857, + "loss": 3.6111, + "theoretical_loss": 4.596433073592289, + "tokens_seen": 131792896 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048487462387161484, + "loss": 3.4538, + "theoretical_loss": 4.596128590824026, + "tokens_seen": 131858432 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004848645937813441, + "loss": 3.2722, + "theoretical_loss": 4.595824301700904, + "tokens_seen": 131923968 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004848545636910732, + "loss": 3.2836, + "theoretical_loss": 4.595520206003663, + "tokens_seen": 131989504 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048484453360080244, + "loss": 3.2864, + "theoretical_loss": 4.595216303513399, + "tokens_seen": 132055040 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004848345035105316, + "loss": 3.3418, + "theoretical_loss": 4.594912594011566, + "tokens_seen": 132120576 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004848244734202608, + "loss": 3.573, + "theoretical_loss": 4.594609077279973, + "tokens_seen": 132186112 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048481444332999, + "loss": 3.4152, + "theoretical_loss": 4.594305753100782, + "tokens_seen": 132251648 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048480441323971917, + "loss": 3.2979, + "theoretical_loss": 4.594002621256511, + "tokens_seen": 132317184 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048479438314944835, + "loss": 3.5056, + "theoretical_loss": 4.59369968153003, + "tokens_seen": 132382720 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004847843530591776, + "loss": 3.6168, + "theoretical_loss": 4.593396933704562, + "tokens_seen": 132448256 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004847743229689067, + "loss": 3.3142, + "theoretical_loss": 4.593094377563681, + "tokens_seen": 132513792 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048476429287863594, + "loss": 3.3908, + "theoretical_loss": 4.592792012891314, + "tokens_seen": 132579328 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048475426278836507, + "loss": 3.3265, + "theoretical_loss": 4.592489839471735, + "tokens_seen": 132644864 + }, + { + "epoch": 0.04, + "objective/train/docs_used": 212401, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.6346824169158936, + "objective/train/theoretical_loss": 4.592187857089571, + "objective/train/tokens_used": 153170400, + "theoretical_loss": 4.592187857089571, + "tokens_seen": 132710400 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004847442326980943, + "loss": 3.6193, + "theoretical_loss": 4.592187857089571, + "tokens_seen": 132710400 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004847342026078235, + "loss": 3.1198, + "theoretical_loss": 4.591886065529795, + "tokens_seen": 132775936 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048472417251755267, + "loss": 3.5852, + "theoretical_loss": 4.591584464577728, + "tokens_seen": 132841472 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048471414242728185, + "loss": 3.5391, + "theoretical_loss": 4.591283054019041, + "tokens_seen": 132907008 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004847041123370111, + "loss": 3.7466, + "theoretical_loss": 4.5909818336397485, + "tokens_seen": 132972544 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004846940822467402, + "loss": 3.5896, + "theoretical_loss": 4.590680803226213, + "tokens_seen": 133038080 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048468405215646945, + "loss": 3.4774, + "theoretical_loss": 4.590379962565141, + "tokens_seen": 133103616 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004846740220661986, + "loss": 3.5418, + "theoretical_loss": 4.590079311443583, + "tokens_seen": 133169152 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004846639919759278, + "loss": 3.5195, + "theoretical_loss": 4.589778849648934, + "tokens_seen": 133234688 + }, + { + "epoch": 0.04, + "learning_rate": 0.000484653961885657, + "loss": 3.3776, + "theoretical_loss": 4.589478576968932, + "tokens_seen": 133300224 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048464393179538617, + "loss": 3.451, + "theoretical_loss": 4.589178493191655, + "tokens_seen": 133365760 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048463390170511535, + "loss": 3.4017, + "theoretical_loss": 4.588878598105527, + "tokens_seen": 133431296 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048462387161484453, + "loss": 3.1066, + "theoretical_loss": 4.588578891499308, + "tokens_seen": 133496832 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004846138415245737, + "loss": 3.5051, + "theoretical_loss": 4.588279373162101, + "tokens_seen": 133562368 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048460381143430295, + "loss": 3.3603, + "theoretical_loss": 4.587980042883347, + "tokens_seen": 133627904 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004845937813440321, + "loss": 3.1542, + "theoretical_loss": 4.587680900452824, + "tokens_seen": 133693440 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004845837512537613, + "loss": 3.3772, + "theoretical_loss": 4.587381945660653, + "tokens_seen": 133758976 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048457372116349044, + "loss": 3.303, + "theoretical_loss": 4.587083178297288, + "tokens_seen": 133824512 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004845636910732197, + "loss": 3.4483, + "theoretical_loss": 4.5867845981535185, + "tokens_seen": 133890048 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048455366098294886, + "loss": 3.4031, + "theoretical_loss": 4.586486205020474, + "tokens_seen": 133955584 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048454363089267804, + "loss": 3.4808, + "theoretical_loss": 4.586187998689616, + "tokens_seen": 134021120 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004845336008024072, + "loss": 3.4596, + "theoretical_loss": 4.585889978952741, + "tokens_seen": 134086656 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048452357071213645, + "loss": 3.2542, + "theoretical_loss": 4.58559214560198, + "tokens_seen": 134152192 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004845135406218656, + "loss": 3.2377, + "theoretical_loss": 4.585294498429796, + "tokens_seen": 134217728 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004845035105315948, + "loss": 3.4138, + "theoretical_loss": 4.584997037228986, + "tokens_seen": 134283264 + }, + { + "epoch": 0.04, + "objective/train/docs_used": 213682, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.553767681121826, + "objective/train/theoretical_loss": 4.584699761792674, + "objective/train/tokens_used": 154808800, + "theoretical_loss": 4.584699761792674, + "tokens_seen": 134348800 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048449348044132394, + "loss": 3.4115, + "theoretical_loss": 4.584699761792674, + "tokens_seen": 134348800 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004844834503510532, + "loss": 3.379, + "theoretical_loss": 4.5844026719143205, + "tokens_seen": 134414336 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048447342026078236, + "loss": 3.2655, + "theoretical_loss": 4.5841057673877135, + "tokens_seen": 134479872 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048446339017051154, + "loss": 3.6094, + "theoretical_loss": 4.5838090480069695, + "tokens_seen": 134545408 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004844533600802407, + "loss": 3.4139, + "theoretical_loss": 4.5835125135665375, + "tokens_seen": 134610944 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004844433299899699, + "loss": 3.241, + "theoretical_loss": 4.583216163861191, + "tokens_seen": 134676480 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004844332998996991, + "loss": 3.4857, + "theoretical_loss": 4.58291999868603, + "tokens_seen": 134742016 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004844232698094283, + "loss": 3.4424, + "theoretical_loss": 4.582624017836489, + "tokens_seen": 134807552 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048441323971915745, + "loss": 3.5497, + "theoretical_loss": 4.582328221108318, + "tokens_seen": 134873088 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004844032096288867, + "loss": 3.4341, + "theoretical_loss": 4.5820326082976, + "tokens_seen": 134938624 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048439317953861586, + "loss": 3.4735, + "theoretical_loss": 4.581737179200739, + "tokens_seen": 135004160 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048438314944834504, + "loss": 3.3947, + "theoretical_loss": 4.581441933614466, + "tokens_seen": 135069696 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004843731193580742, + "loss": 3.6985, + "theoretical_loss": 4.581146871335832, + "tokens_seen": 135135232 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004843630892678034, + "loss": 3.4447, + "theoretical_loss": 4.580851992162214, + "tokens_seen": 135200768 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004843530591775326, + "loss": 3.2804, + "theoretical_loss": 4.5805572958913086, + "tokens_seen": 135266304 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004843430290872618, + "loss": 3.2166, + "theoretical_loss": 4.580262782321135, + "tokens_seen": 135331840 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048433299899699095, + "loss": 3.1952, + "theoretical_loss": 4.579968451250032, + "tokens_seen": 135397376 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004843229689067202, + "loss": 3.6351, + "theoretical_loss": 4.579674302476661, + "tokens_seen": 135462912 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004843129388164493, + "loss": 3.5732, + "theoretical_loss": 4.579380335800001, + "tokens_seen": 135528448 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048430290872617855, + "loss": 3.2331, + "theoretical_loss": 4.579086551019348, + "tokens_seen": 135593984 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048429287863590773, + "loss": 3.3013, + "theoretical_loss": 4.5787929479343195, + "tokens_seen": 135659520 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004842828485456369, + "loss": 3.3744, + "theoretical_loss": 4.578499526344848, + "tokens_seen": 135725056 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004842728184553661, + "loss": 3.4363, + "theoretical_loss": 4.578206286051184, + "tokens_seen": 135790592 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048426278836509527, + "loss": 3.4997, + "theoretical_loss": 4.5779132268538945, + "tokens_seen": 135856128 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048425275827482445, + "loss": 3.3039, + "theoretical_loss": 4.577620348553859, + "tokens_seen": 135921664 + }, + { + "epoch": 0.04, + "objective/train/docs_used": 214514, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.6967029571533203, + "objective/train/theoretical_loss": 4.577327650952276, + "objective/train/tokens_used": 156447200, + "theoretical_loss": 4.577327650952276, + "tokens_seen": 135987200 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004842427281845537, + "loss": 3.7825, + "theoretical_loss": 4.577327650952276, + "tokens_seen": 135987200 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048423269809428287, + "loss": 3.2603, + "theoretical_loss": 4.5770351338506545, + "tokens_seen": 136052736 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048422266800401205, + "loss": 3.4404, + "theoretical_loss": 4.57674279705082, + "tokens_seen": 136118272 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004842126379137413, + "loss": 3.39, + "theoretical_loss": 4.57645064035491, + "tokens_seen": 136183808 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004842026078234704, + "loss": 3.1578, + "theoretical_loss": 4.576158663565371, + "tokens_seen": 136249344 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048419257773319965, + "loss": 3.1521, + "theoretical_loss": 4.575866866484967, + "tokens_seen": 136314880 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004841825476429288, + "loss": 3.3245, + "theoretical_loss": 4.575575248916767, + "tokens_seen": 136380416 + }, + { + "epoch": 0.04, + "learning_rate": 0.000484172517552658, + "loss": 3.2286, + "theoretical_loss": 4.575283810664155, + "tokens_seen": 136445952 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004841624874623872, + "loss": 3.1527, + "theoretical_loss": 4.574992551530822, + "tokens_seen": 136511488 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048415245737211637, + "loss": 3.3615, + "theoretical_loss": 4.574701471320768, + "tokens_seen": 136577024 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048414242728184555, + "loss": 3.2323, + "theoretical_loss": 4.574410569838304, + "tokens_seen": 136642560 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048413239719157473, + "loss": 3.5178, + "theoretical_loss": 4.574119846888045, + "tokens_seen": 136708096 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004841223671013039, + "loss": 3.4339, + "theoretical_loss": 4.573829302274915, + "tokens_seen": 136773632 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048411233701103315, + "loss": 3.3785, + "theoretical_loss": 4.573538935804146, + "tokens_seen": 136839168 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004841023069207623, + "loss": 3.4305, + "theoretical_loss": 4.573248747281273, + "tokens_seen": 136904704 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004840922768304915, + "loss": 3.7156, + "theoretical_loss": 4.5729587365121365, + "tokens_seen": 136970240 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048408224674022064, + "loss": 3.1826, + "theoretical_loss": 4.572668903302886, + "tokens_seen": 137035776 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004840722166499499, + "loss": 3.291, + "theoretical_loss": 4.572379247459969, + "tokens_seen": 137101312 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048406218655967906, + "loss": 3.4151, + "theoretical_loss": 4.57208976879014, + "tokens_seen": 137166848 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048405215646940824, + "loss": 3.4274, + "theoretical_loss": 4.571800467100456, + "tokens_seen": 137232384 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004840421263791374, + "loss": 3.5986, + "theoretical_loss": 4.5715113421982725, + "tokens_seen": 137297920 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048403209628886665, + "loss": 3.6963, + "theoretical_loss": 4.571222393891253, + "tokens_seen": 137363456 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004840220661985958, + "loss": 3.3796, + "theoretical_loss": 4.570933621987356, + "tokens_seen": 137428992 + }, + { + "epoch": 0.04, + "learning_rate": 0.000484012036108325, + "loss": 3.1538, + "theoretical_loss": 4.570645026294844, + "tokens_seen": 137494528 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048400200601805414, + "loss": 3.3521, + "theoretical_loss": 4.570356606622278, + "tokens_seen": 137560064 + }, + { + "epoch": 0.04, + "objective/train/docs_used": 215745, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.805065631866455, + "objective/train/theoretical_loss": 4.570068362778516, + "objective/train/tokens_used": 158085600, + "theoretical_loss": 4.570068362778516, + "tokens_seen": 137625600 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004839919759277834, + "loss": 3.7218, + "theoretical_loss": 4.570068362778516, + "tokens_seen": 137625600 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048398194583751256, + "loss": 3.2555, + "theoretical_loss": 4.569780294572718, + "tokens_seen": 137691136 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048397191574724174, + "loss": 3.5768, + "theoretical_loss": 4.569492401814339, + "tokens_seen": 137756672 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004839618856569709, + "loss": 3.6795, + "theoretical_loss": 4.569204684313133, + "tokens_seen": 137822208 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004839518555667001, + "loss": 3.3141, + "theoretical_loss": 4.568917141879149, + "tokens_seen": 137887744 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004839418254764293, + "loss": 3.3816, + "theoretical_loss": 4.568629774322736, + "tokens_seen": 137953280 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004839317953861585, + "loss": 3.3705, + "theoretical_loss": 4.568342581454532, + "tokens_seen": 138018816 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048392176529588765, + "loss": 3.5989, + "theoretical_loss": 4.568055563085476, + "tokens_seen": 138084352 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004839117352056169, + "loss": 3.3891, + "theoretical_loss": 4.567768719026797, + "tokens_seen": 138149888 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048390170511534606, + "loss": 3.2672, + "theoretical_loss": 4.567482049090019, + "tokens_seen": 138215424 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048389167502507524, + "loss": 3.3265, + "theoretical_loss": 4.567195553086961, + "tokens_seen": 138280960 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004838816449348044, + "loss": 3.2513, + "theoretical_loss": 4.566909230829729, + "tokens_seen": 138346496 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004838716148445336, + "loss": 3.4663, + "theoretical_loss": 4.566623082130729, + "tokens_seen": 138412032 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004838615847542628, + "loss": 3.5158, + "theoretical_loss": 4.566337106802651, + "tokens_seen": 138477568 + }, + { + "epoch": 0.04, + "learning_rate": 0.000483851554663992, + "loss": 3.558, + "theoretical_loss": 4.56605130465848, + "tokens_seen": 138543104 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048384152457372115, + "loss": 3.5835, + "theoretical_loss": 4.565765675511487, + "tokens_seen": 138608640 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004838314944834504, + "loss": 3.4223, + "theoretical_loss": 4.565480219175237, + "tokens_seen": 138674176 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004838214643931795, + "loss": 3.6202, + "theoretical_loss": 4.56519493546358, + "tokens_seen": 138739712 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048381143430290875, + "loss": 3.3628, + "theoretical_loss": 4.56490982419066, + "tokens_seen": 138805248 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048380140421263793, + "loss": 3.5243, + "theoretical_loss": 4.564624885170902, + "tokens_seen": 138870784 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004837913741223671, + "loss": 3.2818, + "theoretical_loss": 4.564340118219022, + "tokens_seen": 138936320 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004837813440320963, + "loss": 3.3265, + "theoretical_loss": 4.56405552315002, + "tokens_seen": 139001856 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048377131394182547, + "loss": 3.5035, + "theoretical_loss": 4.563771099779187, + "tokens_seen": 139067392 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048376128385155465, + "loss": 3.5528, + "theoretical_loss": 4.563486847922093, + "tokens_seen": 139132928 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004837512537612839, + "loss": 3.5953, + "theoretical_loss": 4.563202767394597, + "tokens_seen": 139198464 + }, + { + "epoch": 0.04, + "objective/train/docs_used": 216543, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.1856539249420166, + "objective/train/theoretical_loss": 4.562918858012843, + "objective/train/tokens_used": 159724000, + "theoretical_loss": 4.562918858012843, + "tokens_seen": 139264000 + }, + { + "epoch": 0.04, + "learning_rate": 0.000483741223671013, + "loss": 3.1963, + "theoretical_loss": 4.562918858012843, + "tokens_seen": 139264000 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048373119358074225, + "loss": 3.7505, + "theoretical_loss": 4.562635119593255, + "tokens_seen": 139329536 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048372116349047143, + "loss": 3.3943, + "theoretical_loss": 4.562351551952542, + "tokens_seen": 139395072 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004837111334002006, + "loss": 3.427, + "theoretical_loss": 4.5620681549076965, + "tokens_seen": 139460608 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004837011033099298, + "loss": 3.3626, + "theoretical_loss": 4.561784928275992, + "tokens_seen": 139526144 + }, + { + "epoch": 0.04, + "learning_rate": 0.000483691073219659, + "loss": 3.2654, + "theoretical_loss": 4.561501871874984, + "tokens_seen": 139591680 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048368104312938816, + "loss": 3.3665, + "theoretical_loss": 4.561218985522507, + "tokens_seen": 139657216 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004836710130391174, + "loss": 3.4153, + "theoretical_loss": 4.560936269036679, + "tokens_seen": 139722752 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004836609829488465, + "loss": 3.5138, + "theoretical_loss": 4.560653722235895, + "tokens_seen": 139788288 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048365095285857575, + "loss": 3.5586, + "theoretical_loss": 4.560371344938831, + "tokens_seen": 139853824 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004836409227683049, + "loss": 3.5356, + "theoretical_loss": 4.560089136964439, + "tokens_seen": 139919360 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004836308926780341, + "loss": 3.6322, + "theoretical_loss": 4.559807098131953, + "tokens_seen": 139984896 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004836208625877633, + "loss": 3.3173, + "theoretical_loss": 4.559525228260882, + "tokens_seen": 140050432 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004836108324974925, + "loss": 3.5052, + "theoretical_loss": 4.559243527171011, + "tokens_seen": 140115968 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048360080240722166, + "loss": 3.6296, + "theoretical_loss": 4.558961994682403, + "tokens_seen": 140181504 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048359077231695084, + "loss": 3.6015, + "theoretical_loss": 4.558680630615397, + "tokens_seen": 140247040 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048358074222668, + "loss": 3.2321, + "theoretical_loss": 4.558399434790607, + "tokens_seen": 140312576 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048357071213640926, + "loss": 3.5938, + "theoretical_loss": 4.558118407028921, + "tokens_seen": 140378112 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004835606820461384, + "loss": 3.2905, + "theoretical_loss": 4.557837547151502, + "tokens_seen": 140443648 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004835506519558676, + "loss": 3.4072, + "theoretical_loss": 4.557556854979786, + "tokens_seen": 140509184 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004835406218655968, + "loss": 3.4305, + "theoretical_loss": 4.5572763303354815, + "tokens_seen": 140574720 + }, + { + "epoch": 0.04, + "learning_rate": 0.000483530591775326, + "loss": 3.4042, + "theoretical_loss": 4.556995973040574, + "tokens_seen": 140640256 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048352056168505516, + "loss": 3.6395, + "theoretical_loss": 4.556715782917314, + "tokens_seen": 140705792 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048351053159478434, + "loss": 3.4382, + "theoretical_loss": 4.556435759788229, + "tokens_seen": 140771328 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004835005015045135, + "loss": 3.4248, + "theoretical_loss": 4.556155903476114, + "tokens_seen": 140836864 + }, + { + "epoch": 0.04, + "objective/train/docs_used": 217936, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.1173794269561768, + "objective/train/theoretical_loss": 4.555876213804037, + "objective/train/tokens_used": 161362400, + "theoretical_loss": 4.555876213804037, + "tokens_seen": 140902400 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048349047141424276, + "loss": 3.3518, + "theoretical_loss": 4.555876213804037, + "tokens_seen": 140902400 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048348044132397194, + "loss": 3.3859, + "theoretical_loss": 4.555596690595333, + "tokens_seen": 140967936 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004834704112337011, + "loss": 3.2391, + "theoretical_loss": 4.555317333673611, + "tokens_seen": 141033472 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004834603811434303, + "loss": 3.2149, + "theoretical_loss": 4.555038142862742, + "tokens_seen": 141099008 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004834503510531595, + "loss": 3.5474, + "theoretical_loss": 4.5547591179868725, + "tokens_seen": 141164544 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004834403209628887, + "loss": 3.4493, + "theoretical_loss": 4.554480258870409, + "tokens_seen": 141230080 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048343029087261785, + "loss": 3.3747, + "theoretical_loss": 4.554201565338033, + "tokens_seen": 141295616 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004834202607823471, + "loss": 3.6331, + "theoretical_loss": 4.5539230372146875, + "tokens_seen": 141361152 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048341023069207626, + "loss": 3.3845, + "theoretical_loss": 4.553644674325584, + "tokens_seen": 141426688 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048340020060180544, + "loss": 3.5964, + "theoretical_loss": 4.553366476496198, + "tokens_seen": 141492224 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004833901705115346, + "loss": 3.3062, + "theoretical_loss": 4.553088443552269, + "tokens_seen": 141557760 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004833801404212638, + "loss": 3.4501, + "theoretical_loss": 4.552810575319806, + "tokens_seen": 141623296 + }, + { + "epoch": 0.04, + "learning_rate": 0.000483370110330993, + "loss": 3.2792, + "theoretical_loss": 4.552532871625077, + "tokens_seen": 141688832 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004833600802407222, + "loss": 3.1857, + "theoretical_loss": 4.5522553322946155, + "tokens_seen": 141754368 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048335005015045135, + "loss": 3.2261, + "theoretical_loss": 4.551977957155217, + "tokens_seen": 141819904 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004833400200601806, + "loss": 3.6914, + "theoretical_loss": 4.5517007460339425, + "tokens_seen": 141885440 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004833299899699097, + "loss": 3.3864, + "theoretical_loss": 4.551423698758111, + "tokens_seen": 141950976 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048331995987963895, + "loss": 3.309, + "theoretical_loss": 4.551146815155304, + "tokens_seen": 142016512 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048330992978936813, + "loss": 3.6274, + "theoretical_loss": 4.550870095053366, + "tokens_seen": 142082048 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004832998996990973, + "loss": 3.3515, + "theoretical_loss": 4.550593538280398, + "tokens_seen": 142147584 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004832898696088265, + "loss": 3.2589, + "theoretical_loss": 4.550317144664766, + "tokens_seen": 142213120 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048327983951855567, + "loss": 3.4483, + "theoretical_loss": 4.55004091403509, + "tokens_seen": 142278656 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048326980942828485, + "loss": 3.3192, + "theoretical_loss": 4.5497648462202545, + "tokens_seen": 142344192 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004832597793380141, + "loss": 3.42, + "theoretical_loss": 4.549488941049397, + "tokens_seen": 142409728 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004832497492477432, + "loss": 3.4013, + "theoretical_loss": 4.549213198351914, + "tokens_seen": 142475264 + }, + { + "epoch": 0.04, + "objective/train/docs_used": 218572, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.226104259490967, + "objective/train/theoretical_loss": 4.548937617957463, + "objective/train/tokens_used": 163000800, + "theoretical_loss": 4.548937617957463, + "tokens_seen": 142540800 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048323971915747245, + "loss": 3.3476, + "theoretical_loss": 4.548937617957463, + "tokens_seen": 142540800 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048322968906720163, + "loss": 3.1424, + "theoretical_loss": 4.548662199695954, + "tokens_seen": 142606336 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004832196589769308, + "loss": 3.4049, + "theoretical_loss": 4.548386943397556, + "tokens_seen": 142671872 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048320962888666, + "loss": 3.3425, + "theoretical_loss": 4.548111848892693, + "tokens_seen": 142737408 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004831995987963892, + "loss": 3.6242, + "theoretical_loss": 4.547836916012042, + "tokens_seen": 142802944 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048318956870611836, + "loss": 3.1838, + "theoretical_loss": 4.547562144586539, + "tokens_seen": 142868480 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004831795386158476, + "loss": 3.4277, + "theoretical_loss": 4.547287534447372, + "tokens_seen": 142934016 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004831695085255767, + "loss": 3.2606, + "theoretical_loss": 4.5470130854259825, + "tokens_seen": 142999552 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048315947843530595, + "loss": 3.3748, + "theoretical_loss": 4.546738797354065, + "tokens_seen": 143065088 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004831494483450351, + "loss": 3.4503, + "theoretical_loss": 4.546464670063569, + "tokens_seen": 143130624 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004831394182547643, + "loss": 3.3705, + "theoretical_loss": 4.546190703386695, + "tokens_seen": 143196160 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004831293881644935, + "loss": 3.4002, + "theoretical_loss": 4.545916897155894, + "tokens_seen": 143261696 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004831193580742227, + "loss": 3.6006, + "theoretical_loss": 4.54564325120387, + "tokens_seen": 143327232 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048310932798395186, + "loss": 3.614, + "theoretical_loss": 4.545369765363578, + "tokens_seen": 143392768 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048309929789368104, + "loss": 3.3347, + "theoretical_loss": 4.545096439468223, + "tokens_seen": 143458304 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004830892678034102, + "loss": 3.4975, + "theoretical_loss": 4.544823273351257, + "tokens_seen": 143523840 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048307923771313946, + "loss": 3.5284, + "theoretical_loss": 4.544550266846388, + "tokens_seen": 143589376 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004830692076228686, + "loss": 3.5193, + "theoretical_loss": 4.544277419787566, + "tokens_seen": 143654912 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004830591775325978, + "loss": 3.3756, + "theoretical_loss": 4.544004732008993, + "tokens_seen": 143720448 + }, + { + "epoch": 0.04, + "learning_rate": 0.000483049147442327, + "loss": 3.0692, + "theoretical_loss": 4.543732203345119, + "tokens_seen": 143785984 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004830391173520562, + "loss": 3.3665, + "theoretical_loss": 4.543459833630639, + "tokens_seen": 143851520 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048302908726178536, + "loss": 3.429, + "theoretical_loss": 4.543187622700497, + "tokens_seen": 143917056 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048301905717151454, + "loss": 3.3939, + "theoretical_loss": 4.542915570389884, + "tokens_seen": 143982592 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004830090270812437, + "loss": 3.4513, + "theoretical_loss": 4.542643676534234, + "tokens_seen": 144048128 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048299899699097296, + "loss": 3.3302, + "theoretical_loss": 4.542371940969231, + "tokens_seen": 144113664 + }, + { + "epoch": 0.04, + "objective/train/docs_used": 219918, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.464306116104126, + "objective/train/theoretical_loss": 4.542100363530799, + "objective/train/tokens_used": 164639200, + "theoretical_loss": 4.542100363530799, + "tokens_seen": 144179200 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004829889669007021, + "loss": 3.5182, + "theoretical_loss": 4.542100363530799, + "tokens_seen": 144179200 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004829789368104313, + "loss": 3.3918, + "theoretical_loss": 4.54182894405511, + "tokens_seen": 144244736 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048296890672016045, + "loss": 3.2309, + "theoretical_loss": 4.5415576823785795, + "tokens_seen": 144310272 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004829588766298897, + "loss": 3.3555, + "theoretical_loss": 4.541286578337866, + "tokens_seen": 144375808 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048294884653961887, + "loss": 3.485, + "theoretical_loss": 4.541015631769872, + "tokens_seen": 144441344 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048293881644934805, + "loss": 3.382, + "theoretical_loss": 4.5407448425117405, + "tokens_seen": 144506880 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048292878635907723, + "loss": 3.5303, + "theoretical_loss": 4.540474210400859, + "tokens_seen": 144572416 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048291875626880646, + "loss": 3.3492, + "theoretical_loss": 4.540203735274855, + "tokens_seen": 144637952 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004829087261785356, + "loss": 3.2747, + "theoretical_loss": 4.5399334169716, + "tokens_seen": 144703488 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004828986960882648, + "loss": 3.4229, + "theoretical_loss": 4.539663255329202, + "tokens_seen": 144769024 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048288866599799395, + "loss": 3.3941, + "theoretical_loss": 4.539393250186015, + "tokens_seen": 144834560 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004828786359077232, + "loss": 3.5768, + "theoretical_loss": 4.539123401380625, + "tokens_seen": 144900096 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048286860581745237, + "loss": 3.376, + "theoretical_loss": 4.538853708751866, + "tokens_seen": 144965632 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048285857572718155, + "loss": 3.1908, + "theoretical_loss": 4.538584172138804, + "tokens_seen": 145031168 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048284854563691073, + "loss": 3.4312, + "theoretical_loss": 4.538314791380748, + "tokens_seen": 145096704 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004828385155466399, + "loss": 3.2382, + "theoretical_loss": 4.538045566317242, + "tokens_seen": 145162240 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004828284854563691, + "loss": 3.2605, + "theoretical_loss": 4.537776496788071, + "tokens_seen": 145227776 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048281845536609833, + "loss": 3.4595, + "theoretical_loss": 4.537507582633253, + "tokens_seen": 145293312 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048280842527582746, + "loss": 3.4714, + "theoretical_loss": 4.537238823693045, + "tokens_seen": 145358848 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004827983951855567, + "loss": 3.4331, + "theoretical_loss": 4.536970219807939, + "tokens_seen": 145424384 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004827883650952858, + "loss": 3.3043, + "theoretical_loss": 4.536701770818665, + "tokens_seen": 145489920 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048277833500501505, + "loss": 3.5044, + "theoretical_loss": 4.536433476566185, + "tokens_seen": 145555456 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048276830491474423, + "loss": 3.4038, + "theoretical_loss": 4.536165336891699, + "tokens_seen": 145620992 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004827582748244734, + "loss": 3.2575, + "theoretical_loss": 4.535897351636638, + "tokens_seen": 145686528 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004827482447342026, + "loss": 3.6485, + "theoretical_loss": 4.53562952064267, + "tokens_seen": 145752064 + }, + { + "epoch": 0.04, + "objective/train/docs_used": 220709, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.8053040504455566, + "objective/train/theoretical_loss": 4.535361843751696, + "objective/train/tokens_used": 166277600, + "theoretical_loss": 4.535361843751696, + "tokens_seen": 145817600 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048273821464393183, + "loss": 3.4441, + "theoretical_loss": 4.535361843751696, + "tokens_seen": 145817600 + }, + { + "epoch": 0.04, + "learning_rate": 0.000482728184553661, + "loss": 3.1346, + "theoretical_loss": 4.535094320805847, + "tokens_seen": 145883136 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004827181544633902, + "loss": 3.3252, + "theoretical_loss": 4.534826951647489, + "tokens_seen": 145948672 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004827081243731194, + "loss": 3.4838, + "theoretical_loss": 4.5345597361192205, + "tokens_seen": 146014208 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048269809428284856, + "loss": 3.6738, + "theoretical_loss": 4.53429267406387, + "tokens_seen": 146079744 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004826880641925778, + "loss": 3.4136, + "theoretical_loss": 4.5340257653244995, + "tokens_seen": 146145280 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004826780341023069, + "loss": 3.5642, + "theoretical_loss": 4.5337590097444, + "tokens_seen": 146210816 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048266800401203615, + "loss": 3.2791, + "theoretical_loss": 4.533492407167093, + "tokens_seen": 146276352 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004826579739217653, + "loss": 3.4304, + "theoretical_loss": 4.53322595743633, + "tokens_seen": 146341888 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004826479438314945, + "loss": 3.4174, + "theoretical_loss": 4.5329596603960916, + "tokens_seen": 146407424 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004826379137412237, + "loss": 3.3817, + "theoretical_loss": 4.53269351589059, + "tokens_seen": 146472960 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004826278836509529, + "loss": 3.2871, + "theoretical_loss": 4.532427523764261, + "tokens_seen": 146538496 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048261785356068206, + "loss": 3.303, + "theoretical_loss": 4.532161683861773, + "tokens_seen": 146604032 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048260782347041124, + "loss": 3.5583, + "theoretical_loss": 4.5318959960280205, + "tokens_seen": 146669568 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004825977933801404, + "loss": 3.0956, + "theoretical_loss": 4.531630460108125, + "tokens_seen": 146735104 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048258776328986966, + "loss": 3.6116, + "theoretical_loss": 4.531365075947434, + "tokens_seen": 146800640 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004825777331995988, + "loss": 3.4407, + "theoretical_loss": 4.531099843391524, + "tokens_seen": 146866176 + }, + { + "epoch": 0.04, + "learning_rate": 0.000482567703109328, + "loss": 3.5475, + "theoretical_loss": 4.5308347622861955, + "tokens_seen": 146931712 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004825576730190572, + "loss": 3.3681, + "theoretical_loss": 4.5305698324774735, + "tokens_seen": 146997248 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004825476429287864, + "loss": 3.3835, + "theoretical_loss": 4.530305053811611, + "tokens_seen": 147062784 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048253761283851556, + "loss": 3.2951, + "theoretical_loss": 4.530040426135084, + "tokens_seen": 147128320 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048252758274824474, + "loss": 3.4417, + "theoretical_loss": 4.529775949294593, + "tokens_seen": 147193856 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004825175526579739, + "loss": 3.1626, + "theoretical_loss": 4.529511623137061, + "tokens_seen": 147259392 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048250752256770316, + "loss": 3.2913, + "theoretical_loss": 4.529247447509637, + "tokens_seen": 147324928 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004824974924774323, + "loss": 3.4218, + "theoretical_loss": 4.528983422259691, + "tokens_seen": 147390464 + }, + { + "epoch": 0.04, + "objective/train/docs_used": 222333, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.542501926422119, + "objective/train/theoretical_loss": 4.528719547234816, + "objective/train/tokens_used": 167916000, + "theoretical_loss": 4.528719547234816, + "tokens_seen": 147456000 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004824874623871615, + "loss": 3.6895, + "theoretical_loss": 4.528719547234816, + "tokens_seen": 147456000 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048247743229689065, + "loss": 3.1272, + "theoretical_loss": 4.528455822282828, + "tokens_seen": 147521536 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004824674022066199, + "loss": 3.3643, + "theoretical_loss": 4.528192247251763, + "tokens_seen": 147587072 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048245737211634907, + "loss": 3.3044, + "theoretical_loss": 4.52792882198988, + "tokens_seen": 147652608 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048244734202607825, + "loss": 3.2101, + "theoretical_loss": 4.527665546345656, + "tokens_seen": 147718144 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048243731193580743, + "loss": 3.4948, + "theoretical_loss": 4.5274024201677925, + "tokens_seen": 147783680 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048242728184553666, + "loss": 3.286, + "theoretical_loss": 4.527139443305209, + "tokens_seen": 147849216 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004824172517552658, + "loss": 3.3363, + "theoretical_loss": 4.526876615607042, + "tokens_seen": 147914752 + }, + { + "epoch": 0.04, + "learning_rate": 0.000482407221664995, + "loss": 3.5136, + "theoretical_loss": 4.526613936922654, + "tokens_seen": 147980288 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048239719157472415, + "loss": 3.4325, + "theoretical_loss": 4.526351407101618, + "tokens_seen": 148045824 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004823871614844534, + "loss": 3.3161, + "theoretical_loss": 4.526089025993732, + "tokens_seen": 148111360 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048237713139418257, + "loss": 3.341, + "theoretical_loss": 4.525826793449008, + "tokens_seen": 148176896 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048236710130391175, + "loss": 3.5225, + "theoretical_loss": 4.525564709317678, + "tokens_seen": 148242432 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048235707121364093, + "loss": 3.511, + "theoretical_loss": 4.525302773450187, + "tokens_seen": 148307968 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004823470411233701, + "loss": 3.2191, + "theoretical_loss": 4.525040985697203, + "tokens_seen": 148373504 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004823370110330993, + "loss": 3.1906, + "theoretical_loss": 4.524779345909604, + "tokens_seen": 148439040 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048232698094282853, + "loss": 3.4767, + "theoretical_loss": 4.524517853938489, + "tokens_seen": 148504576 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048231695085255766, + "loss": 3.328, + "theoretical_loss": 4.524256509635169, + "tokens_seen": 148570112 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004823069207622869, + "loss": 3.5906, + "theoretical_loss": 4.523995312851174, + "tokens_seen": 148635648 + }, + { + "epoch": 0.05, + "learning_rate": 0.000482296890672016, + "loss": 3.1019, + "theoretical_loss": 4.523734263438241, + "tokens_seen": 148701184 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048228686058174525, + "loss": 3.4911, + "theoretical_loss": 4.52347336124833, + "tokens_seen": 148766720 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048227683049147443, + "loss": 3.5419, + "theoretical_loss": 4.52321260613361, + "tokens_seen": 148832256 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004822668004012036, + "loss": 3.4556, + "theoretical_loss": 4.522951997946466, + "tokens_seen": 148897792 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004822567703109328, + "loss": 3.438, + "theoretical_loss": 4.522691536539492, + "tokens_seen": 148963328 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048224674022066203, + "loss": 3.5173, + "theoretical_loss": 4.522431221765498, + "tokens_seen": 149028864 + }, + { + "epoch": 0.05, + "objective/train/docs_used": 222973, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.872528314590454, + "objective/train/theoretical_loss": 4.522171053477507, + "objective/train/tokens_used": 169554400, + "theoretical_loss": 4.522171053477507, + "tokens_seen": 149094400 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048223671013039116, + "loss": 3.1564, + "theoretical_loss": 4.522171053477507, + "tokens_seen": 149094400 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004822266800401204, + "loss": 3.4409, + "theoretical_loss": 4.5219110315287505, + "tokens_seen": 149159936 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004822166499498495, + "loss": 3.4656, + "theoretical_loss": 4.521651155772675, + "tokens_seen": 149225472 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048220661985957876, + "loss": 3.5566, + "theoretical_loss": 4.521391426062934, + "tokens_seen": 149291008 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048219658976930794, + "loss": 3.3884, + "theoretical_loss": 4.521131842253396, + "tokens_seen": 149356544 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004821865596790371, + "loss": 3.6665, + "theoretical_loss": 4.520872404198139, + "tokens_seen": 149422080 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004821765295887663, + "loss": 3.4762, + "theoretical_loss": 4.520613111751445, + "tokens_seen": 149487616 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004821664994984955, + "loss": 3.3686, + "theoretical_loss": 4.520353964767814, + "tokens_seen": 149553152 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048215646940822466, + "loss": 3.5076, + "theoretical_loss": 4.5200949631019505, + "tokens_seen": 149618688 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004821464393179539, + "loss": 3.4631, + "theoretical_loss": 4.519836106608768, + "tokens_seen": 149684224 + }, + { + "epoch": 0.05, + "learning_rate": 0.000482136409227683, + "loss": 3.2705, + "theoretical_loss": 4.519577395143388, + "tokens_seen": 149749760 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048212637913741226, + "loss": 3.5931, + "theoretical_loss": 4.519318828561142, + "tokens_seen": 149815296 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004821163490471414, + "loss": 3.3086, + "theoretical_loss": 4.519060406717565, + "tokens_seen": 149880832 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004821063189568706, + "loss": 3.2906, + "theoretical_loss": 4.518802129468405, + "tokens_seen": 149946368 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004820962888665998, + "loss": 3.3824, + "theoretical_loss": 4.51854399666961, + "tokens_seen": 150011904 + }, + { + "epoch": 0.05, + "learning_rate": 0.000482086258776329, + "loss": 3.3789, + "theoretical_loss": 4.518286008177341, + "tokens_seen": 150077440 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048207622868605816, + "loss": 3.284, + "theoretical_loss": 4.51802816384796, + "tokens_seen": 150142976 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004820661985957874, + "loss": 3.2642, + "theoretical_loss": 4.517770463538038, + "tokens_seen": 150208512 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048205616850551653, + "loss": 3.3741, + "theoretical_loss": 4.517512907104347, + "tokens_seen": 150274048 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048204613841524576, + "loss": 3.4381, + "theoretical_loss": 4.517255494403868, + "tokens_seen": 150339584 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004820361083249749, + "loss": 3.3189, + "theoretical_loss": 4.516998225293785, + "tokens_seen": 150405120 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004820260782347041, + "loss": 3.3564, + "theoretical_loss": 4.516741099631485, + "tokens_seen": 150470656 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004820160481444333, + "loss": 3.2927, + "theoretical_loss": 4.51648411727456, + "tokens_seen": 150536192 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004820060180541625, + "loss": 3.1088, + "theoretical_loss": 4.5162272780808035, + "tokens_seen": 150601728 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048199598796389167, + "loss": 3.3767, + "theoretical_loss": 4.515970581908216, + "tokens_seen": 150667264 + }, + { + "epoch": 0.05, + "objective/train/docs_used": 224256, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.578481435775757, + "objective/train/theoretical_loss": 4.515714028614996, + "objective/train/tokens_used": 171192800, + "theoretical_loss": 4.515714028614996, + "tokens_seen": 150732800 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048198595787362085, + "loss": 3.3921, + "theoretical_loss": 4.515714028614996, + "tokens_seen": 150732800 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004819759277833501, + "loss": 3.6899, + "theoretical_loss": 4.515457618059546, + "tokens_seen": 150798336 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048196589769307927, + "loss": 3.365, + "theoretical_loss": 4.515201350100471, + "tokens_seen": 150863872 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048195586760280845, + "loss": 3.4232, + "theoretical_loss": 4.514945224596577, + "tokens_seen": 150929408 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048194583751253763, + "loss": 3.2746, + "theoretical_loss": 4.5146892414068684, + "tokens_seen": 150994944 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048193580742226686, + "loss": 3.3062, + "theoretical_loss": 4.514433400390554, + "tokens_seen": 151060480 + }, + { + "epoch": 0.05, + "learning_rate": 0.000481925777331996, + "loss": 3.5527, + "theoretical_loss": 4.514177701407042, + "tokens_seen": 151126016 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004819157472417252, + "loss": 3.4158, + "theoretical_loss": 4.51392214431594, + "tokens_seen": 151191552 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048190571715145435, + "loss": 3.3167, + "theoretical_loss": 4.513666728977054, + "tokens_seen": 151257088 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004818956870611836, + "loss": 3.261, + "theoretical_loss": 4.51341145525039, + "tokens_seen": 151322624 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048188565697091277, + "loss": 3.2799, + "theoretical_loss": 4.513156322996155, + "tokens_seen": 151388160 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048187562688064195, + "loss": 3.0428, + "theoretical_loss": 4.512901332074751, + "tokens_seen": 151453696 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048186559679037113, + "loss": 3.4207, + "theoretical_loss": 4.5126464823467805, + "tokens_seen": 151519232 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004818555667001003, + "loss": 3.4559, + "theoretical_loss": 4.512391773673042, + "tokens_seen": 151584768 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004818455366098295, + "loss": 3.4414, + "theoretical_loss": 4.5121372059145335, + "tokens_seen": 151650304 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048183550651955873, + "loss": 3.1644, + "theoretical_loss": 4.511882778932447, + "tokens_seen": 151715840 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048182547642928786, + "loss": 3.1517, + "theoretical_loss": 4.511628492588174, + "tokens_seen": 151781376 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004818154463390171, + "loss": 3.3785, + "theoretical_loss": 4.5113743467433, + "tokens_seen": 151846912 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004818054162487462, + "loss": 3.5209, + "theoretical_loss": 4.511120341259608, + "tokens_seen": 151912448 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048179538615847545, + "loss": 3.5419, + "theoretical_loss": 4.510866475999077, + "tokens_seen": 151977984 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048178535606820463, + "loss": 3.4455, + "theoretical_loss": 4.510612750823878, + "tokens_seen": 152043520 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004817753259779338, + "loss": 3.1911, + "theoretical_loss": 4.5103591655963795, + "tokens_seen": 152109056 + }, + { + "epoch": 0.05, + "learning_rate": 0.000481765295887663, + "loss": 3.3913, + "theoretical_loss": 4.510105720179144, + "tokens_seen": 152174592 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048175526579739223, + "loss": 3.1391, + "theoretical_loss": 4.5098524144349295, + "tokens_seen": 152240128 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048174523570712136, + "loss": 3.2863, + "theoretical_loss": 4.509599248226683, + "tokens_seen": 152305664 + }, + { + "epoch": 0.05, + "objective/train/docs_used": 224258, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.519541025161743, + "objective/train/theoretical_loss": 4.509346221417552, + "objective/train/tokens_used": 172831200, + "theoretical_loss": 4.509346221417552, + "tokens_seen": 152371200 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004817352056168506, + "loss": 3.1877, + "theoretical_loss": 4.509346221417552, + "tokens_seen": 152371200 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004817251755265797, + "loss": 3.5031, + "theoretical_loss": 4.509093333870869, + "tokens_seen": 152436736 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048171514543630896, + "loss": 3.4969, + "theoretical_loss": 4.508840585450166, + "tokens_seen": 152502272 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048170511534603814, + "loss": 3.4823, + "theoretical_loss": 4.508587976019164, + "tokens_seen": 152567808 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004816950852557673, + "loss": 3.6068, + "theoretical_loss": 4.508335505441774, + "tokens_seen": 152633344 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004816850551654965, + "loss": 3.2556, + "theoretical_loss": 4.508083173582105, + "tokens_seen": 152698880 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004816750250752257, + "loss": 3.4433, + "theoretical_loss": 4.507830980304451, + "tokens_seen": 152764416 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048166499498495486, + "loss": 3.4769, + "theoretical_loss": 4.5075789254733, + "tokens_seen": 152829952 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004816549648946841, + "loss": 3.3988, + "theoretical_loss": 4.507327008953329, + "tokens_seen": 152895488 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004816449348044132, + "loss": 3.4335, + "theoretical_loss": 4.507075230609407, + "tokens_seen": 152961024 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048163490471414246, + "loss": 3.6205, + "theoretical_loss": 4.506823590306591, + "tokens_seen": 153026560 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004816248746238716, + "loss": 3.2815, + "theoretical_loss": 4.506572087910127, + "tokens_seen": 153092096 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004816148445336008, + "loss": 3.424, + "theoretical_loss": 4.506320723285455, + "tokens_seen": 153157632 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048160481444333, + "loss": 3.234, + "theoretical_loss": 4.506069496298198, + "tokens_seen": 153223168 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004815947843530592, + "loss": 3.5371, + "theoretical_loss": 4.5058184068141705, + "tokens_seen": 153288704 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048158475426278837, + "loss": 3.4785, + "theoretical_loss": 4.505567454699373, + "tokens_seen": 153354240 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004815747241725176, + "loss": 3.3644, + "theoretical_loss": 4.505316639819997, + "tokens_seen": 153419776 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048156469408224673, + "loss": 3.3134, + "theoretical_loss": 4.505065962042418, + "tokens_seen": 153485312 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048155466399197596, + "loss": 3.2391, + "theoretical_loss": 4.504815421233202, + "tokens_seen": 153550848 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004815446339017051, + "loss": 3.4874, + "theoretical_loss": 4.504565017259097, + "tokens_seen": 153616384 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004815346038114343, + "loss": 3.3429, + "theoretical_loss": 4.504314749987044, + "tokens_seen": 153681920 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004815245737211635, + "loss": 3.2746, + "theoretical_loss": 4.504064619284163, + "tokens_seen": 153747456 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004815145436308927, + "loss": 3.4142, + "theoretical_loss": 4.503814625017766, + "tokens_seen": 153812992 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048150451354062187, + "loss": 3.1693, + "theoretical_loss": 4.5035647670553445, + "tokens_seen": 153878528 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048149448345035105, + "loss": 3.3487, + "theoretical_loss": 4.503315045264581, + "tokens_seen": 153944064 + }, + { + "epoch": 0.05, + "objective/train/docs_used": 224258, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.4950404167175293, + "objective/train/theoretical_loss": 4.503065459513339, + "objective/train/tokens_used": 174469600, + "theoretical_loss": 4.503065459513339, + "tokens_seen": 154009600 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048148445336008023, + "loss": 3.3584, + "theoretical_loss": 4.503065459513339, + "tokens_seen": 154009600 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048147442326980947, + "loss": 3.3349, + "theoretical_loss": 4.502816009669665, + "tokens_seen": 154075136 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004814643931795386, + "loss": 3.4341, + "theoretical_loss": 4.502566695601795, + "tokens_seen": 154140672 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048145436308926783, + "loss": 3.5226, + "theoretical_loss": 4.502317517178142, + "tokens_seen": 154206208 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048144433299899696, + "loss": 3.5991, + "theoretical_loss": 4.502068474267309, + "tokens_seen": 154271744 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004814343029087262, + "loss": 3.3909, + "theoretical_loss": 4.501819566738076, + "tokens_seen": 154337280 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048142427281845537, + "loss": 3.2495, + "theoretical_loss": 4.501570794459411, + "tokens_seen": 154402816 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048141424272818455, + "loss": 3.3363, + "theoretical_loss": 4.501322157300461, + "tokens_seen": 154468352 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048140421263791373, + "loss": 3.2894, + "theoretical_loss": 4.501073655130554, + "tokens_seen": 154533888 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048139418254764297, + "loss": 3.352, + "theoretical_loss": 4.500825287819205, + "tokens_seen": 154599424 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004813841524573721, + "loss": 3.268, + "theoretical_loss": 4.500577055236104, + "tokens_seen": 154664960 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048137412236710133, + "loss": 3.3665, + "theoretical_loss": 4.500328957251128, + "tokens_seen": 154730496 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048136409227683046, + "loss": 3.2677, + "theoretical_loss": 4.500080993734329, + "tokens_seen": 154796032 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004813540621865597, + "loss": 3.2372, + "theoretical_loss": 4.499833164555944, + "tokens_seen": 154861568 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004813440320962889, + "loss": 3.5336, + "theoretical_loss": 4.499585469586387, + "tokens_seen": 154927104 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048133400200601806, + "loss": 3.2678, + "theoretical_loss": 4.499337908696255, + "tokens_seen": 154992640 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048132397191574724, + "loss": 3.2836, + "theoretical_loss": 4.499090481756321, + "tokens_seen": 155058176 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004813139418254764, + "loss": 3.4351, + "theoretical_loss": 4.498843188637538, + "tokens_seen": 155123712 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004813039117352056, + "loss": 3.361, + "theoretical_loss": 4.498596029211041, + "tokens_seen": 155189248 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048129388164493483, + "loss": 3.3694, + "theoretical_loss": 4.498349003348137, + "tokens_seen": 155254784 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048128385155466396, + "loss": 3.4364, + "theoretical_loss": 4.4981021109203185, + "tokens_seen": 155320320 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004812738214643932, + "loss": 3.3872, + "theoretical_loss": 4.49785535179925, + "tokens_seen": 155385856 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004812637913741223, + "loss": 3.4293, + "theoretical_loss": 4.497608725856776, + "tokens_seen": 155451392 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048125376128385156, + "loss": 3.2014, + "theoretical_loss": 4.497362232964919, + "tokens_seen": 155516928 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048124373119358074, + "loss": 3.1822, + "theoretical_loss": 4.497115872995876, + "tokens_seen": 155582464 + }, + { + "epoch": 0.05, + "objective/train/docs_used": 225763, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.025784730911255, + "objective/train/theoretical_loss": 4.496869645822022, + "objective/train/tokens_used": 176108000, + "theoretical_loss": 4.496869645822022, + "tokens_seen": 155648000 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004812337011033099, + "loss": 3.4056, + "theoretical_loss": 4.496869645822022, + "tokens_seen": 155648000 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048122367101303916, + "loss": 3.2566, + "theoretical_loss": 4.496623551315908, + "tokens_seen": 155713536 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048121364092276834, + "loss": 3.3016, + "theoretical_loss": 4.496377589350261, + "tokens_seen": 155779072 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004812036108324975, + "loss": 3.3609, + "theoretical_loss": 4.496131759797984, + "tokens_seen": 155844608 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004811935807422267, + "loss": 3.1468, + "theoretical_loss": 4.495886062532153, + "tokens_seen": 155910144 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004811835506519559, + "loss": 3.6107, + "theoretical_loss": 4.495640497426023, + "tokens_seen": 155975680 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048117352056168506, + "loss": 3.3142, + "theoretical_loss": 4.495395064353019, + "tokens_seen": 156041216 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004811634904714143, + "loss": 3.3829, + "theoretical_loss": 4.4951497631867445, + "tokens_seen": 156106752 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004811534603811434, + "loss": 3.4919, + "theoretical_loss": 4.494904593800973, + "tokens_seen": 156172288 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048114343029087266, + "loss": 3.4889, + "theoretical_loss": 4.4946595560696565, + "tokens_seen": 156237824 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004811334002006018, + "loss": 3.4904, + "theoretical_loss": 4.494414649866915, + "tokens_seen": 156303360 + }, + { + "epoch": 0.05, + "learning_rate": 0.000481123370110331, + "loss": 3.24, + "theoretical_loss": 4.494169875067046, + "tokens_seen": 156368896 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004811133400200602, + "loss": 3.2623, + "theoretical_loss": 4.493925231544516, + "tokens_seen": 156434432 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004811033099297894, + "loss": 3.172, + "theoretical_loss": 4.493680719173968, + "tokens_seen": 156499968 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048109327983951857, + "loss": 3.5037, + "theoretical_loss": 4.4934363378302145, + "tokens_seen": 156565504 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004810832497492478, + "loss": 3.3995, + "theoretical_loss": 4.493192087388239, + "tokens_seen": 156631040 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048107321965897693, + "loss": 3.2766, + "theoretical_loss": 4.4929479677232, + "tokens_seen": 156696576 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048106318956870616, + "loss": 3.2331, + "theoretical_loss": 4.4927039787104235, + "tokens_seen": 156762112 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004810531594784353, + "loss": 3.4322, + "theoretical_loss": 4.4924601202254095, + "tokens_seen": 156827648 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004810431293881645, + "loss": 3.2955, + "theoretical_loss": 4.492216392143826, + "tokens_seen": 156893184 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004810330992978937, + "loss": 3.5907, + "theoretical_loss": 4.491972794341514, + "tokens_seen": 156958720 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004810230692076229, + "loss": 3.2891, + "theoretical_loss": 4.49172932669448, + "tokens_seen": 157024256 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048101303911735207, + "loss": 3.4427, + "theoretical_loss": 4.491485989078906, + "tokens_seen": 157089792 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048100300902708125, + "loss": 3.3958, + "theoretical_loss": 4.491242781371138, + "tokens_seen": 157155328 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048099297893681043, + "loss": 3.4415, + "theoretical_loss": 4.490999703447697, + "tokens_seen": 157220864 + }, + { + "epoch": 0.05, + "objective/train/docs_used": 226317, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.2634286880493164, + "objective/train/theoretical_loss": 4.4907567551852665, + "objective/train/tokens_used": 177746400, + "theoretical_loss": 4.4907567551852665, + "tokens_seen": 157286400 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048098294884653967, + "loss": 3.2088, + "theoretical_loss": 4.4907567551852665, + "tokens_seen": 157286400 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004809729187562688, + "loss": 3.4287, + "theoretical_loss": 4.490513936460702, + "tokens_seen": 157351936 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048096288866599803, + "loss": 3.0838, + "theoretical_loss": 4.490271247151027, + "tokens_seen": 157417472 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048095285857572716, + "loss": 3.349, + "theoretical_loss": 4.490028687133432, + "tokens_seen": 157483008 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004809428284854564, + "loss": 3.5244, + "theoretical_loss": 4.489786256285276, + "tokens_seen": 157548544 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048093279839518557, + "loss": 3.3918, + "theoretical_loss": 4.489543954484084, + "tokens_seen": 157614080 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048092276830491475, + "loss": 3.4131, + "theoretical_loss": 4.489301781607551, + "tokens_seen": 157679616 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048091273821464393, + "loss": 3.3327, + "theoretical_loss": 4.489059737533534, + "tokens_seen": 157745152 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048090270812437317, + "loss": 3.4728, + "theoretical_loss": 4.48881782214006, + "tokens_seen": 157810688 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004808926780341023, + "loss": 3.2697, + "theoretical_loss": 4.48857603530532, + "tokens_seen": 157876224 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048088264794383153, + "loss": 3.3249, + "theoretical_loss": 4.488334376907673, + "tokens_seen": 157941760 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048087261785356066, + "loss": 3.4597, + "theoretical_loss": 4.4880928468256425, + "tokens_seen": 158007296 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004808625877632899, + "loss": 3.5744, + "theoretical_loss": 4.487851444937916, + "tokens_seen": 158072832 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004808525576730191, + "loss": 3.3033, + "theoretical_loss": 4.487610171123347, + "tokens_seen": 158138368 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048084252758274826, + "loss": 3.3512, + "theoretical_loss": 4.487369025260954, + "tokens_seen": 158203904 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048083249749247744, + "loss": 3.0467, + "theoretical_loss": 4.48712800722992, + "tokens_seen": 158269440 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004808224674022066, + "loss": 3.4032, + "theoretical_loss": 4.48688711690959, + "tokens_seen": 158334976 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004808124373119358, + "loss": 3.3788, + "theoretical_loss": 4.486646354179475, + "tokens_seen": 158400512 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048080240722166503, + "loss": 3.4075, + "theoretical_loss": 4.48640571891925, + "tokens_seen": 158466048 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048079237713139416, + "loss": 3.4903, + "theoretical_loss": 4.48616521100875, + "tokens_seen": 158531584 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004807823470411234, + "loss": 3.3345, + "theoretical_loss": 4.485924830327974, + "tokens_seen": 158597120 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004807723169508525, + "loss": 3.3844, + "theoretical_loss": 4.485684576757087, + "tokens_seen": 158662656 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048076228686058176, + "loss": 3.4555, + "theoretical_loss": 4.485444450176413, + "tokens_seen": 158728192 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048075225677031094, + "loss": 3.5851, + "theoretical_loss": 4.485204450466437, + "tokens_seen": 158793728 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004807422266800401, + "loss": 3.4905, + "theoretical_loss": 4.484964577507808, + "tokens_seen": 158859264 + }, + { + "epoch": 0.05, + "objective/train/docs_used": 227589, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5188961029052734, + "objective/train/theoretical_loss": 4.484724831181337, + "objective/train/tokens_used": 179384800, + "theoretical_loss": 4.484724831181337, + "tokens_seen": 158924800 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004807321965897693, + "loss": 2.9862, + "theoretical_loss": 4.484724831181337, + "tokens_seen": 158924800 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048072216649949854, + "loss": 3.3628, + "theoretical_loss": 4.4844852113679945, + "tokens_seen": 158990336 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048071213640922766, + "loss": 3.5553, + "theoretical_loss": 4.484245717948913, + "tokens_seen": 159055872 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004807021063189569, + "loss": 3.0604, + "theoretical_loss": 4.484006350805385, + "tokens_seen": 159121408 + }, + { + "epoch": 0.05, + "learning_rate": 0.000480692076228686, + "loss": 3.1705, + "theoretical_loss": 4.483767109818862, + "tokens_seen": 159186944 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048068204613841526, + "loss": 3.4456, + "theoretical_loss": 4.483527994870958, + "tokens_seen": 159252480 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048067201604814444, + "loss": 2.9752, + "theoretical_loss": 4.483289005843445, + "tokens_seen": 159318016 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004806619859578736, + "loss": 3.2205, + "theoretical_loss": 4.483050142618255, + "tokens_seen": 159383552 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004806519558676028, + "loss": 3.5844, + "theoretical_loss": 4.482811405077482, + "tokens_seen": 159449088 + }, + { + "epoch": 0.05, + "learning_rate": 0.000480641925777332, + "loss": 3.5894, + "theoretical_loss": 4.482572793103373, + "tokens_seen": 159514624 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048063189568706117, + "loss": 3.3414, + "theoretical_loss": 4.482334306578339, + "tokens_seen": 159580160 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004806218655967904, + "loss": 3.3352, + "theoretical_loss": 4.482095945384946, + "tokens_seen": 159645696 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048061183550651953, + "loss": 3.4557, + "theoretical_loss": 4.481857709405919, + "tokens_seen": 159711232 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048060180541624877, + "loss": 3.2868, + "theoretical_loss": 4.4816195985241425, + "tokens_seen": 159776768 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048059177532597795, + "loss": 3.1668, + "theoretical_loss": 4.481381612622657, + "tokens_seen": 159842304 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048058174523570713, + "loss": 3.4842, + "theoretical_loss": 4.481143751584659, + "tokens_seen": 159907840 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004805717151454363, + "loss": 3.4874, + "theoretical_loss": 4.480906015293505, + "tokens_seen": 159973376 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004805616850551655, + "loss": 3.6502, + "theoretical_loss": 4.480668403632706, + "tokens_seen": 160038912 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048055165496489467, + "loss": 3.4696, + "theoretical_loss": 4.480430916485929, + "tokens_seen": 160104448 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004805416248746239, + "loss": 3.4085, + "theoretical_loss": 4.480193553736999, + "tokens_seen": 160169984 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048053159478435303, + "loss": 3.296, + "theoretical_loss": 4.479956315269897, + "tokens_seen": 160235520 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048052156469408227, + "loss": 3.3078, + "theoretical_loss": 4.479719200968757, + "tokens_seen": 160301056 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004805115346038114, + "loss": 3.4221, + "theoretical_loss": 4.479482210717871, + "tokens_seen": 160366592 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048050150451354063, + "loss": 3.3827, + "theoretical_loss": 4.479245344401685, + "tokens_seen": 160432128 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004804914744232698, + "loss": 3.3653, + "theoretical_loss": 4.479008601904798, + "tokens_seen": 160497664 + }, + { + "epoch": 0.05, + "objective/train/docs_used": 228344, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.3748011589050293, + "objective/train/theoretical_loss": 4.478771983111967, + "objective/train/tokens_used": 181023200, + "theoretical_loss": 4.478771983111967, + "tokens_seen": 160563200 + }, + { + "epoch": 0.05, + "learning_rate": 0.000480481444332999, + "loss": 3.298, + "theoretical_loss": 4.478771983111967, + "tokens_seen": 160563200 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048047141424272823, + "loss": 3.2943, + "theoretical_loss": 4.478535487908101, + "tokens_seen": 160628736 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048046138415245736, + "loss": 3.4003, + "theoretical_loss": 4.478299116178265, + "tokens_seen": 160694272 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004804513540621866, + "loss": 3.6019, + "theoretical_loss": 4.478062867807674, + "tokens_seen": 160759808 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048044132397191577, + "loss": 3.4234, + "theoretical_loss": 4.4778267426817, + "tokens_seen": 160825344 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048043129388164495, + "loss": 3.3492, + "theoretical_loss": 4.477590740685867, + "tokens_seen": 160890880 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048042126379137413, + "loss": 3.3196, + "theoretical_loss": 4.47735486170585, + "tokens_seen": 160956416 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048041123370110337, + "loss": 3.2311, + "theoretical_loss": 4.47711910562748, + "tokens_seen": 161021952 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004804012036108325, + "loss": 3.2435, + "theoretical_loss": 4.4768834723367394, + "tokens_seen": 161087488 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048039117352056173, + "loss": 3.3293, + "theoretical_loss": 4.4766479617197605, + "tokens_seen": 161153024 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048038114343029086, + "loss": 3.3614, + "theoretical_loss": 4.476412573662829, + "tokens_seen": 161218560 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004803711133400201, + "loss": 3.4204, + "theoretical_loss": 4.4761773080523835, + "tokens_seen": 161284096 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004803610832497493, + "loss": 3.266, + "theoretical_loss": 4.475942164775013, + "tokens_seen": 161349632 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048035105315947846, + "loss": 3.3162, + "theoretical_loss": 4.475707143717455, + "tokens_seen": 161415168 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048034102306920764, + "loss": 3.2002, + "theoretical_loss": 4.475472244766601, + "tokens_seen": 161480704 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004803309929789368, + "loss": 3.2076, + "theoretical_loss": 4.475237467809492, + "tokens_seen": 161546240 + }, + { + "epoch": 0.05, + "learning_rate": 0.000480320962888666, + "loss": 3.3402, + "theoretical_loss": 4.47500281273332, + "tokens_seen": 161611776 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048031093279839523, + "loss": 3.405, + "theoretical_loss": 4.474768279425424, + "tokens_seen": 161677312 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048030090270812436, + "loss": 3.3508, + "theoretical_loss": 4.474533867773299, + "tokens_seen": 161742848 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004802908726178536, + "loss": 3.2164, + "theoretical_loss": 4.474299577664581, + "tokens_seen": 161808384 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004802808425275827, + "loss": 3.2979, + "theoretical_loss": 4.474065408987063, + "tokens_seen": 161873920 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048027081243731196, + "loss": 3.16, + "theoretical_loss": 4.473831361628682, + "tokens_seen": 161939456 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048026078234704114, + "loss": 3.4809, + "theoretical_loss": 4.473597435477526, + "tokens_seen": 162004992 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004802507522567703, + "loss": 3.3, + "theoretical_loss": 4.473363630421831, + "tokens_seen": 162070528 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004802407221664995, + "loss": 3.0521, + "theoretical_loss": 4.473129946349982, + "tokens_seen": 162136064 + }, + { + "epoch": 0.05, + "objective/train/docs_used": 229659, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.4476068019866943, + "objective/train/theoretical_loss": 4.472896383150508, + "objective/train/tokens_used": 182661600, + "theoretical_loss": 4.472896383150508, + "tokens_seen": 162201600 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048023069207622874, + "loss": 3.5461, + "theoretical_loss": 4.472896383150508, + "tokens_seen": 162201600 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048022066198595786, + "loss": 3.4397, + "theoretical_loss": 4.472662940712091, + "tokens_seen": 162267136 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004802106318956871, + "loss": 3.5161, + "theoretical_loss": 4.472429618923558, + "tokens_seen": 162332672 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048020060180541623, + "loss": 3.2219, + "theoretical_loss": 4.472196417673883, + "tokens_seen": 162398208 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048019057171514546, + "loss": 3.2822, + "theoretical_loss": 4.471963336852187, + "tokens_seen": 162463744 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048018054162487464, + "loss": 3.2824, + "theoretical_loss": 4.471730376347738, + "tokens_seen": 162529280 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004801705115346038, + "loss": 3.3961, + "theoretical_loss": 4.4714975360499505, + "tokens_seen": 162594816 + }, + { + "epoch": 0.05, + "learning_rate": 0.000480160481444333, + "loss": 3.5644, + "theoretical_loss": 4.471264815848384, + "tokens_seen": 162660352 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004801504513540622, + "loss": 3.3472, + "theoretical_loss": 4.471032215632746, + "tokens_seen": 162725888 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048014042126379137, + "loss": 3.6874, + "theoretical_loss": 4.470799735292889, + "tokens_seen": 162791424 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004801303911735206, + "loss": 3.1886, + "theoretical_loss": 4.470567374718808, + "tokens_seen": 162856960 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048012036108324973, + "loss": 3.1756, + "theoretical_loss": 4.470335133800649, + "tokens_seen": 162922496 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048011033099297897, + "loss": 3.3402, + "theoretical_loss": 4.470103012428696, + "tokens_seen": 162988032 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048010030090270815, + "loss": 3.2792, + "theoretical_loss": 4.469871010493383, + "tokens_seen": 163053568 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048009027081243733, + "loss": 3.3714, + "theoretical_loss": 4.469639127885287, + "tokens_seen": 163119104 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004800802407221665, + "loss": 3.269, + "theoretical_loss": 4.4694073644951295, + "tokens_seen": 163184640 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004800702106318957, + "loss": 3.2126, + "theoretical_loss": 4.469175720213771, + "tokens_seen": 163250176 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048006018054162487, + "loss": 3.2064, + "theoretical_loss": 4.468944194932225, + "tokens_seen": 163315712 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004800501504513541, + "loss": 3.6002, + "theoretical_loss": 4.468712788541639, + "tokens_seen": 163381248 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048004012036108323, + "loss": 3.4055, + "theoretical_loss": 4.46848150093331, + "tokens_seen": 163446784 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048003009027081247, + "loss": 3.4893, + "theoretical_loss": 4.468250331998676, + "tokens_seen": 163512320 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004800200601805416, + "loss": 3.422, + "theoretical_loss": 4.468019281629316, + "tokens_seen": 163577856 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048001003009027083, + "loss": 3.19, + "theoretical_loss": 4.467788349716955, + "tokens_seen": 163643392 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048, + "loss": 3.137, + "theoretical_loss": 4.467557536153457, + "tokens_seen": 163708928 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004799899699097292, + "loss": 3.2519, + "theoretical_loss": 4.467326840830829, + "tokens_seen": 163774464 + }, + { + "debugging/Self-BLEU-5": 0.28281542061774223, + "debugging/distinct-1-grams": 0.7326597736651554, + "debugging/distinct-2-grams": 0.9067393674201281, + "debugging/entropy-1-grams": 5.525931828352874, + "debugging/entropy-2-grams": 6.085041606065698, + "debugging/length": 468.25, + "debugging/num_segments": 8, + "debugging/score": 0.00998370262687916, + "debugging/score_std": 0.008263218592709398, + "epoch": 0.05, + "objective/train/docs_used": 230304, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.3832874298095703, + "objective/train/theoretical_loss": 4.467096263641219, + "objective/train/tokens_used": 184300000, + "theoretical_loss": 4.467096263641219, + "tokens_seen": 163840000 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004799799398194584, + "loss": 3.1978, + "theoretical_loss": 4.467096263641219, + "tokens_seen": 163840000 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047996990972918756, + "loss": 3.112, + "theoretical_loss": 4.466865804476919, + "tokens_seen": 163905536 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047995987963891674, + "loss": 3.2774, + "theoretical_loss": 4.466635463230359, + "tokens_seen": 163971072 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047994984954864597, + "loss": 3.3353, + "theoretical_loss": 4.466405239794113, + "tokens_seen": 164036608 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004799398194583751, + "loss": 3.2677, + "theoretical_loss": 4.466175134060894, + "tokens_seen": 164102144 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047992978936810433, + "loss": 3.4542, + "theoretical_loss": 4.465945145923554, + "tokens_seen": 164167680 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004799197592778335, + "loss": 3.4061, + "theoretical_loss": 4.4657152752750875, + "tokens_seen": 164233216 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004799097291875627, + "loss": 3.3211, + "theoretical_loss": 4.465485522008629, + "tokens_seen": 164298752 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004798996990972919, + "loss": 3.1246, + "theoretical_loss": 4.465255886017452, + "tokens_seen": 164364288 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047988966900702106, + "loss": 3.3521, + "theoretical_loss": 4.465026367194971, + "tokens_seen": 164429824 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047987963891675024, + "loss": 3.406, + "theoretical_loss": 4.464796965434738, + "tokens_seen": 164495360 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004798696088264795, + "loss": 3.2096, + "theoretical_loss": 4.464567680630443, + "tokens_seen": 164560896 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004798595787362086, + "loss": 3.3375, + "theoretical_loss": 4.464338512675919, + "tokens_seen": 164626432 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047984954864593784, + "loss": 3.3015, + "theoretical_loss": 4.464109461465133, + "tokens_seen": 164691968 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047983951855566696, + "loss": 3.1817, + "theoretical_loss": 4.4638805268921935, + "tokens_seen": 164757504 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004798294884653962, + "loss": 3.3159, + "theoretical_loss": 4.463651708851346, + "tokens_seen": 164823040 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004798194583751254, + "loss": 3.3364, + "theoretical_loss": 4.463423007236974, + "tokens_seen": 164888576 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047980942828485456, + "loss": 3.5, + "theoretical_loss": 4.4631944219436, + "tokens_seen": 164954112 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047979939819458374, + "loss": 3.5072, + "theoretical_loss": 4.462965952865879, + "tokens_seen": 165019648 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004797893681043129, + "loss": 3.541, + "theoretical_loss": 4.46273759989861, + "tokens_seen": 165085184 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004797793380140421, + "loss": 3.034, + "theoretical_loss": 4.462509362936723, + "tokens_seen": 165150720 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047976930792377134, + "loss": 3.3458, + "theoretical_loss": 4.46228124187529, + "tokens_seen": 165216256 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047975927783350047, + "loss": 3.3679, + "theoretical_loss": 4.462053236609516, + "tokens_seen": 165281792 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004797492477432297, + "loss": 3.3828, + "theoretical_loss": 4.461825347034742, + "tokens_seen": 165347328 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004797392176529589, + "loss": 3.3819, + "theoretical_loss": 4.461597573046449, + "tokens_seen": 165412864 + }, + { + "epoch": 0.05, + "objective/train/docs_used": 231649, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.920825958251953, + "objective/train/theoretical_loss": 4.461369914540247, + "objective/train/tokens_used": 185938400, + "theoretical_loss": 4.461369914540247, + "tokens_seen": 165478400 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047972918756268807, + "loss": 3.1735, + "theoretical_loss": 4.461369914540247, + "tokens_seen": 165478400 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004797191574724173, + "loss": 3.4608, + "theoretical_loss": 4.4611423714118885, + "tokens_seen": 165543936 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047970912738214643, + "loss": 3.1453, + "theoretical_loss": 4.460914943557256, + "tokens_seen": 165609472 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047969909729187566, + "loss": 3.3385, + "theoretical_loss": 4.460687630872371, + "tokens_seen": 165675008 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047968906720160484, + "loss": 3.3347, + "theoretical_loss": 4.46046043325339, + "tokens_seen": 165740544 + }, + { + "epoch": 0.05, + "learning_rate": 0.000479679037111334, + "loss": 3.1779, + "theoretical_loss": 4.460233350596599, + "tokens_seen": 165806080 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004796690070210632, + "loss": 3.3418, + "theoretical_loss": 4.460006382798425, + "tokens_seen": 165871616 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004796589769307924, + "loss": 3.1079, + "theoretical_loss": 4.459779529755423, + "tokens_seen": 165937152 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047964894684052157, + "loss": 3.378, + "theoretical_loss": 4.459552791364288, + "tokens_seen": 166002688 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004796389167502508, + "loss": 3.5243, + "theoretical_loss": 4.459326167521844, + "tokens_seen": 166068224 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047962888665997993, + "loss": 3.4306, + "theoretical_loss": 4.4590996581250515, + "tokens_seen": 166133760 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047961885656970917, + "loss": 3.3535, + "theoretical_loss": 4.458873263071002, + "tokens_seen": 166199296 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047960882647943835, + "loss": 3.311, + "theoretical_loss": 4.458646982256921, + "tokens_seen": 166264832 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047959879638916753, + "loss": 3.0939, + "theoretical_loss": 4.458420815580169, + "tokens_seen": 166330368 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004795887662988967, + "loss": 3.1869, + "theoretical_loss": 4.458194762938234, + "tokens_seen": 166395904 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004795787362086259, + "loss": 3.3403, + "theoretical_loss": 4.457968824228743, + "tokens_seen": 166461440 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047956870611835507, + "loss": 3.3175, + "theoretical_loss": 4.457742999349449, + "tokens_seen": 166526976 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004795586760280843, + "loss": 3.4267, + "theoretical_loss": 4.4575172881982414, + "tokens_seen": 166592512 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047954864593781343, + "loss": 3.3691, + "theoretical_loss": 4.457291690673139, + "tokens_seen": 166658048 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047953861584754267, + "loss": 3.4988, + "theoretical_loss": 4.457066206672291, + "tokens_seen": 166723584 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004795285857572718, + "loss": 3.1917, + "theoretical_loss": 4.456840836093983, + "tokens_seen": 166789120 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047951855566700103, + "loss": 3.1793, + "theoretical_loss": 4.456615578836625, + "tokens_seen": 166854656 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004795085255767302, + "loss": 3.4376, + "theoretical_loss": 4.456390434798762, + "tokens_seen": 166920192 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004794984954864594, + "loss": 3.3839, + "theoretical_loss": 4.45616540387907, + "tokens_seen": 166985728 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004794884653961886, + "loss": 3.4359, + "theoretical_loss": 4.4559404859763525, + "tokens_seen": 167051264 + }, + { + "epoch": 0.05, + "objective/train/docs_used": 232239, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.8191466331481934, + "objective/train/theoretical_loss": 4.455715680989545, + "objective/train/tokens_used": 187576800, + "theoretical_loss": 4.455715680989545, + "tokens_seen": 167116800 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047947843530591776, + "loss": 3.3247, + "theoretical_loss": 4.455715680989545, + "tokens_seen": 167116800 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047946840521564694, + "loss": 3.442, + "theoretical_loss": 4.455490988817713, + "tokens_seen": 167182336 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047945837512537617, + "loss": 3.4273, + "theoretical_loss": 4.4552664093600525, + "tokens_seen": 167247872 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004794483450351053, + "loss": 3.3412, + "theoretical_loss": 4.455041942515887, + "tokens_seen": 167313408 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047943831494483453, + "loss": 3.2788, + "theoretical_loss": 4.454817588184669, + "tokens_seen": 167378944 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004794282848545637, + "loss": 3.3126, + "theoretical_loss": 4.454593346265984, + "tokens_seen": 167444480 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004794182547642929, + "loss": 3.5354, + "theoretical_loss": 4.454369216659542, + "tokens_seen": 167510016 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004794082246740221, + "loss": 3.2904, + "theoretical_loss": 4.454145199265183, + "tokens_seen": 167575552 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047939819458375126, + "loss": 3.3353, + "theoretical_loss": 4.453921293982877, + "tokens_seen": 167641088 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047938816449348044, + "loss": 3.3379, + "theoretical_loss": 4.453697500712722, + "tokens_seen": 167706624 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004793781344032097, + "loss": 3.1888, + "theoretical_loss": 4.453473819354942, + "tokens_seen": 167772160 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004793681043129388, + "loss": 3.3098, + "theoretical_loss": 4.453250249809889, + "tokens_seen": 167837696 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047935807422266804, + "loss": 3.3648, + "theoretical_loss": 4.453026791978045, + "tokens_seen": 167903232 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047934804413239716, + "loss": 3.4082, + "theoretical_loss": 4.4528034457600185, + "tokens_seen": 167968768 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004793380140421264, + "loss": 3.472, + "theoretical_loss": 4.452580211056542, + "tokens_seen": 168034304 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004793279839518556, + "loss": 3.1374, + "theoretical_loss": 4.452357087768481, + "tokens_seen": 168099840 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047931795386158476, + "loss": 3.3576, + "theoretical_loss": 4.45213407579682, + "tokens_seen": 168165376 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047930792377131394, + "loss": 3.4063, + "theoretical_loss": 4.451911175042679, + "tokens_seen": 168230912 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004792978936810431, + "loss": 3.5888, + "theoretical_loss": 4.451688385407296, + "tokens_seen": 168296448 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004792878635907723, + "loss": 3.4353, + "theoretical_loss": 4.451465706792041, + "tokens_seen": 168361984 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047927783350050154, + "loss": 3.6028, + "theoretical_loss": 4.4512431390984055, + "tokens_seen": 168427520 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047926780341023067, + "loss": 3.5768, + "theoretical_loss": 4.451020682228011, + "tokens_seen": 168493056 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004792577733199599, + "loss": 3.1725, + "theoretical_loss": 4.450798336082601, + "tokens_seen": 168558592 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004792477432296891, + "loss": 3.0739, + "theoretical_loss": 4.450576100564046, + "tokens_seen": 168624128 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047923771313941827, + "loss": 3.16, + "theoretical_loss": 4.450353975574341, + "tokens_seen": 168689664 + }, + { + "epoch": 0.05, + "objective/train/docs_used": 233635, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.160566568374634, + "objective/train/theoretical_loss": 4.450131961015606, + "objective/train/tokens_used": 189215200, + "theoretical_loss": 4.450131961015606, + "tokens_seen": 168755200 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047922768304914745, + "loss": 3.2785, + "theoretical_loss": 4.450131961015606, + "tokens_seen": 168755200 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047921765295887663, + "loss": 3.401, + "theoretical_loss": 4.449910056790086, + "tokens_seen": 168820736 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004792076228686058, + "loss": 3.122, + "theoretical_loss": 4.44968826280015, + "tokens_seen": 168886272 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047919759277833504, + "loss": 3.4615, + "theoretical_loss": 4.4494665789482895, + "tokens_seen": 168951808 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047918756268806417, + "loss": 3.2752, + "theoretical_loss": 4.449245005137125, + "tokens_seen": 169017344 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004791775325977934, + "loss": 3.2905, + "theoretical_loss": 4.449023541269395, + "tokens_seen": 169082880 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047916750250752253, + "loss": 3.3756, + "theoretical_loss": 4.448802187247966, + "tokens_seen": 169148416 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047915747241725177, + "loss": 3.5968, + "theoretical_loss": 4.448580942975825, + "tokens_seen": 169213952 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047914744232698095, + "loss": 3.2809, + "theoretical_loss": 4.448359808356084, + "tokens_seen": 169279488 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047913741223671013, + "loss": 3.5359, + "theoretical_loss": 4.448138783291979, + "tokens_seen": 169345024 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004791273821464393, + "loss": 3.4119, + "theoretical_loss": 4.447917867686863, + "tokens_seen": 169410560 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047911735205616855, + "loss": 3.4354, + "theoretical_loss": 4.44769706144422, + "tokens_seen": 169476096 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004791073219658977, + "loss": 3.4738, + "theoretical_loss": 4.44747636446765, + "tokens_seen": 169541632 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004790972918756269, + "loss": 3.6068, + "theoretical_loss": 4.447255776660878, + "tokens_seen": 169607168 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047908726178535604, + "loss": 3.2765, + "theoretical_loss": 4.44703529792775, + "tokens_seen": 169672704 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047907723169508527, + "loss": 3.2577, + "theoretical_loss": 4.446814928172234, + "tokens_seen": 169738240 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047906720160481445, + "loss": 3.3136, + "theoretical_loss": 4.446594667298421, + "tokens_seen": 169803776 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047905717151454363, + "loss": 3.4342, + "theoretical_loss": 4.446374515210521, + "tokens_seen": 169869312 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004790471414242728, + "loss": 3.305, + "theoretical_loss": 4.446154471812866, + "tokens_seen": 169934848 + }, + { + "epoch": 0.05, + "learning_rate": 0.000479037111334002, + "loss": 3.4233, + "theoretical_loss": 4.445934537009911, + "tokens_seen": 170000384 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004790270812437312, + "loss": 3.5624, + "theoretical_loss": 4.445714710706228, + "tokens_seen": 170065920 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004790170511534604, + "loss": 3.4564, + "theoretical_loss": 4.445494992806513, + "tokens_seen": 170131456 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047900702106318954, + "loss": 3.4515, + "theoretical_loss": 4.44527538321558, + "tokens_seen": 170196992 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004789969909729188, + "loss": 3.3197, + "theoretical_loss": 4.445055881838365, + "tokens_seen": 170262528 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004789869608826479, + "loss": 3.4949, + "theoretical_loss": 4.444836488579924, + "tokens_seen": 170328064 + }, + { + "epoch": 0.05, + "objective/train/docs_used": 234408, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.2099924087524414, + "objective/train/theoretical_loss": 4.44461720334543, + "objective/train/tokens_used": 190853600, + "theoretical_loss": 4.44461720334543, + "tokens_seen": 170393600 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047897693079237714, + "loss": 3.2463, + "theoretical_loss": 4.44461720334543, + "tokens_seen": 170393600 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047896690070210637, + "loss": 3.4087, + "theoretical_loss": 4.444398026040179, + "tokens_seen": 170459136 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004789568706118355, + "loss": 3.4015, + "theoretical_loss": 4.444178956569585, + "tokens_seen": 170524672 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047894684052156473, + "loss": 3.2236, + "theoretical_loss": 4.443959994839181, + "tokens_seen": 170590208 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004789368104312939, + "loss": 3.3624, + "theoretical_loss": 4.44374114075462, + "tokens_seen": 170655744 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004789267803410231, + "loss": 3.4993, + "theoretical_loss": 4.443522394221671, + "tokens_seen": 170721280 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004789167502507523, + "loss": 3.3937, + "theoretical_loss": 4.443303755146225, + "tokens_seen": 170786816 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047890672016048146, + "loss": 3.2099, + "theoretical_loss": 4.443085223434291, + "tokens_seen": 170852352 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047889669007021064, + "loss": 3.2229, + "theoretical_loss": 4.442866798991993, + "tokens_seen": 170917888 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004788866599799399, + "loss": 3.38, + "theoretical_loss": 4.442648481725577, + "tokens_seen": 170983424 + }, + { + "epoch": 0.05, + "learning_rate": 0.000478876629889669, + "loss": 3.5082, + "theoretical_loss": 4.442430271541404, + "tokens_seen": 171048960 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047886659979939824, + "loss": 3.2701, + "theoretical_loss": 4.442212168345956, + "tokens_seen": 171114496 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047885656970912736, + "loss": 3.405, + "theoretical_loss": 4.4419941720458285, + "tokens_seen": 171180032 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004788465396188566, + "loss": 3.0497, + "theoretical_loss": 4.441776282547736, + "tokens_seen": 171245568 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004788365095285858, + "loss": 3.2281, + "theoretical_loss": 4.441558499758511, + "tokens_seen": 171311104 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047882647943831496, + "loss": 3.3353, + "theoretical_loss": 4.441340823585101, + "tokens_seen": 171376640 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047881644934804414, + "loss": 3.2502, + "theoretical_loss": 4.441123253934572, + "tokens_seen": 171442176 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004788064192577733, + "loss": 3.36, + "theoretical_loss": 4.440905790714105, + "tokens_seen": 171507712 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004787963891675025, + "loss": 3.2246, + "theoretical_loss": 4.440688433830999, + "tokens_seen": 171573248 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047878635907723174, + "loss": 3.1875, + "theoretical_loss": 4.440471183192667, + "tokens_seen": 171638784 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047877632898696087, + "loss": 3.2297, + "theoretical_loss": 4.440254038706639, + "tokens_seen": 171704320 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004787662988966901, + "loss": 3.4318, + "theoretical_loss": 4.440037000280561, + "tokens_seen": 171769856 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004787562688064193, + "loss": 3.298, + "theoretical_loss": 4.439820067822195, + "tokens_seen": 171835392 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047874623871614847, + "loss": 3.5067, + "theoretical_loss": 4.439603241239416, + "tokens_seen": 171900928 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047873620862587765, + "loss": 3.1999, + "theoretical_loss": 4.439386520440218, + "tokens_seen": 171966464 + }, + { + "epoch": 0.05, + "objective/train/docs_used": 235647, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.488489866256714, + "objective/train/theoretical_loss": 4.439169905332706, + "objective/train/tokens_used": 192492000, + "theoretical_loss": 4.439169905332706, + "tokens_seen": 172032000 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047872617853560683, + "loss": 3.6249, + "theoretical_loss": 4.439169905332706, + "tokens_seen": 172032000 + }, + { + "epoch": 0.05, + "learning_rate": 0.000478716148445336, + "loss": 3.4185, + "theoretical_loss": 4.438953395825102, + "tokens_seen": 172097536 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047870611835506524, + "loss": 3.4638, + "theoretical_loss": 4.438736991825744, + "tokens_seen": 172163072 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047869608826479437, + "loss": 2.9901, + "theoretical_loss": 4.438520693243079, + "tokens_seen": 172228608 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004786860581745236, + "loss": 3.3508, + "theoretical_loss": 4.4383044999856756, + "tokens_seen": 172294144 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047867602808425273, + "loss": 3.006, + "theoretical_loss": 4.438088411962211, + "tokens_seen": 172359680 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047866599799398197, + "loss": 3.2261, + "theoretical_loss": 4.437872429081477, + "tokens_seen": 172425216 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047865596790371115, + "loss": 3.3128, + "theoretical_loss": 4.437656551252381, + "tokens_seen": 172490752 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047864593781344033, + "loss": 3.1811, + "theoretical_loss": 4.4374407783839445, + "tokens_seen": 172556288 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004786359077231695, + "loss": 3.6537, + "theoretical_loss": 4.437225110385297, + "tokens_seen": 172621824 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047862587763289875, + "loss": 3.3197, + "theoretical_loss": 4.4370095471656885, + "tokens_seen": 172687360 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004786158475426279, + "loss": 3.2659, + "theoretical_loss": 4.436794088634477, + "tokens_seen": 172752896 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004786058174523571, + "loss": 3.432, + "theoretical_loss": 4.4365787347011345, + "tokens_seen": 172818432 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047859578736208624, + "loss": 3.4144, + "theoretical_loss": 4.436363485275246, + "tokens_seen": 172883968 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047858575727181547, + "loss": 3.4017, + "theoretical_loss": 4.436148340266508, + "tokens_seen": 172949504 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047857572718154465, + "loss": 3.7135, + "theoretical_loss": 4.435933299584729, + "tokens_seen": 173015040 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047856569709127383, + "loss": 3.2931, + "theoretical_loss": 4.4357183631398325, + "tokens_seen": 173080576 + }, + { + "epoch": 0.05, + "learning_rate": 0.000478555667001003, + "loss": 3.3342, + "theoretical_loss": 4.435503530841849, + "tokens_seen": 173146112 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004785456369107322, + "loss": 3.1036, + "theoretical_loss": 4.435288802600926, + "tokens_seen": 173211648 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004785356068204614, + "loss": 3.4167, + "theoretical_loss": 4.4350741783273175, + "tokens_seen": 173277184 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004785255767301906, + "loss": 3.3526, + "theoretical_loss": 4.434859657931392, + "tokens_seen": 173342720 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047851554663991974, + "loss": 3.3756, + "theoretical_loss": 4.434645241323629, + "tokens_seen": 173408256 + }, + { + "epoch": 0.05, + "learning_rate": 0.000478505516549649, + "loss": 3.3329, + "theoretical_loss": 4.434430928414617, + "tokens_seen": 173473792 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004784954864593781, + "loss": 3.2552, + "theoretical_loss": 4.434216719115057, + "tokens_seen": 173539328 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047848545636910734, + "loss": 3.1126, + "theoretical_loss": 4.43400261333576, + "tokens_seen": 173604864 + }, + { + "epoch": 0.05, + "objective/train/docs_used": 236140, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.18463397026062, + "objective/train/theoretical_loss": 4.433788610987646, + "objective/train/tokens_used": 194130400, + "theoretical_loss": 4.433788610987646, + "tokens_seen": 173670400 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004784754262788365, + "loss": 3.3947, + "theoretical_loss": 4.433788610987646, + "tokens_seen": 173670400 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004784653961885657, + "loss": 3.4461, + "theoretical_loss": 4.433574711981749, + "tokens_seen": 173735936 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004784553660982949, + "loss": 3.2924, + "theoretical_loss": 4.433360916229209, + "tokens_seen": 173801472 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004784453360080241, + "loss": 3.3511, + "theoretical_loss": 4.433147223641278, + "tokens_seen": 173867008 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047843530591775324, + "loss": 3.1406, + "theoretical_loss": 4.432933634129318, + "tokens_seen": 173932544 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004784252758274825, + "loss": 3.3653, + "theoretical_loss": 4.4327201476047975, + "tokens_seen": 173998080 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004784152457372116, + "loss": 3.1941, + "theoretical_loss": 4.432506763979299, + "tokens_seen": 174063616 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047840521564694084, + "loss": 3.2088, + "theoretical_loss": 4.432293483164512, + "tokens_seen": 174129152 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047839518555667, + "loss": 3.4825, + "theoretical_loss": 4.432080305072233, + "tokens_seen": 174194688 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004783851554663992, + "loss": 3.1969, + "theoretical_loss": 4.43186722961437, + "tokens_seen": 174260224 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004783751253761284, + "loss": 3.3585, + "theoretical_loss": 4.431654256702938, + "tokens_seen": 174325760 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047836509528585756, + "loss": 3.2937, + "theoretical_loss": 4.431441386250063, + "tokens_seen": 174391296 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047835506519558675, + "loss": 3.3733, + "theoretical_loss": 4.4312286181679745, + "tokens_seen": 174456832 + }, + { + "epoch": 0.05, + "learning_rate": 0.000478345035105316, + "loss": 3.1708, + "theoretical_loss": 4.431015952369016, + "tokens_seen": 174522368 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004783350050150451, + "loss": 3.4721, + "theoretical_loss": 4.430803388765636, + "tokens_seen": 174587904 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047832497492477434, + "loss": 3.3855, + "theoretical_loss": 4.430590927270388, + "tokens_seen": 174653440 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047831494483450347, + "loss": 3.2205, + "theoretical_loss": 4.430378567795938, + "tokens_seen": 174718976 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004783049147442327, + "loss": 3.3129, + "theoretical_loss": 4.430166310255057, + "tokens_seen": 174784512 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004782948846539619, + "loss": 3.6459, + "theoretical_loss": 4.429954154560624, + "tokens_seen": 174850048 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047828485456369107, + "loss": 3.3131, + "theoretical_loss": 4.429742100625624, + "tokens_seen": 174915584 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047827482447342025, + "loss": 3.2811, + "theoretical_loss": 4.429530148363151, + "tokens_seen": 174981120 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004782647943831495, + "loss": 3.2996, + "theoretical_loss": 4.429318297686402, + "tokens_seen": 175046656 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004782547642928786, + "loss": 3.3326, + "theoretical_loss": 4.429106548508685, + "tokens_seen": 175112192 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047824473420260785, + "loss": 3.1878, + "theoretical_loss": 4.428894900743411, + "tokens_seen": 175177728 + }, + { + "epoch": 0.05, + "learning_rate": 0.000478234704112337, + "loss": 3.214, + "theoretical_loss": 4.428683354304098, + "tokens_seen": 175243264 + }, + { + "epoch": 0.05, + "objective/train/docs_used": 237488, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5905725955963135, + "objective/train/theoretical_loss": 4.428471909104372, + "objective/train/tokens_used": 195768800, + "theoretical_loss": 4.428471909104372, + "tokens_seen": 175308800 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004782246740220662, + "loss": 3.1411, + "theoretical_loss": 4.428471909104372, + "tokens_seen": 175308800 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047821464393179544, + "loss": 3.3937, + "theoretical_loss": 4.428260565057964, + "tokens_seen": 175374336 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047820461384152457, + "loss": 3.3474, + "theoretical_loss": 4.428049322078708, + "tokens_seen": 175439872 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004781945837512538, + "loss": 3.3299, + "theoretical_loss": 4.427838180080547, + "tokens_seen": 175505408 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047818455366098293, + "loss": 3.3159, + "theoretical_loss": 4.4276271389775275, + "tokens_seen": 175570944 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047817452357071217, + "loss": 3.3342, + "theoretical_loss": 4.427416198683803, + "tokens_seen": 175636480 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047816449348044135, + "loss": 3.2288, + "theoretical_loss": 4.427205359113629, + "tokens_seen": 175702016 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047815446339017053, + "loss": 3.5414, + "theoretical_loss": 4.42699462018137, + "tokens_seen": 175767552 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004781444332998997, + "loss": 3.389, + "theoretical_loss": 4.42678398180149, + "tokens_seen": 175833088 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047813440320962895, + "loss": 3.2056, + "theoretical_loss": 4.426573443888563, + "tokens_seen": 175898624 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004781243731193581, + "loss": 3.2972, + "theoretical_loss": 4.426363006357263, + "tokens_seen": 175964160 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004781143430290873, + "loss": 3.4419, + "theoretical_loss": 4.426152669122374, + "tokens_seen": 176029696 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047810431293881644, + "loss": 3.285, + "theoretical_loss": 4.425942432098774, + "tokens_seen": 176095232 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047809428284854567, + "loss": 3.3969, + "theoretical_loss": 4.425732295201455, + "tokens_seen": 176160768 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047808425275827485, + "loss": 3.2307, + "theoretical_loss": 4.425522258345508, + "tokens_seen": 176226304 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047807422266800403, + "loss": 3.4295, + "theoretical_loss": 4.425312321446127, + "tokens_seen": 176291840 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004780641925777332, + "loss": 3.1458, + "theoretical_loss": 4.425102484418613, + "tokens_seen": 176357376 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004780541624874624, + "loss": 3.3972, + "theoretical_loss": 4.424892747178365, + "tokens_seen": 176422912 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004780441323971916, + "loss": 3.527, + "theoretical_loss": 4.42468310964089, + "tokens_seen": 176488448 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004780341023069208, + "loss": 3.4912, + "theoretical_loss": 4.424473571721794, + "tokens_seen": 176553984 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047802407221664994, + "loss": 3.2588, + "theoretical_loss": 4.42426413333679, + "tokens_seen": 176619520 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004780140421263792, + "loss": 3.398, + "theoretical_loss": 4.424054794401689, + "tokens_seen": 176685056 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004780040120361083, + "loss": 3.3066, + "theoretical_loss": 4.423845554832406, + "tokens_seen": 176750592 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047799398194583754, + "loss": 3.4778, + "theoretical_loss": 4.42363641454496, + "tokens_seen": 176816128 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004779839518555667, + "loss": 3.4736, + "theoretical_loss": 4.423427373455471, + "tokens_seen": 176881664 + }, + { + "epoch": 0.05, + "objective/train/docs_used": 238146, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.402047634124756, + "objective/train/theoretical_loss": 4.42321843148016, + "objective/train/tokens_used": 197407200, + "theoretical_loss": 4.42321843148016, + "tokens_seen": 176947200 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004779739217652959, + "loss": 3.4011, + "theoretical_loss": 4.42321843148016, + "tokens_seen": 176947200 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004779638916750251, + "loss": 3.2247, + "theoretical_loss": 4.423009588535351, + "tokens_seen": 177012736 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004779538615847543, + "loss": 3.5019, + "theoretical_loss": 4.422800844537466, + "tokens_seen": 177078272 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047794383149448344, + "loss": 3.3271, + "theoretical_loss": 4.422592199403036, + "tokens_seen": 177143808 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004779338014042127, + "loss": 3.1612, + "theoretical_loss": 4.422383653048685, + "tokens_seen": 177209344 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004779237713139418, + "loss": 3.5149, + "theoretical_loss": 4.422175205391145, + "tokens_seen": 177274880 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047791374122367104, + "loss": 3.4788, + "theoretical_loss": 4.421966856347243, + "tokens_seen": 177340416 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004779037111334002, + "loss": 3.3556, + "theoretical_loss": 4.421758605833912, + "tokens_seen": 177405952 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004778936810431294, + "loss": 3.2858, + "theoretical_loss": 4.421550453768181, + "tokens_seen": 177471488 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004778836509528586, + "loss": 3.2064, + "theoretical_loss": 4.421342400067183, + "tokens_seen": 177537024 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047787362086258776, + "loss": 3.2573, + "theoretical_loss": 4.42113444464815, + "tokens_seen": 177602560 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047786359077231695, + "loss": 3.1305, + "theoretical_loss": 4.420926587428411, + "tokens_seen": 177668096 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004778535606820462, + "loss": 3.3526, + "theoretical_loss": 4.420718828325403, + "tokens_seen": 177733632 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004778435305917753, + "loss": 3.4669, + "theoretical_loss": 4.420511167256656, + "tokens_seen": 177799168 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047783350050150454, + "loss": 3.2721, + "theoretical_loss": 4.4203036041398, + "tokens_seen": 177864704 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047782347041123367, + "loss": 3.4746, + "theoretical_loss": 4.420096138892568, + "tokens_seen": 177930240 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004778134403209629, + "loss": 3.2158, + "theoretical_loss": 4.419888771432789, + "tokens_seen": 177995776 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004778034102306921, + "loss": 3.5373, + "theoretical_loss": 4.419681501678395, + "tokens_seen": 178061312 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047779338014042127, + "loss": 3.2966, + "theoretical_loss": 4.419474329547413, + "tokens_seen": 178126848 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047778335005015045, + "loss": 3.4379, + "theoretical_loss": 4.419267254957971, + "tokens_seen": 178192384 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004777733199598797, + "loss": 3.5645, + "theoretical_loss": 4.419060277828295, + "tokens_seen": 178257920 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004777632898696088, + "loss": 3.3582, + "theoretical_loss": 4.41885339807671, + "tokens_seen": 178323456 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047775325977933805, + "loss": 3.2314, + "theoretical_loss": 4.4186466156216415, + "tokens_seen": 178388992 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004777432296890672, + "loss": 3.3675, + "theoretical_loss": 4.418439930381609, + "tokens_seen": 178454528 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004777331995987964, + "loss": 3.1722, + "theoretical_loss": 4.418233342275233, + "tokens_seen": 178520064 + }, + { + "epoch": 0.05, + "objective/train/docs_used": 239453, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.536768674850464, + "objective/train/theoretical_loss": 4.418026851221231, + "objective/train/tokens_used": 199045600, + "theoretical_loss": 4.418026851221231, + "tokens_seen": 178585600 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004777231695085256, + "loss": 3.5599, + "theoretical_loss": 4.418026851221231, + "tokens_seen": 178585600 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047771313941825477, + "loss": 3.3849, + "theoretical_loss": 4.4178204571384185, + "tokens_seen": 178651136 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047770310932798395, + "loss": 3.2237, + "theoretical_loss": 4.41761415994571, + "tokens_seen": 178716672 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047769307923771313, + "loss": 3.3179, + "theoretical_loss": 4.417407959562116, + "tokens_seen": 178782208 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004776830491474423, + "loss": 3.0157, + "theoretical_loss": 4.417201855906742, + "tokens_seen": 178847744 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047767301905717155, + "loss": 3.4205, + "theoretical_loss": 4.416995848898797, + "tokens_seen": 178913280 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004776629889669007, + "loss": 3.5319, + "theoretical_loss": 4.4167899384575815, + "tokens_seen": 178978816 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004776529588766299, + "loss": 3.3096, + "theoretical_loss": 4.416584124502495, + "tokens_seen": 179044352 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047764292878635904, + "loss": 3.5886, + "theoretical_loss": 4.416378406953033, + "tokens_seen": 179109888 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004776328986960883, + "loss": 3.4965, + "theoretical_loss": 4.41617278572879, + "tokens_seen": 179175424 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047762286860581746, + "loss": 3.2969, + "theoretical_loss": 4.4159672607494524, + "tokens_seen": 179240960 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047761283851554664, + "loss": 3.3703, + "theoretical_loss": 4.415761831934808, + "tokens_seen": 179306496 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004776028084252758, + "loss": 3.363, + "theoretical_loss": 4.415556499204737, + "tokens_seen": 179372032 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047759277833500505, + "loss": 3.4131, + "theoretical_loss": 4.415351262479216, + "tokens_seen": 179437568 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004775827482447342, + "loss": 3.2837, + "theoretical_loss": 4.415146121678321, + "tokens_seen": 179503104 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004775727181544634, + "loss": 3.3687, + "theoretical_loss": 4.414941076722219, + "tokens_seen": 179568640 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047756268806419254, + "loss": 3.2755, + "theoretical_loss": 4.4147361275311745, + "tokens_seen": 179634176 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004775526579739218, + "loss": 3.4791, + "theoretical_loss": 4.414531274025548, + "tokens_seen": 179699712 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047754262788365096, + "loss": 3.3726, + "theoretical_loss": 4.414326516125795, + "tokens_seen": 179765248 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047753259779338014, + "loss": 3.4113, + "theoretical_loss": 4.414121853752466, + "tokens_seen": 179830784 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004775225677031093, + "loss": 3.3512, + "theoretical_loss": 4.413917286826205, + "tokens_seen": 179896320 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004775125376128385, + "loss": 3.4069, + "theoretical_loss": 4.413712815267752, + "tokens_seen": 179961856 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004775025075225677, + "loss": 3.1901, + "theoretical_loss": 4.413508438997944, + "tokens_seen": 180027392 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004774924774322969, + "loss": 3.5482, + "theoretical_loss": 4.4133041579377075, + "tokens_seen": 180092928 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047748244734202605, + "loss": 3.3269, + "theoretical_loss": 4.413099972008068, + "tokens_seen": 180158464 + }, + { + "epoch": 0.05, + "objective/train/docs_used": 240101, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.189091444015503, + "objective/train/theoretical_loss": 4.412895881130142, + "objective/train/tokens_used": 200684000, + "theoretical_loss": 4.412895881130142, + "tokens_seen": 180224000 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004774724172517553, + "loss": 3.2624, + "theoretical_loss": 4.412895881130142, + "tokens_seen": 180224000 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004774623871614845, + "loss": 3.4578, + "theoretical_loss": 4.412691885225141, + "tokens_seen": 180289536 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047745235707121364, + "loss": 3.3567, + "theoretical_loss": 4.412487984214373, + "tokens_seen": 180355072 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004774423269809429, + "loss": 3.2258, + "theoretical_loss": 4.412284178019235, + "tokens_seen": 180420608 + }, + { + "epoch": 0.05, + "learning_rate": 0.000477432296890672, + "loss": 3.2944, + "theoretical_loss": 4.412080466561221, + "tokens_seen": 180486144 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047742226680040124, + "loss": 3.3614, + "theoretical_loss": 4.411876849761917, + "tokens_seen": 180551680 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004774122367101304, + "loss": 3.1865, + "theoretical_loss": 4.411673327543005, + "tokens_seen": 180617216 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004774022066198596, + "loss": 3.517, + "theoretical_loss": 4.4114698998262565, + "tokens_seen": 180682752 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004773921765295888, + "loss": 3.5934, + "theoretical_loss": 4.411266566533539, + "tokens_seen": 180748288 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047738214643931797, + "loss": 3.0994, + "theoretical_loss": 4.41106332758681, + "tokens_seen": 180813824 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047737211634904715, + "loss": 3.6293, + "theoretical_loss": 4.41086018290812, + "tokens_seen": 180879360 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004773620862587764, + "loss": 3.3159, + "theoretical_loss": 4.410657132419617, + "tokens_seen": 180944896 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004773520561685055, + "loss": 3.2151, + "theoretical_loss": 4.410454176043537, + "tokens_seen": 181010432 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047734202607823474, + "loss": 3.3866, + "theoretical_loss": 4.410251313702208, + "tokens_seen": 181075968 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047733199598796387, + "loss": 3.3264, + "theoretical_loss": 4.410048545318052, + "tokens_seen": 181141504 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004773219658976931, + "loss": 3.3692, + "theoretical_loss": 4.409845870813582, + "tokens_seen": 181207040 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004773119358074223, + "loss": 3.4274, + "theoretical_loss": 4.409643290111404, + "tokens_seen": 181272576 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047730190571715147, + "loss": 3.2483, + "theoretical_loss": 4.409440803134215, + "tokens_seen": 181338112 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047729187562688065, + "loss": 3.2365, + "theoretical_loss": 4.409238409804804, + "tokens_seen": 181403648 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004772818455366099, + "loss": 3.2074, + "theoretical_loss": 4.409036110046051, + "tokens_seen": 181469184 + }, + { + "epoch": 0.06, + "learning_rate": 0.000477271815446339, + "loss": 3.0851, + "theoretical_loss": 4.408833903780926, + "tokens_seen": 181534720 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047726178535606825, + "loss": 3.2768, + "theoretical_loss": 4.408631790932494, + "tokens_seen": 181600256 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004772517552657974, + "loss": 3.7316, + "theoretical_loss": 4.408429771423909, + "tokens_seen": 181665792 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004772417251755266, + "loss": 3.1774, + "theoretical_loss": 4.408227845178414, + "tokens_seen": 181731328 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004772316950852558, + "loss": 3.3076, + "theoretical_loss": 4.408026012119344, + "tokens_seen": 181796864 + }, + { + "epoch": 0.06, + "objective/train/docs_used": 241412, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.7472660541534424, + "objective/train/theoretical_loss": 4.407824272170128, + "objective/train/tokens_used": 202322400, + "theoretical_loss": 4.407824272170128, + "tokens_seen": 181862400 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047722166499498497, + "loss": 3.2786, + "theoretical_loss": 4.407824272170128, + "tokens_seen": 181862400 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047721163490471415, + "loss": 3.4129, + "theoretical_loss": 4.407622625254279, + "tokens_seen": 181927936 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047720160481444333, + "loss": 2.838, + "theoretical_loss": 4.407421071295406, + "tokens_seen": 181993472 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004771915747241725, + "loss": 3.2404, + "theoretical_loss": 4.407219610217206, + "tokens_seen": 182059008 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047718154463390175, + "loss": 3.4927, + "theoretical_loss": 4.407018241943467, + "tokens_seen": 182124544 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004771715145436309, + "loss": 3.1684, + "theoretical_loss": 4.406816966398064, + "tokens_seen": 182190080 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004771614844533601, + "loss": 3.3542, + "theoretical_loss": 4.406615783504965, + "tokens_seen": 182255616 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047715145436308924, + "loss": 3.2972, + "theoretical_loss": 4.4064146931882275, + "tokens_seen": 182321152 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004771414242728185, + "loss": 3.0567, + "theoretical_loss": 4.406213695371996, + "tokens_seen": 182386688 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047713139418254766, + "loss": 3.2913, + "theoretical_loss": 4.406012789980506, + "tokens_seen": 182452224 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047712136409227684, + "loss": 3.425, + "theoretical_loss": 4.405811976938084, + "tokens_seen": 182517760 + }, + { + "epoch": 0.06, + "learning_rate": 0.000477111334002006, + "loss": 3.3947, + "theoretical_loss": 4.405611256169143, + "tokens_seen": 182583296 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047710130391173525, + "loss": 3.2788, + "theoretical_loss": 4.405410627598185, + "tokens_seen": 182648832 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004770912738214644, + "loss": 3.2396, + "theoretical_loss": 4.405210091149802, + "tokens_seen": 182714368 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004770812437311936, + "loss": 3.6049, + "theoretical_loss": 4.405009646748674, + "tokens_seen": 182779904 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047707121364092274, + "loss": 3.2774, + "theoretical_loss": 4.404809294319572, + "tokens_seen": 182845440 + }, + { + "epoch": 0.06, + "learning_rate": 0.000477061183550652, + "loss": 3.1352, + "theoretical_loss": 4.40460903378735, + "tokens_seen": 182910976 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047705115346038116, + "loss": 3.5256, + "theoretical_loss": 4.404408865076955, + "tokens_seen": 182976512 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047704112337011034, + "loss": 3.3179, + "theoretical_loss": 4.404208788113422, + "tokens_seen": 183042048 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004770310932798395, + "loss": 3.368, + "theoretical_loss": 4.404008802821871, + "tokens_seen": 183107584 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004770210631895687, + "loss": 3.4253, + "theoretical_loss": 4.4038089091275125, + "tokens_seen": 183173120 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004770110330992979, + "loss": 3.3234, + "theoretical_loss": 4.403609106955645, + "tokens_seen": 183238656 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004770010030090271, + "loss": 3.3547, + "theoretical_loss": 4.403409396231651, + "tokens_seen": 183304192 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047699097291875625, + "loss": 3.3308, + "theoretical_loss": 4.403209776881004, + "tokens_seen": 183369728 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004769809428284855, + "loss": 3.3826, + "theoretical_loss": 4.403010248829265, + "tokens_seen": 183435264 + }, + { + "epoch": 0.06, + "objective/train/docs_used": 241901, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.666914463043213, + "objective/train/theoretical_loss": 4.4028108120020795, + "objective/train/tokens_used": 203960800, + "theoretical_loss": 4.4028108120020795, + "tokens_seen": 183500800 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047697091273821466, + "loss": 3.347, + "theoretical_loss": 4.4028108120020795, + "tokens_seen": 183500800 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047696088264794384, + "loss": 3.3715, + "theoretical_loss": 4.402611466325182, + "tokens_seen": 183566336 + }, + { + "epoch": 0.06, + "learning_rate": 0.000476950852557673, + "loss": 3.5634, + "theoretical_loss": 4.4024122117243945, + "tokens_seen": 183631872 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004769408224674022, + "loss": 3.0663, + "theoretical_loss": 4.402213048125624, + "tokens_seen": 183697408 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004769307923771314, + "loss": 3.059, + "theoretical_loss": 4.4020139754548655, + "tokens_seen": 183762944 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004769207622868606, + "loss": 3.1995, + "theoretical_loss": 4.401814993638199, + "tokens_seen": 183828480 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047691073219658975, + "loss": 3.1642, + "theoretical_loss": 4.4016161026017935, + "tokens_seen": 183894016 + }, + { + "epoch": 0.06, + "learning_rate": 0.000476900702106319, + "loss": 2.9642, + "theoretical_loss": 4.401417302271902, + "tokens_seen": 183959552 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004768906720160481, + "loss": 3.405, + "theoretical_loss": 4.401218592574865, + "tokens_seen": 184025088 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047688064192577735, + "loss": 3.362, + "theoretical_loss": 4.401019973437108, + "tokens_seen": 184090624 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047687061183550653, + "loss": 3.416, + "theoretical_loss": 4.400821444785143, + "tokens_seen": 184156160 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004768605817452357, + "loss": 3.1617, + "theoretical_loss": 4.400623006545567, + "tokens_seen": 184221696 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004768505516549649, + "loss": 3.4902, + "theoretical_loss": 4.400424658645065, + "tokens_seen": 184287232 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047684052156469407, + "loss": 3.2861, + "theoretical_loss": 4.400226401010404, + "tokens_seen": 184352768 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047683049147442325, + "loss": 3.3124, + "theoretical_loss": 4.40002823356844, + "tokens_seen": 184418304 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004768204613841525, + "loss": 2.9837, + "theoretical_loss": 4.39983015624611, + "tokens_seen": 184483840 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004768104312938816, + "loss": 3.2424, + "theoretical_loss": 4.39963216897044, + "tokens_seen": 184549376 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047680040120361085, + "loss": 3.2719, + "theoretical_loss": 4.3994342716685395, + "tokens_seen": 184614912 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047679037111334003, + "loss": 3.1485, + "theoretical_loss": 4.399236464267602, + "tokens_seen": 184680448 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004767803410230692, + "loss": 3.3108, + "theoretical_loss": 4.399038746694908, + "tokens_seen": 184745984 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004767703109327984, + "loss": 3.135, + "theoretical_loss": 4.398841118877819, + "tokens_seen": 184811520 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004767602808425276, + "loss": 3.4858, + "theoretical_loss": 4.398643580743785, + "tokens_seen": 184877056 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047675025075225676, + "loss": 3.272, + "theoretical_loss": 4.398446132220338, + "tokens_seen": 184942592 + }, + { + "epoch": 0.06, + "learning_rate": 0.000476740220661986, + "loss": 3.4381, + "theoretical_loss": 4.3982487732350934, + "tokens_seen": 185008128 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004767301905717151, + "loss": 3.3967, + "theoretical_loss": 4.398051503715753, + "tokens_seen": 185073664 + }, + { + "epoch": 0.06, + "objective/train/docs_used": 243537, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.3711562156677246, + "objective/train/theoretical_loss": 4.397854323590102, + "objective/train/tokens_used": 205599200, + "theoretical_loss": 4.397854323590102, + "tokens_seen": 185139200 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047672016048144435, + "loss": 3.0928, + "theoretical_loss": 4.397854323590102, + "tokens_seen": 185139200 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047671013039117353, + "loss": 3.2136, + "theoretical_loss": 4.397657232786008, + "tokens_seen": 185204736 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004767001003009027, + "loss": 3.3584, + "theoretical_loss": 4.397460231231424, + "tokens_seen": 185270272 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047669007021063195, + "loss": 3.4504, + "theoretical_loss": 4.397263318854384, + "tokens_seen": 185335808 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004766800401203611, + "loss": 3.3679, + "theoretical_loss": 4.39706649558301, + "tokens_seen": 185401344 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004766700100300903, + "loss": 3.254, + "theoretical_loss": 4.396869761345503, + "tokens_seen": 185466880 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047665997993981944, + "loss": 3.0167, + "theoretical_loss": 4.396673116070147, + "tokens_seen": 185532416 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004766499498495487, + "loss": 3.397, + "theoretical_loss": 4.396476559685315, + "tokens_seen": 185597952 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047663991975927786, + "loss": 3.1028, + "theoretical_loss": 4.396280092119455, + "tokens_seen": 185663488 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047662988966900704, + "loss": 3.3503, + "theoretical_loss": 4.3960837133011035, + "tokens_seen": 185729024 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004766198595787362, + "loss": 3.2027, + "theoretical_loss": 4.395887423158877, + "tokens_seen": 185794560 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047660982948846545, + "loss": 3.36, + "theoretical_loss": 4.395691221621476, + "tokens_seen": 185860096 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004765997993981946, + "loss": 3.4191, + "theoretical_loss": 4.395495108617682, + "tokens_seen": 185925632 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004765897693079238, + "loss": 3.4474, + "theoretical_loss": 4.39529908407636, + "tokens_seen": 185991168 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047657973921765294, + "loss": 3.2397, + "theoretical_loss": 4.3951031479264575, + "tokens_seen": 186056704 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004765697091273822, + "loss": 3.4729, + "theoretical_loss": 4.394907300097002, + "tokens_seen": 186122240 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047655967903711136, + "loss": 3.2495, + "theoretical_loss": 4.394711540517106, + "tokens_seen": 186187776 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047654964894684054, + "loss": 3.4181, + "theoretical_loss": 4.39451586911596, + "tokens_seen": 186253312 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004765396188565697, + "loss": 3.5468, + "theoretical_loss": 4.39432028582284, + "tokens_seen": 186318848 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004765295887662989, + "loss": 3.111, + "theoretical_loss": 4.394124790567101, + "tokens_seen": 186384384 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004765195586760281, + "loss": 3.3198, + "theoretical_loss": 4.3939293832781825, + "tokens_seen": 186449920 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004765095285857573, + "loss": 3.2602, + "theoretical_loss": 4.393734063885599, + "tokens_seen": 186515456 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047649949849548645, + "loss": 3.0038, + "theoretical_loss": 4.3935388323189555, + "tokens_seen": 186580992 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004764894684052157, + "loss": 3.371, + "theoretical_loss": 4.39334368850793, + "tokens_seen": 186646528 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047647943831494486, + "loss": 3.4606, + "theoretical_loss": 4.3931486323822835, + "tokens_seen": 186712064 + }, + { + "epoch": 0.06, + "objective/train/docs_used": 246703, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.3764562606811523, + "objective/train/theoretical_loss": 4.392953663871862, + "objective/train/tokens_used": 207237600, + "theoretical_loss": 4.392953663871862, + "tokens_seen": 186777600 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047646940822467404, + "loss": 3.2311, + "theoretical_loss": 4.392953663871862, + "tokens_seen": 186777600 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004764593781344032, + "loss": 3.2135, + "theoretical_loss": 4.392758782906586, + "tokens_seen": 186843136 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004764493480441324, + "loss": 3.5582, + "theoretical_loss": 4.392563989416462, + "tokens_seen": 186908672 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004764393179538616, + "loss": 3.4679, + "theoretical_loss": 4.392369283331574, + "tokens_seen": 186974208 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004764292878635908, + "loss": 3.3004, + "theoretical_loss": 4.392174664582085, + "tokens_seen": 187039744 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047641925777331995, + "loss": 3.3981, + "theoretical_loss": 4.391980133098244, + "tokens_seen": 187105280 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004764092276830492, + "loss": 3.248, + "theoretical_loss": 4.391785688810373, + "tokens_seen": 187170816 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004763991975927783, + "loss": 3.6385, + "theoretical_loss": 4.391591331648879, + "tokens_seen": 187236352 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047638916750250755, + "loss": 3.2769, + "theoretical_loss": 4.391397061544247, + "tokens_seen": 187301888 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047637913741223673, + "loss": 3.1636, + "theoretical_loss": 4.391202878427042, + "tokens_seen": 187367424 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004763691073219659, + "loss": 3.3382, + "theoretical_loss": 4.3910087822279085, + "tokens_seen": 187432960 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004763590772316951, + "loss": 3.3282, + "theoretical_loss": 4.390814772877571, + "tokens_seen": 187498496 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047634904714142427, + "loss": 3.4497, + "theoretical_loss": 4.390620850306832, + "tokens_seen": 187564032 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047633901705115345, + "loss": 3.5448, + "theoretical_loss": 4.390427014446575, + "tokens_seen": 187629568 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004763289869608827, + "loss": 3.5607, + "theoretical_loss": 4.390233265227764, + "tokens_seen": 187695104 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004763189568706118, + "loss": 3.2674, + "theoretical_loss": 4.390039602581437, + "tokens_seen": 187760640 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047630892678034105, + "loss": 3.3578, + "theoretical_loss": 4.389846026438715, + "tokens_seen": 187826176 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047629889669007023, + "loss": 3.3827, + "theoretical_loss": 4.3896525367307975, + "tokens_seen": 187891712 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004762888665997994, + "loss": 3.2597, + "theoretical_loss": 4.389459133388962, + "tokens_seen": 187957248 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004762788365095286, + "loss": 3.1751, + "theoretical_loss": 4.3892658163445635, + "tokens_seen": 188022784 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004762688064192578, + "loss": 3.3537, + "theoretical_loss": 4.389072585529037, + "tokens_seen": 188088320 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047625877632898696, + "loss": 2.8638, + "theoretical_loss": 4.388879440873897, + "tokens_seen": 188153856 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004762487462387162, + "loss": 3.4062, + "theoretical_loss": 4.388686382310732, + "tokens_seen": 188219392 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004762387161484453, + "loss": 3.1585, + "theoretical_loss": 4.388493409771213, + "tokens_seen": 188284928 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047622868605817455, + "loss": 3.134, + "theoretical_loss": 4.388300523187087, + "tokens_seen": 188350464 + }, + { + "epoch": 0.06, + "objective/train/docs_used": 251659, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.317779541015625, + "objective/train/theoretical_loss": 4.3881077224901786, + "objective/train/tokens_used": 208876000, + "theoretical_loss": 4.3881077224901786, + "tokens_seen": 188416000 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004762186559679037, + "loss": 3.4901, + "theoretical_loss": 4.3881077224901786, + "tokens_seen": 188416000 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004762086258776329, + "loss": 3.4647, + "theoretical_loss": 4.38791500761239, + "tokens_seen": 188481536 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004761985957873621, + "loss": 3.1316, + "theoretical_loss": 4.387722378485703, + "tokens_seen": 188547072 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004761885656970913, + "loss": 3.1812, + "theoretical_loss": 4.3875298350421765, + "tokens_seen": 188612608 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047617853560682046, + "loss": 3.2664, + "theoretical_loss": 4.387337377213943, + "tokens_seen": 188678144 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047616850551654964, + "loss": 3.4667, + "theoretical_loss": 4.387145004933218, + "tokens_seen": 188743680 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004761584754262788, + "loss": 3.1033, + "theoretical_loss": 4.38695271813229, + "tokens_seen": 188809216 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047614844533600806, + "loss": 3.425, + "theoretical_loss": 4.386760516743526, + "tokens_seen": 188874752 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004761384152457372, + "loss": 3.301, + "theoretical_loss": 4.38656840069937, + "tokens_seen": 188940288 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004761283851554664, + "loss": 3.3707, + "theoretical_loss": 4.386376369932344, + "tokens_seen": 189005824 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004761183550651956, + "loss": 3.3374, + "theoretical_loss": 4.386184424375044, + "tokens_seen": 189071360 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004761083249749248, + "loss": 3.279, + "theoretical_loss": 4.385992563960145, + "tokens_seen": 189136896 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047609829488465396, + "loss": 3.101, + "theoretical_loss": 4.385800788620397, + "tokens_seen": 189202432 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047608826479438314, + "loss": 3.205, + "theoretical_loss": 4.385609098288628, + "tokens_seen": 189267968 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004760782347041123, + "loss": 3.2755, + "theoretical_loss": 4.385417492897741, + "tokens_seen": 189333504 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047606820461384156, + "loss": 3.3262, + "theoretical_loss": 4.385225972380715, + "tokens_seen": 189399040 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004760581745235707, + "loss": 3.2513, + "theoretical_loss": 4.385034536670606, + "tokens_seen": 189464576 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004760481444332999, + "loss": 3.422, + "theoretical_loss": 4.384843185700544, + "tokens_seen": 189530112 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047603811434302905, + "loss": 3.6571, + "theoretical_loss": 4.384651919403739, + "tokens_seen": 189595648 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004760280842527583, + "loss": 3.1847, + "theoretical_loss": 4.384460737713471, + "tokens_seen": 189661184 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047601805416248746, + "loss": 3.3644, + "theoretical_loss": 4.384269640563101, + "tokens_seen": 189726720 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047600802407221665, + "loss": 3.4833, + "theoretical_loss": 4.384078627886062, + "tokens_seen": 189792256 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004759979939819458, + "loss": 3.2601, + "theoretical_loss": 4.383887699615863, + "tokens_seen": 189857792 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047598796389167506, + "loss": 3.3371, + "theoretical_loss": 4.38369685568609, + "tokens_seen": 189923328 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004759779338014042, + "loss": 3.2443, + "theoretical_loss": 4.383506096030401, + "tokens_seen": 189988864 + }, + { + "epoch": 0.06, + "objective/train/docs_used": 256551, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.438608169555664, + "objective/train/theoretical_loss": 4.383315420582533, + "objective/train/tokens_used": 210514400, + "theoretical_loss": 4.383315420582533, + "tokens_seen": 190054400 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004759679037111334, + "loss": 3.2904, + "theoretical_loss": 4.383315420582533, + "tokens_seen": 190054400 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004759578736208626, + "loss": 3.0384, + "theoretical_loss": 4.383124829276294, + "tokens_seen": 190119936 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004759478435305918, + "loss": 3.1615, + "theoretical_loss": 4.38293432204557, + "tokens_seen": 190185472 + }, + { + "epoch": 0.06, + "learning_rate": 0.000475937813440321, + "loss": 3.4141, + "theoretical_loss": 4.382743898824321, + "tokens_seen": 190251008 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047592778335005015, + "loss": 3.2814, + "theoretical_loss": 4.3825535595465785, + "tokens_seen": 190316544 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004759177532597794, + "loss": 3.3771, + "theoretical_loss": 4.382363304146453, + "tokens_seen": 190382080 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004759077231695085, + "loss": 3.4016, + "theoretical_loss": 4.382173132558126, + "tokens_seen": 190447616 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047589769307923775, + "loss": 3.306, + "theoretical_loss": 4.381983044715856, + "tokens_seen": 190513152 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047588766298896693, + "loss": 3.2248, + "theoretical_loss": 4.381793040553973, + "tokens_seen": 190578688 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004758776328986961, + "loss": 3.2466, + "theoretical_loss": 4.381603120006883, + "tokens_seen": 190644224 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004758676028084253, + "loss": 3.325, + "theoretical_loss": 4.381413283009065, + "tokens_seen": 190709760 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047585757271815447, + "loss": 3.1018, + "theoretical_loss": 4.381223529495073, + "tokens_seen": 190775296 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047584754262788365, + "loss": 3.2762, + "theoretical_loss": 4.381033859399532, + "tokens_seen": 190840832 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004758375125376129, + "loss": 3.5844, + "theoretical_loss": 4.380844272657145, + "tokens_seen": 190906368 + }, + { + "epoch": 0.06, + "learning_rate": 0.000475827482447342, + "loss": 3.3999, + "theoretical_loss": 4.380654769202683, + "tokens_seen": 190971904 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047581745235707125, + "loss": 3.3852, + "theoretical_loss": 4.380465348970995, + "tokens_seen": 191037440 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047580742226680043, + "loss": 3.2935, + "theoretical_loss": 4.380276011897003, + "tokens_seen": 191102976 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004757973921765296, + "loss": 3.2371, + "theoretical_loss": 4.380086757915698, + "tokens_seen": 191168512 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004757873620862588, + "loss": 3.5679, + "theoretical_loss": 4.379897586962148, + "tokens_seen": 191234048 + }, + { + "epoch": 0.06, + "learning_rate": 0.000475777331995988, + "loss": 3.338, + "theoretical_loss": 4.379708498971494, + "tokens_seen": 191299584 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047576730190571716, + "loss": 3.1515, + "theoretical_loss": 4.379519493878948, + "tokens_seen": 191365120 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004757572718154464, + "loss": 3.1375, + "theoretical_loss": 4.379330571619795, + "tokens_seen": 191430656 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004757472417251755, + "loss": 3.2006, + "theoretical_loss": 4.379141732129394, + "tokens_seen": 191496192 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047573721163490475, + "loss": 3.5164, + "theoretical_loss": 4.378952975343175, + "tokens_seen": 191561728 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004757271815446339, + "loss": 3.4549, + "theoretical_loss": 4.378764301196642, + "tokens_seen": 191627264 + }, + { + "epoch": 0.06, + "objective/train/docs_used": 261606, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.4967339038848877, + "objective/train/theoretical_loss": 4.37857570962537, + "objective/train/tokens_used": 212152800, + "theoretical_loss": 4.37857570962537, + "tokens_seen": 191692800 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004757171514543631, + "loss": 3.2403, + "theoretical_loss": 4.37857570962537, + "tokens_seen": 191692800 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004757071213640923, + "loss": 3.305, + "theoretical_loss": 4.378387200565006, + "tokens_seen": 191758336 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004756970912738215, + "loss": 3.175, + "theoretical_loss": 4.378198773951272, + "tokens_seen": 191823872 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047568706118355066, + "loss": 3.3906, + "theoretical_loss": 4.378010429719957, + "tokens_seen": 191889408 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047567703109327984, + "loss": 3.4164, + "theoretical_loss": 4.377822167806928, + "tokens_seen": 191954944 + }, + { + "epoch": 0.06, + "learning_rate": 0.000475667001003009, + "loss": 3.2804, + "theoretical_loss": 4.377633988148117, + "tokens_seen": 192020480 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047565697091273826, + "loss": 3.2627, + "theoretical_loss": 4.377445890679534, + "tokens_seen": 192086016 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004756469408224674, + "loss": 3.1732, + "theoretical_loss": 4.377257875337257, + "tokens_seen": 192151552 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004756369107321966, + "loss": 3.4658, + "theoretical_loss": 4.377069942057436, + "tokens_seen": 192217088 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004756268806419258, + "loss": 3.3786, + "theoretical_loss": 4.376882090776293, + "tokens_seen": 192282624 + }, + { + "epoch": 0.06, + "learning_rate": 0.000475616850551655, + "loss": 3.1876, + "theoretical_loss": 4.376694321430121, + "tokens_seen": 192348160 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047560682046138416, + "loss": 3.2698, + "theoretical_loss": 4.376506633955286, + "tokens_seen": 192413696 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047559679037111334, + "loss": 3.3082, + "theoretical_loss": 4.376319028288219, + "tokens_seen": 192479232 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004755867602808425, + "loss": 3.4333, + "theoretical_loss": 4.37613150436543, + "tokens_seen": 192544768 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047557673019057176, + "loss": 3.4606, + "theoretical_loss": 4.375944062123496, + "tokens_seen": 192610304 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004755667001003009, + "loss": 3.0892, + "theoretical_loss": 4.375756701499063, + "tokens_seen": 192675840 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004755566700100301, + "loss": 3.2806, + "theoretical_loss": 4.3755694224288515, + "tokens_seen": 192741376 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047554663991975925, + "loss": 3.4885, + "theoretical_loss": 4.375382224849648, + "tokens_seen": 192806912 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004755366098294885, + "loss": 3.217, + "theoretical_loss": 4.375195108698316, + "tokens_seen": 192872448 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047552657973921766, + "loss": 3.277, + "theoretical_loss": 4.375008073911781, + "tokens_seen": 192937984 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047551654964894685, + "loss": 3.6578, + "theoretical_loss": 4.374821120427047, + "tokens_seen": 193003520 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047550651955867603, + "loss": 3.4625, + "theoretical_loss": 4.374634248181182, + "tokens_seen": 193069056 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047549648946840526, + "loss": 3.3939, + "theoretical_loss": 4.3744474571113265, + "tokens_seen": 193134592 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004754864593781344, + "loss": 3.4205, + "theoretical_loss": 4.374260747154692, + "tokens_seen": 193200128 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004754764292878636, + "loss": 3.0036, + "theoretical_loss": 4.374074118248559, + "tokens_seen": 193265664 + }, + { + "epoch": 0.06, + "objective/train/docs_used": 266689, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.22265625, + "objective/train/theoretical_loss": 4.373887570330275, + "objective/train/tokens_used": 213791200, + "theoretical_loss": 4.373887570330275, + "tokens_seen": 193331200 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047546639919759275, + "loss": 3.3538, + "theoretical_loss": 4.373887570330275, + "tokens_seen": 193331200 + }, + { + "epoch": 0.06, + "learning_rate": 0.000475456369107322, + "loss": 3.0157, + "theoretical_loss": 4.373701103337263, + "tokens_seen": 193396736 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047544633901705117, + "loss": 3.3198, + "theoretical_loss": 4.373514717207009, + "tokens_seen": 193462272 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047543630892678035, + "loss": 3.303, + "theoretical_loss": 4.373328411877073, + "tokens_seen": 193527808 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047542627883650953, + "loss": 3.4534, + "theoretical_loss": 4.373142187285083, + "tokens_seen": 193593344 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004754162487462387, + "loss": 3.442, + "theoretical_loss": 4.372956043368736, + "tokens_seen": 193658880 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004754062186559679, + "loss": 3.0814, + "theoretical_loss": 4.372769980065797, + "tokens_seen": 193724416 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047539618856569713, + "loss": 3.2632, + "theoretical_loss": 4.372583997314104, + "tokens_seen": 193789952 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047538615847542625, + "loss": 3.0204, + "theoretical_loss": 4.372398095051559, + "tokens_seen": 193855488 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004753761283851555, + "loss": 3.4108, + "theoretical_loss": 4.372212273216136, + "tokens_seen": 193921024 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004753660982948846, + "loss": 3.3715, + "theoretical_loss": 4.372026531745877, + "tokens_seen": 193986560 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047535606820461385, + "loss": 3.1585, + "theoretical_loss": 4.371840870578891, + "tokens_seen": 194052096 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047534603811434303, + "loss": 3.3463, + "theoretical_loss": 4.37165528965336, + "tokens_seen": 194117632 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004753360080240722, + "loss": 3.1645, + "theoretical_loss": 4.371469788907529, + "tokens_seen": 194183168 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004753259779338014, + "loss": 3.542, + "theoretical_loss": 4.371284368279714, + "tokens_seen": 194248704 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047531594784353063, + "loss": 3.4428, + "theoretical_loss": 4.3710990277083, + "tokens_seen": 194314240 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047530591775325976, + "loss": 3.2136, + "theoretical_loss": 4.3709137671317375, + "tokens_seen": 194379776 + }, + { + "epoch": 0.06, + "learning_rate": 0.000475295887662989, + "loss": 3.5123, + "theoretical_loss": 4.37072858648855, + "tokens_seen": 194445312 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004752858575727181, + "loss": 3.4088, + "theoretical_loss": 4.370543485717322, + "tokens_seen": 194510848 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047527582748244736, + "loss": 3.2998, + "theoretical_loss": 4.370358464756713, + "tokens_seen": 194576384 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047526579739217654, + "loss": 3.0609, + "theoretical_loss": 4.370173523545443, + "tokens_seen": 194641920 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004752557673019057, + "loss": 3.1257, + "theoretical_loss": 4.3699886620223065, + "tokens_seen": 194707456 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004752457372116349, + "loss": 3.3654, + "theoretical_loss": 4.369803880126162, + "tokens_seen": 194772992 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004752357071213641, + "loss": 3.362, + "theoretical_loss": 4.3696191777959354, + "tokens_seen": 194838528 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047522567703109326, + "loss": 3.3996, + "theoretical_loss": 4.369434554970621, + "tokens_seen": 194904064 + }, + { + "epoch": 0.06, + "objective/train/docs_used": 269630, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.62288236618042, + "objective/train/theoretical_loss": 4.369250011589279, + "objective/train/tokens_used": 215429600, + "theoretical_loss": 4.369250011589279, + "tokens_seen": 194969600 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004752156469408225, + "loss": 3.4504, + "theoretical_loss": 4.369250011589279, + "tokens_seen": 194969600 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004752056168505517, + "loss": 3.3655, + "theoretical_loss": 4.369065547591038, + "tokens_seen": 195035136 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047519558676028086, + "loss": 3.2661, + "theoretical_loss": 4.368881162915095, + "tokens_seen": 195100672 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047518555667001004, + "loss": 3.3123, + "theoretical_loss": 4.36869685750071, + "tokens_seen": 195166208 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004751755265797392, + "loss": 3.2071, + "theoretical_loss": 4.3685126312872145, + "tokens_seen": 195231744 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047516549648946846, + "loss": 3.5141, + "theoretical_loss": 4.368328484214002, + "tokens_seen": 195297280 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004751554663991976, + "loss": 3.4057, + "theoretical_loss": 4.368144416220538, + "tokens_seen": 195362816 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004751454363089268, + "loss": 3.34, + "theoretical_loss": 4.3679604272463495, + "tokens_seen": 195428352 + }, + { + "epoch": 0.06, + "learning_rate": 0.000475135406218656, + "loss": 3.2522, + "theoretical_loss": 4.367776517231033, + "tokens_seen": 195493888 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004751253761283852, + "loss": 3.1821, + "theoretical_loss": 4.367592686114252, + "tokens_seen": 195559424 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047511534603811436, + "loss": 3.3659, + "theoretical_loss": 4.367408933835733, + "tokens_seen": 195624960 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047510531594784354, + "loss": 3.3274, + "theoretical_loss": 4.367225260335272, + "tokens_seen": 195690496 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004750952858575727, + "loss": 3.1391, + "theoretical_loss": 4.36704166555273, + "tokens_seen": 195756032 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047508525576730196, + "loss": 3.2901, + "theoretical_loss": 4.366858149428032, + "tokens_seen": 195821568 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004750752256770311, + "loss": 3.0, + "theoretical_loss": 4.366674711901173, + "tokens_seen": 195887104 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004750651955867603, + "loss": 3.3856, + "theoretical_loss": 4.366491352912211, + "tokens_seen": 195952640 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047505516549648945, + "loss": 3.369, + "theoretical_loss": 4.366308072401271, + "tokens_seen": 196018176 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004750451354062187, + "loss": 3.1375, + "theoretical_loss": 4.366124870308541, + "tokens_seen": 196083712 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047503510531594787, + "loss": 3.3023, + "theoretical_loss": 4.365941746574278, + "tokens_seen": 196149248 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047502507522567705, + "loss": 3.4409, + "theoretical_loss": 4.3657587011388035, + "tokens_seen": 196214784 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047501504513540623, + "loss": 3.4023, + "theoretical_loss": 4.365575733942503, + "tokens_seen": 196280320 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047500501504513546, + "loss": 3.1508, + "theoretical_loss": 4.365392844925829, + "tokens_seen": 196345856 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004749949849548646, + "loss": 3.2641, + "theoretical_loss": 4.365210034029298, + "tokens_seen": 196411392 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004749849548645938, + "loss": 3.2689, + "theoretical_loss": 4.365027301193491, + "tokens_seen": 196476928 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047497492477432295, + "loss": 3.3907, + "theoretical_loss": 4.364844646359056, + "tokens_seen": 196542464 + }, + { + "epoch": 0.06, + "objective/train/docs_used": 270311, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.453747272491455, + "objective/train/theoretical_loss": 4.364662069466704, + "objective/train/tokens_used": 217068000, + "theoretical_loss": 4.364662069466704, + "tokens_seen": 196608000 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004749648946840522, + "loss": 3.4165, + "theoretical_loss": 4.364662069466704, + "tokens_seen": 196608000 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047495486459378137, + "loss": 3.2901, + "theoretical_loss": 4.364479570457213, + "tokens_seen": 196673536 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047494483450351055, + "loss": 3.3529, + "theoretical_loss": 4.364297149271423, + "tokens_seen": 196739072 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047493480441323973, + "loss": 3.4174, + "theoretical_loss": 4.3641148058502415, + "tokens_seen": 196804608 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004749247743229689, + "loss": 3.203, + "theoretical_loss": 4.363932540134638, + "tokens_seen": 196870144 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004749147442326981, + "loss": 3.1933, + "theoretical_loss": 4.363750352065647, + "tokens_seen": 196935680 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047490471414242733, + "loss": 3.2939, + "theoretical_loss": 4.363568241584368, + "tokens_seen": 197001216 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047489468405215646, + "loss": 3.1258, + "theoretical_loss": 4.363386208631966, + "tokens_seen": 197066752 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004748846539618857, + "loss": 3.2392, + "theoretical_loss": 4.363204253149667, + "tokens_seen": 197132288 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004748746238716148, + "loss": 3.283, + "theoretical_loss": 4.3630223750787644, + "tokens_seen": 197197824 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047486459378134405, + "loss": 3.3828, + "theoretical_loss": 4.362840574360612, + "tokens_seen": 197263360 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047485456369107323, + "loss": 3.0395, + "theoretical_loss": 4.362658850936631, + "tokens_seen": 197328896 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004748445336008024, + "loss": 3.2909, + "theoretical_loss": 4.362477204748305, + "tokens_seen": 197394432 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004748345035105316, + "loss": 3.1958, + "theoretical_loss": 4.362295635737179, + "tokens_seen": 197459968 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047482447342026083, + "loss": 2.9771, + "theoretical_loss": 4.362114143844867, + "tokens_seen": 197525504 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047481444332998996, + "loss": 3.289, + "theoretical_loss": 4.3619327290130405, + "tokens_seen": 197591040 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004748044132397192, + "loss": 3.2772, + "theoretical_loss": 4.3617513911834385, + "tokens_seen": 197656576 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004747943831494483, + "loss": 3.149, + "theoretical_loss": 4.361570130297863, + "tokens_seen": 197722112 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047478435305917756, + "loss": 3.3416, + "theoretical_loss": 4.3613889462981765, + "tokens_seen": 197787648 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047477432296890674, + "loss": 3.4184, + "theoretical_loss": 4.361207839126308, + "tokens_seen": 197853184 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004747642928786359, + "loss": 3.3547, + "theoretical_loss": 4.361026808724247, + "tokens_seen": 197918720 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004747542627883651, + "loss": 3.1468, + "theoretical_loss": 4.360845855034049, + "tokens_seen": 197984256 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004747442326980943, + "loss": 3.2702, + "theoretical_loss": 4.360664977997828, + "tokens_seen": 198049792 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047473420260782346, + "loss": 2.9289, + "theoretical_loss": 4.360484177557766, + "tokens_seen": 198115328 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004747241725175527, + "loss": 3.3127, + "theoretical_loss": 4.360303453656103, + "tokens_seen": 198180864 + }, + { + "epoch": 0.06, + "objective/train/docs_used": 271760, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.187415838241577, + "objective/train/theoretical_loss": 4.360122806235145, + "objective/train/tokens_used": 218706400, + "theoretical_loss": 4.360122806235145, + "tokens_seen": 198246400 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004747141424272818, + "loss": 3.2371, + "theoretical_loss": 4.360122806235145, + "tokens_seen": 198246400 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047470411233701106, + "loss": 3.1704, + "theoretical_loss": 4.359942235237257, + "tokens_seen": 198311936 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004746940822467402, + "loss": 3.375, + "theoretical_loss": 4.359761740604871, + "tokens_seen": 198377472 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004746840521564694, + "loss": 3.0178, + "theoretical_loss": 4.359581322280479, + "tokens_seen": 198443008 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004746740220661986, + "loss": 2.9285, + "theoretical_loss": 4.359400980206634, + "tokens_seen": 198508544 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004746639919759278, + "loss": 3.3454, + "theoretical_loss": 4.359220714325954, + "tokens_seen": 198574080 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047465396188565696, + "loss": 3.3081, + "theoretical_loss": 4.359040524581116, + "tokens_seen": 198639616 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004746439317953862, + "loss": 3.4269, + "theoretical_loss": 4.358860410914861, + "tokens_seen": 198705152 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004746339017051153, + "loss": 3.2979, + "theoretical_loss": 4.358680373269993, + "tokens_seen": 198770688 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047462387161484456, + "loss": 3.2567, + "theoretical_loss": 4.358500411589375, + "tokens_seen": 198836224 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004746138415245737, + "loss": 3.373, + "theoretical_loss": 4.358320525815934, + "tokens_seen": 198901760 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004746038114343029, + "loss": 3.4155, + "theoretical_loss": 4.358140715892658, + "tokens_seen": 198967296 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004745937813440321, + "loss": 3.5005, + "theoretical_loss": 4.357960981762595, + "tokens_seen": 199032832 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004745837512537613, + "loss": 3.145, + "theoretical_loss": 4.357781323368857, + "tokens_seen": 199098368 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047457372116349047, + "loss": 3.1356, + "theoretical_loss": 4.357601740654617, + "tokens_seen": 199163904 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047456369107321965, + "loss": 3.3813, + "theoretical_loss": 4.357422233563106, + "tokens_seen": 199229440 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047455366098294883, + "loss": 3.2667, + "theoretical_loss": 4.357242802037623, + "tokens_seen": 199294976 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047454363089267807, + "loss": 3.5508, + "theoretical_loss": 4.35706344602152, + "tokens_seen": 199360512 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004745336008024072, + "loss": 3.3062, + "theoretical_loss": 4.356884165458217, + "tokens_seen": 199426048 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047452357071213643, + "loss": 3.3419, + "theoretical_loss": 4.356704960291191, + "tokens_seen": 199491584 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047451354062186555, + "loss": 3.1005, + "theoretical_loss": 4.35652583046398, + "tokens_seen": 199557120 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004745035105315948, + "loss": 3.1586, + "theoretical_loss": 4.356346775920185, + "tokens_seen": 199622656 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047449348044132397, + "loss": 2.9514, + "theoretical_loss": 4.356167796603467, + "tokens_seen": 199688192 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047448345035105315, + "loss": 3.5216, + "theoretical_loss": 4.355988892457546, + "tokens_seen": 199753728 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004744734202607824, + "loss": 3.4248, + "theoretical_loss": 4.355810063426204, + "tokens_seen": 199819264 + }, + { + "epoch": 0.06, + "objective/train/docs_used": 272554, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.5305120944976807, + "objective/train/theoretical_loss": 4.355631309453283, + "objective/train/tokens_used": 220344800, + "theoretical_loss": 4.355631309453283, + "tokens_seen": 199884800 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047446339017051157, + "loss": 3.2918, + "theoretical_loss": 4.355631309453283, + "tokens_seen": 199884800 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047445336008024075, + "loss": 3.3689, + "theoretical_loss": 4.355452630482685, + "tokens_seen": 199950336 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047444332998996993, + "loss": 3.1337, + "theoretical_loss": 4.355274026458375, + "tokens_seen": 200015872 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004744332998996991, + "loss": 3.1743, + "theoretical_loss": 4.355095497324373, + "tokens_seen": 200081408 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004744232698094283, + "loss": 3.3082, + "theoretical_loss": 4.354917043024765, + "tokens_seen": 200146944 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047441323971915753, + "loss": 3.4852, + "theoretical_loss": 4.354738663503692, + "tokens_seen": 200212480 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047440320962888666, + "loss": 3.2587, + "theoretical_loss": 4.354560358705358, + "tokens_seen": 200278016 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004743931795386159, + "loss": 2.9646, + "theoretical_loss": 4.354382128574027, + "tokens_seen": 200343552 + }, + { + "epoch": 0.06, + "learning_rate": 0.000474383149448345, + "loss": 3.1041, + "theoretical_loss": 4.35420397305402, + "tokens_seen": 200409088 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047437311935807425, + "loss": 3.532, + "theoretical_loss": 4.35402589208972, + "tokens_seen": 200474624 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047436308926780343, + "loss": 3.3644, + "theoretical_loss": 4.353847885625571, + "tokens_seen": 200540160 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004743530591775326, + "loss": 3.3038, + "theoretical_loss": 4.353669953606072, + "tokens_seen": 200605696 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004743430290872618, + "loss": 3.1885, + "theoretical_loss": 4.353492095975787, + "tokens_seen": 200671232 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047433299899699103, + "loss": 3.1056, + "theoretical_loss": 4.353314312679333, + "tokens_seen": 200736768 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047432296890672016, + "loss": 3.1679, + "theoretical_loss": 4.353136603661392, + "tokens_seen": 200802304 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004743129388164494, + "loss": 3.3277, + "theoretical_loss": 4.352958968866704, + "tokens_seen": 200867840 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004743029087261785, + "loss": 3.3515, + "theoretical_loss": 4.352781408240065, + "tokens_seen": 200933376 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047429287863590776, + "loss": 3.4612, + "theoretical_loss": 4.352603921726334, + "tokens_seen": 200998912 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047428284854563694, + "loss": 3.2698, + "theoretical_loss": 4.352426509270425, + "tokens_seen": 201064448 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004742728184553661, + "loss": 3.2832, + "theoretical_loss": 4.352249170817315, + "tokens_seen": 201129984 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004742627883650953, + "loss": 3.2757, + "theoretical_loss": 4.352071906312037, + "tokens_seen": 201195520 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004742527582748245, + "loss": 3.2153, + "theoretical_loss": 4.351894715699684, + "tokens_seen": 201261056 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047424272818455366, + "loss": 3.5588, + "theoretical_loss": 4.351717598925406, + "tokens_seen": 201326592 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004742326980942829, + "loss": 3.1392, + "theoretical_loss": 4.351540555934414, + "tokens_seen": 201392128 + }, + { + "epoch": 0.06, + "learning_rate": 0.000474222668004012, + "loss": 3.2376, + "theoretical_loss": 4.351363586671976, + "tokens_seen": 201457664 + }, + { + "epoch": 0.06, + "objective/train/docs_used": 274046, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8291563987731934, + "objective/train/theoretical_loss": 4.351186691083417, + "objective/train/tokens_used": 221983200, + "theoretical_loss": 4.351186691083417, + "tokens_seen": 201523200 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047421263791374126, + "loss": 3.179, + "theoretical_loss": 4.351186691083417, + "tokens_seen": 201523200 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004742026078234704, + "loss": 3.047, + "theoretical_loss": 4.351009869114124, + "tokens_seen": 201588736 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004741925777331996, + "loss": 3.2204, + "theoretical_loss": 4.350833120709539, + "tokens_seen": 201654272 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004741825476429288, + "loss": 3.2638, + "theoretical_loss": 4.350656445815164, + "tokens_seen": 201719808 + }, + { + "epoch": 0.06, + "learning_rate": 0.000474172517552658, + "loss": 3.3704, + "theoretical_loss": 4.350479844376557, + "tokens_seen": 201785344 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047416248746238716, + "loss": 3.2403, + "theoretical_loss": 4.350303316339337, + "tokens_seen": 201850880 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004741524573721164, + "loss": 3.1801, + "theoretical_loss": 4.350126861649178, + "tokens_seen": 201916416 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004741424272818455, + "loss": 3.0457, + "theoretical_loss": 4.349950480251813, + "tokens_seen": 201981952 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047413239719157476, + "loss": 3.3629, + "theoretical_loss": 4.349774172093033, + "tokens_seen": 202047488 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004741223671013039, + "loss": 3.2814, + "theoretical_loss": 4.349597937118687, + "tokens_seen": 202113024 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004741123370110331, + "loss": 3.6036, + "theoretical_loss": 4.3494217752746795, + "tokens_seen": 202178560 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004741023069207623, + "loss": 3.2269, + "theoretical_loss": 4.349245686506976, + "tokens_seen": 202244096 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004740922768304915, + "loss": 3.4228, + "theoretical_loss": 4.349069670761597, + "tokens_seen": 202309632 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047408224674022067, + "loss": 3.4884, + "theoretical_loss": 4.348893727984619, + "tokens_seen": 202375168 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047407221664994985, + "loss": 3.2129, + "theoretical_loss": 4.348717858122178, + "tokens_seen": 202440704 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047406218655967903, + "loss": 3.1145, + "theoretical_loss": 4.348542061120469, + "tokens_seen": 202506240 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047405215646940827, + "loss": 3.4848, + "theoretical_loss": 4.348366336925739, + "tokens_seen": 202571776 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004740421263791374, + "loss": 3.3138, + "theoretical_loss": 4.3481906854842975, + "tokens_seen": 202637312 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047403209628886663, + "loss": 3.3594, + "theoretical_loss": 4.348015106742507, + "tokens_seen": 202702848 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047402206619859575, + "loss": 3.4342, + "theoretical_loss": 4.347839600646786, + "tokens_seen": 202768384 + }, + { + "epoch": 0.06, + "learning_rate": 0.000474012036108325, + "loss": 3.2728, + "theoretical_loss": 4.347664167143615, + "tokens_seen": 202833920 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047400200601805417, + "loss": 3.491, + "theoretical_loss": 4.347488806179528, + "tokens_seen": 202899456 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047399197592778335, + "loss": 3.2498, + "theoretical_loss": 4.347313517701114, + "tokens_seen": 202964992 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047398194583751253, + "loss": 3.2725, + "theoretical_loss": 4.347138301655021, + "tokens_seen": 203030528 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047397191574724177, + "loss": 3.2918, + "theoretical_loss": 4.346963157987954, + "tokens_seen": 203096064 + }, + { + "epoch": 0.06, + "objective/train/docs_used": 274684, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.094592332839966, + "objective/train/theoretical_loss": 4.346788086646671, + "objective/train/tokens_used": 223621600, + "theoretical_loss": 4.346788086646671, + "tokens_seen": 203161600 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004739618856569709, + "loss": 3.2821, + "theoretical_loss": 4.346788086646671, + "tokens_seen": 203161600 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047395185556670013, + "loss": 3.1994, + "theoretical_loss": 4.346613087577991, + "tokens_seen": 203227136 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047394182547642926, + "loss": 3.276, + "theoretical_loss": 4.346438160728785, + "tokens_seen": 203292672 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004739317953861585, + "loss": 3.4814, + "theoretical_loss": 4.346263306045983, + "tokens_seen": 203358208 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004739217652958877, + "loss": 3.3933, + "theoretical_loss": 4.346088523476569, + "tokens_seen": 203423744 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047391173520561686, + "loss": 3.1577, + "theoretical_loss": 4.345913812967584, + "tokens_seen": 203489280 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047390170511534604, + "loss": 3.2551, + "theoretical_loss": 4.345739174466127, + "tokens_seen": 203554816 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004738916750250752, + "loss": 3.3351, + "theoretical_loss": 4.345564607919348, + "tokens_seen": 203620352 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004738816449348044, + "loss": 3.2087, + "theoretical_loss": 4.3453901132744575, + "tokens_seen": 203685888 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047387161484453363, + "loss": 3.5707, + "theoretical_loss": 4.345215690478719, + "tokens_seen": 203751424 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047386158475426276, + "loss": 3.2015, + "theoretical_loss": 4.345041339479453, + "tokens_seen": 203816960 + }, + { + "epoch": 0.06, + "learning_rate": 0.000473851554663992, + "loss": 3.2998, + "theoretical_loss": 4.3448670602240345, + "tokens_seen": 203882496 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004738415245737211, + "loss": 3.2642, + "theoretical_loss": 4.344692852659895, + "tokens_seen": 203948032 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047383149448345036, + "loss": 3.307, + "theoretical_loss": 4.34451871673452, + "tokens_seen": 204013568 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047382146439317954, + "loss": 3.3531, + "theoretical_loss": 4.344344652395451, + "tokens_seen": 204079104 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004738114343029087, + "loss": 3.3066, + "theoretical_loss": 4.3441706595902865, + "tokens_seen": 204144640 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004738014042126379, + "loss": 3.0495, + "theoretical_loss": 4.343996738266677, + "tokens_seen": 204210176 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047379137412236714, + "loss": 3.3121, + "theoretical_loss": 4.343822888372331, + "tokens_seen": 204275712 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047378134403209626, + "loss": 3.32, + "theoretical_loss": 4.343649109855009, + "tokens_seen": 204341248 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004737713139418255, + "loss": 3.2897, + "theoretical_loss": 4.343475402662529, + "tokens_seen": 204406784 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004737612838515546, + "loss": 3.325, + "theoretical_loss": 4.343301766742763, + "tokens_seen": 204472320 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047375125376128386, + "loss": 3.2351, + "theoretical_loss": 4.343128202043638, + "tokens_seen": 204537856 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047374122367101304, + "loss": 3.4129, + "theoretical_loss": 4.342954708513136, + "tokens_seen": 204603392 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004737311935807422, + "loss": 3.2237, + "theoretical_loss": 4.342781286099291, + "tokens_seen": 204668928 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047372116349047146, + "loss": 3.224, + "theoretical_loss": 4.3426079347501965, + "tokens_seen": 204734464 + }, + { + "epoch": 0.06, + "objective/train/docs_used": 275206, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.4942283630371094, + "objective/train/theoretical_loss": 4.342434654413995, + "objective/train/tokens_used": 225260000, + "theoretical_loss": 4.342434654413995, + "tokens_seen": 204800000 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004737111334002006, + "loss": 3.4119, + "theoretical_loss": 4.342434654413995, + "tokens_seen": 204800000 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004737011033099298, + "loss": 3.3759, + "theoretical_loss": 4.342261445038888, + "tokens_seen": 204865536 + }, + { + "epoch": 0.06, + "learning_rate": 0.000473691073219659, + "loss": 3.1406, + "theoretical_loss": 4.342088306573128, + "tokens_seen": 204931072 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004736810431293882, + "loss": 3.2741, + "theoretical_loss": 4.341915238965026, + "tokens_seen": 204996608 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047367101303911736, + "loss": 3.2403, + "theoretical_loss": 4.34174224216294, + "tokens_seen": 205062144 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004736609829488466, + "loss": 3.1585, + "theoretical_loss": 4.34156931611529, + "tokens_seen": 205127680 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004736509528585757, + "loss": 3.2647, + "theoretical_loss": 4.341396460770547, + "tokens_seen": 205193216 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047364092276830496, + "loss": 3.3709, + "theoretical_loss": 4.341223676077232, + "tokens_seen": 205258752 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004736308926780341, + "loss": 3.2875, + "theoretical_loss": 4.341050961983926, + "tokens_seen": 205324288 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004736208625877633, + "loss": 3.2638, + "theoretical_loss": 4.340878318439261, + "tokens_seen": 205389824 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004736108324974925, + "loss": 3.563, + "theoretical_loss": 4.340705745391922, + "tokens_seen": 205455360 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004736008024072217, + "loss": 3.3661, + "theoretical_loss": 4.3405332427906504, + "tokens_seen": 205520896 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047359077231695087, + "loss": 3.5278, + "theoretical_loss": 4.340360810584238, + "tokens_seen": 205586432 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047358074222668005, + "loss": 3.3486, + "theoretical_loss": 4.340188448721532, + "tokens_seen": 205651968 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047357071213640923, + "loss": 3.3382, + "theoretical_loss": 4.3400161571514335, + "tokens_seen": 205717504 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047356068204613847, + "loss": 3.156, + "theoretical_loss": 4.339843935822895, + "tokens_seen": 205783040 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004735506519558676, + "loss": 3.4644, + "theoretical_loss": 4.339671784684923, + "tokens_seen": 205848576 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047354062186559683, + "loss": 3.0953, + "theoretical_loss": 4.339499703686579, + "tokens_seen": 205914112 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047353059177532595, + "loss": 3.1301, + "theoretical_loss": 4.339327692776977, + "tokens_seen": 205979648 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004735205616850552, + "loss": 3.3351, + "theoretical_loss": 4.339155751905282, + "tokens_seen": 206045184 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047351053159478437, + "loss": 3.3348, + "theoretical_loss": 4.338983881020713, + "tokens_seen": 206110720 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047350050150451355, + "loss": 3.3371, + "theoretical_loss": 4.338812080072545, + "tokens_seen": 206176256 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047349047141424273, + "loss": 3.2751, + "theoretical_loss": 4.338640349010101, + "tokens_seen": 206241792 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047348044132397197, + "loss": 3.1762, + "theoretical_loss": 4.3384686877827585, + "tokens_seen": 206307328 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004734704112337011, + "loss": 3.3436, + "theoretical_loss": 4.338297096339951, + "tokens_seen": 206372864 + }, + { + "epoch": 0.06, + "objective/train/docs_used": 276466, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.012336015701294, + "objective/train/theoretical_loss": 4.33812557463116, + "objective/train/tokens_used": 226898400, + "theoretical_loss": 4.33812557463116, + "tokens_seen": 206438400 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047346038114343033, + "loss": 3.0366, + "theoretical_loss": 4.33812557463116, + "tokens_seen": 206438400 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047345035105315946, + "loss": 3.2347, + "theoretical_loss": 4.3379541226059235, + "tokens_seen": 206503936 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004734403209628887, + "loss": 3.1413, + "theoretical_loss": 4.337782740213827, + "tokens_seen": 206569472 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004734302908726179, + "loss": 3.0938, + "theoretical_loss": 4.337611427404514, + "tokens_seen": 206635008 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047342026078234706, + "loss": 3.0309, + "theoretical_loss": 4.337440184127679, + "tokens_seen": 206700544 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047341023069207624, + "loss": 3.3016, + "theoretical_loss": 4.337269010333065, + "tokens_seen": 206766080 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004734002006018054, + "loss": 3.315, + "theoretical_loss": 4.337097905970471, + "tokens_seen": 206831616 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004733901705115346, + "loss": 3.5151, + "theoretical_loss": 4.336926870989748, + "tokens_seen": 206897152 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047338014042126383, + "loss": 3.2054, + "theoretical_loss": 4.336755905340797, + "tokens_seen": 206962688 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047337011033099296, + "loss": 3.1878, + "theoretical_loss": 4.336585008973573, + "tokens_seen": 207028224 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004733600802407222, + "loss": 3.2425, + "theoretical_loss": 4.336414181838082, + "tokens_seen": 207093760 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004733500501504513, + "loss": 3.5244, + "theoretical_loss": 4.336243423884382, + "tokens_seen": 207159296 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047334002006018056, + "loss": 3.153, + "theoretical_loss": 4.336072735062583, + "tokens_seen": 207224832 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047332998996990974, + "loss": 3.2415, + "theoretical_loss": 4.335902115322847, + "tokens_seen": 207290368 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004733199598796389, + "loss": 3.0556, + "theoretical_loss": 4.335731564615387, + "tokens_seen": 207355904 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004733099297893681, + "loss": 3.1923, + "theoretical_loss": 4.335561082890468, + "tokens_seen": 207421440 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047329989969909734, + "loss": 3.2044, + "theoretical_loss": 4.335390670098407, + "tokens_seen": 207486976 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047328986960882646, + "loss": 3.5337, + "theoretical_loss": 4.335220326189571, + "tokens_seen": 207552512 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004732798395185557, + "loss": 3.2499, + "theoretical_loss": 4.335050051114379, + "tokens_seen": 207618048 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004732698094282848, + "loss": 3.1384, + "theoretical_loss": 4.334879844823304, + "tokens_seen": 207683584 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047325977933801406, + "loss": 3.1074, + "theoretical_loss": 4.334709707266865, + "tokens_seen": 207749120 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047324974924774324, + "loss": 3.1987, + "theoretical_loss": 4.334539638395636, + "tokens_seen": 207814656 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004732397191574724, + "loss": 3.1185, + "theoretical_loss": 4.334369638160242, + "tokens_seen": 207880192 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004732296890672016, + "loss": 3.2328, + "theoretical_loss": 4.334199706511358, + "tokens_seen": 207945728 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004732196589769308, + "loss": 3.0938, + "theoretical_loss": 4.334029843399709, + "tokens_seen": 208011264 + }, + { + "epoch": 0.06, + "objective/train/docs_used": 277178, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.992138385772705, + "objective/train/theoretical_loss": 4.333860048776074, + "objective/train/tokens_used": 228536800, + "theoretical_loss": 4.333860048776074, + "tokens_seen": 208076800 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047320962888665997, + "loss": 3.1004, + "theoretical_loss": 4.333860048776074, + "tokens_seen": 208076800 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004731995987963892, + "loss": 3.2221, + "theoretical_loss": 4.33369032259128, + "tokens_seen": 208142336 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047318956870611833, + "loss": 3.2591, + "theoretical_loss": 4.333520664796206, + "tokens_seen": 208207872 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047317953861584757, + "loss": 3.4427, + "theoretical_loss": 4.33335107534178, + "tokens_seen": 208273408 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047316950852557675, + "loss": 3.2497, + "theoretical_loss": 4.333181554178985, + "tokens_seen": 208338944 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047315947843530593, + "loss": 3.293, + "theoretical_loss": 4.3330121012588485, + "tokens_seen": 208404480 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004731494483450351, + "loss": 3.5665, + "theoretical_loss": 4.332842716532454, + "tokens_seen": 208470016 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004731394182547643, + "loss": 3.0097, + "theoretical_loss": 4.332673399950932, + "tokens_seen": 208535552 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047312938816449347, + "loss": 3.1251, + "theoretical_loss": 4.332504151465464, + "tokens_seen": 208601088 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004731193580742227, + "loss": 3.0934, + "theoretical_loss": 4.332334971027284, + "tokens_seen": 208666624 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047310932798395183, + "loss": 3.4214, + "theoretical_loss": 4.332165858587672, + "tokens_seen": 208732160 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047309929789368107, + "loss": 3.0211, + "theoretical_loss": 4.331996814097963, + "tokens_seen": 208797696 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004730892678034102, + "loss": 3.2003, + "theoretical_loss": 4.331827837509538, + "tokens_seen": 208863232 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047307923771313943, + "loss": 3.3196, + "theoretical_loss": 4.331658928773831, + "tokens_seen": 208928768 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004730692076228686, + "loss": 3.2202, + "theoretical_loss": 4.331490087842324, + "tokens_seen": 208994304 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004730591775325978, + "loss": 3.1351, + "theoretical_loss": 4.33132131466655, + "tokens_seen": 209059840 + }, + { + "epoch": 0.06, + "learning_rate": 0.000473049147442327, + "loss": 3.212, + "theoretical_loss": 4.3311526091980905, + "tokens_seen": 209125376 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047303911735205615, + "loss": 3.2426, + "theoretical_loss": 4.330983971388578, + "tokens_seen": 209190912 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047302908726178534, + "loss": 3.2021, + "theoretical_loss": 4.330815401189695, + "tokens_seen": 209256448 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047301905717151457, + "loss": 3.2868, + "theoretical_loss": 4.330646898553173, + "tokens_seen": 209321984 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004730090270812437, + "loss": 3.2098, + "theoretical_loss": 4.330478463430792, + "tokens_seen": 209387520 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047299899699097293, + "loss": 3.0737, + "theoretical_loss": 4.330310095774383, + "tokens_seen": 209453056 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004729889669007021, + "loss": 3.3302, + "theoretical_loss": 4.330141795535828, + "tokens_seen": 209518592 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004729789368104313, + "loss": 3.1422, + "theoretical_loss": 4.329973562667053, + "tokens_seen": 209584128 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047296890672016053, + "loss": 3.3357, + "theoretical_loss": 4.3298053971200385, + "tokens_seen": 209649664 + }, + { + "epoch": 0.06, + "objective/train/docs_used": 278592, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8049657344818115, + "objective/train/theoretical_loss": 4.329637298846812, + "objective/train/tokens_used": 230175200, + "theoretical_loss": 4.329637298846812, + "tokens_seen": 209715200 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047295887662988966, + "loss": 2.8862, + "theoretical_loss": 4.329637298846812, + "tokens_seen": 209715200 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004729488465396189, + "loss": 3.3877, + "theoretical_loss": 4.329469267799451, + "tokens_seen": 209780736 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004729388164493481, + "loss": 3.2828, + "theoretical_loss": 4.32930130393008, + "tokens_seen": 209846272 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047292878635907726, + "loss": 3.2585, + "theoretical_loss": 4.329133407190876, + "tokens_seen": 209911808 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047291875626880644, + "loss": 3.21, + "theoretical_loss": 4.3289655775340625, + "tokens_seen": 209977344 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004729087261785356, + "loss": 3.2672, + "theoretical_loss": 4.328797814911912, + "tokens_seen": 210042880 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004728986960882648, + "loss": 3.3022, + "theoretical_loss": 4.328630119276747, + "tokens_seen": 210108416 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047288866599799403, + "loss": 3.0099, + "theoretical_loss": 4.328462490580938, + "tokens_seen": 210173952 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047287863590772316, + "loss": 3.2968, + "theoretical_loss": 4.328294928776903, + "tokens_seen": 210239488 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004728686058174524, + "loss": 3.1646, + "theoretical_loss": 4.328127433817112, + "tokens_seen": 210305024 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004728585757271815, + "loss": 3.2345, + "theoretical_loss": 4.327960005654081, + "tokens_seen": 210370560 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047284854563691076, + "loss": 3.3589, + "theoretical_loss": 4.327792644240374, + "tokens_seen": 210436096 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047283851554663994, + "loss": 3.0922, + "theoretical_loss": 4.327625349528605, + "tokens_seen": 210501632 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004728284854563691, + "loss": 3.3172, + "theoretical_loss": 4.327458121471436, + "tokens_seen": 210567168 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004728184553660983, + "loss": 3.3431, + "theoretical_loss": 4.3272909600215765, + "tokens_seen": 210632704 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047280842527582754, + "loss": 3.1134, + "theoretical_loss": 4.327123865131786, + "tokens_seen": 210698240 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047279839518555666, + "loss": 3.1697, + "theoretical_loss": 4.326956836754871, + "tokens_seen": 210763776 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004727883650952859, + "loss": 3.4017, + "theoretical_loss": 4.326789874843685, + "tokens_seen": 210829312 + }, + { + "epoch": 0.06, + "learning_rate": 0.000472778335005015, + "loss": 3.1166, + "theoretical_loss": 4.326622979351132, + "tokens_seen": 210894848 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047276830491474426, + "loss": 3.0739, + "theoretical_loss": 4.326456150230163, + "tokens_seen": 210960384 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047275827482447344, + "loss": 3.2434, + "theoretical_loss": 4.326289387433776, + "tokens_seen": 211025920 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004727482447342026, + "loss": 3.1457, + "theoretical_loss": 4.326122690915017, + "tokens_seen": 211091456 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004727382146439318, + "loss": 3.5019, + "theoretical_loss": 4.325956060626982, + "tokens_seen": 211156992 + }, + { + "epoch": 0.06, + "learning_rate": 0.000472728184553661, + "loss": 3.1354, + "theoretical_loss": 4.325789496522812, + "tokens_seen": 211222528 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047271815446339017, + "loss": 3.1215, + "theoretical_loss": 4.325622998555697, + "tokens_seen": 211288064 + }, + { + "epoch": 0.06, + "objective/train/docs_used": 279319, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.034355640411377, + "objective/train/theoretical_loss": 4.3254565666788745, + "objective/train/tokens_used": 231813600, + "theoretical_loss": 4.3254565666788745, + "tokens_seen": 211353600 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004727081243731194, + "loss": 3.3646, + "theoretical_loss": 4.3254565666788745, + "tokens_seen": 211353600 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047269809428284853, + "loss": 3.1249, + "theoretical_loss": 4.325290200845629, + "tokens_seen": 211419136 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047268806419257777, + "loss": 2.903, + "theoretical_loss": 4.3251239010092934, + "tokens_seen": 211484672 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047267803410230695, + "loss": 2.9262, + "theoretical_loss": 4.324957667123249, + "tokens_seen": 211550208 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047266800401203613, + "loss": 3.339, + "theoretical_loss": 4.32479149914092, + "tokens_seen": 211615744 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004726579739217653, + "loss": 3.2618, + "theoretical_loss": 4.324625397015783, + "tokens_seen": 211681280 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004726479438314945, + "loss": 3.1819, + "theoretical_loss": 4.3244593607013595, + "tokens_seen": 211746816 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047263791374122367, + "loss": 2.9808, + "theoretical_loss": 4.324293390151218, + "tokens_seen": 211812352 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004726278836509529, + "loss": 3.3896, + "theoretical_loss": 4.324127485318975, + "tokens_seen": 211877888 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047261785356068203, + "loss": 3.5236, + "theoretical_loss": 4.323961646158294, + "tokens_seen": 211943424 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047260782347041127, + "loss": 3.0748, + "theoretical_loss": 4.323795872622884, + "tokens_seen": 212008960 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004725977933801404, + "loss": 3.2701, + "theoretical_loss": 4.323630164666502, + "tokens_seen": 212074496 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047258776328986963, + "loss": 3.0506, + "theoretical_loss": 4.323464522242954, + "tokens_seen": 212140032 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004725777331995988, + "loss": 3.3146, + "theoretical_loss": 4.323298945306089, + "tokens_seen": 212205568 + }, + { + "epoch": 0.06, + "learning_rate": 0.000472567703109328, + "loss": 3.3863, + "theoretical_loss": 4.3231334338098035, + "tokens_seen": 212271104 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004725576730190572, + "loss": 3.5083, + "theoretical_loss": 4.322967987708043, + "tokens_seen": 212336640 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047254764292878636, + "loss": 3.0861, + "theoretical_loss": 4.322802606954798, + "tokens_seen": 212402176 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047253761283851554, + "loss": 3.2103, + "theoretical_loss": 4.322637291504106, + "tokens_seen": 212467712 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047252758274824477, + "loss": 3.2339, + "theoretical_loss": 4.32247204131005, + "tokens_seen": 212533248 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004725175526579739, + "loss": 3.3825, + "theoretical_loss": 4.322306856326761, + "tokens_seen": 212598784 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047250752256770313, + "loss": 3.2205, + "theoretical_loss": 4.322141736508415, + "tokens_seen": 212664320 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004724974924774323, + "loss": 3.2196, + "theoretical_loss": 4.321976681809236, + "tokens_seen": 212729856 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004724874623871615, + "loss": 3.1189, + "theoretical_loss": 4.321811692183491, + "tokens_seen": 212795392 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004724774322968907, + "loss": 3.2039, + "theoretical_loss": 4.321646767585497, + "tokens_seen": 212860928 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047246740220661986, + "loss": 3.1653, + "theoretical_loss": 4.3214819079696145, + "tokens_seen": 212926464 + }, + { + "epoch": 0.06, + "objective/train/docs_used": 280599, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.3949620723724365, + "objective/train/theoretical_loss": 4.321317113290252, + "objective/train/tokens_used": 233452000, + "theoretical_loss": 4.321317113290252, + "tokens_seen": 212992000 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047245737211634904, + "loss": 3.3408, + "theoretical_loss": 4.321317113290252, + "tokens_seen": 212992000 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004724473420260783, + "loss": 3.1235, + "theoretical_loss": 4.321152383501863, + "tokens_seen": 213057536 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004724373119358074, + "loss": 3.2593, + "theoretical_loss": 4.320987718558945, + "tokens_seen": 213123072 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047242728184553664, + "loss": 3.3497, + "theoretical_loss": 4.320823118416046, + "tokens_seen": 213188608 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047241725175526576, + "loss": 3.5142, + "theoretical_loss": 4.320658583027755, + "tokens_seen": 213254144 + }, + { + "epoch": 0.06, + "learning_rate": 0.000472407221664995, + "loss": 3.4399, + "theoretical_loss": 4.32049411234871, + "tokens_seen": 213319680 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004723971915747242, + "loss": 3.2726, + "theoretical_loss": 4.3203297063335935, + "tokens_seen": 213385216 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047238716148445336, + "loss": 3.2783, + "theoretical_loss": 4.320165364937134, + "tokens_seen": 213450752 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047237713139418254, + "loss": 3.2021, + "theoretical_loss": 4.320001088114105, + "tokens_seen": 213516288 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004723671013039117, + "loss": 3.2311, + "theoretical_loss": 4.319836875819325, + "tokens_seen": 213581824 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004723570712136409, + "loss": 2.9671, + "theoretical_loss": 4.31967272800766, + "tokens_seen": 213647360 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047234704112337014, + "loss": 3.1442, + "theoretical_loss": 4.319508644634021, + "tokens_seen": 213712896 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047233701103309927, + "loss": 2.8623, + "theoretical_loss": 4.319344625653361, + "tokens_seen": 213778432 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004723269809428285, + "loss": 3.1813, + "theoretical_loss": 4.319180671020684, + "tokens_seen": 213843968 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004723169508525577, + "loss": 3.1773, + "theoretical_loss": 4.319016780691033, + "tokens_seen": 213909504 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047230692076228686, + "loss": 3.2263, + "theoretical_loss": 4.318852954619501, + "tokens_seen": 213975040 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047229689067201605, + "loss": 3.2179, + "theoretical_loss": 4.318689192761225, + "tokens_seen": 214040576 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004722868605817452, + "loss": 3.3982, + "theoretical_loss": 4.318525495071385, + "tokens_seen": 214106112 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004722768304914744, + "loss": 3.2654, + "theoretical_loss": 4.318361861505207, + "tokens_seen": 214171648 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047226680040120364, + "loss": 3.0422, + "theoretical_loss": 4.318198292017964, + "tokens_seen": 214237184 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047225677031093277, + "loss": 3.2805, + "theoretical_loss": 4.318034786564971, + "tokens_seen": 214302720 + }, + { + "epoch": 0.06, + "learning_rate": 0.000472246740220662, + "loss": 3.115, + "theoretical_loss": 4.31787134510159, + "tokens_seen": 214368256 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047223671013039113, + "loss": 3.4185, + "theoretical_loss": 4.3177079675832255, + "tokens_seen": 214433792 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047222668004012037, + "loss": 3.2431, + "theoretical_loss": 4.317544653965329, + "tokens_seen": 214499328 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004722166499498496, + "loss": 3.2717, + "theoretical_loss": 4.3173814042033944, + "tokens_seen": 214564864 + }, + { + "epoch": 0.07, + "objective/train/docs_used": 281081, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.1554346084594727, + "objective/train/theoretical_loss": 4.317218218252963, + "objective/train/tokens_used": 235090400, + "theoretical_loss": 4.317218218252963, + "tokens_seen": 214630400 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047220661985957873, + "loss": 3.2487, + "theoretical_loss": 4.317218218252963, + "tokens_seen": 214630400 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047219658976930797, + "loss": 3.2627, + "theoretical_loss": 4.317055096069618, + "tokens_seen": 214695936 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047218655967903715, + "loss": 3.132, + "theoretical_loss": 4.316892037608987, + "tokens_seen": 214761472 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047217652958876633, + "loss": 3.2713, + "theoretical_loss": 4.316729042826745, + "tokens_seen": 214827008 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004721664994984955, + "loss": 3.2406, + "theoretical_loss": 4.316566111678609, + "tokens_seen": 214892544 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004721564694082247, + "loss": 3.3419, + "theoretical_loss": 4.316403244120339, + "tokens_seen": 214958080 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047214643931795387, + "loss": 3.0689, + "theoretical_loss": 4.3162404401077445, + "tokens_seen": 215023616 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004721364092276831, + "loss": 3.3983, + "theoretical_loss": 4.316077699596671, + "tokens_seen": 215089152 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047212637913741223, + "loss": 3.0837, + "theoretical_loss": 4.315915022543016, + "tokens_seen": 215154688 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047211634904714147, + "loss": 3.1345, + "theoretical_loss": 4.315752408902716, + "tokens_seen": 215220224 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004721063189568706, + "loss": 3.2947, + "theoretical_loss": 4.315589858631755, + "tokens_seen": 215285760 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047209628886659983, + "loss": 3.2647, + "theoretical_loss": 4.315427371686157, + "tokens_seen": 215351296 + }, + { + "epoch": 0.07, + "learning_rate": 0.000472086258776329, + "loss": 3.1737, + "theoretical_loss": 4.315264948021994, + "tokens_seen": 215416832 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004720762286860582, + "loss": 3.1847, + "theoretical_loss": 4.315102587595379, + "tokens_seen": 215482368 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004720661985957874, + "loss": 3.0922, + "theoretical_loss": 4.31494029036247, + "tokens_seen": 215547904 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047205616850551656, + "loss": 3.287, + "theoretical_loss": 4.314778056279468, + "tokens_seen": 215613440 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047204613841524574, + "loss": 2.8694, + "theoretical_loss": 4.314615885302619, + "tokens_seen": 215678976 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047203610832497497, + "loss": 3.373, + "theoretical_loss": 4.314453777388209, + "tokens_seen": 215744512 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004720260782347041, + "loss": 3.2566, + "theoretical_loss": 4.314291732492573, + "tokens_seen": 215810048 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047201604814443333, + "loss": 2.9877, + "theoretical_loss": 4.314129750572087, + "tokens_seen": 215875584 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004720060180541625, + "loss": 3.3091, + "theoretical_loss": 4.3139678315831675, + "tokens_seen": 215941120 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004719959879638917, + "loss": 3.0783, + "theoretical_loss": 4.313805975482278, + "tokens_seen": 216006656 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004719859578736209, + "loss": 3.1844, + "theoretical_loss": 4.313644182225926, + "tokens_seen": 216072192 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047197592778335006, + "loss": 3.2515, + "theoretical_loss": 4.313482451770659, + "tokens_seen": 216137728 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047196589769307924, + "loss": 2.9306, + "theoretical_loss": 4.313320784073069, + "tokens_seen": 216203264 + }, + { + "epoch": 0.07, + "objective/train/docs_used": 282379, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.2120323181152344, + "objective/train/theoretical_loss": 4.3131591790897925, + "objective/train/tokens_used": 236728800, + "theoretical_loss": 4.3131591790897925, + "tokens_seen": 216268800 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004719558676028085, + "loss": 3.1633, + "theoretical_loss": 4.3131591790897925, + "tokens_seen": 216268800 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004719458375125376, + "loss": 3.3385, + "theoretical_loss": 4.3129976367775065, + "tokens_seen": 216334336 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047193580742226684, + "loss": 3.1109, + "theoretical_loss": 4.312836157092934, + "tokens_seen": 216399872 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047192577733199596, + "loss": 3.3772, + "theoretical_loss": 4.312674739992839, + "tokens_seen": 216465408 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004719157472417252, + "loss": 3.3361, + "theoretical_loss": 4.31251338543403, + "tokens_seen": 216530944 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004719057171514544, + "loss": 3.3251, + "theoretical_loss": 4.312352093373354, + "tokens_seen": 216596480 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047189568706118356, + "loss": 3.0503, + "theoretical_loss": 4.312190863767708, + "tokens_seen": 216662016 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047188565697091274, + "loss": 3.0291, + "theoretical_loss": 4.312029696574027, + "tokens_seen": 216727552 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004718756268806419, + "loss": 3.0042, + "theoretical_loss": 4.311868591749287, + "tokens_seen": 216793088 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004718655967903711, + "loss": 3.0656, + "theoretical_loss": 4.311707549250514, + "tokens_seen": 216858624 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047185556670010034, + "loss": 3.1287, + "theoretical_loss": 4.311546569034767, + "tokens_seen": 216924160 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047184553660982947, + "loss": 3.1405, + "theoretical_loss": 4.311385651059155, + "tokens_seen": 216989696 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004718355065195587, + "loss": 3.425, + "theoretical_loss": 4.311224795280825, + "tokens_seen": 217055232 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004718254764292879, + "loss": 3.282, + "theoretical_loss": 4.3110640016569715, + "tokens_seen": 217120768 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047181544633901706, + "loss": 3.4067, + "theoretical_loss": 4.310903270144825, + "tokens_seen": 217186304 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047180541624874625, + "loss": 3.1557, + "theoretical_loss": 4.310742600701664, + "tokens_seen": 217251840 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004717953861584754, + "loss": 3.1474, + "theoretical_loss": 4.310581993284805, + "tokens_seen": 217317376 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004717853560682046, + "loss": 3.2756, + "theoretical_loss": 4.310421447851609, + "tokens_seen": 217382912 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047177532597793384, + "loss": 3.4118, + "theoretical_loss": 4.310260964359479, + "tokens_seen": 217448448 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047176529588766297, + "loss": 3.2428, + "theoretical_loss": 4.310100542765858, + "tokens_seen": 217513984 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004717552657973922, + "loss": 3.2141, + "theoretical_loss": 4.309940183028236, + "tokens_seen": 217579520 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047174523570712133, + "loss": 3.3118, + "theoretical_loss": 4.309779885104139, + "tokens_seen": 217645056 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047173520561685057, + "loss": 3.276, + "theoretical_loss": 4.309619648951139, + "tokens_seen": 217710592 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047172517552657975, + "loss": 2.8709, + "theoretical_loss": 4.3094594745268475, + "tokens_seen": 217776128 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047171514543630893, + "loss": 3.3411, + "theoretical_loss": 4.30929936178892, + "tokens_seen": 217841664 + }, + { + "epoch": 0.07, + "objective/train/docs_used": 283045, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.1448984146118164, + "objective/train/theoretical_loss": 4.309139310695053, + "objective/train/tokens_used": 238367200, + "theoretical_loss": 4.309139310695053, + "tokens_seen": 217907200 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004717051153460381, + "loss": 3.247, + "theoretical_loss": 4.309139310695053, + "tokens_seen": 217907200 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047169508525576735, + "loss": 2.9108, + "theoretical_loss": 4.308979321202983, + "tokens_seen": 217972736 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004716850551654965, + "loss": 3.2046, + "theoretical_loss": 4.308819393270491, + "tokens_seen": 218038272 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004716750250752257, + "loss": 3.076, + "theoretical_loss": 4.308659526855396, + "tokens_seen": 218103808 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047166499498495484, + "loss": 3.245, + "theoretical_loss": 4.308499721915563, + "tokens_seen": 218169344 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047165496489468407, + "loss": 3.2956, + "theoretical_loss": 4.308339978408897, + "tokens_seen": 218234880 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047164493480441325, + "loss": 3.0581, + "theoretical_loss": 4.308180296293341, + "tokens_seen": 218300416 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047163490471414243, + "loss": 3.1606, + "theoretical_loss": 4.308020675526883, + "tokens_seen": 218365952 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004716248746238716, + "loss": 3.2829, + "theoretical_loss": 4.307861116067554, + "tokens_seen": 218431488 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004716148445336008, + "loss": 3.3331, + "theoretical_loss": 4.30770161787342, + "tokens_seen": 218497024 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047160481444333, + "loss": 3.2521, + "theoretical_loss": 4.307542180902594, + "tokens_seen": 218562560 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004715947843530592, + "loss": 3.0335, + "theoretical_loss": 4.307382805113228, + "tokens_seen": 218628096 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047158475426278834, + "loss": 2.9783, + "theoretical_loss": 4.307223490463516, + "tokens_seen": 218693632 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004715747241725176, + "loss": 3.1517, + "theoretical_loss": 4.307064236911692, + "tokens_seen": 218759168 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004715646940822467, + "loss": 3.002, + "theoretical_loss": 4.30690504441603, + "tokens_seen": 218824704 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047155466399197594, + "loss": 3.13, + "theoretical_loss": 4.306745912934849, + "tokens_seen": 218890240 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004715446339017051, + "loss": 3.2757, + "theoretical_loss": 4.306586842426504, + "tokens_seen": 218955776 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004715346038114343, + "loss": 3.1729, + "theoretical_loss": 4.306427832849394, + "tokens_seen": 219021312 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004715245737211635, + "loss": 2.8273, + "theoretical_loss": 4.306268884161959, + "tokens_seen": 219086848 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004715145436308927, + "loss": 3.0639, + "theoretical_loss": 4.306109996322679, + "tokens_seen": 219152384 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047150451354062184, + "loss": 3.2826, + "theoretical_loss": 4.305951169290073, + "tokens_seen": 219217920 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004714944834503511, + "loss": 3.2331, + "theoretical_loss": 4.305792403022703, + "tokens_seen": 219283456 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004714844533600802, + "loss": 3.4157, + "theoretical_loss": 4.305633697479171, + "tokens_seen": 219348992 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047147442326980944, + "loss": 3.2171, + "theoretical_loss": 4.305475052618119, + "tokens_seen": 219414528 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004714643931795387, + "loss": 3.271, + "theoretical_loss": 4.30531646839823, + "tokens_seen": 219480064 + }, + { + "epoch": 0.07, + "objective/train/docs_used": 284124, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.518115997314453, + "objective/train/theoretical_loss": 4.305157944778228, + "objective/train/tokens_used": 240005600, + "theoretical_loss": 4.305157944778228, + "tokens_seen": 219545600 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004714543630892678, + "loss": 3.2974, + "theoretical_loss": 4.305157944778228, + "tokens_seen": 219545600 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047144433299899704, + "loss": 3.1396, + "theoretical_loss": 4.304999481716876, + "tokens_seen": 219611136 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047143430290872616, + "loss": 3.3291, + "theoretical_loss": 4.304841079172979, + "tokens_seen": 219676672 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004714242728184554, + "loss": 3.3637, + "theoretical_loss": 4.30468273710538, + "tokens_seen": 219742208 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004714142427281846, + "loss": 3.22, + "theoretical_loss": 4.304524455472965, + "tokens_seen": 219807744 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047140421263791376, + "loss": 3.2553, + "theoretical_loss": 4.304366234234659, + "tokens_seen": 219873280 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047139418254764294, + "loss": 3.2947, + "theoretical_loss": 4.304208073349426, + "tokens_seen": 219938816 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004713841524573721, + "loss": 3.2111, + "theoretical_loss": 4.304049972776271, + "tokens_seen": 220004352 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004713741223671013, + "loss": 3.2684, + "theoretical_loss": 4.30389193247424, + "tokens_seen": 220069888 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047136409227683054, + "loss": 3.1529, + "theoretical_loss": 4.303733952402419, + "tokens_seen": 220135424 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047135406218655967, + "loss": 3.1474, + "theoretical_loss": 4.303576032519931, + "tokens_seen": 220200960 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004713440320962889, + "loss": 3.0742, + "theoretical_loss": 4.303418172785943, + "tokens_seen": 220266496 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004713340020060181, + "loss": 3.1944, + "theoretical_loss": 4.303260373159659, + "tokens_seen": 220332032 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047132397191574726, + "loss": 3.0874, + "theoretical_loss": 4.303102633600322, + "tokens_seen": 220397568 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047131394182547645, + "loss": 3.0249, + "theoretical_loss": 4.30294495406722, + "tokens_seen": 220463104 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047130391173520563, + "loss": 3.434, + "theoretical_loss": 4.3027873345196745, + "tokens_seen": 220528640 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004712938816449348, + "loss": 3.4907, + "theoretical_loss": 4.302629774917049, + "tokens_seen": 220594176 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047128385155466404, + "loss": 3.1007, + "theoretical_loss": 4.302472275218748, + "tokens_seen": 220659712 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047127382146439317, + "loss": 3.3572, + "theoretical_loss": 4.302314835384214, + "tokens_seen": 220725248 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004712637913741224, + "loss": 3.2629, + "theoretical_loss": 4.30215745537293, + "tokens_seen": 220790784 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047125376128385153, + "loss": 3.0362, + "theoretical_loss": 4.302000135144416, + "tokens_seen": 220856320 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047124373119358077, + "loss": 3.1044, + "theoretical_loss": 4.301842874658235, + "tokens_seen": 220921856 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047123370110330995, + "loss": 3.2226, + "theoretical_loss": 4.301685673873987, + "tokens_seen": 220987392 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047122367101303913, + "loss": 3.2153, + "theoretical_loss": 4.301528532751312, + "tokens_seen": 221052928 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004712136409227683, + "loss": 3.5554, + "theoretical_loss": 4.301371451249888, + "tokens_seen": 221118464 + }, + { + "epoch": 0.07, + "objective/train/docs_used": 284674, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.90860652923584, + "objective/train/theoretical_loss": 4.301214429329433, + "objective/train/tokens_used": 241644000, + "theoretical_loss": 4.301214429329433, + "tokens_seen": 221184000 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047120361083249755, + "loss": 3.2309, + "theoretical_loss": 4.301214429329433, + "tokens_seen": 221184000 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004711935807422267, + "loss": 3.2685, + "theoretical_loss": 4.301057466949707, + "tokens_seen": 221249536 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004711835506519559, + "loss": 3.1091, + "theoretical_loss": 4.300900564070504, + "tokens_seen": 221315072 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047117352056168504, + "loss": 3.0911, + "theoretical_loss": 4.30074372065166, + "tokens_seen": 221380608 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047116349047141427, + "loss": 3.3026, + "theoretical_loss": 4.300586936653049, + "tokens_seen": 221446144 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047115346038114345, + "loss": 3.2759, + "theoretical_loss": 4.300430212034587, + "tokens_seen": 221511680 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047114343029087263, + "loss": 3.1257, + "theoretical_loss": 4.300273546756223, + "tokens_seen": 221577216 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004711334002006018, + "loss": 3.1675, + "theoretical_loss": 4.300116940777951, + "tokens_seen": 221642752 + }, + { + "epoch": 0.07, + "learning_rate": 0.000471123370110331, + "loss": 2.9268, + "theoretical_loss": 4.299960394059799, + "tokens_seen": 221708288 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004711133400200602, + "loss": 3.1183, + "theoretical_loss": 4.299803906561835, + "tokens_seen": 221773824 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004711033099297894, + "loss": 3.3301, + "theoretical_loss": 4.29964747824417, + "tokens_seen": 221839360 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047109327983951854, + "loss": 3.0193, + "theoretical_loss": 4.299491109066947, + "tokens_seen": 221904896 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004710832497492478, + "loss": 3.4411, + "theoretical_loss": 4.299334798990351, + "tokens_seen": 221970432 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004710732196589769, + "loss": 3.2296, + "theoretical_loss": 4.2991785479746065, + "tokens_seen": 222035968 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047106318956870614, + "loss": 3.0854, + "theoretical_loss": 4.299022355979974, + "tokens_seen": 222101504 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004710531594784353, + "loss": 3.2255, + "theoretical_loss": 4.298866222966755, + "tokens_seen": 222167040 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004710431293881645, + "loss": 3.2013, + "theoretical_loss": 4.298710148895286, + "tokens_seen": 222232576 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004710330992978937, + "loss": 3.2448, + "theoretical_loss": 4.298554133725946, + "tokens_seen": 222298112 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004710230692076229, + "loss": 3.1119, + "theoretical_loss": 4.298398177419149, + "tokens_seen": 222363648 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047101303911735204, + "loss": 3.1882, + "theoretical_loss": 4.298242279935349, + "tokens_seen": 222429184 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004710030090270813, + "loss": 3.0531, + "theoretical_loss": 4.2980864412350375, + "tokens_seen": 222494720 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004709929789368104, + "loss": 3.0501, + "theoretical_loss": 4.297930661278745, + "tokens_seen": 222560256 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047098294884653964, + "loss": 3.1275, + "theoretical_loss": 4.297774940027038, + "tokens_seen": 222625792 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004709729187562688, + "loss": 3.3449, + "theoretical_loss": 4.297619277440523, + "tokens_seen": 222691328 + }, + { + "epoch": 0.07, + "learning_rate": 0.000470962888665998, + "loss": 3.2312, + "theoretical_loss": 4.297463673479846, + "tokens_seen": 222756864 + }, + { + "epoch": 0.07, + "objective/train/docs_used": 286126, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6270840167999268, + "objective/train/theoretical_loss": 4.297308128105687, + "objective/train/tokens_used": 243282400, + "theoretical_loss": 4.297308128105687, + "tokens_seen": 222822400 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004709528585757272, + "loss": 3.1513, + "theoretical_loss": 4.297308128105687, + "tokens_seen": 222822400 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047094282848545636, + "loss": 3.1221, + "theoretical_loss": 4.297152641278767, + "tokens_seen": 222887936 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047093279839518555, + "loss": 3.2143, + "theoretical_loss": 4.296997212959842, + "tokens_seen": 222953472 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004709227683049148, + "loss": 3.0636, + "theoretical_loss": 4.296841843109711, + "tokens_seen": 223019008 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004709127382146439, + "loss": 3.2735, + "theoretical_loss": 4.296686531689204, + "tokens_seen": 223084544 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047090270812437314, + "loss": 2.9135, + "theoretical_loss": 4.296531278659193, + "tokens_seen": 223150080 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047089267803410227, + "loss": 3.1243, + "theoretical_loss": 4.296376083980589, + "tokens_seen": 223215616 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004708826479438315, + "loss": 3.0309, + "theoretical_loss": 4.296220947614337, + "tokens_seen": 223281152 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004708726178535607, + "loss": 3.154, + "theoretical_loss": 4.296065869521421, + "tokens_seen": 223346688 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047086258776328987, + "loss": 2.8156, + "theoretical_loss": 4.295910849662862, + "tokens_seen": 223412224 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047085255767301905, + "loss": 3.0104, + "theoretical_loss": 4.2957558879997215, + "tokens_seen": 223477760 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004708425275827483, + "loss": 3.1348, + "theoretical_loss": 4.295600984493093, + "tokens_seen": 223543296 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004708324974924774, + "loss": 3.047, + "theoretical_loss": 4.295446139104112, + "tokens_seen": 223608832 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047082246740220665, + "loss": 3.3838, + "theoretical_loss": 4.295291351793951, + "tokens_seen": 223674368 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004708124373119358, + "loss": 3.0171, + "theoretical_loss": 4.295136622523817, + "tokens_seen": 223739904 + }, + { + "epoch": 0.07, + "learning_rate": 0.000470802407221665, + "loss": 3.3862, + "theoretical_loss": 4.294981951254956, + "tokens_seen": 223805440 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004707923771313942, + "loss": 3.1766, + "theoretical_loss": 4.294827337948651, + "tokens_seen": 223870976 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047078234704112337, + "loss": 3.0689, + "theoretical_loss": 4.294672782566224, + "tokens_seen": 223936512 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047077231695085255, + "loss": 3.1193, + "theoretical_loss": 4.29451828506903, + "tokens_seen": 224002048 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047076228686058173, + "loss": 3.4053, + "theoretical_loss": 4.294363845418465, + "tokens_seen": 224067584 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004707522567703109, + "loss": 3.1876, + "theoretical_loss": 4.29420946357596, + "tokens_seen": 224133120 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047074222668004015, + "loss": 3.2133, + "theoretical_loss": 4.294055139502985, + "tokens_seen": 224198656 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004707321965897693, + "loss": 3.2064, + "theoretical_loss": 4.293900873161043, + "tokens_seen": 224264192 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004707221664994985, + "loss": 3.295, + "theoretical_loss": 4.293746664511678, + "tokens_seen": 224329728 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047071213640922775, + "loss": 2.9466, + "theoretical_loss": 4.293592513516469, + "tokens_seen": 224395264 + }, + { + "epoch": 0.07, + "objective/train/docs_used": 286836, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.5235793590545654, + "objective/train/theoretical_loss": 4.293438420137031, + "objective/train/tokens_used": 244920800, + "theoretical_loss": 4.293438420137031, + "tokens_seen": 224460800 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004707021063189569, + "loss": 3.2967, + "theoretical_loss": 4.293438420137031, + "tokens_seen": 224460800 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004706920762286861, + "loss": 3.1972, + "theoretical_loss": 4.293284384335017, + "tokens_seen": 224526336 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047068204613841524, + "loss": 3.2584, + "theoretical_loss": 4.293130406072118, + "tokens_seen": 224591872 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047067201604814447, + "loss": 3.1142, + "theoretical_loss": 4.292976485310057, + "tokens_seen": 224657408 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047066198595787365, + "loss": 3.1867, + "theoretical_loss": 4.2928226220106005, + "tokens_seen": 224722944 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047065195586760283, + "loss": 3.1763, + "theoretical_loss": 4.292668816135545, + "tokens_seen": 224788480 + }, + { + "epoch": 0.07, + "learning_rate": 0.000470641925777332, + "loss": 3.5007, + "theoretical_loss": 4.292515067646727, + "tokens_seen": 224854016 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004706318956870612, + "loss": 2.8707, + "theoretical_loss": 4.29236137650602, + "tokens_seen": 224919552 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004706218655967904, + "loss": 3.2017, + "theoretical_loss": 4.2922077426753305, + "tokens_seen": 224985088 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004706118355065196, + "loss": 3.0439, + "theoretical_loss": 4.292054166116605, + "tokens_seen": 225050624 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047060180541624874, + "loss": 3.2209, + "theoretical_loss": 4.291900646791825, + "tokens_seen": 225116160 + }, + { + "epoch": 0.07, + "learning_rate": 0.000470591775325978, + "loss": 3.2066, + "theoretical_loss": 4.2917471846630075, + "tokens_seen": 225181696 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004705817452357071, + "loss": 2.9519, + "theoretical_loss": 4.291593779692207, + "tokens_seen": 225247232 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047057171514543634, + "loss": 3.1154, + "theoretical_loss": 4.291440431841513, + "tokens_seen": 225312768 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004705616850551655, + "loss": 3.181, + "theoretical_loss": 4.291287141073053, + "tokens_seen": 225378304 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004705516549648947, + "loss": 2.9915, + "theoretical_loss": 4.291133907348989, + "tokens_seen": 225443840 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004705416248746239, + "loss": 3.1199, + "theoretical_loss": 4.29098073063152, + "tokens_seen": 225509376 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004705315947843531, + "loss": 3.3002, + "theoretical_loss": 4.29082761088288, + "tokens_seen": 225574912 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047052156469408224, + "loss": 2.7099, + "theoretical_loss": 4.290674548065338, + "tokens_seen": 225640448 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004705115346038115, + "loss": 3.1867, + "theoretical_loss": 4.290521542141203, + "tokens_seen": 225705984 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004705015045135406, + "loss": 3.0885, + "theoretical_loss": 4.290368593072817, + "tokens_seen": 225771520 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047049147442326984, + "loss": 3.1099, + "theoretical_loss": 4.290215700822556, + "tokens_seen": 225837056 + }, + { + "epoch": 0.07, + "learning_rate": 0.000470481444332999, + "loss": 3.1279, + "theoretical_loss": 4.290062865352837, + "tokens_seen": 225902592 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004704714142427282, + "loss": 3.0228, + "theoretical_loss": 4.289910086626108, + "tokens_seen": 225968128 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004704613841524574, + "loss": 3.3652, + "theoretical_loss": 4.289757364604855, + "tokens_seen": 226033664 + }, + { + "epoch": 0.07, + "objective/train/docs_used": 288143, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.2444591522216797, + "objective/train/theoretical_loss": 4.2896046992515995, + "objective/train/tokens_used": 246559200, + "theoretical_loss": 4.2896046992515995, + "tokens_seen": 226099200 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047045135406218656, + "loss": 3.2585, + "theoretical_loss": 4.2896046992515995, + "tokens_seen": 226099200 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047044132397191575, + "loss": 2.841, + "theoretical_loss": 4.289452090528897, + "tokens_seen": 226164736 + }, + { + "epoch": 0.07, + "learning_rate": 0.000470431293881645, + "loss": 3.1428, + "theoretical_loss": 4.289299538399341, + "tokens_seen": 226230272 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004704212637913741, + "loss": 3.229, + "theoretical_loss": 4.28914704282556, + "tokens_seen": 226295808 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047041123370110334, + "loss": 3.003, + "theoretical_loss": 4.288994603770215, + "tokens_seen": 226361344 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047040120361083247, + "loss": 3.1008, + "theoretical_loss": 4.288842221196007, + "tokens_seen": 226426880 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004703911735205617, + "loss": 3.0763, + "theoretical_loss": 4.28868989506567, + "tokens_seen": 226492416 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004703811434302909, + "loss": 3.1798, + "theoretical_loss": 4.288537625341974, + "tokens_seen": 226557952 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047037111334002007, + "loss": 3.5773, + "theoretical_loss": 4.288385411987722, + "tokens_seen": 226623488 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047036108324974925, + "loss": 3.156, + "theoretical_loss": 4.288233254965755, + "tokens_seen": 226689024 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004703510531594785, + "loss": 2.9742, + "theoretical_loss": 4.2880811542389505, + "tokens_seen": 226754560 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004703410230692076, + "loss": 2.9553, + "theoretical_loss": 4.287929109770217, + "tokens_seen": 226820096 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047033099297893685, + "loss": 3.1251, + "theoretical_loss": 4.287777121522501, + "tokens_seen": 226885632 + }, + { + "epoch": 0.07, + "learning_rate": 0.000470320962888666, + "loss": 3.0942, + "theoretical_loss": 4.287625189458781, + "tokens_seen": 226951168 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004703109327983952, + "loss": 3.2267, + "theoretical_loss": 4.287473313542077, + "tokens_seen": 227016704 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004703009027081244, + "loss": 3.0715, + "theoretical_loss": 4.287321493735438, + "tokens_seen": 227082240 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047029087261785357, + "loss": 3.2224, + "theoretical_loss": 4.287169730001949, + "tokens_seen": 227147776 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047028084252758275, + "loss": 3.1082, + "theoretical_loss": 4.287018022304733, + "tokens_seen": 227213312 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047027081243731193, + "loss": 3.2017, + "theoretical_loss": 4.286866370606943, + "tokens_seen": 227278848 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004702607823470411, + "loss": 2.8765, + "theoretical_loss": 4.286714774871772, + "tokens_seen": 227344384 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047025075225677035, + "loss": 2.7387, + "theoretical_loss": 4.286563235062444, + "tokens_seen": 227409920 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004702407221664995, + "loss": 2.8634, + "theoretical_loss": 4.28641175114222, + "tokens_seen": 227475456 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004702306920762287, + "loss": 2.7575, + "theoretical_loss": 4.286260323074394, + "tokens_seen": 227540992 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047022066198595784, + "loss": 3.0389, + "theoretical_loss": 4.286108950822296, + "tokens_seen": 227606528 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004702106318956871, + "loss": 3.0842, + "theoretical_loss": 4.285957634349289, + "tokens_seen": 227672064 + }, + { + "epoch": 0.07, + "objective/train/docs_used": 288537, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.2806308269500732, + "objective/train/theoretical_loss": 4.285806373618774, + "objective/train/tokens_used": 248197600, + "theoretical_loss": 4.285806373618774, + "tokens_seen": 227737600 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047020060180541626, + "loss": 3.402, + "theoretical_loss": 4.285806373618774, + "tokens_seen": 227737600 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047019057171514544, + "loss": 3.2526, + "theoretical_loss": 4.285655168594182, + "tokens_seen": 227803136 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004701805416248746, + "loss": 3.2338, + "theoretical_loss": 4.285504019238982, + "tokens_seen": 227868672 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047017051153460385, + "loss": 3.1311, + "theoretical_loss": 4.285352925516676, + "tokens_seen": 227934208 + }, + { + "epoch": 0.07, + "learning_rate": 0.000470160481444333, + "loss": 3.2231, + "theoretical_loss": 4.2852018873908, + "tokens_seen": 227999744 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004701504513540622, + "loss": 3.1156, + "theoretical_loss": 4.285050904824925, + "tokens_seen": 228065280 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047014042126379134, + "loss": 3.2022, + "theoretical_loss": 4.284899977782658, + "tokens_seen": 228130816 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004701303911735206, + "loss": 3.1321, + "theoretical_loss": 4.284749106227636, + "tokens_seen": 228196352 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047012036108324976, + "loss": 3.1009, + "theoretical_loss": 4.284598290123535, + "tokens_seen": 228261888 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047011033099297894, + "loss": 3.2566, + "theoretical_loss": 4.284447529434061, + "tokens_seen": 228327424 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004701003009027081, + "loss": 3.1493, + "theoretical_loss": 4.284296824122959, + "tokens_seen": 228392960 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004700902708124373, + "loss": 3.1941, + "theoretical_loss": 4.284146174154003, + "tokens_seen": 228458496 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004700802407221665, + "loss": 2.9355, + "theoretical_loss": 4.283995579491004, + "tokens_seen": 228524032 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004700702106318957, + "loss": 3.098, + "theoretical_loss": 4.283845040097807, + "tokens_seen": 228589568 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047006018054162484, + "loss": 3.013, + "theoretical_loss": 4.28369455593829, + "tokens_seen": 228655104 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004700501504513541, + "loss": 3.2876, + "theoretical_loss": 4.2835441269763646, + "tokens_seen": 228720640 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047004012036108326, + "loss": 3.1562, + "theoretical_loss": 4.283393753175979, + "tokens_seen": 228786176 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047003009027081244, + "loss": 3.1474, + "theoretical_loss": 4.283243434501112, + "tokens_seen": 228851712 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004700200601805416, + "loss": 3.1408, + "theoretical_loss": 4.283093170915778, + "tokens_seen": 228917248 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004700100300902708, + "loss": 3.0895, + "theoretical_loss": 4.282942962384023, + "tokens_seen": 228982784 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047, + "loss": 3.1473, + "theoretical_loss": 4.282792808869932, + "tokens_seen": 229048320 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004699899699097292, + "loss": 3.2697, + "theoretical_loss": 4.282642710337618, + "tokens_seen": 229113856 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046997993981945835, + "loss": 2.9681, + "theoretical_loss": 4.28249266675123, + "tokens_seen": 229179392 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004699699097291876, + "loss": 3.0361, + "theoretical_loss": 4.282342678074951, + "tokens_seen": 229244928 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046995987963891676, + "loss": 3.0598, + "theoretical_loss": 4.2821927442729955, + "tokens_seen": 229310464 + }, + { + "epoch": 0.07, + "objective/train/docs_used": 289830, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.521411418914795, + "objective/train/theoretical_loss": 4.282042865309616, + "objective/train/tokens_used": 249836000, + "theoretical_loss": 4.282042865309616, + "tokens_seen": 229376000 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046994984954864595, + "loss": 3.1125, + "theoretical_loss": 4.282042865309616, + "tokens_seen": 229376000 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004699398194583752, + "loss": 2.9385, + "theoretical_loss": 4.281893041149093, + "tokens_seen": 229441536 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004699297893681043, + "loss": 3.3297, + "theoretical_loss": 4.2817432717557455, + "tokens_seen": 229507072 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046991975927783354, + "loss": 2.8433, + "theoretical_loss": 4.28159355709392, + "tokens_seen": 229572608 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046990972918756267, + "loss": 3.0393, + "theoretical_loss": 4.281443897128004, + "tokens_seen": 229638144 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004698996990972919, + "loss": 3.0533, + "theoretical_loss": 4.2812942918224115, + "tokens_seen": 229703680 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004698896690070211, + "loss": 3.0605, + "theoretical_loss": 4.281144741141593, + "tokens_seen": 229769216 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046987963891675027, + "loss": 3.2703, + "theoretical_loss": 4.280995245050032, + "tokens_seen": 229834752 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046986960882647945, + "loss": 3.296, + "theoretical_loss": 4.2808458035122445, + "tokens_seen": 229900288 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004698595787362087, + "loss": 3.1057, + "theoretical_loss": 4.2806964164927805, + "tokens_seen": 229965824 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004698495486459378, + "loss": 3.2996, + "theoretical_loss": 4.280547083956224, + "tokens_seen": 230031360 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046983951855566705, + "loss": 3.1414, + "theoretical_loss": 4.280397805867188, + "tokens_seen": 230096896 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004698294884653962, + "loss": 2.9225, + "theoretical_loss": 4.280248582190324, + "tokens_seen": 230162432 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004698194583751254, + "loss": 3.1511, + "theoretical_loss": 4.280099412890312, + "tokens_seen": 230227968 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004698094282848546, + "loss": 3.36, + "theoretical_loss": 4.279950297931869, + "tokens_seen": 230293504 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046979939819458377, + "loss": 3.1327, + "theoretical_loss": 4.27980123727974, + "tokens_seen": 230359040 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046978936810431295, + "loss": 3.1655, + "theoretical_loss": 4.279652230898709, + "tokens_seen": 230424576 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046977933801404213, + "loss": 2.9951, + "theoretical_loss": 4.279503278753586, + "tokens_seen": 230490112 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004697693079237713, + "loss": 3.1871, + "theoretical_loss": 4.27935438080922, + "tokens_seen": 230555648 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046975927783350055, + "loss": 3.413, + "theoretical_loss": 4.27920553703049, + "tokens_seen": 230621184 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004697492477432297, + "loss": 3.0407, + "theoretical_loss": 4.279056747382306, + "tokens_seen": 230686720 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004697392176529589, + "loss": 3.2621, + "theoretical_loss": 4.278908011829613, + "tokens_seen": 230752256 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046972918756268804, + "loss": 3.0509, + "theoretical_loss": 4.27875933033739, + "tokens_seen": 230817792 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004697191574724173, + "loss": 3.1669, + "theoretical_loss": 4.278610702870646, + "tokens_seen": 230883328 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046970912738214646, + "loss": 3.0121, + "theoretical_loss": 4.278462129394423, + "tokens_seen": 230948864 + }, + { + "epoch": 0.07, + "objective/train/docs_used": 290404, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.064453125, + "objective/train/theoretical_loss": 4.278313609873795, + "objective/train/tokens_used": 251474400, + "theoretical_loss": 4.278313609873795, + "tokens_seen": 231014400 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046969909729187564, + "loss": 3.1929, + "theoretical_loss": 4.278313609873795, + "tokens_seen": 231014400 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004696890672016048, + "loss": 3.1449, + "theoretical_loss": 4.278165144273871, + "tokens_seen": 231079936 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046967903711133405, + "loss": 3.1433, + "theoretical_loss": 4.27801673255979, + "tokens_seen": 231145472 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004696690070210632, + "loss": 3.1905, + "theoretical_loss": 4.277868374696725, + "tokens_seen": 231211008 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004696589769307924, + "loss": 3.2421, + "theoretical_loss": 4.277720070649879, + "tokens_seen": 231276544 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046964894684052154, + "loss": 3.3793, + "theoretical_loss": 4.277571820384491, + "tokens_seen": 231342080 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004696389167502508, + "loss": 3.0555, + "theoretical_loss": 4.277423623865829, + "tokens_seen": 231407616 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046962888665997996, + "loss": 3.0292, + "theoretical_loss": 4.277275481059195, + "tokens_seen": 231473152 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046961885656970914, + "loss": 3.2957, + "theoretical_loss": 4.2771273919299215, + "tokens_seen": 231538688 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004696088264794383, + "loss": 3.2876, + "theoretical_loss": 4.276979356443377, + "tokens_seen": 231604224 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004695987963891675, + "loss": 3.3123, + "theoretical_loss": 4.276831374564957, + "tokens_seen": 231669760 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004695887662988967, + "loss": 3.0926, + "theoretical_loss": 4.276683446260093, + "tokens_seen": 231735296 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004695787362086259, + "loss": 3.0816, + "theoretical_loss": 4.276535571494247, + "tokens_seen": 231800832 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046956870611835505, + "loss": 2.8716, + "theoretical_loss": 4.276387750232913, + "tokens_seen": 231866368 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004695586760280843, + "loss": 3.5923, + "theoretical_loss": 4.276239982441617, + "tokens_seen": 231931904 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046954864593781346, + "loss": 3.1727, + "theoretical_loss": 4.276092268085918, + "tokens_seen": 231997440 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046953861584754264, + "loss": 3.0979, + "theoretical_loss": 4.275944607131406, + "tokens_seen": 232062976 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004695285857572718, + "loss": 3.179, + "theoretical_loss": 4.275796999543703, + "tokens_seen": 232128512 + }, + { + "epoch": 0.07, + "learning_rate": 0.000469518555667001, + "loss": 3.0949, + "theoretical_loss": 4.275649445288461, + "tokens_seen": 232194048 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004695085255767302, + "loss": 3.2259, + "theoretical_loss": 4.275501944331367, + "tokens_seen": 232259584 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004694984954864594, + "loss": 3.2972, + "theoretical_loss": 4.275354496638139, + "tokens_seen": 232325120 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046948846539618855, + "loss": 3.5002, + "theoretical_loss": 4.275207102174525, + "tokens_seen": 232390656 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004694784353059178, + "loss": 2.9256, + "theoretical_loss": 4.275059760906305, + "tokens_seen": 232456192 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004694684052156469, + "loss": 3.3357, + "theoretical_loss": 4.2749124727992935, + "tokens_seen": 232521728 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046945837512537615, + "loss": 3.146, + "theoretical_loss": 4.274765237819333, + "tokens_seen": 232587264 + }, + { + "epoch": 0.07, + "objective/train/docs_used": 291491, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.0524933338165283, + "objective/train/theoretical_loss": 4.274618055932298, + "objective/train/tokens_used": 253112800, + "theoretical_loss": 4.274618055932298, + "tokens_seen": 232652800 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004694483450351053, + "loss": 2.9909, + "theoretical_loss": 4.274618055932298, + "tokens_seen": 232652800 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004694383149448345, + "loss": 3.1934, + "theoretical_loss": 4.2744709271040975, + "tokens_seen": 232718336 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004694282848545637, + "loss": 3.075, + "theoretical_loss": 4.27432385130067, + "tokens_seen": 232783872 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046941825476429287, + "loss": 3.3298, + "theoretical_loss": 4.274176828487984, + "tokens_seen": 232849408 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046940822467402205, + "loss": 3.1214, + "theoretical_loss": 4.2740298586320415, + "tokens_seen": 232914944 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004693981945837513, + "loss": 3.2709, + "theoretical_loss": 4.273882941698876, + "tokens_seen": 232980480 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004693881644934804, + "loss": 3.351, + "theoretical_loss": 4.27373607765455, + "tokens_seen": 233046016 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046937813440320965, + "loss": 3.24, + "theoretical_loss": 4.2735892664651605, + "tokens_seen": 233111552 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046936810431293883, + "loss": 3.1293, + "theoretical_loss": 4.273442508096833, + "tokens_seen": 233177088 + }, + { + "epoch": 0.07, + "learning_rate": 0.000469358074222668, + "loss": 3.2126, + "theoretical_loss": 4.273295802515726, + "tokens_seen": 233242624 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004693480441323972, + "loss": 3.1292, + "theoretical_loss": 4.273149149688028, + "tokens_seen": 233308160 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004693380140421264, + "loss": 3.0769, + "theoretical_loss": 4.27300254957996, + "tokens_seen": 233373696 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046932798395185555, + "loss": 3.026, + "theoretical_loss": 4.272856002157772, + "tokens_seen": 233439232 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004693179538615848, + "loss": 3.335, + "theoretical_loss": 4.272709507387748, + "tokens_seen": 233504768 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004693079237713139, + "loss": 3.1597, + "theoretical_loss": 4.2725630652362, + "tokens_seen": 233570304 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046929789368104315, + "loss": 3.1306, + "theoretical_loss": 4.272416675669473, + "tokens_seen": 233635840 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004692878635907723, + "loss": 3.2993, + "theoretical_loss": 4.272270338653942, + "tokens_seen": 233701376 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004692778335005015, + "loss": 3.2775, + "theoretical_loss": 4.272124054156014, + "tokens_seen": 233766912 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004692678034102307, + "loss": 3.0821, + "theoretical_loss": 4.271977822142125, + "tokens_seen": 233832448 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004692577733199599, + "loss": 3.1264, + "theoretical_loss": 4.271831642578745, + "tokens_seen": 233897984 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046924774322968906, + "loss": 3.0504, + "theoretical_loss": 4.27168551543237, + "tokens_seen": 233963520 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046923771313941824, + "loss": 3.29, + "theoretical_loss": 4.271539440669532, + "tokens_seen": 234029056 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004692276830491474, + "loss": 3.2624, + "theoretical_loss": 4.27139341825679, + "tokens_seen": 234094592 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046921765295887666, + "loss": 3.288, + "theoretical_loss": 4.271247448160736, + "tokens_seen": 234160128 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046920762286860584, + "loss": 3.2345, + "theoretical_loss": 4.27110153034799, + "tokens_seen": 234225664 + }, + { + "epoch": 0.07, + "objective/train/docs_used": 292210, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.441500663757324, + "objective/train/theoretical_loss": 4.270955664785207, + "objective/train/tokens_used": 254751200, + "theoretical_loss": 4.270955664785207, + "tokens_seen": 234291200 + }, + { + "epoch": 0.07, + "learning_rate": 0.000469197592778335, + "loss": 3.2181, + "theoretical_loss": 4.270955664785207, + "tokens_seen": 234291200 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046918756268806425, + "loss": 3.0661, + "theoretical_loss": 4.2708098514390676, + "tokens_seen": 234356736 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004691775325977934, + "loss": 3.0524, + "theoretical_loss": 4.270664090276286, + "tokens_seen": 234422272 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004691675025075226, + "loss": 3.2508, + "theoretical_loss": 4.2705183812636065, + "tokens_seen": 234487808 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046915747241725174, + "loss": 3.3243, + "theoretical_loss": 4.270372724367803, + "tokens_seen": 234553344 + }, + { + "epoch": 0.07, + "learning_rate": 0.000469147442326981, + "loss": 3.2352, + "theoretical_loss": 4.270227119555681, + "tokens_seen": 234618880 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046913741223671016, + "loss": 3.1664, + "theoretical_loss": 4.270081566794076, + "tokens_seen": 234684416 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046912738214643934, + "loss": 3.2272, + "theoretical_loss": 4.269936066049852, + "tokens_seen": 234749952 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004691173520561685, + "loss": 3.403, + "theoretical_loss": 4.269790617289907, + "tokens_seen": 234815488 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004691073219658977, + "loss": 3.171, + "theoretical_loss": 4.269645220481166, + "tokens_seen": 234881024 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004690972918756269, + "loss": 3.1504, + "theoretical_loss": 4.269499875590587, + "tokens_seen": 234946560 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004690872617853561, + "loss": 3.3593, + "theoretical_loss": 4.269354582585156, + "tokens_seen": 235012096 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046907723169508525, + "loss": 3.4094, + "theoretical_loss": 4.269209341431889, + "tokens_seen": 235077632 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004690672016048145, + "loss": 3.0117, + "theoretical_loss": 4.269064152097835, + "tokens_seen": 235143168 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046905717151454366, + "loss": 3.1555, + "theoretical_loss": 4.26891901455007, + "tokens_seen": 235208704 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046904714142427284, + "loss": 3.3753, + "theoretical_loss": 4.268773928755701, + "tokens_seen": 235274240 + }, + { + "epoch": 0.07, + "learning_rate": 0.000469037111334002, + "loss": 3.1484, + "theoretical_loss": 4.268628894681868, + "tokens_seen": 235339776 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004690270812437312, + "loss": 3.2826, + "theoretical_loss": 4.268483912295735, + "tokens_seen": 235405312 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004690170511534604, + "loss": 3.267, + "theoretical_loss": 4.268338981564502, + "tokens_seen": 235470848 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004690070210631896, + "loss": 3.0986, + "theoretical_loss": 4.268194102455395, + "tokens_seen": 235536384 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046899699097291875, + "loss": 3.0481, + "theoretical_loss": 4.26804927493567, + "tokens_seen": 235601920 + }, + { + "epoch": 0.07, + "learning_rate": 0.000468986960882648, + "loss": 3.3586, + "theoretical_loss": 4.267904498972618, + "tokens_seen": 235667456 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004689769307923771, + "loss": 3.1682, + "theoretical_loss": 4.267759774533552, + "tokens_seen": 235732992 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046896690070210635, + "loss": 3.1601, + "theoretical_loss": 4.267615101585821, + "tokens_seen": 235798528 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046895687061183553, + "loss": 3.3558, + "theoretical_loss": 4.267470480096801, + "tokens_seen": 235864064 + }, + { + "epoch": 0.07, + "objective/train/docs_used": 293604, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7816476821899414, + "objective/train/theoretical_loss": 4.267325910033897, + "objective/train/tokens_used": 256389600, + "theoretical_loss": 4.267325910033897, + "tokens_seen": 235929600 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004689468405215647, + "loss": 3.0005, + "theoretical_loss": 4.267325910033897, + "tokens_seen": 235929600 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004689368104312939, + "loss": 3.3922, + "theoretical_loss": 4.267181391364547, + "tokens_seen": 235995136 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046892678034102307, + "loss": 3.2998, + "theoretical_loss": 4.267036924056215, + "tokens_seen": 236060672 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046891675025075225, + "loss": 3.0282, + "theoretical_loss": 4.266892508076397, + "tokens_seen": 236126208 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004689067201604815, + "loss": 3.2748, + "theoretical_loss": 4.266748143392617, + "tokens_seen": 236191744 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004688966900702106, + "loss": 3.0778, + "theoretical_loss": 4.26660382997243, + "tokens_seen": 236257280 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046888665997993985, + "loss": 3.3932, + "theoretical_loss": 4.26645956778342, + "tokens_seen": 236322816 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046887662988966903, + "loss": 3.5158, + "theoretical_loss": 4.2663153567932, + "tokens_seen": 236388352 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004688665997993982, + "loss": 2.9413, + "theoretical_loss": 4.266171196969412, + "tokens_seen": 236453888 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004688565697091274, + "loss": 3.0881, + "theoretical_loss": 4.2660270882797295, + "tokens_seen": 236519424 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004688465396188566, + "loss": 3.3507, + "theoretical_loss": 4.265883030691853, + "tokens_seen": 236584960 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046883650952858575, + "loss": 3.2892, + "theoretical_loss": 4.265739024173515, + "tokens_seen": 236650496 + }, + { + "epoch": 0.07, + "learning_rate": 0.000468826479438315, + "loss": 3.1898, + "theoretical_loss": 4.265595068692473, + "tokens_seen": 236716032 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004688164493480441, + "loss": 3.0696, + "theoretical_loss": 4.26545116421652, + "tokens_seen": 236781568 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046880641925777335, + "loss": 3.385, + "theoretical_loss": 4.265307310713471, + "tokens_seen": 236847104 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004687963891675025, + "loss": 3.3006, + "theoretical_loss": 4.2651635081511765, + "tokens_seen": 236912640 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004687863590772317, + "loss": 3.1971, + "theoretical_loss": 4.265019756497512, + "tokens_seen": 236978176 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004687763289869609, + "loss": 3.0931, + "theoretical_loss": 4.264876055720386, + "tokens_seen": 237043712 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004687662988966901, + "loss": 3.1088, + "theoretical_loss": 4.264732405787731, + "tokens_seen": 237109248 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046875626880641926, + "loss": 3.0559, + "theoretical_loss": 4.264588806667513, + "tokens_seen": 237174784 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046874623871614844, + "loss": 3.3792, + "theoretical_loss": 4.264445258327724, + "tokens_seen": 237240320 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004687362086258776, + "loss": 3.2076, + "theoretical_loss": 4.264301760736389, + "tokens_seen": 237305856 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046872617853560686, + "loss": 3.2832, + "theoretical_loss": 4.264158313861557, + "tokens_seen": 237371392 + }, + { + "epoch": 0.07, + "learning_rate": 0.000468716148445336, + "loss": 3.3468, + "theoretical_loss": 4.264014917671309, + "tokens_seen": 237436928 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004687061183550652, + "loss": 3.0554, + "theoretical_loss": 4.2638715721337554, + "tokens_seen": 237502464 + }, + { + "epoch": 0.07, + "objective/train/docs_used": 294906, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.298981189727783, + "objective/train/theoretical_loss": 4.263728277217032, + "objective/train/tokens_used": 258028000, + "theoretical_loss": 4.263728277217032, + "tokens_seen": 237568000 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004686960882647944, + "loss": 3.2936, + "theoretical_loss": 4.263728277217032, + "tokens_seen": 237568000 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004686860581745236, + "loss": 3.0715, + "theoretical_loss": 4.263585032889306, + "tokens_seen": 237633536 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046867602808425276, + "loss": 3.2912, + "theoretical_loss": 4.263441839118776, + "tokens_seen": 237699072 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046866599799398194, + "loss": 2.9441, + "theoretical_loss": 4.2632986958736625, + "tokens_seen": 237764608 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004686559679037111, + "loss": 3.467, + "theoretical_loss": 4.263155603122221, + "tokens_seen": 237830144 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046864593781344036, + "loss": 3.2158, + "theoretical_loss": 4.263012560832733, + "tokens_seen": 237895680 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004686359077231695, + "loss": 3.2119, + "theoretical_loss": 4.262869568973508, + "tokens_seen": 237961216 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004686258776328987, + "loss": 3.2893, + "theoretical_loss": 4.262726627512886, + "tokens_seen": 238026752 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046861584754262785, + "loss": 3.4987, + "theoretical_loss": 4.262583736419234, + "tokens_seen": 238092288 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004686058174523571, + "loss": 3.271, + "theoretical_loss": 4.26244089566095, + "tokens_seen": 238157824 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046859578736208626, + "loss": 3.2722, + "theoretical_loss": 4.262298105206456, + "tokens_seen": 238223360 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046858575727181545, + "loss": 3.3172, + "theoretical_loss": 4.262155365024207, + "tokens_seen": 238288896 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004685757271815446, + "loss": 3.2976, + "theoretical_loss": 4.262012675082685, + "tokens_seen": 238354432 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004685656970912738, + "loss": 3.229, + "theoretical_loss": 4.261870035350399, + "tokens_seen": 238419968 + }, + { + "epoch": 0.07, + "learning_rate": 0.000468555667001003, + "loss": 3.5022, + "theoretical_loss": 4.261727445795888, + "tokens_seen": 238485504 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004685456369107322, + "loss": 3.1887, + "theoretical_loss": 4.26158490638772, + "tokens_seen": 238551040 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046853560682046135, + "loss": 3.1939, + "theoretical_loss": 4.261442417094488, + "tokens_seen": 238616576 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004685255767301906, + "loss": 3.0544, + "theoretical_loss": 4.261299977884816, + "tokens_seen": 238682112 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046851554663991977, + "loss": 3.1491, + "theoretical_loss": 4.2611575887273565, + "tokens_seen": 238747648 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046850551654964895, + "loss": 3.215, + "theoretical_loss": 4.261015249590789, + "tokens_seen": 238813184 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046849548645937813, + "loss": 3.2087, + "theoretical_loss": 4.260872960443822, + "tokens_seen": 238878720 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004684854563691073, + "loss": 3.0708, + "theoretical_loss": 4.260730721255191, + "tokens_seen": 238944256 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004684754262788365, + "loss": 2.872, + "theoretical_loss": 4.260588531993662, + "tokens_seen": 239009792 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046846539618856573, + "loss": 3.329, + "theoretical_loss": 4.260446392628026, + "tokens_seen": 239075328 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004684553660982949, + "loss": 3.2693, + "theoretical_loss": 4.2603043031271035, + "tokens_seen": 239140864 + }, + { + "epoch": 0.07, + "objective/train/docs_used": 295763, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9719858169555664, + "objective/train/theoretical_loss": 4.260162263459744, + "objective/train/tokens_used": 259666400, + "theoretical_loss": 4.260162263459744, + "tokens_seen": 239206400 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004684453360080241, + "loss": 3.251, + "theoretical_loss": 4.260162263459744, + "tokens_seen": 239206400 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046843530591775327, + "loss": 3.3018, + "theoretical_loss": 4.260020273594824, + "tokens_seen": 239271936 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046842527582748245, + "loss": 2.9441, + "theoretical_loss": 4.259878333501247, + "tokens_seen": 239337472 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004684152457372117, + "loss": 3.3198, + "theoretical_loss": 4.259736443147946, + "tokens_seen": 239403008 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004684052156469408, + "loss": 3.1598, + "theoretical_loss": 4.259594602503881, + "tokens_seen": 239468544 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046839518555667005, + "loss": 3.4569, + "theoretical_loss": 4.259452811538041, + "tokens_seen": 239534080 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046838515546639923, + "loss": 3.3406, + "theoretical_loss": 4.259311070219441, + "tokens_seen": 239599616 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004683751253761284, + "loss": 3.0996, + "theoretical_loss": 4.259169378517125, + "tokens_seen": 239665152 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004683650952858576, + "loss": 3.2584, + "theoretical_loss": 4.259027736400165, + "tokens_seen": 239730688 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004683550651955868, + "loss": 3.4028, + "theoretical_loss": 4.258886143837661, + "tokens_seen": 239796224 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046834503510531595, + "loss": 3.2419, + "theoretical_loss": 4.258744600798739, + "tokens_seen": 239861760 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004683350050150452, + "loss": 3.121, + "theoretical_loss": 4.2586031072525525, + "tokens_seen": 239927296 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004683249749247743, + "loss": 3.1897, + "theoretical_loss": 4.258461663168285, + "tokens_seen": 239992832 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046831494483450355, + "loss": 3.0282, + "theoretical_loss": 4.258320268515147, + "tokens_seen": 240058368 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004683049147442327, + "loss": 3.1978, + "theoretical_loss": 4.258178923262376, + "tokens_seen": 240123904 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004682948846539619, + "loss": 3.2214, + "theoretical_loss": 4.258037627379235, + "tokens_seen": 240189440 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004682848545636911, + "loss": 3.3644, + "theoretical_loss": 4.257896380835018, + "tokens_seen": 240254976 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004682748244734203, + "loss": 3.1584, + "theoretical_loss": 4.257755183599045, + "tokens_seen": 240320512 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046826479438314946, + "loss": 3.1607, + "theoretical_loss": 4.257614035640662, + "tokens_seen": 240386048 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046825476429287864, + "loss": 2.992, + "theoretical_loss": 4.257472936929246, + "tokens_seen": 240451584 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004682447342026078, + "loss": 3.399, + "theoretical_loss": 4.257331887434198, + "tokens_seen": 240517120 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046823470411233706, + "loss": 3.076, + "theoretical_loss": 4.257190887124946, + "tokens_seen": 240582656 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004682246740220662, + "loss": 3.1499, + "theoretical_loss": 4.25704993597095, + "tokens_seen": 240648192 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004682146439317954, + "loss": 3.3741, + "theoretical_loss": 4.256909033941691, + "tokens_seen": 240713728 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004682046138415246, + "loss": 3.2284, + "theoretical_loss": 4.256768181006683, + "tokens_seen": 240779264 + }, + { + "epoch": 0.07, + "objective/train/docs_used": 296516, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.0105109214782715, + "objective/train/theoretical_loss": 4.2566273771354615, + "objective/train/tokens_used": 261304800, + "theoretical_loss": 4.2566273771354615, + "tokens_seen": 240844800 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004681945837512538, + "loss": 3.02, + "theoretical_loss": 4.2566273771354615, + "tokens_seen": 240844800 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046818455366098296, + "loss": 3.3288, + "theoretical_loss": 4.256486622297595, + "tokens_seen": 240910336 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046817452357071214, + "loss": 3.397, + "theoretical_loss": 4.256345916462674, + "tokens_seen": 240975872 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004681644934804413, + "loss": 3.0537, + "theoretical_loss": 4.256205259600321, + "tokens_seen": 241041408 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046815446339017056, + "loss": 3.077, + "theoretical_loss": 4.256064651680182, + "tokens_seen": 241106944 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004681444332998997, + "loss": 3.2307, + "theoretical_loss": 4.255924092671931, + "tokens_seen": 241172480 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004681344032096289, + "loss": 2.9942, + "theoretical_loss": 4.255783582545269, + "tokens_seen": 241238016 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046812437311935805, + "loss": 3.281, + "theoretical_loss": 4.255643121269924, + "tokens_seen": 241303552 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004681143430290873, + "loss": 3.2357, + "theoretical_loss": 4.255502708815651, + "tokens_seen": 241369088 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046810431293881646, + "loss": 3.098, + "theoretical_loss": 4.255362345152234, + "tokens_seen": 241434624 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046809428284854565, + "loss": 3.0425, + "theoretical_loss": 4.255222030249479, + "tokens_seen": 241500160 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004680842527582748, + "loss": 3.1521, + "theoretical_loss": 4.255081764077224, + "tokens_seen": 241565696 + }, + { + "epoch": 0.07, + "learning_rate": 0.000468074222668004, + "loss": 3.3244, + "theoretical_loss": 4.25494154660533, + "tokens_seen": 241631232 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004680641925777332, + "loss": 3.0535, + "theoretical_loss": 4.254801377803689, + "tokens_seen": 241696768 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004680541624874624, + "loss": 3.0864, + "theoretical_loss": 4.254661257642215, + "tokens_seen": 241762304 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046804413239719155, + "loss": 3.1507, + "theoretical_loss": 4.254521186090852, + "tokens_seen": 241827840 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004680341023069208, + "loss": 3.1771, + "theoretical_loss": 4.254381163119568, + "tokens_seen": 241893376 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046802407221664997, + "loss": 3.3228, + "theoretical_loss": 4.254241188698361, + "tokens_seen": 241958912 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046801404212637915, + "loss": 3.1643, + "theoretical_loss": 4.2541012627972545, + "tokens_seen": 242024448 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046800401203610833, + "loss": 3.0985, + "theoretical_loss": 4.2539613853862965, + "tokens_seen": 242089984 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004679939819458375, + "loss": 3.1079, + "theoretical_loss": 4.253821556435565, + "tokens_seen": 242155520 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004679839518555667, + "loss": 3.1036, + "theoretical_loss": 4.253681775915161, + "tokens_seen": 242221056 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046797392176529593, + "loss": 3.1256, + "theoretical_loss": 4.253542043795215, + "tokens_seen": 242286592 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046796389167502505, + "loss": 3.0556, + "theoretical_loss": 4.253402360045882, + "tokens_seen": 242352128 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004679538615847543, + "loss": 3.2891, + "theoretical_loss": 4.253262724637346, + "tokens_seen": 242417664 + }, + { + "epoch": 0.07, + "objective/train/docs_used": 297983, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.877701997756958, + "objective/train/theoretical_loss": 4.253123137539814, + "objective/train/tokens_used": 262943200, + "theoretical_loss": 4.253123137539814, + "tokens_seen": 242483200 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004679438314944834, + "loss": 3.1046, + "theoretical_loss": 4.253123137539814, + "tokens_seen": 242483200 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046793380140421265, + "loss": 3.1167, + "theoretical_loss": 4.252983598723521, + "tokens_seen": 242548736 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046792377131394183, + "loss": 3.4114, + "theoretical_loss": 4.25284410815873, + "tokens_seen": 242614272 + }, + { + "epoch": 0.07, + "learning_rate": 0.000467913741223671, + "loss": 3.323, + "theoretical_loss": 4.2527046658157275, + "tokens_seen": 242679808 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004679037111334002, + "loss": 2.809, + "theoretical_loss": 4.252565271664828, + "tokens_seen": 242745344 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046789368104312943, + "loss": 3.1643, + "theoretical_loss": 4.252425925676373, + "tokens_seen": 242810880 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046788365095285856, + "loss": 3.4548, + "theoretical_loss": 4.252286627820727, + "tokens_seen": 242876416 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004678736208625878, + "loss": 3.2465, + "theoretical_loss": 4.252147378068285, + "tokens_seen": 242941952 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004678635907723169, + "loss": 3.2615, + "theoretical_loss": 4.252008176389465, + "tokens_seen": 243007488 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046785356068204616, + "loss": 3.1489, + "theoretical_loss": 4.251869022754712, + "tokens_seen": 243073024 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046784353059177534, + "loss": 3.2578, + "theoretical_loss": 4.251729917134498, + "tokens_seen": 243138560 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004678335005015045, + "loss": 3.0778, + "theoretical_loss": 4.251590859499322, + "tokens_seen": 243204096 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004678234704112337, + "loss": 3.046, + "theoretical_loss": 4.251451849819704, + "tokens_seen": 243269632 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004678134403209629, + "loss": 3.255, + "theoretical_loss": 4.251312888066197, + "tokens_seen": 243335168 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046780341023069206, + "loss": 2.9979, + "theoretical_loss": 4.251173974209375, + "tokens_seen": 243400704 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004677933801404213, + "loss": 3.0556, + "theoretical_loss": 4.251035108219839, + "tokens_seen": 243466240 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004677833500501504, + "loss": 3.2338, + "theoretical_loss": 4.250896290068218, + "tokens_seen": 243531776 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046777331995987966, + "loss": 3.1279, + "theoretical_loss": 4.250757519725165, + "tokens_seen": 243597312 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004677632898696088, + "loss": 3.2705, + "theoretical_loss": 4.25061879716136, + "tokens_seen": 243662848 + }, + { + "epoch": 0.07, + "learning_rate": 0.000467753259779338, + "loss": 3.1101, + "theoretical_loss": 4.250480122347507, + "tokens_seen": 243728384 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004677432296890672, + "loss": 2.9749, + "theoretical_loss": 4.250341495254337, + "tokens_seen": 243793920 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004677331995987964, + "loss": 3.0554, + "theoretical_loss": 4.250202915852608, + "tokens_seen": 243859456 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046772316950852556, + "loss": 2.9862, + "theoretical_loss": 4.250064384113102, + "tokens_seen": 243924992 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004677131394182548, + "loss": 2.8649, + "theoretical_loss": 4.249925900006627, + "tokens_seen": 243990528 + }, + { + "epoch": 0.07, + "learning_rate": 0.000467703109327984, + "loss": 3.533, + "theoretical_loss": 4.249787463504019, + "tokens_seen": 244056064 + }, + { + "epoch": 0.07, + "objective/train/docs_used": 298552, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.992788553237915, + "objective/train/theoretical_loss": 4.249649074576134, + "objective/train/tokens_used": 264581600, + "theoretical_loss": 4.249649074576134, + "tokens_seen": 244121600 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046769307923771316, + "loss": 3.0907, + "theoretical_loss": 4.249649074576134, + "tokens_seen": 244121600 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046768304914744234, + "loss": 2.9822, + "theoretical_loss": 4.249510733193862, + "tokens_seen": 244187136 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004676730190571715, + "loss": 3.1675, + "theoretical_loss": 4.249372439328111, + "tokens_seen": 244252672 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046766298896690076, + "loss": 3.1947, + "theoretical_loss": 4.249234192949818, + "tokens_seen": 244318208 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004676529588766299, + "loss": 3.1614, + "theoretical_loss": 4.249095994029947, + "tokens_seen": 244383744 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004676429287863591, + "loss": 3.2011, + "theoretical_loss": 4.248957842539484, + "tokens_seen": 244449280 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046763289869608825, + "loss": 3.1561, + "theoretical_loss": 4.248819738449442, + "tokens_seen": 244514816 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004676228686058175, + "loss": 3.3678, + "theoretical_loss": 4.2486816817308615, + "tokens_seen": 244580352 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046761283851554666, + "loss": 3.1089, + "theoretical_loss": 4.248543672354805, + "tokens_seen": 244645888 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046760280842527585, + "loss": 3.2738, + "theoretical_loss": 4.248405710292364, + "tokens_seen": 244711424 + }, + { + "epoch": 0.07, + "learning_rate": 0.000467592778335005, + "loss": 3.3828, + "theoretical_loss": 4.248267795514652, + "tokens_seen": 244776960 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004675827482447342, + "loss": 3.1759, + "theoretical_loss": 4.248129927992808, + "tokens_seen": 244842496 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004675727181544634, + "loss": 3.004, + "theoretical_loss": 4.247992107698002, + "tokens_seen": 244908032 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004675626880641926, + "loss": 3.2486, + "theoretical_loss": 4.247854334601421, + "tokens_seen": 244973568 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046755265797392175, + "loss": 3.2047, + "theoretical_loss": 4.247716608674283, + "tokens_seen": 245039104 + }, + { + "epoch": 0.07, + "learning_rate": 0.000467542627883651, + "loss": 3.0704, + "theoretical_loss": 4.247578929887829, + "tokens_seen": 245104640 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046753259779338017, + "loss": 2.9287, + "theoretical_loss": 4.247441298213326, + "tokens_seen": 245170176 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046752256770310935, + "loss": 3.0956, + "theoretical_loss": 4.247303713622067, + "tokens_seen": 245235712 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046751253761283853, + "loss": 3.0942, + "theoretical_loss": 4.247166176085367, + "tokens_seen": 245301248 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004675025075225677, + "loss": 3.0354, + "theoretical_loss": 4.247028685574569, + "tokens_seen": 245366784 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004674924774322969, + "loss": 3.135, + "theoretical_loss": 4.246891242061041, + "tokens_seen": 245432320 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046748244734202613, + "loss": 3.0614, + "theoretical_loss": 4.246753845516174, + "tokens_seen": 245497856 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046747241725175525, + "loss": 3.1953, + "theoretical_loss": 4.246616495911388, + "tokens_seen": 245563392 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004674623871614845, + "loss": 2.9679, + "theoretical_loss": 4.246479193218123, + "tokens_seen": 245628928 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004674523570712136, + "loss": 3.3495, + "theoretical_loss": 4.246341937407848, + "tokens_seen": 245694464 + }, + { + "epoch": 0.07, + "objective/train/docs_used": 299755, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.3899707794189453, + "objective/train/theoretical_loss": 4.246204728452055, + "objective/train/tokens_used": 266220000, + "theoretical_loss": 4.246204728452055, + "tokens_seen": 245760000 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046744232698094285, + "loss": 3.138, + "theoretical_loss": 4.246204728452055, + "tokens_seen": 245760000 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046743229689067203, + "loss": 3.3381, + "theoretical_loss": 4.246067566322259, + "tokens_seen": 245825536 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004674222668004012, + "loss": 2.9391, + "theoretical_loss": 4.245930450990007, + "tokens_seen": 245891072 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004674122367101304, + "loss": 3.2446, + "theoretical_loss": 4.245793382426861, + "tokens_seen": 245956608 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046740220661985963, + "loss": 3.1792, + "theoretical_loss": 4.245656360604417, + "tokens_seen": 246022144 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046739217652958876, + "loss": 3.3674, + "theoretical_loss": 4.24551938549429, + "tokens_seen": 246087680 + }, + { + "epoch": 0.07, + "learning_rate": 0.000467382146439318, + "loss": 3.1692, + "theoretical_loss": 4.2453824570681205, + "tokens_seen": 246153216 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004673721163490471, + "loss": 3.3341, + "theoretical_loss": 4.245245575297577, + "tokens_seen": 246218752 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046736208625877636, + "loss": 3.3756, + "theoretical_loss": 4.2451087401543495, + "tokens_seen": 246284288 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046735205616850554, + "loss": 3.2973, + "theoretical_loss": 4.244971951610154, + "tokens_seen": 246349824 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004673420260782347, + "loss": 3.376, + "theoretical_loss": 4.24483520963673, + "tokens_seen": 246415360 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004673319959879639, + "loss": 2.9486, + "theoretical_loss": 4.244698514205844, + "tokens_seen": 246480896 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004673219658976931, + "loss": 3.0086, + "theoretical_loss": 4.244561865289285, + "tokens_seen": 246546432 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046731193580742226, + "loss": 3.1113, + "theoretical_loss": 4.244425262858867, + "tokens_seen": 246611968 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004673019057171515, + "loss": 3.2665, + "theoretical_loss": 4.2442887068864295, + "tokens_seen": 246677504 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004672918756268806, + "loss": 3.1233, + "theoretical_loss": 4.244152197343835, + "tokens_seen": 246743040 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046728184553660986, + "loss": 3.3192, + "theoretical_loss": 4.244015734202973, + "tokens_seen": 246808576 + }, + { + "epoch": 0.07, + "learning_rate": 0.000467271815446339, + "loss": 3.3794, + "theoretical_loss": 4.243879317435755, + "tokens_seen": 246874112 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004672617853560682, + "loss": 2.863, + "theoretical_loss": 4.243742947014117, + "tokens_seen": 246939648 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004672517552657974, + "loss": 3.3378, + "theoretical_loss": 4.243606622910021, + "tokens_seen": 247005184 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004672417251755266, + "loss": 2.9326, + "theoretical_loss": 4.243470345095453, + "tokens_seen": 247070720 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046723169508525576, + "loss": 3.1262, + "theoretical_loss": 4.2433341135424225, + "tokens_seen": 247136256 + }, + { + "epoch": 0.07, + "learning_rate": 0.000467221664994985, + "loss": 3.2178, + "theoretical_loss": 4.243197928222964, + "tokens_seen": 247201792 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004672116349047141, + "loss": 3.2335, + "theoretical_loss": 4.243061789109136, + "tokens_seen": 247267328 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046720160481444336, + "loss": 3.1028, + "theoretical_loss": 4.242925696173021, + "tokens_seen": 247332864 + }, + { + "epoch": 0.07, + "objective/train/docs_used": 300199, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.2579872608184814, + "objective/train/theoretical_loss": 4.2427896493867285, + "objective/train/tokens_used": 267858400, + "theoretical_loss": 4.2427896493867285, + "tokens_seen": 247398400 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004671915747241725, + "loss": 3.0951, + "theoretical_loss": 4.2427896493867285, + "tokens_seen": 247398400 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004671815446339017, + "loss": 3.0132, + "theoretical_loss": 4.242653648722387, + "tokens_seen": 247463936 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004671715145436309, + "loss": 3.3596, + "theoretical_loss": 4.242517694152154, + "tokens_seen": 247529472 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004671614844533601, + "loss": 2.881, + "theoretical_loss": 4.24238178564821, + "tokens_seen": 247595008 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046715145436308927, + "loss": 3.3289, + "theoretical_loss": 4.242245923182756, + "tokens_seen": 247660544 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046714142427281845, + "loss": 3.1006, + "theoretical_loss": 4.242110106728022, + "tokens_seen": 247726080 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046713139418254763, + "loss": 3.3098, + "theoretical_loss": 4.241974336256261, + "tokens_seen": 247791616 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046712136409227686, + "loss": 3.4344, + "theoretical_loss": 4.241838611739748, + "tokens_seen": 247857152 + }, + { + "epoch": 0.08, + "learning_rate": 0.000467111334002006, + "loss": 3.3127, + "theoretical_loss": 4.241702933150783, + "tokens_seen": 247922688 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004671013039117352, + "loss": 3.2709, + "theoretical_loss": 4.241567300461693, + "tokens_seen": 247988224 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046709127382146435, + "loss": 3.2324, + "theoretical_loss": 4.241431713644823, + "tokens_seen": 248053760 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004670812437311936, + "loss": 2.8616, + "theoretical_loss": 4.241296172672547, + "tokens_seen": 248119296 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046707121364092277, + "loss": 3.1879, + "theoretical_loss": 4.24116067751726, + "tokens_seen": 248184832 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046706118355065195, + "loss": 3.2202, + "theoretical_loss": 4.241025228151383, + "tokens_seen": 248250368 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046705115346038113, + "loss": 3.2631, + "theoretical_loss": 4.24088982454736, + "tokens_seen": 248315904 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046704112337011037, + "loss": 3.0812, + "theoretical_loss": 4.240754466677659, + "tokens_seen": 248381440 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004670310932798395, + "loss": 3.2234, + "theoretical_loss": 4.240619154514771, + "tokens_seen": 248446976 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046702106318956873, + "loss": 3.2165, + "theoretical_loss": 4.240483888031212, + "tokens_seen": 248512512 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046701103309929786, + "loss": 2.9284, + "theoretical_loss": 4.240348667199521, + "tokens_seen": 248578048 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004670010030090271, + "loss": 3.1256, + "theoretical_loss": 4.240213491992261, + "tokens_seen": 248643584 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004669909729187563, + "loss": 3.1306, + "theoretical_loss": 4.240078362382019, + "tokens_seen": 248709120 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046698094282848545, + "loss": 3.1289, + "theoretical_loss": 4.239943278341404, + "tokens_seen": 248774656 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046697091273821464, + "loss": 3.2737, + "theoretical_loss": 4.239808239843052, + "tokens_seen": 248840192 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004669608826479438, + "loss": 3.1408, + "theoretical_loss": 4.239673246859619, + "tokens_seen": 248905728 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046695085255767305, + "loss": 3.2095, + "theoretical_loss": 4.239538299363788, + "tokens_seen": 248971264 + }, + { + "epoch": 0.08, + "objective/train/docs_used": 301486, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.3037164211273193, + "objective/train/theoretical_loss": 4.239403397328261, + "objective/train/tokens_used": 269496800, + "theoretical_loss": 4.239403397328261, + "tokens_seen": 249036800 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046694082246740223, + "loss": 3.109, + "theoretical_loss": 4.239403397328261, + "tokens_seen": 249036800 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004669307923771314, + "loss": 3.2687, + "theoretical_loss": 4.239268540725769, + "tokens_seen": 249102336 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004669207622868606, + "loss": 3.2338, + "theoretical_loss": 4.239133729529064, + "tokens_seen": 249167872 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046691073219658983, + "loss": 3.023, + "theoretical_loss": 4.2389989637109196, + "tokens_seen": 249233408 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046690070210631896, + "loss": 3.2841, + "theoretical_loss": 4.2388642432441355, + "tokens_seen": 249298944 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004668906720160482, + "loss": 3.4195, + "theoretical_loss": 4.238729568101535, + "tokens_seen": 249364480 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004668806419257773, + "loss": 3.0559, + "theoretical_loss": 4.238594938255963, + "tokens_seen": 249430016 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046687061183550656, + "loss": 3.1747, + "theoretical_loss": 4.2384603536802885, + "tokens_seen": 249495552 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046686058174523574, + "loss": 3.1241, + "theoretical_loss": 4.238325814347404, + "tokens_seen": 249561088 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004668505516549649, + "loss": 3.3317, + "theoretical_loss": 4.238191320230227, + "tokens_seen": 249626624 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004668405215646941, + "loss": 3.127, + "theoretical_loss": 4.238056871301695, + "tokens_seen": 249692160 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004668304914744233, + "loss": 3.1163, + "theoretical_loss": 4.237922467534771, + "tokens_seen": 249757696 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046682046138415246, + "loss": 3.1401, + "theoretical_loss": 4.237788108902441, + "tokens_seen": 249823232 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004668104312938817, + "loss": 3.285, + "theoretical_loss": 4.237653795377714, + "tokens_seen": 249888768 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004668004012036108, + "loss": 3.4912, + "theoretical_loss": 4.237519526933622, + "tokens_seen": 249954304 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046679037111334006, + "loss": 2.9408, + "theoretical_loss": 4.2373853035432205, + "tokens_seen": 250019840 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004667803410230692, + "loss": 3.312, + "theoretical_loss": 4.237251125179588, + "tokens_seen": 250085376 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004667703109327984, + "loss": 3.14, + "theoretical_loss": 4.237116991815826, + "tokens_seen": 250150912 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004667602808425276, + "loss": 3.1893, + "theoretical_loss": 4.23698290342506, + "tokens_seen": 250216448 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004667502507522568, + "loss": 3.1478, + "theoretical_loss": 4.236848859980437, + "tokens_seen": 250281984 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046674022066198596, + "loss": 3.0303, + "theoretical_loss": 4.23671486145513, + "tokens_seen": 250347520 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004667301905717152, + "loss": 3.1047, + "theoretical_loss": 4.236580907822331, + "tokens_seen": 250413056 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004667201604814443, + "loss": 3.0878, + "theoretical_loss": 4.236446999055257, + "tokens_seen": 250478592 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046671013039117356, + "loss": 3.123, + "theoretical_loss": 4.2363131351271495, + "tokens_seen": 250544128 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004667001003009027, + "loss": 3.1892, + "theoretical_loss": 4.2361793160112695, + "tokens_seen": 250609664 + }, + { + "epoch": 0.08, + "objective/train/docs_used": 302298, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.1085546016693115, + "objective/train/theoretical_loss": 4.236045541680905, + "objective/train/tokens_used": 271135200, + "theoretical_loss": 4.236045541680905, + "tokens_seen": 250675200 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004666900702106319, + "loss": 3.3128, + "theoretical_loss": 4.236045541680905, + "tokens_seen": 250675200 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004666800401203611, + "loss": 3.097, + "theoretical_loss": 4.235911812109363, + "tokens_seen": 250740736 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004666700100300903, + "loss": 3.054, + "theoretical_loss": 4.235778127269976, + "tokens_seen": 250806272 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046665997993981947, + "loss": 3.2811, + "theoretical_loss": 4.235644487136098, + "tokens_seen": 250871808 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046664994984954865, + "loss": 3.2177, + "theoretical_loss": 4.235510891681108, + "tokens_seen": 250937344 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046663991975927783, + "loss": 3.1458, + "theoretical_loss": 4.235377340878404, + "tokens_seen": 251002880 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046662988966900706, + "loss": 3.2887, + "theoretical_loss": 4.23524383470141, + "tokens_seen": 251068416 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004666198595787362, + "loss": 3.5902, + "theoretical_loss": 4.235110373123572, + "tokens_seen": 251133952 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046660982948846543, + "loss": 3.2736, + "theoretical_loss": 4.2349769561183574, + "tokens_seen": 251199488 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046659979939819455, + "loss": 3.3576, + "theoretical_loss": 4.2348435836592575, + "tokens_seen": 251265024 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004665897693079238, + "loss": 3.0621, + "theoretical_loss": 4.234710255719786, + "tokens_seen": 251330560 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046657973921765297, + "loss": 3.2782, + "theoretical_loss": 4.234576972273481, + "tokens_seen": 251396096 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046656970912738215, + "loss": 3.1883, + "theoretical_loss": 4.234443733293899, + "tokens_seen": 251461632 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046655967903711133, + "loss": 3.2245, + "theoretical_loss": 4.234310538754624, + "tokens_seen": 251527168 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046654964894684057, + "loss": 3.2696, + "theoretical_loss": 4.2341773886292575, + "tokens_seen": 251592704 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004665396188565697, + "loss": 3.1524, + "theoretical_loss": 4.234044282891429, + "tokens_seen": 251658240 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046652958876629893, + "loss": 3.0471, + "theoretical_loss": 4.233911221514787, + "tokens_seen": 251723776 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046651955867602806, + "loss": 3.1227, + "theoretical_loss": 4.233778204473002, + "tokens_seen": 251789312 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004665095285857573, + "loss": 3.2876, + "theoretical_loss": 4.23364523173977, + "tokens_seen": 251854848 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004664994984954865, + "loss": 3.2065, + "theoretical_loss": 4.233512303288807, + "tokens_seen": 251920384 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046648946840521565, + "loss": 2.8681, + "theoretical_loss": 4.233379419093851, + "tokens_seen": 251985920 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046647943831494484, + "loss": 3.0819, + "theoretical_loss": 4.233246579128666, + "tokens_seen": 252051456 + }, + { + "epoch": 0.08, + "learning_rate": 0.000466469408224674, + "loss": 3.1822, + "theoretical_loss": 4.233113783367033, + "tokens_seen": 252116992 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004664593781344032, + "loss": 3.1466, + "theoretical_loss": 4.232981031782761, + "tokens_seen": 252182528 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046644934804413243, + "loss": 3.3105, + "theoretical_loss": 4.232848324349677, + "tokens_seen": 252248064 + }, + { + "epoch": 0.08, + "objective/train/docs_used": 303567, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.1599833965301514, + "objective/train/theoretical_loss": 4.232715661041632, + "objective/train/tokens_used": 272773600, + "theoretical_loss": 4.232715661041632, + "tokens_seen": 252313600 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046643931795386156, + "loss": 3.1736, + "theoretical_loss": 4.232715661041632, + "tokens_seen": 252313600 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004664292878635908, + "loss": 3.1568, + "theoretical_loss": 4.232583041832499, + "tokens_seen": 252379136 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004664192577733199, + "loss": 3.1369, + "theoretical_loss": 4.232450466696174, + "tokens_seen": 252444672 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046640922768304916, + "loss": 3.0532, + "theoretical_loss": 4.2323179356065745, + "tokens_seen": 252510208 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046639919759277834, + "loss": 2.9697, + "theoretical_loss": 4.23218544853764, + "tokens_seen": 252575744 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004663891675025075, + "loss": 3.31, + "theoretical_loss": 4.232053005463333, + "tokens_seen": 252641280 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004663791374122367, + "loss": 3.2688, + "theoretical_loss": 4.231920606357638, + "tokens_seen": 252706816 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046636910732196594, + "loss": 3.0768, + "theoretical_loss": 4.231788251194559, + "tokens_seen": 252772352 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046635907723169506, + "loss": 3.1853, + "theoretical_loss": 4.231655939948127, + "tokens_seen": 252837888 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004663490471414243, + "loss": 3.0185, + "theoretical_loss": 4.231523672592392, + "tokens_seen": 252903424 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004663390170511534, + "loss": 3.2061, + "theoretical_loss": 4.231391449101425, + "tokens_seen": 252968960 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046632898696088266, + "loss": 3.0254, + "theoretical_loss": 4.231259269449322, + "tokens_seen": 253034496 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046631895687061184, + "loss": 3.4604, + "theoretical_loss": 4.231127133610198, + "tokens_seen": 253100032 + }, + { + "epoch": 0.08, + "learning_rate": 0.000466308926780341, + "loss": 3.1453, + "theoretical_loss": 4.230995041558194, + "tokens_seen": 253165568 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004662988966900702, + "loss": 3.1521, + "theoretical_loss": 4.230862993267468, + "tokens_seen": 253231104 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004662888665997994, + "loss": 3.2638, + "theoretical_loss": 4.230730988712205, + "tokens_seen": 253296640 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046627883650952857, + "loss": 3.3752, + "theoretical_loss": 4.230599027866606, + "tokens_seen": 253362176 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004662688064192578, + "loss": 3.1583, + "theoretical_loss": 4.2304671107048994, + "tokens_seen": 253427712 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046625877632898693, + "loss": 3.2989, + "theoretical_loss": 4.2303352372013325, + "tokens_seen": 253493248 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046624874623871616, + "loss": 3.2402, + "theoretical_loss": 4.230203407330176, + "tokens_seen": 253558784 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046623871614844535, + "loss": 3.1902, + "theoretical_loss": 4.230071621065721, + "tokens_seen": 253624320 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004662286860581745, + "loss": 2.866, + "theoretical_loss": 4.2299398783822815, + "tokens_seen": 253689856 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004662186559679037, + "loss": 3.2554, + "theoretical_loss": 4.229808179254192, + "tokens_seen": 253755392 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004662086258776329, + "loss": 3.2293, + "theoretical_loss": 4.22967652365581, + "tokens_seen": 253820928 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004661985957873621, + "loss": 2.9234, + "theoretical_loss": 4.229544911561513, + "tokens_seen": 253886464 + }, + { + "epoch": 0.08, + "objective/train/docs_used": 304269, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.1997804641723633, + "objective/train/theoretical_loss": 4.229413342945703, + "objective/train/tokens_used": 274412000, + "theoretical_loss": 4.229413342945703, + "tokens_seen": 253952000 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004661885656970913, + "loss": 3.353, + "theoretical_loss": 4.229413342945703, + "tokens_seen": 253952000 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004661785356068205, + "loss": 3.1267, + "theoretical_loss": 4.229281817782801, + "tokens_seen": 254017536 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046616850551654967, + "loss": 3.2084, + "theoretical_loss": 4.229150336047251, + "tokens_seen": 254083072 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046615847542627885, + "loss": 3.3788, + "theoretical_loss": 4.229018897713519, + "tokens_seen": 254148608 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046614844533600803, + "loss": 3.0708, + "theoretical_loss": 4.22888750275609, + "tokens_seen": 254214144 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046613841524573727, + "loss": 3.0028, + "theoretical_loss": 4.228756151149475, + "tokens_seen": 254279680 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004661283851554664, + "loss": 3.3021, + "theoretical_loss": 4.228624842868202, + "tokens_seen": 254345216 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046611835506519563, + "loss": 3.0294, + "theoretical_loss": 4.228493577886824, + "tokens_seen": 254410752 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046610832497492475, + "loss": 3.1577, + "theoretical_loss": 4.228362356179913, + "tokens_seen": 254476288 + }, + { + "epoch": 0.08, + "learning_rate": 0.000466098294884654, + "loss": 3.0022, + "theoretical_loss": 4.228231177722063, + "tokens_seen": 254541824 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046608826479438317, + "loss": 3.0895, + "theoretical_loss": 4.228100042487892, + "tokens_seen": 254607360 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046607823470411235, + "loss": 3.1647, + "theoretical_loss": 4.227968950452035, + "tokens_seen": 254672896 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046606820461384153, + "loss": 3.2423, + "theoretical_loss": 4.227837901589153, + "tokens_seen": 254738432 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046605817452357077, + "loss": 3.056, + "theoretical_loss": 4.227706895873924, + "tokens_seen": 254803968 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004660481444332999, + "loss": 2.8991, + "theoretical_loss": 4.227575933281051, + "tokens_seen": 254869504 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046603811434302913, + "loss": 3.0, + "theoretical_loss": 4.227445013785257, + "tokens_seen": 254935040 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046602808425275826, + "loss": 3.0826, + "theoretical_loss": 4.227314137361285, + "tokens_seen": 255000576 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004660180541624875, + "loss": 3.2037, + "theoretical_loss": 4.227183303983901, + "tokens_seen": 255066112 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004660080240722167, + "loss": 3.2881, + "theoretical_loss": 4.227052513627893, + "tokens_seen": 255131648 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046599799398194586, + "loss": 3.1299, + "theoretical_loss": 4.226921766268067, + "tokens_seen": 255197184 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046598796389167504, + "loss": 3.0684, + "theoretical_loss": 4.226791061879253, + "tokens_seen": 255262720 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004659779338014042, + "loss": 3.0469, + "theoretical_loss": 4.226660400436302, + "tokens_seen": 255328256 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004659679037111334, + "loss": 3.0472, + "theoretical_loss": 4.226529781914084, + "tokens_seen": 255393792 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046595787362086263, + "loss": 3.2079, + "theoretical_loss": 4.226399206287493, + "tokens_seen": 255459328 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046594784353059176, + "loss": 3.1358, + "theoretical_loss": 4.226268673531442, + "tokens_seen": 255524864 + }, + { + "epoch": 0.08, + "objective/train/docs_used": 305849, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.743330240249634, + "objective/train/theoretical_loss": 4.226138183620867, + "objective/train/tokens_used": 276050400, + "theoretical_loss": 4.226138183620867, + "tokens_seen": 255590400 + }, + { + "epoch": 0.08, + "learning_rate": 0.000465937813440321, + "loss": 3.1548, + "theoretical_loss": 4.226138183620867, + "tokens_seen": 255590400 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004659277833500501, + "loss": 3.0213, + "theoretical_loss": 4.226007736530723, + "tokens_seen": 255655936 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046591775325977936, + "loss": 3.1984, + "theoretical_loss": 4.225877332235987, + "tokens_seen": 255721472 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046590772316950854, + "loss": 3.1091, + "theoretical_loss": 4.225746970711657, + "tokens_seen": 255787008 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004658976930792377, + "loss": 3.2381, + "theoretical_loss": 4.225616651932753, + "tokens_seen": 255852544 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004658876629889669, + "loss": 3.2222, + "theoretical_loss": 4.225486375874315, + "tokens_seen": 255918080 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046587763289869614, + "loss": 3.1889, + "theoretical_loss": 4.225356142511402, + "tokens_seen": 255983616 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046586760280842526, + "loss": 3.0673, + "theoretical_loss": 4.225225951819099, + "tokens_seen": 256049152 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004658575727181545, + "loss": 3.1606, + "theoretical_loss": 4.225095803772507, + "tokens_seen": 256114688 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004658475426278836, + "loss": 3.2571, + "theoretical_loss": 4.22496569834675, + "tokens_seen": 256180224 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046583751253761286, + "loss": 3.3425, + "theoretical_loss": 4.224835635516973, + "tokens_seen": 256245760 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046582748244734204, + "loss": 3.0092, + "theoretical_loss": 4.224705615258341, + "tokens_seen": 256311296 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004658174523570712, + "loss": 3.2458, + "theoretical_loss": 4.224575637546041, + "tokens_seen": 256376832 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004658074222668004, + "loss": 2.9137, + "theoretical_loss": 4.224445702355279, + "tokens_seen": 256442368 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004657973921765296, + "loss": 3.1672, + "theoretical_loss": 4.2243158096612845, + "tokens_seen": 256507904 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046578736208625877, + "loss": 3.1599, + "theoretical_loss": 4.224185959439305, + "tokens_seen": 256573440 + }, + { + "epoch": 0.08, + "learning_rate": 0.000465777331995988, + "loss": 3.1255, + "theoretical_loss": 4.22405615166461, + "tokens_seen": 256638976 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046576730190571713, + "loss": 3.1789, + "theoretical_loss": 4.22392638631249, + "tokens_seen": 256704512 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046575727181544636, + "loss": 3.1087, + "theoretical_loss": 4.223796663358255, + "tokens_seen": 256770048 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046574724172517555, + "loss": 3.2464, + "theoretical_loss": 4.223666982777237, + "tokens_seen": 256835584 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004657372116349047, + "loss": 3.327, + "theoretical_loss": 4.223537344544788, + "tokens_seen": 256901120 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004657271815446339, + "loss": 3.1453, + "theoretical_loss": 4.223407748636282, + "tokens_seen": 256966656 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004657171514543631, + "loss": 3.0814, + "theoretical_loss": 4.22327819502711, + "tokens_seen": 257032192 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046570712136409227, + "loss": 3.1326, + "theoretical_loss": 4.223148683692687, + "tokens_seen": 257097728 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004656970912738215, + "loss": 3.2236, + "theoretical_loss": 4.223019214608446, + "tokens_seen": 257163264 + }, + { + "epoch": 0.08, + "objective/train/docs_used": 306364, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.629127264022827, + "objective/train/theoretical_loss": 4.222889787749845, + "objective/train/tokens_used": 277688800, + "theoretical_loss": 4.222889787749845, + "tokens_seen": 257228800 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046568706118355063, + "loss": 3.0022, + "theoretical_loss": 4.222889787749845, + "tokens_seen": 257228800 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046567703109327987, + "loss": 3.2723, + "theoretical_loss": 4.222760403092358, + "tokens_seen": 257294336 + }, + { + "epoch": 0.08, + "learning_rate": 0.000465667001003009, + "loss": 3.1032, + "theoretical_loss": 4.22263106061148, + "tokens_seen": 257359872 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046565697091273823, + "loss": 3.2668, + "theoretical_loss": 4.222501760282729, + "tokens_seen": 257425408 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004656469408224674, + "loss": 3.3694, + "theoretical_loss": 4.22237250208164, + "tokens_seen": 257490944 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004656369107321966, + "loss": 3.0323, + "theoretical_loss": 4.222243285983772, + "tokens_seen": 257556480 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004656268806419258, + "loss": 3.0638, + "theoretical_loss": 4.222114111964703, + "tokens_seen": 257622016 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046561685055165495, + "loss": 3.2751, + "theoretical_loss": 4.221984980000029, + "tokens_seen": 257687552 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046560682046138414, + "loss": 3.1497, + "theoretical_loss": 4.2218558900653695, + "tokens_seen": 257753088 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046559679037111337, + "loss": 3.2393, + "theoretical_loss": 4.221726842136364, + "tokens_seen": 257818624 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004655867602808425, + "loss": 3.3169, + "theoretical_loss": 4.2215978361886695, + "tokens_seen": 257884160 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046557673019057173, + "loss": 3.1882, + "theoretical_loss": 4.221468872197967, + "tokens_seen": 257949696 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004655667001003009, + "loss": 3.0225, + "theoretical_loss": 4.221339950139956, + "tokens_seen": 258015232 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004655566700100301, + "loss": 3.1899, + "theoretical_loss": 4.221211069990357, + "tokens_seen": 258080768 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004655466399197593, + "loss": 3.2241, + "theoretical_loss": 4.221082231724908, + "tokens_seen": 258146304 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046553660982948846, + "loss": 3.1808, + "theoretical_loss": 4.22095343531937, + "tokens_seen": 258211840 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046552657973921764, + "loss": 3.4274, + "theoretical_loss": 4.220824680749525, + "tokens_seen": 258277376 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004655165496489469, + "loss": 3.0687, + "theoretical_loss": 4.220695967991171, + "tokens_seen": 258342912 + }, + { + "epoch": 0.08, + "learning_rate": 0.000465506519558676, + "loss": 3.1397, + "theoretical_loss": 4.220567297020131, + "tokens_seen": 258408448 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046549648946840524, + "loss": 3.1805, + "theoretical_loss": 4.220438667812244, + "tokens_seen": 258473984 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046548645937813436, + "loss": 3.1492, + "theoretical_loss": 4.220310080343373, + "tokens_seen": 258539520 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004654764292878636, + "loss": 2.7064, + "theoretical_loss": 4.220181534589398, + "tokens_seen": 258605056 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004654663991975928, + "loss": 3.0719, + "theoretical_loss": 4.22005303052622, + "tokens_seen": 258670592 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046545636910732196, + "loss": 3.2042, + "theoretical_loss": 4.219924568129759, + "tokens_seen": 258736128 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004654463390170512, + "loss": 3.0768, + "theoretical_loss": 4.219796147375957, + "tokens_seen": 258801664 + }, + { + "epoch": 0.08, + "objective/train/docs_used": 307092, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.476236343383789, + "objective/train/theoretical_loss": 4.219667768240775, + "objective/train/tokens_used": 279327200, + "theoretical_loss": 4.219667768240775, + "tokens_seen": 258867200 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004654363089267803, + "loss": 3.1775, + "theoretical_loss": 4.219667768240775, + "tokens_seen": 258867200 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046542627883650956, + "loss": 2.9703, + "theoretical_loss": 4.219539430700195, + "tokens_seen": 258932736 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046541624874623874, + "loss": 3.0814, + "theoretical_loss": 4.2194111347302155, + "tokens_seen": 258998272 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004654062186559679, + "loss": 3.124, + "theoretical_loss": 4.219282880306859, + "tokens_seen": 259063808 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004653961885656971, + "loss": 2.9946, + "theoretical_loss": 4.219154667406166, + "tokens_seen": 259129344 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046538615847542634, + "loss": 3.1049, + "theoretical_loss": 4.219026496004198, + "tokens_seen": 259194880 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046537612838515546, + "loss": 3.06, + "theoretical_loss": 4.218898366077035, + "tokens_seen": 259260416 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004653660982948847, + "loss": 3.0953, + "theoretical_loss": 4.218770277600775, + "tokens_seen": 259325952 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004653560682046138, + "loss": 2.8445, + "theoretical_loss": 4.218642230551541, + "tokens_seen": 259391488 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046534603811434306, + "loss": 3.0274, + "theoretical_loss": 4.218514224905472, + "tokens_seen": 259457024 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046533600802407224, + "loss": 2.9485, + "theoretical_loss": 4.218386260638727, + "tokens_seen": 259522560 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004653259779338014, + "loss": 3.1876, + "theoretical_loss": 4.2182583377274865, + "tokens_seen": 259588096 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004653159478435306, + "loss": 3.1376, + "theoretical_loss": 4.218130456147948, + "tokens_seen": 259653632 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004653059177532598, + "loss": 3.1214, + "theoretical_loss": 4.218002615876332, + "tokens_seen": 259719168 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046529588766298897, + "loss": 3.2295, + "theoretical_loss": 4.217874816888877, + "tokens_seen": 259784704 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004652858575727182, + "loss": 3.0723, + "theoretical_loss": 4.217747059161839, + "tokens_seen": 259850240 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046527582748244733, + "loss": 2.9696, + "theoretical_loss": 4.217619342671498, + "tokens_seen": 259915776 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046526579739217656, + "loss": 3.0124, + "theoretical_loss": 4.2174916673941505, + "tokens_seen": 259981312 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046525576730190575, + "loss": 2.8966, + "theoretical_loss": 4.217364033306113, + "tokens_seen": 260046848 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004652457372116349, + "loss": 3.1619, + "theoretical_loss": 4.217236440383724, + "tokens_seen": 260112384 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004652357071213641, + "loss": 2.9974, + "theoretical_loss": 4.217108888603337, + "tokens_seen": 260177920 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004652256770310933, + "loss": 2.9334, + "theoretical_loss": 4.21698137794133, + "tokens_seen": 260243456 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046521564694082247, + "loss": 3.2643, + "theoretical_loss": 4.216853908374097, + "tokens_seen": 260308992 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004652056168505517, + "loss": 3.3784, + "theoretical_loss": 4.216726479878052, + "tokens_seen": 260374528 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046519558676028083, + "loss": 3.1801, + "theoretical_loss": 4.216599092429631, + "tokens_seen": 260440064 + }, + { + "epoch": 0.08, + "objective/train/docs_used": 308211, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.457490921020508, + "objective/train/theoretical_loss": 4.216471746005286, + "objective/train/tokens_used": 280965600, + "theoretical_loss": 4.216471746005286, + "tokens_seen": 260505600 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046518555667001007, + "loss": 3.1286, + "theoretical_loss": 4.216471746005286, + "tokens_seen": 260505600 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004651755265797392, + "loss": 2.9917, + "theoretical_loss": 4.216344440581491, + "tokens_seen": 260571136 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046516549648946843, + "loss": 3.0801, + "theoretical_loss": 4.2162171761347365, + "tokens_seen": 260636672 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004651554663991976, + "loss": 3.1188, + "theoretical_loss": 4.2160899526415365, + "tokens_seen": 260702208 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004651454363089268, + "loss": 3.0944, + "theoretical_loss": 4.215962770078422, + "tokens_seen": 260767744 + }, + { + "epoch": 0.08, + "learning_rate": 0.000465135406218656, + "loss": 3.3147, + "theoretical_loss": 4.215835628421942, + "tokens_seen": 260833280 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046512537612838515, + "loss": 3.1019, + "theoretical_loss": 4.215708527648667, + "tokens_seen": 260898816 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046511534603811434, + "loss": 2.8503, + "theoretical_loss": 4.215581467735187, + "tokens_seen": 260964352 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046510531594784357, + "loss": 3.2366, + "theoretical_loss": 4.215454448658109, + "tokens_seen": 261029888 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004650952858575727, + "loss": 3.1674, + "theoretical_loss": 4.215327470394062, + "tokens_seen": 261095424 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046508525576730193, + "loss": 3.1104, + "theoretical_loss": 4.215200532919691, + "tokens_seen": 261160960 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004650752256770311, + "loss": 3.197, + "theoretical_loss": 4.215073636211664, + "tokens_seen": 261226496 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004650651955867603, + "loss": 3.1214, + "theoretical_loss": 4.214946780246666, + "tokens_seen": 261292032 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004650551654964895, + "loss": 3.0783, + "theoretical_loss": 4.214819965001401, + "tokens_seen": 261357568 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046504513540621866, + "loss": 3.0267, + "theoretical_loss": 4.214693190452593, + "tokens_seen": 261423104 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046503510531594784, + "loss": 3.0135, + "theoretical_loss": 4.214566456576984, + "tokens_seen": 261488640 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004650250752256771, + "loss": 2.9927, + "theoretical_loss": 4.214439763351336, + "tokens_seen": 261554176 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004650150451354062, + "loss": 2.9392, + "theoretical_loss": 4.214313110752431, + "tokens_seen": 261619712 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046500501504513544, + "loss": 2.9189, + "theoretical_loss": 4.214186498757069, + "tokens_seen": 261685248 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046499498495486456, + "loss": 3.0795, + "theoretical_loss": 4.214059927342068, + "tokens_seen": 261750784 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004649849548645938, + "loss": 3.0373, + "theoretical_loss": 4.213933396484267, + "tokens_seen": 261816320 + }, + { + "epoch": 0.08, + "learning_rate": 0.000464974924774323, + "loss": 3.1087, + "theoretical_loss": 4.213806906160523, + "tokens_seen": 261881856 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046496489468405216, + "loss": 3.2205, + "theoretical_loss": 4.213680456347712, + "tokens_seen": 261947392 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046495486459378134, + "loss": 3.438, + "theoretical_loss": 4.213554047022729, + "tokens_seen": 262012928 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004649448345035105, + "loss": 3.1792, + "theoretical_loss": 4.213427678162489, + "tokens_seen": 262078464 + }, + { + "epoch": 0.08, + "objective/train/docs_used": 308640, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.578385353088379, + "objective/train/theoretical_loss": 4.213301349743924, + "objective/train/tokens_used": 282604000, + "theoretical_loss": 4.213301349743924, + "tokens_seen": 262144000 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004649348044132397, + "loss": 3.2087, + "theoretical_loss": 4.213301349743924, + "tokens_seen": 262144000 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046492477432296894, + "loss": 3.0893, + "theoretical_loss": 4.2131750617439865, + "tokens_seen": 262209536 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046491474423269807, + "loss": 3.1618, + "theoretical_loss": 4.213048814139647, + "tokens_seen": 262275072 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004649047141424273, + "loss": 2.9118, + "theoretical_loss": 4.212922606907895, + "tokens_seen": 262340608 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004648946840521565, + "loss": 3.2177, + "theoretical_loss": 4.21279644002574, + "tokens_seen": 262406144 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046488465396188566, + "loss": 3.1045, + "theoretical_loss": 4.212670313470209, + "tokens_seen": 262471680 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046487462387161485, + "loss": 3.0042, + "theoretical_loss": 4.212544227218347, + "tokens_seen": 262537216 + }, + { + "epoch": 0.08, + "learning_rate": 0.000464864593781344, + "loss": 3.0721, + "theoretical_loss": 4.21241818124722, + "tokens_seen": 262602752 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004648545636910732, + "loss": 3.1759, + "theoretical_loss": 4.212292175533912, + "tokens_seen": 262668288 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046484453360080244, + "loss": 3.2897, + "theoretical_loss": 4.212166210055526, + "tokens_seen": 262733824 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046483450351053157, + "loss": 3.1411, + "theoretical_loss": 4.212040284789181, + "tokens_seen": 262799360 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004648244734202608, + "loss": 3.0484, + "theoretical_loss": 4.211914399712019, + "tokens_seen": 262864896 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046481444332998993, + "loss": 3.0421, + "theoretical_loss": 4.211788554801198, + "tokens_seen": 262930432 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046480441323971917, + "loss": 3.1647, + "theoretical_loss": 4.211662750033895, + "tokens_seen": 262995968 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046479438314944835, + "loss": 2.9247, + "theoretical_loss": 4.211536985387307, + "tokens_seen": 263061504 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046478435305917753, + "loss": 3.1853, + "theoretical_loss": 4.211411260838647, + "tokens_seen": 263127040 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004647743229689067, + "loss": 3.0719, + "theoretical_loss": 4.2112855763651496, + "tokens_seen": 263192576 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046476429287863595, + "loss": 3.1743, + "theoretical_loss": 4.211159931944065, + "tokens_seen": 263258112 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004647542627883651, + "loss": 3.3044, + "theoretical_loss": 4.211034327552666, + "tokens_seen": 263323648 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004647442326980943, + "loss": 3.172, + "theoretical_loss": 4.210908763168239, + "tokens_seen": 263389184 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046473420260782344, + "loss": 2.8232, + "theoretical_loss": 4.210783238768093, + "tokens_seen": 263454720 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046472417251755267, + "loss": 2.8838, + "theoretical_loss": 4.210657754329553, + "tokens_seen": 263520256 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004647141424272819, + "loss": 3.1246, + "theoretical_loss": 4.210532309829965, + "tokens_seen": 263585792 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046470411233701103, + "loss": 3.0826, + "theoretical_loss": 4.21040690524669, + "tokens_seen": 263651328 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046469408224674027, + "loss": 3.0629, + "theoretical_loss": 4.21028154055711, + "tokens_seen": 263716864 + }, + { + "epoch": 0.08, + "objective/train/docs_used": 309875, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5037975311279297, + "objective/train/theoretical_loss": 4.2101562157386265, + "objective/train/tokens_used": 284242400, + "theoretical_loss": 4.2101562157386265, + "tokens_seen": 263782400 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004646840521564694, + "loss": 2.8095, + "theoretical_loss": 4.2101562157386265, + "tokens_seen": 263782400 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046467402206619863, + "loss": 3.2129, + "theoretical_loss": 4.210030930768655, + "tokens_seen": 263847936 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004646639919759278, + "loss": 3.3535, + "theoretical_loss": 4.2099056856246335, + "tokens_seen": 263913472 + }, + { + "epoch": 0.08, + "learning_rate": 0.000464653961885657, + "loss": 3.0754, + "theoretical_loss": 4.209780480284017, + "tokens_seen": 263979008 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004646439317953862, + "loss": 3.2122, + "theoretical_loss": 4.209655314724279, + "tokens_seen": 264044544 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046463390170511535, + "loss": 2.9998, + "theoretical_loss": 4.209530188922911, + "tokens_seen": 264110080 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046462387161484454, + "loss": 3.2189, + "theoretical_loss": 4.209405102857422, + "tokens_seen": 264175616 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046461384152457377, + "loss": 3.1448, + "theoretical_loss": 4.209280056505342, + "tokens_seen": 264241152 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004646038114343029, + "loss": 3.1553, + "theoretical_loss": 4.209155049844217, + "tokens_seen": 264306688 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046459378134403213, + "loss": 3.0602, + "theoretical_loss": 4.209030082851612, + "tokens_seen": 264372224 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004645837512537613, + "loss": 3.1091, + "theoretical_loss": 4.208905155505109, + "tokens_seen": 264437760 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004645737211634905, + "loss": 2.961, + "theoretical_loss": 4.20878026778231, + "tokens_seen": 264503296 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004645636910732197, + "loss": 2.9389, + "theoretical_loss": 4.208655419660834, + "tokens_seen": 264568832 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046455366098294886, + "loss": 3.2368, + "theoretical_loss": 4.208530611118321, + "tokens_seen": 264634368 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046454363089267804, + "loss": 3.2025, + "theoretical_loss": 4.208405842132423, + "tokens_seen": 264699904 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004645336008024073, + "loss": 3.3101, + "theoretical_loss": 4.208281112680817, + "tokens_seen": 264765440 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004645235707121364, + "loss": 2.9905, + "theoretical_loss": 4.208156422741195, + "tokens_seen": 264830976 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046451354062186564, + "loss": 3.3088, + "theoretical_loss": 4.208031772291265, + "tokens_seen": 264896512 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046450351053159476, + "loss": 3.1124, + "theoretical_loss": 4.207907161308757, + "tokens_seen": 264962048 + }, + { + "epoch": 0.08, + "learning_rate": 0.000464493480441324, + "loss": 3.011, + "theoretical_loss": 4.2077825897714165, + "tokens_seen": 265027584 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004644834503510532, + "loss": 3.1281, + "theoretical_loss": 4.207658057657008, + "tokens_seen": 265093120 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046447342026078236, + "loss": 3.2965, + "theoretical_loss": 4.207533564943316, + "tokens_seen": 265158656 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046446339017051154, + "loss": 3.0647, + "theoretical_loss": 4.207409111608138, + "tokens_seen": 265224192 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004644533600802407, + "loss": 2.9901, + "theoretical_loss": 4.2072846976292935, + "tokens_seen": 265289728 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004644433299899699, + "loss": 3.0768, + "theoretical_loss": 4.2071603229846195, + "tokens_seen": 265355264 + }, + { + "epoch": 0.08, + "objective/train/docs_used": 311046, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.1395962238311768, + "objective/train/theoretical_loss": 4.20703598765197, + "objective/train/tokens_used": 285880800, + "theoretical_loss": 4.20703598765197, + "tokens_seen": 265420800 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046443329989969914, + "loss": 3.1445, + "theoretical_loss": 4.20703598765197, + "tokens_seen": 265420800 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046442326980942827, + "loss": 3.0687, + "theoretical_loss": 4.206911691609217, + "tokens_seen": 265486336 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004644132397191575, + "loss": 2.8642, + "theoretical_loss": 4.206787434834251, + "tokens_seen": 265551872 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004644032096288867, + "loss": 3.3262, + "theoretical_loss": 4.20666321730498, + "tokens_seen": 265617408 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046439317953861586, + "loss": 3.095, + "theoretical_loss": 4.206539038999329, + "tokens_seen": 265682944 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046438314944834505, + "loss": 2.9848, + "theoretical_loss": 4.206414899895244, + "tokens_seen": 265748480 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004643731193580742, + "loss": 3.1623, + "theoretical_loss": 4.206290799970685, + "tokens_seen": 265814016 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004643630892678034, + "loss": 3.1056, + "theoretical_loss": 4.206166739203632, + "tokens_seen": 265879552 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046435305917753264, + "loss": 3.2523, + "theoretical_loss": 4.206042717572082, + "tokens_seen": 265945088 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046434302908726177, + "loss": 2.9739, + "theoretical_loss": 4.20591873505405, + "tokens_seen": 266010624 + }, + { + "epoch": 0.08, + "learning_rate": 0.000464332998996991, + "loss": 3.0009, + "theoretical_loss": 4.20579479162757, + "tokens_seen": 266076160 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046432296890672013, + "loss": 3.1793, + "theoretical_loss": 4.205670887270691, + "tokens_seen": 266141696 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046431293881644937, + "loss": 3.0394, + "theoretical_loss": 4.205547021961482, + "tokens_seen": 266207232 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046430290872617855, + "loss": 3.04, + "theoretical_loss": 4.205423195678029, + "tokens_seen": 266272768 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046429287863590773, + "loss": 2.7836, + "theoretical_loss": 4.205299408398435, + "tokens_seen": 266338304 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004642828485456369, + "loss": 3.4339, + "theoretical_loss": 4.2051756601008226, + "tokens_seen": 266403840 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046427281845536615, + "loss": 3.2731, + "theoretical_loss": 4.20505195076333, + "tokens_seen": 266469376 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004642627883650953, + "loss": 2.9399, + "theoretical_loss": 4.204928280364115, + "tokens_seen": 266534912 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004642527582748245, + "loss": 2.6865, + "theoretical_loss": 4.20480464888135, + "tokens_seen": 266600448 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046424272818455364, + "loss": 3.1693, + "theoretical_loss": 4.204681056293228, + "tokens_seen": 266665984 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046423269809428287, + "loss": 3.178, + "theoretical_loss": 4.204557502577957, + "tokens_seen": 266731520 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046422266800401205, + "loss": 3.0357, + "theoretical_loss": 4.204433987713767, + "tokens_seen": 266797056 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046421263791374123, + "loss": 3.1383, + "theoretical_loss": 4.2043105116789, + "tokens_seen": 266862592 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004642026078234704, + "loss": 3.1161, + "theoretical_loss": 4.204187074451617, + "tokens_seen": 266928128 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004641925777331996, + "loss": 3.0181, + "theoretical_loss": 4.204063676010202, + "tokens_seen": 266993664 + }, + { + "epoch": 0.08, + "objective/train/docs_used": 311732, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.2471554279327393, + "objective/train/theoretical_loss": 4.203940316332948, + "objective/train/tokens_used": 287519200, + "theoretical_loss": 4.203940316332948, + "tokens_seen": 267059200 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004641825476429288, + "loss": 3.2104, + "theoretical_loss": 4.203940316332948, + "tokens_seen": 267059200 + }, + { + "epoch": 0.08, + "learning_rate": 0.000464172517552658, + "loss": 3.0556, + "theoretical_loss": 4.203816995398171, + "tokens_seen": 267124736 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046416248746238714, + "loss": 2.944, + "theoretical_loss": 4.203693713184203, + "tokens_seen": 267190272 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004641524573721164, + "loss": 3.212, + "theoretical_loss": 4.203570469669392, + "tokens_seen": 267255808 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004641424272818455, + "loss": 3.2094, + "theoretical_loss": 4.203447264832107, + "tokens_seen": 267321344 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046413239719157474, + "loss": 3.4161, + "theoretical_loss": 4.203324098650731, + "tokens_seen": 267386880 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004641223671013039, + "loss": 3.1592, + "theoretical_loss": 4.203200971103666, + "tokens_seen": 267452416 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004641123370110331, + "loss": 3.1786, + "theoretical_loss": 4.20307788216933, + "tokens_seen": 267517952 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004641023069207623, + "loss": 3.2507, + "theoretical_loss": 4.202954831826159, + "tokens_seen": 267583488 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004640922768304915, + "loss": 3.132, + "theoretical_loss": 4.202831820052609, + "tokens_seen": 267649024 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046408224674022064, + "loss": 3.032, + "theoretical_loss": 4.202708846827148, + "tokens_seen": 267714560 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004640722166499499, + "loss": 2.9754, + "theoretical_loss": 4.202585912128266, + "tokens_seen": 267780096 + }, + { + "epoch": 0.08, + "learning_rate": 0.000464062186559679, + "loss": 3.0707, + "theoretical_loss": 4.202463015934468, + "tokens_seen": 267845632 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046405215646940824, + "loss": 2.8317, + "theoretical_loss": 4.202340158224277, + "tokens_seen": 267911168 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004640421263791374, + "loss": 3.2203, + "theoretical_loss": 4.202217338976231, + "tokens_seen": 267976704 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004640320962888666, + "loss": 2.7621, + "theoretical_loss": 4.2020945581688895, + "tokens_seen": 268042240 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004640220661985958, + "loss": 2.9985, + "theoretical_loss": 4.201971815780826, + "tokens_seen": 268107776 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046401203610832496, + "loss": 3.1052, + "theoretical_loss": 4.201849111790631, + "tokens_seen": 268173312 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046400200601805414, + "loss": 3.1588, + "theoretical_loss": 4.201726446176915, + "tokens_seen": 268238848 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004639919759277834, + "loss": 3.1762, + "theoretical_loss": 4.201603818918302, + "tokens_seen": 268304384 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004639819458375125, + "loss": 2.9964, + "theoretical_loss": 4.201481229993435, + "tokens_seen": 268369920 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046397191574724174, + "loss": 2.9145, + "theoretical_loss": 4.201358679380976, + "tokens_seen": 268435456 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004639618856569709, + "loss": 3.1938, + "theoretical_loss": 4.201236167059601, + "tokens_seen": 268500992 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004639518555667001, + "loss": 2.9321, + "theoretical_loss": 4.201113693008002, + "tokens_seen": 268566528 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046394182547642934, + "loss": 2.843, + "theoretical_loss": 4.200991257204894, + "tokens_seen": 268632064 + }, + { + "epoch": 0.08, + "objective/train/docs_used": 312511, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.1893441677093506, + "objective/train/theoretical_loss": 4.2008688596290025, + "objective/train/tokens_used": 289157600, + "theoretical_loss": 4.2008688596290025, + "tokens_seen": 268697600 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046393179538615847, + "loss": 3.3237, + "theoretical_loss": 4.2008688596290025, + "tokens_seen": 268697600 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004639217652958877, + "loss": 3.1753, + "theoretical_loss": 4.200746500259073, + "tokens_seen": 268763136 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004639117352056169, + "loss": 2.9467, + "theoretical_loss": 4.200624179073869, + "tokens_seen": 268828672 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046390170511534606, + "loss": 3.0959, + "theoretical_loss": 4.2005018960521685, + "tokens_seen": 268894208 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046389167502507525, + "loss": 3.1224, + "theoretical_loss": 4.200379651172769, + "tokens_seen": 268959744 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004638816449348044, + "loss": 3.1511, + "theoretical_loss": 4.200257444414483, + "tokens_seen": 269025280 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004638716148445336, + "loss": 3.2735, + "theoretical_loss": 4.200135275756139, + "tokens_seen": 269090816 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046386158475426284, + "loss": 3.0709, + "theoretical_loss": 4.200013145176587, + "tokens_seen": 269156352 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046385155466399197, + "loss": 3.3529, + "theoretical_loss": 4.199891052654689, + "tokens_seen": 269221888 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004638415245737212, + "loss": 3.1699, + "theoretical_loss": 4.199768998169326, + "tokens_seen": 269287424 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046383149448345033, + "loss": 3.1224, + "theoretical_loss": 4.199646981699395, + "tokens_seen": 269352960 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046382146439317957, + "loss": 3.1028, + "theoretical_loss": 4.199525003223812, + "tokens_seen": 269418496 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046381143430290875, + "loss": 2.9704, + "theoretical_loss": 4.199403062721506, + "tokens_seen": 269484032 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046380140421263793, + "loss": 3.1699, + "theoretical_loss": 4.199281160171427, + "tokens_seen": 269549568 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004637913741223671, + "loss": 2.773, + "theoretical_loss": 4.1991592955525405, + "tokens_seen": 269615104 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046378134403209635, + "loss": 3.1007, + "theoretical_loss": 4.199037468843825, + "tokens_seen": 269680640 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004637713139418255, + "loss": 3.3052, + "theoretical_loss": 4.198915680024282, + "tokens_seen": 269746176 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004637612838515547, + "loss": 2.9844, + "theoretical_loss": 4.198793929072925, + "tokens_seen": 269811712 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046375125376128384, + "loss": 3.1632, + "theoretical_loss": 4.198672215968785, + "tokens_seen": 269877248 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046374122367101307, + "loss": 3.0686, + "theoretical_loss": 4.198550540690912, + "tokens_seen": 269942784 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046373119358074225, + "loss": 2.9668, + "theoretical_loss": 4.198428903218371, + "tokens_seen": 270008320 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046372116349047143, + "loss": 2.9512, + "theoretical_loss": 4.198307303530243, + "tokens_seen": 270073856 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004637111334002006, + "loss": 3.3384, + "theoretical_loss": 4.198185741605628, + "tokens_seen": 270139392 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004637011033099298, + "loss": 3.2329, + "theoretical_loss": 4.19806421742364, + "tokens_seen": 270204928 + }, + { + "epoch": 0.08, + "learning_rate": 0.000463691073219659, + "loss": 2.7934, + "theoretical_loss": 4.197942730963412, + "tokens_seen": 270270464 + }, + { + "epoch": 0.08, + "objective/train/docs_used": 313588, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.1681175231933594, + "objective/train/theoretical_loss": 4.19782128220409, + "objective/train/tokens_used": 290796000, + "theoretical_loss": 4.19782128220409, + "tokens_seen": 270336000 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004636810431293882, + "loss": 3.1274, + "theoretical_loss": 4.19782128220409, + "tokens_seen": 270336000 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046367101303911734, + "loss": 3.231, + "theoretical_loss": 4.19769987112484, + "tokens_seen": 270401536 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004636609829488466, + "loss": 2.9321, + "theoretical_loss": 4.1975784977048445, + "tokens_seen": 270467072 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004636509528585757, + "loss": 3.1377, + "theoretical_loss": 4.1974571619233, + "tokens_seen": 270532608 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046364092276830494, + "loss": 2.8866, + "theoretical_loss": 4.197335863759422, + "tokens_seen": 270598144 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004636308926780341, + "loss": 3.0662, + "theoretical_loss": 4.1972146031924416, + "tokens_seen": 270663680 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004636208625877633, + "loss": 2.9822, + "theoretical_loss": 4.197093380201606, + "tokens_seen": 270729216 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004636108324974925, + "loss": 3.1604, + "theoretical_loss": 4.196972194766179, + "tokens_seen": 270794752 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004636008024072217, + "loss": 2.7832, + "theoretical_loss": 4.196851046865442, + "tokens_seen": 270860288 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046359077231695084, + "loss": 3.0355, + "theoretical_loss": 4.1967299364786905, + "tokens_seen": 270925824 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004635807422266801, + "loss": 3.1407, + "theoretical_loss": 4.196608863585239, + "tokens_seen": 270991360 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004635707121364092, + "loss": 2.861, + "theoretical_loss": 4.1964878281644165, + "tokens_seen": 271056896 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046356068204613844, + "loss": 3.2408, + "theoretical_loss": 4.19636683019557, + "tokens_seen": 271122432 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004635506519558676, + "loss": 3.0655, + "theoretical_loss": 4.196245869658061, + "tokens_seen": 271187968 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004635406218655968, + "loss": 2.9336, + "theoretical_loss": 4.1961249465312696, + "tokens_seen": 271253504 + }, + { + "epoch": 0.08, + "learning_rate": 0.000463530591775326, + "loss": 2.9904, + "theoretical_loss": 4.196004060794589, + "tokens_seen": 271319040 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046352056168505516, + "loss": 2.8734, + "theoretical_loss": 4.195883212427433, + "tokens_seen": 271384576 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046351053159478434, + "loss": 3.0506, + "theoretical_loss": 4.195762401409229, + "tokens_seen": 271450112 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004635005015045136, + "loss": 2.8748, + "theoretical_loss": 4.19564162771942, + "tokens_seen": 271515648 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004634904714142427, + "loss": 3.0623, + "theoretical_loss": 4.195520891337466, + "tokens_seen": 271581184 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046348044132397194, + "loss": 3.0601, + "theoretical_loss": 4.195400192242845, + "tokens_seen": 271646720 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046347041123370107, + "loss": 2.7883, + "theoretical_loss": 4.19527953041505, + "tokens_seen": 271712256 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004634603811434303, + "loss": 3.1026, + "theoretical_loss": 4.19515890583359, + "tokens_seen": 271777792 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004634503510531595, + "loss": 3.1229, + "theoretical_loss": 4.195038318477989, + "tokens_seen": 271843328 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046344032096288867, + "loss": 2.9915, + "theoretical_loss": 4.194917768327789, + "tokens_seen": 271908864 + }, + { + "epoch": 0.08, + "objective/train/docs_used": 314289, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.050889253616333, + "objective/train/theoretical_loss": 4.194797255362549, + "objective/train/tokens_used": 292434400, + "theoretical_loss": 4.194797255362549, + "tokens_seen": 271974400 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046343029087261785, + "loss": 3.1228, + "theoretical_loss": 4.194797255362549, + "tokens_seen": 271974400 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004634202607823471, + "loss": 3.1203, + "theoretical_loss": 4.194676779561841, + "tokens_seen": 272039936 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004634102306920762, + "loss": 2.7881, + "theoretical_loss": 4.194556340905256, + "tokens_seen": 272105472 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046340020060180545, + "loss": 3.0799, + "theoretical_loss": 4.194435939372401, + "tokens_seen": 272171008 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046339017051153457, + "loss": 3.032, + "theoretical_loss": 4.194315574942896, + "tokens_seen": 272236544 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004633801404212638, + "loss": 3.2427, + "theoretical_loss": 4.194195247596381, + "tokens_seen": 272302080 + }, + { + "epoch": 0.08, + "learning_rate": 0.000463370110330993, + "loss": 2.9247, + "theoretical_loss": 4.19407495731251, + "tokens_seen": 272367616 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046336008024072217, + "loss": 2.9747, + "theoretical_loss": 4.193954704070952, + "tokens_seen": 272433152 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046335005015045135, + "loss": 3.1666, + "theoretical_loss": 4.193834487851396, + "tokens_seen": 272498688 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046334002006018053, + "loss": 3.0714, + "theoretical_loss": 4.193714308633542, + "tokens_seen": 272564224 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004633299899699097, + "loss": 3.0147, + "theoretical_loss": 4.1935941663971095, + "tokens_seen": 272629760 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046331995987963895, + "loss": 3.1997, + "theoretical_loss": 4.193474061121833, + "tokens_seen": 272695296 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004633099297893681, + "loss": 2.8764, + "theoretical_loss": 4.193353992787463, + "tokens_seen": 272760832 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004632998996990973, + "loss": 3.2184, + "theoretical_loss": 4.193233961373766, + "tokens_seen": 272826368 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046328986960882644, + "loss": 3.1254, + "theoretical_loss": 4.1931139668605235, + "tokens_seen": 272891904 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004632798395185557, + "loss": 3.0326, + "theoretical_loss": 4.192994009227535, + "tokens_seen": 272957440 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046326980942828485, + "loss": 3.0574, + "theoretical_loss": 4.192874088454613, + "tokens_seen": 273022976 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046325977933801404, + "loss": 3.1175, + "theoretical_loss": 4.19275420452159, + "tokens_seen": 273088512 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004632497492477432, + "loss": 3.0634, + "theoretical_loss": 4.192634357408309, + "tokens_seen": 273154048 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046323971915747245, + "loss": 2.7333, + "theoretical_loss": 4.192514547094634, + "tokens_seen": 273219584 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004632296890672016, + "loss": 2.9853, + "theoretical_loss": 4.192394773560441, + "tokens_seen": 273285120 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004632196589769308, + "loss": 3.1364, + "theoretical_loss": 4.192275036785625, + "tokens_seen": 273350656 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046320962888666, + "loss": 2.8319, + "theoretical_loss": 4.192155336750094, + "tokens_seen": 273416192 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004631995987963892, + "loss": 3.0046, + "theoretical_loss": 4.192035673433773, + "tokens_seen": 273481728 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004631895687061184, + "loss": 3.1155, + "theoretical_loss": 4.191916046816605, + "tokens_seen": 273547264 + }, + { + "epoch": 0.08, + "objective/train/docs_used": 314822, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.1304068565368652, + "objective/train/theoretical_loss": 4.191796456878544, + "objective/train/tokens_used": 294072800, + "theoretical_loss": 4.191796456878544, + "tokens_seen": 273612800 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046317953861584754, + "loss": 2.943, + "theoretical_loss": 4.191796456878544, + "tokens_seen": 273612800 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004631695085255768, + "loss": 2.8587, + "theoretical_loss": 4.191676903599563, + "tokens_seen": 273678336 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004631594784353059, + "loss": 3.1101, + "theoretical_loss": 4.191557386959651, + "tokens_seen": 273743872 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046314944834503514, + "loss": 3.1475, + "theoretical_loss": 4.191437906938811, + "tokens_seen": 273809408 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004631394182547643, + "loss": 3.0439, + "theoretical_loss": 4.191318463517062, + "tokens_seen": 273874944 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004631293881644935, + "loss": 3.1469, + "theoretical_loss": 4.19119905667444, + "tokens_seen": 273940480 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004631193580742227, + "loss": 3.1943, + "theoretical_loss": 4.191079686390996, + "tokens_seen": 274006016 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004631093279839519, + "loss": 3.1121, + "theoretical_loss": 4.190960352646796, + "tokens_seen": 274071552 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046309929789368104, + "loss": 3.0344, + "theoretical_loss": 4.190841055421921, + "tokens_seen": 274137088 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004630892678034103, + "loss": 3.0578, + "theoretical_loss": 4.19072179469647, + "tokens_seen": 274202624 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004630792377131394, + "loss": 3.1027, + "theoretical_loss": 4.190602570450556, + "tokens_seen": 274268160 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046306920762286864, + "loss": 3.0957, + "theoretical_loss": 4.190483382664308, + "tokens_seen": 274333696 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004630591775325978, + "loss": 2.8938, + "theoretical_loss": 4.19036423131787, + "tokens_seen": 274399232 + }, + { + "epoch": 0.08, + "learning_rate": 0.000463049147442327, + "loss": 2.8262, + "theoretical_loss": 4.190245116391403, + "tokens_seen": 274464768 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004630391173520562, + "loss": 2.9228, + "theoretical_loss": 4.190126037865082, + "tokens_seen": 274530304 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046302908726178536, + "loss": 3.0006, + "theoretical_loss": 4.190006995719098, + "tokens_seen": 274595840 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046301905717151455, + "loss": 2.9758, + "theoretical_loss": 4.1898879899336565, + "tokens_seen": 274661376 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004630090270812438, + "loss": 3.2112, + "theoretical_loss": 4.189769020488981, + "tokens_seen": 274726912 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004629989969909729, + "loss": 3.0473, + "theoretical_loss": 4.189650087365309, + "tokens_seen": 274792448 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046298896690070214, + "loss": 2.8983, + "theoretical_loss": 4.189531190542893, + "tokens_seen": 274857984 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046297893681043127, + "loss": 2.7118, + "theoretical_loss": 4.189412330002001, + "tokens_seen": 274923520 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004629689067201605, + "loss": 2.9403, + "theoretical_loss": 4.189293505722918, + "tokens_seen": 274989056 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004629588766298897, + "loss": 2.9515, + "theoretical_loss": 4.189174717685942, + "tokens_seen": 275054592 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046294884653961887, + "loss": 2.7063, + "theoretical_loss": 4.189055965871389, + "tokens_seen": 275120128 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046293881644934805, + "loss": 3.0041, + "theoretical_loss": 4.188937250259587, + "tokens_seen": 275185664 + }, + { + "epoch": 0.08, + "objective/train/docs_used": 318374, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.3687500953674316, + "objective/train/theoretical_loss": 4.188818570830883, + "objective/train/tokens_used": 295711200, + "theoretical_loss": 4.188818570830883, + "tokens_seen": 275251200 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004629287863590773, + "loss": 2.9334, + "theoretical_loss": 4.188818570830883, + "tokens_seen": 275251200 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004629187562688064, + "loss": 2.9519, + "theoretical_loss": 4.188699927565638, + "tokens_seen": 275316736 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046290872617853565, + "loss": 3.24, + "theoretical_loss": 4.188581320444228, + "tokens_seen": 275382272 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046289869608826477, + "loss": 3.0205, + "theoretical_loss": 4.1884627494470426, + "tokens_seen": 275447808 + }, + { + "epoch": 0.08, + "learning_rate": 0.000462888665997994, + "loss": 3.1389, + "theoretical_loss": 4.1883442145544905, + "tokens_seen": 275513344 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004628786359077232, + "loss": 3.117, + "theoretical_loss": 4.188225715746992, + "tokens_seen": 275578880 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046286860581745237, + "loss": 3.0967, + "theoretical_loss": 4.188107253004986, + "tokens_seen": 275644416 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046285857572718155, + "loss": 3.2164, + "theoretical_loss": 4.187988826308925, + "tokens_seen": 275709952 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046284854563691073, + "loss": 3.0597, + "theoretical_loss": 4.187870435639275, + "tokens_seen": 275775488 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004628385155466399, + "loss": 2.9352, + "theoretical_loss": 4.18775208097652, + "tokens_seen": 275841024 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046282848545636915, + "loss": 3.2597, + "theoretical_loss": 4.187633762301159, + "tokens_seen": 275906560 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004628184553660983, + "loss": 3.0023, + "theoretical_loss": 4.187515479593704, + "tokens_seen": 275972096 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004628084252758275, + "loss": 2.9841, + "theoretical_loss": 4.187397232834683, + "tokens_seen": 276037632 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046279839518555664, + "loss": 3.3956, + "theoretical_loss": 4.187279022004642, + "tokens_seen": 276103168 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004627883650952859, + "loss": 3.2086, + "theoretical_loss": 4.1871608470841375, + "tokens_seen": 276168704 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046277833500501505, + "loss": 2.8913, + "theoretical_loss": 4.1870427080537445, + "tokens_seen": 276234240 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046276830491474424, + "loss": 2.9785, + "theoretical_loss": 4.1869246048940525, + "tokens_seen": 276299776 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004627582748244734, + "loss": 3.138, + "theoretical_loss": 4.186806537585666, + "tokens_seen": 276365312 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046274824473420265, + "loss": 2.9597, + "theoretical_loss": 4.186688506109202, + "tokens_seen": 276430848 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004627382146439318, + "loss": 3.1554, + "theoretical_loss": 4.186570510445296, + "tokens_seen": 276496384 + }, + { + "epoch": 0.08, + "learning_rate": 0.000462728184553661, + "loss": 3.1408, + "theoretical_loss": 4.186452550574599, + "tokens_seen": 276561920 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046271815446339014, + "loss": 2.8797, + "theoretical_loss": 4.186334626477774, + "tokens_seen": 276627456 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004627081243731194, + "loss": 2.892, + "theoretical_loss": 4.186216738135501, + "tokens_seen": 276692992 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046269809428284856, + "loss": 2.6408, + "theoretical_loss": 4.186098885528473, + "tokens_seen": 276758528 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046268806419257774, + "loss": 3.0981, + "theoretical_loss": 4.185981068637401, + "tokens_seen": 276824064 + }, + { + "epoch": 0.08, + "objective/train/docs_used": 320606, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.31486439704895, + "objective/train/theoretical_loss": 4.185863287443008, + "objective/train/tokens_used": 296742368, + "theoretical_loss": 4.185863287443008, + "tokens_seen": 276889600 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004626780341023069, + "loss": 3.0755, + "theoretical_loss": 4.185863287443008, + "tokens_seen": 276889600 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004626680040120361, + "loss": 2.9726, + "theoretical_loss": 4.185745541926035, + "tokens_seen": 276955136 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004626579739217653, + "loss": 3.1442, + "theoretical_loss": 4.185627832067237, + "tokens_seen": 277020672 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004626479438314945, + "loss": 3.17, + "theoretical_loss": 4.1855101578473795, + "tokens_seen": 277086208 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046263791374122364, + "loss": 3.0287, + "theoretical_loss": 4.18539251924725, + "tokens_seen": 277151744 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004626278836509529, + "loss": 2.8911, + "theoretical_loss": 4.185274916247646, + "tokens_seen": 277217280 + }, + { + "epoch": 0.08, + "learning_rate": 0.000462617853560682, + "loss": 2.869, + "theoretical_loss": 4.185157348829383, + "tokens_seen": 277282816 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046260782347041124, + "loss": 3.1912, + "theoretical_loss": 4.185039816973289, + "tokens_seen": 277348352 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004625977933801404, + "loss": 2.8488, + "theoretical_loss": 4.184922320660207, + "tokens_seen": 277413888 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004625877632898696, + "loss": 3.0904, + "theoretical_loss": 4.184804859870997, + "tokens_seen": 277479424 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004625777331995988, + "loss": 3.0076, + "theoretical_loss": 4.184687434586531, + "tokens_seen": 277544960 + }, + { + "epoch": 0.08, + "learning_rate": 0.000462567703109328, + "loss": 2.8916, + "theoretical_loss": 4.184570044787698, + "tokens_seen": 277610496 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046255767301905715, + "loss": 2.8376, + "theoretical_loss": 4.1844526904554, + "tokens_seen": 277676032 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004625476429287864, + "loss": 2.6536, + "theoretical_loss": 4.184335371570556, + "tokens_seen": 277741568 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004625376128385155, + "loss": 2.97, + "theoretical_loss": 4.184218088114097, + "tokens_seen": 277807104 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046252758274824475, + "loss": 3.0261, + "theoretical_loss": 4.1841008400669715, + "tokens_seen": 277872640 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004625175526579739, + "loss": 3.0633, + "theoretical_loss": 4.183983627410142, + "tokens_seen": 277938176 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004625075225677031, + "loss": 3.0716, + "theoretical_loss": 4.183866450124584, + "tokens_seen": 278003712 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004624974924774323, + "loss": 3.0718, + "theoretical_loss": 4.18374930819129, + "tokens_seen": 278069248 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046248746238716147, + "loss": 3.0414, + "theoretical_loss": 4.183632201591264, + "tokens_seen": 278134784 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046247743229689065, + "loss": 2.8162, + "theoretical_loss": 4.18351513030553, + "tokens_seen": 278200320 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004624674022066199, + "loss": 3.1109, + "theoretical_loss": 4.1833980943151206, + "tokens_seen": 278265856 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046245737211634907, + "loss": 3.0844, + "theoretical_loss": 4.183281093601087, + "tokens_seen": 278331392 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046244734202607825, + "loss": 3.2941, + "theoretical_loss": 4.183164128144495, + "tokens_seen": 278396928 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004624373119358075, + "loss": 3.0747, + "theoretical_loss": 4.183047197926422, + "tokens_seen": 278462464 + }, + { + "epoch": 0.08, + "objective/train/docs_used": 320606, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.0842607021331787, + "objective/train/theoretical_loss": 4.182930302927963, + "objective/train/tokens_used": 296742368, + "theoretical_loss": 4.182930302927963, + "tokens_seen": 278528000 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004624272818455366, + "loss": 2.8917, + "theoretical_loss": 4.182930302927963, + "tokens_seen": 278528000 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046241725175526585, + "loss": 3.085, + "theoretical_loss": 4.182813443130227, + "tokens_seen": 278593536 + }, + { + "epoch": 0.08, + "learning_rate": 0.000462407221664995, + "loss": 2.8507, + "theoretical_loss": 4.182696618514337, + "tokens_seen": 278659072 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004623971915747242, + "loss": 2.9847, + "theoretical_loss": 4.18257982906143, + "tokens_seen": 278724608 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004623871614844534, + "loss": 3.0537, + "theoretical_loss": 4.1824630747526585, + "tokens_seen": 278790144 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046237713139418257, + "loss": 2.8442, + "theoretical_loss": 4.182346355569189, + "tokens_seen": 278855680 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046236710130391175, + "loss": 3.2378, + "theoretical_loss": 4.182229671492204, + "tokens_seen": 278921216 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046235707121364093, + "loss": 2.8921, + "theoretical_loss": 4.1821130225028975, + "tokens_seen": 278986752 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004623470411233701, + "loss": 2.9071, + "theoretical_loss": 4.1819964085824815, + "tokens_seen": 279052288 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046233701103309935, + "loss": 2.8913, + "theoretical_loss": 4.181879829712178, + "tokens_seen": 279117824 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004623269809428285, + "loss": 3.2138, + "theoretical_loss": 4.181763285873231, + "tokens_seen": 279183360 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004623169508525577, + "loss": 3.0489, + "theoretical_loss": 4.181646777046889, + "tokens_seen": 279248896 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046230692076228684, + "loss": 2.8127, + "theoretical_loss": 4.181530303214423, + "tokens_seen": 279314432 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004622968906720161, + "loss": 3.1112, + "theoretical_loss": 4.181413864357115, + "tokens_seen": 279379968 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046228686058174525, + "loss": 2.9245, + "theoretical_loss": 4.181297460456262, + "tokens_seen": 279445504 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046227683049147444, + "loss": 2.6736, + "theoretical_loss": 4.181181091493174, + "tokens_seen": 279511040 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004622668004012036, + "loss": 3.0354, + "theoretical_loss": 4.181064757449178, + "tokens_seen": 279576576 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046225677031093285, + "loss": 3.3787, + "theoretical_loss": 4.180948458305615, + "tokens_seen": 279642112 + }, + { + "epoch": 0.08, + "learning_rate": 0.000462246740220662, + "loss": 2.9473, + "theoretical_loss": 4.180832194043836, + "tokens_seen": 279707648 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004622367101303912, + "loss": 2.9305, + "theoretical_loss": 4.180715964645213, + "tokens_seen": 279773184 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046222668004012034, + "loss": 3.0333, + "theoretical_loss": 4.180599770091126, + "tokens_seen": 279838720 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004622166499498496, + "loss": 2.9703, + "theoretical_loss": 4.180483610362975, + "tokens_seen": 279904256 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046220661985957876, + "loss": 2.7939, + "theoretical_loss": 4.18036748544217, + "tokens_seen": 279969792 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046219658976930794, + "loss": 3.0377, + "theoretical_loss": 4.180251395310137, + "tokens_seen": 280035328 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004621865596790371, + "loss": 3.0576, + "theoretical_loss": 4.1801353399483165, + "tokens_seen": 280100864 + }, + { + "epoch": 0.08, + "objective/train/docs_used": 320606, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.94958758354187, + "objective/train/theoretical_loss": 4.180019319338163, + "objective/train/tokens_used": 296742368, + "theoretical_loss": 4.180019319338163, + "tokens_seen": 280166400 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004621765295887663, + "loss": 3.0546, + "theoretical_loss": 4.180019319338163, + "tokens_seen": 280166400 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004621664994984955, + "loss": 2.875, + "theoretical_loss": 4.179903333461144, + "tokens_seen": 280231936 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004621564694082247, + "loss": 3.0677, + "theoretical_loss": 4.179787382298744, + "tokens_seen": 280297472 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046214643931795384, + "loss": 3.1748, + "theoretical_loss": 4.179671465832458, + "tokens_seen": 280363008 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004621364092276831, + "loss": 3.0775, + "theoretical_loss": 4.179555584043799, + "tokens_seen": 280428544 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004621263791374122, + "loss": 3.1528, + "theoretical_loss": 4.17943973691429, + "tokens_seen": 280494080 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046211634904714144, + "loss": 2.9587, + "theoretical_loss": 4.179323924425472, + "tokens_seen": 280559616 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004621063189568706, + "loss": 2.9175, + "theoretical_loss": 4.179208146558899, + "tokens_seen": 280625152 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004620962888665998, + "loss": 2.9214, + "theoretical_loss": 4.1790924032961385, + "tokens_seen": 280690688 + }, + { + "epoch": 0.09, + "learning_rate": 0.000462086258776329, + "loss": 2.9464, + "theoretical_loss": 4.178976694618772, + "tokens_seen": 280756224 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004620762286860582, + "loss": 3.1425, + "theoretical_loss": 4.178861020508395, + "tokens_seen": 280821760 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046206619859578735, + "loss": 2.8949, + "theoretical_loss": 4.178745380946619, + "tokens_seen": 280887296 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004620561685055166, + "loss": 2.9406, + "theoretical_loss": 4.178629775915066, + "tokens_seen": 280952832 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004620461384152457, + "loss": 2.9719, + "theoretical_loss": 4.178514205395376, + "tokens_seen": 281018368 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046203610832497495, + "loss": 2.9724, + "theoretical_loss": 4.178398669369201, + "tokens_seen": 281083904 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004620260782347041, + "loss": 3.0159, + "theoretical_loss": 4.178283167818206, + "tokens_seen": 281149440 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004620160481444333, + "loss": 3.1142, + "theoretical_loss": 4.178167700724073, + "tokens_seen": 281214976 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004620060180541625, + "loss": 3.1041, + "theoretical_loss": 4.178052268068494, + "tokens_seen": 281280512 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046199598796389167, + "loss": 2.9121, + "theoretical_loss": 4.177936869833179, + "tokens_seen": 281346048 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046198595787362085, + "loss": 2.8584, + "theoretical_loss": 4.17782150599985, + "tokens_seen": 281411584 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004619759277833501, + "loss": 2.9394, + "theoretical_loss": 4.1777061765502435, + "tokens_seen": 281477120 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004619658976930792, + "loss": 3.0848, + "theoretical_loss": 4.1775908814661085, + "tokens_seen": 281542656 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046195586760280845, + "loss": 2.7208, + "theoretical_loss": 4.17747562072921, + "tokens_seen": 281608192 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046194583751253763, + "loss": 3.0835, + "theoretical_loss": 4.177360394321325, + "tokens_seen": 281673728 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004619358074222668, + "loss": 3.1928, + "theoretical_loss": 4.177245202224246, + "tokens_seen": 281739264 + }, + { + "epoch": 0.09, + "objective/train/docs_used": 320606, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.1719119548797607, + "objective/train/theoretical_loss": 4.17713004441978, + "objective/train/tokens_used": 296742368, + "theoretical_loss": 4.17713004441978, + "tokens_seen": 281804800 + }, + { + "epoch": 0.09, + "learning_rate": 0.000461925777331996, + "loss": 2.9915, + "theoretical_loss": 4.17713004441978, + "tokens_seen": 281804800 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004619157472417252, + "loss": 2.9478, + "theoretical_loss": 4.177014920889745, + "tokens_seen": 281870336 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046190571715145435, + "loss": 3.0954, + "theoretical_loss": 4.176899831615974, + "tokens_seen": 281935872 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004618956870611836, + "loss": 2.9708, + "theoretical_loss": 4.176784776580316, + "tokens_seen": 282001408 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004618856569709127, + "loss": 3.0337, + "theoretical_loss": 4.176669755764632, + "tokens_seen": 282066944 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046187562688064195, + "loss": 3.261, + "theoretical_loss": 4.176554769150796, + "tokens_seen": 282132480 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004618655967903711, + "loss": 3.0503, + "theoretical_loss": 4.176439816720697, + "tokens_seen": 282198016 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004618555667001003, + "loss": 2.8879, + "theoretical_loss": 4.1763248984562376, + "tokens_seen": 282263552 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004618455366098295, + "loss": 2.9648, + "theoretical_loss": 4.176210014339335, + "tokens_seen": 282329088 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004618355065195587, + "loss": 2.673, + "theoretical_loss": 4.17609516435192, + "tokens_seen": 282394624 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046182547642928786, + "loss": 3.2389, + "theoretical_loss": 4.1759803484759335, + "tokens_seen": 282460160 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046181544633901704, + "loss": 2.9818, + "theoretical_loss": 4.175865566693336, + "tokens_seen": 282525696 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004618054162487462, + "loss": 2.7571, + "theoretical_loss": 4.175750818986098, + "tokens_seen": 282591232 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046179538615847545, + "loss": 2.9559, + "theoretical_loss": 4.1756361053362046, + "tokens_seen": 282656768 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004617853560682046, + "loss": 3.1227, + "theoretical_loss": 4.1755214257256545, + "tokens_seen": 282722304 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004617753259779338, + "loss": 3.1455, + "theoretical_loss": 4.17540678013646, + "tokens_seen": 282787840 + }, + { + "epoch": 0.09, + "learning_rate": 0.000461765295887663, + "loss": 3.044, + "theoretical_loss": 4.175292168550648, + "tokens_seen": 282853376 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004617552657973922, + "loss": 2.7818, + "theoretical_loss": 4.175177590950257, + "tokens_seen": 282918912 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046174523570712136, + "loss": 2.7973, + "theoretical_loss": 4.175063047317342, + "tokens_seen": 282984448 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046173520561685054, + "loss": 3.1903, + "theoretical_loss": 4.174948537633968, + "tokens_seen": 283049984 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004617251755265797, + "loss": 3.2353, + "theoretical_loss": 4.174834061882218, + "tokens_seen": 283115520 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046171514543630896, + "loss": 3.0471, + "theoretical_loss": 4.1747196200441845, + "tokens_seen": 283181056 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046170511534603814, + "loss": 2.8341, + "theoretical_loss": 4.174605212101977, + "tokens_seen": 283246592 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004616950852557673, + "loss": 2.8812, + "theoretical_loss": 4.174490838037716, + "tokens_seen": 283312128 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004616850551654965, + "loss": 2.9857, + "theoretical_loss": 4.174376497833537, + "tokens_seen": 283377664 + }, + { + "epoch": 0.09, + "objective/train/docs_used": 320606, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5323517322540283, + "objective/train/theoretical_loss": 4.174262191471587, + "objective/train/tokens_used": 296742368, + "theoretical_loss": 4.174262191471587, + "tokens_seen": 283443200 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004616750250752257, + "loss": 2.8143, + "theoretical_loss": 4.174262191471587, + "tokens_seen": 283443200 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004616649949849549, + "loss": 3.11, + "theoretical_loss": 4.17414791893403, + "tokens_seen": 283508736 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046165496489468404, + "loss": 3.0364, + "theoretical_loss": 4.17403368020304, + "tokens_seen": 283574272 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004616449348044133, + "loss": 3.2199, + "theoretical_loss": 4.173919475260808, + "tokens_seen": 283639808 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004616349047141424, + "loss": 3.0457, + "theoretical_loss": 4.173805304089536, + "tokens_seen": 283705344 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046162487462387164, + "loss": 2.9339, + "theoretical_loss": 4.173691166671439, + "tokens_seen": 283770880 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004616148445336008, + "loss": 3.1206, + "theoretical_loss": 4.173577062988748, + "tokens_seen": 283836416 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046160481444333, + "loss": 2.7604, + "theoretical_loss": 4.173462993023706, + "tokens_seen": 283901952 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004615947843530592, + "loss": 2.7994, + "theoretical_loss": 4.173348956758568, + "tokens_seen": 283967488 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004615847542627884, + "loss": 2.885, + "theoretical_loss": 4.173234954175605, + "tokens_seen": 284033024 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046157472417251755, + "loss": 2.7621, + "theoretical_loss": 4.173120985257102, + "tokens_seen": 284098560 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004615646940822468, + "loss": 3.1411, + "theoretical_loss": 4.173007049985352, + "tokens_seen": 284164096 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004615546639919759, + "loss": 2.7966, + "theoretical_loss": 4.172893148342667, + "tokens_seen": 284229632 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046154463390170515, + "loss": 2.9813, + "theoretical_loss": 4.172779280311372, + "tokens_seen": 284295168 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004615346038114343, + "loss": 3.0168, + "theoretical_loss": 4.172665445873801, + "tokens_seen": 284360704 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004615245737211635, + "loss": 2.9192, + "theoretical_loss": 4.172551645012307, + "tokens_seen": 284426240 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004615145436308927, + "loss": 2.8819, + "theoretical_loss": 4.1724378777092515, + "tokens_seen": 284491776 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046150451354062187, + "loss": 2.5965, + "theoretical_loss": 4.172324143947012, + "tokens_seen": 284557312 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046149448345035105, + "loss": 3.0194, + "theoretical_loss": 4.172210443707979, + "tokens_seen": 284622848 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004614844533600803, + "loss": 2.7609, + "theoretical_loss": 4.1720967769745565, + "tokens_seen": 284688384 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004614744232698094, + "loss": 2.9768, + "theoretical_loss": 4.171983143729159, + "tokens_seen": 284753920 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046146439317953865, + "loss": 2.7417, + "theoretical_loss": 4.1718695439542195, + "tokens_seen": 284819456 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046145436308926783, + "loss": 3.1363, + "theoretical_loss": 4.17175597763218, + "tokens_seen": 284884992 + }, + { + "epoch": 0.09, + "learning_rate": 0.000461444332998997, + "loss": 2.9471, + "theoretical_loss": 4.171642444745497, + "tokens_seen": 284950528 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004614343029087262, + "loss": 3.2626, + "theoretical_loss": 4.1715289452766395, + "tokens_seen": 285016064 + }, + { + "epoch": 0.09, + "objective/train/docs_used": 320606, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.09801983833313, + "objective/train/theoretical_loss": 4.1714154792080915, + "objective/train/tokens_used": 296742368, + "theoretical_loss": 4.1714154792080915, + "tokens_seen": 285081600 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004614242728184554, + "loss": 3.0522, + "theoretical_loss": 4.1714154792080915, + "tokens_seen": 285081600 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046141424272818455, + "loss": 2.7098, + "theoretical_loss": 4.171302046522349, + "tokens_seen": 285147136 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004614042126379138, + "loss": 3.1704, + "theoretical_loss": 4.171188647201921, + "tokens_seen": 285212672 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004613941825476429, + "loss": 3.0845, + "theoretical_loss": 4.1710752812293315, + "tokens_seen": 285278208 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046138415245737215, + "loss": 2.8987, + "theoretical_loss": 4.170961948587115, + "tokens_seen": 285343744 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004613741223671013, + "loss": 2.7445, + "theoretical_loss": 4.17084864925782, + "tokens_seen": 285409280 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004613640922768305, + "loss": 3.0298, + "theoretical_loss": 4.1707353832240095, + "tokens_seen": 285474816 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004613540621865597, + "loss": 2.9432, + "theoretical_loss": 4.170622150468258, + "tokens_seen": 285540352 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004613440320962889, + "loss": 3.1768, + "theoretical_loss": 4.170508950973154, + "tokens_seen": 285605888 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046133400200601806, + "loss": 2.7724, + "theoretical_loss": 4.1703957847213, + "tokens_seen": 285671424 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046132397191574724, + "loss": 2.7251, + "theoretical_loss": 4.170282651695308, + "tokens_seen": 285736960 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004613139418254764, + "loss": 2.9493, + "theoretical_loss": 4.170169551877808, + "tokens_seen": 285802496 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046130391173520566, + "loss": 2.8802, + "theoretical_loss": 4.170056485251439, + "tokens_seen": 285868032 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004612938816449348, + "loss": 3.0992, + "theoretical_loss": 4.169943451798856, + "tokens_seen": 285933568 + }, + { + "epoch": 0.09, + "learning_rate": 0.000461283851554664, + "loss": 2.9054, + "theoretical_loss": 4.169830451502724, + "tokens_seen": 285999104 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004612738214643932, + "loss": 2.8996, + "theoretical_loss": 4.169717484345725, + "tokens_seen": 286064640 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004612637913741224, + "loss": 2.7219, + "theoretical_loss": 4.1696045503105506, + "tokens_seen": 286130176 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046125376128385156, + "loss": 3.0844, + "theoretical_loss": 4.169491649379905, + "tokens_seen": 286195712 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046124373119358074, + "loss": 3.1184, + "theoretical_loss": 4.169378781536509, + "tokens_seen": 286261248 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004612337011033099, + "loss": 2.7949, + "theoretical_loss": 4.169265946763095, + "tokens_seen": 286326784 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046122367101303916, + "loss": 3.0171, + "theoretical_loss": 4.169153145042405, + "tokens_seen": 286392320 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004612136409227683, + "loss": 3.0261, + "theoretical_loss": 4.169040376357199, + "tokens_seen": 286457856 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004612036108324975, + "loss": 2.9589, + "theoretical_loss": 4.168927640690246, + "tokens_seen": 286523392 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046119358074222665, + "loss": 2.9269, + "theoretical_loss": 4.16881493802433, + "tokens_seen": 286588928 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004611835506519559, + "loss": 2.916, + "theoretical_loss": 4.168702268342248, + "tokens_seen": 286654464 + }, + { + "epoch": 0.09, + "objective/train/docs_used": 320606, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.830076217651367, + "objective/train/theoretical_loss": 4.168589631626808, + "objective/train/tokens_used": 296742368, + "theoretical_loss": 4.168589631626808, + "tokens_seen": 286720000 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046117352056168506, + "loss": 2.9636, + "theoretical_loss": 4.168589631626808, + "tokens_seen": 286720000 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046116349047141425, + "loss": 2.7699, + "theoretical_loss": 4.168477027860833, + "tokens_seen": 286785536 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004611534603811434, + "loss": 3.0801, + "theoretical_loss": 4.168364457027158, + "tokens_seen": 286851072 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004611434302908726, + "loss": 3.0063, + "theoretical_loss": 4.168251919108632, + "tokens_seen": 286916608 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004611334002006018, + "loss": 2.8081, + "theoretical_loss": 4.168139414088113, + "tokens_seen": 286982144 + }, + { + "epoch": 0.09, + "learning_rate": 0.000461123370110331, + "loss": 3.0765, + "theoretical_loss": 4.168026941948478, + "tokens_seen": 287047680 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046111334002006015, + "loss": 3.1996, + "theoretical_loss": 4.167914502672611, + "tokens_seen": 287113216 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004611033099297894, + "loss": 2.7884, + "theoretical_loss": 4.1678020962434115, + "tokens_seen": 287178752 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046109327983951857, + "loss": 2.9895, + "theoretical_loss": 4.167689722643792, + "tokens_seen": 287244288 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046108324974924775, + "loss": 3.0916, + "theoretical_loss": 4.1675773818566775, + "tokens_seen": 287309824 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046107321965897693, + "loss": 2.9913, + "theoretical_loss": 4.167465073865006, + "tokens_seen": 287375360 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004610631895687061, + "loss": 2.8553, + "theoretical_loss": 4.167352798651726, + "tokens_seen": 287440896 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004610531594784353, + "loss": 3.0485, + "theoretical_loss": 4.167240556199802, + "tokens_seen": 287506432 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004610431293881645, + "loss": 3.1886, + "theoretical_loss": 4.167128346492211, + "tokens_seen": 287571968 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046103309929789365, + "loss": 2.9215, + "theoretical_loss": 4.16701616951194, + "tokens_seen": 287637504 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004610230692076229, + "loss": 3.0773, + "theoretical_loss": 4.1669040252419896, + "tokens_seen": 287703040 + }, + { + "epoch": 0.09, + "learning_rate": 0.000461013039117352, + "loss": 3.2776, + "theoretical_loss": 4.166791913665375, + "tokens_seen": 287768576 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046100300902708125, + "loss": 3.0554, + "theoretical_loss": 4.166679834765123, + "tokens_seen": 287834112 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046099297893681043, + "loss": 2.9973, + "theoretical_loss": 4.166567788524272, + "tokens_seen": 287899648 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004609829488465396, + "loss": 3.2166, + "theoretical_loss": 4.166455774925875, + "tokens_seen": 287965184 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004609729187562688, + "loss": 3.1155, + "theoretical_loss": 4.166343793952995, + "tokens_seen": 288030720 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046096288866599803, + "loss": 2.9107, + "theoretical_loss": 4.166231845588712, + "tokens_seen": 288096256 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004609528585757272, + "loss": 2.8911, + "theoretical_loss": 4.166119929816113, + "tokens_seen": 288161792 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004609428284854564, + "loss": 3.0025, + "theoretical_loss": 4.166008046618303, + "tokens_seen": 288227328 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004609327983951856, + "loss": 2.9673, + "theoretical_loss": 4.1658961959783944, + "tokens_seen": 288292864 + }, + { + "epoch": 0.09, + "objective/train/docs_used": 320606, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.154595136642456, + "objective/train/theoretical_loss": 4.165784377879517, + "objective/train/tokens_used": 296742368, + "theoretical_loss": 4.165784377879517, + "tokens_seen": 288358400 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046092276830491475, + "loss": 3.198, + "theoretical_loss": 4.165784377879517, + "tokens_seen": 288358400 + }, + { + "epoch": 0.09, + "learning_rate": 0.000460912738214644, + "loss": 3.1049, + "theoretical_loss": 4.165672592304811, + "tokens_seen": 288423936 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004609027081243731, + "loss": 2.9952, + "theoretical_loss": 4.165560839237429, + "tokens_seen": 288489472 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046089267803410235, + "loss": 3.0699, + "theoretical_loss": 4.165449118660536, + "tokens_seen": 288555008 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004608826479438315, + "loss": 3.0712, + "theoretical_loss": 4.16533743055731, + "tokens_seen": 288620544 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004608726178535607, + "loss": 3.1647, + "theoretical_loss": 4.165225774910941, + "tokens_seen": 288686080 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004608625877632899, + "loss": 3.2739, + "theoretical_loss": 4.165114151704634, + "tokens_seen": 288751616 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004608525576730191, + "loss": 3.0161, + "theoretical_loss": 4.165002560921601, + "tokens_seen": 288817152 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046084252758274826, + "loss": 3.1432, + "theoretical_loss": 4.164891002545073, + "tokens_seen": 288882688 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046083249749247744, + "loss": 2.9712, + "theoretical_loss": 4.16477947655829, + "tokens_seen": 288948224 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004608224674022066, + "loss": 2.8887, + "theoretical_loss": 4.164667982944504, + "tokens_seen": 289013760 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046081243731193586, + "loss": 2.7605, + "theoretical_loss": 4.164556521686981, + "tokens_seen": 289079296 + }, + { + "epoch": 0.09, + "learning_rate": 0.000460802407221665, + "loss": 3.1795, + "theoretical_loss": 4.1644450927689975, + "tokens_seen": 289144832 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004607923771313942, + "loss": 2.902, + "theoretical_loss": 4.164333696173846, + "tokens_seen": 289210368 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004607823470411234, + "loss": 3.0938, + "theoretical_loss": 4.164222331884827, + "tokens_seen": 289275904 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004607723169508526, + "loss": 2.9628, + "theoretical_loss": 4.164110999885256, + "tokens_seen": 289341440 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046076228686058176, + "loss": 2.8941, + "theoretical_loss": 4.163999700158462, + "tokens_seen": 289406976 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046075225677031094, + "loss": 3.0364, + "theoretical_loss": 4.163888432687784, + "tokens_seen": 289472512 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004607422266800401, + "loss": 3.1758, + "theoretical_loss": 4.163777197456573, + "tokens_seen": 289538048 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046073219658976936, + "loss": 2.7997, + "theoretical_loss": 4.163665994448197, + "tokens_seen": 289603584 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004607221664994985, + "loss": 2.9896, + "theoretical_loss": 4.163554823646027, + "tokens_seen": 289669120 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004607121364092277, + "loss": 2.9085, + "theoretical_loss": 4.163443685033458, + "tokens_seen": 289734656 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046070210631895685, + "loss": 2.8829, + "theoretical_loss": 4.163332578593889, + "tokens_seen": 289800192 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004606920762286861, + "loss": 2.846, + "theoretical_loss": 4.163221504310734, + "tokens_seen": 289865728 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046068204613841526, + "loss": 2.9586, + "theoretical_loss": 4.1631104621674195, + "tokens_seen": 289931264 + }, + { + "epoch": 0.09, + "objective/train/docs_used": 320606, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.6529836654663086, + "objective/train/theoretical_loss": 4.162999452147384, + "objective/train/tokens_used": 296742368, + "theoretical_loss": 4.162999452147384, + "tokens_seen": 289996800 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046067201604814445, + "loss": 2.9531, + "theoretical_loss": 4.162999452147384, + "tokens_seen": 289996800 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004606619859578736, + "loss": 3.323, + "theoretical_loss": 4.1628884742340775, + "tokens_seen": 290062336 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004606519558676028, + "loss": 3.1529, + "theoretical_loss": 4.162777528410963, + "tokens_seen": 290127872 + }, + { + "epoch": 0.09, + "learning_rate": 0.000460641925777332, + "loss": 3.1177, + "theoretical_loss": 4.162666614661518, + "tokens_seen": 290193408 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004606318956870612, + "loss": 2.821, + "theoretical_loss": 4.162555732969227, + "tokens_seen": 290258944 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046062186559679035, + "loss": 2.7999, + "theoretical_loss": 4.162444883317591, + "tokens_seen": 290324480 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004606118355065196, + "loss": 3.0117, + "theoretical_loss": 4.162334065690123, + "tokens_seen": 290390016 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046060180541624877, + "loss": 2.6194, + "theoretical_loss": 4.162223280070345, + "tokens_seen": 290455552 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046059177532597795, + "loss": 2.7548, + "theoretical_loss": 4.1621125264417955, + "tokens_seen": 290521088 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046058174523570713, + "loss": 2.8914, + "theoretical_loss": 4.162001804788021, + "tokens_seen": 290586624 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004605717151454363, + "loss": 2.9791, + "theoretical_loss": 4.161891115092583, + "tokens_seen": 290652160 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004605616850551655, + "loss": 3.3375, + "theoretical_loss": 4.161780457339055, + "tokens_seen": 290717696 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004605516549648947, + "loss": 3.075, + "theoretical_loss": 4.161669831511022, + "tokens_seen": 290783232 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046054162487462385, + "loss": 3.0684, + "theoretical_loss": 4.16155923759208, + "tokens_seen": 290848768 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004605315947843531, + "loss": 3.2013, + "theoretical_loss": 4.161448675565838, + "tokens_seen": 290914304 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004605215646940822, + "loss": 2.8147, + "theoretical_loss": 4.161338145415918, + "tokens_seen": 290979840 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046051153460381145, + "loss": 2.8885, + "theoretical_loss": 4.161227647125955, + "tokens_seen": 291045376 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046050150451354063, + "loss": 2.9984, + "theoretical_loss": 4.161117180679591, + "tokens_seen": 291110912 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004604914744232698, + "loss": 2.9891, + "theoretical_loss": 4.161006746060488, + "tokens_seen": 291176448 + }, + { + "epoch": 0.09, + "learning_rate": 0.000460481444332999, + "loss": 2.8484, + "theoretical_loss": 4.160896343252311, + "tokens_seen": 291241984 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046047141424272823, + "loss": 2.7959, + "theoretical_loss": 4.160785972238745, + "tokens_seen": 291307520 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046046138415245736, + "loss": 3.0276, + "theoretical_loss": 4.160675633003484, + "tokens_seen": 291373056 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004604513540621866, + "loss": 2.9596, + "theoretical_loss": 4.16056532553023, + "tokens_seen": 291438592 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004604413239719157, + "loss": 2.9352, + "theoretical_loss": 4.160455049802706, + "tokens_seen": 291504128 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046043129388164495, + "loss": 3.0643, + "theoretical_loss": 4.1603448058046375, + "tokens_seen": 291569664 + }, + { + "epoch": 0.09, + "objective/train/docs_used": 320606, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.340557098388672, + "objective/train/theoretical_loss": 4.160234593519768, + "objective/train/tokens_used": 296742368, + "theoretical_loss": 4.160234593519768, + "tokens_seen": 291635200 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046042126379137414, + "loss": 3.2364, + "theoretical_loss": 4.160234593519768, + "tokens_seen": 291635200 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004604112337011033, + "loss": 3.0605, + "theoretical_loss": 4.160124412931852, + "tokens_seen": 291700736 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004604012036108325, + "loss": 2.8302, + "theoretical_loss": 4.160014264024654, + "tokens_seen": 291766272 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004603911735205617, + "loss": 2.7241, + "theoretical_loss": 4.159904146781952, + "tokens_seen": 291831808 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046038114343029086, + "loss": 2.7337, + "theoretical_loss": 4.159794061187536, + "tokens_seen": 291897344 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004603711133400201, + "loss": 3.0597, + "theoretical_loss": 4.1596840072252075, + "tokens_seen": 291962880 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004603610832497492, + "loss": 2.8511, + "theoretical_loss": 4.159573984878779, + "tokens_seen": 292028416 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046035105315947846, + "loss": 3.1022, + "theoretical_loss": 4.159463994132079, + "tokens_seen": 292093952 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004603410230692076, + "loss": 3.2257, + "theoretical_loss": 4.15935403496894, + "tokens_seen": 292159488 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004603309929789368, + "loss": 3.1807, + "theoretical_loss": 4.159244107373215, + "tokens_seen": 292225024 + }, + { + "epoch": 0.09, + "learning_rate": 0.000460320962888666, + "loss": 2.9728, + "theoretical_loss": 4.159134211328765, + "tokens_seen": 292290560 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004603109327983952, + "loss": 2.9467, + "theoretical_loss": 4.159024346819461, + "tokens_seen": 292356096 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046030090270812436, + "loss": 3.1047, + "theoretical_loss": 4.158914513829189, + "tokens_seen": 292421632 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004602908726178536, + "loss": 3.202, + "theoretical_loss": 4.158804712341845, + "tokens_seen": 292487168 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004602808425275827, + "loss": 3.202, + "theoretical_loss": 4.158694942341338, + "tokens_seen": 292552704 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046027081243731196, + "loss": 3.0828, + "theoretical_loss": 4.1585852038115885, + "tokens_seen": 292618240 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004602607823470411, + "loss": 2.9317, + "theoretical_loss": 4.1584754967365285, + "tokens_seen": 292683776 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004602507522567703, + "loss": 2.9311, + "theoretical_loss": 4.1583658211001016, + "tokens_seen": 292749312 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004602407221664995, + "loss": 3.1576, + "theoretical_loss": 4.158256176886264, + "tokens_seen": 292814848 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004602306920762287, + "loss": 2.8076, + "theoretical_loss": 4.158146564078982, + "tokens_seen": 292880384 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046022066198595787, + "loss": 3.1682, + "theoretical_loss": 4.158036982662237, + "tokens_seen": 292945920 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046021063189568705, + "loss": 2.8877, + "theoretical_loss": 4.157927432620018, + "tokens_seen": 293011456 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004602006018054163, + "loss": 2.8661, + "theoretical_loss": 4.157817913936329, + "tokens_seen": 293076992 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046019057171514546, + "loss": 3.0495, + "theoretical_loss": 4.157708426595184, + "tokens_seen": 293142528 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046018054162487465, + "loss": 3.0307, + "theoretical_loss": 4.157598970580608, + "tokens_seen": 293208064 + }, + { + "epoch": 0.09, + "objective/train/docs_used": 320606, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6987125873565674, + "objective/train/theoretical_loss": 4.157489545876642, + "objective/train/tokens_used": 296742368, + "theoretical_loss": 4.157489545876642, + "tokens_seen": 293273600 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004601705115346038, + "loss": 2.8924, + "theoretical_loss": 4.157489545876642, + "tokens_seen": 293273600 + }, + { + "epoch": 0.09, + "learning_rate": 0.000460160481444333, + "loss": 2.7811, + "theoretical_loss": 4.157380152467333, + "tokens_seen": 293339136 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004601504513540622, + "loss": 2.929, + "theoretical_loss": 4.157270790336742, + "tokens_seen": 293404672 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004601404212637914, + "loss": 3.0652, + "theoretical_loss": 4.157161459468944, + "tokens_seen": 293470208 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046013039117352055, + "loss": 3.0149, + "theoretical_loss": 4.157052159848023, + "tokens_seen": 293535744 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004601203610832498, + "loss": 2.8461, + "theoretical_loss": 4.156942891458074, + "tokens_seen": 293601280 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046011033099297897, + "loss": 2.9769, + "theoretical_loss": 4.156833654283207, + "tokens_seen": 293666816 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046010030090270815, + "loss": 3.1259, + "theoretical_loss": 4.15672444830754, + "tokens_seen": 293732352 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046009027081243733, + "loss": 3.0044, + "theoretical_loss": 4.156615273515205, + "tokens_seen": 293797888 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004600802407221665, + "loss": 3.1242, + "theoretical_loss": 4.156506129890344, + "tokens_seen": 293863424 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004600702106318957, + "loss": 2.8938, + "theoretical_loss": 4.156397017417111, + "tokens_seen": 293928960 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046006018054162493, + "loss": 2.9038, + "theoretical_loss": 4.156287936079675, + "tokens_seen": 293994496 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046005015045135405, + "loss": 2.9051, + "theoretical_loss": 4.156178885862209, + "tokens_seen": 294060032 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004600401203610833, + "loss": 2.9019, + "theoretical_loss": 4.156069866748906, + "tokens_seen": 294125568 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004600300902708124, + "loss": 2.8916, + "theoretical_loss": 4.155960878723965, + "tokens_seen": 294191104 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046002006018054165, + "loss": 2.8845, + "theoretical_loss": 4.155851921771598, + "tokens_seen": 294256640 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046001003009027083, + "loss": 2.8652, + "theoretical_loss": 4.155742995876029, + "tokens_seen": 294322176 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046, + "loss": 2.8854, + "theoretical_loss": 4.155634101021494, + "tokens_seen": 294387712 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004599899699097292, + "loss": 3.0241, + "theoretical_loss": 4.155525237192238, + "tokens_seen": 294453248 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045997993981945843, + "loss": 3.2217, + "theoretical_loss": 4.155416404372522, + "tokens_seen": 294518784 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045996990972918756, + "loss": 2.8677, + "theoretical_loss": 4.155307602546614, + "tokens_seen": 294584320 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004599598796389168, + "loss": 2.9117, + "theoretical_loss": 4.155198831698795, + "tokens_seen": 294649856 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004599498495486459, + "loss": 2.8769, + "theoretical_loss": 4.155090091813358, + "tokens_seen": 294715392 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045993981945837515, + "loss": 3.2191, + "theoretical_loss": 4.154981382874608, + "tokens_seen": 294780928 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045992978936810434, + "loss": 2.9602, + "theoretical_loss": 4.154872704866859, + "tokens_seen": 294846464 + }, + { + "epoch": 0.09, + "objective/train/docs_used": 320606, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.4592936038970947, + "objective/train/theoretical_loss": 4.15476405777444, + "objective/train/tokens_used": 296742368, + "theoretical_loss": 4.15476405777444, + "tokens_seen": 294912000 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004599197592778335, + "loss": 3.0343, + "theoretical_loss": 4.15476405777444, + "tokens_seen": 294912000 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004599097291875627, + "loss": 2.9556, + "theoretical_loss": 4.154655441581687, + "tokens_seen": 294977536 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004598996990972919, + "loss": 2.9162, + "theoretical_loss": 4.154546856272952, + "tokens_seen": 295043072 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045988966900702106, + "loss": 2.8154, + "theoretical_loss": 4.154438301832596, + "tokens_seen": 295108608 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004598796389167503, + "loss": 3.1434, + "theoretical_loss": 4.154329778244991, + "tokens_seen": 295174144 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004598696088264794, + "loss": 2.7265, + "theoretical_loss": 4.154221285494521, + "tokens_seen": 295239680 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045985957873620866, + "loss": 3.1865, + "theoretical_loss": 4.154112823565582, + "tokens_seen": 295305216 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004598495486459378, + "loss": 2.9111, + "theoretical_loss": 4.15400439244258, + "tokens_seen": 295370752 + }, + { + "epoch": 0.09, + "learning_rate": 0.000459839518555667, + "loss": 2.9867, + "theoretical_loss": 4.153895992109935, + "tokens_seen": 295436288 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004598294884653962, + "loss": 2.9011, + "theoretical_loss": 4.153787622552073, + "tokens_seen": 295501824 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004598194583751254, + "loss": 3.0471, + "theoretical_loss": 4.153679283753439, + "tokens_seen": 295567360 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045980942828485456, + "loss": 2.909, + "theoretical_loss": 4.15357097569848, + "tokens_seen": 295632896 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004597993981945838, + "loss": 2.7348, + "theoretical_loss": 4.153462698371665, + "tokens_seen": 295698432 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004597893681043129, + "loss": 3.0335, + "theoretical_loss": 4.1533544517574645, + "tokens_seen": 295763968 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045977933801404216, + "loss": 2.9854, + "theoretical_loss": 4.153246235840367, + "tokens_seen": 295829504 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004597693079237713, + "loss": 2.8577, + "theoretical_loss": 4.153138050604868, + "tokens_seen": 295895040 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004597592778335005, + "loss": 3.1334, + "theoretical_loss": 4.153029896035476, + "tokens_seen": 295960576 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004597492477432297, + "loss": 3.1153, + "theoretical_loss": 4.152921772116712, + "tokens_seen": 296026112 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004597392176529589, + "loss": 3.0102, + "theoretical_loss": 4.152813678833106, + "tokens_seen": 296091648 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045972918756268807, + "loss": 2.846, + "theoretical_loss": 4.152705616169202, + "tokens_seen": 296157184 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045971915747241725, + "loss": 3.0878, + "theoretical_loss": 4.15259758410955, + "tokens_seen": 296222720 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045970912738214643, + "loss": 3.0873, + "theoretical_loss": 4.152489582638719, + "tokens_seen": 296288256 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045969909729187566, + "loss": 3.111, + "theoretical_loss": 4.152381611741281, + "tokens_seen": 296353792 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004596890672016048, + "loss": 2.9683, + "theoretical_loss": 4.152273671401824, + "tokens_seen": 296419328 + }, + { + "epoch": 0.09, + "learning_rate": 0.000459679037111334, + "loss": 3.2746, + "theoretical_loss": 4.152165761604948, + "tokens_seen": 296484864 + }, + { + "epoch": 0.09, + "objective/train/docs_used": 320606, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.3823139667510986, + "objective/train/theoretical_loss": 4.152057882335261, + "objective/train/tokens_used": 296742368, + "theoretical_loss": 4.152057882335261, + "tokens_seen": 296550400 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045966900702106315, + "loss": 2.9545, + "theoretical_loss": 4.152057882335261, + "tokens_seen": 296550400 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004596589769307924, + "loss": 2.8693, + "theoretical_loss": 4.151950033577383, + "tokens_seen": 296615936 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045964894684052157, + "loss": 3.2989, + "theoretical_loss": 4.151842215315947, + "tokens_seen": 296681472 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045963891675025075, + "loss": 3.0708, + "theoretical_loss": 4.151734427535594, + "tokens_seen": 296747008 + }, + { + "epoch": 1.0, + "learning_rate": 0.00045962888665997993, + "loss": 3.9868, + "theoretical_loss": 4.151601419005685, + "tokens_seen": 296827904 + }, + { + "epoch": 1.0, + "learning_rate": 0.00045961885656970917, + "loss": 3.0976, + "theoretical_loss": 4.151493699276069, + "tokens_seen": 296893440 + }, + { + "epoch": 1.0, + "learning_rate": 0.0004596088264794383, + "loss": 3.2042, + "theoretical_loss": 4.151386009977943, + "tokens_seen": 296958976 + }, + { + "epoch": 1.0, + "learning_rate": 0.00045959879638916753, + "loss": 3.1598, + "theoretical_loss": 4.151278351095997, + "tokens_seen": 297024512 + }, + { + "epoch": 1.0, + "learning_rate": 0.00045958876629889666, + "loss": 3.2069, + "theoretical_loss": 4.15117072261493, + "tokens_seen": 297090048 + }, + { + "epoch": 1.0, + "learning_rate": 0.0004595787362086259, + "loss": 2.9252, + "theoretical_loss": 4.151063124519455, + "tokens_seen": 297155584 + }, + { + "epoch": 1.0, + "learning_rate": 0.0004595687061183551, + "loss": 3.1758, + "theoretical_loss": 4.150955556794295, + "tokens_seen": 297221120 + }, + { + "epoch": 1.0, + "learning_rate": 0.00045955867602808425, + "loss": 3.1888, + "theoretical_loss": 4.150848019424184, + "tokens_seen": 297286656 + }, + { + "epoch": 1.0, + "learning_rate": 0.00045954864593781344, + "loss": 3.1973, + "theoretical_loss": 4.150740512393868, + "tokens_seen": 297352192 + }, + { + "epoch": 1.0, + "learning_rate": 0.0004595386158475426, + "loss": 3.1925, + "theoretical_loss": 4.1506330356881005, + "tokens_seen": 297417728 + }, + { + "epoch": 1.0, + "learning_rate": 0.0004595285857572718, + "loss": 3.2762, + "theoretical_loss": 4.150525589291652, + "tokens_seen": 297483264 + }, + { + "epoch": 1.0, + "learning_rate": 0.00045951855566700103, + "loss": 3.2021, + "theoretical_loss": 4.150418173189299, + "tokens_seen": 297548800 + }, + { + "epoch": 1.0, + "learning_rate": 0.00045950852557673016, + "loss": 3.1805, + "theoretical_loss": 4.1503107873658305, + "tokens_seen": 297614336 + }, + { + "epoch": 1.0, + "learning_rate": 0.0004594984954864594, + "loss": 3.3438, + "theoretical_loss": 4.150203431806046, + "tokens_seen": 297679872 + }, + { + "epoch": 1.0, + "learning_rate": 0.0004594884653961885, + "loss": 3.1938, + "theoretical_loss": 4.150096106494758, + "tokens_seen": 297745408 + }, + { + "epoch": 1.0, + "learning_rate": 0.00045947843530591776, + "loss": 3.1626, + "theoretical_loss": 4.149988811416788, + "tokens_seen": 297810944 + }, + { + "epoch": 1.0, + "learning_rate": 0.00045946840521564694, + "loss": 3.1176, + "theoretical_loss": 4.149881546556971, + "tokens_seen": 297876480 + }, + { + "epoch": 1.0, + "learning_rate": 0.0004594583751253761, + "loss": 3.0244, + "theoretical_loss": 4.149774311900147, + "tokens_seen": 297942016 + }, + { + "epoch": 1.0, + "learning_rate": 0.00045944834503510536, + "loss": 3.0722, + "theoretical_loss": 4.149667107431174, + "tokens_seen": 298007552 + }, + { + "epoch": 1.0, + "learning_rate": 0.00045943831494483454, + "loss": 3.0816, + "theoretical_loss": 4.149559933134916, + "tokens_seen": 298073088 + }, + { + "epoch": 1.0, + "learning_rate": 0.0004594282848545637, + "loss": 3.2045, + "theoretical_loss": 4.149452788996252, + "tokens_seen": 298138624 + }, + { + "epoch": 1.0, + "objective/train/docs_used": 388086, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.5071983337402344, + "objective/train/theoretical_loss": 4.149372450674081, + "objective/train/tokens_used": 318647776, + "theoretical_loss": 4.149372450674081, + "tokens_seen": 298187776 + }, + { + "epoch": 1.0, + "learning_rate": 0.0004594182547642929, + "loss": 3.0893, + "theoretical_loss": 4.149345675000067, + "tokens_seen": 298204160 + }, + { + "epoch": 1.0, + "learning_rate": 0.0004594082246740221, + "loss": 3.2211, + "theoretical_loss": 4.149238591131261, + "tokens_seen": 298269696 + }, + { + "epoch": 1.0, + "learning_rate": 0.00045939819458375126, + "loss": 3.1047, + "theoretical_loss": 4.149131537374743, + "tokens_seen": 298335232 + }, + { + "epoch": 1.0, + "learning_rate": 0.0004593881644934805, + "loss": 3.1427, + "theoretical_loss": 4.149024513715434, + "tokens_seen": 298400768 + }, + { + "epoch": 1.0, + "learning_rate": 0.0004593781344032096, + "loss": 3.2841, + "theoretical_loss": 4.148917520138264, + "tokens_seen": 298466304 + }, + { + "epoch": 1.0, + "learning_rate": 0.00045936810431293886, + "loss": 3.1813, + "theoretical_loss": 4.148810556628176, + "tokens_seen": 298531840 + }, + { + "epoch": 1.0, + "learning_rate": 0.000459358074222668, + "loss": 3.0694, + "theoretical_loss": 4.148703623170123, + "tokens_seen": 298597376 + }, + { + "epoch": 1.0, + "learning_rate": 0.0004593480441323972, + "loss": 3.0848, + "theoretical_loss": 4.148596719749067, + "tokens_seen": 298662912 + }, + { + "epoch": 1.0, + "learning_rate": 0.0004593380140421264, + "loss": 3.1606, + "theoretical_loss": 4.148489846349984, + "tokens_seen": 298728448 + }, + { + "epoch": 1.0, + "learning_rate": 0.0004593279839518556, + "loss": 3.001, + "theoretical_loss": 4.14838300295786, + "tokens_seen": 298793984 + }, + { + "epoch": 1.0, + "learning_rate": 0.00045931795386158476, + "loss": 3.2776, + "theoretical_loss": 4.148276189557689, + "tokens_seen": 298859520 + }, + { + "epoch": 1.0, + "learning_rate": 0.000459307923771314, + "loss": 3.1101, + "theoretical_loss": 4.148169406134479, + "tokens_seen": 298925056 + }, + { + "epoch": 1.0, + "learning_rate": 0.0004592978936810431, + "loss": 3.2241, + "theoretical_loss": 4.148062652673248, + "tokens_seen": 298990592 + }, + { + "epoch": 1.0, + "learning_rate": 0.00045928786359077236, + "loss": 3.2768, + "theoretical_loss": 4.147955929159024, + "tokens_seen": 299056128 + }, + { + "epoch": 1.0, + "learning_rate": 0.0004592778335005015, + "loss": 3.3648, + "theoretical_loss": 4.147849235576846, + "tokens_seen": 299121664 + }, + { + "epoch": 1.0, + "learning_rate": 0.0004592678034102307, + "loss": 3.039, + "theoretical_loss": 4.147742571911765, + "tokens_seen": 299187200 + }, + { + "epoch": 1.0, + "learning_rate": 0.0004592577733199599, + "loss": 3.0152, + "theoretical_loss": 4.147635938148841, + "tokens_seen": 299252736 + }, + { + "epoch": 1.0, + "learning_rate": 0.0004592477432296891, + "loss": 3.1572, + "theoretical_loss": 4.147529334273145, + "tokens_seen": 299318272 + }, + { + "epoch": 1.0, + "learning_rate": 0.00045923771313941827, + "loss": 3.2587, + "theoretical_loss": 4.147422760269759, + "tokens_seen": 299383808 + }, + { + "epoch": 1.0, + "learning_rate": 0.00045922768304914745, + "loss": 3.0774, + "theoretical_loss": 4.147316216123777, + "tokens_seen": 299449344 + }, + { + "epoch": 1.0, + "learning_rate": 0.00045921765295887663, + "loss": 3.2984, + "theoretical_loss": 4.147209701820302, + "tokens_seen": 299514880 + }, + { + "epoch": 1.0, + "learning_rate": 0.00045920762286860586, + "loss": 3.0177, + "theoretical_loss": 4.147103217344448, + "tokens_seen": 299580416 + }, + { + "epoch": 1.0, + "learning_rate": 0.000459197592778335, + "loss": 3.1661, + "theoretical_loss": 4.14699676268134, + "tokens_seen": 299645952 + }, + { + "epoch": 1.0, + "learning_rate": 0.0004591875626880642, + "loss": 2.9458, + "theoretical_loss": 4.146890337816114, + "tokens_seen": 299711488 + }, + { + "epoch": 1.0, + "learning_rate": 0.00045917753259779335, + "loss": 2.9969, + "theoretical_loss": 4.146783942733915, + "tokens_seen": 299777024 + }, + { + "epoch": 1.0, + "objective/train/docs_used": 393339, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.2248926162719727, + "objective/train/theoretical_loss": 4.1467041659584485, + "objective/train/tokens_used": 320286176, + "theoretical_loss": 4.1467041659584485, + "tokens_seen": 299826176 + }, + { + "epoch": 1.0, + "learning_rate": 0.0004591675025075226, + "loss": 3.0643, + "theoretical_loss": 4.146677577419902, + "tokens_seen": 299842560 + }, + { + "epoch": 1.0, + "learning_rate": 0.00045915747241725177, + "loss": 3.0779, + "theoretical_loss": 4.14657124185924, + "tokens_seen": 299908096 + }, + { + "epoch": 1.0, + "learning_rate": 0.00045914744232698095, + "loss": 3.1616, + "theoretical_loss": 4.14646493603711, + "tokens_seen": 299973632 + }, + { + "epoch": 1.0, + "learning_rate": 0.00045913741223671013, + "loss": 3.0127, + "theoretical_loss": 4.146358659938699, + "tokens_seen": 300039168 + }, + { + "epoch": 1.0, + "learning_rate": 0.00045912738214643937, + "loss": 3.0727, + "theoretical_loss": 4.146252413549207, + "tokens_seen": 300104704 + }, + { + "epoch": 1.0, + "learning_rate": 0.0004591173520561685, + "loss": 3.2627, + "theoretical_loss": 4.146146196853843, + "tokens_seen": 300170240 + }, + { + "epoch": 1.0, + "learning_rate": 0.00045910732196589773, + "loss": 3.1219, + "theoretical_loss": 4.146040009837829, + "tokens_seen": 300235776 + }, + { + "epoch": 1.0, + "learning_rate": 0.00045909729187562686, + "loss": 3.2786, + "theoretical_loss": 4.145933852486396, + "tokens_seen": 300301312 + }, + { + "epoch": 1.0, + "learning_rate": 0.0004590872617853561, + "loss": 3.0808, + "theoretical_loss": 4.145827724784784, + "tokens_seen": 300366848 + }, + { + "epoch": 1.0, + "learning_rate": 0.0004590772316950853, + "loss": 3.054, + "theoretical_loss": 4.145721626718247, + "tokens_seen": 300432384 + }, + { + "epoch": 1.0, + "learning_rate": 0.00045906720160481445, + "loss": 3.0884, + "theoretical_loss": 4.1456155582720475, + "tokens_seen": 300497920 + }, + { + "epoch": 1.0, + "learning_rate": 0.00045905717151454364, + "loss": 3.0052, + "theoretical_loss": 4.145509519431459, + "tokens_seen": 300563456 + }, + { + "epoch": 1.0, + "learning_rate": 0.0004590471414242728, + "loss": 2.9324, + "theoretical_loss": 4.145403510181765, + "tokens_seen": 300628992 + }, + { + "epoch": 1.0, + "learning_rate": 0.000459037111334002, + "loss": 3.1987, + "theoretical_loss": 4.145297530508261, + "tokens_seen": 300694528 + }, + { + "epoch": 1.0, + "learning_rate": 0.00045902708124373123, + "loss": 3.245, + "theoretical_loss": 4.145191580396251, + "tokens_seen": 300760064 + }, + { + "epoch": 1.0, + "learning_rate": 0.00045901705115346036, + "loss": 3.2895, + "theoretical_loss": 4.145085659831049, + "tokens_seen": 300825600 + }, + { + "epoch": 1.0, + "learning_rate": 0.0004590070210631896, + "loss": 3.3486, + "theoretical_loss": 4.144979768797985, + "tokens_seen": 300891136 + }, + { + "epoch": 1.0, + "learning_rate": 0.0004589969909729187, + "loss": 3.0806, + "theoretical_loss": 4.14487390728239, + "tokens_seen": 300956672 + }, + { + "epoch": 1.0, + "learning_rate": 0.00045898696088264796, + "loss": 2.9642, + "theoretical_loss": 4.144768075269616, + "tokens_seen": 301022208 + }, + { + "epoch": 1.0, + "learning_rate": 0.00045897693079237714, + "loss": 3.1238, + "theoretical_loss": 4.144662272745018, + "tokens_seen": 301087744 + }, + { + "epoch": 1.0, + "learning_rate": 0.0004589669007021063, + "loss": 3.0449, + "theoretical_loss": 4.144556499693964, + "tokens_seen": 301153280 + }, + { + "epoch": 1.0, + "learning_rate": 0.0004589568706118355, + "loss": 3.1233, + "theoretical_loss": 4.144450756101832, + "tokens_seen": 301218816 + }, + { + "epoch": 1.0, + "learning_rate": 0.00045894684052156474, + "loss": 3.1373, + "theoretical_loss": 4.144345041954011, + "tokens_seen": 301284352 + }, + { + "epoch": 1.0, + "learning_rate": 0.00045893681043129386, + "loss": 3.047, + "theoretical_loss": 4.1442393572359, + "tokens_seen": 301349888 + }, + { + "epoch": 1.0, + "learning_rate": 0.0004589267803410231, + "loss": 3.0807, + "theoretical_loss": 4.14413370193291, + "tokens_seen": 301415424 + }, + { + "epoch": 1.0, + "objective/train/docs_used": 398212, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.0288240909576416, + "objective/train/theoretical_loss": 4.144054479750567, + "objective/train/tokens_used": 321924576, + "theoretical_loss": 4.144054479750567, + "tokens_seen": 301464576 + }, + { + "epoch": 1.0, + "learning_rate": 0.0004589167502507522, + "loss": 3.0242, + "theoretical_loss": 4.144028076030458, + "tokens_seen": 301480960 + }, + { + "epoch": 1.0, + "learning_rate": 0.00045890672016048146, + "loss": 3.2372, + "theoretical_loss": 4.143922479513977, + "tokens_seen": 301546496 + }, + { + "epoch": 1.0, + "learning_rate": 0.00045889669007021064, + "loss": 3.141, + "theoretical_loss": 4.143816912368906, + "tokens_seen": 301612032 + }, + { + "epoch": 1.0, + "learning_rate": 0.0004588866599799398, + "loss": 3.2979, + "theoretical_loss": 4.143711374580697, + "tokens_seen": 301677568 + }, + { + "epoch": 1.0, + "learning_rate": 0.000458876629889669, + "loss": 3.1576, + "theoretical_loss": 4.143605866134811, + "tokens_seen": 301743104 + }, + { + "epoch": 1.0, + "learning_rate": 0.0004588665997993982, + "loss": 3.1577, + "theoretical_loss": 4.1435003870167195, + "tokens_seen": 301808640 + }, + { + "epoch": 1.0, + "learning_rate": 0.00045885656970912737, + "loss": 3.1194, + "theoretical_loss": 4.143394937211906, + "tokens_seen": 301874176 + }, + { + "epoch": 1.0, + "learning_rate": 0.0004588465396188566, + "loss": 3.0641, + "theoretical_loss": 4.143289516705861, + "tokens_seen": 301939712 + }, + { + "epoch": 1.0, + "learning_rate": 0.00045883650952858573, + "loss": 3.1287, + "theoretical_loss": 4.14318412548409, + "tokens_seen": 302005248 + }, + { + "epoch": 1.0, + "learning_rate": 0.00045882647943831496, + "loss": 3.1244, + "theoretical_loss": 4.143078763532104, + "tokens_seen": 302070784 + }, + { + "epoch": 1.0, + "learning_rate": 0.00045881644934804415, + "loss": 3.2705, + "theoretical_loss": 4.1429734308354265, + "tokens_seen": 302136320 + }, + { + "epoch": 1.0, + "learning_rate": 0.0004588064192577733, + "loss": 3.2505, + "theoretical_loss": 4.142868127379592, + "tokens_seen": 302201856 + }, + { + "epoch": 1.0, + "learning_rate": 0.0004587963891675025, + "loss": 3.0527, + "theoretical_loss": 4.142762853150145, + "tokens_seen": 302267392 + }, + { + "epoch": 1.0, + "learning_rate": 0.0004587863590772317, + "loss": 3.0989, + "theoretical_loss": 4.142657608132638, + "tokens_seen": 302332928 + }, + { + "epoch": 1.0, + "learning_rate": 0.00045877632898696087, + "loss": 3.3695, + "theoretical_loss": 4.142552392312638, + "tokens_seen": 302398464 + }, + { + "epoch": 1.0, + "learning_rate": 0.0004587662988966901, + "loss": 3.1914, + "theoretical_loss": 4.142447205675717, + "tokens_seen": 302464000 + }, + { + "epoch": 1.0, + "learning_rate": 0.00045875626880641923, + "loss": 3.3743, + "theoretical_loss": 4.142342048207462, + "tokens_seen": 302529536 + }, + { + "epoch": 1.0, + "learning_rate": 0.00045874623871614847, + "loss": 3.2538, + "theoretical_loss": 4.1422369198934685, + "tokens_seen": 302595072 + }, + { + "epoch": 1.0, + "learning_rate": 0.0004587362086258776, + "loss": 3.2982, + "theoretical_loss": 4.142131820719342, + "tokens_seen": 302660608 + }, + { + "epoch": 1.0, + "learning_rate": 0.00045872617853560683, + "loss": 3.1992, + "theoretical_loss": 4.142026750670697, + "tokens_seen": 302726144 + }, + { + "epoch": 1.0, + "learning_rate": 0.000458716148445336, + "loss": 3.1915, + "theoretical_loss": 4.14192170973316, + "tokens_seen": 302791680 + }, + { + "epoch": 1.0, + "learning_rate": 0.0004587061183550652, + "loss": 3.3022, + "theoretical_loss": 4.141816697892368, + "tokens_seen": 302857216 + }, + { + "epoch": 1.0, + "learning_rate": 0.0004586960882647944, + "loss": 3.347, + "theoretical_loss": 4.141711715133967, + "tokens_seen": 302922752 + }, + { + "epoch": 1.0, + "learning_rate": 0.00045868605817452355, + "loss": 3.2762, + "theoretical_loss": 4.1416067614436125, + "tokens_seen": 302988288 + }, + { + "epoch": 1.0, + "learning_rate": 0.0004586760280842528, + "loss": 3.1287, + "theoretical_loss": 4.141501836806973, + "tokens_seen": 303053824 + }, + { + "epoch": 1.0, + "objective/train/docs_used": 403119, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8905088901519775, + "objective/train/theoretical_loss": 4.141423162387376, + "objective/train/tokens_used": 323562976, + "theoretical_loss": 4.141423162387376, + "tokens_seen": 303102976 + }, + { + "epoch": 1.0, + "learning_rate": 0.00045866599799398197, + "loss": 3.0808, + "theoretical_loss": 4.141396941209724, + "tokens_seen": 303119360 + }, + { + "epoch": 1.0, + "learning_rate": 0.00045865596790371115, + "loss": 3.2722, + "theoretical_loss": 4.141292074637554, + "tokens_seen": 303184896 + }, + { + "epoch": 1.0, + "learning_rate": 0.00045864593781344033, + "loss": 3.4675, + "theoretical_loss": 4.141187237076158, + "tokens_seen": 303250432 + }, + { + "epoch": 1.0, + "learning_rate": 0.00045863590772316957, + "loss": 3.2572, + "theoretical_loss": 4.141082428511247, + "tokens_seen": 303315968 + }, + { + "epoch": 1.0, + "learning_rate": 0.0004586258776328987, + "loss": 3.2468, + "theoretical_loss": 4.140977648928534, + "tokens_seen": 303381504 + }, + { + "epoch": 1.0, + "learning_rate": 0.00045861584754262793, + "loss": 3.025, + "theoretical_loss": 4.14087289831375, + "tokens_seen": 303447040 + }, + { + "epoch": 1.0, + "learning_rate": 0.00045860581745235706, + "loss": 3.2632, + "theoretical_loss": 4.140768176652632, + "tokens_seen": 303512576 + }, + { + "epoch": 1.0, + "learning_rate": 0.0004585957873620863, + "loss": 3.4513, + "theoretical_loss": 4.1406634839309255, + "tokens_seen": 303578112 + }, + { + "epoch": 1.0, + "learning_rate": 0.0004585857572718155, + "loss": 3.307, + "theoretical_loss": 4.140558820134391, + "tokens_seen": 303643648 + }, + { + "epoch": 1.0, + "learning_rate": 0.00045857572718154465, + "loss": 3.2101, + "theoretical_loss": 4.140454185248797, + "tokens_seen": 303709184 + }, + { + "epoch": 1.0, + "learning_rate": 0.00045856569709127384, + "loss": 3.3302, + "theoretical_loss": 4.140349579259919, + "tokens_seen": 303774720 + }, + { + "epoch": 1.0, + "learning_rate": 0.000458555667001003, + "loss": 3.2633, + "theoretical_loss": 4.140245002153547, + "tokens_seen": 303840256 + }, + { + "epoch": 1.0, + "learning_rate": 0.0004585456369107322, + "loss": 3.0562, + "theoretical_loss": 4.140140453915478, + "tokens_seen": 303905792 + }, + { + "epoch": 1.0, + "learning_rate": 0.00045853560682046143, + "loss": 3.1859, + "theoretical_loss": 4.140035934531521, + "tokens_seen": 303971328 + }, + { + "epoch": 1.0, + "learning_rate": 0.00045852557673019056, + "loss": 3.1359, + "theoretical_loss": 4.139931443987494, + "tokens_seen": 304036864 + }, + { + "epoch": 1.0, + "learning_rate": 0.0004585155466399198, + "loss": 3.4099, + "theoretical_loss": 4.139826982269225, + "tokens_seen": 304102400 + }, + { + "epoch": 1.0, + "learning_rate": 0.0004585055165496489, + "loss": 3.2253, + "theoretical_loss": 4.139722549362553, + "tokens_seen": 304167936 + }, + { + "epoch": 1.0, + "learning_rate": 0.00045849548645937816, + "loss": 3.1415, + "theoretical_loss": 4.139618145253326, + "tokens_seen": 304233472 + }, + { + "epoch": 1.0, + "learning_rate": 0.00045848545636910734, + "loss": 3.0069, + "theoretical_loss": 4.139513769927402, + "tokens_seen": 304299008 + }, + { + "epoch": 1.0, + "learning_rate": 0.0004584754262788365, + "loss": 3.3609, + "theoretical_loss": 4.139409423370649, + "tokens_seen": 304364544 + }, + { + "epoch": 1.0, + "learning_rate": 0.0004584653961885657, + "loss": 3.3372, + "theoretical_loss": 4.139305105568946, + "tokens_seen": 304430080 + }, + { + "epoch": 1.0, + "learning_rate": 0.00045845536609829494, + "loss": 3.124, + "theoretical_loss": 4.139200816508181, + "tokens_seen": 304495616 + }, + { + "epoch": 1.0, + "learning_rate": 0.00045844533600802406, + "loss": 3.1771, + "theoretical_loss": 4.139096556174252, + "tokens_seen": 304561152 + }, + { + "epoch": 1.0, + "learning_rate": 0.0004584353059177533, + "loss": 3.2137, + "theoretical_loss": 4.138992324553068, + "tokens_seen": 304626688 + }, + { + "epoch": 1.0, + "learning_rate": 0.0004584252758274824, + "loss": 3.2045, + "theoretical_loss": 4.138888121630545, + "tokens_seen": 304692224 + }, + { + "epoch": 1.0, + "objective/train/docs_used": 408173, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.1989989280700684, + "objective/train/theoretical_loss": 4.138809988263685, + "objective/train/tokens_used": 325201376, + "theoretical_loss": 4.138809988263685, + "tokens_seen": 304741376 + }, + { + "epoch": 1.0, + "learning_rate": 0.00045841524573721166, + "loss": 3.3398, + "theoretical_loss": 4.138783947392613, + "tokens_seen": 304757760 + }, + { + "epoch": 1.0, + "learning_rate": 0.00045840521564694084, + "loss": 3.1126, + "theoretical_loss": 4.13867980182521, + "tokens_seen": 304823296 + }, + { + "epoch": 1.0, + "learning_rate": 0.00045839518555667, + "loss": 3.1801, + "theoretical_loss": 4.138575684914282, + "tokens_seen": 304888832 + }, + { + "epoch": 1.0, + "learning_rate": 0.0004583851554663992, + "loss": 3.2703, + "theoretical_loss": 4.138471596645789, + "tokens_seen": 304954368 + }, + { + "epoch": 1.0, + "learning_rate": 0.0004583751253761284, + "loss": 3.2164, + "theoretical_loss": 4.138367537005697, + "tokens_seen": 305019904 + }, + { + "epoch": 1.0, + "learning_rate": 0.00045836509528585757, + "loss": 3.3049, + "theoretical_loss": 4.1382635059799835, + "tokens_seen": 305085440 + }, + { + "epoch": 1.0, + "learning_rate": 0.0004583550651955868, + "loss": 2.9696, + "theoretical_loss": 4.138159503554638, + "tokens_seen": 305150976 + }, + { + "epoch": 1.0, + "learning_rate": 0.00045834503510531593, + "loss": 3.3098, + "theoretical_loss": 4.138055529715655, + "tokens_seen": 305216512 + }, + { + "epoch": 1.0, + "learning_rate": 0.00045833500501504516, + "loss": 3.1387, + "theoretical_loss": 4.137951584449044, + "tokens_seen": 305282048 + }, + { + "epoch": 1.0, + "learning_rate": 0.00045832497492477435, + "loss": 3.1771, + "theoretical_loss": 4.13784766774082, + "tokens_seen": 305347584 + }, + { + "epoch": 1.0, + "learning_rate": 0.0004583149448345035, + "loss": 3.171, + "theoretical_loss": 4.137743779577011, + "tokens_seen": 305413120 + }, + { + "epoch": 1.0, + "learning_rate": 0.0004583049147442327, + "loss": 3.009, + "theoretical_loss": 4.137639919943655, + "tokens_seen": 305478656 + }, + { + "epoch": 1.0, + "learning_rate": 0.0004582948846539619, + "loss": 3.0046, + "theoretical_loss": 4.137536088826796, + "tokens_seen": 305544192 + }, + { + "epoch": 1.0, + "learning_rate": 0.00045828485456369107, + "loss": 3.0266, + "theoretical_loss": 4.1374322862124915, + "tokens_seen": 305609728 + }, + { + "epoch": 1.0, + "learning_rate": 0.0004582748244734203, + "loss": 2.9813, + "theoretical_loss": 4.137328512086807, + "tokens_seen": 305675264 + }, + { + "epoch": 1.0, + "learning_rate": 0.00045826479438314943, + "loss": 3.0888, + "theoretical_loss": 4.1372247664358195, + "tokens_seen": 305740800 + }, + { + "epoch": 1.0, + "learning_rate": 0.00045825476429287867, + "loss": 2.9889, + "theoretical_loss": 4.137121049245614, + "tokens_seen": 305806336 + }, + { + "epoch": 1.0, + "learning_rate": 0.0004582447342026078, + "loss": 2.9691, + "theoretical_loss": 4.137017360502286, + "tokens_seen": 305871872 + }, + { + "epoch": 1.0, + "learning_rate": 0.00045823470411233703, + "loss": 2.893, + "theoretical_loss": 4.136913700191942, + "tokens_seen": 305937408 + }, + { + "epoch": 1.0, + "learning_rate": 0.0004582246740220662, + "loss": 3.1183, + "theoretical_loss": 4.136810068300694, + "tokens_seen": 306002944 + }, + { + "epoch": 1.0, + "learning_rate": 0.0004582146439317954, + "loss": 3.2618, + "theoretical_loss": 4.13670646481467, + "tokens_seen": 306068480 + }, + { + "epoch": 1.0, + "learning_rate": 0.00045820461384152457, + "loss": 3.2482, + "theoretical_loss": 4.1366028897200025, + "tokens_seen": 306134016 + }, + { + "epoch": 1.0, + "learning_rate": 0.00045819458375125375, + "loss": 3.1096, + "theoretical_loss": 4.1364993430028365, + "tokens_seen": 306199552 + }, + { + "epoch": 1.0, + "learning_rate": 0.00045818455366098294, + "loss": 3.4132, + "theoretical_loss": 4.136395824649327, + "tokens_seen": 306265088 + }, + { + "epoch": 1.0, + "learning_rate": 0.00045817452357071217, + "loss": 3.1493, + "theoretical_loss": 4.136292334645636, + "tokens_seen": 306330624 + }, + { + "epoch": 1.0, + "objective/train/docs_used": 413344, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.146451950073242, + "objective/train/theoretical_loss": 4.136214735739118, + "objective/train/tokens_used": 326839776, + "theoretical_loss": 4.136214735739118, + "tokens_seen": 306379776 + }, + { + "epoch": 1.0, + "learning_rate": 0.0004581644934804413, + "loss": 3.0853, + "theoretical_loss": 4.136188872977938, + "tokens_seen": 306396160 + }, + { + "epoch": 1.0, + "learning_rate": 0.00045815446339017053, + "loss": 3.0507, + "theoretical_loss": 4.1360854396324145, + "tokens_seen": 306461696 + }, + { + "epoch": 1.0, + "learning_rate": 0.0004581444332998997, + "loss": 3.0544, + "theoretical_loss": 4.13598203459526, + "tokens_seen": 306527232 + }, + { + "epoch": 1.0, + "learning_rate": 0.0004581344032096289, + "loss": 3.0592, + "theoretical_loss": 4.135878657852677, + "tokens_seen": 306592768 + }, + { + "epoch": 1.0, + "learning_rate": 0.0004581243731193581, + "loss": 3.1452, + "theoretical_loss": 4.135775309390876, + "tokens_seen": 306658304 + }, + { + "epoch": 1.0, + "learning_rate": 0.00045811434302908726, + "loss": 2.9714, + "theoretical_loss": 4.135671989196081, + "tokens_seen": 306723840 + }, + { + "epoch": 1.0, + "learning_rate": 0.00045810431293881644, + "loss": 3.279, + "theoretical_loss": 4.135568697254522, + "tokens_seen": 306789376 + }, + { + "epoch": 1.0, + "learning_rate": 0.0004580942828485457, + "loss": 3.2136, + "theoretical_loss": 4.135465433552441, + "tokens_seen": 306854912 + }, + { + "epoch": 1.0, + "learning_rate": 0.0004580842527582748, + "loss": 3.1741, + "theoretical_loss": 4.135362198076088, + "tokens_seen": 306920448 + }, + { + "epoch": 1.0, + "learning_rate": 0.00045807422266800404, + "loss": 3.2101, + "theoretical_loss": 4.135258990811723, + "tokens_seen": 306985984 + }, + { + "epoch": 1.0, + "learning_rate": 0.00045806419257773316, + "loss": 3.0828, + "theoretical_loss": 4.1351558117456175, + "tokens_seen": 307051520 + }, + { + "epoch": 1.0, + "learning_rate": 0.0004580541624874624, + "loss": 3.0378, + "theoretical_loss": 4.13505266086405, + "tokens_seen": 307117056 + }, + { + "epoch": 1.0, + "learning_rate": 0.0004580441323971916, + "loss": 3.0833, + "theoretical_loss": 4.134949538153309, + "tokens_seen": 307182592 + }, + { + "epoch": 1.0, + "learning_rate": 0.00045803410230692076, + "loss": 3.0974, + "theoretical_loss": 4.134846443599694, + "tokens_seen": 307248128 + }, + { + "epoch": 1.0, + "learning_rate": 0.00045802407221664994, + "loss": 3.1697, + "theoretical_loss": 4.1347433771895155, + "tokens_seen": 307313664 + }, + { + "epoch": 1.0, + "learning_rate": 0.0004580140421263791, + "loss": 3.1566, + "theoretical_loss": 4.134640338909088, + "tokens_seen": 307379200 + }, + { + "epoch": 1.0, + "learning_rate": 0.0004580040120361083, + "loss": 2.9549, + "theoretical_loss": 4.134537328744742, + "tokens_seen": 307444736 + }, + { + "epoch": 1.0, + "learning_rate": 0.00045799398194583754, + "loss": 3.0834, + "theoretical_loss": 4.134434346682812, + "tokens_seen": 307510272 + }, + { + "epoch": 1.0, + "learning_rate": 0.00045798395185556667, + "loss": 2.92, + "theoretical_loss": 4.134331392709647, + "tokens_seen": 307575808 + }, + { + "epoch": 1.0, + "learning_rate": 0.0004579739217652959, + "loss": 3.018, + "theoretical_loss": 4.134228466811601, + "tokens_seen": 307641344 + }, + { + "epoch": 1.0, + "learning_rate": 0.0004579638916750251, + "loss": 3.0011, + "theoretical_loss": 4.1341255689750405, + "tokens_seen": 307706880 + }, + { + "epoch": 1.0, + "learning_rate": 0.00045795386158475426, + "loss": 3.1451, + "theoretical_loss": 4.134022699186342, + "tokens_seen": 307772416 + }, + { + "epoch": 1.0, + "learning_rate": 0.0004579438314944835, + "loss": 3.0998, + "theoretical_loss": 4.133919857431889, + "tokens_seen": 307837952 + }, + { + "epoch": 1.0, + "learning_rate": 0.0004579338014042126, + "loss": 3.0204, + "theoretical_loss": 4.133817043698075, + "tokens_seen": 307903488 + }, + { + "epoch": 1.0, + "learning_rate": 0.00045792377131394186, + "loss": 3.0224, + "theoretical_loss": 4.133714257971306, + "tokens_seen": 307969024 + }, + { + "epoch": 1.0, + "objective/train/docs_used": 418392, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.903918981552124, + "objective/train/theoretical_loss": 4.133637187047677, + "objective/train/tokens_used": 328478176, + "theoretical_loss": 4.133637187047677, + "tokens_seen": 308018176 + }, + { + "epoch": 1.0, + "learning_rate": 0.00045791374122367104, + "loss": 2.9841, + "theoretical_loss": 4.133611500237993, + "tokens_seen": 308034560 + }, + { + "epoch": 1.0, + "learning_rate": 0.0004579037111334002, + "loss": 2.9786, + "theoretical_loss": 4.133508770484561, + "tokens_seen": 308100096 + }, + { + "epoch": 1.0, + "learning_rate": 0.0004578936810431294, + "loss": 3.0327, + "theoretical_loss": 4.13340606869744, + "tokens_seen": 308165632 + }, + { + "epoch": 1.0, + "learning_rate": 0.0004578836509528586, + "loss": 3.1817, + "theoretical_loss": 4.133303394863072, + "tokens_seen": 308231168 + }, + { + "epoch": 1.0, + "learning_rate": 0.00045787362086258777, + "loss": 2.8875, + "theoretical_loss": 4.133200748967909, + "tokens_seen": 308296704 + }, + { + "epoch": 1.0, + "learning_rate": 0.000457863590772317, + "loss": 3.3509, + "theoretical_loss": 4.133098130998412, + "tokens_seen": 308362240 + }, + { + "epoch": 1.0, + "learning_rate": 0.00045785356068204613, + "loss": 3.14, + "theoretical_loss": 4.1329955409410495, + "tokens_seen": 308427776 + }, + { + "epoch": 1.0, + "learning_rate": 0.00045784353059177536, + "loss": 3.1157, + "theoretical_loss": 4.132892978782301, + "tokens_seen": 308493312 + }, + { + "epoch": 1.0, + "learning_rate": 0.00045783350050150455, + "loss": 3.2767, + "theoretical_loss": 4.1327904445086565, + "tokens_seen": 308558848 + }, + { + "epoch": 1.0, + "learning_rate": 0.0004578234704112337, + "loss": 3.1308, + "theoretical_loss": 4.132687938106614, + "tokens_seen": 308624384 + }, + { + "epoch": 1.0, + "learning_rate": 0.0004578134403209629, + "loss": 2.9828, + "theoretical_loss": 4.132585459562681, + "tokens_seen": 308689920 + }, + { + "epoch": 1.0, + "learning_rate": 0.0004578034102306921, + "loss": 3.2487, + "theoretical_loss": 4.132483008863374, + "tokens_seen": 308755456 + }, + { + "epoch": 1.0, + "learning_rate": 0.00045779338014042127, + "loss": 2.948, + "theoretical_loss": 4.13238058599522, + "tokens_seen": 308820992 + }, + { + "epoch": 1.0, + "learning_rate": 0.0004577833500501505, + "loss": 3.2227, + "theoretical_loss": 4.132278190944755, + "tokens_seen": 308886528 + }, + { + "epoch": 1.0, + "learning_rate": 0.00045777331995987963, + "loss": 3.115, + "theoretical_loss": 4.132175823698523, + "tokens_seen": 308952064 + }, + { + "epoch": 1.0, + "learning_rate": 0.00045776328986960887, + "loss": 3.1668, + "theoretical_loss": 4.1320734842430795, + "tokens_seen": 309017600 + }, + { + "epoch": 1.0, + "learning_rate": 0.000457753259779338, + "loss": 3.0767, + "theoretical_loss": 4.131971172564989, + "tokens_seen": 309083136 + }, + { + "epoch": 1.0, + "learning_rate": 0.00045774322968906723, + "loss": 2.9926, + "theoretical_loss": 4.131868888650824, + "tokens_seen": 309148672 + }, + { + "epoch": 1.0, + "learning_rate": 0.0004577331995987964, + "loss": 3.0552, + "theoretical_loss": 4.1317666324871665, + "tokens_seen": 309214208 + }, + { + "epoch": 1.0, + "learning_rate": 0.0004577231695085256, + "loss": 3.0081, + "theoretical_loss": 4.13166440406061, + "tokens_seen": 309279744 + }, + { + "epoch": 1.0, + "learning_rate": 0.0004577131394182548, + "loss": 2.9077, + "theoretical_loss": 4.131562203357753, + "tokens_seen": 309345280 + }, + { + "epoch": 1.0, + "learning_rate": 0.00045770310932798395, + "loss": 3.0159, + "theoretical_loss": 4.131460030365209, + "tokens_seen": 309410816 + }, + { + "epoch": 1.0, + "learning_rate": 0.00045769307923771314, + "loss": 3.2991, + "theoretical_loss": 4.131357885069596, + "tokens_seen": 309476352 + }, + { + "epoch": 1.0, + "learning_rate": 0.00045768304914744237, + "loss": 3.2686, + "theoretical_loss": 4.131255767457543, + "tokens_seen": 309541888 + }, + { + "epoch": 1.0, + "learning_rate": 0.0004576730190571715, + "loss": 2.9812, + "theoretical_loss": 4.13115367751569, + "tokens_seen": 309607424 + }, + { + "epoch": 1.0, + "objective/train/docs_used": 423122, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.105621099472046, + "objective/train/theoretical_loss": 4.131077128209836, + "objective/train/tokens_used": 330116576, + "theoretical_loss": 4.131077128209836, + "tokens_seen": 309656576 + }, + { + "epoch": 1.0, + "learning_rate": 0.00045766298896690073, + "loss": 3.2017, + "theoretical_loss": 4.1310516152306835, + "tokens_seen": 309672960 + }, + { + "epoch": 1.0, + "learning_rate": 0.0004576529588766299, + "loss": 3.1174, + "theoretical_loss": 4.130949580589181, + "tokens_seen": 309738496 + }, + { + "epoch": 1.0, + "learning_rate": 0.0004576429287863591, + "loss": 3.0827, + "theoretical_loss": 4.130847573577848, + "tokens_seen": 309804032 + }, + { + "epoch": 1.0, + "learning_rate": 0.0004576328986960883, + "loss": 3.1589, + "theoretical_loss": 4.13074559418336, + "tokens_seen": 309869568 + }, + { + "epoch": 1.0, + "learning_rate": 0.00045762286860581746, + "loss": 3.2419, + "theoretical_loss": 4.1306436423924024, + "tokens_seen": 309935104 + }, + { + "epoch": 1.0, + "learning_rate": 0.00045761283851554664, + "loss": 3.0622, + "theoretical_loss": 4.130541718191669, + "tokens_seen": 310000640 + }, + { + "epoch": 1.0, + "learning_rate": 0.0004576028084252759, + "loss": 3.0627, + "theoretical_loss": 4.130439821567863, + "tokens_seen": 310066176 + }, + { + "epoch": 1.0, + "learning_rate": 0.000457592778335005, + "loss": 2.9596, + "theoretical_loss": 4.130337952507695, + "tokens_seen": 310131712 + }, + { + "epoch": 1.0, + "learning_rate": 0.00045758274824473424, + "loss": 3.1407, + "theoretical_loss": 4.130236110997889, + "tokens_seen": 310197248 + }, + { + "epoch": 1.0, + "learning_rate": 0.00045757271815446336, + "loss": 3.0654, + "theoretical_loss": 4.130134297025175, + "tokens_seen": 310262784 + }, + { + "epoch": 1.0, + "learning_rate": 0.0004575626880641926, + "loss": 3.1397, + "theoretical_loss": 4.130032510576292, + "tokens_seen": 310328320 + }, + { + "epoch": 1.0, + "learning_rate": 0.0004575526579739218, + "loss": 3.171, + "theoretical_loss": 4.129930751637991, + "tokens_seen": 310393856 + }, + { + "epoch": 1.0, + "learning_rate": 0.00045754262788365096, + "loss": 3.2332, + "theoretical_loss": 4.129829020197029, + "tokens_seen": 310459392 + }, + { + "epoch": 1.0, + "learning_rate": 0.00045753259779338014, + "loss": 3.0021, + "theoretical_loss": 4.1297273162401735, + "tokens_seen": 310524928 + }, + { + "epoch": 1.0, + "learning_rate": 0.0004575225677031093, + "loss": 2.9555, + "theoretical_loss": 4.1296256397542015, + "tokens_seen": 310590464 + }, + { + "epoch": 1.0, + "learning_rate": 0.0004575125376128385, + "loss": 3.1127, + "theoretical_loss": 4.1295239907258985, + "tokens_seen": 310656000 + }, + { + "epoch": 1.0, + "learning_rate": 0.00045750250752256774, + "loss": 3.0528, + "theoretical_loss": 4.12942236914206, + "tokens_seen": 310721536 + }, + { + "epoch": 1.0, + "learning_rate": 0.00045749247743229687, + "loss": 3.0231, + "theoretical_loss": 4.129320774989489, + "tokens_seen": 310787072 + }, + { + "epoch": 1.0, + "learning_rate": 0.0004574824473420261, + "loss": 3.1342, + "theoretical_loss": 4.129219208255002, + "tokens_seen": 310852608 + }, + { + "epoch": 1.0, + "learning_rate": 0.0004574724172517553, + "loss": 3.113, + "theoretical_loss": 4.129117668925415, + "tokens_seen": 310918144 + }, + { + "epoch": 1.0, + "learning_rate": 0.00045746238716148446, + "loss": 2.9051, + "theoretical_loss": 4.129016156987566, + "tokens_seen": 310983680 + }, + { + "epoch": 1.0, + "learning_rate": 0.00045745235707121364, + "loss": 2.9875, + "theoretical_loss": 4.128914672428293, + "tokens_seen": 311049216 + }, + { + "epoch": 1.0, + "learning_rate": 0.0004574423269809428, + "loss": 3.06, + "theoretical_loss": 4.128813215234444, + "tokens_seen": 311114752 + }, + { + "epoch": 1.0, + "learning_rate": 0.000457432296890672, + "loss": 3.0676, + "theoretical_loss": 4.128711785392879, + "tokens_seen": 311180288 + }, + { + "epoch": 1.0, + "learning_rate": 0.00045742226680040124, + "loss": 3.1011, + "theoretical_loss": 4.128610382890467, + "tokens_seen": 311245824 + }, + { + "epoch": 1.0, + "objective/train/docs_used": 428239, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.340728282928467, + "objective/train/theoretical_loss": 4.128534348947081, + "objective/train/tokens_used": 331754976, + "theoretical_loss": 4.128534348947081, + "tokens_seen": 311294976 + }, + { + "epoch": 1.0, + "learning_rate": 0.00045741223671013037, + "loss": 3.1619, + "theoretical_loss": 4.128509007714083, + "tokens_seen": 311311360 + }, + { + "epoch": 1.0, + "learning_rate": 0.0004574022066198596, + "loss": 3.0496, + "theoretical_loss": 4.128407659850614, + "tokens_seen": 311376896 + }, + { + "epoch": 1.0, + "learning_rate": 0.00045739217652958873, + "loss": 2.8642, + "theoretical_loss": 4.128306339286954, + "tokens_seen": 311442432 + }, + { + "epoch": 1.0, + "learning_rate": 0.00045738214643931797, + "loss": 3.0832, + "theoretical_loss": 4.1282050460100095, + "tokens_seen": 311507968 + }, + { + "epoch": 1.0, + "learning_rate": 0.00045737211634904715, + "loss": 3.1083, + "theoretical_loss": 4.12810378000669, + "tokens_seen": 311573504 + }, + { + "epoch": 1.0, + "learning_rate": 0.00045736208625877633, + "loss": 2.8529, + "theoretical_loss": 4.12800254126392, + "tokens_seen": 311639040 + }, + { + "epoch": 1.0, + "learning_rate": 0.0004573520561685055, + "loss": 3.1388, + "theoretical_loss": 4.12790132976863, + "tokens_seen": 311704576 + }, + { + "epoch": 1.0, + "learning_rate": 0.00045734202607823475, + "loss": 2.9444, + "theoretical_loss": 4.127800145507759, + "tokens_seen": 311770112 + }, + { + "epoch": 1.0, + "learning_rate": 0.00045733199598796387, + "loss": 2.8887, + "theoretical_loss": 4.127698988468259, + "tokens_seen": 311835648 + }, + { + "epoch": 1.0, + "learning_rate": 0.0004573219658976931, + "loss": 3.1635, + "theoretical_loss": 4.127597858637085, + "tokens_seen": 311901184 + }, + { + "epoch": 1.0, + "learning_rate": 0.00045731193580742223, + "loss": 3.0093, + "theoretical_loss": 4.1274967560012055, + "tokens_seen": 311966720 + }, + { + "epoch": 1.0, + "learning_rate": 0.00045730190571715147, + "loss": 3.1035, + "theoretical_loss": 4.127395680547597, + "tokens_seen": 312032256 + }, + { + "epoch": 1.0, + "learning_rate": 0.00045729187562688065, + "loss": 3.207, + "theoretical_loss": 4.127294632263243, + "tokens_seen": 312097792 + }, + { + "epoch": 1.0, + "learning_rate": 0.00045728184553660983, + "loss": 3.0375, + "theoretical_loss": 4.127193611135139, + "tokens_seen": 312163328 + }, + { + "epoch": 1.0, + "learning_rate": 0.000457271815446339, + "loss": 2.8885, + "theoretical_loss": 4.1270926171502875, + "tokens_seen": 312228864 + }, + { + "epoch": 1.0, + "learning_rate": 0.0004572617853560682, + "loss": 3.0487, + "theoretical_loss": 4.1269916502957, + "tokens_seen": 312294400 + }, + { + "epoch": 1.0, + "learning_rate": 0.0004572517552657974, + "loss": 3.0653, + "theoretical_loss": 4.126890710558398, + "tokens_seen": 312359936 + }, + { + "epoch": 1.0, + "learning_rate": 0.0004572417251755266, + "loss": 2.9576, + "theoretical_loss": 4.126789797925411, + "tokens_seen": 312425472 + }, + { + "epoch": 1.0, + "learning_rate": 0.00045723169508525574, + "loss": 3.2364, + "theoretical_loss": 4.126688912383777, + "tokens_seen": 312491008 + }, + { + "epoch": 1.0, + "learning_rate": 0.000457221664994985, + "loss": 3.2365, + "theoretical_loss": 4.126588053920545, + "tokens_seen": 312556544 + }, + { + "epoch": 1.0, + "learning_rate": 0.0004572116349047141, + "loss": 3.1703, + "theoretical_loss": 4.126487222522771, + "tokens_seen": 312622080 + }, + { + "epoch": 1.0, + "learning_rate": 0.00045720160481444334, + "loss": 3.1546, + "theoretical_loss": 4.12638641817752, + "tokens_seen": 312687616 + }, + { + "epoch": 1.0, + "learning_rate": 0.00045719157472417257, + "loss": 3.2756, + "theoretical_loss": 4.126285640871867, + "tokens_seen": 312753152 + }, + { + "epoch": 1.0, + "learning_rate": 0.0004571815446339017, + "loss": 3.1369, + "theoretical_loss": 4.126184890592894, + "tokens_seen": 312818688 + }, + { + "epoch": 1.0, + "learning_rate": 0.00045717151454363093, + "loss": 2.9366, + "theoretical_loss": 4.126084167327694, + "tokens_seen": 312884224 + }, + { + "epoch": 1.0, + "objective/train/docs_used": 433098, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.1568994522094727, + "objective/train/theoretical_loss": 4.126008642598823, + "objective/train/tokens_used": 333393376, + "theoretical_loss": 4.126008642598823, + "tokens_seen": 312933376 + }, + { + "epoch": 1.0, + "learning_rate": 0.0004571614844533601, + "loss": 3.0267, + "theoretical_loss": 4.125983471063368, + "tokens_seen": 312949760 + }, + { + "epoch": 1.0, + "learning_rate": 0.0004571514543630893, + "loss": 3.0007, + "theoretical_loss": 4.125882801787025, + "tokens_seen": 313015296 + }, + { + "epoch": 1.0, + "learning_rate": 0.0004571414242728185, + "loss": 3.1487, + "theoretical_loss": 4.125782159485785, + "tokens_seen": 313080832 + }, + { + "epoch": 1.0, + "learning_rate": 0.00045713139418254766, + "loss": 3.1603, + "theoretical_loss": 4.125681544146774, + "tokens_seen": 313146368 + }, + { + "epoch": 1.0, + "learning_rate": 0.00045712136409227684, + "loss": 3.1322, + "theoretical_loss": 4.12558095575713, + "tokens_seen": 313211904 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004571113340020061, + "loss": 3.1128, + "theoretical_loss": 4.125480394303996, + "tokens_seen": 313277440 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004571013039117352, + "loss": 3.0045, + "theoretical_loss": 4.125379859774528, + "tokens_seen": 313342976 + }, + { + "epoch": 1.01, + "learning_rate": 0.00045709127382146444, + "loss": 3.0059, + "theoretical_loss": 4.1252793521558875, + "tokens_seen": 313408512 + }, + { + "epoch": 1.01, + "learning_rate": 0.00045708124373119356, + "loss": 2.9702, + "theoretical_loss": 4.125178871435247, + "tokens_seen": 313474048 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004570712136409228, + "loss": 3.2466, + "theoretical_loss": 4.125078417599785, + "tokens_seen": 313539584 + }, + { + "epoch": 1.01, + "learning_rate": 0.000457061183550652, + "loss": 2.8958, + "theoretical_loss": 4.124977990636694, + "tokens_seen": 313605120 + }, + { + "epoch": 1.01, + "learning_rate": 0.00045705115346038116, + "loss": 2.9426, + "theoretical_loss": 4.124877590533169, + "tokens_seen": 313670656 + }, + { + "epoch": 1.01, + "learning_rate": 0.00045704112337011034, + "loss": 3.1383, + "theoretical_loss": 4.124777217276417, + "tokens_seen": 313736192 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004570310932798395, + "loss": 2.9676, + "theoretical_loss": 4.124676870853656, + "tokens_seen": 313801728 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004570210631895687, + "loss": 2.6934, + "theoretical_loss": 4.124576551252107, + "tokens_seen": 313867264 + }, + { + "epoch": 1.01, + "learning_rate": 0.00045701103309929794, + "loss": 2.9895, + "theoretical_loss": 4.124476258459005, + "tokens_seen": 313932800 + }, + { + "epoch": 1.01, + "learning_rate": 0.00045700100300902707, + "loss": 3.3011, + "theoretical_loss": 4.124375992461592, + "tokens_seen": 313998336 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004569909729187563, + "loss": 2.9931, + "theoretical_loss": 4.124275753247116, + "tokens_seen": 314063872 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004569809428284855, + "loss": 3.0106, + "theoretical_loss": 4.124175540802839, + "tokens_seen": 314129408 + }, + { + "epoch": 1.01, + "learning_rate": 0.00045697091273821466, + "loss": 3.0805, + "theoretical_loss": 4.124075355116028, + "tokens_seen": 314194944 + }, + { + "epoch": 1.01, + "learning_rate": 0.00045696088264794384, + "loss": 3.1193, + "theoretical_loss": 4.123975196173959, + "tokens_seen": 314260480 + }, + { + "epoch": 1.01, + "learning_rate": 0.000456950852557673, + "loss": 2.9561, + "theoretical_loss": 4.123875063963917, + "tokens_seen": 314326016 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004569408224674022, + "loss": 3.0758, + "theoretical_loss": 4.123774958473197, + "tokens_seen": 314391552 + }, + { + "epoch": 1.01, + "learning_rate": 0.00045693079237713144, + "loss": 3.1406, + "theoretical_loss": 4.123674879689101, + "tokens_seen": 314457088 + }, + { + "epoch": 1.01, + "learning_rate": 0.00045692076228686057, + "loss": 3.1404, + "theoretical_loss": 4.123574827598942, + "tokens_seen": 314522624 + }, + { + "epoch": 1.01, + "objective/train/docs_used": 438319, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.2052862644195557, + "objective/train/theoretical_loss": 4.12349980604159, + "objective/train/tokens_used": 335031776, + "theoretical_loss": 4.12349980604159, + "tokens_seen": 314571776 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004569107321965898, + "loss": 3.1546, + "theoretical_loss": 4.123474802190039, + "tokens_seen": 314588160 + }, + { + "epoch": 1.01, + "learning_rate": 0.00045690070210631893, + "loss": 3.1286, + "theoretical_loss": 4.123374803449719, + "tokens_seen": 314653696 + }, + { + "epoch": 1.01, + "learning_rate": 0.00045689067201604817, + "loss": 3.022, + "theoretical_loss": 4.123274831365322, + "tokens_seen": 314719232 + }, + { + "epoch": 1.01, + "learning_rate": 0.00045688064192577735, + "loss": 3.0278, + "theoretical_loss": 4.123174885924192, + "tokens_seen": 314784768 + }, + { + "epoch": 1.01, + "learning_rate": 0.00045687061183550653, + "loss": 3.1057, + "theoretical_loss": 4.123074967113686, + "tokens_seen": 314850304 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004568605817452357, + "loss": 3.2642, + "theoretical_loss": 4.122975074921165, + "tokens_seen": 314915840 + }, + { + "epoch": 1.01, + "learning_rate": 0.00045685055165496495, + "loss": 2.8919, + "theoretical_loss": 4.122875209334002, + "tokens_seen": 314981376 + }, + { + "epoch": 1.01, + "learning_rate": 0.00045684052156469407, + "loss": 3.119, + "theoretical_loss": 4.122775370339577, + "tokens_seen": 315046912 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004568304914744233, + "loss": 3.188, + "theoretical_loss": 4.12267555792528, + "tokens_seen": 315112448 + }, + { + "epoch": 1.01, + "learning_rate": 0.00045682046138415243, + "loss": 3.0429, + "theoretical_loss": 4.122575772078507, + "tokens_seen": 315177984 + }, + { + "epoch": 1.01, + "learning_rate": 0.00045681043129388167, + "loss": 3.2101, + "theoretical_loss": 4.122476012786667, + "tokens_seen": 315243520 + }, + { + "epoch": 1.01, + "learning_rate": 0.00045680040120361085, + "loss": 3.2976, + "theoretical_loss": 4.122376280037173, + "tokens_seen": 315309056 + }, + { + "epoch": 1.01, + "learning_rate": 0.00045679037111334003, + "loss": 3.0171, + "theoretical_loss": 4.122276573817448, + "tokens_seen": 315374592 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004567803410230692, + "loss": 3.0178, + "theoretical_loss": 4.122176894114926, + "tokens_seen": 315440128 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004567703109327984, + "loss": 3.2494, + "theoretical_loss": 4.122077240917046, + "tokens_seen": 315505664 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004567602808425276, + "loss": 2.9009, + "theoretical_loss": 4.121977614211259, + "tokens_seen": 315571200 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004567502507522568, + "loss": 3.2204, + "theoretical_loss": 4.12187801398502, + "tokens_seen": 315636736 + }, + { + "epoch": 1.01, + "learning_rate": 0.00045674022066198594, + "loss": 3.1532, + "theoretical_loss": 4.121778440225798, + "tokens_seen": 315702272 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004567301905717152, + "loss": 3.0529, + "theoretical_loss": 4.121678892921066, + "tokens_seen": 315767808 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004567201604814443, + "loss": 2.8736, + "theoretical_loss": 4.121579372058308, + "tokens_seen": 315833344 + }, + { + "epoch": 1.01, + "learning_rate": 0.00045671013039117354, + "loss": 3.103, + "theoretical_loss": 4.121479877625016, + "tokens_seen": 315898880 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004567001003009027, + "loss": 3.0901, + "theoretical_loss": 4.12138040960869, + "tokens_seen": 315964416 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004566900702106319, + "loss": 2.9924, + "theoretical_loss": 4.12128096799684, + "tokens_seen": 316029952 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004566800401203611, + "loss": 3.1131, + "theoretical_loss": 4.1211815527769815, + "tokens_seen": 316095488 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004566700100300903, + "loss": 3.2011, + "theoretical_loss": 4.1210821639366415, + "tokens_seen": 316161024 + }, + { + "epoch": 1.01, + "objective/train/docs_used": 440664, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.3091273307800293, + "objective/train/theoretical_loss": 4.121007639610445, + "objective/train/tokens_used": 336670176, + "theoretical_loss": 4.121007639610445, + "tokens_seen": 316210176 + }, + { + "epoch": 1.01, + "learning_rate": 0.00045665997993981944, + "loss": 3.1517, + "theoretical_loss": 4.120982801463353, + "tokens_seen": 316226560 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004566499498495487, + "loss": 3.0728, + "theoretical_loss": 4.1208834653446615, + "tokens_seen": 316292096 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004566399197592778, + "loss": 3.2325, + "theoretical_loss": 4.120784155568115, + "tokens_seen": 316357632 + }, + { + "epoch": 1.01, + "learning_rate": 0.00045662988966900704, + "loss": 3.1194, + "theoretical_loss": 4.120684872121277, + "tokens_seen": 316423168 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004566198595787362, + "loss": 3.1201, + "theoretical_loss": 4.120585614991712, + "tokens_seen": 316488704 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004566098294884654, + "loss": 3.1454, + "theoretical_loss": 4.120486384166998, + "tokens_seen": 316554240 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004565997993981946, + "loss": 2.9571, + "theoretical_loss": 4.120387179634721, + "tokens_seen": 316619776 + }, + { + "epoch": 1.01, + "learning_rate": 0.00045658976930792376, + "loss": 3.1203, + "theoretical_loss": 4.120288001382474, + "tokens_seen": 316685312 + }, + { + "epoch": 1.01, + "learning_rate": 0.00045657973921765294, + "loss": 3.1862, + "theoretical_loss": 4.1201888493978585, + "tokens_seen": 316750848 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004565697091273822, + "loss": 3.0073, + "theoretical_loss": 4.120089723668484, + "tokens_seen": 316816384 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004565596790371113, + "loss": 3.1276, + "theoretical_loss": 4.1199906241819715, + "tokens_seen": 316881920 + }, + { + "epoch": 1.01, + "learning_rate": 0.00045654964894684054, + "loss": 3.1453, + "theoretical_loss": 4.119891550925946, + "tokens_seen": 316947456 + }, + { + "epoch": 1.01, + "learning_rate": 0.00045653961885656967, + "loss": 3.1468, + "theoretical_loss": 4.119792503888043, + "tokens_seen": 317012992 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004565295887662989, + "loss": 3.0982, + "theoretical_loss": 4.119693483055908, + "tokens_seen": 317078528 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004565195586760281, + "loss": 2.9135, + "theoretical_loss": 4.119594488417192, + "tokens_seen": 317144064 + }, + { + "epoch": 1.01, + "learning_rate": 0.00045650952858575727, + "loss": 3.1709, + "theoretical_loss": 4.119495519959556, + "tokens_seen": 317209600 + }, + { + "epoch": 1.01, + "learning_rate": 0.00045649949849548645, + "loss": 3.1193, + "theoretical_loss": 4.119396577670669, + "tokens_seen": 317275136 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004564894684052157, + "loss": 3.1036, + "theoretical_loss": 4.119297661538209, + "tokens_seen": 317340672 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004564794383149448, + "loss": 2.9798, + "theoretical_loss": 4.11919877154986, + "tokens_seen": 317406208 + }, + { + "epoch": 1.01, + "learning_rate": 0.00045646940822467405, + "loss": 3.0293, + "theoretical_loss": 4.119099907693318, + "tokens_seen": 317471744 + }, + { + "epoch": 1.01, + "learning_rate": 0.00045645937813440317, + "loss": 3.0721, + "theoretical_loss": 4.119001069956284, + "tokens_seen": 317537280 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004564493480441324, + "loss": 3.3166, + "theoretical_loss": 4.11890225832647, + "tokens_seen": 317602816 + }, + { + "epoch": 1.01, + "learning_rate": 0.00045643931795386164, + "loss": 2.9832, + "theoretical_loss": 4.118803472791592, + "tokens_seen": 317668352 + }, + { + "epoch": 1.01, + "learning_rate": 0.00045642928786359077, + "loss": 3.1933, + "theoretical_loss": 4.1187047133393815, + "tokens_seen": 317733888 + }, + { + "epoch": 1.01, + "learning_rate": 0.00045641925777332, + "loss": 3.0072, + "theoretical_loss": 4.118605979957572, + "tokens_seen": 317799424 + }, + { + "epoch": 1.01, + "objective/train/docs_used": 445676, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.2730116844177246, + "objective/train/theoretical_loss": 4.118531947022541, + "objective/train/tokens_used": 338308576, + "theoretical_loss": 4.118531947022541, + "tokens_seen": 317848576 + }, + { + "epoch": 1.01, + "learning_rate": 0.00045640922768304913, + "loss": 3.2766, + "theoretical_loss": 4.118507272633906, + "tokens_seen": 317864960 + }, + { + "epoch": 1.01, + "learning_rate": 0.00045639919759277837, + "loss": 3.1235, + "theoretical_loss": 4.118408591356138, + "tokens_seen": 317930496 + }, + { + "epoch": 1.01, + "learning_rate": 0.00045638916750250755, + "loss": 2.7607, + "theoretical_loss": 4.118309936112027, + "tokens_seen": 317996032 + }, + { + "epoch": 1.01, + "learning_rate": 0.00045637913741223673, + "loss": 3.1125, + "theoretical_loss": 4.118211306889343, + "tokens_seen": 318061568 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004563691073219659, + "loss": 3.1064, + "theoretical_loss": 4.1181127036758625, + "tokens_seen": 318127104 + }, + { + "epoch": 1.01, + "learning_rate": 0.00045635907723169515, + "loss": 3.0764, + "theoretical_loss": 4.118014126459371, + "tokens_seen": 318192640 + }, + { + "epoch": 1.01, + "learning_rate": 0.00045634904714142427, + "loss": 3.127, + "theoretical_loss": 4.11791557522766, + "tokens_seen": 318258176 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004563390170511535, + "loss": 3.0333, + "theoretical_loss": 4.117817049968535, + "tokens_seen": 318323712 + }, + { + "epoch": 1.01, + "learning_rate": 0.00045632898696088263, + "loss": 3.21, + "theoretical_loss": 4.117718550669802, + "tokens_seen": 318389248 + }, + { + "epoch": 1.01, + "learning_rate": 0.00045631895687061187, + "loss": 3.0226, + "theoretical_loss": 4.117620077319282, + "tokens_seen": 318454784 + }, + { + "epoch": 1.01, + "learning_rate": 0.00045630892678034105, + "loss": 3.2592, + "theoretical_loss": 4.117521629904801, + "tokens_seen": 318520320 + }, + { + "epoch": 1.01, + "learning_rate": 0.00045629889669007023, + "loss": 2.9392, + "theoretical_loss": 4.117423208414193, + "tokens_seen": 318585856 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004562888665997994, + "loss": 3.0316, + "theoretical_loss": 4.117324812835301, + "tokens_seen": 318651392 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004562788365095286, + "loss": 3.1168, + "theoretical_loss": 4.117226443155976, + "tokens_seen": 318716928 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004562688064192578, + "loss": 3.0642, + "theoretical_loss": 4.117128099364078, + "tokens_seen": 318782464 + }, + { + "epoch": 1.01, + "learning_rate": 0.000456258776328987, + "loss": 2.8883, + "theoretical_loss": 4.1170297814474734, + "tokens_seen": 318848000 + }, + { + "epoch": 1.01, + "learning_rate": 0.00045624874623871614, + "loss": 3.0011, + "theoretical_loss": 4.116931489394039, + "tokens_seen": 318913536 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004562387161484454, + "loss": 2.9764, + "theoretical_loss": 4.116833223191657, + "tokens_seen": 318979072 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004562286860581745, + "loss": 2.9898, + "theoretical_loss": 4.116734982828222, + "tokens_seen": 319044608 + }, + { + "epoch": 1.01, + "learning_rate": 0.00045621865596790374, + "loss": 3.1809, + "theoretical_loss": 4.116636768291631, + "tokens_seen": 319110144 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004562086258776329, + "loss": 3.1512, + "theoretical_loss": 4.116538579569793, + "tokens_seen": 319175680 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004561985957873621, + "loss": 3.0586, + "theoretical_loss": 4.116440416650627, + "tokens_seen": 319241216 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004561885656970913, + "loss": 3.0317, + "theoretical_loss": 4.116342279522055, + "tokens_seen": 319306752 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004561785356068205, + "loss": 3.145, + "theoretical_loss": 4.11624416817201, + "tokens_seen": 319372288 + }, + { + "epoch": 1.01, + "learning_rate": 0.00045616850551654964, + "loss": 2.9133, + "theoretical_loss": 4.1161460825884335, + "tokens_seen": 319437824 + }, + { + "epoch": 1.01, + "objective/train/docs_used": 450671, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.054537773132324, + "objective/train/theoretical_loss": 4.116072535302745, + "objective/train/tokens_used": 339946976, + "theoretical_loss": 4.116072535302745, + "tokens_seen": 319486976 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004561584754262789, + "loss": 2.8832, + "theoretical_loss": 4.116048022759274, + "tokens_seen": 319503360 + }, + { + "epoch": 1.01, + "learning_rate": 0.000456148445336008, + "loss": 3.0356, + "theoretical_loss": 4.115949988672488, + "tokens_seen": 319568896 + }, + { + "epoch": 1.01, + "learning_rate": 0.00045613841524573724, + "loss": 3.0226, + "theoretical_loss": 4.115851980316042, + "tokens_seen": 319634432 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004561283851554664, + "loss": 2.753, + "theoretical_loss": 4.115753997677907, + "tokens_seen": 319699968 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004561183550651956, + "loss": 3.0136, + "theoretical_loss": 4.115656040746067, + "tokens_seen": 319765504 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004561083249749248, + "loss": 3.0264, + "theoretical_loss": 4.11555810950851, + "tokens_seen": 319831040 + }, + { + "epoch": 1.01, + "learning_rate": 0.00045609829488465396, + "loss": 3.0399, + "theoretical_loss": 4.1154602039532335, + "tokens_seen": 319896576 + }, + { + "epoch": 1.01, + "learning_rate": 0.00045608826479438314, + "loss": 2.8733, + "theoretical_loss": 4.115362324068242, + "tokens_seen": 319962112 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004560782347041124, + "loss": 3.123, + "theoretical_loss": 4.115264469841551, + "tokens_seen": 320027648 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004560682046138415, + "loss": 3.013, + "theoretical_loss": 4.115166641261181, + "tokens_seen": 320093184 + }, + { + "epoch": 1.01, + "learning_rate": 0.00045605817452357074, + "loss": 3.0235, + "theoretical_loss": 4.115068838315162, + "tokens_seen": 320158720 + }, + { + "epoch": 1.01, + "learning_rate": 0.00045604814443329987, + "loss": 2.9962, + "theoretical_loss": 4.1149710609915315, + "tokens_seen": 320224256 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004560381143430291, + "loss": 3.0287, + "theoretical_loss": 4.114873309278335, + "tokens_seen": 320289792 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004560280842527583, + "loss": 2.9697, + "theoretical_loss": 4.114775583163627, + "tokens_seen": 320355328 + }, + { + "epoch": 1.01, + "learning_rate": 0.00045601805416248747, + "loss": 2.974, + "theoretical_loss": 4.114677882635468, + "tokens_seen": 320420864 + }, + { + "epoch": 1.01, + "learning_rate": 0.00045600802407221665, + "loss": 3.1124, + "theoretical_loss": 4.114580207681929, + "tokens_seen": 320486400 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004559979939819459, + "loss": 3.2231, + "theoretical_loss": 4.114482558291087, + "tokens_seen": 320551936 + }, + { + "epoch": 1.01, + "learning_rate": 0.000455987963891675, + "loss": 2.8844, + "theoretical_loss": 4.114384934451029, + "tokens_seen": 320617472 + }, + { + "epoch": 1.01, + "learning_rate": 0.00045597793380140425, + "loss": 3.105, + "theoretical_loss": 4.114287336149847, + "tokens_seen": 320683008 + }, + { + "epoch": 1.01, + "learning_rate": 0.00045596790371113337, + "loss": 2.9922, + "theoretical_loss": 4.114189763375643, + "tokens_seen": 320748544 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004559578736208626, + "loss": 3.102, + "theoretical_loss": 4.1140922161165285, + "tokens_seen": 320814080 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004559478435305918, + "loss": 3.1323, + "theoretical_loss": 4.11399469436062, + "tokens_seen": 320879616 + }, + { + "epoch": 1.01, + "learning_rate": 0.00045593781344032097, + "loss": 2.9403, + "theoretical_loss": 4.113897198096042, + "tokens_seen": 320945152 + }, + { + "epoch": 1.01, + "learning_rate": 0.00045592778335005015, + "loss": 3.1139, + "theoretical_loss": 4.11379972731093, + "tokens_seen": 321010688 + }, + { + "epoch": 1.01, + "learning_rate": 0.00045591775325977933, + "loss": 2.9772, + "theoretical_loss": 4.113702281993424, + "tokens_seen": 321076224 + }, + { + "epoch": 1.01, + "objective/train/docs_used": 452128, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.911926031112671, + "objective/train/theoretical_loss": 4.113629214711283, + "objective/train/tokens_used": 341585376, + "theoretical_loss": 4.113629214711283, + "tokens_seen": 321125376 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004559077231695085, + "loss": 2.9501, + "theoretical_loss": 4.113604862131675, + "tokens_seen": 321141760 + }, + { + "epoch": 1.01, + "learning_rate": 0.00045589769307923775, + "loss": 2.9992, + "theoretical_loss": 4.113507467713839, + "tokens_seen": 321207296 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004558876629889669, + "loss": 3.1123, + "theoretical_loss": 4.113410098728081, + "tokens_seen": 321272832 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004558776328986961, + "loss": 3.1029, + "theoretical_loss": 4.1133127551625766, + "tokens_seen": 321338368 + }, + { + "epoch": 1.01, + "learning_rate": 0.00045586760280842524, + "loss": 3.1395, + "theoretical_loss": 4.113215437005504, + "tokens_seen": 321403904 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004558575727181545, + "loss": 2.9044, + "theoretical_loss": 4.113118144245054, + "tokens_seen": 321469440 + }, + { + "epoch": 1.01, + "learning_rate": 0.00045584754262788365, + "loss": 3.1156, + "theoretical_loss": 4.113020876869424, + "tokens_seen": 321534976 + }, + { + "epoch": 1.01, + "learning_rate": 0.00045583751253761284, + "loss": 3.1783, + "theoretical_loss": 4.112923634866817, + "tokens_seen": 321600512 + }, + { + "epoch": 1.01, + "learning_rate": 0.000455827482447342, + "loss": 3.0768, + "theoretical_loss": 4.112826418225447, + "tokens_seen": 321666048 + }, + { + "epoch": 1.01, + "learning_rate": 0.00045581745235707125, + "loss": 2.8923, + "theoretical_loss": 4.112729226933536, + "tokens_seen": 321731584 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004558074222668004, + "loss": 2.8801, + "theoretical_loss": 4.112632060979309, + "tokens_seen": 321797120 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004557973921765296, + "loss": 2.9205, + "theoretical_loss": 4.112534920351004, + "tokens_seen": 321862656 + }, + { + "epoch": 1.01, + "learning_rate": 0.00045578736208625874, + "loss": 2.9779, + "theoretical_loss": 4.112437805036866, + "tokens_seen": 321928192 + }, + { + "epoch": 1.01, + "learning_rate": 0.000455777331995988, + "loss": 3.0556, + "theoretical_loss": 4.112340715025147, + "tokens_seen": 321993728 + }, + { + "epoch": 1.01, + "learning_rate": 0.00045576730190571716, + "loss": 2.9769, + "theoretical_loss": 4.112243650304106, + "tokens_seen": 322059264 + }, + { + "epoch": 1.01, + "learning_rate": 0.00045575727181544634, + "loss": 3.1575, + "theoretical_loss": 4.11214661086201, + "tokens_seen": 322124800 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004557472417251755, + "loss": 3.2281, + "theoretical_loss": 4.112049596687136, + "tokens_seen": 322190336 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004557372116349047, + "loss": 3.2923, + "theoretical_loss": 4.111952607767767, + "tokens_seen": 322255872 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004557271815446339, + "loss": 3.1374, + "theoretical_loss": 4.111855644092194, + "tokens_seen": 322321408 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004557171514543631, + "loss": 2.9752, + "theoretical_loss": 4.111758705648716, + "tokens_seen": 322386944 + }, + { + "epoch": 1.01, + "learning_rate": 0.00045570712136409224, + "loss": 2.9913, + "theoretical_loss": 4.11166179242564, + "tokens_seen": 322452480 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004556970912738215, + "loss": 2.955, + "theoretical_loss": 4.11156490441128, + "tokens_seen": 322518016 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004556870611835507, + "loss": 3.1211, + "theoretical_loss": 4.111468041593958, + "tokens_seen": 322583552 + }, + { + "epoch": 1.01, + "learning_rate": 0.00045567703109327984, + "loss": 2.7598, + "theoretical_loss": 4.111371203962006, + "tokens_seen": 322649088 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004556670010030091, + "loss": 3.1663, + "theoretical_loss": 4.11127439150376, + "tokens_seen": 322714624 + }, + { + "epoch": 1.01, + "objective/train/docs_used": 452464, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7435483932495117, + "objective/train/theoretical_loss": 4.111201798673309, + "objective/train/tokens_used": 343223776, + "theoretical_loss": 4.111201798673309, + "tokens_seen": 322763776 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004556569709127382, + "loss": 3.0986, + "theoretical_loss": 4.111177604207566, + "tokens_seen": 322780160 + }, + { + "epoch": 1.01, + "learning_rate": 0.00045564694082246744, + "loss": 3.123, + "theoretical_loss": 4.111080842061779, + "tokens_seen": 322845696 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004556369107321966, + "loss": 2.9448, + "theoretical_loss": 4.110984105054758, + "tokens_seen": 322911232 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004556268806419258, + "loss": 2.6903, + "theoretical_loss": 4.110887393174874, + "tokens_seen": 322976768 + }, + { + "epoch": 1.01, + "learning_rate": 0.000455616850551655, + "loss": 3.3049, + "theoretical_loss": 4.110790706410502, + "tokens_seen": 323042304 + }, + { + "epoch": 1.01, + "learning_rate": 0.00045560682046138416, + "loss": 3.0815, + "theoretical_loss": 4.110694044750028, + "tokens_seen": 323107840 + }, + { + "epoch": 1.01, + "learning_rate": 0.00045559679037111334, + "loss": 3.0381, + "theoretical_loss": 4.110597408181842, + "tokens_seen": 323173376 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004555867602808426, + "loss": 3.1317, + "theoretical_loss": 4.110500796694346, + "tokens_seen": 323238912 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004555767301905717, + "loss": 2.9622, + "theoretical_loss": 4.110404210275947, + "tokens_seen": 323304448 + }, + { + "epoch": 1.01, + "learning_rate": 0.00045556670010030094, + "loss": 3.0428, + "theoretical_loss": 4.110307648915059, + "tokens_seen": 323369984 + }, + { + "epoch": 1.01, + "learning_rate": 0.00045555667001003007, + "loss": 3.0631, + "theoretical_loss": 4.110211112600107, + "tokens_seen": 323435520 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004555466399197593, + "loss": 3.0437, + "theoretical_loss": 4.11011460131952, + "tokens_seen": 323501056 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004555366098294885, + "loss": 3.08, + "theoretical_loss": 4.110018115061737, + "tokens_seen": 323566592 + }, + { + "epoch": 1.01, + "learning_rate": 0.00045552657973921767, + "loss": 2.8587, + "theoretical_loss": 4.109921653815205, + "tokens_seen": 323632128 + }, + { + "epoch": 1.01, + "learning_rate": 0.00045551654964894685, + "loss": 2.8589, + "theoretical_loss": 4.109825217568375, + "tokens_seen": 323697664 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004555065195586761, + "loss": 2.9758, + "theoretical_loss": 4.109728806309711, + "tokens_seen": 323763200 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004554964894684052, + "loss": 3.0621, + "theoretical_loss": 4.1096324200276815, + "tokens_seen": 323828736 + }, + { + "epoch": 1.01, + "learning_rate": 0.00045548645937813445, + "loss": 3.2006, + "theoretical_loss": 4.109536058710763, + "tokens_seen": 323894272 + }, + { + "epoch": 1.01, + "learning_rate": 0.00045547642928786357, + "loss": 3.2291, + "theoretical_loss": 4.109439722347439, + "tokens_seen": 323959808 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004554663991975928, + "loss": 3.0339, + "theoretical_loss": 4.109343410926202, + "tokens_seen": 324025344 + }, + { + "epoch": 1.01, + "learning_rate": 0.000455456369107322, + "loss": 3.1038, + "theoretical_loss": 4.109247124435552, + "tokens_seen": 324090880 + }, + { + "epoch": 1.01, + "learning_rate": 0.00045544633901705117, + "loss": 3.2273, + "theoretical_loss": 4.109150862863997, + "tokens_seen": 324156416 + }, + { + "epoch": 1.01, + "learning_rate": 0.00045543630892678035, + "loss": 3.0505, + "theoretical_loss": 4.1090546262000505, + "tokens_seen": 324221952 + }, + { + "epoch": 1.01, + "learning_rate": 0.00045542627883650953, + "loss": 3.101, + "theoretical_loss": 4.108958414432235, + "tokens_seen": 324287488 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004554162487462387, + "loss": 3.1216, + "theoretical_loss": 4.10886222754908, + "tokens_seen": 324353024 + }, + { + "epoch": 1.01, + "objective/train/docs_used": 453833, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.195937395095825, + "objective/train/theoretical_loss": 4.108790103710379, + "objective/train/tokens_used": 344862176, + "theoretical_loss": 4.108790103710379, + "tokens_seen": 324402176 + }, + { + "epoch": 1.01, + "learning_rate": 0.00045540621865596795, + "loss": 3.2008, + "theoretical_loss": 4.108766065539125, + "tokens_seen": 324418560 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004553961885656971, + "loss": 3.0328, + "theoretical_loss": 4.108669928390915, + "tokens_seen": 324484096 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004553861584754263, + "loss": 3.0984, + "theoretical_loss": 4.108573816093003, + "tokens_seen": 324549632 + }, + { + "epoch": 1.01, + "learning_rate": 0.00045537612838515544, + "loss": 2.8293, + "theoretical_loss": 4.108477728633948, + "tokens_seen": 324615168 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004553660982948847, + "loss": 3.2014, + "theoretical_loss": 4.108381666002318, + "tokens_seen": 324680704 + }, + { + "epoch": 1.01, + "learning_rate": 0.00045535606820461385, + "loss": 3.0183, + "theoretical_loss": 4.10828562818669, + "tokens_seen": 324746240 + }, + { + "epoch": 1.01, + "learning_rate": 0.00045534603811434304, + "loss": 3.1596, + "theoretical_loss": 4.108189615175646, + "tokens_seen": 324811776 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004553360080240722, + "loss": 2.7589, + "theoretical_loss": 4.1080936269577775, + "tokens_seen": 324877312 + }, + { + "epoch": 1.01, + "learning_rate": 0.00045532597793380145, + "loss": 2.8328, + "theoretical_loss": 4.107997663521683, + "tokens_seen": 324942848 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004553159478435306, + "loss": 2.7061, + "theoretical_loss": 4.107901724855966, + "tokens_seen": 325008384 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004553059177532598, + "loss": 2.8632, + "theoretical_loss": 4.107805810949244, + "tokens_seen": 325073920 + }, + { + "epoch": 1.01, + "learning_rate": 0.00045529588766298894, + "loss": 2.9472, + "theoretical_loss": 4.107709921790134, + "tokens_seen": 325139456 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004552858575727182, + "loss": 2.9933, + "theoretical_loss": 4.1076140573672655, + "tokens_seen": 325204992 + }, + { + "epoch": 1.01, + "learning_rate": 0.00045527582748244736, + "loss": 3.0292, + "theoretical_loss": 4.107518217669275, + "tokens_seen": 325270528 + }, + { + "epoch": 1.01, + "learning_rate": 0.00045526579739217654, + "loss": 2.9263, + "theoretical_loss": 4.107422402684806, + "tokens_seen": 325336064 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004552557673019057, + "loss": 3.1346, + "theoretical_loss": 4.107326612402509, + "tokens_seen": 325401600 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004552457372116349, + "loss": 3.1047, + "theoretical_loss": 4.107230846811042, + "tokens_seen": 325467136 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004552357071213641, + "loss": 3.0838, + "theoretical_loss": 4.107135105899072, + "tokens_seen": 325532672 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004552256770310933, + "loss": 3.2631, + "theoretical_loss": 4.107039389655271, + "tokens_seen": 325598208 + }, + { + "epoch": 1.01, + "learning_rate": 0.00045521564694082244, + "loss": 2.9412, + "theoretical_loss": 4.106943698068321, + "tokens_seen": 325663744 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004552056168505517, + "loss": 3.1445, + "theoretical_loss": 4.106848031126909, + "tokens_seen": 325729280 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004551955867602808, + "loss": 2.9177, + "theoretical_loss": 4.106752388819732, + "tokens_seen": 325794816 + }, + { + "epoch": 1.01, + "learning_rate": 0.00045518555667001004, + "loss": 2.8788, + "theoretical_loss": 4.106656771135494, + "tokens_seen": 325860352 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004551755265797392, + "loss": 3.0533, + "theoretical_loss": 4.106561178062903, + "tokens_seen": 325925888 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004551654964894684, + "loss": 3.2877, + "theoretical_loss": 4.10646560959068, + "tokens_seen": 325991424 + }, + { + "epoch": 1.01, + "objective/train/docs_used": 455028, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.201298236846924, + "objective/train/theoretical_loss": 4.10639394937372, + "objective/train/tokens_used": 346500576, + "theoretical_loss": 4.10639394937372, + "tokens_seen": 326040576 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004551554663991976, + "loss": 3.1642, + "theoretical_loss": 4.106370065707549, + "tokens_seen": 326056960 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004551454363089268, + "loss": 2.8599, + "theoretical_loss": 4.1062745464022425, + "tokens_seen": 326122496 + }, + { + "epoch": 1.01, + "learning_rate": 0.00045513540621865595, + "loss": 2.9966, + "theoretical_loss": 4.106179051663502, + "tokens_seen": 326188032 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004551253761283852, + "loss": 3.0048, + "theoretical_loss": 4.106083581480076, + "tokens_seen": 326253568 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004551153460381143, + "loss": 3.1238, + "theoretical_loss": 4.105988135840717, + "tokens_seen": 326319104 + }, + { + "epoch": 1.01, + "learning_rate": 0.00045510531594784354, + "loss": 2.9494, + "theoretical_loss": 4.105892714734191, + "tokens_seen": 326384640 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004550952858575727, + "loss": 2.8344, + "theoretical_loss": 4.105797318149266, + "tokens_seen": 326450176 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004550852557673019, + "loss": 3.2039, + "theoretical_loss": 4.105701946074721, + "tokens_seen": 326515712 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004550752256770311, + "loss": 3.1371, + "theoretical_loss": 4.105606598499339, + "tokens_seen": 326581248 + }, + { + "epoch": 1.01, + "learning_rate": 0.00045506519558676027, + "loss": 3.0273, + "theoretical_loss": 4.105511275411914, + "tokens_seen": 326646784 + }, + { + "epoch": 1.01, + "learning_rate": 0.00045505516549648945, + "loss": 2.9177, + "theoretical_loss": 4.105415976801243, + "tokens_seen": 326712320 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004550451354062187, + "loss": 3.1699, + "theoretical_loss": 4.105320702656137, + "tokens_seen": 326777856 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004550351053159478, + "loss": 3.0427, + "theoretical_loss": 4.105225452965406, + "tokens_seen": 326843392 + }, + { + "epoch": 1.01, + "learning_rate": 0.00045502507522567705, + "loss": 3.0283, + "theoretical_loss": 4.105130227717876, + "tokens_seen": 326908928 + }, + { + "epoch": 1.01, + "learning_rate": 0.00045501504513540623, + "loss": 3.0798, + "theoretical_loss": 4.105035026902372, + "tokens_seen": 326974464 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004550050150451354, + "loss": 3.1388, + "theoretical_loss": 4.104939850507733, + "tokens_seen": 327040000 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004549949849548646, + "loss": 2.9548, + "theoretical_loss": 4.104844698522801, + "tokens_seen": 327105536 + }, + { + "epoch": 1.01, + "learning_rate": 0.00045498495486459377, + "loss": 3.1927, + "theoretical_loss": 4.104749570936429, + "tokens_seen": 327171072 + }, + { + "epoch": 1.01, + "learning_rate": 0.00045497492477432295, + "loss": 2.939, + "theoretical_loss": 4.104654467737474, + "tokens_seen": 327236608 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004549648946840522, + "loss": 3.1495, + "theoretical_loss": 4.104559388914802, + "tokens_seen": 327302144 + }, + { + "epoch": 1.01, + "learning_rate": 0.00045495486459378137, + "loss": 3.1292, + "theoretical_loss": 4.104464334457286, + "tokens_seen": 327367680 + }, + { + "epoch": 1.01, + "learning_rate": 0.00045494483450351055, + "loss": 3.1137, + "theoretical_loss": 4.104369304353806, + "tokens_seen": 327433216 + }, + { + "epoch": 1.01, + "learning_rate": 0.00045493480441323973, + "loss": 3.1885, + "theoretical_loss": 4.104274298593252, + "tokens_seen": 327498752 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004549247743229689, + "loss": 3.1253, + "theoretical_loss": 4.1041793171645145, + "tokens_seen": 327564288 + }, + { + "epoch": 1.01, + "learning_rate": 0.00045491474423269815, + "loss": 2.9915, + "theoretical_loss": 4.104084360056499, + "tokens_seen": 327629824 + }, + { + "debugging/Self-BLEU-5": 0.6136948618909069, + "debugging/distinct-1-grams": 0.7435805096055491, + "debugging/distinct-2-grams": 0.8810311841604012, + "debugging/entropy-1-grams": 6.325230867052365, + "debugging/entropy-2-grams": 7.6356135460409735, + "debugging/length": 511.3030303030303, + "debugging/num_segments": 33, + "debugging/score": 0.0005877098914164371, + "debugging/score_std": 0.0013283427361280467, + "epoch": 1.01, + "objective/train/docs_used": 455830, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.0165796279907227, + "objective/train/theoretical_loss": 4.1040131581792885, + "objective/train/tokens_used": 348138976, + "theoretical_loss": 4.1040131581792885, + "tokens_seen": 327678976 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004549047141424273, + "loss": 2.8767, + "theoretical_loss": 4.103989427258114, + "tokens_seen": 327695360 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004548946840521565, + "loss": 3.0342, + "theoretical_loss": 4.103894518758277, + "tokens_seen": 327760896 + }, + { + "epoch": 1.01, + "learning_rate": 0.00045488465396188564, + "loss": 3.0731, + "theoretical_loss": 4.1037996345459105, + "tokens_seen": 327826432 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004548746238716149, + "loss": 3.1697, + "theoretical_loss": 4.103704774609946, + "tokens_seen": 327891968 + }, + { + "epoch": 1.01, + "learning_rate": 0.00045486459378134405, + "loss": 3.1106, + "theoretical_loss": 4.103609938939324, + "tokens_seen": 327957504 + }, + { + "epoch": 1.01, + "learning_rate": 0.00045485456369107324, + "loss": 3.1378, + "theoretical_loss": 4.103515127522988, + "tokens_seen": 328023040 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004548445336008024, + "loss": 2.9621, + "theoretical_loss": 4.103420340349893, + "tokens_seen": 328088576 + }, + { + "epoch": 1.01, + "learning_rate": 0.00045483450351053165, + "loss": 2.9669, + "theoretical_loss": 4.103325577408997, + "tokens_seen": 328154112 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004548244734202608, + "loss": 3.0197, + "theoretical_loss": 4.103230838689269, + "tokens_seen": 328219648 + }, + { + "epoch": 1.01, + "learning_rate": 0.00045481444332999, + "loss": 3.0518, + "theoretical_loss": 4.103136124179684, + "tokens_seen": 328285184 + }, + { + "epoch": 1.01, + "learning_rate": 0.00045480441323971914, + "loss": 3.2399, + "theoretical_loss": 4.103041433869223, + "tokens_seen": 328350720 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004547943831494484, + "loss": 3.0063, + "theoretical_loss": 4.102946767746875, + "tokens_seen": 328416256 + }, + { + "epoch": 1.01, + "learning_rate": 0.00045478435305917756, + "loss": 2.907, + "theoretical_loss": 4.102852125801638, + "tokens_seen": 328481792 + }, + { + "epoch": 1.01, + "learning_rate": 0.00045477432296890674, + "loss": 3.197, + "theoretical_loss": 4.102757508022513, + "tokens_seen": 328547328 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004547642928786359, + "loss": 3.2354, + "theoretical_loss": 4.102662914398513, + "tokens_seen": 328612864 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004547542627883651, + "loss": 2.9079, + "theoretical_loss": 4.102568344918655, + "tokens_seen": 328678400 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004547442326980943, + "loss": 2.8512, + "theoretical_loss": 4.102473799571964, + "tokens_seen": 328743936 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004547342026078235, + "loss": 3.0689, + "theoretical_loss": 4.102379278347472, + "tokens_seen": 328809472 + }, + { + "epoch": 1.01, + "learning_rate": 0.00045472417251755264, + "loss": 2.9659, + "theoretical_loss": 4.10228478123422, + "tokens_seen": 328875008 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004547141424272819, + "loss": 3.2956, + "theoretical_loss": 4.102190308221253, + "tokens_seen": 328940544 + }, + { + "epoch": 1.01, + "learning_rate": 0.000454704112337011, + "loss": 3.0001, + "theoretical_loss": 4.102095859297625, + "tokens_seen": 329006080 + }, + { + "epoch": 1.01, + "learning_rate": 0.00045469408224674024, + "loss": 3.0739, + "theoretical_loss": 4.102001434452398, + "tokens_seen": 329071616 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004546840521564694, + "loss": 3.0976, + "theoretical_loss": 4.101907033674639, + "tokens_seen": 329137152 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004546740220661986, + "loss": 3.0629, + "theoretical_loss": 4.101812656953424, + "tokens_seen": 329202688 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004546639919759278, + "loss": 2.9582, + "theoretical_loss": 4.101718304277834, + "tokens_seen": 329268224 + }, + { + "epoch": 1.01, + "objective/train/docs_used": 456654, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.36710524559021, + "objective/train/theoretical_loss": 4.10164755554452, + "objective/train/tokens_used": 349777376, + "theoretical_loss": 4.10164755554452, + "tokens_seen": 329317376 + }, + { + "epoch": 1.01, + "learning_rate": 0.000454653961885657, + "loss": 2.9395, + "theoretical_loss": 4.10162397563696, + "tokens_seen": 329333760 + }, + { + "epoch": 1.01, + "learning_rate": 0.00045464393179538615, + "loss": 3.1366, + "theoretical_loss": 4.101529671019898, + "tokens_seen": 329399296 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004546339017051154, + "loss": 2.7547, + "theoretical_loss": 4.101435390415752, + "tokens_seen": 329464832 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004546238716148445, + "loss": 3.222, + "theoretical_loss": 4.101341133813632, + "tokens_seen": 329530368 + }, + { + "epoch": 1.01, + "learning_rate": 0.00045461384152457374, + "loss": 2.9597, + "theoretical_loss": 4.101246901202655, + "tokens_seen": 329595904 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004546038114343029, + "loss": 3.0324, + "theoretical_loss": 4.101152692571949, + "tokens_seen": 329661440 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004545937813440321, + "loss": 3.1991, + "theoretical_loss": 4.101058507910645, + "tokens_seen": 329726976 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004545837512537613, + "loss": 3.0136, + "theoretical_loss": 4.100964347207881, + "tokens_seen": 329792512 + }, + { + "epoch": 1.01, + "learning_rate": 0.00045457372116349047, + "loss": 3.1253, + "theoretical_loss": 4.100870210452804, + "tokens_seen": 329858048 + }, + { + "epoch": 1.01, + "learning_rate": 0.00045456369107321965, + "loss": 3.1968, + "theoretical_loss": 4.100776097634567, + "tokens_seen": 329923584 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004545536609829489, + "loss": 3.0509, + "theoretical_loss": 4.100682008742331, + "tokens_seen": 329989120 + }, + { + "epoch": 1.01, + "learning_rate": 0.000454543630892678, + "loss": 2.961, + "theoretical_loss": 4.100587943765264, + "tokens_seen": 330054656 + }, + { + "epoch": 1.01, + "learning_rate": 0.00045453360080240725, + "loss": 2.9026, + "theoretical_loss": 4.100493902692539, + "tokens_seen": 330120192 + }, + { + "epoch": 1.01, + "learning_rate": 0.00045452357071213643, + "loss": 3.0556, + "theoretical_loss": 4.100399885513339, + "tokens_seen": 330185728 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004545135406218656, + "loss": 3.03, + "theoretical_loss": 4.10030589221685, + "tokens_seen": 330251264 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004545035105315948, + "loss": 2.9057, + "theoretical_loss": 4.100211922792271, + "tokens_seen": 330316800 + }, + { + "epoch": 1.01, + "learning_rate": 0.00045449348044132397, + "loss": 3.1829, + "theoretical_loss": 4.100117977228804, + "tokens_seen": 330382336 + }, + { + "epoch": 1.01, + "learning_rate": 0.00045448345035105315, + "loss": 2.9077, + "theoretical_loss": 4.1000240555156555, + "tokens_seen": 330447872 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004544734202607824, + "loss": 3.0517, + "theoretical_loss": 4.099930157642047, + "tokens_seen": 330513408 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004544633901705115, + "loss": 2.9146, + "theoretical_loss": 4.099836283597199, + "tokens_seen": 330578944 + }, + { + "epoch": 1.01, + "learning_rate": 0.00045445336008024075, + "loss": 2.989, + "theoretical_loss": 4.099742433370342, + "tokens_seen": 330644480 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004544433299899699, + "loss": 2.8763, + "theoretical_loss": 4.0996486069507165, + "tokens_seen": 330710016 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004544332998996991, + "loss": 2.9498, + "theoretical_loss": 4.099554804327565, + "tokens_seen": 330775552 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004544232698094283, + "loss": 2.8131, + "theoretical_loss": 4.09946102549014, + "tokens_seen": 330841088 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004544132397191575, + "loss": 2.9849, + "theoretical_loss": 4.099367270427699, + "tokens_seen": 330906624 + }, + { + "epoch": 1.01, + "objective/train/docs_used": 458161, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8100082874298096, + "objective/train/theoretical_loss": 4.099296969726746, + "objective/train/tokens_used": 351415776, + "theoretical_loss": 4.099296969726746, + "tokens_seen": 330955776 + }, + { + "epoch": 1.01, + "learning_rate": 0.00045440320962888666, + "loss": 2.9439, + "theoretical_loss": 4.099273539129509, + "tokens_seen": 330972160 + }, + { + "epoch": 1.01, + "learning_rate": 0.00045439317953861584, + "loss": 3.0252, + "theoretical_loss": 4.099179831584843, + "tokens_seen": 331037696 + }, + { + "epoch": 1.01, + "learning_rate": 0.000454383149448345, + "loss": 3.0193, + "theoretical_loss": 4.09908614778298, + "tokens_seen": 331103232 + }, + { + "epoch": 1.01, + "learning_rate": 0.00045437311935807425, + "loss": 2.9958, + "theoretical_loss": 4.098992487713207, + "tokens_seen": 331168768 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004543630892678034, + "loss": 3.0734, + "theoretical_loss": 4.098898851364815, + "tokens_seen": 331234304 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004543530591775326, + "loss": 2.9824, + "theoretical_loss": 4.098805238727109, + "tokens_seen": 331299840 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004543430290872618, + "loss": 3.2124, + "theoretical_loss": 4.098711649789392, + "tokens_seen": 331365376 + }, + { + "epoch": 1.01, + "learning_rate": 0.000454332998996991, + "loss": 2.8041, + "theoretical_loss": 4.098618084540981, + "tokens_seen": 331430912 + }, + { + "epoch": 1.01, + "learning_rate": 0.00045432296890672016, + "loss": 2.8408, + "theoretical_loss": 4.098524542971197, + "tokens_seen": 331496448 + }, + { + "epoch": 1.01, + "learning_rate": 0.00045431293881644934, + "loss": 3.1551, + "theoretical_loss": 4.098431025069368, + "tokens_seen": 331561984 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004543029087261785, + "loss": 3.0807, + "theoretical_loss": 4.098337530824828, + "tokens_seen": 331627520 + }, + { + "epoch": 1.01, + "learning_rate": 0.00045429287863590776, + "loss": 2.9272, + "theoretical_loss": 4.09824406022692, + "tokens_seen": 331693056 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004542828485456369, + "loss": 3.0841, + "theoretical_loss": 4.098150613264993, + "tokens_seen": 331758592 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004542728184553661, + "loss": 3.2841, + "theoretical_loss": 4.0980571899284035, + "tokens_seen": 331824128 + }, + { + "epoch": 1.01, + "learning_rate": 0.00045426278836509525, + "loss": 2.9232, + "theoretical_loss": 4.097963790206513, + "tokens_seen": 331889664 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004542527582748245, + "loss": 2.9302, + "theoretical_loss": 4.097870414088691, + "tokens_seen": 331955200 + }, + { + "epoch": 1.01, + "learning_rate": 0.00045424272818455366, + "loss": 3.2363, + "theoretical_loss": 4.097777061564315, + "tokens_seen": 332020736 + }, + { + "epoch": 1.01, + "learning_rate": 0.00045423269809428284, + "loss": 3.1058, + "theoretical_loss": 4.097683732622768, + "tokens_seen": 332086272 + }, + { + "epoch": 1.01, + "learning_rate": 0.000454222668004012, + "loss": 2.9857, + "theoretical_loss": 4.09759042725344, + "tokens_seen": 332151808 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004542126379137412, + "loss": 3.1798, + "theoretical_loss": 4.097497145445729, + "tokens_seen": 332217344 + }, + { + "epoch": 1.01, + "learning_rate": 0.00045420260782347044, + "loss": 2.9651, + "theoretical_loss": 4.097403887189038, + "tokens_seen": 332282880 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004541925777331996, + "loss": 3.1198, + "theoretical_loss": 4.097310652472778, + "tokens_seen": 332348416 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004541825476429288, + "loss": 2.9496, + "theoretical_loss": 4.097217441286367, + "tokens_seen": 332413952 + }, + { + "epoch": 1.01, + "learning_rate": 0.000454172517552658, + "loss": 3.0268, + "theoretical_loss": 4.09712425361923, + "tokens_seen": 332479488 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004541624874623872, + "loss": 2.9316, + "theoretical_loss": 4.097031089460796, + "tokens_seen": 332545024 + }, + { + "epoch": 1.01, + "objective/train/docs_used": 459492, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.3007261753082275, + "objective/train/theoretical_loss": 4.096961231763205, + "objective/train/tokens_used": 353054176, + "theoretical_loss": 4.096961231763205, + "tokens_seen": 332594176 + }, + { + "epoch": 1.01, + "learning_rate": 0.00045415245737211635, + "loss": 3.137, + "theoretical_loss": 4.096937948800506, + "tokens_seen": 332610560 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004541424272818456, + "loss": 3.1584, + "theoretical_loss": 4.096844831627804, + "tokens_seen": 332676096 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004541323971915747, + "loss": 3.3038, + "theoretical_loss": 4.096751737932141, + "tokens_seen": 332741632 + }, + { + "epoch": 1.01, + "learning_rate": 0.00045412236710130395, + "loss": 2.9974, + "theoretical_loss": 4.096658667702978, + "tokens_seen": 332807168 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004541123370110331, + "loss": 2.7968, + "theoretical_loss": 4.096565620929778, + "tokens_seen": 332872704 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004541023069207623, + "loss": 2.9963, + "theoretical_loss": 4.096472597602014, + "tokens_seen": 332938240 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004540922768304915, + "loss": 3.0869, + "theoretical_loss": 4.096379597709166, + "tokens_seen": 333003776 + }, + { + "epoch": 1.01, + "learning_rate": 0.00045408224674022067, + "loss": 3.1367, + "theoretical_loss": 4.0962866212407185, + "tokens_seen": 333069312 + }, + { + "epoch": 1.01, + "learning_rate": 0.00045407221664994985, + "loss": 2.9563, + "theoretical_loss": 4.096193668186165, + "tokens_seen": 333134848 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004540621865596791, + "loss": 2.7802, + "theoretical_loss": 4.096100738535004, + "tokens_seen": 333200384 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004540521564694082, + "loss": 3.09, + "theoretical_loss": 4.096007832276742, + "tokens_seen": 333265920 + }, + { + "epoch": 1.01, + "learning_rate": 0.00045404212637913745, + "loss": 2.9668, + "theoretical_loss": 4.0959149494008935, + "tokens_seen": 333331456 + }, + { + "epoch": 1.01, + "learning_rate": 0.00045403209628886663, + "loss": 3.0414, + "theoretical_loss": 4.095822089896976, + "tokens_seen": 333396992 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004540220661985958, + "loss": 2.8439, + "theoretical_loss": 4.0957292537545165, + "tokens_seen": 333462528 + }, + { + "epoch": 1.01, + "learning_rate": 0.000454012036108325, + "loss": 3.1205, + "theoretical_loss": 4.09563644096305, + "tokens_seen": 333528064 + }, + { + "epoch": 1.01, + "learning_rate": 0.00045400200601805417, + "loss": 3.0406, + "theoretical_loss": 4.095543651512115, + "tokens_seen": 333593600 + }, + { + "epoch": 1.01, + "learning_rate": 0.00045399197592778335, + "loss": 3.1992, + "theoretical_loss": 4.095450885391257, + "tokens_seen": 333659136 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004539819458375126, + "loss": 2.9049, + "theoretical_loss": 4.095358142590031, + "tokens_seen": 333724672 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004539719157472417, + "loss": 2.8475, + "theoretical_loss": 4.095265423097998, + "tokens_seen": 333790208 + }, + { + "epoch": 1.01, + "learning_rate": 0.00045396188565697095, + "loss": 3.0034, + "theoretical_loss": 4.095172726904723, + "tokens_seen": 333855744 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004539518555667001, + "loss": 3.0068, + "theoretical_loss": 4.09508005399978, + "tokens_seen": 333921280 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004539418254764293, + "loss": 2.9264, + "theoretical_loss": 4.094987404372751, + "tokens_seen": 333986816 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004539317953861585, + "loss": 2.9861, + "theoretical_loss": 4.094894778013221, + "tokens_seen": 334052352 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004539217652958877, + "loss": 2.9869, + "theoretical_loss": 4.094802174910784, + "tokens_seen": 334117888 + }, + { + "epoch": 1.01, + "learning_rate": 0.00045391173520561686, + "loss": 3.2001, + "theoretical_loss": 4.094709595055042, + "tokens_seen": 334183424 + }, + { + "epoch": 1.01, + "objective/train/docs_used": 460159, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9135870933532715, + "objective/train/theoretical_loss": 4.094640175412626, + "objective/train/tokens_used": 354692576, + "theoretical_loss": 4.094640175412626, + "tokens_seen": 334232576 + }, + { + "epoch": 1.01, + "learning_rate": 0.00045390170511534604, + "loss": 3.0121, + "theoretical_loss": 4.0946170384356, + "tokens_seen": 334248960 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004538916750250752, + "loss": 2.7814, + "theoretical_loss": 4.094524505042074, + "tokens_seen": 334314496 + }, + { + "epoch": 1.01, + "learning_rate": 0.00045388164493480445, + "loss": 2.9696, + "theoretical_loss": 4.094431994864082, + "tokens_seen": 334380032 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004538716148445336, + "loss": 2.9823, + "theoretical_loss": 4.094339507891252, + "tokens_seen": 334445568 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004538615847542628, + "loss": 3.209, + "theoretical_loss": 4.094247044113219, + "tokens_seen": 334511104 + }, + { + "epoch": 1.01, + "learning_rate": 0.000453851554663992, + "loss": 3.0497, + "theoretical_loss": 4.094154603519623, + "tokens_seen": 334576640 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004538415245737212, + "loss": 3.0694, + "theoretical_loss": 4.0940621861001105, + "tokens_seen": 334642176 + }, + { + "epoch": 1.01, + "learning_rate": 0.00045383149448345036, + "loss": 2.9195, + "theoretical_loss": 4.093969791844335, + "tokens_seen": 334707712 + }, + { + "epoch": 1.01, + "learning_rate": 0.00045382146439317954, + "loss": 2.8378, + "theoretical_loss": 4.093877420741958, + "tokens_seen": 334773248 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004538114343029087, + "loss": 3.0146, + "theoretical_loss": 4.093785072782646, + "tokens_seen": 334838784 + }, + { + "epoch": 1.01, + "learning_rate": 0.00045380140421263796, + "loss": 3.0526, + "theoretical_loss": 4.093692747956072, + "tokens_seen": 334904320 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004537913741223671, + "loss": 3.1662, + "theoretical_loss": 4.0936004462519175, + "tokens_seen": 334969856 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004537813440320963, + "loss": 2.9986, + "theoretical_loss": 4.093508167659869, + "tokens_seen": 335035392 + }, + { + "epoch": 1.01, + "learning_rate": 0.00045377131394182545, + "loss": 3.1044, + "theoretical_loss": 4.09341591216962, + "tokens_seen": 335100928 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004537612838515547, + "loss": 2.893, + "theoretical_loss": 4.0933236797708705, + "tokens_seen": 335166464 + }, + { + "epoch": 1.01, + "learning_rate": 0.00045375125376128386, + "loss": 2.9955, + "theoretical_loss": 4.093231470453327, + "tokens_seen": 335232000 + }, + { + "epoch": 1.01, + "learning_rate": 0.00045374122367101304, + "loss": 3.1863, + "theoretical_loss": 4.093139284206703, + "tokens_seen": 335297536 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004537311935807422, + "loss": 3.2446, + "theoretical_loss": 4.09304712102072, + "tokens_seen": 335363072 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004537211634904714, + "loss": 3.3968, + "theoretical_loss": 4.092954980885102, + "tokens_seen": 335428608 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004537111334002006, + "loss": 3.0159, + "theoretical_loss": 4.092862863789584, + "tokens_seen": 335494144 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004537011033099298, + "loss": 3.1544, + "theoretical_loss": 4.092770769723905, + "tokens_seen": 335559680 + }, + { + "epoch": 1.01, + "learning_rate": 0.00045369107321965895, + "loss": 3.0962, + "theoretical_loss": 4.092678698677811, + "tokens_seen": 335625216 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004536810431293882, + "loss": 3.0672, + "theoretical_loss": 4.0925866506410555, + "tokens_seen": 335690752 + }, + { + "epoch": 1.01, + "learning_rate": 0.00045367101303911737, + "loss": 3.2044, + "theoretical_loss": 4.092494625603397, + "tokens_seen": 335756288 + }, + { + "epoch": 1.01, + "learning_rate": 0.00045366098294884655, + "loss": 2.981, + "theoretical_loss": 4.0924026235546025, + "tokens_seen": 335821824 + }, + { + "epoch": 1.01, + "objective/train/docs_used": 460945, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6775364875793457, + "objective/train/theoretical_loss": 4.092333637098296, + "objective/train/tokens_used": 356330976, + "theoretical_loss": 4.092333637098296, + "tokens_seen": 335870976 + }, + { + "epoch": 1.01, + "learning_rate": 0.00045365095285857573, + "loss": 2.8384, + "theoretical_loss": 4.092310644484444, + "tokens_seen": 335887360 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004536409227683049, + "loss": 2.7994, + "theoretical_loss": 4.092218688382701, + "tokens_seen": 335952896 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004536308926780341, + "loss": 2.7465, + "theoretical_loss": 4.092126755239159, + "tokens_seen": 336018432 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004536208625877633, + "loss": 3.0426, + "theoretical_loss": 4.092034845043608, + "tokens_seen": 336083968 + }, + { + "epoch": 1.01, + "learning_rate": 0.00045361083249749245, + "loss": 3.0685, + "theoretical_loss": 4.09194295778585, + "tokens_seen": 336149504 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004536008024072217, + "loss": 2.9471, + "theoretical_loss": 4.091851093455689, + "tokens_seen": 336215040 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004535907723169508, + "loss": 2.9211, + "theoretical_loss": 4.091759252042936, + "tokens_seen": 336280576 + }, + { + "epoch": 1.01, + "learning_rate": 0.00045358074222668005, + "loss": 2.8346, + "theoretical_loss": 4.091667433537408, + "tokens_seen": 336346112 + }, + { + "epoch": 1.01, + "learning_rate": 0.00045357071213640923, + "loss": 2.8476, + "theoretical_loss": 4.0915756379289325, + "tokens_seen": 336411648 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004535606820461384, + "loss": 2.9507, + "theoretical_loss": 4.09148386520734, + "tokens_seen": 336477184 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004535506519558676, + "loss": 3.0561, + "theoretical_loss": 4.091392115362467, + "tokens_seen": 336542720 + }, + { + "epoch": 1.01, + "learning_rate": 0.00045354062186559683, + "loss": 2.7804, + "theoretical_loss": 4.091300388384158, + "tokens_seen": 336608256 + }, + { + "epoch": 1.01, + "learning_rate": 0.00045353059177532596, + "loss": 3.1013, + "theoretical_loss": 4.091208684262265, + "tokens_seen": 336673792 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004535205616850552, + "loss": 3.2401, + "theoretical_loss": 4.0911170029866435, + "tokens_seen": 336739328 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004535105315947843, + "loss": 3.0821, + "theoretical_loss": 4.091025344547158, + "tokens_seen": 336804864 + }, + { + "epoch": 1.01, + "learning_rate": 0.00045350050150451355, + "loss": 3.2724, + "theoretical_loss": 4.090933708933679, + "tokens_seen": 336870400 + }, + { + "epoch": 1.01, + "learning_rate": 0.00045349047141424274, + "loss": 3.2233, + "theoretical_loss": 4.0908420961360825, + "tokens_seen": 336935936 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004534804413239719, + "loss": 2.8302, + "theoretical_loss": 4.090750506144251, + "tokens_seen": 337001472 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004534704112337011, + "loss": 2.9115, + "theoretical_loss": 4.090658938948074, + "tokens_seen": 337067008 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004534603811434303, + "loss": 3.08, + "theoretical_loss": 4.090567394537449, + "tokens_seen": 337132544 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004534503510531595, + "loss": 2.9024, + "theoretical_loss": 4.090475872902277, + "tokens_seen": 337198080 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004534403209628887, + "loss": 3.1429, + "theoretical_loss": 4.0903843740324675, + "tokens_seen": 337263616 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004534302908726179, + "loss": 2.8771, + "theoretical_loss": 4.090292897917935, + "tokens_seen": 337329152 + }, + { + "epoch": 1.01, + "learning_rate": 0.00045342026078234706, + "loss": 3.1249, + "theoretical_loss": 4.090201444548601, + "tokens_seen": 337394688 + }, + { + "epoch": 1.01, + "learning_rate": 0.00045341023069207624, + "loss": 3.1044, + "theoretical_loss": 4.090110013914395, + "tokens_seen": 337460224 + }, + { + "epoch": 1.01, + "objective/train/docs_used": 462430, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.748074531555176, + "objective/train/theoretical_loss": 4.090041455852612, + "objective/train/tokens_used": 357969376, + "theoretical_loss": 4.090041455852612, + "tokens_seen": 337509376 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004534002006018054, + "loss": 2.9316, + "theoretical_loss": 4.09001860600525, + "tokens_seen": 337525760 + }, + { + "epoch": 1.01, + "learning_rate": 0.00045339017051153465, + "loss": 2.849, + "theoretical_loss": 4.0899272208111075, + "tokens_seen": 337591296 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004533801404212638, + "loss": 3.1926, + "theoretical_loss": 4.089835858321916, + "tokens_seen": 337656832 + }, + { + "epoch": 1.01, + "learning_rate": 0.000453370110330993, + "loss": 3.0211, + "theoretical_loss": 4.089744518527627, + "tokens_seen": 337722368 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004533600802407222, + "loss": 2.9728, + "theoretical_loss": 4.089653201418201, + "tokens_seen": 337787904 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004533500501504514, + "loss": 3.1214, + "theoretical_loss": 4.089561906983606, + "tokens_seen": 337853440 + }, + { + "epoch": 1.01, + "learning_rate": 0.00045334002006018056, + "loss": 3.0502, + "theoretical_loss": 4.089470635213814, + "tokens_seen": 337918976 + }, + { + "epoch": 1.01, + "learning_rate": 0.00045332998996990974, + "loss": 3.1409, + "theoretical_loss": 4.089379386098804, + "tokens_seen": 337984512 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004533199598796389, + "loss": 2.8961, + "theoretical_loss": 4.089288159628562, + "tokens_seen": 338050048 + }, + { + "epoch": 1.01, + "learning_rate": 0.00045330992978936816, + "loss": 2.851, + "theoretical_loss": 4.08919695579308, + "tokens_seen": 338115584 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004532998996990973, + "loss": 2.8503, + "theoretical_loss": 4.089105774582356, + "tokens_seen": 338181120 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004532898696088265, + "loss": 3.0414, + "theoretical_loss": 4.089014615986394, + "tokens_seen": 338246656 + }, + { + "epoch": 1.01, + "learning_rate": 0.00045327983951855565, + "loss": 2.8973, + "theoretical_loss": 4.0889234799952066, + "tokens_seen": 338312192 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004532698094282849, + "loss": 2.8654, + "theoretical_loss": 4.088832366598811, + "tokens_seen": 338377728 + }, + { + "epoch": 1.01, + "learning_rate": 0.00045325977933801406, + "loss": 2.9145, + "theoretical_loss": 4.08874127578723, + "tokens_seen": 338443264 + }, + { + "epoch": 1.01, + "learning_rate": 0.00045324974924774324, + "loss": 2.9358, + "theoretical_loss": 4.0886502075504945, + "tokens_seen": 338508800 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004532397191574724, + "loss": 3.1151, + "theoretical_loss": 4.08855916187864, + "tokens_seen": 338574336 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004532296890672016, + "loss": 3.2944, + "theoretical_loss": 4.08846813876171, + "tokens_seen": 338639872 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004532196589769308, + "loss": 2.8268, + "theoretical_loss": 4.088377138189754, + "tokens_seen": 338705408 + }, + { + "epoch": 1.01, + "learning_rate": 0.00045320962888666, + "loss": 3.0523, + "theoretical_loss": 4.088286160152827, + "tokens_seen": 338770944 + }, + { + "epoch": 1.01, + "learning_rate": 0.00045319959879638915, + "loss": 2.8429, + "theoretical_loss": 4.088195204640989, + "tokens_seen": 338836480 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004531895687061184, + "loss": 2.8917, + "theoretical_loss": 4.088104271644311, + "tokens_seen": 338902016 + }, + { + "epoch": 1.01, + "learning_rate": 0.00045317953861584757, + "loss": 3.2186, + "theoretical_loss": 4.088013361152865, + "tokens_seen": 338967552 + }, + { + "epoch": 1.01, + "learning_rate": 0.00045316950852557675, + "loss": 2.8986, + "theoretical_loss": 4.087922473156732, + "tokens_seen": 339033088 + }, + { + "epoch": 1.01, + "learning_rate": 0.00045315947843530593, + "loss": 3.0517, + "theoretical_loss": 4.087831607646, + "tokens_seen": 339098624 + }, + { + "epoch": 1.01, + "objective/train/docs_used": 463251, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.839505434036255, + "objective/train/theoretical_loss": 4.087763473263035, + "objective/train/tokens_used": 359607776, + "theoretical_loss": 4.087763473263035, + "tokens_seen": 339147776 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004531494483450351, + "loss": 2.941, + "theoretical_loss": 4.087740764610761, + "tokens_seen": 339164160 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004531394182547643, + "loss": 2.9092, + "theoretical_loss": 4.0876499440411145, + "tokens_seen": 339229696 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004531293881644935, + "loss": 2.8542, + "theoretical_loss": 4.087559145927166, + "tokens_seen": 339295232 + }, + { + "epoch": 1.01, + "learning_rate": 0.00045311935807422265, + "loss": 3.1179, + "theoretical_loss": 4.087468370259028, + "tokens_seen": 339360768 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004531093279839519, + "loss": 2.9362, + "theoretical_loss": 4.087377617026819, + "tokens_seen": 339426304 + }, + { + "epoch": 1.01, + "learning_rate": 0.000453099297893681, + "loss": 2.8019, + "theoretical_loss": 4.087286886220663, + "tokens_seen": 339491840 + }, + { + "epoch": 1.01, + "learning_rate": 0.00045308926780341025, + "loss": 3.1268, + "theoretical_loss": 4.087196177830691, + "tokens_seen": 339557376 + }, + { + "epoch": 1.01, + "learning_rate": 0.00045307923771313943, + "loss": 2.732, + "theoretical_loss": 4.08710549184704, + "tokens_seen": 339622912 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004530692076228686, + "loss": 3.1866, + "theoretical_loss": 4.087014828259853, + "tokens_seen": 339688448 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004530591775325978, + "loss": 2.9075, + "theoretical_loss": 4.086924187059279, + "tokens_seen": 339753984 + }, + { + "epoch": 1.01, + "learning_rate": 0.00045304914744232703, + "loss": 3.0424, + "theoretical_loss": 4.086833568235474, + "tokens_seen": 339819520 + }, + { + "epoch": 1.01, + "learning_rate": 0.00045303911735205616, + "loss": 3.0533, + "theoretical_loss": 4.086742971778601, + "tokens_seen": 339885056 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004530290872617854, + "loss": 3.0583, + "theoretical_loss": 4.086652397678827, + "tokens_seen": 339950592 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004530190571715145, + "loss": 2.9185, + "theoretical_loss": 4.086561845926326, + "tokens_seen": 340016128 + }, + { + "epoch": 1.01, + "learning_rate": 0.00045300902708124375, + "loss": 2.9725, + "theoretical_loss": 4.086471316511281, + "tokens_seen": 340081664 + }, + { + "epoch": 1.01, + "learning_rate": 0.00045299899699097294, + "loss": 2.917, + "theoretical_loss": 4.086380809423876, + "tokens_seen": 340147200 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004529889669007021, + "loss": 3.1826, + "theoretical_loss": 4.086290324654303, + "tokens_seen": 340212736 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004529789368104313, + "loss": 2.8471, + "theoretical_loss": 4.086199862192766, + "tokens_seen": 340278272 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004529689067201605, + "loss": 3.1347, + "theoretical_loss": 4.086109422029466, + "tokens_seen": 340343808 + }, + { + "epoch": 1.01, + "learning_rate": 0.00045295887662988966, + "loss": 2.7183, + "theoretical_loss": 4.0860190041546165, + "tokens_seen": 340409344 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004529488465396189, + "loss": 2.9848, + "theoretical_loss": 4.085928608558435, + "tokens_seen": 340474880 + }, + { + "epoch": 1.01, + "learning_rate": 0.000452938816449348, + "loss": 3.2565, + "theoretical_loss": 4.085838235231145, + "tokens_seen": 340540416 + }, + { + "epoch": 1.01, + "learning_rate": 0.00045292878635907726, + "loss": 3.0739, + "theoretical_loss": 4.085747884162976, + "tokens_seen": 340605952 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004529187562688064, + "loss": 2.9197, + "theoretical_loss": 4.0856575553441665, + "tokens_seen": 340671488 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004529087261785356, + "loss": 3.154, + "theoretical_loss": 4.085567248764956, + "tokens_seen": 340737024 + }, + { + "epoch": 1.01, + "objective/train/docs_used": 464753, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.337740182876587, + "objective/train/theoretical_loss": 4.08549953341942, + "objective/train/tokens_used": 361246176, + "theoretical_loss": 4.08549953341942, + "tokens_seen": 340786176 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004528986960882648, + "loss": 3.2774, + "theoretical_loss": 4.085476964415595, + "tokens_seen": 340802560 + }, + { + "epoch": 1.01, + "learning_rate": 0.000452888665997994, + "loss": 3.0845, + "theoretical_loss": 4.085386702286338, + "tokens_seen": 340868096 + }, + { + "epoch": 1.01, + "learning_rate": 0.00045287863590772316, + "loss": 2.9111, + "theoretical_loss": 4.085296462367445, + "tokens_seen": 340933632 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004528686058174524, + "loss": 3.3461, + "theoretical_loss": 4.085206244649184, + "tokens_seen": 340999168 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004528585757271815, + "loss": 2.8499, + "theoretical_loss": 4.085116049121828, + "tokens_seen": 341064704 + }, + { + "epoch": 1.01, + "learning_rate": 0.00045284854563691076, + "loss": 3.4213, + "theoretical_loss": 4.085025875775655, + "tokens_seen": 341130240 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004528385155466399, + "loss": 2.9534, + "theoretical_loss": 4.0849357246009514, + "tokens_seen": 341195776 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004528284854563691, + "loss": 3.0351, + "theoretical_loss": 4.084845595588009, + "tokens_seen": 341261312 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004528184553660983, + "loss": 3.1023, + "theoretical_loss": 4.084755488727124, + "tokens_seen": 341326848 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004528084252758275, + "loss": 2.9196, + "theoretical_loss": 4.084665404008602, + "tokens_seen": 341392384 + }, + { + "epoch": 1.01, + "learning_rate": 0.00045279839518555667, + "loss": 2.8665, + "theoretical_loss": 4.084575341422752, + "tokens_seen": 341457920 + }, + { + "epoch": 1.01, + "learning_rate": 0.00045278836509528585, + "loss": 3.187, + "theoretical_loss": 4.084485300959889, + "tokens_seen": 341523456 + }, + { + "epoch": 1.01, + "learning_rate": 0.00045277833500501503, + "loss": 3.1001, + "theoretical_loss": 4.084395282610337, + "tokens_seen": 341588992 + }, + { + "epoch": 1.01, + "learning_rate": 0.00045276830491474426, + "loss": 3.0675, + "theoretical_loss": 4.0843052863644225, + "tokens_seen": 341654528 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004527582748244734, + "loss": 3.1473, + "theoretical_loss": 4.08421531221248, + "tokens_seen": 341720064 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004527482447342026, + "loss": 3.0618, + "theoretical_loss": 4.084125360144849, + "tokens_seen": 341785600 + }, + { + "epoch": 1.01, + "learning_rate": 0.00045273821464393175, + "loss": 3.083, + "theoretical_loss": 4.084035430151879, + "tokens_seen": 341851136 + }, + { + "epoch": 1.01, + "learning_rate": 0.000452728184553661, + "loss": 2.9518, + "theoretical_loss": 4.083945522223919, + "tokens_seen": 341916672 + }, + { + "epoch": 1.01, + "learning_rate": 0.00045271815446339017, + "loss": 3.136, + "theoretical_loss": 4.083855636351329, + "tokens_seen": 341982208 + }, + { + "epoch": 1.01, + "learning_rate": 0.00045270812437311935, + "loss": 2.9707, + "theoretical_loss": 4.0837657725244725, + "tokens_seen": 342047744 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004526980942828486, + "loss": 2.9953, + "theoretical_loss": 4.083675930733721, + "tokens_seen": 342113280 + }, + { + "epoch": 1.01, + "learning_rate": 0.00045268806419257777, + "loss": 2.6718, + "theoretical_loss": 4.083586110969451, + "tokens_seen": 342178816 + }, + { + "epoch": 1.01, + "learning_rate": 0.00045267803410230695, + "loss": 3.0025, + "theoretical_loss": 4.083496313222046, + "tokens_seen": 342244352 + }, + { + "epoch": 1.01, + "learning_rate": 0.00045266800401203613, + "loss": 3.0224, + "theoretical_loss": 4.083406537481893, + "tokens_seen": 342309888 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004526579739217653, + "loss": 2.7411, + "theoretical_loss": 4.083316783739388, + "tokens_seen": 342375424 + }, + { + "epoch": 1.01, + "objective/train/docs_used": 465448, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.2629733085632324, + "objective/train/theoretical_loss": 4.083249482862691, + "objective/train/tokens_used": 362884576, + "theoretical_loss": 4.083249482862691, + "tokens_seen": 342424576 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004526479438314945, + "loss": 2.895, + "theoretical_loss": 4.083227051984932, + "tokens_seen": 342440960 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004526379137412237, + "loss": 2.8273, + "theoretical_loss": 4.08313734220893, + "tokens_seen": 342506496 + }, + { + "epoch": 1.01, + "learning_rate": 0.00045262788365095285, + "loss": 2.9571, + "theoretical_loss": 4.083047654401797, + "tokens_seen": 342572032 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004526178535606821, + "loss": 2.6863, + "theoretical_loss": 4.082957988553951, + "tokens_seen": 342637568 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004526078234704112, + "loss": 2.9279, + "theoretical_loss": 4.082868344655816, + "tokens_seen": 342703104 + }, + { + "epoch": 1.01, + "learning_rate": 0.00045259779338014045, + "loss": 3.321, + "theoretical_loss": 4.082778722697825, + "tokens_seen": 342768640 + }, + { + "epoch": 1.01, + "learning_rate": 0.00045258776328986963, + "loss": 3.1226, + "theoretical_loss": 4.082689122670413, + "tokens_seen": 342834176 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004525777331995988, + "loss": 3.1412, + "theoretical_loss": 4.082599544564024, + "tokens_seen": 342899712 + }, + { + "epoch": 1.01, + "learning_rate": 0.000452567703109328, + "loss": 2.9624, + "theoretical_loss": 4.082509988369106, + "tokens_seen": 342965248 + }, + { + "epoch": 1.01, + "learning_rate": 0.00045255767301905723, + "loss": 3.0756, + "theoretical_loss": 4.0824204540761135, + "tokens_seen": 343030784 + }, + { + "epoch": 1.01, + "learning_rate": 0.00045254764292878636, + "loss": 2.9775, + "theoretical_loss": 4.082330941675508, + "tokens_seen": 343096320 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004525376128385156, + "loss": 2.9741, + "theoretical_loss": 4.082241451157757, + "tokens_seen": 343161856 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004525275827482447, + "loss": 3.0864, + "theoretical_loss": 4.0821519825133326, + "tokens_seen": 343227392 + }, + { + "epoch": 1.01, + "learning_rate": 0.00045251755265797395, + "loss": 3.3031, + "theoretical_loss": 4.082062535732713, + "tokens_seen": 343292928 + }, + { + "epoch": 1.01, + "learning_rate": 0.00045250752256770314, + "loss": 2.8638, + "theoretical_loss": 4.081973110806383, + "tokens_seen": 343358464 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004524974924774323, + "loss": 3.0286, + "theoretical_loss": 4.081883707724835, + "tokens_seen": 343424000 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004524874623871615, + "loss": 2.8336, + "theoretical_loss": 4.081794326478563, + "tokens_seen": 343489536 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004524774322968907, + "loss": 3.2582, + "theoretical_loss": 4.081704967058071, + "tokens_seen": 343555072 + }, + { + "epoch": 1.01, + "learning_rate": 0.00045246740220661986, + "loss": 3.1308, + "theoretical_loss": 4.081615629453868, + "tokens_seen": 343620608 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004524573721163491, + "loss": 3.0641, + "theoretical_loss": 4.081526313656466, + "tokens_seen": 343686144 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004524473420260782, + "loss": 3.0581, + "theoretical_loss": 4.081437019656389, + "tokens_seen": 343751680 + }, + { + "epoch": 1.01, + "learning_rate": 0.00045243731193580746, + "loss": 3.0062, + "theoretical_loss": 4.081347747444161, + "tokens_seen": 343817216 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004524272818455366, + "loss": 2.7659, + "theoretical_loss": 4.081258497010314, + "tokens_seen": 343882752 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004524172517552658, + "loss": 3.0713, + "theoretical_loss": 4.081169268345387, + "tokens_seen": 343948288 + }, + { + "epoch": 1.01, + "learning_rate": 0.000452407221664995, + "loss": 3.0026, + "theoretical_loss": 4.081080061439923, + "tokens_seen": 344013824 + }, + { + "epoch": 1.01, + "objective/train/docs_used": 466745, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.469646692276001, + "objective/train/theoretical_loss": 4.0810131705347885, + "objective/train/tokens_used": 364522976, + "theoretical_loss": 4.0810131705347885, + "tokens_seen": 344062976 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004523971915747242, + "loss": 3.0142, + "theoretical_loss": 4.080990876284473, + "tokens_seen": 344079360 + }, + { + "epoch": 1.01, + "learning_rate": 0.00045238716148445336, + "loss": 3.1162, + "theoretical_loss": 4.0809017128695935, + "tokens_seen": 344144896 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004523771313941826, + "loss": 3.3286, + "theoretical_loss": 4.080812571185845, + "tokens_seen": 344210432 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004523671013039117, + "loss": 2.7948, + "theoretical_loss": 4.080723451223795, + "tokens_seen": 344275968 + }, + { + "epoch": 1.01, + "learning_rate": 0.00045235707121364096, + "loss": 3.0005, + "theoretical_loss": 4.080634352974018, + "tokens_seen": 344341504 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004523470411233701, + "loss": 3.055, + "theoretical_loss": 4.080545276427092, + "tokens_seen": 344407040 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004523370110330993, + "loss": 3.0788, + "theoretical_loss": 4.080456221573604, + "tokens_seen": 344472576 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004523269809428285, + "loss": 2.9626, + "theoretical_loss": 4.080367188404144, + "tokens_seen": 344538112 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004523169508525577, + "loss": 2.8527, + "theoretical_loss": 4.080278176909309, + "tokens_seen": 344603648 + }, + { + "epoch": 1.01, + "learning_rate": 0.00045230692076228687, + "loss": 3.0898, + "theoretical_loss": 4.080189187079703, + "tokens_seen": 344669184 + }, + { + "epoch": 1.01, + "learning_rate": 0.00045229689067201605, + "loss": 3.1361, + "theoretical_loss": 4.080100218905933, + "tokens_seen": 344734720 + }, + { + "epoch": 1.01, + "learning_rate": 0.00045228686058174523, + "loss": 3.1205, + "theoretical_loss": 4.080011272378616, + "tokens_seen": 344800256 + }, + { + "epoch": 1.01, + "learning_rate": 0.00045227683049147446, + "loss": 3.0475, + "theoretical_loss": 4.07992234748837, + "tokens_seen": 344865792 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004522668004012036, + "loss": 2.9471, + "theoretical_loss": 4.079833444225824, + "tokens_seen": 344931328 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004522567703109328, + "loss": 3.1468, + "theoretical_loss": 4.079744562581608, + "tokens_seen": 344996864 + }, + { + "epoch": 1.01, + "learning_rate": 0.00045224674022066195, + "loss": 3.1542, + "theoretical_loss": 4.0796557025463605, + "tokens_seen": 345062400 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004522367101303912, + "loss": 2.932, + "theoretical_loss": 4.079566864110725, + "tokens_seen": 345127936 + }, + { + "epoch": 1.01, + "learning_rate": 0.00045222668004012037, + "loss": 2.7915, + "theoretical_loss": 4.0794780472653525, + "tokens_seen": 345193472 + }, + { + "epoch": 1.01, + "learning_rate": 0.00045221664994984955, + "loss": 3.0188, + "theoretical_loss": 4.0793892520008965, + "tokens_seen": 345259008 + }, + { + "epoch": 1.01, + "learning_rate": 0.00045220661985957873, + "loss": 3.2555, + "theoretical_loss": 4.07930047830802, + "tokens_seen": 345324544 + }, + { + "epoch": 1.01, + "learning_rate": 0.00045219658976930797, + "loss": 2.9997, + "theoretical_loss": 4.079211726177389, + "tokens_seen": 345390080 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004521865596790371, + "loss": 2.8722, + "theoretical_loss": 4.079122995599677, + "tokens_seen": 345455616 + }, + { + "epoch": 1.01, + "learning_rate": 0.00045217652958876633, + "loss": 2.816, + "theoretical_loss": 4.079034286565563, + "tokens_seen": 345521152 + }, + { + "epoch": 1.01, + "learning_rate": 0.00045216649949849546, + "loss": 3.0846, + "theoretical_loss": 4.078945599065731, + "tokens_seen": 345586688 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004521564694082247, + "loss": 3.057, + "theoretical_loss": 4.078856933090871, + "tokens_seen": 345652224 + }, + { + "epoch": 1.01, + "objective/train/docs_used": 467437, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.132908344268799, + "objective/train/theoretical_loss": 4.078790447729892, + "objective/train/tokens_used": 366161376, + "theoretical_loss": 4.078790447729892, + "tokens_seen": 345701376 + }, + { + "epoch": 1.01, + "learning_rate": 0.00045214643931795387, + "loss": 3.0499, + "theoretical_loss": 4.07876828863168, + "tokens_seen": 345717760 + }, + { + "epoch": 1.01, + "learning_rate": 0.00045213640922768305, + "loss": 2.9937, + "theoretical_loss": 4.078679665678859, + "tokens_seen": 345783296 + }, + { + "epoch": 1.01, + "learning_rate": 0.00045212637913741223, + "loss": 3.0931, + "theoretical_loss": 4.078591064223116, + "tokens_seen": 345848832 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004521163490471414, + "loss": 2.9227, + "theoretical_loss": 4.078502484255164, + "tokens_seen": 345914368 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004521063189568706, + "loss": 2.871, + "theoretical_loss": 4.078413925765724, + "tokens_seen": 345979904 + }, + { + "epoch": 1.01, + "learning_rate": 0.00045209628886659983, + "loss": 3.1182, + "theoretical_loss": 4.078325388745519, + "tokens_seen": 346045440 + }, + { + "epoch": 1.01, + "learning_rate": 0.00045208625877632896, + "loss": 3.0976, + "theoretical_loss": 4.07823687318528, + "tokens_seen": 346110976 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004520762286860582, + "loss": 2.9415, + "theoretical_loss": 4.078148379075744, + "tokens_seen": 346176512 + }, + { + "epoch": 1.01, + "learning_rate": 0.0004520661985957873, + "loss": 2.9781, + "theoretical_loss": 4.078059906407653, + "tokens_seen": 346242048 + }, + { + "epoch": 1.02, + "learning_rate": 0.00045205616850551656, + "loss": 2.825, + "theoretical_loss": 4.077971455171755, + "tokens_seen": 346307584 + }, + { + "epoch": 1.02, + "learning_rate": 0.00045204613841524574, + "loss": 2.8616, + "theoretical_loss": 4.077883025358804, + "tokens_seen": 346373120 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004520361083249749, + "loss": 2.8784, + "theoretical_loss": 4.07779461695956, + "tokens_seen": 346438656 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004520260782347041, + "loss": 2.9527, + "theoretical_loss": 4.077706229964786, + "tokens_seen": 346504192 + }, + { + "epoch": 1.02, + "learning_rate": 0.00045201604814443334, + "loss": 3.3518, + "theoretical_loss": 4.077617864365255, + "tokens_seen": 346569728 + }, + { + "epoch": 1.02, + "learning_rate": 0.00045200601805416246, + "loss": 2.7224, + "theoretical_loss": 4.077529520151743, + "tokens_seen": 346635264 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004519959879638917, + "loss": 3.0247, + "theoretical_loss": 4.077441197315032, + "tokens_seen": 346700800 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004519859578736208, + "loss": 3.023, + "theoretical_loss": 4.0773528958459115, + "tokens_seen": 346766336 + }, + { + "epoch": 1.02, + "learning_rate": 0.00045197592778335006, + "loss": 2.9417, + "theoretical_loss": 4.0772646157351735, + "tokens_seen": 346831872 + }, + { + "epoch": 1.02, + "learning_rate": 0.00045196589769307924, + "loss": 2.6767, + "theoretical_loss": 4.077176356973618, + "tokens_seen": 346897408 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004519558676028084, + "loss": 3.0087, + "theoretical_loss": 4.077088119552052, + "tokens_seen": 346962944 + }, + { + "epoch": 1.02, + "learning_rate": 0.00045194583751253766, + "loss": 3.1533, + "theoretical_loss": 4.076999903461283, + "tokens_seen": 347028480 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004519358074222668, + "loss": 2.7099, + "theoretical_loss": 4.07691170869213, + "tokens_seen": 347094016 + }, + { + "epoch": 1.02, + "learning_rate": 0.000451925777331996, + "loss": 2.8064, + "theoretical_loss": 4.076823535235415, + "tokens_seen": 347159552 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004519157472417252, + "loss": 3.2354, + "theoretical_loss": 4.0767353830819655, + "tokens_seen": 347225088 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004519057171514544, + "loss": 2.6779, + "theoretical_loss": 4.076647252222616, + "tokens_seen": 347290624 + }, + { + "epoch": 1.02, + "objective/train/docs_used": 469057, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.269904375076294, + "objective/train/theoretical_loss": 4.076581168046844, + "objective/train/tokens_used": 367799776, + "theoretical_loss": 4.076581168046844, + "tokens_seen": 347339776 + }, + { + "epoch": 1.02, + "learning_rate": 0.00045189568706118356, + "loss": 3.1138, + "theoretical_loss": 4.076559142648204, + "tokens_seen": 347356160 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004518856569709128, + "loss": 2.8085, + "theoretical_loss": 4.076471054349575, + "tokens_seen": 347421696 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004518756268806419, + "loss": 3.0591, + "theoretical_loss": 4.076382987317581, + "tokens_seen": 347487232 + }, + { + "epoch": 1.02, + "learning_rate": 0.00045186559679037116, + "loss": 3.0139, + "theoretical_loss": 4.076294941543078, + "tokens_seen": 347552768 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004518555667001003, + "loss": 3.0779, + "theoretical_loss": 4.076206917016927, + "tokens_seen": 347618304 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004518455366098295, + "loss": 2.8424, + "theoretical_loss": 4.076118913729996, + "tokens_seen": 347683840 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004518355065195587, + "loss": 2.9054, + "theoretical_loss": 4.0760309316731576, + "tokens_seen": 347749376 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004518254764292879, + "loss": 3.2079, + "theoretical_loss": 4.075942970837292, + "tokens_seen": 347814912 + }, + { + "epoch": 1.02, + "learning_rate": 0.00045181544633901707, + "loss": 2.9092, + "theoretical_loss": 4.075855031213283, + "tokens_seen": 347880448 + }, + { + "epoch": 1.02, + "learning_rate": 0.00045180541624874625, + "loss": 3.1271, + "theoretical_loss": 4.075767112792021, + "tokens_seen": 347945984 + }, + { + "epoch": 1.02, + "learning_rate": 0.00045179538615847543, + "loss": 2.9086, + "theoretical_loss": 4.075679215564401, + "tokens_seen": 348011520 + }, + { + "epoch": 1.02, + "learning_rate": 0.00045178535606820466, + "loss": 2.8916, + "theoretical_loss": 4.075591339521326, + "tokens_seen": 348077056 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004517753259779338, + "loss": 3.1885, + "theoretical_loss": 4.0755034846537015, + "tokens_seen": 348142592 + }, + { + "epoch": 1.02, + "learning_rate": 0.000451765295887663, + "loss": 2.7859, + "theoretical_loss": 4.07541565095244, + "tokens_seen": 348208128 + }, + { + "epoch": 1.02, + "learning_rate": 0.00045175526579739215, + "loss": 3.1342, + "theoretical_loss": 4.0753278384084615, + "tokens_seen": 348273664 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004517452357071214, + "loss": 3.0243, + "theoretical_loss": 4.075240047012688, + "tokens_seen": 348339200 + }, + { + "epoch": 1.02, + "learning_rate": 0.00045173520561685057, + "loss": 2.9966, + "theoretical_loss": 4.07515227675605, + "tokens_seen": 348404736 + }, + { + "epoch": 1.02, + "learning_rate": 0.00045172517552657975, + "loss": 3.2115, + "theoretical_loss": 4.075064527629483, + "tokens_seen": 348470272 + }, + { + "epoch": 1.02, + "learning_rate": 0.00045171514543630893, + "loss": 3.2221, + "theoretical_loss": 4.074976799623926, + "tokens_seen": 348535808 + }, + { + "epoch": 1.02, + "learning_rate": 0.00045170511534603817, + "loss": 2.8883, + "theoretical_loss": 4.074889092730325, + "tokens_seen": 348601344 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004516950852557673, + "loss": 3.1443, + "theoretical_loss": 4.074801406939635, + "tokens_seen": 348666880 + }, + { + "epoch": 1.02, + "learning_rate": 0.00045168505516549653, + "loss": 3.1144, + "theoretical_loss": 4.07471374224281, + "tokens_seen": 348732416 + }, + { + "epoch": 1.02, + "learning_rate": 0.00045167502507522566, + "loss": 2.9944, + "theoretical_loss": 4.074626098630815, + "tokens_seen": 348797952 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004516649949849549, + "loss": 2.8234, + "theoretical_loss": 4.074538476094617, + "tokens_seen": 348863488 + }, + { + "epoch": 1.02, + "learning_rate": 0.00045165496489468407, + "loss": 2.834, + "theoretical_loss": 4.0744508746251915, + "tokens_seen": 348929024 + }, + { + "epoch": 1.02, + "objective/train/docs_used": 469778, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.0074195861816406, + "objective/train/theoretical_loss": 4.074385187342765, + "objective/train/tokens_used": 369438176, + "theoretical_loss": 4.074385187342765, + "tokens_seen": 348978176 + }, + { + "epoch": 1.02, + "learning_rate": 0.00045164493480441325, + "loss": 3.1902, + "theoretical_loss": 4.0743632942135175, + "tokens_seen": 348994560 + }, + { + "epoch": 1.02, + "learning_rate": 0.00045163490471414244, + "loss": 3.0386, + "theoretical_loss": 4.07427573485058, + "tokens_seen": 349060096 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004516248746238716, + "loss": 2.8443, + "theoretical_loss": 4.07418819652737, + "tokens_seen": 349125632 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004516148445336008, + "loss": 2.8372, + "theoretical_loss": 4.074100679234883, + "tokens_seen": 349191168 + }, + { + "epoch": 1.02, + "learning_rate": 0.00045160481444333003, + "loss": 3.1183, + "theoretical_loss": 4.074013182964123, + "tokens_seen": 349256704 + }, + { + "epoch": 1.02, + "learning_rate": 0.00045159478435305916, + "loss": 3.104, + "theoretical_loss": 4.0739257077060955, + "tokens_seen": 349322240 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004515847542627884, + "loss": 3.0652, + "theoretical_loss": 4.073838253451814, + "tokens_seen": 349387776 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004515747241725175, + "loss": 3.0714, + "theoretical_loss": 4.073750820192296, + "tokens_seen": 349453312 + }, + { + "epoch": 1.02, + "learning_rate": 0.00045156469408224676, + "loss": 2.9575, + "theoretical_loss": 4.073663407918566, + "tokens_seen": 349518848 + }, + { + "epoch": 1.02, + "learning_rate": 0.00045155466399197594, + "loss": 3.0506, + "theoretical_loss": 4.073576016621656, + "tokens_seen": 349584384 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004515446339017051, + "loss": 2.8519, + "theoretical_loss": 4.073488646292597, + "tokens_seen": 349649920 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004515346038114343, + "loss": 2.8341, + "theoretical_loss": 4.073401296922432, + "tokens_seen": 349715456 + }, + { + "epoch": 1.02, + "learning_rate": 0.00045152457372116354, + "loss": 3.1848, + "theoretical_loss": 4.073313968502206, + "tokens_seen": 349780992 + }, + { + "epoch": 1.02, + "learning_rate": 0.00045151454363089266, + "loss": 2.9554, + "theoretical_loss": 4.0732266610229715, + "tokens_seen": 349846528 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004515045135406219, + "loss": 2.7127, + "theoretical_loss": 4.073139374475784, + "tokens_seen": 349912064 + }, + { + "epoch": 1.02, + "learning_rate": 0.000451494483450351, + "loss": 3.185, + "theoretical_loss": 4.073052108851707, + "tokens_seen": 349977600 + }, + { + "epoch": 1.02, + "learning_rate": 0.00045148445336008026, + "loss": 3.3413, + "theoretical_loss": 4.072964864141809, + "tokens_seen": 350043136 + }, + { + "epoch": 1.02, + "learning_rate": 0.00045147442326980944, + "loss": 3.0749, + "theoretical_loss": 4.072877640337162, + "tokens_seen": 350108672 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004514643931795386, + "loss": 3.016, + "theoretical_loss": 4.072790437428846, + "tokens_seen": 350174208 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004514543630892678, + "loss": 3.1049, + "theoretical_loss": 4.072703255407946, + "tokens_seen": 350239744 + }, + { + "epoch": 1.02, + "learning_rate": 0.000451444332998997, + "loss": 3.0196, + "theoretical_loss": 4.07261609426555, + "tokens_seen": 350305280 + }, + { + "epoch": 1.02, + "learning_rate": 0.00045143430290872617, + "loss": 2.9661, + "theoretical_loss": 4.072528953992756, + "tokens_seen": 350370816 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004514242728184554, + "loss": 2.7829, + "theoretical_loss": 4.072441834580663, + "tokens_seen": 350436352 + }, + { + "epoch": 1.02, + "learning_rate": 0.00045141424272818453, + "loss": 3.1238, + "theoretical_loss": 4.072354736020377, + "tokens_seen": 350501888 + }, + { + "epoch": 1.02, + "learning_rate": 0.00045140421263791376, + "loss": 3.0012, + "theoretical_loss": 4.07226765830301, + "tokens_seen": 350567424 + }, + { + "epoch": 1.02, + "objective/train/docs_used": 471047, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.1343419551849365, + "objective/train/theoretical_loss": 4.072202363687808, + "objective/train/tokens_used": 371076576, + "theoretical_loss": 4.072202363687808, + "tokens_seen": 350616576 + }, + { + "epoch": 1.02, + "learning_rate": 0.00045139418254764294, + "loss": 3.0134, + "theoretical_loss": 4.072180601419681, + "tokens_seen": 350632960 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004513841524573721, + "loss": 2.9707, + "theoretical_loss": 4.072093565361511, + "tokens_seen": 350698496 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004513741223671013, + "loss": 2.9995, + "theoretical_loss": 4.072006550119628, + "tokens_seen": 350764032 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004513640922768305, + "loss": 2.9866, + "theoretical_loss": 4.071919555685166, + "tokens_seen": 350829568 + }, + { + "epoch": 1.02, + "learning_rate": 0.00045135406218655967, + "loss": 2.9988, + "theoretical_loss": 4.071832582049264, + "tokens_seen": 350895104 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004513440320962889, + "loss": 2.6705, + "theoretical_loss": 4.071745629203066, + "tokens_seen": 350960640 + }, + { + "epoch": 1.02, + "learning_rate": 0.00045133400200601803, + "loss": 3.159, + "theoretical_loss": 4.071658697137722, + "tokens_seen": 351026176 + }, + { + "epoch": 1.02, + "learning_rate": 0.00045132397191574727, + "loss": 2.6714, + "theoretical_loss": 4.071571785844387, + "tokens_seen": 351091712 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004513139418254764, + "loss": 3.0984, + "theoretical_loss": 4.071484895314223, + "tokens_seen": 351157248 + }, + { + "epoch": 1.02, + "learning_rate": 0.00045130391173520563, + "loss": 3.1536, + "theoretical_loss": 4.071398025538394, + "tokens_seen": 351222784 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004512938816449348, + "loss": 2.8315, + "theoretical_loss": 4.071311176508073, + "tokens_seen": 351288320 + }, + { + "epoch": 1.02, + "learning_rate": 0.000451283851554664, + "loss": 3.047, + "theoretical_loss": 4.071224348214435, + "tokens_seen": 351353856 + }, + { + "epoch": 1.02, + "learning_rate": 0.00045127382146439317, + "loss": 3.2294, + "theoretical_loss": 4.071137540648665, + "tokens_seen": 351419392 + }, + { + "epoch": 1.02, + "learning_rate": 0.00045126379137412235, + "loss": 3.1709, + "theoretical_loss": 4.0710507538019485, + "tokens_seen": 351484928 + }, + { + "epoch": 1.02, + "learning_rate": 0.00045125376128385153, + "loss": 3.2218, + "theoretical_loss": 4.070963987665479, + "tokens_seen": 351550464 + }, + { + "epoch": 1.02, + "learning_rate": 0.00045124373119358077, + "loss": 2.9715, + "theoretical_loss": 4.0708772422304556, + "tokens_seen": 351616000 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004512337011033099, + "loss": 3.0497, + "theoretical_loss": 4.070790517488081, + "tokens_seen": 351681536 + }, + { + "epoch": 1.02, + "learning_rate": 0.00045122367101303913, + "loss": 2.9379, + "theoretical_loss": 4.070703813429566, + "tokens_seen": 351747072 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004512136409227683, + "loss": 2.9688, + "theoretical_loss": 4.070617130046124, + "tokens_seen": 351812608 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004512036108324975, + "loss": 3.1147, + "theoretical_loss": 4.070530467328975, + "tokens_seen": 351878144 + }, + { + "epoch": 1.02, + "learning_rate": 0.00045119358074222673, + "loss": 2.9876, + "theoretical_loss": 4.070443825269344, + "tokens_seen": 351943680 + }, + { + "epoch": 1.02, + "learning_rate": 0.00045118355065195586, + "loss": 3.2065, + "theoretical_loss": 4.070357203858462, + "tokens_seen": 352009216 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004511735205616851, + "loss": 3.0601, + "theoretical_loss": 4.070270603087565, + "tokens_seen": 352074752 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004511634904714143, + "loss": 3.2063, + "theoretical_loss": 4.0701840229478945, + "tokens_seen": 352140288 + }, + { + "epoch": 1.02, + "learning_rate": 0.00045115346038114345, + "loss": 2.9421, + "theoretical_loss": 4.070097463430697, + "tokens_seen": 352205824 + }, + { + "epoch": 1.02, + "objective/train/docs_used": 471658, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.1030523777008057, + "objective/train/theoretical_loss": 4.070032557321034, + "objective/train/tokens_used": 372714976, + "theoretical_loss": 4.070032557321034, + "tokens_seen": 352254976 + }, + { + "epoch": 1.02, + "learning_rate": 0.00045114343029087264, + "loss": 2.8468, + "theoretical_loss": 4.0700109245272245, + "tokens_seen": 352271360 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004511334002006018, + "loss": 3.0807, + "theoretical_loss": 4.0699244062287345, + "tokens_seen": 352336896 + }, + { + "epoch": 1.02, + "learning_rate": 0.000451123370110331, + "loss": 2.9094, + "theoretical_loss": 4.069837908526489, + "tokens_seen": 352402432 + }, + { + "epoch": 1.02, + "learning_rate": 0.00045111334002006023, + "loss": 3.0076, + "theoretical_loss": 4.069751431411758, + "tokens_seen": 352467968 + }, + { + "epoch": 1.02, + "learning_rate": 0.00045110330992978936, + "loss": 3.1799, + "theoretical_loss": 4.0696649748758125, + "tokens_seen": 352533504 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004510932798395186, + "loss": 3.1436, + "theoretical_loss": 4.069578538909933, + "tokens_seen": 352599040 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004510832497492477, + "loss": 2.9335, + "theoretical_loss": 4.069492123505402, + "tokens_seen": 352664576 + }, + { + "epoch": 1.02, + "learning_rate": 0.00045107321965897696, + "loss": 3.0457, + "theoretical_loss": 4.069405728653509, + "tokens_seen": 352730112 + }, + { + "epoch": 1.02, + "learning_rate": 0.00045106318956870614, + "loss": 3.0376, + "theoretical_loss": 4.0693193543455495, + "tokens_seen": 352795648 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004510531594784353, + "loss": 3.1604, + "theoretical_loss": 4.069233000572823, + "tokens_seen": 352861184 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004510431293881645, + "loss": 3.2364, + "theoretical_loss": 4.069146667326635, + "tokens_seen": 352926720 + }, + { + "epoch": 1.02, + "learning_rate": 0.00045103309929789374, + "loss": 2.9682, + "theoretical_loss": 4.069060354598296, + "tokens_seen": 352992256 + }, + { + "epoch": 1.02, + "learning_rate": 0.00045102306920762286, + "loss": 3.0206, + "theoretical_loss": 4.06897406237912, + "tokens_seen": 353057792 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004510130391173521, + "loss": 2.8432, + "theoretical_loss": 4.0688877906604315, + "tokens_seen": 353123328 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004510030090270812, + "loss": 2.9897, + "theoretical_loss": 4.068801539433554, + "tokens_seen": 353188864 + }, + { + "epoch": 1.02, + "learning_rate": 0.00045099297893681046, + "loss": 3.0886, + "theoretical_loss": 4.06871530868982, + "tokens_seen": 353254400 + }, + { + "epoch": 1.02, + "learning_rate": 0.00045098294884653964, + "loss": 3.1712, + "theoretical_loss": 4.068629098420567, + "tokens_seen": 353319936 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004509729187562688, + "loss": 2.9892, + "theoretical_loss": 4.068542908617136, + "tokens_seen": 353385472 + }, + { + "epoch": 1.02, + "learning_rate": 0.000450962888665998, + "loss": 3.1108, + "theoretical_loss": 4.068456739270876, + "tokens_seen": 353451008 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004509528585757272, + "loss": 3.0438, + "theoretical_loss": 4.068370590373139, + "tokens_seen": 353516544 + }, + { + "epoch": 1.02, + "learning_rate": 0.00045094282848545637, + "loss": 2.8643, + "theoretical_loss": 4.068284461915282, + "tokens_seen": 353582080 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004509327983951856, + "loss": 2.9281, + "theoretical_loss": 4.06819835388867, + "tokens_seen": 353647616 + }, + { + "epoch": 1.02, + "learning_rate": 0.00045092276830491473, + "loss": 3.2311, + "theoretical_loss": 4.06811226628467, + "tokens_seen": 353713152 + }, + { + "epoch": 1.02, + "learning_rate": 0.00045091273821464396, + "loss": 3.1787, + "theoretical_loss": 4.068026199094657, + "tokens_seen": 353778688 + }, + { + "epoch": 1.02, + "learning_rate": 0.00045090270812437314, + "loss": 3.188, + "theoretical_loss": 4.067940152310008, + "tokens_seen": 353844224 + }, + { + "epoch": 1.02, + "objective/train/docs_used": 473221, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.438317060470581, + "objective/train/theoretical_loss": 4.06787563060736, + "objective/train/tokens_used": 374353376, + "theoretical_loss": 4.06787563060736, + "tokens_seen": 353893376 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004508926780341023, + "loss": 3.0398, + "theoretical_loss": 4.06785412592211, + "tokens_seen": 353909760 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004508826479438315, + "loss": 3.107, + "theoretical_loss": 4.06776811992235, + "tokens_seen": 353975296 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004508726178535607, + "loss": 3.1091, + "theoretical_loss": 4.067682134302124, + "tokens_seen": 354040832 + }, + { + "epoch": 1.02, + "learning_rate": 0.00045086258776328987, + "loss": 3.1053, + "theoretical_loss": 4.06759616905283, + "tokens_seen": 354106368 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004508525576730191, + "loss": 3.3487, + "theoretical_loss": 4.0675102241658765, + "tokens_seen": 354171904 + }, + { + "epoch": 1.02, + "learning_rate": 0.00045084252758274823, + "loss": 3.0064, + "theoretical_loss": 4.0674242996326715, + "tokens_seen": 354237440 + }, + { + "epoch": 1.02, + "learning_rate": 0.00045083249749247747, + "loss": 2.8039, + "theoretical_loss": 4.06733839544463, + "tokens_seen": 354302976 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004508224674022066, + "loss": 3.0391, + "theoretical_loss": 4.067252511593175, + "tokens_seen": 354368512 + }, + { + "epoch": 1.02, + "learning_rate": 0.00045081243731193583, + "loss": 2.8456, + "theoretical_loss": 4.067166648069731, + "tokens_seen": 354434048 + }, + { + "epoch": 1.02, + "learning_rate": 0.000450802407221665, + "loss": 2.9981, + "theoretical_loss": 4.067080804865728, + "tokens_seen": 354499584 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004507923771313942, + "loss": 2.7936, + "theoretical_loss": 4.066994981972604, + "tokens_seen": 354565120 + }, + { + "epoch": 1.02, + "learning_rate": 0.00045078234704112337, + "loss": 3.2045, + "theoretical_loss": 4.066909179381801, + "tokens_seen": 354630656 + }, + { + "epoch": 1.02, + "learning_rate": 0.00045077231695085255, + "loss": 2.9505, + "theoretical_loss": 4.066823397084764, + "tokens_seen": 354696192 + }, + { + "epoch": 1.02, + "learning_rate": 0.00045076228686058173, + "loss": 2.6534, + "theoretical_loss": 4.066737635072946, + "tokens_seen": 354761728 + }, + { + "epoch": 1.02, + "learning_rate": 0.00045075225677031097, + "loss": 2.7746, + "theoretical_loss": 4.066651893337804, + "tokens_seen": 354827264 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004507422266800401, + "loss": 3.0798, + "theoretical_loss": 4.066566171870799, + "tokens_seen": 354892800 + }, + { + "epoch": 1.02, + "learning_rate": 0.00045073219658976933, + "loss": 3.0763, + "theoretical_loss": 4.066480470663401, + "tokens_seen": 354958336 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004507221664994985, + "loss": 3.2006, + "theoretical_loss": 4.06639478970708, + "tokens_seen": 355023872 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004507121364092277, + "loss": 2.9487, + "theoretical_loss": 4.066309128993316, + "tokens_seen": 355089408 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004507021063189569, + "loss": 3.2167, + "theoretical_loss": 4.0662234885135895, + "tokens_seen": 355154944 + }, + { + "epoch": 1.02, + "learning_rate": 0.00045069207622868606, + "loss": 2.998, + "theoretical_loss": 4.066137868259391, + "tokens_seen": 355220480 + }, + { + "epoch": 1.02, + "learning_rate": 0.00045068204613841524, + "loss": 3.0349, + "theoretical_loss": 4.0660522682222116, + "tokens_seen": 355286016 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004506720160481445, + "loss": 3.0877, + "theoretical_loss": 4.065966688393551, + "tokens_seen": 355351552 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004506619859578736, + "loss": 2.874, + "theoretical_loss": 4.065881128764912, + "tokens_seen": 355417088 + }, + { + "epoch": 1.02, + "learning_rate": 0.00045065195586760284, + "loss": 3.1169, + "theoretical_loss": 4.065795589327804, + "tokens_seen": 355482624 + }, + { + "epoch": 1.02, + "objective/train/docs_used": 473851, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.1546261310577393, + "objective/train/theoretical_loss": 4.065731447995559, + "objective/train/tokens_used": 375991776, + "theoretical_loss": 4.065731447995559, + "tokens_seen": 355531776 + }, + { + "epoch": 1.02, + "learning_rate": 0.00045064192577733196, + "loss": 2.968, + "theoretical_loss": 4.06571007007374, + "tokens_seen": 355548160 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004506318956870612, + "loss": 3.0017, + "theoretical_loss": 4.065624570994239, + "tokens_seen": 355613696 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004506218655967904, + "loss": 3.1213, + "theoretical_loss": 4.065539092080826, + "tokens_seen": 355679232 + }, + { + "epoch": 1.02, + "learning_rate": 0.00045061183550651956, + "loss": 3.0196, + "theoretical_loss": 4.0654536333250295, + "tokens_seen": 355744768 + }, + { + "epoch": 1.02, + "learning_rate": 0.00045060180541624874, + "loss": 2.9044, + "theoretical_loss": 4.065368194718383, + "tokens_seen": 355810304 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004505917753259779, + "loss": 3.0828, + "theoretical_loss": 4.065282776252427, + "tokens_seen": 355875840 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004505817452357071, + "loss": 2.883, + "theoretical_loss": 4.065197377918706, + "tokens_seen": 355941376 + }, + { + "epoch": 1.02, + "learning_rate": 0.00045057171514543634, + "loss": 2.8719, + "theoretical_loss": 4.065111999708767, + "tokens_seen": 356006912 + }, + { + "epoch": 1.02, + "learning_rate": 0.00045056168505516547, + "loss": 2.7669, + "theoretical_loss": 4.065026641614169, + "tokens_seen": 356072448 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004505516549648947, + "loss": 2.9525, + "theoretical_loss": 4.064941303626469, + "tokens_seen": 356137984 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004505416248746239, + "loss": 2.9686, + "theoretical_loss": 4.064855985737234, + "tokens_seen": 356203520 + }, + { + "epoch": 1.02, + "learning_rate": 0.00045053159478435306, + "loss": 3.0237, + "theoretical_loss": 4.064770687938031, + "tokens_seen": 356269056 + }, + { + "epoch": 1.02, + "learning_rate": 0.00045052156469408224, + "loss": 3.1213, + "theoretical_loss": 4.064685410220437, + "tokens_seen": 356334592 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004505115346038114, + "loss": 3.1203, + "theoretical_loss": 4.064600152576032, + "tokens_seen": 356400128 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004505015045135406, + "loss": 3.2214, + "theoretical_loss": 4.0645149149964, + "tokens_seen": 356465664 + }, + { + "epoch": 1.02, + "learning_rate": 0.00045049147442326984, + "loss": 2.9673, + "theoretical_loss": 4.064429697473134, + "tokens_seen": 356531200 + }, + { + "epoch": 1.02, + "learning_rate": 0.00045048144433299897, + "loss": 2.9293, + "theoretical_loss": 4.064344499997826, + "tokens_seen": 356596736 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004504714142427282, + "loss": 2.7086, + "theoretical_loss": 4.06425932256208, + "tokens_seen": 356662272 + }, + { + "epoch": 1.02, + "learning_rate": 0.00045046138415245733, + "loss": 3.1569, + "theoretical_loss": 4.064174165157499, + "tokens_seen": 356727808 + }, + { + "epoch": 1.02, + "learning_rate": 0.00045045135406218657, + "loss": 2.777, + "theoretical_loss": 4.064089027775694, + "tokens_seen": 356793344 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004504413239719158, + "loss": 2.8872, + "theoretical_loss": 4.064003910408281, + "tokens_seen": 356858880 + }, + { + "epoch": 1.02, + "learning_rate": 0.00045043129388164493, + "loss": 3.1839, + "theoretical_loss": 4.06391881304688, + "tokens_seen": 356924416 + }, + { + "epoch": 1.02, + "learning_rate": 0.00045042126379137416, + "loss": 3.1245, + "theoretical_loss": 4.063833735683118, + "tokens_seen": 356989952 + }, + { + "epoch": 1.02, + "learning_rate": 0.00045041123370110334, + "loss": 2.964, + "theoretical_loss": 4.063748678308624, + "tokens_seen": 357055488 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004504012036108325, + "loss": 3.1374, + "theoretical_loss": 4.063663640915035, + "tokens_seen": 357121024 + }, + { + "epoch": 1.02, + "objective/train/docs_used": 475208, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.0372159481048584, + "objective/train/theoretical_loss": 4.063599875977284, + "objective/train/tokens_used": 377630176, + "theoretical_loss": 4.063599875977284, + "tokens_seen": 357170176 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004503911735205617, + "loss": 3.083, + "theoretical_loss": 4.063578623493992, + "tokens_seen": 357186560 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004503811434302909, + "loss": 3.0497, + "theoretical_loss": 4.06349362603714, + "tokens_seen": 357252096 + }, + { + "epoch": 1.02, + "learning_rate": 0.00045037111334002007, + "loss": 3.3375, + "theoretical_loss": 4.0634086485361305, + "tokens_seen": 357317632 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004503610832497493, + "loss": 2.9054, + "theoretical_loss": 4.063323690982619, + "tokens_seen": 357383168 + }, + { + "epoch": 1.02, + "learning_rate": 0.00045035105315947843, + "loss": 2.9316, + "theoretical_loss": 4.0632387533682675, + "tokens_seen": 357448704 + }, + { + "epoch": 1.02, + "learning_rate": 0.00045034102306920767, + "loss": 3.0126, + "theoretical_loss": 4.06315383568474, + "tokens_seen": 357514240 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004503309929789368, + "loss": 3.0289, + "theoretical_loss": 4.063068937923709, + "tokens_seen": 357579776 + }, + { + "epoch": 1.02, + "learning_rate": 0.00045032096288866603, + "loss": 2.7174, + "theoretical_loss": 4.062984060076849, + "tokens_seen": 357645312 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004503109327983952, + "loss": 3.0905, + "theoretical_loss": 4.062899202135844, + "tokens_seen": 357710848 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004503009027081244, + "loss": 2.8817, + "theoretical_loss": 4.062814364092376, + "tokens_seen": 357776384 + }, + { + "epoch": 1.02, + "learning_rate": 0.00045029087261785357, + "loss": 2.7171, + "theoretical_loss": 4.062729545938138, + "tokens_seen": 357841920 + }, + { + "epoch": 1.02, + "learning_rate": 0.00045028084252758275, + "loss": 2.963, + "theoretical_loss": 4.062644747664827, + "tokens_seen": 357907456 + }, + { + "epoch": 1.02, + "learning_rate": 0.00045027081243731193, + "loss": 3.0307, + "theoretical_loss": 4.062559969264141, + "tokens_seen": 357972992 + }, + { + "epoch": 1.02, + "learning_rate": 0.00045026078234704117, + "loss": 3.2462, + "theoretical_loss": 4.062475210727789, + "tokens_seen": 358038528 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004502507522567703, + "loss": 2.9787, + "theoretical_loss": 4.06239047204748, + "tokens_seen": 358104064 + }, + { + "epoch": 1.02, + "learning_rate": 0.00045024072216649953, + "loss": 2.927, + "theoretical_loss": 4.06230575321493, + "tokens_seen": 358169600 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004502306920762287, + "loss": 3.0136, + "theoretical_loss": 4.06222105422186, + "tokens_seen": 358235136 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004502206619859579, + "loss": 3.1244, + "theoretical_loss": 4.062136375059996, + "tokens_seen": 358300672 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004502106318956871, + "loss": 2.8825, + "theoretical_loss": 4.062051715721069, + "tokens_seen": 358366208 + }, + { + "epoch": 1.02, + "learning_rate": 0.00045020060180541626, + "loss": 3.108, + "theoretical_loss": 4.061967076196815, + "tokens_seen": 358431744 + }, + { + "epoch": 1.02, + "learning_rate": 0.00045019057171514544, + "loss": 3.1374, + "theoretical_loss": 4.061882456478973, + "tokens_seen": 358497280 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004501805416248747, + "loss": 3.17, + "theoretical_loss": 4.0617978565592905, + "tokens_seen": 358562816 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004501705115346038, + "loss": 3.2081, + "theoretical_loss": 4.061713276429517, + "tokens_seen": 358628352 + }, + { + "epoch": 1.02, + "learning_rate": 0.00045016048144433304, + "loss": 3.1158, + "theoretical_loss": 4.061628716081408, + "tokens_seen": 358693888 + }, + { + "epoch": 1.02, + "learning_rate": 0.00045015045135406216, + "loss": 3.176, + "theoretical_loss": 4.061544175506725, + "tokens_seen": 358759424 + }, + { + "epoch": 1.02, + "objective/train/docs_used": 475771, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.862164258956909, + "objective/train/theoretical_loss": 4.061480783047069, + "objective/train/tokens_used": 379268576, + "theoretical_loss": 4.061480783047069, + "tokens_seen": 358808576 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004501404212637914, + "loss": 3.0135, + "theoretical_loss": 4.061459654697233, + "tokens_seen": 358824960 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004501303911735206, + "loss": 3.2575, + "theoretical_loss": 4.061375153644701, + "tokens_seen": 358890496 + }, + { + "epoch": 1.02, + "learning_rate": 0.00045012036108324976, + "loss": 3.0728, + "theoretical_loss": 4.061290672340906, + "tokens_seen": 358956032 + }, + { + "epoch": 1.02, + "learning_rate": 0.00045011033099297894, + "loss": 3.1692, + "theoretical_loss": 4.061206210777627, + "tokens_seen": 359021568 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004501003009027081, + "loss": 3.0361, + "theoretical_loss": 4.0611217689466494, + "tokens_seen": 359087104 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004500902708124373, + "loss": 3.0446, + "theoretical_loss": 4.061037346839764, + "tokens_seen": 359152640 + }, + { + "epoch": 1.02, + "learning_rate": 0.00045008024072216654, + "loss": 3.0256, + "theoretical_loss": 4.060952944448765, + "tokens_seen": 359218176 + }, + { + "epoch": 1.02, + "learning_rate": 0.00045007021063189567, + "loss": 3.0506, + "theoretical_loss": 4.060868561765452, + "tokens_seen": 359283712 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004500601805416249, + "loss": 3.0259, + "theoretical_loss": 4.060784198781631, + "tokens_seen": 359349248 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004500501504513541, + "loss": 3.1077, + "theoretical_loss": 4.06069985548911, + "tokens_seen": 359414784 + }, + { + "epoch": 1.02, + "learning_rate": 0.00045004012036108326, + "loss": 3.1462, + "theoretical_loss": 4.060615531879705, + "tokens_seen": 359480320 + }, + { + "epoch": 1.02, + "learning_rate": 0.00045003009027081244, + "loss": 3.0044, + "theoretical_loss": 4.0605312279452335, + "tokens_seen": 359545856 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004500200601805416, + "loss": 2.947, + "theoretical_loss": 4.060446943677523, + "tokens_seen": 359611392 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004500100300902708, + "loss": 3.0459, + "theoretical_loss": 4.060362679068399, + "tokens_seen": 359676928 + }, + { + "epoch": 1.02, + "learning_rate": 0.00045000000000000004, + "loss": 3.2938, + "theoretical_loss": 4.060278434109699, + "tokens_seen": 359742464 + }, + { + "epoch": 1.02, + "learning_rate": 0.00044998996990972917, + "loss": 2.9767, + "theoretical_loss": 4.060194208793259, + "tokens_seen": 359808000 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004499799398194584, + "loss": 2.9244, + "theoretical_loss": 4.060110003110925, + "tokens_seen": 359873536 + }, + { + "epoch": 1.02, + "learning_rate": 0.00044996990972918753, + "loss": 3.1669, + "theoretical_loss": 4.0600258170545445, + "tokens_seen": 359939072 + }, + { + "epoch": 1.02, + "learning_rate": 0.00044995987963891677, + "loss": 3.0841, + "theoretical_loss": 4.059941650615972, + "tokens_seen": 360004608 + }, + { + "epoch": 1.02, + "learning_rate": 0.00044994984954864595, + "loss": 3.0194, + "theoretical_loss": 4.059857503787066, + "tokens_seen": 360070144 + }, + { + "epoch": 1.02, + "learning_rate": 0.00044993981945837513, + "loss": 2.9446, + "theoretical_loss": 4.059773376559689, + "tokens_seen": 360135680 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004499297893681043, + "loss": 2.8132, + "theoretical_loss": 4.05968926892571, + "tokens_seen": 360201216 + }, + { + "epoch": 1.02, + "learning_rate": 0.00044991975927783355, + "loss": 3.0256, + "theoretical_loss": 4.059605180877001, + "tokens_seen": 360266752 + }, + { + "epoch": 1.02, + "learning_rate": 0.00044990972918756267, + "loss": 3.0306, + "theoretical_loss": 4.05952111240544, + "tokens_seen": 360332288 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004498996990972919, + "loss": 2.9097, + "theoretical_loss": 4.0594370635029104, + "tokens_seen": 360397824 + }, + { + "epoch": 1.02, + "objective/train/docs_used": 477253, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.3396573066711426, + "objective/train/theoretical_loss": 4.059374039663311, + "objective/train/tokens_used": 380906976, + "theoretical_loss": 4.059374039663311, + "tokens_seen": 360446976 + }, + { + "epoch": 1.02, + "learning_rate": 0.00044988966900702103, + "loss": 3.1348, + "theoretical_loss": 4.0593530341613, + "tokens_seen": 360463360 + }, + { + "epoch": 1.02, + "learning_rate": 0.00044987963891675027, + "loss": 2.9424, + "theoretical_loss": 4.059269024372501, + "tokens_seen": 360528896 + }, + { + "epoch": 1.02, + "learning_rate": 0.00044986960882647945, + "loss": 3.1762, + "theoretical_loss": 4.05918503412841, + "tokens_seen": 360594432 + }, + { + "epoch": 1.02, + "learning_rate": 0.00044985957873620863, + "loss": 3.2145, + "theoretical_loss": 4.059101063420929, + "tokens_seen": 360659968 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004498495486459378, + "loss": 2.899, + "theoretical_loss": 4.059017112241966, + "tokens_seen": 360725504 + }, + { + "epoch": 1.02, + "learning_rate": 0.000449839518555667, + "loss": 3.1533, + "theoretical_loss": 4.058933180583431, + "tokens_seen": 360791040 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004498294884653962, + "loss": 3.015, + "theoretical_loss": 4.0588492684372435, + "tokens_seen": 360856576 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004498194583751254, + "loss": 3.1045, + "theoretical_loss": 4.058765375795321, + "tokens_seen": 360922112 + }, + { + "epoch": 1.02, + "learning_rate": 0.00044980942828485454, + "loss": 3.291, + "theoretical_loss": 4.058681502649593, + "tokens_seen": 360987648 + }, + { + "epoch": 1.02, + "learning_rate": 0.00044979939819458377, + "loss": 2.8875, + "theoretical_loss": 4.058597648991988, + "tokens_seen": 361053184 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004497893681043129, + "loss": 2.8632, + "theoretical_loss": 4.058513814814443, + "tokens_seen": 361118720 + }, + { + "epoch": 1.02, + "learning_rate": 0.00044977933801404213, + "loss": 2.8654, + "theoretical_loss": 4.058430000108898, + "tokens_seen": 361184256 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004497693079237713, + "loss": 3.1835, + "theoretical_loss": 4.058346204867299, + "tokens_seen": 361249792 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004497592778335005, + "loss": 3.1647, + "theoretical_loss": 4.058262429081596, + "tokens_seen": 361315328 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004497492477432297, + "loss": 3.0466, + "theoretical_loss": 4.058178672743744, + "tokens_seen": 361380864 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004497392176529589, + "loss": 3.0466, + "theoretical_loss": 4.058094935845703, + "tokens_seen": 361446400 + }, + { + "epoch": 1.02, + "learning_rate": 0.00044972918756268804, + "loss": 3.0096, + "theoretical_loss": 4.058011218379436, + "tokens_seen": 361511936 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004497191574724173, + "loss": 3.2179, + "theoretical_loss": 4.057927520336913, + "tokens_seen": 361577472 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004497091273821464, + "loss": 3.4683, + "theoretical_loss": 4.05784384171011, + "tokens_seen": 361643008 + }, + { + "epoch": 1.02, + "learning_rate": 0.00044969909729187564, + "loss": 3.0278, + "theoretical_loss": 4.057760182491003, + "tokens_seen": 361708544 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004496890672016049, + "loss": 2.9469, + "theoretical_loss": 4.057676542671577, + "tokens_seen": 361774080 + }, + { + "epoch": 1.02, + "learning_rate": 0.000449679037111334, + "loss": 3.0088, + "theoretical_loss": 4.0575929222438205, + "tokens_seen": 361839616 + }, + { + "epoch": 1.02, + "learning_rate": 0.00044966900702106324, + "loss": 2.9364, + "theoretical_loss": 4.057509321199726, + "tokens_seen": 361905152 + }, + { + "epoch": 1.02, + "learning_rate": 0.00044965897693079236, + "loss": 2.808, + "theoretical_loss": 4.057425739531292, + "tokens_seen": 361970688 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004496489468405216, + "loss": 3.017, + "theoretical_loss": 4.057342177230519, + "tokens_seen": 362036224 + }, + { + "epoch": 1.02, + "objective/train/docs_used": 478095, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.1937191486358643, + "objective/train/theoretical_loss": 4.057279518210161, + "objective/train/tokens_used": 382545376, + "theoretical_loss": 4.057279518210161, + "tokens_seen": 362085376 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004496389167502508, + "loss": 3.2181, + "theoretical_loss": 4.057258634289418, + "tokens_seen": 362101760 + }, + { + "epoch": 1.02, + "learning_rate": 0.00044962888665997996, + "loss": 2.8883, + "theoretical_loss": 4.057175110699999, + "tokens_seen": 362167296 + }, + { + "epoch": 1.02, + "learning_rate": 0.00044961885656970914, + "loss": 2.9905, + "theoretical_loss": 4.057091606454279, + "tokens_seen": 362232832 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004496088264794383, + "loss": 2.9349, + "theoretical_loss": 4.057008121544279, + "tokens_seen": 362298368 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004495987963891675, + "loss": 3.0119, + "theoretical_loss": 4.056924655962027, + "tokens_seen": 362363904 + }, + { + "epoch": 1.02, + "learning_rate": 0.00044958876629889674, + "loss": 2.8529, + "theoretical_loss": 4.056841209699553, + "tokens_seen": 362429440 + }, + { + "epoch": 1.02, + "learning_rate": 0.00044957873620862587, + "loss": 3.0478, + "theoretical_loss": 4.0567577827488925, + "tokens_seen": 362494976 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004495687061183551, + "loss": 3.0898, + "theoretical_loss": 4.056674375102086, + "tokens_seen": 362560512 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004495586760280843, + "loss": 2.9169, + "theoretical_loss": 4.05659098675118, + "tokens_seen": 362626048 + }, + { + "epoch": 1.02, + "learning_rate": 0.00044954864593781346, + "loss": 2.6805, + "theoretical_loss": 4.056507617688223, + "tokens_seen": 362691584 + }, + { + "epoch": 1.02, + "learning_rate": 0.00044953861584754264, + "loss": 3.002, + "theoretical_loss": 4.05642426790527, + "tokens_seen": 362757120 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004495285857572718, + "loss": 3.2354, + "theoretical_loss": 4.056340937394381, + "tokens_seen": 362822656 + }, + { + "epoch": 1.02, + "learning_rate": 0.000449518555667001, + "loss": 2.8529, + "theoretical_loss": 4.0562576261476195, + "tokens_seen": 362888192 + }, + { + "epoch": 1.02, + "learning_rate": 0.00044950852557673024, + "loss": 3.0567, + "theoretical_loss": 4.056174334157054, + "tokens_seen": 362953728 + }, + { + "epoch": 1.02, + "learning_rate": 0.00044949849548645937, + "loss": 3.1938, + "theoretical_loss": 4.056091061414759, + "tokens_seen": 363019264 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004494884653961886, + "loss": 2.914, + "theoretical_loss": 4.0560078079128115, + "tokens_seen": 363084800 + }, + { + "epoch": 1.02, + "learning_rate": 0.00044947843530591773, + "loss": 2.9784, + "theoretical_loss": 4.055924573643295, + "tokens_seen": 363150336 + }, + { + "epoch": 1.02, + "learning_rate": 0.00044946840521564697, + "loss": 2.9387, + "theoretical_loss": 4.055841358598297, + "tokens_seen": 363215872 + }, + { + "epoch": 1.02, + "learning_rate": 0.00044945837512537615, + "loss": 3.182, + "theoretical_loss": 4.055758162769909, + "tokens_seen": 363281408 + }, + { + "epoch": 1.02, + "learning_rate": 0.00044944834503510533, + "loss": 3.0795, + "theoretical_loss": 4.055674986150228, + "tokens_seen": 363346944 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004494383149448345, + "loss": 3.0805, + "theoretical_loss": 4.055591828731356, + "tokens_seen": 363412480 + }, + { + "epoch": 1.02, + "learning_rate": 0.00044942828485456375, + "loss": 3.2346, + "theoretical_loss": 4.0555086905054, + "tokens_seen": 363478016 + }, + { + "epoch": 1.02, + "learning_rate": 0.00044941825476429287, + "loss": 3.0509, + "theoretical_loss": 4.05542557146447, + "tokens_seen": 363543552 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004494082246740221, + "loss": 2.9644, + "theoretical_loss": 4.055342471600682, + "tokens_seen": 363609088 + }, + { + "epoch": 1.02, + "learning_rate": 0.00044939819458375123, + "loss": 3.0934, + "theoretical_loss": 4.055259390906155, + "tokens_seen": 363674624 + }, + { + "epoch": 1.02, + "objective/train/docs_used": 479451, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.949183702468872, + "objective/train/theoretical_loss": 4.05519709296035, + "objective/train/tokens_used": 384183776, + "theoretical_loss": 4.05519709296035, + "tokens_seen": 363723776 + }, + { + "epoch": 1.02, + "learning_rate": 0.00044938816449348047, + "loss": 2.8857, + "theoretical_loss": 4.055176329373015, + "tokens_seen": 363740160 + }, + { + "epoch": 1.02, + "learning_rate": 0.00044937813440320965, + "loss": 2.8673, + "theoretical_loss": 4.055093286993392, + "tokens_seen": 363805696 + }, + { + "epoch": 1.02, + "learning_rate": 0.00044936810431293883, + "loss": 2.9403, + "theoretical_loss": 4.055010263759419, + "tokens_seen": 363871232 + }, + { + "epoch": 1.02, + "learning_rate": 0.000449358074222668, + "loss": 2.9892, + "theoretical_loss": 4.054927259663235, + "tokens_seen": 363936768 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004493480441323972, + "loss": 3.1854, + "theoretical_loss": 4.054844274696984, + "tokens_seen": 364002304 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004493380140421264, + "loss": 3.1805, + "theoretical_loss": 4.0547613088528145, + "tokens_seen": 364067840 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004493279839518556, + "loss": 2.9269, + "theoretical_loss": 4.054678362122878, + "tokens_seen": 364133376 + }, + { + "epoch": 1.02, + "learning_rate": 0.00044931795386158474, + "loss": 3.0968, + "theoretical_loss": 4.0545954344993325, + "tokens_seen": 364198912 + }, + { + "epoch": 1.02, + "learning_rate": 0.00044930792377131397, + "loss": 3.1839, + "theoretical_loss": 4.05451252597434, + "tokens_seen": 364264448 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004492978936810431, + "loss": 3.0138, + "theoretical_loss": 4.054429636540068, + "tokens_seen": 364329984 + }, + { + "epoch": 1.02, + "learning_rate": 0.00044928786359077234, + "loss": 2.8682, + "theoretical_loss": 4.0543467661886865, + "tokens_seen": 364395520 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004492778335005015, + "loss": 2.8514, + "theoretical_loss": 4.054263914912372, + "tokens_seen": 364461056 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004492678034102307, + "loss": 2.8017, + "theoretical_loss": 4.0541810827033045, + "tokens_seen": 364526592 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004492577733199599, + "loss": 3.0815, + "theoretical_loss": 4.054098269553669, + "tokens_seen": 364592128 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004492477432296891, + "loss": 2.8874, + "theoretical_loss": 4.054015475455656, + "tokens_seen": 364657664 + }, + { + "epoch": 1.02, + "learning_rate": 0.00044923771313941824, + "loss": 3.159, + "theoretical_loss": 4.053932700401459, + "tokens_seen": 364723200 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004492276830491475, + "loss": 3.033, + "theoretical_loss": 4.053849944383279, + "tokens_seen": 364788736 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004492176529588766, + "loss": 3.0449, + "theoretical_loss": 4.0537672073933155, + "tokens_seen": 364854272 + }, + { + "epoch": 1.02, + "learning_rate": 0.00044920762286860584, + "loss": 3.2518, + "theoretical_loss": 4.05368448942378, + "tokens_seen": 364919808 + }, + { + "epoch": 1.02, + "learning_rate": 0.000449197592778335, + "loss": 3.0096, + "theoretical_loss": 4.053601790466884, + "tokens_seen": 364985344 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004491875626880642, + "loss": 3.032, + "theoretical_loss": 4.053519110514845, + "tokens_seen": 365050880 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004491775325977934, + "loss": 3.2143, + "theoretical_loss": 4.053436449559886, + "tokens_seen": 365116416 + }, + { + "epoch": 1.02, + "learning_rate": 0.00044916750250752256, + "loss": 2.92, + "theoretical_loss": 4.05335380759423, + "tokens_seen": 365181952 + }, + { + "epoch": 1.02, + "learning_rate": 0.00044915747241725174, + "loss": 3.1182, + "theoretical_loss": 4.053271184610111, + "tokens_seen": 365247488 + }, + { + "epoch": 1.02, + "learning_rate": 0.000449147442326981, + "loss": 2.9261, + "theoretical_loss": 4.053188580599764, + "tokens_seen": 365313024 + }, + { + "epoch": 1.02, + "objective/train/docs_used": 480072, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.2834551334381104, + "objective/train/theoretical_loss": 4.053126640038872, + "objective/train/tokens_used": 385822176, + "theoretical_loss": 4.053126640038872, + "tokens_seen": 365362176 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004491374122367101, + "loss": 2.9938, + "theoretical_loss": 4.053105995555429, + "tokens_seen": 365378560 + }, + { + "epoch": 1.02, + "learning_rate": 0.00044912738214643934, + "loss": 2.9087, + "theoretical_loss": 4.0530234294693495, + "tokens_seen": 365444096 + }, + { + "epoch": 1.02, + "learning_rate": 0.00044911735205616847, + "loss": 2.8535, + "theoretical_loss": 4.052940882333777, + "tokens_seen": 365509632 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004491073219658977, + "loss": 3.0883, + "theoretical_loss": 4.052858354140964, + "tokens_seen": 365575168 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004490972918756269, + "loss": 2.8664, + "theoretical_loss": 4.052775844883168, + "tokens_seen": 365640704 + }, + { + "epoch": 1.02, + "learning_rate": 0.00044908726178535607, + "loss": 3.0148, + "theoretical_loss": 4.0526933545526544, + "tokens_seen": 365706240 + }, + { + "epoch": 1.02, + "learning_rate": 0.00044907723169508525, + "loss": 3.1162, + "theoretical_loss": 4.052610883141689, + "tokens_seen": 365771776 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004490672016048145, + "loss": 3.2736, + "theoretical_loss": 4.052528430642544, + "tokens_seen": 365837312 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004490571715145436, + "loss": 2.9012, + "theoretical_loss": 4.052445997047497, + "tokens_seen": 365902848 + }, + { + "epoch": 1.02, + "learning_rate": 0.00044904714142427284, + "loss": 3.0513, + "theoretical_loss": 4.052363582348827, + "tokens_seen": 365968384 + }, + { + "epoch": 1.02, + "learning_rate": 0.00044903711133400197, + "loss": 3.3379, + "theoretical_loss": 4.052281186538821, + "tokens_seen": 366033920 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004490270812437312, + "loss": 3.041, + "theoretical_loss": 4.052198809609769, + "tokens_seen": 366099456 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004490170511534604, + "loss": 2.924, + "theoretical_loss": 4.0521164515539665, + "tokens_seen": 366164992 + }, + { + "epoch": 1.02, + "learning_rate": 0.00044900702106318957, + "loss": 2.9846, + "theoretical_loss": 4.052034112363711, + "tokens_seen": 366230528 + }, + { + "epoch": 1.02, + "learning_rate": 0.00044899699097291875, + "loss": 3.087, + "theoretical_loss": 4.051951792031308, + "tokens_seen": 366296064 + }, + { + "epoch": 1.02, + "learning_rate": 0.00044898696088264793, + "loss": 3.0914, + "theoretical_loss": 4.051869490549064, + "tokens_seen": 366361600 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004489769307923771, + "loss": 2.9215, + "theoretical_loss": 4.051787207909294, + "tokens_seen": 366427136 + }, + { + "epoch": 1.02, + "learning_rate": 0.00044896690070210635, + "loss": 2.9358, + "theoretical_loss": 4.051704944104313, + "tokens_seen": 366492672 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004489568706118355, + "loss": 3.0429, + "theoretical_loss": 4.051622699126444, + "tokens_seen": 366558208 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004489468405215647, + "loss": 3.0332, + "theoretical_loss": 4.0515404729680125, + "tokens_seen": 366623744 + }, + { + "epoch": 1.02, + "learning_rate": 0.00044893681043129395, + "loss": 3.057, + "theoretical_loss": 4.051458265621351, + "tokens_seen": 366689280 + }, + { + "epoch": 1.02, + "learning_rate": 0.00044892678034102307, + "loss": 3.0415, + "theoretical_loss": 4.0513760770787925, + "tokens_seen": 366754816 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004489167502507523, + "loss": 3.3661, + "theoretical_loss": 4.051293907332678, + "tokens_seen": 366820352 + }, + { + "epoch": 1.02, + "learning_rate": 0.00044890672016048143, + "loss": 3.2608, + "theoretical_loss": 4.051211756375352, + "tokens_seen": 366885888 + }, + { + "epoch": 1.02, + "learning_rate": 0.00044889669007021067, + "loss": 3.042, + "theoretical_loss": 4.051129624199163, + "tokens_seen": 366951424 + }, + { + "epoch": 1.02, + "objective/train/docs_used": 481451, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.963195323944092, + "objective/train/theoretical_loss": 4.051068037387542, + "objective/train/tokens_used": 387460576, + "theoretical_loss": 4.051068037387542, + "tokens_seen": 367000576 + }, + { + "epoch": 1.02, + "learning_rate": 0.00044888665997993985, + "loss": 3.0687, + "theoretical_loss": 4.051047510796463, + "tokens_seen": 367016960 + }, + { + "epoch": 1.02, + "learning_rate": 0.00044887662988966903, + "loss": 2.9519, + "theoretical_loss": 4.050965416159612, + "tokens_seen": 367082496 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004488665997993982, + "loss": 2.8079, + "theoretical_loss": 4.05088334028097, + "tokens_seen": 367148032 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004488565697091274, + "loss": 3.114, + "theoretical_loss": 4.050801283152905, + "tokens_seen": 367213568 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004488465396188566, + "loss": 2.8896, + "theoretical_loss": 4.050719244767788, + "tokens_seen": 367279104 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004488365095285858, + "loss": 2.8136, + "theoretical_loss": 4.0506372251179945, + "tokens_seen": 367344640 + }, + { + "epoch": 1.02, + "learning_rate": 0.00044882647943831494, + "loss": 2.8483, + "theoretical_loss": 4.050555224195904, + "tokens_seen": 367410176 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004488164493480442, + "loss": 3.2117, + "theoretical_loss": 4.050473241993901, + "tokens_seen": 367475712 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004488064192577733, + "loss": 3.0052, + "theoretical_loss": 4.0503912785043745, + "tokens_seen": 367541248 + }, + { + "epoch": 1.02, + "learning_rate": 0.00044879638916750254, + "loss": 2.9371, + "theoretical_loss": 4.050309333719719, + "tokens_seen": 367606784 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004487863590772317, + "loss": 3.207, + "theoretical_loss": 4.050227407632331, + "tokens_seen": 367672320 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004487763289869609, + "loss": 3.1296, + "theoretical_loss": 4.050145500234613, + "tokens_seen": 367737856 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004487662988966901, + "loss": 2.7027, + "theoretical_loss": 4.050063611518974, + "tokens_seen": 367803392 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004487562688064193, + "loss": 3.1566, + "theoretical_loss": 4.049981741477821, + "tokens_seen": 367868928 + }, + { + "epoch": 1.02, + "learning_rate": 0.00044874623871614844, + "loss": 2.958, + "theoretical_loss": 4.049899890103573, + "tokens_seen": 367934464 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004487362086258777, + "loss": 3.1707, + "theoretical_loss": 4.049818057388649, + "tokens_seen": 368000000 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004487261785356068, + "loss": 3.0476, + "theoretical_loss": 4.049736243325473, + "tokens_seen": 368065536 + }, + { + "epoch": 1.02, + "learning_rate": 0.00044871614844533604, + "loss": 2.7808, + "theoretical_loss": 4.049654447906473, + "tokens_seen": 368131072 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004487061183550652, + "loss": 3.0115, + "theoretical_loss": 4.049572671124086, + "tokens_seen": 368196608 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004486960882647944, + "loss": 3.1322, + "theoretical_loss": 4.049490912970745, + "tokens_seen": 368262144 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004486860581745236, + "loss": 2.9311, + "theoretical_loss": 4.049409173438896, + "tokens_seen": 368327680 + }, + { + "epoch": 1.02, + "learning_rate": 0.00044867602808425276, + "loss": 3.1052, + "theoretical_loss": 4.049327452520982, + "tokens_seen": 368393216 + }, + { + "epoch": 1.02, + "learning_rate": 0.00044866599799398194, + "loss": 2.8961, + "theoretical_loss": 4.049245750209458, + "tokens_seen": 368458752 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004486559679037112, + "loss": 2.943, + "theoretical_loss": 4.049164066496776, + "tokens_seen": 368524288 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004486459378134403, + "loss": 2.9977, + "theoretical_loss": 4.049082401375397, + "tokens_seen": 368589824 + }, + { + "epoch": 1.02, + "objective/train/docs_used": 482108, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.12469744682312, + "objective/train/theoretical_loss": 4.049021164730373, + "objective/train/tokens_used": 389098976, + "theoretical_loss": 4.049021164730373, + "tokens_seen": 368638976 + }, + { + "epoch": 1.02, + "learning_rate": 0.00044863590772316954, + "loss": 3.2333, + "theoretical_loss": 4.049000754837786, + "tokens_seen": 368655360 + }, + { + "epoch": 1.02, + "learning_rate": 0.00044862587763289867, + "loss": 2.8482, + "theoretical_loss": 4.04891912687641, + "tokens_seen": 368720896 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004486158475426279, + "loss": 2.9425, + "theoretical_loss": 4.048837517483744, + "tokens_seen": 368786432 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004486058174523571, + "loss": 3.0541, + "theoretical_loss": 4.048755926652262, + "tokens_seen": 368851968 + }, + { + "epoch": 1.02, + "learning_rate": 0.00044859578736208627, + "loss": 2.9036, + "theoretical_loss": 4.04867435437445, + "tokens_seen": 368917504 + }, + { + "epoch": 1.02, + "learning_rate": 0.00044858575727181545, + "loss": 3.1034, + "theoretical_loss": 4.0485928006427905, + "tokens_seen": 368983040 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004485757271815447, + "loss": 3.0932, + "theoretical_loss": 4.0485112654497755, + "tokens_seen": 369048576 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004485656970912738, + "loss": 3.0221, + "theoretical_loss": 4.048429748787899, + "tokens_seen": 369114112 + }, + { + "epoch": 1.02, + "learning_rate": 0.00044855566700100304, + "loss": 2.7535, + "theoretical_loss": 4.048348250649662, + "tokens_seen": 369179648 + }, + { + "epoch": 1.02, + "learning_rate": 0.00044854563691073217, + "loss": 3.1027, + "theoretical_loss": 4.048266771027565, + "tokens_seen": 369245184 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004485356068204614, + "loss": 3.3042, + "theoretical_loss": 4.048185309914119, + "tokens_seen": 369310720 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004485255767301906, + "loss": 3.091, + "theoretical_loss": 4.048103867301836, + "tokens_seen": 369376256 + }, + { + "epoch": 1.02, + "learning_rate": 0.00044851554663991977, + "loss": 2.9852, + "theoretical_loss": 4.04802244318323, + "tokens_seen": 369441792 + }, + { + "epoch": 1.02, + "learning_rate": 0.00044850551654964895, + "loss": 3.1036, + "theoretical_loss": 4.047941037550825, + "tokens_seen": 369507328 + }, + { + "epoch": 1.02, + "learning_rate": 0.00044849548645937813, + "loss": 2.9854, + "theoretical_loss": 4.047859650397145, + "tokens_seen": 369572864 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004484854563691073, + "loss": 3.1245, + "theoretical_loss": 4.04777828171472, + "tokens_seen": 369638400 + }, + { + "epoch": 1.02, + "learning_rate": 0.00044847542627883655, + "loss": 2.9639, + "theoretical_loss": 4.047696931496084, + "tokens_seen": 369703936 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004484653961885657, + "loss": 3.0576, + "theoretical_loss": 4.047615599733775, + "tokens_seen": 369769472 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004484553660982949, + "loss": 3.1132, + "theoretical_loss": 4.047534286420337, + "tokens_seen": 369835008 + }, + { + "epoch": 1.02, + "learning_rate": 0.00044844533600802404, + "loss": 3.0274, + "theoretical_loss": 4.047452991548315, + "tokens_seen": 369900544 + }, + { + "epoch": 1.02, + "learning_rate": 0.00044843530591775327, + "loss": 3.2649, + "theoretical_loss": 4.047371715110262, + "tokens_seen": 369966080 + }, + { + "epoch": 1.02, + "learning_rate": 0.00044842527582748245, + "loss": 2.865, + "theoretical_loss": 4.047290457098733, + "tokens_seen": 370031616 + }, + { + "epoch": 1.02, + "learning_rate": 0.00044841524573721163, + "loss": 3.3263, + "theoretical_loss": 4.047209217506289, + "tokens_seen": 370097152 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004484052156469408, + "loss": 2.9046, + "theoretical_loss": 4.047127996325493, + "tokens_seen": 370162688 + }, + { + "epoch": 1.02, + "learning_rate": 0.00044839518555667005, + "loss": 3.1456, + "theoretical_loss": 4.0470467935489145, + "tokens_seen": 370228224 + }, + { + "epoch": 1.02, + "objective/train/docs_used": 483097, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9543488025665283, + "objective/train/theoretical_loss": 4.04698590353978, + "objective/train/tokens_used": 390737376, + "theoretical_loss": 4.04698590353978, + "tokens_seen": 370277376 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004483851554663992, + "loss": 2.8715, + "theoretical_loss": 4.046965609169126, + "tokens_seen": 370293760 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004483751253761284, + "loss": 2.9675, + "theoretical_loss": 4.046884443178706, + "tokens_seen": 370359296 + }, + { + "epoch": 1.02, + "learning_rate": 0.00044836509528585754, + "loss": 3.2167, + "theoretical_loss": 4.046803295570235, + "tokens_seen": 370424832 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004483550651955868, + "loss": 3.1818, + "theoretical_loss": 4.046722166336298, + "tokens_seen": 370490368 + }, + { + "epoch": 1.02, + "learning_rate": 0.00044834503510531596, + "loss": 3.1305, + "theoretical_loss": 4.046641055469488, + "tokens_seen": 370555904 + }, + { + "epoch": 1.02, + "learning_rate": 0.00044833500501504514, + "loss": 2.8626, + "theoretical_loss": 4.046559962962398, + "tokens_seen": 370621440 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004483249749247743, + "loss": 2.9436, + "theoretical_loss": 4.046478888807626, + "tokens_seen": 370686976 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004483149448345035, + "loss": 3.2137, + "theoretical_loss": 4.046397832997776, + "tokens_seen": 370752512 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004483049147442327, + "loss": 3.1725, + "theoretical_loss": 4.046316795525456, + "tokens_seen": 370818048 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004482948846539619, + "loss": 3.1907, + "theoretical_loss": 4.046235776383276, + "tokens_seen": 370883584 + }, + { + "epoch": 1.02, + "learning_rate": 0.00044828485456369104, + "loss": 2.9513, + "theoretical_loss": 4.046154775563854, + "tokens_seen": 370949120 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004482748244734203, + "loss": 3.2516, + "theoretical_loss": 4.046073793059808, + "tokens_seen": 371014656 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004482647943831494, + "loss": 3.1746, + "theoretical_loss": 4.045992828863764, + "tokens_seen": 371080192 + }, + { + "epoch": 1.02, + "learning_rate": 0.00044825476429287864, + "loss": 3.0957, + "theoretical_loss": 4.045911882968351, + "tokens_seen": 371145728 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004482447342026078, + "loss": 3.1395, + "theoretical_loss": 4.045830955366201, + "tokens_seen": 371211264 + }, + { + "epoch": 1.02, + "learning_rate": 0.000448234704112337, + "loss": 3.0624, + "theoretical_loss": 4.045750046049952, + "tokens_seen": 371276800 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004482246740220662, + "loss": 2.9705, + "theoretical_loss": 4.045669155012246, + "tokens_seen": 371342336 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004482146439317954, + "loss": 2.8102, + "theoretical_loss": 4.045588282245728, + "tokens_seen": 371407872 + }, + { + "epoch": 1.02, + "learning_rate": 0.00044820461384152455, + "loss": 3.1322, + "theoretical_loss": 4.045507427743048, + "tokens_seen": 371473408 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004481945837512538, + "loss": 3.0764, + "theoretical_loss": 4.045426591496861, + "tokens_seen": 371538944 + }, + { + "epoch": 1.02, + "learning_rate": 0.00044818455366098296, + "loss": 3.0931, + "theoretical_loss": 4.045345773499825, + "tokens_seen": 371604480 + }, + { + "epoch": 1.02, + "learning_rate": 0.00044817452357071214, + "loss": 3.1696, + "theoretical_loss": 4.045264973744604, + "tokens_seen": 371670016 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004481644934804414, + "loss": 2.9068, + "theoretical_loss": 4.045184192223863, + "tokens_seen": 371735552 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004481544633901705, + "loss": 3.0335, + "theoretical_loss": 4.045103428930275, + "tokens_seen": 371801088 + }, + { + "epoch": 1.02, + "learning_rate": 0.00044814443329989974, + "loss": 3.166, + "theoretical_loss": 4.045022683856517, + "tokens_seen": 371866624 + }, + { + "epoch": 1.02, + "objective/train/docs_used": 483681, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6399648189544678, + "objective/train/theoretical_loss": 4.044962137003555, + "objective/train/tokens_used": 392375776, + "theoretical_loss": 4.044962137003555, + "tokens_seen": 371915776 + }, + { + "epoch": 1.02, + "learning_rate": 0.00044813440320962887, + "loss": 2.9421, + "theoretical_loss": 4.044941956995265, + "tokens_seen": 371932160 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004481243731193581, + "loss": 3.0246, + "theoretical_loss": 4.044861248339206, + "tokens_seen": 371997696 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004481143430290873, + "loss": 3.1591, + "theoretical_loss": 4.044780557881026, + "tokens_seen": 372063232 + }, + { + "epoch": 1.02, + "learning_rate": 0.00044810431293881647, + "loss": 2.8982, + "theoretical_loss": 4.04469988561342, + "tokens_seen": 372128768 + }, + { + "epoch": 1.02, + "learning_rate": 0.00044809428284854565, + "loss": 2.7799, + "theoretical_loss": 4.044619231529083, + "tokens_seen": 372194304 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004480842527582749, + "loss": 3.1344, + "theoretical_loss": 4.0445385956207165, + "tokens_seen": 372259840 + }, + { + "epoch": 1.02, + "learning_rate": 0.000448074222668004, + "loss": 2.8566, + "theoretical_loss": 4.0444579778810255, + "tokens_seen": 372325376 + }, + { + "epoch": 1.02, + "learning_rate": 0.00044806419257773324, + "loss": 2.9545, + "theoretical_loss": 4.04437737830272, + "tokens_seen": 372390912 + }, + { + "epoch": 1.02, + "learning_rate": 0.00044805416248746237, + "loss": 3.1218, + "theoretical_loss": 4.044296796878512, + "tokens_seen": 372456448 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004480441323971916, + "loss": 3.2154, + "theoretical_loss": 4.044216233601119, + "tokens_seen": 372521984 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004480341023069208, + "loss": 3.1728, + "theoretical_loss": 4.044135688463267, + "tokens_seen": 372587520 + }, + { + "epoch": 1.02, + "learning_rate": 0.00044802407221664997, + "loss": 2.8922, + "theoretical_loss": 4.044055161457678, + "tokens_seen": 372653056 + }, + { + "epoch": 1.02, + "learning_rate": 0.00044801404212637915, + "loss": 3.1435, + "theoretical_loss": 4.0439746525770826, + "tokens_seen": 372718592 + }, + { + "epoch": 1.02, + "learning_rate": 0.00044800401203610833, + "loss": 2.858, + "theoretical_loss": 4.043894161814217, + "tokens_seen": 372784128 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004479939819458375, + "loss": 2.9771, + "theoretical_loss": 4.043813689161819, + "tokens_seen": 372849664 + }, + { + "epoch": 1.02, + "learning_rate": 0.00044798395185556675, + "loss": 3.1192, + "theoretical_loss": 4.043733234612633, + "tokens_seen": 372915200 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004479739217652959, + "loss": 3.0501, + "theoretical_loss": 4.043652798159405, + "tokens_seen": 372980736 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004479638916750251, + "loss": 3.1939, + "theoretical_loss": 4.0435723797948855, + "tokens_seen": 373046272 + }, + { + "epoch": 1.02, + "learning_rate": 0.00044795386158475424, + "loss": 3.0302, + "theoretical_loss": 4.043491979511831, + "tokens_seen": 373111808 + }, + { + "epoch": 1.02, + "learning_rate": 0.00044794383149448347, + "loss": 2.9471, + "theoretical_loss": 4.043411597303, + "tokens_seen": 373177344 + }, + { + "epoch": 1.02, + "learning_rate": 0.00044793380140421265, + "loss": 2.924, + "theoretical_loss": 4.043331233161158, + "tokens_seen": 373242880 + }, + { + "epoch": 1.02, + "learning_rate": 0.00044792377131394183, + "loss": 2.9933, + "theoretical_loss": 4.043250887079072, + "tokens_seen": 373308416 + }, + { + "epoch": 1.02, + "learning_rate": 0.000447913741223671, + "loss": 3.0149, + "theoretical_loss": 4.043170559049514, + "tokens_seen": 373373952 + }, + { + "epoch": 1.02, + "learning_rate": 0.00044790371113340025, + "loss": 2.4948, + "theoretical_loss": 4.04309024906526, + "tokens_seen": 373439488 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004478936810431294, + "loss": 3.101, + "theoretical_loss": 4.043009957119092, + "tokens_seen": 373505024 + }, + { + "epoch": 1.02, + "objective/train/docs_used": 485048, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.050812244415283, + "objective/train/theoretical_loss": 4.042949749992618, + "objective/train/tokens_used": 394014176, + "theoretical_loss": 4.042949749992618, + "tokens_seen": 373554176 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004478836509528586, + "loss": 3.1085, + "theoretical_loss": 4.042929683203793, + "tokens_seen": 373570560 + }, + { + "epoch": 1.02, + "learning_rate": 0.00044787362086258774, + "loss": 3.0532, + "theoretical_loss": 4.042849427312152, + "tokens_seen": 373636096 + }, + { + "epoch": 1.02, + "learning_rate": 0.000447863590772317, + "loss": 3.0551, + "theoretical_loss": 4.042769189436964, + "tokens_seen": 373701632 + }, + { + "epoch": 1.02, + "learning_rate": 0.00044785356068204616, + "loss": 3.2651, + "theoretical_loss": 4.042688969571023, + "tokens_seen": 373767168 + }, + { + "epoch": 1.02, + "learning_rate": 0.00044784353059177534, + "loss": 3.2799, + "theoretical_loss": 4.042608767707131, + "tokens_seen": 373832704 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004478335005015045, + "loss": 2.9216, + "theoretical_loss": 4.042528583838095, + "tokens_seen": 373898240 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004478234704112337, + "loss": 2.8679, + "theoretical_loss": 4.042448417956722, + "tokens_seen": 373963776 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004478134403209629, + "loss": 3.1769, + "theoretical_loss": 4.042368270055828, + "tokens_seen": 374029312 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004478034102306921, + "loss": 3.1326, + "theoretical_loss": 4.042288140128229, + "tokens_seen": 374094848 + }, + { + "epoch": 1.02, + "learning_rate": 0.00044779338014042124, + "loss": 3.1083, + "theoretical_loss": 4.042208028166748, + "tokens_seen": 374160384 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004477833500501505, + "loss": 3.0138, + "theoretical_loss": 4.04212793416421, + "tokens_seen": 374225920 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004477733199598796, + "loss": 2.8839, + "theoretical_loss": 4.042047858113446, + "tokens_seen": 374291456 + }, + { + "epoch": 1.02, + "learning_rate": 0.00044776328986960884, + "loss": 3.1341, + "theoretical_loss": 4.041967800007289, + "tokens_seen": 374356992 + }, + { + "epoch": 1.02, + "learning_rate": 0.000447753259779338, + "loss": 2.8194, + "theoretical_loss": 4.041887759838579, + "tokens_seen": 374422528 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004477432296890672, + "loss": 3.1354, + "theoretical_loss": 4.041807737600157, + "tokens_seen": 374488064 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004477331995987964, + "loss": 2.9551, + "theoretical_loss": 4.041727733284871, + "tokens_seen": 374553600 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004477231695085256, + "loss": 3.1074, + "theoretical_loss": 4.04164774688557, + "tokens_seen": 374619136 + }, + { + "epoch": 1.02, + "learning_rate": 0.00044771313941825475, + "loss": 3.0928, + "theoretical_loss": 4.04156777839511, + "tokens_seen": 374684672 + }, + { + "epoch": 1.02, + "learning_rate": 0.000447703109327984, + "loss": 2.9912, + "theoretical_loss": 4.041487827806349, + "tokens_seen": 374750208 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004476930792377131, + "loss": 3.1209, + "theoretical_loss": 4.04140789511215, + "tokens_seen": 374815744 + }, + { + "epoch": 1.02, + "learning_rate": 0.00044768304914744234, + "loss": 3.1041, + "theoretical_loss": 4.041327980305381, + "tokens_seen": 374881280 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004476730190571715, + "loss": 3.141, + "theoretical_loss": 4.041248083378912, + "tokens_seen": 374946816 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004476629889669007, + "loss": 3.1131, + "theoretical_loss": 4.041168204325619, + "tokens_seen": 375012352 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004476529588766299, + "loss": 3.1638, + "theoretical_loss": 4.041088343138382, + "tokens_seen": 375077888 + }, + { + "epoch": 1.02, + "learning_rate": 0.00044764292878635907, + "loss": 3.3661, + "theoretical_loss": 4.041008499810082, + "tokens_seen": 375143424 + }, + { + "epoch": 1.02, + "objective/train/docs_used": 485815, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.0911853313446045, + "objective/train/theoretical_loss": 4.040948629029507, + "objective/train/tokens_used": 395652576, + "theoretical_loss": 4.040948629029507, + "tokens_seen": 375192576 + }, + { + "epoch": 1.02, + "learning_rate": 0.00044763289869608825, + "loss": 3.0192, + "theoretical_loss": 4.040928674333609, + "tokens_seen": 375208960 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004476228686058175, + "loss": 3.1861, + "theoretical_loss": 4.040848866701853, + "tokens_seen": 375274496 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004476128385155466, + "loss": 3.2503, + "theoretical_loss": 4.04076907690771, + "tokens_seen": 375340032 + }, + { + "epoch": 1.02, + "learning_rate": 0.00044760280842527585, + "loss": 3.1357, + "theoretical_loss": 4.040689304944079, + "tokens_seen": 375405568 + }, + { + "epoch": 1.02, + "learning_rate": 0.00044759277833500503, + "loss": 3.0223, + "theoretical_loss": 4.0406095508038655, + "tokens_seen": 375471104 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004475827482447342, + "loss": 2.9539, + "theoretical_loss": 4.040529814479976, + "tokens_seen": 375536640 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004475727181544634, + "loss": 3.148, + "theoretical_loss": 4.040450095965323, + "tokens_seen": 375602176 + }, + { + "epoch": 1.02, + "learning_rate": 0.00044756268806419257, + "loss": 2.8196, + "theoretical_loss": 4.040370395252821, + "tokens_seen": 375667712 + }, + { + "epoch": 1.02, + "learning_rate": 0.00044755265797392175, + "loss": 3.2455, + "theoretical_loss": 4.040290712335391, + "tokens_seen": 375733248 + }, + { + "epoch": 1.02, + "learning_rate": 0.000447542627883651, + "loss": 3.2465, + "theoretical_loss": 4.0402110472059585, + "tokens_seen": 375798784 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004475325977933801, + "loss": 3.1096, + "theoretical_loss": 4.040131399857449, + "tokens_seen": 375864320 + }, + { + "epoch": 1.02, + "learning_rate": 0.00044752256770310935, + "loss": 3.0396, + "theoretical_loss": 4.040051770282796, + "tokens_seen": 375929856 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004475125376128385, + "loss": 3.2297, + "theoretical_loss": 4.039972158474936, + "tokens_seen": 375995392 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004475025075225677, + "loss": 3.0009, + "theoretical_loss": 4.0398925644268076, + "tokens_seen": 376060928 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004474924774322969, + "loss": 2.9304, + "theoretical_loss": 4.039812988131357, + "tokens_seen": 376126464 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004474824473420261, + "loss": 3.1235, + "theoretical_loss": 4.03973342958153, + "tokens_seen": 376192000 + }, + { + "epoch": 1.02, + "learning_rate": 0.00044747241725175526, + "loss": 2.9068, + "theoretical_loss": 4.039653888770282, + "tokens_seen": 376257536 + }, + { + "epoch": 1.02, + "learning_rate": 0.00044746238716148444, + "loss": 3.0073, + "theoretical_loss": 4.039574365690567, + "tokens_seen": 376323072 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004474523570712136, + "loss": 2.8717, + "theoretical_loss": 4.0394948603353455, + "tokens_seen": 376388608 + }, + { + "epoch": 1.02, + "learning_rate": 0.00044744232698094285, + "loss": 2.926, + "theoretical_loss": 4.039415372697583, + "tokens_seen": 376454144 + }, + { + "epoch": 1.02, + "learning_rate": 0.00044743229689067204, + "loss": 2.7771, + "theoretical_loss": 4.039335902770247, + "tokens_seen": 376519680 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004474222668004012, + "loss": 2.9918, + "theoretical_loss": 4.03925645054631, + "tokens_seen": 376585216 + }, + { + "epoch": 1.02, + "learning_rate": 0.00044741223671013045, + "loss": 2.9529, + "theoretical_loss": 4.039177016018749, + "tokens_seen": 376650752 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004474022066198596, + "loss": 2.9578, + "theoretical_loss": 4.039097599180543, + "tokens_seen": 376716288 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004473921765295888, + "loss": 2.9367, + "theoretical_loss": 4.039018200024678, + "tokens_seen": 376781824 + }, + { + "epoch": 1.02, + "objective/train/docs_used": 486601, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.832075834274292, + "objective/train/theoretical_loss": 4.038958662257597, + "objective/train/tokens_used": 397290976, + "theoretical_loss": 4.038958662257597, + "tokens_seen": 376830976 + }, + { + "epoch": 1.02, + "learning_rate": 0.00044738214643931794, + "loss": 2.9877, + "theoretical_loss": 4.038938818544143, + "tokens_seen": 376847360 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004473721163490472, + "loss": 2.8239, + "theoretical_loss": 4.038859454731927, + "tokens_seen": 376912896 + }, + { + "epoch": 1.02, + "learning_rate": 0.00044736208625877636, + "loss": 3.158, + "theoretical_loss": 4.0387801085810295, + "tokens_seen": 376978432 + }, + { + "epoch": 1.02, + "learning_rate": 0.00044735205616850554, + "loss": 3.0955, + "theoretical_loss": 4.038700780084451, + "tokens_seen": 377043968 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004473420260782347, + "loss": 3.0649, + "theoretical_loss": 4.038621469235193, + "tokens_seen": 377109504 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004473319959879639, + "loss": 2.9452, + "theoretical_loss": 4.038542176026267, + "tokens_seen": 377175040 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004473219658976931, + "loss": 3.0891, + "theoretical_loss": 4.038462900450684, + "tokens_seen": 377240576 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004473119358074223, + "loss": 3.1298, + "theoretical_loss": 4.03838364250146, + "tokens_seen": 377306112 + }, + { + "epoch": 1.02, + "learning_rate": 0.00044730190571715144, + "loss": 3.1469, + "theoretical_loss": 4.038304402171616, + "tokens_seen": 377371648 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004472918756268807, + "loss": 3.1611, + "theoretical_loss": 4.038225179454177, + "tokens_seen": 377437184 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004472818455366098, + "loss": 3.165, + "theoretical_loss": 4.038145974342169, + "tokens_seen": 377502720 + }, + { + "epoch": 1.02, + "learning_rate": 0.00044727181544633904, + "loss": 2.916, + "theoretical_loss": 4.038066786828626, + "tokens_seen": 377568256 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004472617853560682, + "loss": 2.9843, + "theoretical_loss": 4.037987616906584, + "tokens_seen": 377633792 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004472517552657974, + "loss": 3.0845, + "theoretical_loss": 4.037908464569083, + "tokens_seen": 377699328 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004472417251755266, + "loss": 3.1241, + "theoretical_loss": 4.037829329809167, + "tokens_seen": 377764864 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004472316950852558, + "loss": 3.1601, + "theoretical_loss": 4.037750212619883, + "tokens_seen": 377830400 + }, + { + "epoch": 1.02, + "learning_rate": 0.00044722166499498495, + "loss": 3.0712, + "theoretical_loss": 4.037671112994286, + "tokens_seen": 377895936 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004472116349047142, + "loss": 2.8286, + "theoretical_loss": 4.037592030925429, + "tokens_seen": 377961472 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004472016048144433, + "loss": 2.8222, + "theoretical_loss": 4.037512966406375, + "tokens_seen": 378027008 + }, + { + "epoch": 1.02, + "learning_rate": 0.00044719157472417254, + "loss": 2.9329, + "theoretical_loss": 4.0374339194301845, + "tokens_seen": 378092544 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004471815446339017, + "loss": 3.0938, + "theoretical_loss": 4.037354889989928, + "tokens_seen": 378158080 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004471715145436309, + "loss": 3.2318, + "theoretical_loss": 4.037275878078676, + "tokens_seen": 378223616 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004471614844533601, + "loss": 2.9692, + "theoretical_loss": 4.037196883689504, + "tokens_seen": 378289152 + }, + { + "epoch": 1.02, + "learning_rate": 0.00044715145436308927, + "loss": 3.0049, + "theoretical_loss": 4.037117906815491, + "tokens_seen": 378354688 + }, + { + "epoch": 1.02, + "learning_rate": 0.00044714142427281845, + "loss": 3.2511, + "theoretical_loss": 4.037038947449723, + "tokens_seen": 378420224 + }, + { + "epoch": 1.02, + "objective/train/docs_used": 487344, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.918959140777588, + "objective/train/theoretical_loss": 4.036979739411024, + "objective/train/tokens_used": 398929376, + "theoretical_loss": 4.036979739411024, + "tokens_seen": 378469376 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004471313941825477, + "loss": 3.0742, + "theoretical_loss": 4.036960005585286, + "tokens_seen": 378485760 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004471213640922768, + "loss": 3.0475, + "theoretical_loss": 4.036881081215271, + "tokens_seen": 378551296 + }, + { + "epoch": 1.02, + "learning_rate": 0.00044711133400200605, + "loss": 2.8166, + "theoretical_loss": 4.036802174332774, + "tokens_seen": 378616832 + }, + { + "epoch": 1.02, + "learning_rate": 0.00044710130391173523, + "loss": 3.1203, + "theoretical_loss": 4.036723284930894, + "tokens_seen": 378682368 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004470912738214644, + "loss": 3.0322, + "theoretical_loss": 4.036644413002735, + "tokens_seen": 378747904 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004470812437311936, + "loss": 3.1208, + "theoretical_loss": 4.036565558541404, + "tokens_seen": 378813440 + }, + { + "epoch": 1.02, + "learning_rate": 0.00044707121364092277, + "loss": 2.895, + "theoretical_loss": 4.03648672154001, + "tokens_seen": 378878976 + }, + { + "epoch": 1.02, + "learning_rate": 0.00044706118355065195, + "loss": 3.1827, + "theoretical_loss": 4.03640790199167, + "tokens_seen": 378944512 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004470511534603812, + "loss": 3.1239, + "theoretical_loss": 4.036329099889502, + "tokens_seen": 379010048 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004470411233701103, + "loss": 3.0161, + "theoretical_loss": 4.03625031522663, + "tokens_seen": 379075584 + }, + { + "epoch": 1.02, + "learning_rate": 0.00044703109327983955, + "loss": 3.126, + "theoretical_loss": 4.036171547996179, + "tokens_seen": 379141120 + }, + { + "epoch": 1.02, + "learning_rate": 0.0004470210631895687, + "loss": 3.2521, + "theoretical_loss": 4.036092798191279, + "tokens_seen": 379206656 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004470110330992979, + "loss": 3.1731, + "theoretical_loss": 4.036014065805067, + "tokens_seen": 379272192 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004470010030090271, + "loss": 3.1899, + "theoretical_loss": 4.035935350830679, + "tokens_seen": 379337728 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004469909729187563, + "loss": 3.0432, + "theoretical_loss": 4.035856653261258, + "tokens_seen": 379403264 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044698094282848546, + "loss": 3.1162, + "theoretical_loss": 4.03577797308995, + "tokens_seen": 379468800 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044697091273821464, + "loss": 2.9898, + "theoretical_loss": 4.035699310309906, + "tokens_seen": 379534336 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004469608826479438, + "loss": 3.2695, + "theoretical_loss": 4.035620664914278, + "tokens_seen": 379599872 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044695085255767305, + "loss": 3.017, + "theoretical_loss": 4.035542036896226, + "tokens_seen": 379665408 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004469408224674022, + "loss": 2.9715, + "theoretical_loss": 4.035463426248908, + "tokens_seen": 379730944 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004469307923771314, + "loss": 2.9837, + "theoretical_loss": 4.0353848329654936, + "tokens_seen": 379796480 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004469207622868606, + "loss": 3.0837, + "theoretical_loss": 4.035306257039149, + "tokens_seen": 379862016 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004469107321965898, + "loss": 2.937, + "theoretical_loss": 4.03522769846305, + "tokens_seen": 379927552 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044690070210631896, + "loss": 3.0365, + "theoretical_loss": 4.0351491572303715, + "tokens_seen": 379993088 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044689067201604814, + "loss": 3.1037, + "theoretical_loss": 4.035070633334295, + "tokens_seen": 380058624 + }, + { + "epoch": 1.03, + "objective/train/docs_used": 488451, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9471192359924316, + "objective/train/theoretical_loss": 4.035011751785283, + "objective/train/tokens_used": 400567776, + "theoretical_loss": 4.035011751785283, + "tokens_seen": 380107776 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004468806419257773, + "loss": 3.079, + "theoretical_loss": 4.034992126768006, + "tokens_seen": 380124160 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044687061183550656, + "loss": 2.9451, + "theoretical_loss": 4.034913637524692, + "tokens_seen": 380189696 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004468605817452357, + "loss": 3.2163, + "theoretical_loss": 4.0348351655975465, + "tokens_seen": 380255232 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004468505516549649, + "loss": 2.9821, + "theoretical_loss": 4.0347567109797655, + "tokens_seen": 380320768 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044684052156469405, + "loss": 3.075, + "theoretical_loss": 4.034678273664548, + "tokens_seen": 380386304 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004468304914744233, + "loss": 2.9896, + "theoretical_loss": 4.0345998536451, + "tokens_seen": 380451840 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044682046138415246, + "loss": 2.702, + "theoretical_loss": 4.034521450914628, + "tokens_seen": 380517376 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044681043129388164, + "loss": 2.9938, + "theoretical_loss": 4.034443065466345, + "tokens_seen": 380582912 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004468004012036108, + "loss": 3.0962, + "theoretical_loss": 4.034364697293466, + "tokens_seen": 380648448 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044679037111334, + "loss": 3.1166, + "theoretical_loss": 4.034286346389209, + "tokens_seen": 380713984 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004467803410230692, + "loss": 3.0233, + "theoretical_loss": 4.034208012746798, + "tokens_seen": 380779520 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004467703109327984, + "loss": 3.026, + "theoretical_loss": 4.03412969635946, + "tokens_seen": 380845056 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044676028084252755, + "loss": 2.9949, + "theoretical_loss": 4.034051397220427, + "tokens_seen": 380910592 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004467502507522568, + "loss": 3.1725, + "theoretical_loss": 4.033973115322932, + "tokens_seen": 380976128 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044674022066198597, + "loss": 3.2379, + "theoretical_loss": 4.033894850660214, + "tokens_seen": 381041664 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044673019057171515, + "loss": 3.2136, + "theoretical_loss": 4.0338166032255165, + "tokens_seen": 381107200 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044672016048144433, + "loss": 3.0469, + "theoretical_loss": 4.033738373012083, + "tokens_seen": 381172736 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004467101303911735, + "loss": 3.164, + "theoretical_loss": 4.0336601600131665, + "tokens_seen": 381238272 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004467001003009027, + "loss": 3.0613, + "theoretical_loss": 4.033581964222018, + "tokens_seen": 381303808 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004466900702106319, + "loss": 3.239, + "theoretical_loss": 4.033503785631897, + "tokens_seen": 381369344 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004466800401203611, + "loss": 2.5838, + "theoretical_loss": 4.033425624236063, + "tokens_seen": 381434880 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004466700100300903, + "loss": 3.1374, + "theoretical_loss": 4.033347480027782, + "tokens_seen": 381500416 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044665997993981947, + "loss": 2.9139, + "theoretical_loss": 4.033269353000324, + "tokens_seen": 381565952 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044664994984954865, + "loss": 3.1137, + "theoretical_loss": 4.03319124314696, + "tokens_seen": 381631488 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004466399197592779, + "loss": 2.9559, + "theoretical_loss": 4.0331131504609665, + "tokens_seen": 381697024 + }, + { + "epoch": 1.03, + "objective/train/docs_used": 489837, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9685873985290527, + "objective/train/theoretical_loss": 4.033054592208517, + "objective/train/tokens_used": 402206176, + "theoretical_loss": 4.033054592208517, + "tokens_seen": 381746176 + }, + { + "epoch": 1.03, + "learning_rate": 0.000446629889669007, + "loss": 3.1153, + "theoretical_loss": 4.033035074935626, + "tokens_seen": 381762560 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044661985957873625, + "loss": 2.9333, + "theoretical_loss": 4.03295701656422, + "tokens_seen": 381828096 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044660982948846543, + "loss": 2.8328, + "theoretical_loss": 4.0328789753400365, + "tokens_seen": 381893632 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004465997993981946, + "loss": 2.9213, + "theoretical_loss": 4.032800951256369, + "tokens_seen": 381959168 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004465897693079238, + "loss": 2.9061, + "theoretical_loss": 4.032722944306511, + "tokens_seen": 382024704 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044657973921765297, + "loss": 3.0426, + "theoretical_loss": 4.032644954483763, + "tokens_seen": 382090240 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044656970912738215, + "loss": 2.8363, + "theoretical_loss": 4.032566981781427, + "tokens_seen": 382155776 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004465596790371114, + "loss": 3.1766, + "theoretical_loss": 4.03248902619281, + "tokens_seen": 382221312 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004465496489468405, + "loss": 2.9881, + "theoretical_loss": 4.032411087711223, + "tokens_seen": 382286848 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044653961885656975, + "loss": 3.2702, + "theoretical_loss": 4.032333166329979, + "tokens_seen": 382352384 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004465295887662989, + "loss": 2.9304, + "theoretical_loss": 4.032255262042397, + "tokens_seen": 382417920 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004465195586760281, + "loss": 2.8589, + "theoretical_loss": 4.0321773748417975, + "tokens_seen": 382483456 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004465095285857573, + "loss": 2.8999, + "theoretical_loss": 4.032099504721508, + "tokens_seen": 382548992 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004464994984954865, + "loss": 3.0983, + "theoretical_loss": 4.032021651674857, + "tokens_seen": 382614528 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044648946840521566, + "loss": 3.0967, + "theoretical_loss": 4.0319438156951755, + "tokens_seen": 382680064 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044647943831494484, + "loss": 3.0136, + "theoretical_loss": 4.031865996775803, + "tokens_seen": 382745600 + }, + { + "epoch": 1.03, + "learning_rate": 0.000446469408224674, + "loss": 3.0933, + "theoretical_loss": 4.031788194910078, + "tokens_seen": 382811136 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044645937813440325, + "loss": 2.915, + "theoretical_loss": 4.031710410091346, + "tokens_seen": 382876672 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004464493480441324, + "loss": 2.9882, + "theoretical_loss": 4.031632642312955, + "tokens_seen": 382942208 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004464393179538616, + "loss": 3.0723, + "theoretical_loss": 4.031554891568255, + "tokens_seen": 383007744 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004464292878635908, + "loss": 3.1333, + "theoretical_loss": 4.031477157850603, + "tokens_seen": 383073280 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044641925777332, + "loss": 3.0266, + "theoretical_loss": 4.031399441153358, + "tokens_seen": 383138816 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044640922768304916, + "loss": 3.0681, + "theoretical_loss": 4.031321741469881, + "tokens_seen": 383204352 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044639919759277834, + "loss": 3.0067, + "theoretical_loss": 4.031244058793542, + "tokens_seen": 383269888 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004463891675025075, + "loss": 2.9739, + "theoretical_loss": 4.031166393117708, + "tokens_seen": 383335424 + }, + { + "epoch": 1.03, + "objective/train/docs_used": 490505, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.2893757820129395, + "objective/train/theoretical_loss": 4.031108155013429, + "objective/train/tokens_used": 403844576, + "theoretical_loss": 4.031108155013429, + "tokens_seen": 383384576 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044637913741223676, + "loss": 3.1077, + "theoretical_loss": 4.031088744435754, + "tokens_seen": 383400960 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004463691073219659, + "loss": 3.155, + "theoretical_loss": 4.0310111127410595, + "tokens_seen": 383466496 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004463590772316951, + "loss": 3.1105, + "theoretical_loss": 4.030933498027005, + "tokens_seen": 383532032 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044634904714142425, + "loss": 3.181, + "theoretical_loss": 4.030855900286974, + "tokens_seen": 383597568 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004463390170511535, + "loss": 2.9759, + "theoretical_loss": 4.030778319514357, + "tokens_seen": 383663104 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044632898696088266, + "loss": 3.1096, + "theoretical_loss": 4.030700755702545, + "tokens_seen": 383728640 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044631895687061184, + "loss": 3.0576, + "theoretical_loss": 4.030623208844937, + "tokens_seen": 383794176 + }, + { + "epoch": 1.03, + "learning_rate": 0.000446308926780341, + "loss": 3.1402, + "theoretical_loss": 4.030545678934931, + "tokens_seen": 383859712 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004462988966900702, + "loss": 3.1986, + "theoretical_loss": 4.030468165965931, + "tokens_seen": 383925248 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004462888665997994, + "loss": 3.0959, + "theoretical_loss": 4.030390669931345, + "tokens_seen": 383990784 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004462788365095286, + "loss": 2.9131, + "theoretical_loss": 4.030313190824582, + "tokens_seen": 384056320 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044626880641925775, + "loss": 3.1382, + "theoretical_loss": 4.030235728639059, + "tokens_seen": 384121856 + }, + { + "epoch": 1.03, + "learning_rate": 0.000446258776328987, + "loss": 3.0705, + "theoretical_loss": 4.030158283368193, + "tokens_seen": 384187392 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044624874623871617, + "loss": 2.8131, + "theoretical_loss": 4.030080855005408, + "tokens_seen": 384252928 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044623871614844535, + "loss": 3.0487, + "theoretical_loss": 4.030003443544127, + "tokens_seen": 384318464 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044622868605817453, + "loss": 3.2123, + "theoretical_loss": 4.0299260489777815, + "tokens_seen": 384384000 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004462186559679037, + "loss": 3.0109, + "theoretical_loss": 4.0298486712998045, + "tokens_seen": 384449536 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004462086258776329, + "loss": 3.0026, + "theoretical_loss": 4.029771310503632, + "tokens_seen": 384515072 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004461985957873621, + "loss": 3.0486, + "theoretical_loss": 4.029693966582705, + "tokens_seen": 384580608 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044618856569709125, + "loss": 3.036, + "theoretical_loss": 4.029616639530467, + "tokens_seen": 384646144 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004461785356068205, + "loss": 2.7565, + "theoretical_loss": 4.029539329340366, + "tokens_seen": 384711680 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004461685055165496, + "loss": 3.1694, + "theoretical_loss": 4.029462036005855, + "tokens_seen": 384777216 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044615847542627885, + "loss": 2.9726, + "theoretical_loss": 4.029384759520386, + "tokens_seen": 384842752 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044614844533600803, + "loss": 2.9842, + "theoretical_loss": 4.0293074998774205, + "tokens_seen": 384908288 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004461384152457372, + "loss": 2.8077, + "theoretical_loss": 4.02923025707042, + "tokens_seen": 384973824 + }, + { + "epoch": 1.03, + "objective/train/docs_used": 491392, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.1775639057159424, + "objective/train/theoretical_loss": 4.029172336009841, + "objective/train/tokens_used": 405482976, + "theoretical_loss": 4.029172336009841, + "tokens_seen": 385022976 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004461283851554664, + "loss": 2.9078, + "theoretical_loss": 4.029153031092851, + "tokens_seen": 385039360 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044611835506519563, + "loss": 2.9131, + "theoretical_loss": 4.029075821938182, + "tokens_seen": 385104896 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044610832497492476, + "loss": 3.0688, + "theoretical_loss": 4.028998629599888, + "tokens_seen": 385170432 + }, + { + "epoch": 1.03, + "learning_rate": 0.000446098294884654, + "loss": 3.0794, + "theoretical_loss": 4.028921454071445, + "tokens_seen": 385235968 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004460882647943831, + "loss": 3.0845, + "theoretical_loss": 4.0288442953463335, + "tokens_seen": 385301504 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044607823470411235, + "loss": 3.1702, + "theoretical_loss": 4.028767153418038, + "tokens_seen": 385367040 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044606820461384153, + "loss": 3.2518, + "theoretical_loss": 4.028690028280046, + "tokens_seen": 385432576 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004460581745235707, + "loss": 2.9465, + "theoretical_loss": 4.02861291992585, + "tokens_seen": 385498112 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004460481444332999, + "loss": 3.3349, + "theoretical_loss": 4.028535828348945, + "tokens_seen": 385563648 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004460381143430291, + "loss": 2.9932, + "theoretical_loss": 4.02845875354283, + "tokens_seen": 385629184 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044602808425275826, + "loss": 3.0814, + "theoretical_loss": 4.0283816955010066, + "tokens_seen": 385694720 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004460180541624875, + "loss": 2.9856, + "theoretical_loss": 4.0283046542169805, + "tokens_seen": 385760256 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004460080240722166, + "loss": 3.2441, + "theoretical_loss": 4.028227629684263, + "tokens_seen": 385825792 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044599799398194586, + "loss": 3.0764, + "theoretical_loss": 4.028150621896366, + "tokens_seen": 385891328 + }, + { + "epoch": 1.03, + "learning_rate": 0.000445987963891675, + "loss": 3.0117, + "theoretical_loss": 4.028073630846807, + "tokens_seen": 385956864 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004459779338014042, + "loss": 2.9714, + "theoretical_loss": 4.0279966565291065, + "tokens_seen": 386022400 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004459679037111334, + "loss": 2.8873, + "theoretical_loss": 4.027919698936788, + "tokens_seen": 386087936 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004459578736208626, + "loss": 3.2267, + "theoretical_loss": 4.02784275806338, + "tokens_seen": 386153472 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044594784353059176, + "loss": 3.2652, + "theoretical_loss": 4.027765833902413, + "tokens_seen": 386219008 + }, + { + "epoch": 1.03, + "learning_rate": 0.000445937813440321, + "loss": 3.0323, + "theoretical_loss": 4.027688926447423, + "tokens_seen": 386284544 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004459277833500502, + "loss": 3.0881, + "theoretical_loss": 4.0276120356919485, + "tokens_seen": 386350080 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044591775325977936, + "loss": 3.2264, + "theoretical_loss": 4.02753516162953, + "tokens_seen": 386415616 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044590772316950854, + "loss": 3.1626, + "theoretical_loss": 4.027458304253715, + "tokens_seen": 386481152 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004458976930792377, + "loss": 3.2694, + "theoretical_loss": 4.027381463558052, + "tokens_seen": 386546688 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044588766298896696, + "loss": 2.8776, + "theoretical_loss": 4.027304639536093, + "tokens_seen": 386612224 + }, + { + "epoch": 1.03, + "objective/train/docs_used": 492041, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7662007808685303, + "objective/train/theoretical_loss": 4.027247032457867, + "objective/train/tokens_used": 407121376, + "theoretical_loss": 4.027247032457867, + "tokens_seen": 386661376 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004458776328986961, + "loss": 2.9938, + "theoretical_loss": 4.027227832181397, + "tokens_seen": 386677760 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004458676028084253, + "loss": 2.8942, + "theoretical_loss": 4.027151041487521, + "tokens_seen": 386743296 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044585757271815445, + "loss": 3.0691, + "theoretical_loss": 4.027074267448031, + "tokens_seen": 386808832 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004458475426278837, + "loss": 3.1324, + "theoretical_loss": 4.026997510056493, + "tokens_seen": 386874368 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044583751253761286, + "loss": 3.1412, + "theoretical_loss": 4.026920769306478, + "tokens_seen": 386939904 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044582748244734204, + "loss": 2.9207, + "theoretical_loss": 4.0268440451915595, + "tokens_seen": 387005440 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004458174523570712, + "loss": 2.9868, + "theoretical_loss": 4.0267673377053175, + "tokens_seen": 387070976 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004458074222668004, + "loss": 3.0294, + "theoretical_loss": 4.026690646841332, + "tokens_seen": 387136512 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004457973921765296, + "loss": 2.9402, + "theoretical_loss": 4.026613972593188, + "tokens_seen": 387202048 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004457873620862588, + "loss": 3.1123, + "theoretical_loss": 4.026537314954475, + "tokens_seen": 387267584 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044577733199598795, + "loss": 3.0472, + "theoretical_loss": 4.026460673918785, + "tokens_seen": 387333120 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004457673019057172, + "loss": 2.9787, + "theoretical_loss": 4.026384049479713, + "tokens_seen": 387398656 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044575727181544637, + "loss": 2.9231, + "theoretical_loss": 4.026307441630858, + "tokens_seen": 387464192 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044574724172517555, + "loss": 2.9785, + "theoretical_loss": 4.026230850365824, + "tokens_seen": 387529728 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044573721163490473, + "loss": 3.0627, + "theoretical_loss": 4.026154275678216, + "tokens_seen": 387595264 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004457271815446339, + "loss": 3.1657, + "theoretical_loss": 4.0260777175616465, + "tokens_seen": 387660800 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004457171514543631, + "loss": 2.9694, + "theoretical_loss": 4.026001176009726, + "tokens_seen": 387726336 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004457071213640923, + "loss": 3.1933, + "theoretical_loss": 4.025924651016073, + "tokens_seen": 387791872 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044569709127382145, + "loss": 2.8754, + "theoretical_loss": 4.025848142574307, + "tokens_seen": 387857408 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004456870611835507, + "loss": 3.1072, + "theoretical_loss": 4.025771650678053, + "tokens_seen": 387922944 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004456770310932798, + "loss": 3.3678, + "theoretical_loss": 4.025695175320939, + "tokens_seen": 387988480 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044566700100300905, + "loss": 3.0147, + "theoretical_loss": 4.025618716496595, + "tokens_seen": 388054016 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044565697091273823, + "loss": 3.3046, + "theoretical_loss": 4.025542274198656, + "tokens_seen": 388119552 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004456469408224674, + "loss": 3.2766, + "theoretical_loss": 4.025465848420762, + "tokens_seen": 388185088 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004456369107321966, + "loss": 3.0493, + "theoretical_loss": 4.025389439156552, + "tokens_seen": 388250624 + }, + { + "epoch": 1.03, + "objective/train/docs_used": 492346, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.0495290756225586, + "objective/train/theoretical_loss": 4.025332143041677, + "objective/train/tokens_used": 408759776, + "theoretical_loss": 4.025332143041677, + "tokens_seen": 388299776 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044562688064192583, + "loss": 3.0685, + "theoretical_loss": 4.025313046399672, + "tokens_seen": 388316160 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044561685055165496, + "loss": 3.2269, + "theoretical_loss": 4.025236670143771, + "tokens_seen": 388381696 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004456068204613842, + "loss": 3.2541, + "theoretical_loss": 4.025160310382502, + "tokens_seen": 388447232 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004455967903711133, + "loss": 2.9198, + "theoretical_loss": 4.02508396710952, + "tokens_seen": 388512768 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044558676028084255, + "loss": 2.9715, + "theoretical_loss": 4.025007640318485, + "tokens_seen": 388578304 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044557673019057173, + "loss": 3.109, + "theoretical_loss": 4.024931330003058, + "tokens_seen": 388643840 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004455667001003009, + "loss": 3.1998, + "theoretical_loss": 4.024855036156907, + "tokens_seen": 388709376 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004455566700100301, + "loss": 3.1695, + "theoretical_loss": 4.024778758773702, + "tokens_seen": 388774912 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004455466399197593, + "loss": 2.9735, + "theoretical_loss": 4.024702497847115, + "tokens_seen": 388840448 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044553660982948846, + "loss": 3.1217, + "theoretical_loss": 4.0246262533708235, + "tokens_seen": 388905984 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004455265797392177, + "loss": 3.1701, + "theoretical_loss": 4.024550025338508, + "tokens_seen": 388971520 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004455165496489468, + "loss": 3.2158, + "theoretical_loss": 4.024473813743851, + "tokens_seen": 389037056 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044550651955867606, + "loss": 3.3209, + "theoretical_loss": 4.024397618580542, + "tokens_seen": 389102592 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004454964894684052, + "loss": 3.1834, + "theoretical_loss": 4.02432143984227, + "tokens_seen": 389168128 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004454864593781344, + "loss": 2.9146, + "theoretical_loss": 4.024245277522729, + "tokens_seen": 389233664 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004454764292878636, + "loss": 3.3165, + "theoretical_loss": 4.0241691316156185, + "tokens_seen": 389299200 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004454663991975928, + "loss": 3.4463, + "theoretical_loss": 4.024093002114638, + "tokens_seen": 389364736 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044545636910732196, + "loss": 3.1539, + "theoretical_loss": 4.0240168890134935, + "tokens_seen": 389430272 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004454463390170512, + "loss": 3.547, + "theoretical_loss": 4.0239407923058925, + "tokens_seen": 389495808 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004454363089267803, + "loss": 3.1451, + "theoretical_loss": 4.023864711985547, + "tokens_seen": 389561344 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044542627883650956, + "loss": 3.1436, + "theoretical_loss": 4.023788648046171, + "tokens_seen": 389626880 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004454162487462387, + "loss": 2.9621, + "theoretical_loss": 4.023712600481484, + "tokens_seen": 389692416 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004454062186559679, + "loss": 3.042, + "theoretical_loss": 4.023636569285208, + "tokens_seen": 389757952 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004453961885656971, + "loss": 3.1537, + "theoretical_loss": 4.023560554451068, + "tokens_seen": 389823488 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004453861584754263, + "loss": 3.1769, + "theoretical_loss": 4.023484555972794, + "tokens_seen": 389889024 + }, + { + "epoch": 1.03, + "objective/train/docs_used": 492346, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.5270936489105225, + "objective/train/theoretical_loss": 4.0234275678438545, + "objective/train/tokens_used": 410398176, + "theoretical_loss": 4.0234275678438545, + "tokens_seen": 389938176 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044537612838515547, + "loss": 3.1988, + "theoretical_loss": 4.023408573844117, + "tokens_seen": 389954560 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044536609829488465, + "loss": 2.8549, + "theoretical_loss": 4.023332608058775, + "tokens_seen": 390020096 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044535606820461383, + "loss": 3.1868, + "theoretical_loss": 4.023256658610505, + "tokens_seen": 390085632 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044534603811434306, + "loss": 3.2635, + "theoretical_loss": 4.02318072549305, + "tokens_seen": 390151168 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004453360080240722, + "loss": 3.1127, + "theoretical_loss": 4.0231048087001575, + "tokens_seen": 390216704 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004453259779338014, + "loss": 3.2178, + "theoretical_loss": 4.023028908225577, + "tokens_seen": 390282240 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044531594784353055, + "loss": 3.1681, + "theoretical_loss": 4.02295302406306, + "tokens_seen": 390347776 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004453059177532598, + "loss": 3.1644, + "theoretical_loss": 4.0228771562063645, + "tokens_seen": 390413312 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044529588766298897, + "loss": 3.3908, + "theoretical_loss": 4.022801304649251, + "tokens_seen": 390478848 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044528585757271815, + "loss": 3.2459, + "theoretical_loss": 4.022725469385481, + "tokens_seen": 390544384 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044527582748244733, + "loss": 2.9103, + "theoretical_loss": 4.022649650408823, + "tokens_seen": 390609920 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044526579739217657, + "loss": 3.2225, + "theoretical_loss": 4.022573847713048, + "tokens_seen": 390675456 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004452557673019057, + "loss": 2.9859, + "theoretical_loss": 4.022498061291927, + "tokens_seen": 390740992 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044524573721163493, + "loss": 3.1762, + "theoretical_loss": 4.022422291139238, + "tokens_seen": 390806528 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044523570712136406, + "loss": 3.3121, + "theoretical_loss": 4.022346537248763, + "tokens_seen": 390872064 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004452256770310933, + "loss": 3.2159, + "theoretical_loss": 4.022270799614285, + "tokens_seen": 390937600 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044521564694082247, + "loss": 3.1719, + "theoretical_loss": 4.02219507822959, + "tokens_seen": 391003136 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044520561685055165, + "loss": 3.325, + "theoretical_loss": 4.022119373088472, + "tokens_seen": 391068672 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044519558676028083, + "loss": 3.2354, + "theoretical_loss": 4.022043684184723, + "tokens_seen": 391134208 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044518555667001, + "loss": 3.0924, + "theoretical_loss": 4.021968011512141, + "tokens_seen": 391199744 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044517552657973925, + "loss": 3.1403, + "theoretical_loss": 4.0218923550645265, + "tokens_seen": 391265280 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044516549648946843, + "loss": 3.2618, + "theoretical_loss": 4.021816714835685, + "tokens_seen": 391330816 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004451554663991976, + "loss": 3.1097, + "theoretical_loss": 4.0217410908194235, + "tokens_seen": 391396352 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004451454363089268, + "loss": 3.0826, + "theoretical_loss": 4.021665483009555, + "tokens_seen": 391461888 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044513540621865603, + "loss": 3.325, + "theoretical_loss": 4.021589891399891, + "tokens_seen": 391527424 + }, + { + "epoch": 1.03, + "objective/train/docs_used": 493102, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.2480216026306152, + "objective/train/theoretical_loss": 4.02153320832031, + "objective/train/tokens_used": 412036576, + "theoretical_loss": 4.02153320832031, + "tokens_seen": 391576576 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044512537612838516, + "loss": 3.1819, + "theoretical_loss": 4.021514315984252, + "tokens_seen": 391592960 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004451153460381144, + "loss": 3.0459, + "theoretical_loss": 4.021438756756459, + "tokens_seen": 391658496 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004451053159478435, + "loss": 3.2661, + "theoretical_loss": 4.021363213710337, + "tokens_seen": 391724032 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044509528585757275, + "loss": 3.1196, + "theoretical_loss": 4.021287686839712, + "tokens_seen": 391789568 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044508525576730194, + "loss": 3.0289, + "theoretical_loss": 4.021212176138419, + "tokens_seen": 391855104 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004450752256770311, + "loss": 3.4092, + "theoretical_loss": 4.02113668160029, + "tokens_seen": 391920640 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004450651955867603, + "loss": 3.0095, + "theoretical_loss": 4.0210612032191655, + "tokens_seen": 391986176 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004450551654964895, + "loss": 3.417, + "theoretical_loss": 4.020985740988887, + "tokens_seen": 392051712 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044504513540621866, + "loss": 3.3539, + "theoretical_loss": 4.020910294903298, + "tokens_seen": 392117248 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004450351053159479, + "loss": 3.3295, + "theoretical_loss": 4.020834864956248, + "tokens_seen": 392182784 + }, + { + "epoch": 1.03, + "learning_rate": 0.000445025075225677, + "loss": 3.4475, + "theoretical_loss": 4.020759451141589, + "tokens_seen": 392248320 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044501504513540626, + "loss": 3.405, + "theoretical_loss": 4.020684053453176, + "tokens_seen": 392313856 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004450050150451354, + "loss": 3.3016, + "theoretical_loss": 4.020608671884868, + "tokens_seen": 392379392 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004449949849548646, + "loss": 3.223, + "theoretical_loss": 4.020533306430527, + "tokens_seen": 392444928 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004449849548645938, + "loss": 3.0813, + "theoretical_loss": 4.020457957084017, + "tokens_seen": 392510464 + }, + { + "epoch": 1.03, + "learning_rate": 0.000444974924774323, + "loss": 3.2733, + "theoretical_loss": 4.0203826238392075, + "tokens_seen": 392576000 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044496489468405216, + "loss": 3.4012, + "theoretical_loss": 4.020307306689972, + "tokens_seen": 392641536 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004449548645937814, + "loss": 2.9446, + "theoretical_loss": 4.020232005630183, + "tokens_seen": 392707072 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004449448345035105, + "loss": 2.8994, + "theoretical_loss": 4.020156720653722, + "tokens_seen": 392772608 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044493480441323976, + "loss": 3.1714, + "theoretical_loss": 4.020081451754469, + "tokens_seen": 392838144 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004449247743229689, + "loss": 3.33, + "theoretical_loss": 4.0200061989263105, + "tokens_seen": 392903680 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004449147442326981, + "loss": 3.2939, + "theoretical_loss": 4.019930962163135, + "tokens_seen": 392969216 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004449047141424273, + "loss": 3.3044, + "theoretical_loss": 4.019855741458835, + "tokens_seen": 393034752 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004448946840521565, + "loss": 3.1703, + "theoretical_loss": 4.019780536807305, + "tokens_seen": 393100288 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044488465396188567, + "loss": 3.1619, + "theoretical_loss": 4.019705348202446, + "tokens_seen": 393165824 + }, + { + "epoch": 1.03, + "objective/train/docs_used": 494542, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9207966327667236, + "objective/train/theoretical_loss": 4.019648967275759, + "objective/train/tokens_used": 413674976, + "theoretical_loss": 4.019648967275759, + "tokens_seen": 393214976 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044487462387161485, + "loss": 3.1161, + "theoretical_loss": 4.019630175638158, + "tokens_seen": 393231360 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044486459378134403, + "loss": 3.1258, + "theoretical_loss": 4.0195550191083464, + "tokens_seen": 393296896 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044485456369107326, + "loss": 3.12, + "theoretical_loss": 4.019479878606921, + "tokens_seen": 393362432 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004448445336008024, + "loss": 3.2797, + "theoretical_loss": 4.019404754127793, + "tokens_seen": 393427968 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004448345035105316, + "loss": 3.1239, + "theoretical_loss": 4.01932964566488, + "tokens_seen": 393493504 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044482447342026075, + "loss": 3.118, + "theoretical_loss": 4.0192545532120985, + "tokens_seen": 393559040 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044481444332999, + "loss": 3.1753, + "theoretical_loss": 4.01917947676337, + "tokens_seen": 393624576 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044480441323971917, + "loss": 3.1646, + "theoretical_loss": 4.019104416312623, + "tokens_seen": 393690112 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044479438314944835, + "loss": 3.0934, + "theoretical_loss": 4.019029371853784, + "tokens_seen": 393755648 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044478435305917753, + "loss": 3.1208, + "theoretical_loss": 4.018954343380786, + "tokens_seen": 393821184 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044477432296890677, + "loss": 3.0607, + "theoretical_loss": 4.018879330887563, + "tokens_seen": 393886720 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004447642928786359, + "loss": 3.2782, + "theoretical_loss": 4.018804334368055, + "tokens_seen": 393952256 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044475426278836513, + "loss": 3.16, + "theoretical_loss": 4.018729353816203, + "tokens_seen": 394017792 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044474423269809426, + "loss": 3.2467, + "theoretical_loss": 4.018654389225954, + "tokens_seen": 394083328 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004447342026078235, + "loss": 3.3955, + "theoretical_loss": 4.018579440591255, + "tokens_seen": 394148864 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044472417251755267, + "loss": 3.2859, + "theoretical_loss": 4.018504507906059, + "tokens_seen": 394214400 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044471414242728185, + "loss": 3.3852, + "theoretical_loss": 4.0184295911643195, + "tokens_seen": 394279936 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044470411233701103, + "loss": 3.437, + "theoretical_loss": 4.018354690359996, + "tokens_seen": 394345472 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004446940822467402, + "loss": 3.277, + "theoretical_loss": 4.01827980548705, + "tokens_seen": 394411008 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004446840521564694, + "loss": 3.3371, + "theoretical_loss": 4.018204936539448, + "tokens_seen": 394476544 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044467402206619863, + "loss": 3.2478, + "theoretical_loss": 4.018130083511156, + "tokens_seen": 394542080 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044466399197592776, + "loss": 3.2044, + "theoretical_loss": 4.018055246396147, + "tokens_seen": 394607616 + }, + { + "epoch": 1.03, + "learning_rate": 0.000444653961885657, + "loss": 3.2246, + "theoretical_loss": 4.017980425188396, + "tokens_seen": 394673152 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004446439317953861, + "loss": 3.0988, + "theoretical_loss": 4.017905619881881, + "tokens_seen": 394738688 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044463390170511536, + "loss": 3.0712, + "theoretical_loss": 4.017830830470583, + "tokens_seen": 394804224 + }, + { + "epoch": 1.03, + "objective/train/docs_used": 495175, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.2577452659606934, + "objective/train/theoretical_loss": 4.017774748839727, + "objective/train/tokens_used": 415313376, + "theoretical_loss": 4.017774748839727, + "tokens_seen": 394853376 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044462387161484454, + "loss": 3.1142, + "theoretical_loss": 4.017756056948487, + "tokens_seen": 394869760 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004446138415245737, + "loss": 3.3469, + "theoretical_loss": 4.017681299309582, + "tokens_seen": 394935296 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004446038114343029, + "loss": 3.1336, + "theoretical_loss": 4.017606557547858, + "tokens_seen": 395000832 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044459378134403214, + "loss": 2.8472, + "theoretical_loss": 4.01753183165731, + "tokens_seen": 395066368 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044458375125376126, + "loss": 3.1497, + "theoretical_loss": 4.017457121631936, + "tokens_seen": 395131904 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004445737211634905, + "loss": 3.2117, + "theoretical_loss": 4.017382427465737, + "tokens_seen": 395197440 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004445636910732196, + "loss": 2.946, + "theoretical_loss": 4.017307749152717, + "tokens_seen": 395262976 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044455366098294886, + "loss": 3.2121, + "theoretical_loss": 4.017233086686884, + "tokens_seen": 395328512 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044454363089267804, + "loss": 2.9614, + "theoretical_loss": 4.017158440062249, + "tokens_seen": 395394048 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004445336008024072, + "loss": 3.1452, + "theoretical_loss": 4.017083809272826, + "tokens_seen": 395459584 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004445235707121364, + "loss": 2.9447, + "theoretical_loss": 4.017009194312633, + "tokens_seen": 395525120 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004445135406218656, + "loss": 2.99, + "theoretical_loss": 4.016934595175689, + "tokens_seen": 395590656 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044450351053159477, + "loss": 3.0288, + "theoretical_loss": 4.01686001185602, + "tokens_seen": 395656192 + }, + { + "epoch": 1.03, + "learning_rate": 0.000444493480441324, + "loss": 3.3945, + "theoretical_loss": 4.016785444347652, + "tokens_seen": 395721728 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044448345035105313, + "loss": 3.2868, + "theoretical_loss": 4.016710892644614, + "tokens_seen": 395787264 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044447342026078236, + "loss": 3.281, + "theoretical_loss": 4.016636356740943, + "tokens_seen": 395852800 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004444633901705115, + "loss": 3.1532, + "theoretical_loss": 4.016561836630672, + "tokens_seen": 395918336 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004444533600802407, + "loss": 3.2487, + "theoretical_loss": 4.016487332307844, + "tokens_seen": 395983872 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044444332998996996, + "loss": 3.1453, + "theoretical_loss": 4.016412843766502, + "tokens_seen": 396049408 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004444332998996991, + "loss": 3.1682, + "theoretical_loss": 4.016338371000691, + "tokens_seen": 396114944 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004444232698094283, + "loss": 3.1292, + "theoretical_loss": 4.016263914004462, + "tokens_seen": 396180480 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004444132397191575, + "loss": 3.4037, + "theoretical_loss": 4.016189472771868, + "tokens_seen": 396246016 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004444032096288867, + "loss": 3.0772, + "theoretical_loss": 4.016115047296965, + "tokens_seen": 396311552 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044439317953861587, + "loss": 3.2491, + "theoretical_loss": 4.016040637573812, + "tokens_seen": 396377088 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044438314944834505, + "loss": 3.2143, + "theoretical_loss": 4.0159662435964725, + "tokens_seen": 396442624 + }, + { + "epoch": 1.03, + "objective/train/docs_used": 496403, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9757843017578125, + "objective/train/theoretical_loss": 4.015910458443088, + "objective/train/tokens_used": 416951776, + "theoretical_loss": 4.015910458443088, + "tokens_seen": 396491776 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044437311935807423, + "loss": 2.9239, + "theoretical_loss": 4.015891865359012, + "tokens_seen": 396508160 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044436308926780346, + "loss": 2.9201, + "theoretical_loss": 4.0158175028555, + "tokens_seen": 396573696 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004443530591775326, + "loss": 3.2929, + "theoretical_loss": 4.015743156080008, + "tokens_seen": 396639232 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004443430290872618, + "loss": 2.9639, + "theoretical_loss": 4.015668825026613, + "tokens_seen": 396704768 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044433299899699095, + "loss": 3.1691, + "theoretical_loss": 4.015594509689393, + "tokens_seen": 396770304 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004443229689067202, + "loss": 3.0509, + "theoretical_loss": 4.015520210062429, + "tokens_seen": 396835840 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044431293881644937, + "loss": 3.3591, + "theoretical_loss": 4.015445926139808, + "tokens_seen": 396901376 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044430290872617855, + "loss": 3.1861, + "theoretical_loss": 4.015371657915617, + "tokens_seen": 396966912 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044429287863590773, + "loss": 3.2149, + "theoretical_loss": 4.015297405383948, + "tokens_seen": 397032448 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044428284854563697, + "loss": 3.2117, + "theoretical_loss": 4.015223168538896, + "tokens_seen": 397097984 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004442728184553661, + "loss": 3.1278, + "theoretical_loss": 4.015148947374559, + "tokens_seen": 397163520 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044426278836509533, + "loss": 3.1822, + "theoretical_loss": 4.015074741885038, + "tokens_seen": 397229056 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044425275827482446, + "loss": 3.0979, + "theoretical_loss": 4.015000552064437, + "tokens_seen": 397294592 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004442427281845537, + "loss": 3.0718, + "theoretical_loss": 4.014926377906864, + "tokens_seen": 397360128 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044423269809428287, + "loss": 3.0584, + "theoretical_loss": 4.01485221940643, + "tokens_seen": 397425664 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044422266800401205, + "loss": 3.2676, + "theoretical_loss": 4.014778076557249, + "tokens_seen": 397491200 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044421263791374123, + "loss": 3.1287, + "theoretical_loss": 4.0147039493534376, + "tokens_seen": 397556736 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004442026078234704, + "loss": 3.184, + "theoretical_loss": 4.014629837789116, + "tokens_seen": 397622272 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004441925777331996, + "loss": 3.3226, + "theoretical_loss": 4.014555741858407, + "tokens_seen": 397687808 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044418254764292883, + "loss": 3.1903, + "theoretical_loss": 4.014481661555439, + "tokens_seen": 397753344 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044417251755265796, + "loss": 3.1214, + "theoretical_loss": 4.0144075968743405, + "tokens_seen": 397818880 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004441624874623872, + "loss": 3.2588, + "theoretical_loss": 4.014333547809246, + "tokens_seen": 397884416 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004441524573721163, + "loss": 3.0024, + "theoretical_loss": 4.01425951435429, + "tokens_seen": 397949952 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044414242728184556, + "loss": 3.0515, + "theoretical_loss": 4.014185496503612, + "tokens_seen": 398015488 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044413239719157474, + "loss": 3.0877, + "theoretical_loss": 4.014111494251355, + "tokens_seen": 398081024 + }, + { + "epoch": 1.03, + "objective/train/docs_used": 497345, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9470760822296143, + "objective/train/theoretical_loss": 4.014056002795105, + "objective/train/tokens_used": 418590176, + "theoretical_loss": 4.014056002795105, + "tokens_seen": 398130176 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004441223671013039, + "loss": 3.09, + "theoretical_loss": 4.014037507591665, + "tokens_seen": 398146560 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004441123370110331, + "loss": 3.3322, + "theoretical_loss": 4.01396353651869, + "tokens_seen": 398212096 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044410230692076234, + "loss": 3.3269, + "theoretical_loss": 4.013889581026583, + "tokens_seen": 398277632 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044409227683049146, + "loss": 3.1256, + "theoretical_loss": 4.013815641109498, + "tokens_seen": 398343168 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004440822467402207, + "loss": 2.9061, + "theoretical_loss": 4.0137417167615945, + "tokens_seen": 398408704 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004440722166499498, + "loss": 2.7541, + "theoretical_loss": 4.013667807977033, + "tokens_seen": 398474240 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044406218655967906, + "loss": 2.8525, + "theoretical_loss": 4.013593914749977, + "tokens_seen": 398539776 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044405215646940824, + "loss": 3.2677, + "theoretical_loss": 4.013520037074597, + "tokens_seen": 398605312 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004440421263791374, + "loss": 3.069, + "theoretical_loss": 4.013446174945062, + "tokens_seen": 398670848 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004440320962888666, + "loss": 3.0376, + "theoretical_loss": 4.013372328355547, + "tokens_seen": 398736384 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004440220661985958, + "loss": 3.0581, + "theoretical_loss": 4.0132984973002275, + "tokens_seen": 398801920 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044401203610832497, + "loss": 3.1568, + "theoretical_loss": 4.013224681773286, + "tokens_seen": 398867456 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004440020060180542, + "loss": 3.2621, + "theoretical_loss": 4.0131508817689046, + "tokens_seen": 398932992 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044399197592778333, + "loss": 3.0157, + "theoretical_loss": 4.013077097281269, + "tokens_seen": 398998528 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044398194583751256, + "loss": 3.3744, + "theoretical_loss": 4.013003328304571, + "tokens_seen": 399064064 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004439719157472417, + "loss": 3.2167, + "theoretical_loss": 4.012929574833002, + "tokens_seen": 399129600 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004439618856569709, + "loss": 3.2892, + "theoretical_loss": 4.012855836860759, + "tokens_seen": 399195136 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004439518555667001, + "loss": 3.3375, + "theoretical_loss": 4.01278211438204, + "tokens_seen": 399260672 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004439418254764293, + "loss": 3.1236, + "theoretical_loss": 4.012708407391049, + "tokens_seen": 399326208 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044393179538615847, + "loss": 2.9568, + "theoretical_loss": 4.012634715881989, + "tokens_seen": 399391744 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004439217652958877, + "loss": 3.2874, + "theoretical_loss": 4.012561039849069, + "tokens_seen": 399457280 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044391173520561683, + "loss": 3.1477, + "theoretical_loss": 4.012487379286502, + "tokens_seen": 399522816 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044390170511534607, + "loss": 2.9509, + "theoretical_loss": 4.012413734188501, + "tokens_seen": 399588352 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004438916750250752, + "loss": 3.1483, + "theoretical_loss": 4.012340104549285, + "tokens_seen": 399653888 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044388164493480443, + "loss": 3.11, + "theoretical_loss": 4.012266490363073, + "tokens_seen": 399719424 + }, + { + "epoch": 1.03, + "objective/train/docs_used": 502454, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.3102781772613525, + "objective/train/theoretical_loss": 4.012211289860975, + "objective/train/tokens_used": 420228576, + "theoretical_loss": 4.012211289860975, + "tokens_seen": 399768576 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004438716148445336, + "loss": 3.2869, + "theoretical_loss": 4.012192891624092, + "tokens_seen": 399784960 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004438615847542628, + "loss": 2.9235, + "theoretical_loss": 4.012119308326567, + "tokens_seen": 399850496 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044385155466399197, + "loss": 3.0863, + "theoretical_loss": 4.012045740464728, + "tokens_seen": 399916032 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044384152457372115, + "loss": 3.0504, + "theoretical_loss": 4.01197218803281, + "tokens_seen": 399981568 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044383149448345033, + "loss": 3.1842, + "theoretical_loss": 4.011898651025048, + "tokens_seen": 400047104 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044382146439317957, + "loss": 3.1272, + "theoretical_loss": 4.011825129435683, + "tokens_seen": 400112640 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004438114343029087, + "loss": 3.1282, + "theoretical_loss": 4.011751623258955, + "tokens_seen": 400178176 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044380140421263793, + "loss": 3.0844, + "theoretical_loss": 4.011678132489112, + "tokens_seen": 400243712 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004437913741223671, + "loss": 2.8656, + "theoretical_loss": 4.011604657120401, + "tokens_seen": 400309248 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004437813440320963, + "loss": 3.2295, + "theoretical_loss": 4.011531197147076, + "tokens_seen": 400374784 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004437713139418255, + "loss": 3.0765, + "theoretical_loss": 4.011457752563391, + "tokens_seen": 400440320 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044376128385155466, + "loss": 3.2233, + "theoretical_loss": 4.011384323363603, + "tokens_seen": 400505856 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044375125376128384, + "loss": 3.3812, + "theoretical_loss": 4.011310909541975, + "tokens_seen": 400571392 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044374122367101307, + "loss": 3.0616, + "theoretical_loss": 4.011237511092769, + "tokens_seen": 400636928 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004437311935807422, + "loss": 3.1718, + "theoretical_loss": 4.011164128010255, + "tokens_seen": 400702464 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044372116349047143, + "loss": 3.0474, + "theoretical_loss": 4.011090760288701, + "tokens_seen": 400768000 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044371113340020056, + "loss": 3.0441, + "theoretical_loss": 4.011017407922381, + "tokens_seen": 400833536 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004437011033099298, + "loss": 3.1871, + "theoretical_loss": 4.010944070905572, + "tokens_seen": 400899072 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044369107321965903, + "loss": 3.3112, + "theoretical_loss": 4.010870749232554, + "tokens_seen": 400964608 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044368104312938816, + "loss": 2.9837, + "theoretical_loss": 4.0107974428976085, + "tokens_seen": 401030144 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004436710130391174, + "loss": 3.4095, + "theoretical_loss": 4.010724151895022, + "tokens_seen": 401095680 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004436609829488465, + "loss": 3.2873, + "theoretical_loss": 4.010650876219083, + "tokens_seen": 401161216 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044365095285857576, + "loss": 3.1268, + "theoretical_loss": 4.010577615864083, + "tokens_seen": 401226752 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044364092276830494, + "loss": 3.0925, + "theoretical_loss": 4.010504370824316, + "tokens_seen": 401292288 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004436308926780341, + "loss": 3.0146, + "theoretical_loss": 4.010431141094083, + "tokens_seen": 401357824 + }, + { + "epoch": 1.03, + "objective/train/docs_used": 507639, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.0078208446502686, + "objective/train/theoretical_loss": 4.01037622883986, + "objective/train/tokens_used": 421866976, + "theoretical_loss": 4.01037622883986, + "tokens_seen": 401406976 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004436208625877633, + "loss": 2.9291, + "theoretical_loss": 4.010357926667682, + "tokens_seen": 401423360 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044361083249749254, + "loss": 3.3317, + "theoretical_loss": 4.0102847275394184, + "tokens_seen": 401488896 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044360080240722166, + "loss": 2.7376, + "theoretical_loss": 4.0102115437035994, + "tokens_seen": 401554432 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004435907723169509, + "loss": 3.2031, + "theoretical_loss": 4.0101383751545345, + "tokens_seen": 401619968 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044358074222668, + "loss": 3.1482, + "theoretical_loss": 4.010065221886537, + "tokens_seen": 401685504 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044357071213640926, + "loss": 3.0891, + "theoretical_loss": 4.0099920838939225, + "tokens_seen": 401751040 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044356068204613844, + "loss": 3.0714, + "theoretical_loss": 4.009918961171012, + "tokens_seen": 401816576 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004435506519558676, + "loss": 3.1246, + "theoretical_loss": 4.009845853712126, + "tokens_seen": 401882112 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004435406218655968, + "loss": 3.2701, + "theoretical_loss": 4.0097727615115915, + "tokens_seen": 401947648 + }, + { + "epoch": 1.03, + "learning_rate": 0.000443530591775326, + "loss": 3.082, + "theoretical_loss": 4.009699684563736, + "tokens_seen": 402013184 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044352056168505517, + "loss": 3.2898, + "theoretical_loss": 4.00962662286289, + "tokens_seen": 402078720 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004435105315947844, + "loss": 3.0541, + "theoretical_loss": 4.00955357640339, + "tokens_seen": 402144256 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044350050150451353, + "loss": 2.9924, + "theoretical_loss": 4.009480545179572, + "tokens_seen": 402209792 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044349047141424276, + "loss": 2.8212, + "theoretical_loss": 4.0094075291857765, + "tokens_seen": 402275328 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004434804413239719, + "loss": 2.8057, + "theoretical_loss": 4.009334528416347, + "tokens_seen": 402340864 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004434704112337011, + "loss": 3.1605, + "theoretical_loss": 4.009261542865631, + "tokens_seen": 402406400 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004434603811434303, + "loss": 3.3208, + "theoretical_loss": 4.009188572527977, + "tokens_seen": 402471936 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004434503510531595, + "loss": 3.2758, + "theoretical_loss": 4.009115617397739, + "tokens_seen": 402537472 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044344032096288867, + "loss": 3.1012, + "theoretical_loss": 4.009042677469271, + "tokens_seen": 402603008 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004434302908726179, + "loss": 3.0349, + "theoretical_loss": 4.008969752736932, + "tokens_seen": 402668544 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044342026078234703, + "loss": 3.3604, + "theoretical_loss": 4.0088968431950835, + "tokens_seen": 402734080 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044341023069207627, + "loss": 3.0932, + "theoretical_loss": 4.008823948838091, + "tokens_seen": 402799616 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004434002006018054, + "loss": 3.0842, + "theoretical_loss": 4.008751069660322, + "tokens_seen": 402865152 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044339017051153463, + "loss": 3.1979, + "theoretical_loss": 4.008678205656146, + "tokens_seen": 402930688 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004433801404212638, + "loss": 3.2418, + "theoretical_loss": 4.008605356819937, + "tokens_seen": 402996224 + }, + { + "epoch": 1.03, + "objective/train/docs_used": 512662, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.06799578666687, + "objective/train/theoretical_loss": 4.0085507301433765, + "objective/train/tokens_used": 423505376, + "theoretical_loss": 4.0085507301433765, + "tokens_seen": 403045376 + }, + { + "epoch": 1.03, + "learning_rate": 0.000443370110330993, + "loss": 3.2713, + "theoretical_loss": 4.008532523146073, + "tokens_seen": 403061760 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044336008024072217, + "loss": 3.0465, + "theoretical_loss": 4.008459704628931, + "tokens_seen": 403127296 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044335005015045135, + "loss": 3.1517, + "theoretical_loss": 4.008386901262897, + "tokens_seen": 403192832 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044334002006018053, + "loss": 3.1538, + "theoretical_loss": 4.0083141130423545, + "tokens_seen": 403258368 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044332998996990977, + "loss": 2.9468, + "theoretical_loss": 4.008241339961692, + "tokens_seen": 403323904 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004433199598796389, + "loss": 3.0336, + "theoretical_loss": 4.008168582015301, + "tokens_seen": 403389440 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044330992978936813, + "loss": 3.1535, + "theoretical_loss": 4.008095839197578, + "tokens_seen": 403454976 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004432998996990973, + "loss": 3.3492, + "theoretical_loss": 4.008023111502919, + "tokens_seen": 403520512 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004432898696088265, + "loss": 3.0574, + "theoretical_loss": 4.007950398925725, + "tokens_seen": 403586048 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004432798395185557, + "loss": 3.1388, + "theoretical_loss": 4.007877701460398, + "tokens_seen": 403651584 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044326980942828486, + "loss": 3.1729, + "theoretical_loss": 4.0078050191013475, + "tokens_seen": 403717120 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044325977933801404, + "loss": 2.9521, + "theoretical_loss": 4.007732351842981, + "tokens_seen": 403782656 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044324974924774327, + "loss": 3.2514, + "theoretical_loss": 4.007659699679711, + "tokens_seen": 403848192 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004432397191574724, + "loss": 2.8846, + "theoretical_loss": 4.007587062605954, + "tokens_seen": 403913728 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044322968906720163, + "loss": 3.2135, + "theoretical_loss": 4.007514440616128, + "tokens_seen": 403979264 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044321965897693076, + "loss": 2.8579, + "theoretical_loss": 4.007441833704654, + "tokens_seen": 404044800 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044320962888666, + "loss": 2.7827, + "theoretical_loss": 4.0073692418659554, + "tokens_seen": 404110336 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004431995987963892, + "loss": 3.0201, + "theoretical_loss": 4.007296665094462, + "tokens_seen": 404175872 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044318956870611836, + "loss": 3.2041, + "theoretical_loss": 4.007224103384603, + "tokens_seen": 404241408 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044317953861584754, + "loss": 2.8284, + "theoretical_loss": 4.007151556730811, + "tokens_seen": 404306944 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004431695085255767, + "loss": 3.0129, + "theoretical_loss": 4.007079025127523, + "tokens_seen": 404372480 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004431594784353059, + "loss": 3.0203, + "theoretical_loss": 4.007006508569178, + "tokens_seen": 404438016 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044314944834503514, + "loss": 3.1886, + "theoretical_loss": 4.0069340070502175, + "tokens_seen": 404503552 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044313941825476427, + "loss": 3.0306, + "theoretical_loss": 4.006861520565088, + "tokens_seen": 404569088 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004431293881644935, + "loss": 3.1072, + "theoretical_loss": 4.006789049108235, + "tokens_seen": 404634624 + }, + { + "epoch": 1.03, + "objective/train/docs_used": 517837, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.787900924682617, + "objective/train/theoretical_loss": 4.006734705374566, + "objective/train/tokens_used": 425143776, + "theoretical_loss": 4.006734705374566, + "tokens_seen": 404683776 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004431193580742227, + "loss": 2.9616, + "theoretical_loss": 4.006716592674112, + "tokens_seen": 404700160 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044310932798395186, + "loss": 3.1183, + "theoretical_loss": 4.006644151257173, + "tokens_seen": 404765696 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044309929789368104, + "loss": 2.8431, + "theoretical_loss": 4.006571724851873, + "tokens_seen": 404831232 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004430892678034102, + "loss": 2.9166, + "theoretical_loss": 4.0064993134526725, + "tokens_seen": 404896768 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004430792377131394, + "loss": 3.0856, + "theoretical_loss": 4.006426917054036, + "tokens_seen": 404962304 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044306920762286864, + "loss": 3.0765, + "theoretical_loss": 4.006354535650425, + "tokens_seen": 405027840 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044305917753259777, + "loss": 3.1055, + "theoretical_loss": 4.0062821692363135, + "tokens_seen": 405093376 + }, + { + "epoch": 1.03, + "learning_rate": 0.000443049147442327, + "loss": 3.0614, + "theoretical_loss": 4.00620981780617, + "tokens_seen": 405158912 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044303911735205613, + "loss": 3.0428, + "theoretical_loss": 4.006137481354468, + "tokens_seen": 405224448 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044302908726178537, + "loss": 3.0897, + "theoretical_loss": 4.006065159875687, + "tokens_seen": 405289984 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044301905717151455, + "loss": 2.9146, + "theoretical_loss": 4.005992853364307, + "tokens_seen": 405355520 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044300902708124373, + "loss": 3.0938, + "theoretical_loss": 4.005920561814811, + "tokens_seen": 405421056 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004429989969909729, + "loss": 3.0032, + "theoretical_loss": 4.005848285221685, + "tokens_seen": 405486592 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004429889669007021, + "loss": 2.8133, + "theoretical_loss": 4.005776023579418, + "tokens_seen": 405552128 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044297893681043127, + "loss": 2.9431, + "theoretical_loss": 4.005703776882503, + "tokens_seen": 405617664 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004429689067201605, + "loss": 3.0047, + "theoretical_loss": 4.005631545125434, + "tokens_seen": 405683200 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044295887662988963, + "loss": 2.989, + "theoretical_loss": 4.00555932830271, + "tokens_seen": 405748736 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044294884653961887, + "loss": 3.2523, + "theoretical_loss": 4.0054871264088305, + "tokens_seen": 405814272 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004429388164493481, + "loss": 3.0664, + "theoretical_loss": 4.0054149394383, + "tokens_seen": 405879808 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044292878635907723, + "loss": 3.2111, + "theoretical_loss": 4.005342767385625, + "tokens_seen": 405945344 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044291875626880647, + "loss": 2.8902, + "theoretical_loss": 4.005270610245315, + "tokens_seen": 406010880 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004429087261785356, + "loss": 3.4227, + "theoretical_loss": 4.005198468011882, + "tokens_seen": 406076416 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044289869608826483, + "loss": 3.1243, + "theoretical_loss": 4.005126340679842, + "tokens_seen": 406141952 + }, + { + "epoch": 1.03, + "learning_rate": 0.000442888665997994, + "loss": 3.1989, + "theoretical_loss": 4.005054228243713, + "tokens_seen": 406207488 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004428786359077232, + "loss": 2.9508, + "theoretical_loss": 4.004982130698017, + "tokens_seen": 406273024 + }, + { + "epoch": 1.03, + "objective/train/docs_used": 518449, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.878129482269287, + "objective/train/theoretical_loss": 4.004928067307296, + "objective/train/tokens_used": 426782176, + "theoretical_loss": 4.004928067307296, + "tokens_seen": 406322176 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044286860581745237, + "loss": 3.0946, + "theoretical_loss": 4.004910048037276, + "tokens_seen": 406338560 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044285857572718155, + "loss": 2.8832, + "theoretical_loss": 4.00483798025602, + "tokens_seen": 406404096 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044284854563691073, + "loss": 3.1023, + "theoretical_loss": 4.0047659273487755, + "tokens_seen": 406469632 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044283851554663997, + "loss": 3.0547, + "theoretical_loss": 4.004693889310077, + "tokens_seen": 406535168 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004428284854563691, + "loss": 3.1845, + "theoretical_loss": 4.00462186613446, + "tokens_seen": 406600704 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044281845536609833, + "loss": 3.0654, + "theoretical_loss": 4.004549857816463, + "tokens_seen": 406666240 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004428084252758275, + "loss": 3.0538, + "theoretical_loss": 4.004477864350627, + "tokens_seen": 406731776 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004427983951855567, + "loss": 3.0571, + "theoretical_loss": 4.004405885731497, + "tokens_seen": 406797312 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004427883650952859, + "loss": 3.0495, + "theoretical_loss": 4.00433392195362, + "tokens_seen": 406862848 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044277833500501506, + "loss": 3.1538, + "theoretical_loss": 4.004261973011546, + "tokens_seen": 406928384 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044276830491474424, + "loss": 2.9353, + "theoretical_loss": 4.004190038899827, + "tokens_seen": 406993920 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044275827482447347, + "loss": 2.8766, + "theoretical_loss": 4.004118119613019, + "tokens_seen": 407059456 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004427482447342026, + "loss": 3.0604, + "theoretical_loss": 4.004046215145682, + "tokens_seen": 407124992 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044273821464393184, + "loss": 3.2647, + "theoretical_loss": 4.003974325492377, + "tokens_seen": 407190528 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044272818455366096, + "loss": 3.1432, + "theoretical_loss": 4.003902450647669, + "tokens_seen": 407256064 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004427181544633902, + "loss": 3.0507, + "theoretical_loss": 4.003830590606123, + "tokens_seen": 407321600 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004427081243731194, + "loss": 3.0707, + "theoretical_loss": 4.003758745362311, + "tokens_seen": 407387136 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044269809428284856, + "loss": 3.1622, + "theoretical_loss": 4.003686914910807, + "tokens_seen": 407452672 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044268806419257774, + "loss": 3.146, + "theoretical_loss": 4.003615099246184, + "tokens_seen": 407518208 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004426780341023069, + "loss": 3.2283, + "theoretical_loss": 4.003543298363022, + "tokens_seen": 407583744 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004426680040120361, + "loss": 2.921, + "theoretical_loss": 4.003471512255905, + "tokens_seen": 407649280 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044265797392176534, + "loss": 3.1143, + "theoretical_loss": 4.003399740919413, + "tokens_seen": 407714816 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044264794383149447, + "loss": 3.0341, + "theoretical_loss": 4.003327984348138, + "tokens_seen": 407780352 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004426379137412237, + "loss": 3.1449, + "theoretical_loss": 4.003256242536667, + "tokens_seen": 407845888 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004426278836509529, + "loss": 2.9538, + "theoretical_loss": 4.003184515479592, + "tokens_seen": 407911424 + }, + { + "epoch": 1.03, + "objective/train/docs_used": 519624, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.0527055263519287, + "objective/train/theoretical_loss": 4.00313072986611, + "objective/train/tokens_used": 428420576, + "theoretical_loss": 4.00313072986611, + "tokens_seen": 407960576 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044261785356068206, + "loss": 3.0662, + "theoretical_loss": 4.003112803171513, + "tokens_seen": 407976960 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044260782347041124, + "loss": 2.9138, + "theoretical_loss": 4.003041105607026, + "tokens_seen": 408042496 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004425977933801404, + "loss": 3.1145, + "theoretical_loss": 4.002969422780733, + "tokens_seen": 408108032 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004425877632898696, + "loss": 3.0576, + "theoretical_loss": 4.002897754687238, + "tokens_seen": 408173568 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044257773319959884, + "loss": 3.2281, + "theoretical_loss": 4.0028261013211495, + "tokens_seen": 408239104 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044256770310932797, + "loss": 3.1774, + "theoretical_loss": 4.002754462677077, + "tokens_seen": 408304640 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004425576730190572, + "loss": 3.0006, + "theoretical_loss": 4.002682838749632, + "tokens_seen": 408370176 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044254764292878633, + "loss": 3.1701, + "theoretical_loss": 4.002611229533432, + "tokens_seen": 408435712 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044253761283851557, + "loss": 3.0723, + "theoretical_loss": 4.002539635023095, + "tokens_seen": 408501248 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044252758274824475, + "loss": 3.0007, + "theoretical_loss": 4.002468055213243, + "tokens_seen": 408566784 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044251755265797393, + "loss": 3.1507, + "theoretical_loss": 4.0023964900985, + "tokens_seen": 408632320 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004425075225677031, + "loss": 3.3785, + "theoretical_loss": 4.002324939673492, + "tokens_seen": 408697856 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004424974924774323, + "loss": 3.0969, + "theoretical_loss": 4.0022534039328495, + "tokens_seen": 408763392 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044248746238716147, + "loss": 3.1206, + "theoretical_loss": 4.002181882871206, + "tokens_seen": 408828928 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004424774322968907, + "loss": 2.8046, + "theoretical_loss": 4.002110376483197, + "tokens_seen": 408894464 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044246740220661983, + "loss": 2.9415, + "theoretical_loss": 4.002038884763459, + "tokens_seen": 408960000 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044245737211634907, + "loss": 2.9706, + "theoretical_loss": 4.001967407706635, + "tokens_seen": 409025536 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044244734202607825, + "loss": 2.9634, + "theoretical_loss": 4.0018959453073695, + "tokens_seen": 409091072 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044243731193580743, + "loss": 3.2558, + "theoretical_loss": 4.001824497560308, + "tokens_seen": 409156608 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004424272818455366, + "loss": 3.1996, + "theoretical_loss": 4.0017530644601, + "tokens_seen": 409222144 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004424172517552658, + "loss": 2.9816, + "theoretical_loss": 4.001681646001399, + "tokens_seen": 409287680 + }, + { + "epoch": 1.03, + "learning_rate": 0.000442407221664995, + "loss": 3.027, + "theoretical_loss": 4.0016102421788595, + "tokens_seen": 409353216 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004423971915747242, + "loss": 2.993, + "theoretical_loss": 4.00153885298714, + "tokens_seen": 409418752 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044238716148445334, + "loss": 2.9679, + "theoretical_loss": 4.001467478420901, + "tokens_seen": 409484288 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044237713139418257, + "loss": 3.3059, + "theoretical_loss": 4.001396118474806, + "tokens_seen": 409549824 + }, + { + "epoch": 1.03, + "objective/train/docs_used": 520212, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.317723512649536, + "objective/train/theoretical_loss": 4.001342608106496, + "objective/train/tokens_used": 430058976, + "theoretical_loss": 4.001342608106496, + "tokens_seen": 409598976 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004423671013039117, + "loss": 2.9615, + "theoretical_loss": 4.001324773143522, + "tokens_seen": 409615360 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044235707121364093, + "loss": 3.0724, + "theoretical_loss": 4.001253442421718, + "tokens_seen": 409680896 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004423470411233701, + "loss": 2.965, + "theoretical_loss": 4.001182126304067, + "tokens_seen": 409746432 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004423370110330993, + "loss": 3.0159, + "theoretical_loss": 4.001110824785242, + "tokens_seen": 409811968 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004423269809428285, + "loss": 2.9357, + "theoretical_loss": 4.001039537859921, + "tokens_seen": 409877504 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004423169508525577, + "loss": 3.4018, + "theoretical_loss": 4.000968265522786, + "tokens_seen": 409943040 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044230692076228684, + "loss": 3.1943, + "theoretical_loss": 4.000897007768518, + "tokens_seen": 410008576 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004422968906720161, + "loss": 2.9675, + "theoretical_loss": 4.000825764591807, + "tokens_seen": 410074112 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004422868605817452, + "loss": 3.1719, + "theoretical_loss": 4.000754535987337, + "tokens_seen": 410139648 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044227683049147444, + "loss": 3.1564, + "theoretical_loss": 4.0006833219498015, + "tokens_seen": 410205184 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004422668004012036, + "loss": 3.1727, + "theoretical_loss": 4.000612122473896, + "tokens_seen": 410270720 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004422567703109328, + "loss": 3.0706, + "theoretical_loss": 4.000540937554316, + "tokens_seen": 410336256 + }, + { + "epoch": 1.03, + "learning_rate": 0.000442246740220662, + "loss": 3.193, + "theoretical_loss": 4.000469767185763, + "tokens_seen": 410401792 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044223671013039116, + "loss": 2.8826, + "theoretical_loss": 4.000398611362938, + "tokens_seen": 410467328 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044222668004012034, + "loss": 3.0742, + "theoretical_loss": 4.0003274700805465, + "tokens_seen": 410532864 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004422166499498496, + "loss": 3.1663, + "theoretical_loss": 4.000256343333298, + "tokens_seen": 410598400 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004422066198595787, + "loss": 3.1183, + "theoretical_loss": 4.000185231115903, + "tokens_seen": 410663936 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044219658976930794, + "loss": 3.0318, + "theoretical_loss": 4.000114133423077, + "tokens_seen": 410729472 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004421865596790371, + "loss": 2.8639, + "theoretical_loss": 4.000043050249533, + "tokens_seen": 410795008 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004421765295887663, + "loss": 3.0937, + "theoretical_loss": 3.999971981589993, + "tokens_seen": 410860544 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044216649949849554, + "loss": 2.9955, + "theoretical_loss": 3.9999009274391772, + "tokens_seen": 410926080 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044215646940822467, + "loss": 3.2867, + "theoretical_loss": 3.9998298877918126, + "tokens_seen": 410991616 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004421464393179539, + "loss": 3.0194, + "theoretical_loss": 3.999758862642625, + "tokens_seen": 411057152 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004421364092276831, + "loss": 3.0622, + "theoretical_loss": 3.9996878519863452, + "tokens_seen": 411122688 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044212637913741226, + "loss": 3.0314, + "theoretical_loss": 3.999616855817707, + "tokens_seen": 411188224 + }, + { + "epoch": 1.03, + "objective/train/docs_used": 521565, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.3985636234283447, + "objective/train/theoretical_loss": 3.9995636181955763, + "objective/train/tokens_used": 431697376, + "theoretical_loss": 3.9995636181955763, + "tokens_seen": 411237376 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044211634904714144, + "loss": 3.0621, + "theoretical_loss": 3.999545874131446, + "tokens_seen": 411253760 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004421063189568706, + "loss": 3.194, + "theoretical_loss": 3.9994749069223, + "tokens_seen": 411319296 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004420962888665998, + "loss": 2.8937, + "theoretical_loss": 3.9994039541850115, + "tokens_seen": 411384832 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044208625877632904, + "loss": 2.9052, + "theoretical_loss": 3.9993330159143237, + "tokens_seen": 411450368 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044207622868605817, + "loss": 3.1371, + "theoretical_loss": 3.999262092104984, + "tokens_seen": 411515904 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004420661985957874, + "loss": 3.1621, + "theoretical_loss": 3.9991911827517415, + "tokens_seen": 411581440 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044205616850551653, + "loss": 2.9365, + "theoretical_loss": 3.9991202878493493, + "tokens_seen": 411646976 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044204613841524577, + "loss": 3.1542, + "theoretical_loss": 3.9990494073925618, + "tokens_seen": 411712512 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044203610832497495, + "loss": 3.4033, + "theoretical_loss": 3.9989785413761365, + "tokens_seen": 411778048 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044202607823470413, + "loss": 3.1086, + "theoretical_loss": 3.9989076897948355, + "tokens_seen": 411843584 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004420160481444333, + "loss": 2.9042, + "theoretical_loss": 3.99883685264342, + "tokens_seen": 411909120 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004420060180541625, + "loss": 3.1093, + "theoretical_loss": 3.9987660299166574, + "tokens_seen": 411974656 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044199598796389167, + "loss": 3.225, + "theoretical_loss": 3.998695221609317, + "tokens_seen": 412040192 + }, + { + "epoch": 1.03, + "learning_rate": 0.0004419859578736209, + "loss": 3.0794, + "theoretical_loss": 3.9986244277161687, + "tokens_seen": 412105728 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044197592778335003, + "loss": 2.9221, + "theoretical_loss": 3.9985536482319874, + "tokens_seen": 412171264 + }, + { + "epoch": 1.03, + "learning_rate": 0.00044196589769307927, + "loss": 3.2153, + "theoretical_loss": 3.99848288315155, + "tokens_seen": 412236800 + }, + { + "epoch": 1.04, + "learning_rate": 0.00044195586760280845, + "loss": 3.1729, + "theoretical_loss": 3.998412132469637, + "tokens_seen": 412302336 + }, + { + "epoch": 1.04, + "learning_rate": 0.00044194583751253763, + "loss": 3.0724, + "theoretical_loss": 3.9983413961810292, + "tokens_seen": 412367872 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004419358074222668, + "loss": 2.8081, + "theoretical_loss": 3.9982706742805125, + "tokens_seen": 412433408 + }, + { + "epoch": 1.04, + "learning_rate": 0.000441925777331996, + "loss": 2.9037, + "theoretical_loss": 3.998199966762875, + "tokens_seen": 412498944 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004419157472417252, + "loss": 3.138, + "theoretical_loss": 3.998129273622907, + "tokens_seen": 412564480 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004419057171514544, + "loss": 2.9746, + "theoretical_loss": 3.9980585948554013, + "tokens_seen": 412630016 + }, + { + "epoch": 1.04, + "learning_rate": 0.00044189568706118354, + "loss": 2.8928, + "theoretical_loss": 3.9979879304551544, + "tokens_seen": 412695552 + }, + { + "epoch": 1.04, + "learning_rate": 0.00044188565697091277, + "loss": 3.0935, + "theoretical_loss": 3.9979172804169654, + "tokens_seen": 412761088 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004418756268806419, + "loss": 3.2098, + "theoretical_loss": 3.997846644735634, + "tokens_seen": 412826624 + }, + { + "epoch": 1.04, + "objective/train/docs_used": 522883, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.032809019088745, + "objective/train/theoretical_loss": 3.997793677393199, + "objective/train/tokens_used": 433335776, + "theoretical_loss": 3.997793677393199, + "tokens_seen": 412875776 + }, + { + "epoch": 1.04, + "learning_rate": 0.00044186559679037113, + "loss": 2.9367, + "theoretical_loss": 3.997776023405966, + "tokens_seen": 412892160 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004418555667001003, + "loss": 3.0454, + "theoretical_loss": 3.997705416422767, + "tokens_seen": 412957696 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004418455366098295, + "loss": 2.9081, + "theoretical_loss": 3.9976348237808477, + "tokens_seen": 413023232 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004418355065195587, + "loss": 3.3134, + "theoretical_loss": 3.9975642454750195, + "tokens_seen": 413088768 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004418254764292879, + "loss": 3.0579, + "theoretical_loss": 3.9974936815000977, + "tokens_seen": 413154304 + }, + { + "epoch": 1.04, + "learning_rate": 0.00044181544633901704, + "loss": 3.1252, + "theoretical_loss": 3.997423131850899, + "tokens_seen": 413219840 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004418054162487463, + "loss": 3.1003, + "theoretical_loss": 3.9973525965222443, + "tokens_seen": 413285376 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004417953861584754, + "loss": 3.0599, + "theoretical_loss": 3.9972820755089566, + "tokens_seen": 413350912 + }, + { + "epoch": 1.04, + "learning_rate": 0.00044178535606820464, + "loss": 2.923, + "theoretical_loss": 3.9972115688058616, + "tokens_seen": 413416448 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004417753259779338, + "loss": 3.0775, + "theoretical_loss": 3.9971410764077877, + "tokens_seen": 413481984 + }, + { + "epoch": 1.04, + "learning_rate": 0.000441765295887663, + "loss": 2.9165, + "theoretical_loss": 3.997070598309566, + "tokens_seen": 413547520 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004417552657973922, + "loss": 2.9974, + "theoretical_loss": 3.9970001345060293, + "tokens_seen": 413613056 + }, + { + "epoch": 1.04, + "learning_rate": 0.00044174523570712136, + "loss": 3.001, + "theoretical_loss": 3.9969296849920153, + "tokens_seen": 413678592 + }, + { + "epoch": 1.04, + "learning_rate": 0.00044173520561685054, + "loss": 3.0008, + "theoretical_loss": 3.9968592497623625, + "tokens_seen": 413744128 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004417251755265798, + "loss": 3.11, + "theoretical_loss": 3.996788828811913, + "tokens_seen": 413809664 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004417151454363089, + "loss": 3.2571, + "theoretical_loss": 3.996718422135511, + "tokens_seen": 413875200 + }, + { + "epoch": 1.04, + "learning_rate": 0.00044170511534603814, + "loss": 2.9652, + "theoretical_loss": 3.996648029728003, + "tokens_seen": 413940736 + }, + { + "epoch": 1.04, + "learning_rate": 0.00044169508525576727, + "loss": 3.0139, + "theoretical_loss": 3.9965776515842397, + "tokens_seen": 414006272 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004416850551654965, + "loss": 2.9294, + "theoretical_loss": 3.9965072876990737, + "tokens_seen": 414071808 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004416750250752257, + "loss": 3.0109, + "theoretical_loss": 3.99643693806736, + "tokens_seen": 414137344 + }, + { + "epoch": 1.04, + "learning_rate": 0.00044166499498495487, + "loss": 3.1063, + "theoretical_loss": 3.9963666026839557, + "tokens_seen": 414202880 + }, + { + "epoch": 1.04, + "learning_rate": 0.00044165496489468405, + "loss": 3.2242, + "theoretical_loss": 3.996296281543722, + "tokens_seen": 414268416 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004416449348044133, + "loss": 2.8582, + "theoretical_loss": 3.996225974641522, + "tokens_seen": 414333952 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004416349047141424, + "loss": 2.8527, + "theoretical_loss": 3.9961556819722217, + "tokens_seen": 414399488 + }, + { + "epoch": 1.04, + "learning_rate": 0.00044162487462387164, + "loss": 3.037, + "theoretical_loss": 3.9960854035306888, + "tokens_seen": 414465024 + }, + { + "epoch": 1.04, + "objective/train/docs_used": 523694, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.1503584384918213, + "objective/train/theoretical_loss": 3.9960327040334263, + "objective/train/tokens_used": 434974176, + "theoretical_loss": 3.9960327040334263, + "tokens_seen": 414514176 + }, + { + "epoch": 1.04, + "learning_rate": 0.00044161484453360077, + "loss": 3.2747, + "theoretical_loss": 3.996015139311795, + "tokens_seen": 414530560 + }, + { + "epoch": 1.04, + "learning_rate": 0.00044160481444333, + "loss": 3.2075, + "theoretical_loss": 3.9959448893104144, + "tokens_seen": 414596096 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004415947843530592, + "loss": 3.0925, + "theoretical_loss": 3.9958746535214233, + "tokens_seen": 414661632 + }, + { + "epoch": 1.04, + "learning_rate": 0.00044158475426278837, + "loss": 3.201, + "theoretical_loss": 3.9958044319397006, + "tokens_seen": 414727168 + }, + { + "epoch": 1.04, + "learning_rate": 0.00044157472417251755, + "loss": 2.9766, + "theoretical_loss": 3.9957342245601275, + "tokens_seen": 414792704 + }, + { + "epoch": 1.04, + "learning_rate": 0.00044156469408224673, + "loss": 3.1882, + "theoretical_loss": 3.99566403137759, + "tokens_seen": 414858240 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004415546639919759, + "loss": 3.0593, + "theoretical_loss": 3.995593852386974, + "tokens_seen": 414923776 + }, + { + "epoch": 1.04, + "learning_rate": 0.00044154463390170515, + "loss": 3.1353, + "theoretical_loss": 3.995523687583169, + "tokens_seen": 414989312 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004415346038114343, + "loss": 3.1151, + "theoretical_loss": 3.9954535369610684, + "tokens_seen": 415054848 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004415245737211635, + "loss": 3.1027, + "theoretical_loss": 3.995383400515567, + "tokens_seen": 415120384 + }, + { + "epoch": 1.04, + "learning_rate": 0.00044151454363089264, + "loss": 3.0921, + "theoretical_loss": 3.9953132782415617, + "tokens_seen": 415185920 + }, + { + "epoch": 1.04, + "learning_rate": 0.00044150451354062187, + "loss": 3.1593, + "theoretical_loss": 3.9952431701339535, + "tokens_seen": 415251456 + }, + { + "epoch": 1.04, + "learning_rate": 0.00044149448345035105, + "loss": 3.2781, + "theoretical_loss": 3.995173076187645, + "tokens_seen": 415316992 + }, + { + "epoch": 1.04, + "learning_rate": 0.00044148445336008023, + "loss": 2.9717, + "theoretical_loss": 3.995102996397542, + "tokens_seen": 415382528 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004414744232698094, + "loss": 3.0149, + "theoretical_loss": 3.995032930758552, + "tokens_seen": 415448064 + }, + { + "epoch": 1.04, + "learning_rate": 0.00044146439317953865, + "loss": 3.0351, + "theoretical_loss": 3.9949628792655876, + "tokens_seen": 415513600 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004414543630892678, + "loss": 3.1212, + "theoretical_loss": 3.9948928419135603, + "tokens_seen": 415579136 + }, + { + "epoch": 1.04, + "learning_rate": 0.000441444332998997, + "loss": 2.9529, + "theoretical_loss": 3.9948228186973873, + "tokens_seen": 415644672 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004414343029087262, + "loss": 3.1814, + "theoretical_loss": 3.9947528096119873, + "tokens_seen": 415710208 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004414242728184554, + "loss": 3.1078, + "theoretical_loss": 3.9946828146522817, + "tokens_seen": 415775744 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004414142427281846, + "loss": 2.9443, + "theoretical_loss": 3.9946128338131945, + "tokens_seen": 415841280 + }, + { + "epoch": 1.04, + "learning_rate": 0.00044140421263791374, + "loss": 2.9845, + "theoretical_loss": 3.9945428670896517, + "tokens_seen": 415906816 + }, + { + "epoch": 1.04, + "learning_rate": 0.00044139418254764297, + "loss": 3.0643, + "theoretical_loss": 3.9944729144765834, + "tokens_seen": 415972352 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004413841524573721, + "loss": 2.9436, + "theoretical_loss": 3.9944029759689204, + "tokens_seen": 416037888 + }, + { + "epoch": 1.04, + "learning_rate": 0.00044137412236710133, + "loss": 2.895, + "theoretical_loss": 3.9943330515615987, + "tokens_seen": 416103424 + }, + { + "epoch": 1.04, + "objective/train/docs_used": 524726, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.1443862915039062, + "objective/train/theoretical_loss": 3.9942806175064094, + "objective/train/tokens_used": 436612576, + "theoretical_loss": 3.9942806175064094, + "tokens_seen": 416152576 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004413640922768305, + "loss": 2.8181, + "theoretical_loss": 3.994263141249554, + "tokens_seen": 416168960 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004413540621865597, + "loss": 3.0632, + "theoretical_loss": 3.994193245027726, + "tokens_seen": 416234496 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004413440320962889, + "loss": 3.2087, + "theoretical_loss": 3.9941233628910586, + "tokens_seen": 416300032 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004413340020060181, + "loss": 3.0843, + "theoretical_loss": 3.9940534948344952, + "tokens_seen": 416365568 + }, + { + "epoch": 1.04, + "learning_rate": 0.00044132397191574724, + "loss": 3.115, + "theoretical_loss": 3.993983640852984, + "tokens_seen": 416431104 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004413139418254765, + "loss": 3.1099, + "theoretical_loss": 3.993913800941475, + "tokens_seen": 416496640 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004413039117352056, + "loss": 3.0741, + "theoretical_loss": 3.9938439750949213, + "tokens_seen": 416562176 + }, + { + "epoch": 1.04, + "learning_rate": 0.00044129388164493484, + "loss": 3.0151, + "theoretical_loss": 3.9937741633082777, + "tokens_seen": 416627712 + }, + { + "epoch": 1.04, + "learning_rate": 0.000441283851554664, + "loss": 3.0126, + "theoretical_loss": 3.9937043655765025, + "tokens_seen": 416693248 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004412738214643932, + "loss": 3.0725, + "theoretical_loss": 3.993634581894556, + "tokens_seen": 416758784 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004412637913741224, + "loss": 3.2187, + "theoretical_loss": 3.9935648122574015, + "tokens_seen": 416824320 + }, + { + "epoch": 1.04, + "learning_rate": 0.00044125376128385156, + "loss": 3.131, + "theoretical_loss": 3.993495056660005, + "tokens_seen": 416889856 + }, + { + "epoch": 1.04, + "learning_rate": 0.00044124373119358074, + "loss": 3.0721, + "theoretical_loss": 3.993425315097335, + "tokens_seen": 416955392 + }, + { + "epoch": 1.04, + "learning_rate": 0.00044123370110331, + "loss": 2.9879, + "theoretical_loss": 3.993355587564362, + "tokens_seen": 417020928 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004412236710130391, + "loss": 3.079, + "theoretical_loss": 3.9932858740560597, + "tokens_seen": 417086464 + }, + { + "epoch": 1.04, + "learning_rate": 0.00044121364092276834, + "loss": 3.1582, + "theoretical_loss": 3.9932161745674035, + "tokens_seen": 417152000 + }, + { + "epoch": 1.04, + "learning_rate": 0.00044120361083249747, + "loss": 3.2366, + "theoretical_loss": 3.9931464890933737, + "tokens_seen": 417217536 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004411935807422267, + "loss": 3.0629, + "theoretical_loss": 3.9930768176289506, + "tokens_seen": 417283072 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004411835506519559, + "loss": 3.1617, + "theoretical_loss": 3.9930071601691184, + "tokens_seen": 417348608 + }, + { + "epoch": 1.04, + "learning_rate": 0.00044117352056168507, + "loss": 3.0412, + "theoretical_loss": 3.9929375167088637, + "tokens_seen": 417414144 + }, + { + "epoch": 1.04, + "learning_rate": 0.00044116349047141425, + "loss": 2.9656, + "theoretical_loss": 3.9928678872431744, + "tokens_seen": 417479680 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004411534603811435, + "loss": 2.9363, + "theoretical_loss": 3.992798271767044, + "tokens_seen": 417545216 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004411434302908726, + "loss": 2.9418, + "theoretical_loss": 3.992728670275465, + "tokens_seen": 417610752 + }, + { + "epoch": 1.04, + "learning_rate": 0.00044113340020060184, + "loss": 3.0558, + "theoretical_loss": 3.992659082763436, + "tokens_seen": 417676288 + }, + { + "epoch": 1.04, + "learning_rate": 0.00044112337011033097, + "loss": 3.0002, + "theoretical_loss": 3.9925895092259545, + "tokens_seen": 417741824 + }, + { + "epoch": 1.04, + "objective/train/docs_used": 525508, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.890538454055786, + "objective/train/theoretical_loss": 3.992537338240634, + "objective/train/tokens_used": 438250976, + "theoretical_loss": 3.992537338240634, + "tokens_seen": 417790976 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004411133400200602, + "loss": 2.9902, + "theoretical_loss": 3.9925199496580235, + "tokens_seen": 417807360 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004411033099297894, + "loss": 3.0079, + "theoretical_loss": 3.9924504040546474, + "tokens_seen": 417872896 + }, + { + "epoch": 1.04, + "learning_rate": 0.00044109327983951857, + "loss": 3.1488, + "theoretical_loss": 3.992380872410833, + "tokens_seen": 417938432 + }, + { + "epoch": 1.04, + "learning_rate": 0.00044108324974924775, + "loss": 3.2423, + "theoretical_loss": 3.9923113547215907, + "tokens_seen": 418003968 + }, + { + "epoch": 1.04, + "learning_rate": 0.00044107321965897693, + "loss": 3.121, + "theoretical_loss": 3.992241850981932, + "tokens_seen": 418069504 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004410631895687061, + "loss": 2.8882, + "theoretical_loss": 3.9921723611868716, + "tokens_seen": 418135040 + }, + { + "epoch": 1.04, + "learning_rate": 0.00044105315947843535, + "loss": 3.145, + "theoretical_loss": 3.992102885331427, + "tokens_seen": 418200576 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004410431293881645, + "loss": 2.9919, + "theoretical_loss": 3.9920334234106187, + "tokens_seen": 418266112 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004410330992978937, + "loss": 2.9014, + "theoretical_loss": 3.991963975419468, + "tokens_seen": 418331648 + }, + { + "epoch": 1.04, + "learning_rate": 0.00044102306920762284, + "loss": 3.1452, + "theoretical_loss": 3.991894541353001, + "tokens_seen": 418397184 + }, + { + "epoch": 1.04, + "learning_rate": 0.00044101303911735207, + "loss": 3.098, + "theoretical_loss": 3.9918251212062454, + "tokens_seen": 418462720 + }, + { + "epoch": 1.04, + "learning_rate": 0.00044100300902708125, + "loss": 2.9439, + "theoretical_loss": 3.99175571497423, + "tokens_seen": 418528256 + }, + { + "epoch": 1.04, + "learning_rate": 0.00044099297893681043, + "loss": 2.8596, + "theoretical_loss": 3.991686322651989, + "tokens_seen": 418593792 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004409829488465396, + "loss": 2.8567, + "theoretical_loss": 3.9916169442345564, + "tokens_seen": 418659328 + }, + { + "epoch": 1.04, + "learning_rate": 0.00044097291875626885, + "loss": 3.0336, + "theoretical_loss": 3.991547579716971, + "tokens_seen": 418724864 + }, + { + "epoch": 1.04, + "learning_rate": 0.000440962888665998, + "loss": 3.01, + "theoretical_loss": 3.9914782290942723, + "tokens_seen": 418790400 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004409528585757272, + "loss": 2.718, + "theoretical_loss": 3.9914088923615036, + "tokens_seen": 418855936 + }, + { + "epoch": 1.04, + "learning_rate": 0.00044094282848545634, + "loss": 2.9002, + "theoretical_loss": 3.9913395695137104, + "tokens_seen": 418921472 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004409327983951856, + "loss": 3.1815, + "theoretical_loss": 3.99127026054594, + "tokens_seen": 418987008 + }, + { + "epoch": 1.04, + "learning_rate": 0.00044092276830491476, + "loss": 3.1064, + "theoretical_loss": 3.991200965453244, + "tokens_seen": 419052544 + }, + { + "epoch": 1.04, + "learning_rate": 0.00044091273821464394, + "loss": 2.9756, + "theoretical_loss": 3.991131684230675, + "tokens_seen": 419118080 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004409027081243731, + "loss": 3.1044, + "theoretical_loss": 3.991062416873288, + "tokens_seen": 419183616 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004408926780341023, + "loss": 2.9907, + "theoretical_loss": 3.990993163376142, + "tokens_seen": 419249152 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004408826479438315, + "loss": 2.5876, + "theoretical_loss": 3.990923923734297, + "tokens_seen": 419314688 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004408726178535607, + "loss": 2.9581, + "theoretical_loss": 3.990854697942816, + "tokens_seen": 419380224 + }, + { + "epoch": 1.04, + "objective/train/docs_used": 526274, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7906696796417236, + "objective/train/theoretical_loss": 3.9908027876855394, + "objective/train/tokens_used": 439889376, + "theoretical_loss": 3.9908027876855394, + "tokens_seen": 419429376 + }, + { + "epoch": 1.04, + "learning_rate": 0.00044086258776328984, + "loss": 2.9342, + "theoretical_loss": 3.990785485996766, + "tokens_seen": 419445760 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004408525576730191, + "loss": 2.8886, + "theoretical_loss": 3.990716287891214, + "tokens_seen": 419511296 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004408425275827482, + "loss": 2.9543, + "theoretical_loss": 3.9906471036212308, + "tokens_seen": 419576832 + }, + { + "epoch": 1.04, + "learning_rate": 0.00044083249749247744, + "loss": 3.0434, + "theoretical_loss": 3.9905779331818905, + "tokens_seen": 419642368 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004408224674022066, + "loss": 2.7933, + "theoretical_loss": 3.9905087765682676, + "tokens_seen": 419707904 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004408124373119358, + "loss": 2.9713, + "theoretical_loss": 3.990439633775442, + "tokens_seen": 419773440 + }, + { + "epoch": 1.04, + "learning_rate": 0.000440802407221665, + "loss": 3.2049, + "theoretical_loss": 3.9903705047984936, + "tokens_seen": 419838976 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004407923771313942, + "loss": 3.0042, + "theoretical_loss": 3.9903013896325064, + "tokens_seen": 419904512 + }, + { + "epoch": 1.04, + "learning_rate": 0.00044078234704112335, + "loss": 3.002, + "theoretical_loss": 3.9902322882725656, + "tokens_seen": 419970048 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004407723169508526, + "loss": 3.2834, + "theoretical_loss": 3.9901632007137593, + "tokens_seen": 420035584 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004407622868605817, + "loss": 3.0347, + "theoretical_loss": 3.9900941269511794, + "tokens_seen": 420101120 + }, + { + "epoch": 1.04, + "learning_rate": 0.00044075225677031094, + "loss": 3.0477, + "theoretical_loss": 3.990025066979919, + "tokens_seen": 420166656 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004407422266800401, + "loss": 2.8861, + "theoretical_loss": 3.9899560207950744, + "tokens_seen": 420232192 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004407321965897693, + "loss": 2.878, + "theoretical_loss": 3.989886988391743, + "tokens_seen": 420297728 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004407221664994985, + "loss": 3.1336, + "theoretical_loss": 3.989817969765027, + "tokens_seen": 420363264 + }, + { + "epoch": 1.04, + "learning_rate": 0.00044071213640922767, + "loss": 3.0215, + "theoretical_loss": 3.9897489649100284, + "tokens_seen": 420428800 + }, + { + "epoch": 1.04, + "learning_rate": 0.00044070210631895685, + "loss": 2.9364, + "theoretical_loss": 3.9896799738218545, + "tokens_seen": 420494336 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004406920762286861, + "loss": 3.0076, + "theoretical_loss": 3.9896109964956135, + "tokens_seen": 420559872 + }, + { + "epoch": 1.04, + "learning_rate": 0.00044068204613841527, + "loss": 2.8764, + "theoretical_loss": 3.9895420329264155, + "tokens_seen": 420625408 + }, + { + "epoch": 1.04, + "learning_rate": 0.00044067201604814445, + "loss": 3.2049, + "theoretical_loss": 3.989473083109375, + "tokens_seen": 420690944 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004406619859578737, + "loss": 3.239, + "theoretical_loss": 3.9894041470396076, + "tokens_seen": 420756480 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004406519558676028, + "loss": 3.1395, + "theoretical_loss": 3.989335224712232, + "tokens_seen": 420822016 + }, + { + "epoch": 1.04, + "learning_rate": 0.00044064192577733204, + "loss": 3.0907, + "theoretical_loss": 3.989266316122369, + "tokens_seen": 420887552 + }, + { + "epoch": 1.04, + "learning_rate": 0.00044063189568706117, + "loss": 3.1008, + "theoretical_loss": 3.989197421265141, + "tokens_seen": 420953088 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004406218655967904, + "loss": 3.0425, + "theoretical_loss": 3.9891285401356757, + "tokens_seen": 421018624 + }, + { + "epoch": 1.04, + "objective/train/docs_used": 527775, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.216331958770752, + "objective/train/theoretical_loss": 3.9890768882944894, + "objective/train/tokens_used": 441527776, + "theoretical_loss": 3.9890768882944894, + "tokens_seen": 421067776 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004406118355065196, + "loss": 3.0228, + "theoretical_loss": 3.9890596727291, + "tokens_seen": 421084160 + }, + { + "epoch": 1.04, + "learning_rate": 0.00044060180541624877, + "loss": 2.7891, + "theoretical_loss": 3.988990819040546, + "tokens_seen": 421149696 + }, + { + "epoch": 1.04, + "learning_rate": 0.00044059177532597795, + "loss": 3.0307, + "theoretical_loss": 3.9889219790651462, + "tokens_seen": 421215232 + }, + { + "epoch": 1.04, + "learning_rate": 0.00044058174523570713, + "loss": 3.0988, + "theoretical_loss": 3.988853152798037, + "tokens_seen": 421280768 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004405717151454363, + "loss": 2.8487, + "theoretical_loss": 3.9887843402343566, + "tokens_seen": 421346304 + }, + { + "epoch": 1.04, + "learning_rate": 0.00044056168505516555, + "loss": 3.0812, + "theoretical_loss": 3.9887155413692463, + "tokens_seen": 421411840 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004405516549648947, + "loss": 3.0347, + "theoretical_loss": 3.9886467561978476, + "tokens_seen": 421477376 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004405416248746239, + "loss": 2.7433, + "theoretical_loss": 3.9885779847153087, + "tokens_seen": 421542912 + }, + { + "epoch": 1.04, + "learning_rate": 0.00044053159478435304, + "loss": 3.1027, + "theoretical_loss": 3.9885092269167766, + "tokens_seen": 421608448 + }, + { + "epoch": 1.04, + "learning_rate": 0.00044052156469408227, + "loss": 3.1315, + "theoretical_loss": 3.9884404827974023, + "tokens_seen": 421673984 + }, + { + "epoch": 1.04, + "learning_rate": 0.00044051153460381145, + "loss": 2.9011, + "theoretical_loss": 3.9883717523523385, + "tokens_seen": 421739520 + }, + { + "epoch": 1.04, + "learning_rate": 0.00044050150451354063, + "loss": 2.9213, + "theoretical_loss": 3.9883030355767417, + "tokens_seen": 421805056 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004404914744232698, + "loss": 3.0712, + "theoretical_loss": 3.98823433246577, + "tokens_seen": 421870592 + }, + { + "epoch": 1.04, + "learning_rate": 0.00044048144433299905, + "loss": 3.0033, + "theoretical_loss": 3.9881656430145833, + "tokens_seen": 421936128 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004404714142427282, + "loss": 3.0171, + "theoretical_loss": 3.988096967218345, + "tokens_seen": 422001664 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004404613841524574, + "loss": 2.9677, + "theoretical_loss": 3.9880283050722207, + "tokens_seen": 422067200 + }, + { + "epoch": 1.04, + "learning_rate": 0.00044045135406218654, + "loss": 2.9678, + "theoretical_loss": 3.9879596565713786, + "tokens_seen": 422132736 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004404413239719158, + "loss": 2.8646, + "theoretical_loss": 3.987891021710989, + "tokens_seen": 422198272 + }, + { + "epoch": 1.04, + "learning_rate": 0.00044043129388164496, + "loss": 3.17, + "theoretical_loss": 3.987822400486225, + "tokens_seen": 422263808 + }, + { + "epoch": 1.04, + "learning_rate": 0.00044042126379137414, + "loss": 3.1096, + "theoretical_loss": 3.9877537928922613, + "tokens_seen": 422329344 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004404112337011033, + "loss": 3.0704, + "theoretical_loss": 3.9876851989242765, + "tokens_seen": 422394880 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004404012036108325, + "loss": 3.0149, + "theoretical_loss": 3.9876166185774506, + "tokens_seen": 422460416 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004403911735205617, + "loss": 3.1474, + "theoretical_loss": 3.987548051846966, + "tokens_seen": 422525952 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004403811434302909, + "loss": 2.9211, + "theoretical_loss": 3.987479498728009, + "tokens_seen": 422591488 + }, + { + "epoch": 1.04, + "learning_rate": 0.00044037111334002004, + "loss": 3.144, + "theoretical_loss": 3.987410959215766, + "tokens_seen": 422657024 + }, + { + "epoch": 1.04, + "objective/train/docs_used": 528406, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4659383296966553, + "objective/train/theoretical_loss": 3.9873595635080967, + "objective/train/tokens_used": 443166176, + "theoretical_loss": 3.9873595635080967, + "tokens_seen": 422706176 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004403610832497493, + "loss": 3.073, + "theoretical_loss": 3.987342433305428, + "tokens_seen": 422722560 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004403510531594784, + "loss": 3.0885, + "theoretical_loss": 3.9872739209921866, + "tokens_seen": 422788096 + }, + { + "epoch": 1.04, + "learning_rate": 0.00044034102306920764, + "loss": 3.0353, + "theoretical_loss": 3.987205422271238, + "tokens_seen": 422853632 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004403309929789368, + "loss": 3.1106, + "theoretical_loss": 3.9871369371377785, + "tokens_seen": 422919168 + }, + { + "epoch": 1.04, + "learning_rate": 0.000440320962888666, + "loss": 2.91, + "theoretical_loss": 3.987068465587009, + "tokens_seen": 422984704 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004403109327983952, + "loss": 2.991, + "theoretical_loss": 3.9870000076141303, + "tokens_seen": 423050240 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004403009027081244, + "loss": 3.0972, + "theoretical_loss": 3.9869315632143483, + "tokens_seen": 423115776 + }, + { + "epoch": 1.04, + "learning_rate": 0.00044029087261785355, + "loss": 3.0585, + "theoretical_loss": 3.9868631323828705, + "tokens_seen": 423181312 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004402808425275828, + "loss": 2.8196, + "theoretical_loss": 3.9867947151149052, + "tokens_seen": 423246848 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004402708124373119, + "loss": 3.1558, + "theoretical_loss": 3.9867263114056657, + "tokens_seen": 423312384 + }, + { + "epoch": 1.04, + "learning_rate": 0.00044026078234704114, + "loss": 3.2312, + "theoretical_loss": 3.9866579212503654, + "tokens_seen": 423377920 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004402507522567703, + "loss": 3.2391, + "theoretical_loss": 3.986589544644222, + "tokens_seen": 423443456 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004402407221664995, + "loss": 3.1316, + "theoretical_loss": 3.986521181582455, + "tokens_seen": 423508992 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004402306920762287, + "loss": 3.2101, + "theoretical_loss": 3.9864528320602854, + "tokens_seen": 423574528 + }, + { + "epoch": 1.04, + "learning_rate": 0.00044022066198595787, + "loss": 2.9653, + "theoretical_loss": 3.9863844960729375, + "tokens_seen": 423640064 + }, + { + "epoch": 1.04, + "learning_rate": 0.00044021063189568705, + "loss": 3.0989, + "theoretical_loss": 3.9863161736156387, + "tokens_seen": 423705600 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004402006018054163, + "loss": 2.7849, + "theoretical_loss": 3.9862478646836172, + "tokens_seen": 423771136 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004401905717151454, + "loss": 2.9124, + "theoretical_loss": 3.986179569272105, + "tokens_seen": 423836672 + }, + { + "epoch": 1.04, + "learning_rate": 0.00044018054162487465, + "loss": 3.0429, + "theoretical_loss": 3.9861112873763354, + "tokens_seen": 423902208 + }, + { + "epoch": 1.04, + "learning_rate": 0.00044017051153460383, + "loss": 2.9653, + "theoretical_loss": 3.986043018991545, + "tokens_seen": 423967744 + }, + { + "epoch": 1.04, + "learning_rate": 0.000440160481444333, + "loss": 3.0573, + "theoretical_loss": 3.9859747641129726, + "tokens_seen": 424033280 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004401504513540622, + "loss": 3.0092, + "theoretical_loss": 3.9859065227358585, + "tokens_seen": 424098816 + }, + { + "epoch": 1.04, + "learning_rate": 0.00044014042126379137, + "loss": 2.9568, + "theoretical_loss": 3.985838294855448, + "tokens_seen": 424164352 + }, + { + "epoch": 1.04, + "learning_rate": 0.00044013039117352055, + "loss": 2.8587, + "theoretical_loss": 3.9857700804669847, + "tokens_seen": 424229888 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004401203610832498, + "loss": 2.9795, + "theoretical_loss": 3.9857018795657186, + "tokens_seen": 424295424 + }, + { + "epoch": 1.04, + "objective/train/docs_used": 530052, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.4916634559631348, + "objective/train/theoretical_loss": 3.985650737737885, + "objective/train/tokens_used": 444804576, + "theoretical_loss": 3.985650737737885, + "tokens_seen": 424344576 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004401103309929789, + "loss": 3.1533, + "theoretical_loss": 3.9856336921469007, + "tokens_seen": 424360960 + }, + { + "epoch": 1.04, + "learning_rate": 0.00044010030090270815, + "loss": 2.9523, + "theoretical_loss": 3.9855655182057825, + "tokens_seen": 424426496 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004400902708124373, + "loss": 2.8567, + "theoretical_loss": 3.9854973577376214, + "tokens_seen": 424492032 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004400802407221665, + "loss": 2.8743, + "theoretical_loss": 3.9854292107376734, + "tokens_seen": 424557568 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004400702106318957, + "loss": 2.8946, + "theoretical_loss": 3.9853610772012007, + "tokens_seen": 424623104 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004400601805416249, + "loss": 2.9646, + "theoretical_loss": 3.985292957123465, + "tokens_seen": 424688640 + }, + { + "epoch": 1.04, + "learning_rate": 0.00044005015045135406, + "loss": 2.9647, + "theoretical_loss": 3.9852248504997325, + "tokens_seen": 424754176 + }, + { + "epoch": 1.04, + "learning_rate": 0.00044004012036108324, + "loss": 2.936, + "theoretical_loss": 3.985156757325269, + "tokens_seen": 424819712 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004400300902708124, + "loss": 2.8415, + "theoretical_loss": 3.9850886775953462, + "tokens_seen": 424885248 + }, + { + "epoch": 1.04, + "learning_rate": 0.00044002006018054165, + "loss": 3.0282, + "theoretical_loss": 3.9850206113052353, + "tokens_seen": 424950784 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004400100300902708, + "loss": 3.0696, + "theoretical_loss": 3.9849525584502112, + "tokens_seen": 425016320 + }, + { + "epoch": 1.04, + "learning_rate": 0.00044, + "loss": 3.1165, + "theoretical_loss": 3.984884519025552, + "tokens_seen": 425081856 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004399899699097292, + "loss": 3.2719, + "theoretical_loss": 3.9848164930265364, + "tokens_seen": 425147392 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004399799398194584, + "loss": 2.9398, + "theoretical_loss": 3.9847484804484457, + "tokens_seen": 425212928 + }, + { + "epoch": 1.04, + "learning_rate": 0.00043996990972918756, + "loss": 2.7438, + "theoretical_loss": 3.9846804812865653, + "tokens_seen": 425278464 + }, + { + "epoch": 1.04, + "learning_rate": 0.00043995987963891674, + "loss": 3.0575, + "theoretical_loss": 3.9846124955361812, + "tokens_seen": 425344000 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004399498495486459, + "loss": 2.8866, + "theoretical_loss": 3.9845445231925827, + "tokens_seen": 425409536 + }, + { + "epoch": 1.04, + "learning_rate": 0.00043993981945837516, + "loss": 3.0092, + "theoretical_loss": 3.984476564251061, + "tokens_seen": 425475072 + }, + { + "epoch": 1.04, + "learning_rate": 0.00043992978936810434, + "loss": 3.1991, + "theoretical_loss": 3.9844086187069108, + "tokens_seen": 425540608 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004399197592778335, + "loss": 3.1595, + "theoretical_loss": 3.9843406865554267, + "tokens_seen": 425606144 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004399097291875627, + "loss": 3.1975, + "theoretical_loss": 3.984272767791908, + "tokens_seen": 425671680 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004398996990972919, + "loss": 2.9193, + "theoretical_loss": 3.984204862411657, + "tokens_seen": 425737216 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004398896690070211, + "loss": 2.92, + "theoretical_loss": 3.9841369704099745, + "tokens_seen": 425802752 + }, + { + "epoch": 1.04, + "learning_rate": 0.00043987963891675024, + "loss": 2.9934, + "theoretical_loss": 3.9840690917821675, + "tokens_seen": 425868288 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004398696088264795, + "loss": 3.1759, + "theoretical_loss": 3.984001226523545, + "tokens_seen": 425933824 + }, + { + "epoch": 1.04, + "objective/train/docs_used": 530759, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.535398006439209, + "objective/train/theoretical_loss": 3.983950336350283, + "objective/train/tokens_used": 446442976, + "theoretical_loss": 3.983950336350283, + "tokens_seen": 425982976 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004398595787362086, + "loss": 2.9939, + "theoretical_loss": 3.983933374629416, + "tokens_seen": 425999360 + }, + { + "epoch": 1.04, + "learning_rate": 0.00043984954864593784, + "loss": 3.0605, + "theoretical_loss": 3.9838655360950925, + "tokens_seen": 426064896 + }, + { + "epoch": 1.04, + "learning_rate": 0.000439839518555667, + "loss": 3.1724, + "theoretical_loss": 3.983797710915892, + "tokens_seen": 426130432 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004398294884653962, + "loss": 2.8873, + "theoretical_loss": 3.9837298990871304, + "tokens_seen": 426195968 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004398194583751254, + "loss": 2.9109, + "theoretical_loss": 3.9836621006041284, + "tokens_seen": 426261504 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004398094282848546, + "loss": 3.051, + "theoretical_loss": 3.9835943154622075, + "tokens_seen": 426327040 + }, + { + "epoch": 1.04, + "learning_rate": 0.00043979939819458375, + "loss": 2.8254, + "theoretical_loss": 3.9835265436566925, + "tokens_seen": 426392576 + }, + { + "epoch": 1.04, + "learning_rate": 0.000439789368104313, + "loss": 3.2306, + "theoretical_loss": 3.9834587851829104, + "tokens_seen": 426458112 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004397793380140421, + "loss": 3.1756, + "theoretical_loss": 3.9833910400361905, + "tokens_seen": 426523648 + }, + { + "epoch": 1.04, + "learning_rate": 0.00043976930792377134, + "loss": 2.9631, + "theoretical_loss": 3.9833233082118644, + "tokens_seen": 426589184 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004397592778335005, + "loss": 3.1489, + "theoretical_loss": 3.9832555897052666, + "tokens_seen": 426654720 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004397492477432297, + "loss": 3.0471, + "theoretical_loss": 3.9831878845117323, + "tokens_seen": 426720256 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004397392176529589, + "loss": 3.0495, + "theoretical_loss": 3.9831201926266018, + "tokens_seen": 426785792 + }, + { + "epoch": 1.04, + "learning_rate": 0.00043972918756268807, + "loss": 3.1702, + "theoretical_loss": 3.983052514045215, + "tokens_seen": 426851328 + }, + { + "epoch": 1.04, + "learning_rate": 0.00043971915747241725, + "loss": 2.9261, + "theoretical_loss": 3.982984848762915, + "tokens_seen": 426916864 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004397091273821465, + "loss": 3.0344, + "theoretical_loss": 3.982917196775049, + "tokens_seen": 426982400 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004396990972918756, + "loss": 3.1101, + "theoretical_loss": 3.9828495580769636, + "tokens_seen": 427047936 + }, + { + "epoch": 1.04, + "learning_rate": 0.00043968906720160485, + "loss": 2.8813, + "theoretical_loss": 3.98278193266401, + "tokens_seen": 427113472 + }, + { + "epoch": 1.04, + "learning_rate": 0.00043967903711133403, + "loss": 2.9827, + "theoretical_loss": 3.9827143205315405, + "tokens_seen": 427179008 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004396690070210632, + "loss": 2.8908, + "theoretical_loss": 3.982646721674911, + "tokens_seen": 427244544 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004396589769307924, + "loss": 3.0251, + "theoretical_loss": 3.9825791360894787, + "tokens_seen": 427310080 + }, + { + "epoch": 1.04, + "learning_rate": 0.00043964894684052157, + "loss": 2.885, + "theoretical_loss": 3.9825115637706023, + "tokens_seen": 427375616 + }, + { + "epoch": 1.04, + "learning_rate": 0.00043963891675025075, + "loss": 2.9362, + "theoretical_loss": 3.982444004713645, + "tokens_seen": 427441152 + }, + { + "epoch": 1.04, + "learning_rate": 0.00043962888665998, + "loss": 2.9634, + "theoretical_loss": 3.982376458913971, + "tokens_seen": 427506688 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004396188565697091, + "loss": 3.0015, + "theoretical_loss": 3.982308926366947, + "tokens_seen": 427572224 + }, + { + "epoch": 1.04, + "objective/train/docs_used": 532253, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.377031087875366, + "objective/train/theoretical_loss": 3.9822582856509445, + "objective/train/tokens_used": 448081376, + "theoretical_loss": 3.9822582856509445, + "tokens_seen": 427621376 + }, + { + "epoch": 1.04, + "learning_rate": 0.00043960882647943835, + "loss": 2.8872, + "theoretical_loss": 3.982241407067942, + "tokens_seen": 427637760 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004395987963891675, + "loss": 3.1085, + "theoretical_loss": 3.9821739010123274, + "tokens_seen": 427703296 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004395887662988967, + "loss": 3.1265, + "theoretical_loss": 3.9821064081954773, + "tokens_seen": 427768832 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004395787362086259, + "loss": 2.9826, + "theoretical_loss": 3.9820389286127678, + "tokens_seen": 427834368 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004395687061183551, + "loss": 3.0233, + "theoretical_loss": 3.981971462259577, + "tokens_seen": 427899904 + }, + { + "epoch": 1.04, + "learning_rate": 0.00043955867602808426, + "loss": 2.8907, + "theoretical_loss": 3.981904009131285, + "tokens_seen": 427965440 + }, + { + "epoch": 1.04, + "learning_rate": 0.00043954864593781344, + "loss": 2.9427, + "theoretical_loss": 3.9818365692232764, + "tokens_seen": 428030976 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004395386158475426, + "loss": 2.6778, + "theoretical_loss": 3.981769142530936, + "tokens_seen": 428096512 + }, + { + "epoch": 1.04, + "learning_rate": 0.00043952858575727185, + "loss": 3.1932, + "theoretical_loss": 3.98170172904965, + "tokens_seen": 428162048 + }, + { + "epoch": 1.04, + "learning_rate": 0.000439518555667001, + "loss": 2.9245, + "theoretical_loss": 3.9816343287748106, + "tokens_seen": 428227584 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004395085255767302, + "loss": 2.8477, + "theoretical_loss": 3.9815669417018085, + "tokens_seen": 428293120 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004394984954864594, + "loss": 3.1402, + "theoretical_loss": 3.981499567826039, + "tokens_seen": 428358656 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004394884653961886, + "loss": 2.8769, + "theoretical_loss": 3.981432207142899, + "tokens_seen": 428424192 + }, + { + "epoch": 1.04, + "learning_rate": 0.00043947843530591776, + "loss": 2.8358, + "theoretical_loss": 3.9813648596477877, + "tokens_seen": 428489728 + }, + { + "epoch": 1.04, + "learning_rate": 0.00043946840521564694, + "loss": 3.0459, + "theoretical_loss": 3.9812975253361067, + "tokens_seen": 428555264 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004394583751253761, + "loss": 3.0235, + "theoretical_loss": 3.9812302042032597, + "tokens_seen": 428620800 + }, + { + "epoch": 1.04, + "learning_rate": 0.00043944834503510536, + "loss": 3.1087, + "theoretical_loss": 3.981162896244653, + "tokens_seen": 428686336 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004394383149448345, + "loss": 3.0048, + "theoretical_loss": 3.9810956014556944, + "tokens_seen": 428751872 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004394282848545637, + "loss": 3.1698, + "theoretical_loss": 3.981028319831796, + "tokens_seen": 428817408 + }, + { + "epoch": 1.04, + "learning_rate": 0.00043941825476429285, + "loss": 3.0103, + "theoretical_loss": 3.9809610513683698, + "tokens_seen": 428882944 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004394082246740221, + "loss": 3.2602, + "theoretical_loss": 3.9808937960608315, + "tokens_seen": 428948480 + }, + { + "epoch": 1.04, + "learning_rate": 0.00043939819458375126, + "loss": 2.9979, + "theoretical_loss": 3.9808265539045986, + "tokens_seen": 429014016 + }, + { + "epoch": 1.04, + "learning_rate": 0.00043938816449348044, + "loss": 2.995, + "theoretical_loss": 3.9807593248950917, + "tokens_seen": 429079552 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004393781344032096, + "loss": 3.0497, + "theoretical_loss": 3.980692109027732, + "tokens_seen": 429145088 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004393681043129388, + "loss": 3.0166, + "theoretical_loss": 3.980624906297945, + "tokens_seen": 429210624 + }, + { + "epoch": 1.04, + "objective/train/docs_used": 533007, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.870386838912964, + "objective/train/theoretical_loss": 3.980574512869385, + "objective/train/tokens_used": 449719776, + "theoretical_loss": 3.980574512869385, + "tokens_seen": 429259776 + }, + { + "epoch": 1.04, + "learning_rate": 0.000439358074222668, + "loss": 2.9974, + "theoretical_loss": 3.9805577167011563, + "tokens_seen": 429276160 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004393480441323972, + "loss": 2.8626, + "theoretical_loss": 3.980490540232797, + "tokens_seen": 429341696 + }, + { + "epoch": 1.04, + "learning_rate": 0.00043933801404212635, + "loss": 2.9723, + "theoretical_loss": 3.9804233768882966, + "tokens_seen": 429407232 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004393279839518556, + "loss": 2.8119, + "theoretical_loss": 3.9803562266630896, + "tokens_seen": 429472768 + }, + { + "epoch": 1.04, + "learning_rate": 0.00043931795386158477, + "loss": 2.9868, + "theoretical_loss": 3.9802890895526124, + "tokens_seen": 429538304 + }, + { + "epoch": 1.04, + "learning_rate": 0.00043930792377131395, + "loss": 2.909, + "theoretical_loss": 3.9802219655523023, + "tokens_seen": 429603840 + }, + { + "epoch": 1.04, + "learning_rate": 0.00043929789368104313, + "loss": 3.1078, + "theoretical_loss": 3.9801548546576004, + "tokens_seen": 429669376 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004392878635907723, + "loss": 2.9401, + "theoretical_loss": 3.9800877568639494, + "tokens_seen": 429734912 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004392778335005015, + "loss": 3.1711, + "theoretical_loss": 3.9800206721667952, + "tokens_seen": 429800448 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004392678034102307, + "loss": 3.2001, + "theoretical_loss": 3.979953600561584, + "tokens_seen": 429865984 + }, + { + "epoch": 1.04, + "learning_rate": 0.00043925777331995985, + "loss": 3.0095, + "theoretical_loss": 3.979886542043766, + "tokens_seen": 429931520 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004392477432296891, + "loss": 3.0156, + "theoretical_loss": 3.979819496608793, + "tokens_seen": 429997056 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004392377131394182, + "loss": 2.8932, + "theoretical_loss": 3.9797524642521194, + "tokens_seen": 430062592 + }, + { + "epoch": 1.04, + "learning_rate": 0.00043922768304914745, + "loss": 3.1111, + "theoretical_loss": 3.979685444969202, + "tokens_seen": 430128128 + }, + { + "epoch": 1.04, + "learning_rate": 0.00043921765295887663, + "loss": 3.0072, + "theoretical_loss": 3.979618438755498, + "tokens_seen": 430193664 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004392076228686058, + "loss": 2.9538, + "theoretical_loss": 3.9795514456064702, + "tokens_seen": 430259200 + }, + { + "epoch": 1.04, + "learning_rate": 0.000439197592778335, + "loss": 3.0761, + "theoretical_loss": 3.979484465517581, + "tokens_seen": 430324736 + }, + { + "epoch": 1.04, + "learning_rate": 0.00043918756268806423, + "loss": 2.9912, + "theoretical_loss": 3.979417498484297, + "tokens_seen": 430390272 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004391775325977934, + "loss": 2.8828, + "theoretical_loss": 3.9793505445020845, + "tokens_seen": 430455808 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004391675025075226, + "loss": 2.9818, + "theoretical_loss": 3.9792836035664148, + "tokens_seen": 430521344 + }, + { + "epoch": 1.04, + "learning_rate": 0.00043915747241725177, + "loss": 3.1237, + "theoretical_loss": 3.9792166756727596, + "tokens_seen": 430586880 + }, + { + "epoch": 1.04, + "learning_rate": 0.00043914744232698095, + "loss": 3.0967, + "theoretical_loss": 3.9791497608165933, + "tokens_seen": 430652416 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004391374122367102, + "loss": 3.1895, + "theoretical_loss": 3.9790828589933933, + "tokens_seen": 430717952 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004391273821464393, + "loss": 2.8996, + "theoretical_loss": 3.9790159701986387, + "tokens_seen": 430783488 + }, + { + "epoch": 1.04, + "learning_rate": 0.00043911735205616855, + "loss": 3.0491, + "theoretical_loss": 3.978949094427811, + "tokens_seen": 430849024 + }, + { + "epoch": 1.04, + "objective/train/docs_used": 534288, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8886990547180176, + "objective/train/theoretical_loss": 3.9788989461439246, + "objective/train/tokens_used": 451358176, + "theoretical_loss": 3.9788989461439246, + "tokens_seen": 430898176 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004391073219658977, + "loss": 2.7626, + "theoretical_loss": 3.978882231676393, + "tokens_seen": 430914560 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004390972918756269, + "loss": 2.8555, + "theoretical_loss": 3.9788153819398717, + "tokens_seen": 430980096 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004390872617853561, + "loss": 3.1994, + "theoretical_loss": 3.978748545213734, + "tokens_seen": 431045632 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004390772316950853, + "loss": 3.0032, + "theoretical_loss": 3.9786817214934715, + "tokens_seen": 431111168 + }, + { + "epoch": 1.04, + "learning_rate": 0.00043906720160481446, + "loss": 2.9658, + "theoretical_loss": 3.9786149107745765, + "tokens_seen": 431176704 + }, + { + "epoch": 1.04, + "learning_rate": 0.00043905717151454364, + "loss": 3.1315, + "theoretical_loss": 3.9785481130525433, + "tokens_seen": 431242240 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004390471414242728, + "loss": 3.1294, + "theoretical_loss": 3.9784813283228693, + "tokens_seen": 431307776 + }, + { + "epoch": 1.04, + "learning_rate": 0.00043903711133400205, + "loss": 3.1198, + "theoretical_loss": 3.978414556581054, + "tokens_seen": 431373312 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004390270812437312, + "loss": 3.0879, + "theoretical_loss": 3.9783477978225994, + "tokens_seen": 431438848 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004390170511534604, + "loss": 3.0557, + "theoretical_loss": 3.9782810520430085, + "tokens_seen": 431504384 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004390070210631896, + "loss": 2.8589, + "theoretical_loss": 3.9782143192377886, + "tokens_seen": 431569920 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004389969909729188, + "loss": 2.7922, + "theoretical_loss": 3.9781475994024467, + "tokens_seen": 431635456 + }, + { + "epoch": 1.04, + "learning_rate": 0.00043898696088264796, + "loss": 2.9462, + "theoretical_loss": 3.9780808925324935, + "tokens_seen": 431700992 + }, + { + "epoch": 1.04, + "learning_rate": 0.00043897693079237714, + "loss": 3.2187, + "theoretical_loss": 3.978014198623443, + "tokens_seen": 431766528 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004389669007021063, + "loss": 2.6678, + "theoretical_loss": 3.977947517670809, + "tokens_seen": 431832064 + }, + { + "epoch": 1.04, + "learning_rate": 0.00043895687061183556, + "loss": 3.0396, + "theoretical_loss": 3.9778808496701092, + "tokens_seen": 431897600 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004389468405215647, + "loss": 3.0952, + "theoretical_loss": 3.9778141946168635, + "tokens_seen": 431963136 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004389368104312939, + "loss": 2.9717, + "theoretical_loss": 3.9777475525065933, + "tokens_seen": 432028672 + }, + { + "epoch": 1.04, + "learning_rate": 0.00043892678034102305, + "loss": 3.0584, + "theoretical_loss": 3.9776809233348223, + "tokens_seen": 432094208 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004389167502507523, + "loss": 2.9005, + "theoretical_loss": 3.977614307097077, + "tokens_seen": 432159744 + }, + { + "epoch": 1.04, + "learning_rate": 0.00043890672016048146, + "loss": 2.6508, + "theoretical_loss": 3.977547703788886, + "tokens_seen": 432225280 + }, + { + "epoch": 1.04, + "learning_rate": 0.00043889669007021064, + "loss": 3.1334, + "theoretical_loss": 3.9774811134057795, + "tokens_seen": 432290816 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004388866599799398, + "loss": 3.1029, + "theoretical_loss": 3.97741453594329, + "tokens_seen": 432356352 + }, + { + "epoch": 1.04, + "learning_rate": 0.000438876629889669, + "loss": 2.8582, + "theoretical_loss": 3.977347971396954, + "tokens_seen": 432421888 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004388665997993982, + "loss": 3.0586, + "theoretical_loss": 3.977281419762307, + "tokens_seen": 432487424 + }, + { + "epoch": 1.04, + "objective/train/docs_used": 535120, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.702747344970703, + "objective/train/theoretical_loss": 3.9772315145069355, + "objective/train/tokens_used": 452996576, + "theoretical_loss": 3.9772315145069355, + "tokens_seen": 432536576 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004388565697091274, + "loss": 2.9321, + "theoretical_loss": 3.9772148810348904, + "tokens_seen": 432552960 + }, + { + "epoch": 1.04, + "learning_rate": 0.00043884653961885655, + "loss": 2.9524, + "theoretical_loss": 3.9771483552102445, + "tokens_seen": 432618496 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004388365095285858, + "loss": 2.8339, + "theoretical_loss": 3.977081842283913, + "tokens_seen": 432684032 + }, + { + "epoch": 1.04, + "learning_rate": 0.00043882647943831497, + "loss": 2.6309, + "theoretical_loss": 3.977015342251444, + "tokens_seen": 432749568 + }, + { + "epoch": 1.04, + "learning_rate": 0.00043881644934804415, + "loss": 2.8433, + "theoretical_loss": 3.976948855108384, + "tokens_seen": 432815104 + }, + { + "epoch": 1.04, + "learning_rate": 0.00043880641925777333, + "loss": 3.1073, + "theoretical_loss": 3.9768823808502836, + "tokens_seen": 432880640 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004387963891675025, + "loss": 2.8288, + "theoretical_loss": 3.976815919472697, + "tokens_seen": 432946176 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004387863590772317, + "loss": 2.9447, + "theoretical_loss": 3.976749470971178, + "tokens_seen": 433011712 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004387763289869609, + "loss": 2.929, + "theoretical_loss": 3.976683035341284, + "tokens_seen": 433077248 + }, + { + "epoch": 1.04, + "learning_rate": 0.00043876629889669005, + "loss": 3.1009, + "theoretical_loss": 3.9766166125785753, + "tokens_seen": 433142784 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004387562688064193, + "loss": 2.8277, + "theoretical_loss": 3.976550202678612, + "tokens_seen": 433208320 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004387462387161484, + "loss": 2.8715, + "theoretical_loss": 3.9764838056369594, + "tokens_seen": 433273856 + }, + { + "epoch": 1.04, + "learning_rate": 0.00043873620862587765, + "loss": 2.9434, + "theoretical_loss": 3.9764174214491828, + "tokens_seen": 433339392 + }, + { + "epoch": 1.04, + "learning_rate": 0.00043872617853560683, + "loss": 3.0301, + "theoretical_loss": 3.97635105011085, + "tokens_seen": 433404928 + }, + { + "epoch": 1.04, + "learning_rate": 0.000438716148445336, + "loss": 2.9667, + "theoretical_loss": 3.976284691617532, + "tokens_seen": 433470464 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004387061183550652, + "loss": 2.8421, + "theoretical_loss": 3.976218345964801, + "tokens_seen": 433536000 + }, + { + "epoch": 1.04, + "learning_rate": 0.00043869608826479443, + "loss": 3.0121, + "theoretical_loss": 3.9761520131482326, + "tokens_seen": 433601536 + }, + { + "epoch": 1.04, + "learning_rate": 0.00043868605817452356, + "loss": 3.2531, + "theoretical_loss": 3.976085693163403, + "tokens_seen": 433667072 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004386760280842528, + "loss": 3.0608, + "theoretical_loss": 3.9760193860058917, + "tokens_seen": 433732608 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004386659979939819, + "loss": 3.1406, + "theoretical_loss": 3.97595309167128, + "tokens_seen": 433798144 + }, + { + "epoch": 1.04, + "learning_rate": 0.00043865596790371115, + "loss": 2.7294, + "theoretical_loss": 3.975886810155152, + "tokens_seen": 433863680 + }, + { + "epoch": 1.04, + "learning_rate": 0.00043864593781344033, + "loss": 2.9871, + "theoretical_loss": 3.9758205414530923, + "tokens_seen": 433929216 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004386359077231695, + "loss": 2.9434, + "theoretical_loss": 3.9757542855606895, + "tokens_seen": 433994752 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004386258776328987, + "loss": 2.856, + "theoretical_loss": 3.975688042473534, + "tokens_seen": 434060288 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004386158475426279, + "loss": 3.0601, + "theoretical_loss": 3.975621812187218, + "tokens_seen": 434125824 + }, + { + "epoch": 1.04, + "objective/train/docs_used": 536351, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.160634756088257, + "objective/train/theoretical_loss": 3.9755721478703814, + "objective/train/tokens_used": 454634976, + "theoretical_loss": 3.9755721478703814, + "tokens_seen": 434174976 + }, + { + "epoch": 1.04, + "learning_rate": 0.00043860581745235706, + "loss": 3.0205, + "theoretical_loss": 3.9755555946973358, + "tokens_seen": 434191360 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004385957873620863, + "loss": 2.9008, + "theoretical_loss": 3.975489389999484, + "tokens_seen": 434256896 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004385857572718154, + "loss": 2.9809, + "theoretical_loss": 3.9754231980892616, + "tokens_seen": 434322432 + }, + { + "epoch": 1.04, + "learning_rate": 0.00043857572718154466, + "loss": 3.0585, + "theoretical_loss": 3.97535701896227, + "tokens_seen": 434387968 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004385656970912738, + "loss": 3.105, + "theoretical_loss": 3.9752908526141115, + "tokens_seen": 434453504 + }, + { + "epoch": 1.04, + "learning_rate": 0.000438555667001003, + "loss": 3.0696, + "theoretical_loss": 3.9752246990403926, + "tokens_seen": 434519040 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004385456369107322, + "loss": 3.1542, + "theoretical_loss": 3.9751585582367195, + "tokens_seen": 434584576 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004385356068204614, + "loss": 2.8699, + "theoretical_loss": 3.9750924301987034, + "tokens_seen": 434650112 + }, + { + "epoch": 1.04, + "learning_rate": 0.00043852557673019056, + "loss": 3.0068, + "theoretical_loss": 3.975026314921955, + "tokens_seen": 434715648 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004385155466399198, + "loss": 2.9832, + "theoretical_loss": 3.974960212402089, + "tokens_seen": 434781184 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004385055165496489, + "loss": 2.9908, + "theoretical_loss": 3.974894122634722, + "tokens_seen": 434846720 + }, + { + "epoch": 1.04, + "learning_rate": 0.00043849548645937816, + "loss": 2.838, + "theoretical_loss": 3.9748280456154714, + "tokens_seen": 434912256 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004384854563691073, + "loss": 3.1155, + "theoretical_loss": 3.9747619813399586, + "tokens_seen": 434977792 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004384754262788365, + "loss": 3.0884, + "theoretical_loss": 3.974695929803806, + "tokens_seen": 435043328 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004384653961885657, + "loss": 3.1434, + "theoretical_loss": 3.9746298910026385, + "tokens_seen": 435108864 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004384553660982949, + "loss": 3.0162, + "theoretical_loss": 3.974563864932083, + "tokens_seen": 435174400 + }, + { + "epoch": 1.04, + "learning_rate": 0.00043844533600802407, + "loss": 2.9452, + "theoretical_loss": 3.9744978515877696, + "tokens_seen": 435239936 + }, + { + "epoch": 1.04, + "learning_rate": 0.00043843530591775325, + "loss": 3.0573, + "theoretical_loss": 3.9744318509653285, + "tokens_seen": 435305472 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004384252758274825, + "loss": 3.0258, + "theoretical_loss": 3.9743658630603935, + "tokens_seen": 435371008 + }, + { + "epoch": 1.04, + "learning_rate": 0.00043841524573721166, + "loss": 2.9923, + "theoretical_loss": 3.9742998878686007, + "tokens_seen": 435436544 + }, + { + "epoch": 1.04, + "learning_rate": 0.00043840521564694084, + "loss": 3.0141, + "theoretical_loss": 3.974233925385588, + "tokens_seen": 435502080 + }, + { + "epoch": 1.04, + "learning_rate": 0.00043839518555667, + "loss": 2.7964, + "theoretical_loss": 3.974167975606995, + "tokens_seen": 435567616 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004383851554663992, + "loss": 2.9255, + "theoretical_loss": 3.974102038528464, + "tokens_seen": 435633152 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004383751253761284, + "loss": 2.8229, + "theoretical_loss": 3.9740361141456395, + "tokens_seen": 435698688 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004383650952858576, + "loss": 3.1127, + "theoretical_loss": 3.9739702024541677, + "tokens_seen": 435764224 + }, + { + "epoch": 1.04, + "objective/train/docs_used": 537149, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.811138868331909, + "objective/train/theoretical_loss": 3.9739207770116467, + "objective/train/tokens_used": 456273376, + "theoretical_loss": 3.9739207770116467, + "tokens_seen": 435813376 + }, + { + "epoch": 1.04, + "learning_rate": 0.00043835506519558675, + "loss": 2.8946, + "theoretical_loss": 3.9739043034496975, + "tokens_seen": 435829760 + }, + { + "epoch": 1.04, + "learning_rate": 0.000438345035105316, + "loss": 2.9828, + "theoretical_loss": 3.9738384171278787, + "tokens_seen": 435895296 + }, + { + "epoch": 1.04, + "learning_rate": 0.00043833500501504517, + "loss": 3.0393, + "theoretical_loss": 3.9737725434843654, + "tokens_seen": 435960832 + }, + { + "epoch": 1.04, + "learning_rate": 0.00043832497492477435, + "loss": 2.9571, + "theoretical_loss": 3.9737066825148126, + "tokens_seen": 436026368 + }, + { + "epoch": 1.04, + "learning_rate": 0.00043831494483450353, + "loss": 3.052, + "theoretical_loss": 3.9736408342148763, + "tokens_seen": 436091904 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004383049147442327, + "loss": 2.8974, + "theoretical_loss": 3.973574998580217, + "tokens_seen": 436157440 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004382948846539619, + "loss": 3.1101, + "theoretical_loss": 3.973509175606495, + "tokens_seen": 436222976 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004382848545636911, + "loss": 2.8409, + "theoretical_loss": 3.9734433652893753, + "tokens_seen": 436288512 + }, + { + "epoch": 1.04, + "learning_rate": 0.00043827482447342025, + "loss": 3.0663, + "theoretical_loss": 3.9733775676245227, + "tokens_seen": 436354048 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004382647943831495, + "loss": 3.0505, + "theoretical_loss": 3.973311782607605, + "tokens_seen": 436419584 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004382547642928786, + "loss": 3.0826, + "theoretical_loss": 3.973246010234292, + "tokens_seen": 436485120 + }, + { + "epoch": 1.04, + "learning_rate": 0.00043824473420260785, + "loss": 3.0302, + "theoretical_loss": 3.973180250500257, + "tokens_seen": 436550656 + }, + { + "epoch": 1.04, + "learning_rate": 0.00043823470411233703, + "loss": 2.9014, + "theoretical_loss": 3.973114503401174, + "tokens_seen": 436616192 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004382246740220662, + "loss": 3.0467, + "theoretical_loss": 3.973048768932718, + "tokens_seen": 436681728 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004382146439317954, + "loss": 3.1691, + "theoretical_loss": 3.972983047090569, + "tokens_seen": 436747264 + }, + { + "epoch": 1.04, + "learning_rate": 0.00043820461384152463, + "loss": 2.8118, + "theoretical_loss": 3.972917337870407, + "tokens_seen": 436812800 + }, + { + "epoch": 1.04, + "learning_rate": 0.00043819458375125376, + "loss": 2.9634, + "theoretical_loss": 3.9728516412679147, + "tokens_seen": 436878336 + }, + { + "epoch": 1.04, + "learning_rate": 0.000438184553660983, + "loss": 3.2248, + "theoretical_loss": 3.9727859572787763, + "tokens_seen": 436943872 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004381745235707121, + "loss": 2.7478, + "theoretical_loss": 3.9727202858986805, + "tokens_seen": 437009408 + }, + { + "epoch": 1.04, + "learning_rate": 0.00043816449348044135, + "loss": 2.9089, + "theoretical_loss": 3.972654627123316, + "tokens_seen": 437074944 + }, + { + "epoch": 1.04, + "learning_rate": 0.00043815446339017053, + "loss": 3.0526, + "theoretical_loss": 3.972588980948373, + "tokens_seen": 437140480 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004381444332998997, + "loss": 3.0026, + "theoretical_loss": 3.9725233473695454, + "tokens_seen": 437206016 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004381344032096289, + "loss": 2.8952, + "theoretical_loss": 3.9724577263825296, + "tokens_seen": 437271552 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004381243731193581, + "loss": 2.9139, + "theoretical_loss": 3.9723921179830217, + "tokens_seen": 437337088 + }, + { + "epoch": 1.04, + "learning_rate": 0.00043811434302908726, + "loss": 2.9151, + "theoretical_loss": 3.972326522166722, + "tokens_seen": 437402624 + }, + { + "epoch": 1.04, + "objective/train/docs_used": 538542, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.2445921897888184, + "objective/train/theoretical_loss": 3.972277333559642, + "objective/train/tokens_used": 457911776, + "theoretical_loss": 3.972277333559642, + "tokens_seen": 437451776 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004381043129388165, + "loss": 3.0233, + "theoretical_loss": 3.972260938929333, + "tokens_seen": 437468160 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004380942828485456, + "loss": 2.7278, + "theoretical_loss": 3.972195368266558, + "tokens_seen": 437533696 + }, + { + "epoch": 1.04, + "learning_rate": 0.00043808425275827486, + "loss": 3.0123, + "theoretical_loss": 3.9721298101741027, + "tokens_seen": 437599232 + }, + { + "epoch": 1.04, + "learning_rate": 0.000438074222668004, + "loss": 2.9663, + "theoretical_loss": 3.972064264647676, + "tokens_seen": 437664768 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004380641925777332, + "loss": 3.1004, + "theoretical_loss": 3.971998731682988, + "tokens_seen": 437730304 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004380541624874624, + "loss": 2.9891, + "theoretical_loss": 3.9719332112757515, + "tokens_seen": 437795840 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004380441323971916, + "loss": 3.0848, + "theoretical_loss": 3.9718677034216796, + "tokens_seen": 437861376 + }, + { + "epoch": 1.04, + "learning_rate": 0.00043803410230692076, + "loss": 2.9775, + "theoretical_loss": 3.97180220811649, + "tokens_seen": 437926912 + }, + { + "epoch": 1.04, + "learning_rate": 0.00043802407221665, + "loss": 2.7554, + "theoretical_loss": 3.9717367253559015, + "tokens_seen": 437992448 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004380140421263791, + "loss": 3.0661, + "theoretical_loss": 3.971671255135634, + "tokens_seen": 438057984 + }, + { + "epoch": 1.04, + "learning_rate": 0.00043800401203610836, + "loss": 3.0481, + "theoretical_loss": 3.971605797451411, + "tokens_seen": 438123520 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004379939819458375, + "loss": 2.7857, + "theoretical_loss": 3.971540352298958, + "tokens_seen": 438189056 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004379839518555667, + "loss": 3.0102, + "theoretical_loss": 3.971474919674001, + "tokens_seen": 438254592 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004379739217652959, + "loss": 2.9069, + "theoretical_loss": 3.97140949957227, + "tokens_seen": 438320128 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004379638916750251, + "loss": 3.0839, + "theoretical_loss": 3.9713440919894962, + "tokens_seen": 438385664 + }, + { + "epoch": 1.04, + "learning_rate": 0.00043795386158475427, + "loss": 2.8571, + "theoretical_loss": 3.971278696921412, + "tokens_seen": 438451200 + }, + { + "epoch": 1.04, + "learning_rate": 0.00043794383149448345, + "loss": 3.0024, + "theoretical_loss": 3.971213314363754, + "tokens_seen": 438516736 + }, + { + "epoch": 1.04, + "learning_rate": 0.00043793380140421263, + "loss": 2.8872, + "theoretical_loss": 3.9711479443122593, + "tokens_seen": 438582272 + }, + { + "epoch": 1.04, + "learning_rate": 0.00043792377131394186, + "loss": 3.0489, + "theoretical_loss": 3.9710825867626673, + "tokens_seen": 438647808 + }, + { + "epoch": 1.04, + "learning_rate": 0.000437913741223671, + "loss": 2.8011, + "theoretical_loss": 3.9710172417107206, + "tokens_seen": 438713344 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004379037111334002, + "loss": 2.8608, + "theoretical_loss": 3.970951909152162, + "tokens_seen": 438778880 + }, + { + "epoch": 1.04, + "learning_rate": 0.00043789368104312935, + "loss": 3.0229, + "theoretical_loss": 3.9708865890827383, + "tokens_seen": 438844416 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004378836509528586, + "loss": 3.0305, + "theoretical_loss": 3.9708212814981962, + "tokens_seen": 438909952 + }, + { + "epoch": 1.04, + "learning_rate": 0.00043787362086258777, + "loss": 2.5924, + "theoretical_loss": 3.970755986394287, + "tokens_seen": 438975488 + }, + { + "epoch": 1.04, + "learning_rate": 0.00043786359077231695, + "loss": 2.9254, + "theoretical_loss": 3.970690703766763, + "tokens_seen": 439041024 + }, + { + "epoch": 1.04, + "objective/train/docs_used": 539178, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.423835277557373, + "objective/train/theoretical_loss": 3.9706417499811923, + "objective/train/tokens_used": 459550176, + "theoretical_loss": 3.9706417499811923, + "tokens_seen": 439090176 + }, + { + "epoch": 1.04, + "learning_rate": 0.00043785356068204613, + "loss": 2.8201, + "theoretical_loss": 3.970625433611377, + "tokens_seen": 439106560 + }, + { + "epoch": 1.04, + "learning_rate": 0.00043784353059177537, + "loss": 3.0262, + "theoretical_loss": 3.9705601759238864, + "tokens_seen": 439172096 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004378335005015045, + "loss": 3.1224, + "theoretical_loss": 3.97049493070005, + "tokens_seen": 439237632 + }, + { + "epoch": 1.04, + "learning_rate": 0.00043782347041123373, + "loss": 3.0464, + "theoretical_loss": 3.970429697935627, + "tokens_seen": 439303168 + }, + { + "epoch": 1.04, + "learning_rate": 0.00043781344032096286, + "loss": 2.9987, + "theoretical_loss": 3.9703644776263802, + "tokens_seen": 439368704 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004378034102306921, + "loss": 3.0031, + "theoretical_loss": 3.9702992697680752, + "tokens_seen": 439434240 + }, + { + "epoch": 1.04, + "learning_rate": 0.00043779338014042127, + "loss": 2.8794, + "theoretical_loss": 3.970234074356477, + "tokens_seen": 439499776 + }, + { + "epoch": 1.04, + "learning_rate": 0.00043778335005015045, + "loss": 3.1284, + "theoretical_loss": 3.9701688913873565, + "tokens_seen": 439565312 + }, + { + "epoch": 1.04, + "learning_rate": 0.00043777331995987963, + "loss": 2.988, + "theoretical_loss": 3.970103720856483, + "tokens_seen": 439630848 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004377632898696088, + "loss": 2.8583, + "theoretical_loss": 3.9700385627596297, + "tokens_seen": 439696384 + }, + { + "epoch": 1.04, + "learning_rate": 0.000437753259779338, + "loss": 3.0441, + "theoretical_loss": 3.969973417092571, + "tokens_seen": 439761920 + }, + { + "epoch": 1.04, + "learning_rate": 0.00043774322968906723, + "loss": 2.8557, + "theoretical_loss": 3.969908283851085, + "tokens_seen": 439827456 + }, + { + "epoch": 1.04, + "learning_rate": 0.00043773319959879636, + "loss": 2.9863, + "theoretical_loss": 3.9698431630309505, + "tokens_seen": 439892992 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004377231695085256, + "loss": 3.1955, + "theoretical_loss": 3.969778054627948, + "tokens_seen": 439958528 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004377131394182547, + "loss": 2.8304, + "theoretical_loss": 3.969712958637862, + "tokens_seen": 440024064 + }, + { + "epoch": 1.04, + "learning_rate": 0.00043770310932798396, + "loss": 2.9701, + "theoretical_loss": 3.969647875056476, + "tokens_seen": 440089600 + }, + { + "epoch": 1.04, + "learning_rate": 0.00043769307923771314, + "loss": 2.954, + "theoretical_loss": 3.9695828038795784, + "tokens_seen": 440155136 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004376830491474423, + "loss": 2.7975, + "theoretical_loss": 3.969517745102958, + "tokens_seen": 440220672 + }, + { + "epoch": 1.04, + "learning_rate": 0.00043767301905717155, + "loss": 2.7506, + "theoretical_loss": 3.9694526987224075, + "tokens_seen": 440286208 + }, + { + "epoch": 1.04, + "learning_rate": 0.00043766298896690073, + "loss": 2.9735, + "theoretical_loss": 3.9693876647337185, + "tokens_seen": 440351744 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004376529588766299, + "loss": 2.7676, + "theoretical_loss": 3.969322643132688, + "tokens_seen": 440417280 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004376429287863591, + "loss": 2.9377, + "theoretical_loss": 3.969257633915113, + "tokens_seen": 440482816 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004376328986960883, + "loss": 2.7283, + "theoretical_loss": 3.9691926370767936, + "tokens_seen": 440548352 + }, + { + "epoch": 1.04, + "learning_rate": 0.00043762286860581746, + "loss": 2.9292, + "theoretical_loss": 3.969127652613531, + "tokens_seen": 440613888 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004376128385155467, + "loss": 3.1896, + "theoretical_loss": 3.9690626805211293, + "tokens_seen": 440679424 + }, + { + "epoch": 1.04, + "objective/train/docs_used": 540524, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.3463034629821777, + "objective/train/theoretical_loss": 3.9690139595676817, + "objective/train/tokens_used": 461188576, + "theoretical_loss": 3.9690139595676817, + "tokens_seen": 440728576 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004376028084252758, + "loss": 3.082, + "theoretical_loss": 3.9689977207953935, + "tokens_seen": 440744960 + }, + { + "epoch": 1.04, + "learning_rate": 0.00043759277833500506, + "loss": 2.7782, + "theoretical_loss": 3.9689327734321327, + "tokens_seen": 440810496 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004375827482447342, + "loss": 3.0849, + "theoretical_loss": 3.9688678384271556, + "tokens_seen": 440876032 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004375727181544634, + "loss": 3.0616, + "theoretical_loss": 3.9688029157762745, + "tokens_seen": 440941568 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004375626880641926, + "loss": 3.1015, + "theoretical_loss": 3.968738005475304, + "tokens_seen": 441007104 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004375526579739218, + "loss": 2.913, + "theoretical_loss": 3.9686731075200594, + "tokens_seen": 441072640 + }, + { + "epoch": 1.04, + "learning_rate": 0.00043754262788365096, + "loss": 2.9542, + "theoretical_loss": 3.9686082219063588, + "tokens_seen": 441138176 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004375325977933802, + "loss": 2.8682, + "theoretical_loss": 3.968543348630022, + "tokens_seen": 441203712 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004375225677031093, + "loss": 2.7976, + "theoretical_loss": 3.968478487686872, + "tokens_seen": 441269248 + }, + { + "epoch": 1.04, + "learning_rate": 0.00043751253761283856, + "loss": 2.8125, + "theoretical_loss": 3.9684136390727325, + "tokens_seen": 441334784 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004375025075225677, + "loss": 2.6787, + "theoretical_loss": 3.9683488027834293, + "tokens_seen": 441400320 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004374924774322969, + "loss": 2.9182, + "theoretical_loss": 3.9682839788147906, + "tokens_seen": 441465856 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004374824473420261, + "loss": 2.6642, + "theoretical_loss": 3.9682191671626477, + "tokens_seen": 441531392 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004374724172517553, + "loss": 2.9396, + "theoretical_loss": 3.968154367822832, + "tokens_seen": 441596928 + }, + { + "epoch": 1.04, + "learning_rate": 0.00043746238716148447, + "loss": 2.8495, + "theoretical_loss": 3.968089580791178, + "tokens_seen": 441662464 + }, + { + "epoch": 1.04, + "learning_rate": 0.00043745235707121365, + "loss": 3.2537, + "theoretical_loss": 3.9680248060635215, + "tokens_seen": 441728000 + }, + { + "epoch": 1.04, + "learning_rate": 0.00043744232698094283, + "loss": 3.1128, + "theoretical_loss": 3.967960043635702, + "tokens_seen": 441793536 + }, + { + "epoch": 1.04, + "learning_rate": 0.00043743229689067206, + "loss": 2.9745, + "theoretical_loss": 3.9678952935035587, + "tokens_seen": 441859072 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004374222668004012, + "loss": 2.9838, + "theoretical_loss": 3.967830555662935, + "tokens_seen": 441924608 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004374122367101304, + "loss": 3.0596, + "theoretical_loss": 3.9677658301096743, + "tokens_seen": 441990144 + }, + { + "epoch": 1.04, + "learning_rate": 0.00043740220661985955, + "loss": 3.0076, + "theoretical_loss": 3.967701116839624, + "tokens_seen": 442055680 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004373921765295888, + "loss": 3.0611, + "theoretical_loss": 3.9676364158486326, + "tokens_seen": 442121216 + }, + { + "epoch": 1.04, + "learning_rate": 0.00043738214643931797, + "loss": 2.8529, + "theoretical_loss": 3.9675717271325492, + "tokens_seen": 442186752 + }, + { + "epoch": 1.04, + "learning_rate": 0.00043737211634904715, + "loss": 2.7567, + "theoretical_loss": 3.967507050687228, + "tokens_seen": 442252288 + }, + { + "epoch": 1.04, + "learning_rate": 0.00043736208625877633, + "loss": 3.0194, + "theoretical_loss": 3.967442386508522, + "tokens_seen": 442317824 + }, + { + "epoch": 1.04, + "objective/train/docs_used": 541315, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.549229145050049, + "objective/train/theoretical_loss": 3.9673938964219673, + "objective/train/tokens_used": 462826976, + "theoretical_loss": 3.9673938964219673, + "tokens_seen": 442366976 + }, + { + "epoch": 1.04, + "learning_rate": 0.00043735205616850557, + "loss": 2.6802, + "theoretical_loss": 3.967377734592289, + "tokens_seen": 442383360 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004373420260782347, + "loss": 2.9142, + "theoretical_loss": 3.9673130949343873, + "tokens_seen": 442448896 + }, + { + "epoch": 1.04, + "learning_rate": 0.00043733199598796393, + "loss": 3.0676, + "theoretical_loss": 3.967248467530677, + "tokens_seen": 442514432 + }, + { + "epoch": 1.04, + "learning_rate": 0.00043732196589769306, + "loss": 3.0618, + "theoretical_loss": 3.967183852377021, + "tokens_seen": 442579968 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004373119358074223, + "loss": 3.0316, + "theoretical_loss": 3.9671192494692837, + "tokens_seen": 442645504 + }, + { + "epoch": 1.04, + "learning_rate": 0.00043730190571715147, + "loss": 3.0938, + "theoretical_loss": 3.9670546588033315, + "tokens_seen": 442711040 + }, + { + "epoch": 1.04, + "learning_rate": 0.00043729187562688065, + "loss": 2.8994, + "theoretical_loss": 3.9669900803750338, + "tokens_seen": 442776576 + }, + { + "epoch": 1.04, + "learning_rate": 0.00043728184553660983, + "loss": 3.0618, + "theoretical_loss": 3.96692551418026, + "tokens_seen": 442842112 + }, + { + "epoch": 1.04, + "learning_rate": 0.000437271815446339, + "loss": 3.1426, + "theoretical_loss": 3.966860960214883, + "tokens_seen": 442907648 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004372617853560682, + "loss": 2.9978, + "theoretical_loss": 3.966796418474779, + "tokens_seen": 442973184 + }, + { + "epoch": 1.04, + "learning_rate": 0.00043725175526579743, + "loss": 2.6247, + "theoretical_loss": 3.966731888955823, + "tokens_seen": 443038720 + }, + { + "epoch": 1.04, + "learning_rate": 0.00043724172517552656, + "loss": 2.8093, + "theoretical_loss": 3.966667371653893, + "tokens_seen": 443104256 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004372316950852558, + "loss": 2.9878, + "theoretical_loss": 3.9666028665648714, + "tokens_seen": 443169792 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004372216649949849, + "loss": 2.8664, + "theoretical_loss": 3.96653837368464, + "tokens_seen": 443235328 + }, + { + "epoch": 1.04, + "learning_rate": 0.00043721163490471416, + "loss": 2.9273, + "theoretical_loss": 3.9664738930090833, + "tokens_seen": 443300864 + }, + { + "epoch": 1.04, + "learning_rate": 0.00043720160481444334, + "loss": 2.9889, + "theoretical_loss": 3.9664094245340875, + "tokens_seen": 443366400 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004371915747241725, + "loss": 2.9439, + "theoretical_loss": 3.9663449682555423, + "tokens_seen": 443431936 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004371815446339017, + "loss": 3.0401, + "theoretical_loss": 3.966280524169337, + "tokens_seen": 443497472 + }, + { + "epoch": 1.04, + "learning_rate": 0.00043717151454363093, + "loss": 3.0757, + "theoretical_loss": 3.966216092271365, + "tokens_seen": 443563008 + }, + { + "epoch": 1.04, + "learning_rate": 0.00043716148445336006, + "loss": 2.9564, + "theoretical_loss": 3.966151672557521, + "tokens_seen": 443628544 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004371514543630893, + "loss": 2.8941, + "theoretical_loss": 3.9660872650237016, + "tokens_seen": 443694080 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004371414242728184, + "loss": 3.0021, + "theoretical_loss": 3.966022869665804, + "tokens_seen": 443759616 + }, + { + "epoch": 1.04, + "learning_rate": 0.00043713139418254766, + "loss": 3.0318, + "theoretical_loss": 3.96595848647973, + "tokens_seen": 443825152 + }, + { + "epoch": 1.04, + "learning_rate": 0.00043712136409227684, + "loss": 3.0716, + "theoretical_loss": 3.9658941154613823, + "tokens_seen": 443890688 + }, + { + "epoch": 1.04, + "learning_rate": 0.000437111334002006, + "loss": 2.9925, + "theoretical_loss": 3.9658297566066647, + "tokens_seen": 443956224 + }, + { + "epoch": 1.04, + "objective/train/docs_used": 542939, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.203843355178833, + "objective/train/theoretical_loss": 3.965781495445546, + "objective/train/tokens_used": 464465376, + "theoretical_loss": 3.965781495445546, + "tokens_seen": 444005376 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004371013039117352, + "loss": 3.0367, + "theoretical_loss": 3.9657654099114836, + "tokens_seen": 444021760 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004370912738214644, + "loss": 3.0506, + "theoretical_loss": 3.965701075371748, + "tokens_seen": 444087296 + }, + { + "epoch": 1.04, + "learning_rate": 0.00043708124373119356, + "loss": 2.9351, + "theoretical_loss": 3.965636752983368, + "tokens_seen": 444152832 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004370712136409228, + "loss": 2.8604, + "theoretical_loss": 3.965572442742256, + "tokens_seen": 444218368 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004370611835506519, + "loss": 3.1753, + "theoretical_loss": 3.9655081446443265, + "tokens_seen": 444283904 + }, + { + "epoch": 1.04, + "learning_rate": 0.00043705115346038116, + "loss": 2.8183, + "theoretical_loss": 3.9654438586854965, + "tokens_seen": 444349440 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004370411233701103, + "loss": 3.1016, + "theoretical_loss": 3.965379584861684, + "tokens_seen": 444414976 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004370310932798395, + "loss": 2.6801, + "theoretical_loss": 3.965315323168808, + "tokens_seen": 444480512 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004370210631895687, + "loss": 2.9384, + "theoretical_loss": 3.9652510736027926, + "tokens_seen": 444546048 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004370110330992979, + "loss": 2.8253, + "theoretical_loss": 3.9651868361595612, + "tokens_seen": 444611584 + }, + { + "epoch": 1.04, + "learning_rate": 0.00043700100300902707, + "loss": 2.925, + "theoretical_loss": 3.9651226108350395, + "tokens_seen": 444677120 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004369909729187563, + "loss": 2.9335, + "theoretical_loss": 3.965058397625157, + "tokens_seen": 444742656 + }, + { + "epoch": 1.04, + "learning_rate": 0.00043698094282848543, + "loss": 2.9976, + "theoretical_loss": 3.9649941965258435, + "tokens_seen": 444808192 + }, + { + "epoch": 1.04, + "learning_rate": 0.00043697091273821467, + "loss": 2.7717, + "theoretical_loss": 3.9649300075330305, + "tokens_seen": 444873728 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004369608826479438, + "loss": 2.8541, + "theoretical_loss": 3.964865830642653, + "tokens_seen": 444939264 + }, + { + "epoch": 1.04, + "learning_rate": 0.00043695085255767303, + "loss": 3.0042, + "theoretical_loss": 3.964801665850646, + "tokens_seen": 445004800 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004369408224674022, + "loss": 3.0289, + "theoretical_loss": 3.964737513152949, + "tokens_seen": 445070336 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004369307923771314, + "loss": 3.0322, + "theoretical_loss": 3.9646733725455006, + "tokens_seen": 445135872 + }, + { + "epoch": 1.04, + "learning_rate": 0.0004369207622868606, + "loss": 3.0371, + "theoretical_loss": 3.964609244024243, + "tokens_seen": 445201408 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043691073219658975, + "loss": 2.8602, + "theoretical_loss": 3.9645451275851205, + "tokens_seen": 445266944 + }, + { + "epoch": 1.05, + "learning_rate": 0.000436900702106319, + "loss": 2.9065, + "theoretical_loss": 3.9644810232240792, + "tokens_seen": 445332480 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043689067201604817, + "loss": 2.9665, + "theoretical_loss": 3.9644169309370665, + "tokens_seen": 445398016 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043688064192577735, + "loss": 2.9056, + "theoretical_loss": 3.964352850720032, + "tokens_seen": 445463552 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043687061183550653, + "loss": 2.8584, + "theoretical_loss": 3.9642887825689286, + "tokens_seen": 445529088 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043686058174523577, + "loss": 2.9096, + "theoretical_loss": 3.964224726479708, + "tokens_seen": 445594624 + }, + { + "epoch": 1.05, + "objective/train/docs_used": 543579, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.705620765686035, + "objective/train/theoretical_loss": 3.964176692325972, + "objective/train/tokens_used": 466103776, + "theoretical_loss": 3.964176692325972, + "tokens_seen": 445643776 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004368505516549649, + "loss": 3.0128, + "theoretical_loss": 3.9641606824483278, + "tokens_seen": 445660160 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043684052156469413, + "loss": 3.0941, + "theoretical_loss": 3.9640966504707444, + "tokens_seen": 445725696 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043683049147442326, + "loss": 2.9789, + "theoretical_loss": 3.964032630542918, + "tokens_seen": 445791232 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004368204613841525, + "loss": 2.81, + "theoretical_loss": 3.96396862266081, + "tokens_seen": 445856768 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043681043129388167, + "loss": 2.9512, + "theoretical_loss": 3.9639046268203835, + "tokens_seen": 445922304 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043680040120361085, + "loss": 3.0781, + "theoretical_loss": 3.963840643017604, + "tokens_seen": 445987840 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043679037111334003, + "loss": 2.7061, + "theoretical_loss": 3.9637766712484384, + "tokens_seen": 446053376 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004367803410230692, + "loss": 3.125, + "theoretical_loss": 3.9637127115088573, + "tokens_seen": 446118912 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004367703109327984, + "loss": 2.981, + "theoretical_loss": 3.963648763794831, + "tokens_seen": 446184448 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043676028084252763, + "loss": 2.7527, + "theoretical_loss": 3.9635848281023325, + "tokens_seen": 446249984 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043675025075225676, + "loss": 2.8724, + "theoretical_loss": 3.9635209044273365, + "tokens_seen": 446315520 + }, + { + "epoch": 1.05, + "learning_rate": 0.000436740220661986, + "loss": 2.9677, + "theoretical_loss": 3.9634569927658214, + "tokens_seen": 446381056 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004367301905717151, + "loss": 2.9814, + "theoretical_loss": 3.9633930931137655, + "tokens_seen": 446446592 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043672016048144436, + "loss": 2.9977, + "theoretical_loss": 3.963329205467149, + "tokens_seen": 446512128 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043671013039117354, + "loss": 2.8533, + "theoretical_loss": 3.9632653298219562, + "tokens_seen": 446577664 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004367001003009027, + "loss": 2.7676, + "theoretical_loss": 3.96320146617417, + "tokens_seen": 446643200 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004366900702106319, + "loss": 3.0384, + "theoretical_loss": 3.9631376145197796, + "tokens_seen": 446708736 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043668004012036113, + "loss": 3.0348, + "theoretical_loss": 3.963073774854771, + "tokens_seen": 446774272 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043667001003009026, + "loss": 3.0659, + "theoretical_loss": 3.9630099471751365, + "tokens_seen": 446839808 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004366599799398195, + "loss": 3.0177, + "theoretical_loss": 3.962946131476868, + "tokens_seen": 446905344 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004366499498495486, + "loss": 2.9298, + "theoretical_loss": 3.9628823277559597, + "tokens_seen": 446970880 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043663991975927786, + "loss": 2.9698, + "theoretical_loss": 3.9628185360084087, + "tokens_seen": 447036416 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043662988966900704, + "loss": 3.0568, + "theoretical_loss": 3.962754756230213, + "tokens_seen": 447101952 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004366198595787362, + "loss": 2.9263, + "theoretical_loss": 3.9626909884173727, + "tokens_seen": 447167488 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004366098294884654, + "loss": 3.1906, + "theoretical_loss": 3.9626272325658896, + "tokens_seen": 447233024 + }, + { + "epoch": 1.05, + "objective/train/docs_used": 544310, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.0421924591064453, + "objective/train/theoretical_loss": 3.962579423524514, + "objective/train/tokens_used": 467742176, + "theoretical_loss": 3.962579423524514, + "tokens_seen": 447282176 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004365997993981946, + "loss": 3.0931, + "theoretical_loss": 3.9625634886717678, + "tokens_seen": 447298560 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043658976930792376, + "loss": 2.9716, + "theoretical_loss": 3.962499756731014, + "tokens_seen": 447364096 + }, + { + "epoch": 1.05, + "learning_rate": 0.000436579739217653, + "loss": 3.0392, + "theoretical_loss": 3.9624360367396347, + "tokens_seen": 447429632 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043656970912738213, + "loss": 2.9594, + "theoretical_loss": 3.9623723286936414, + "tokens_seen": 447495168 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043655967903711136, + "loss": 3.1369, + "theoretical_loss": 3.9623086325890444, + "tokens_seen": 447560704 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004365496489468405, + "loss": 2.9239, + "theoretical_loss": 3.9622449484218585, + "tokens_seen": 447626240 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004365396188565697, + "loss": 3.2243, + "theoretical_loss": 3.9621812761880975, + "tokens_seen": 447691776 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004365295887662989, + "loss": 2.8872, + "theoretical_loss": 3.9621176158837814, + "tokens_seen": 447757312 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004365195586760281, + "loss": 3.1265, + "theoretical_loss": 3.9620539675049278, + "tokens_seen": 447822848 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043650952858575727, + "loss": 2.9101, + "theoretical_loss": 3.9619903310475575, + "tokens_seen": 447888384 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004364994984954865, + "loss": 2.9973, + "theoretical_loss": 3.9619267065076955, + "tokens_seen": 447953920 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043648946840521563, + "loss": 2.9611, + "theoretical_loss": 3.961863093881366, + "tokens_seen": 448019456 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043647943831494487, + "loss": 3.2598, + "theoretical_loss": 3.9617994931645955, + "tokens_seen": 448084992 + }, + { + "epoch": 1.05, + "learning_rate": 0.000436469408224674, + "loss": 2.9044, + "theoretical_loss": 3.9617359043534135, + "tokens_seen": 448150528 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043645937813440323, + "loss": 2.9648, + "theoretical_loss": 3.961672327443851, + "tokens_seen": 448216064 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004364493480441324, + "loss": 2.9154, + "theoretical_loss": 3.96160876243194, + "tokens_seen": 448281600 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004364393179538616, + "loss": 2.9511, + "theoretical_loss": 3.961545209313715, + "tokens_seen": 448347136 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043642928786359077, + "loss": 2.8572, + "theoretical_loss": 3.961481668085214, + "tokens_seen": 448412672 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043641925777331995, + "loss": 3.2968, + "theoretical_loss": 3.9614181387424745, + "tokens_seen": 448478208 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043640922768304913, + "loss": 2.8314, + "theoretical_loss": 3.9613546212815365, + "tokens_seen": 448543744 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043639919759277837, + "loss": 2.9011, + "theoretical_loss": 3.961291115698442, + "tokens_seen": 448609280 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004363891675025075, + "loss": 2.757, + "theoretical_loss": 3.9612276219892366, + "tokens_seen": 448674816 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043637913741223673, + "loss": 2.8699, + "theoretical_loss": 3.9611641401499647, + "tokens_seen": 448740352 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004363691073219659, + "loss": 3.0423, + "theoretical_loss": 3.961100670176675, + "tokens_seen": 448805888 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004363590772316951, + "loss": 2.9675, + "theoretical_loss": 3.961037212065418, + "tokens_seen": 448871424 + }, + { + "epoch": 1.05, + "objective/train/docs_used": 544864, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.588275909423828, + "objective/train/theoretical_loss": 3.9609896262640576, + "objective/train/tokens_used": 469380576, + "theoretical_loss": 3.9609896262640576, + "tokens_seen": 448920576 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004363490471414243, + "loss": 2.9927, + "theoretical_loss": 3.960973765812244, + "tokens_seen": 448936960 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043633901705115346, + "loss": 3.0827, + "theoretical_loss": 3.9609103314132064, + "tokens_seen": 449002496 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043632898696088264, + "loss": 3.0377, + "theoretical_loss": 3.960846908864362, + "tokens_seen": 449068032 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043631895687061187, + "loss": 2.8994, + "theoretical_loss": 3.9607834981617676, + "tokens_seen": 449133568 + }, + { + "epoch": 1.05, + "learning_rate": 0.000436308926780341, + "loss": 2.9719, + "theoretical_loss": 3.9607200993014824, + "tokens_seen": 449199104 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043629889669007023, + "loss": 2.9095, + "theoretical_loss": 3.960656712279567, + "tokens_seen": 449264640 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043628886659979936, + "loss": 2.9354, + "theoretical_loss": 3.960593337092086, + "tokens_seen": 449330176 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004362788365095286, + "loss": 3.012, + "theoretical_loss": 3.9605299737351025, + "tokens_seen": 449395712 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004362688064192578, + "loss": 3.1365, + "theoretical_loss": 3.9604666222046845, + "tokens_seen": 449461248 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043625877632898696, + "loss": 2.8716, + "theoretical_loss": 3.9604032824968995, + "tokens_seen": 449526784 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043624874623871614, + "loss": 2.8419, + "theoretical_loss": 3.960339954607819, + "tokens_seen": 449592320 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004362387161484453, + "loss": 3.0657, + "theoretical_loss": 3.9602766385335153, + "tokens_seen": 449657856 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004362286860581745, + "loss": 3.0267, + "theoretical_loss": 3.9602133342700623, + "tokens_seen": 449723392 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043621865596790374, + "loss": 3.0393, + "theoretical_loss": 3.9601500418135367, + "tokens_seen": 449788928 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043620862587763286, + "loss": 3.1022, + "theoretical_loss": 3.960086761160016, + "tokens_seen": 449854464 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004361985957873621, + "loss": 2.9125, + "theoretical_loss": 3.960023492305581, + "tokens_seen": 449920000 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004361885656970913, + "loss": 2.9061, + "theoretical_loss": 3.959960235246312, + "tokens_seen": 449985536 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043617853560682046, + "loss": 3.0702, + "theoretical_loss": 3.9598969899782936, + "tokens_seen": 450051072 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004361685055165497, + "loss": 2.9372, + "theoretical_loss": 3.9598337564976114, + "tokens_seen": 450116608 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004361584754262788, + "loss": 3.0303, + "theoretical_loss": 3.9597705348003527, + "tokens_seen": 450182144 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043614844533600806, + "loss": 2.8191, + "theoretical_loss": 3.959707324882607, + "tokens_seen": 450247680 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043613841524573724, + "loss": 3.0286, + "theoretical_loss": 3.9596441267404647, + "tokens_seen": 450313216 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004361283851554664, + "loss": 2.9917, + "theoretical_loss": 3.95958094037002, + "tokens_seen": 450378752 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004361183550651956, + "loss": 3.238, + "theoretical_loss": 3.959517765767366, + "tokens_seen": 450444288 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004361083249749248, + "loss": 2.9412, + "theoretical_loss": 3.9594546029286013, + "tokens_seen": 450509824 + }, + { + "epoch": 1.05, + "objective/train/docs_used": 544864, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.3638360500335693, + "objective/train/theoretical_loss": 3.9594072385172328, + "objective/train/tokens_used": 471018976, + "theoretical_loss": 3.9594072385172328, + "tokens_seen": 450558976 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043609829488465397, + "loss": 2.9778, + "theoretical_loss": 3.9593914518498234, + "tokens_seen": 450575360 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004360882647943832, + "loss": 3.1076, + "theoretical_loss": 3.9593283125271332, + "tokens_seen": 450640896 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043607823470411233, + "loss": 3.0533, + "theoretical_loss": 3.959265184956633, + "tokens_seen": 450706432 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043606820461384156, + "loss": 3.0076, + "theoretical_loss": 3.9592020691344265, + "tokens_seen": 450771968 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004360581745235707, + "loss": 3.2244, + "theoretical_loss": 3.9591389650566207, + "tokens_seen": 450837504 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004360481444332999, + "loss": 3.0192, + "theoretical_loss": 3.959075872719322, + "tokens_seen": 450903040 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004360381143430291, + "loss": 3.082, + "theoretical_loss": 3.9590127921186413, + "tokens_seen": 450968576 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004360280842527583, + "loss": 3.1096, + "theoretical_loss": 3.95894972325069, + "tokens_seen": 451034112 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043601805416248747, + "loss": 2.8023, + "theoretical_loss": 3.9588866661115816, + "tokens_seen": 451099648 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004360080240722167, + "loss": 3.124, + "theoretical_loss": 3.958823620697431, + "tokens_seen": 451165184 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043599799398194583, + "loss": 2.9392, + "theoretical_loss": 3.958760587004355, + "tokens_seen": 451230720 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043598796389167507, + "loss": 3.0512, + "theoretical_loss": 3.958697565028474, + "tokens_seen": 451296256 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004359779338014042, + "loss": 2.9383, + "theoretical_loss": 3.958634554765908, + "tokens_seen": 451361792 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043596790371113343, + "loss": 3.0894, + "theoretical_loss": 3.95857155621278, + "tokens_seen": 451427328 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004359578736208626, + "loss": 3.1975, + "theoretical_loss": 3.958508569365214, + "tokens_seen": 451492864 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004359478435305918, + "loss": 3.0222, + "theoretical_loss": 3.9584455942193366, + "tokens_seen": 451558400 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043593781344032097, + "loss": 2.9316, + "theoretical_loss": 3.9583826307712764, + "tokens_seen": 451623936 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043592778335005015, + "loss": 3.0144, + "theoretical_loss": 3.958319679017163, + "tokens_seen": 451689472 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043591775325977933, + "loss": 2.9815, + "theoretical_loss": 3.9582567389531285, + "tokens_seen": 451755008 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043590772316950857, + "loss": 2.9248, + "theoretical_loss": 3.9581938105753065, + "tokens_seen": 451820544 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004358976930792377, + "loss": 2.924, + "theoretical_loss": 3.958130893879833, + "tokens_seen": 451886080 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043588766298896693, + "loss": 2.9269, + "theoretical_loss": 3.9580679888628456, + "tokens_seen": 451951616 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004358776328986961, + "loss": 3.0172, + "theoretical_loss": 3.958005095520483, + "tokens_seen": 452017152 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004358676028084253, + "loss": 3.0345, + "theoretical_loss": 3.9579422138488862, + "tokens_seen": 452082688 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004358575727181545, + "loss": 3.2033, + "theoretical_loss": 3.9578793438441986, + "tokens_seen": 452148224 + }, + { + "epoch": 1.05, + "objective/train/docs_used": 545628, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9914121627807617, + "objective/train/theoretical_loss": 3.9578321989947725, + "objective/train/tokens_used": 472657376, + "theoretical_loss": 3.9578321989947725, + "tokens_seen": 452197376 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043584754262788366, + "loss": 2.9125, + "theoretical_loss": 3.9578164855025646, + "tokens_seen": 452213760 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043583751253761284, + "loss": 2.7379, + "theoretical_loss": 3.9577536388201313, + "tokens_seen": 452279296 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043582748244734207, + "loss": 3.1926, + "theoretical_loss": 3.957690803793047, + "tokens_seen": 452344832 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004358174523570712, + "loss": 3.1275, + "theoretical_loss": 3.9576279804174614, + "tokens_seen": 452410368 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043580742226680043, + "loss": 3.0179, + "theoretical_loss": 3.957565168689528, + "tokens_seen": 452475904 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043579739217652956, + "loss": 3.0056, + "theoretical_loss": 3.9575023686053985, + "tokens_seen": 452541440 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004357873620862588, + "loss": 3.1728, + "theoretical_loss": 3.957439580161231, + "tokens_seen": 452606976 + }, + { + "epoch": 1.05, + "learning_rate": 0.000435777331995988, + "loss": 3.027, + "theoretical_loss": 3.9573768033531813, + "tokens_seen": 452672512 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043576730190571716, + "loss": 2.8647, + "theoretical_loss": 3.95731403817741, + "tokens_seen": 452738048 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043575727181544634, + "loss": 2.8473, + "theoretical_loss": 3.9572512846300776, + "tokens_seen": 452803584 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004357472417251755, + "loss": 2.6547, + "theoretical_loss": 3.9571885427073474, + "tokens_seen": 452869120 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004357372116349047, + "loss": 2.7046, + "theoretical_loss": 3.9571258124053843, + "tokens_seen": 452934656 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043572718154463394, + "loss": 2.9029, + "theoretical_loss": 3.957063093720355, + "tokens_seen": 453000192 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043571715145436306, + "loss": 3.0426, + "theoretical_loss": 3.957000386648428, + "tokens_seen": 453065728 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004357071213640923, + "loss": 2.8844, + "theoretical_loss": 3.9569376911857734, + "tokens_seen": 453131264 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004356970912738215, + "loss": 3.0411, + "theoretical_loss": 3.9568750073285637, + "tokens_seen": 453196800 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043568706118355066, + "loss": 2.8895, + "theoretical_loss": 3.9568123350729727, + "tokens_seen": 453262336 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043567703109327984, + "loss": 3.0225, + "theoretical_loss": 3.956749674415176, + "tokens_seen": 453327872 + }, + { + "epoch": 1.05, + "learning_rate": 0.000435667001003009, + "loss": 3.0718, + "theoretical_loss": 3.9566870253513513, + "tokens_seen": 453393408 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004356569709127382, + "loss": 3.0213, + "theoretical_loss": 3.956624387877678, + "tokens_seen": 453458944 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043564694082246744, + "loss": 3.0282, + "theoretical_loss": 3.956561761990338, + "tokens_seen": 453524480 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043563691073219657, + "loss": 2.9475, + "theoretical_loss": 3.9564991476855136, + "tokens_seen": 453590016 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004356268806419258, + "loss": 2.8315, + "theoretical_loss": 3.956436544959389, + "tokens_seen": 453655552 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043561685055165493, + "loss": 3.0489, + "theoretical_loss": 3.9563739538081517, + "tokens_seen": 453721088 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043560682046138417, + "loss": 2.9775, + "theoretical_loss": 3.9563113742279907, + "tokens_seen": 453786624 + }, + { + "epoch": 1.05, + "objective/train/docs_used": 546923, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7837202548980713, + "objective/train/theoretical_loss": 3.956264447134096, + "objective/train/tokens_used": 474295776, + "theoretical_loss": 3.956264447134096, + "tokens_seen": 453835776 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043559679037111335, + "loss": 2.8202, + "theoretical_loss": 3.956248806215095, + "tokens_seen": 453852160 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043558676028084253, + "loss": 2.867, + "theoretical_loss": 3.9561862497656572, + "tokens_seen": 453917696 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004355767301905717, + "loss": 2.8822, + "theoretical_loss": 3.9561237048758713, + "tokens_seen": 453983232 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004355667001003009, + "loss": 2.8726, + "theoretical_loss": 3.9560611715419327, + "tokens_seen": 454048768 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043555667001003007, + "loss": 3.0744, + "theoretical_loss": 3.955998649760039, + "tokens_seen": 454114304 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004355466399197593, + "loss": 2.7415, + "theoretical_loss": 3.9559361395263895, + "tokens_seen": 454179840 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043553660982948843, + "loss": 2.862, + "theoretical_loss": 3.9558736408371855, + "tokens_seen": 454245376 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043552657973921767, + "loss": 2.7807, + "theoretical_loss": 3.9558111536886287, + "tokens_seen": 454310912 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043551654964894685, + "loss": 2.9495, + "theoretical_loss": 3.955748678076925, + "tokens_seen": 454376448 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043550651955867603, + "loss": 3.0138, + "theoretical_loss": 3.9556862139982805, + "tokens_seen": 454441984 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004354964894684052, + "loss": 2.9964, + "theoretical_loss": 3.955623761448903, + "tokens_seen": 454507520 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004354864593781344, + "loss": 2.9618, + "theoretical_loss": 3.955561320425004, + "tokens_seen": 454573056 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004354764292878636, + "loss": 2.9674, + "theoretical_loss": 3.9554988909227933, + "tokens_seen": 454638592 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004354663991975928, + "loss": 2.9339, + "theoretical_loss": 3.955436472938486, + "tokens_seen": 454704128 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043545636910732194, + "loss": 2.8827, + "theoretical_loss": 3.9553740664682966, + "tokens_seen": 454769664 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043544633901705117, + "loss": 2.8341, + "theoretical_loss": 3.9553116715084427, + "tokens_seen": 454835200 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004354363089267803, + "loss": 3.045, + "theoretical_loss": 3.955249288055143, + "tokens_seen": 454900736 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043542627883650953, + "loss": 3.152, + "theoretical_loss": 3.9551869161046187, + "tokens_seen": 454966272 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043541624874623877, + "loss": 2.8714, + "theoretical_loss": 3.955124555653092, + "tokens_seen": 455031808 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004354062186559679, + "loss": 2.9861, + "theoretical_loss": 3.9550622066967875, + "tokens_seen": 455097344 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043539618856569713, + "loss": 3.0678, + "theoretical_loss": 3.9549998692319317, + "tokens_seen": 455162880 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004353861584754263, + "loss": 3.1052, + "theoretical_loss": 3.9549375432547516, + "tokens_seen": 455228416 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004353761283851555, + "loss": 3.0939, + "theoretical_loss": 3.9548752287614772, + "tokens_seen": 455293952 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004353660982948847, + "loss": 3.2468, + "theoretical_loss": 3.9548129257483406, + "tokens_seen": 455359488 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043535606820461386, + "loss": 2.7976, + "theoretical_loss": 3.9547506342115737, + "tokens_seen": 455425024 + }, + { + "epoch": 1.05, + "objective/train/docs_used": 547608, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.1719532012939453, + "objective/train/theoretical_loss": 3.9547039230881023, + "objective/train/tokens_used": 475934176, + "theoretical_loss": 3.9547039230881023, + "tokens_seen": 455474176 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043534603811434304, + "loss": 3.0834, + "theoretical_loss": 3.954688354147413, + "tokens_seen": 455490560 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043533600802407227, + "loss": 2.8907, + "theoretical_loss": 3.9546260855520945, + "tokens_seen": 455556096 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004353259779338014, + "loss": 3.0901, + "theoretical_loss": 3.9545638284218567, + "tokens_seen": 455621632 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043531594784353063, + "loss": 2.9834, + "theoretical_loss": 3.9545015827529406, + "tokens_seen": 455687168 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043530591775325976, + "loss": 2.9154, + "theoretical_loss": 3.9544393485415874, + "tokens_seen": 455752704 + }, + { + "epoch": 1.05, + "learning_rate": 0.000435295887662989, + "loss": 3.0887, + "theoretical_loss": 3.954377125784042, + "tokens_seen": 455818240 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004352858575727182, + "loss": 2.8017, + "theoretical_loss": 3.954314914476549, + "tokens_seen": 455883776 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043527582748244736, + "loss": 2.9674, + "theoretical_loss": 3.9542527146153565, + "tokens_seen": 455949312 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043526579739217654, + "loss": 2.8448, + "theoretical_loss": 3.954190526196714, + "tokens_seen": 456014848 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004352557673019057, + "loss": 2.9596, + "theoretical_loss": 3.9541283492168717, + "tokens_seen": 456080384 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004352457372116349, + "loss": 3.2475, + "theoretical_loss": 3.954066183672083, + "tokens_seen": 456145920 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043523570712136414, + "loss": 2.9562, + "theoretical_loss": 3.9540040295586016, + "tokens_seen": 456211456 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043522567703109326, + "loss": 3.0325, + "theoretical_loss": 3.9539418868726846, + "tokens_seen": 456276992 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004352156469408225, + "loss": 2.9111, + "theoretical_loss": 3.95387975561059, + "tokens_seen": 456342528 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004352056168505517, + "loss": 3.1916, + "theoretical_loss": 3.9538176357685764, + "tokens_seen": 456408064 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043519558676028086, + "loss": 2.9947, + "theoretical_loss": 3.953755527342907, + "tokens_seen": 456473600 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043518555667001004, + "loss": 2.9115, + "theoretical_loss": 3.953693430329844, + "tokens_seen": 456539136 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004351755265797392, + "loss": 3.0481, + "theoretical_loss": 3.953631344725653, + "tokens_seen": 456604672 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004351654964894684, + "loss": 2.8957, + "theoretical_loss": 3.953569270526601, + "tokens_seen": 456670208 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043515546639919764, + "loss": 2.9168, + "theoretical_loss": 3.953507207728956, + "tokens_seen": 456735744 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043514543630892677, + "loss": 2.9412, + "theoretical_loss": 3.9534451563289883, + "tokens_seen": 456801280 + }, + { + "epoch": 1.05, + "learning_rate": 0.000435135406218656, + "loss": 2.9884, + "theoretical_loss": 3.9533831163229705, + "tokens_seen": 456866816 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043512537612838513, + "loss": 3.1086, + "theoretical_loss": 3.953321087707177, + "tokens_seen": 456932352 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043511534603811437, + "loss": 3.0378, + "theoretical_loss": 3.9532590704778823, + "tokens_seen": 456997888 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043510531594784355, + "loss": 3.1419, + "theoretical_loss": 3.9531970646313637, + "tokens_seen": 457063424 + }, + { + "epoch": 1.05, + "objective/train/docs_used": 548950, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.220716714859009, + "objective/train/theoretical_loss": 3.9531505677141836, + "objective/train/tokens_used": 477572576, + "theoretical_loss": 3.9531505677141836, + "tokens_seen": 457112576 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043509528585757273, + "loss": 3.0836, + "theoretical_loss": 3.953135070163901, + "tokens_seen": 457128960 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004350852557673019, + "loss": 2.9593, + "theoretical_loss": 3.9530730870717745, + "tokens_seen": 457194496 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004350752256770311, + "loss": 3.2264, + "theoretical_loss": 3.9530111153512677, + "tokens_seen": 457260032 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043506519558676027, + "loss": 3.0847, + "theoretical_loss": 3.9529491549986644, + "tokens_seen": 457325568 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004350551654964895, + "loss": 2.9372, + "theoretical_loss": 3.9528872060102502, + "tokens_seen": 457391104 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043504513540621863, + "loss": 3.154, + "theoretical_loss": 3.952825268382314, + "tokens_seen": 457456640 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043503510531594787, + "loss": 3.0226, + "theoretical_loss": 3.9527633421111448, + "tokens_seen": 457522176 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043502507522567705, + "loss": 3.0597, + "theoretical_loss": 3.952701427193033, + "tokens_seen": 457587712 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043501504513540623, + "loss": 3.0086, + "theoretical_loss": 3.9526395236242737, + "tokens_seen": 457653248 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004350050150451354, + "loss": 3.2069, + "theoretical_loss": 3.9525776314011605, + "tokens_seen": 457718784 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004349949849548646, + "loss": 3.0413, + "theoretical_loss": 3.95251575051999, + "tokens_seen": 457784320 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004349849548645938, + "loss": 2.9921, + "theoretical_loss": 3.9524538809770604, + "tokens_seen": 457849856 + }, + { + "epoch": 1.05, + "learning_rate": 0.000434974924774323, + "loss": 3.0149, + "theoretical_loss": 3.952392022768672, + "tokens_seen": 457915392 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043496489468405214, + "loss": 2.9887, + "theoretical_loss": 3.952330175891127, + "tokens_seen": 457980928 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043495486459378137, + "loss": 2.9086, + "theoretical_loss": 3.952268340340728, + "tokens_seen": 458046464 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004349448345035105, + "loss": 3.1181, + "theoretical_loss": 3.952206516113781, + "tokens_seen": 458112000 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043493480441323973, + "loss": 3.0618, + "theoretical_loss": 3.952144703206592, + "tokens_seen": 458177536 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004349247743229689, + "loss": 2.952, + "theoretical_loss": 3.952082901615471, + "tokens_seen": 458243072 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004349147442326981, + "loss": 3.0284, + "theoretical_loss": 3.9520211113367276, + "tokens_seen": 458308608 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004349047141424273, + "loss": 3.1181, + "theoretical_loss": 3.9519593323666746, + "tokens_seen": 458374144 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004348946840521565, + "loss": 2.913, + "theoretical_loss": 3.951897564701625, + "tokens_seen": 458439680 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043488465396188564, + "loss": 2.6729, + "theoretical_loss": 3.951835808337895, + "tokens_seen": 458505216 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004348746238716149, + "loss": 3.3398, + "theoretical_loss": 3.9517740632718015, + "tokens_seen": 458570752 + }, + { + "epoch": 1.05, + "learning_rate": 0.000434864593781344, + "loss": 2.8925, + "theoretical_loss": 3.9517123294996646, + "tokens_seen": 458636288 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043485456369107324, + "loss": 2.9762, + "theoretical_loss": 3.951650607017804, + "tokens_seen": 458701824 + }, + { + "epoch": 1.05, + "objective/train/docs_used": 550265, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.620121955871582, + "objective/train/theoretical_loss": 3.9516043225634405, + "objective/train/tokens_used": 479210976, + "theoretical_loss": 3.9516043225634405, + "tokens_seen": 458750976 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004348445336008024, + "loss": 3.1754, + "theoretical_loss": 3.9515888958225425, + "tokens_seen": 458767360 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004348345035105316, + "loss": 2.9674, + "theoretical_loss": 3.951527195910205, + "tokens_seen": 458832896 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004348244734202608, + "loss": 3.0026, + "theoretical_loss": 3.9514655072771165, + "tokens_seen": 458898432 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043481444332998996, + "loss": 2.9806, + "theoretical_loss": 3.951403829919606, + "tokens_seen": 458963968 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043480441323971914, + "loss": 3.0238, + "theoretical_loss": 3.951342163834001, + "tokens_seen": 459029504 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004347943831494484, + "loss": 2.9712, + "theoretical_loss": 3.9512805090166347, + "tokens_seen": 459095040 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004347843530591775, + "loss": 2.9981, + "theoretical_loss": 3.9512188654638387, + "tokens_seen": 459160576 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043477432296890674, + "loss": 2.9099, + "theoretical_loss": 3.951157233171948, + "tokens_seen": 459226112 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043476429287863587, + "loss": 3.0506, + "theoretical_loss": 3.9510956121372978, + "tokens_seen": 459291648 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004347542627883651, + "loss": 2.9484, + "theoretical_loss": 3.951034002356228, + "tokens_seen": 459357184 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004347442326980943, + "loss": 2.9222, + "theoretical_loss": 3.9509724038250775, + "tokens_seen": 459422720 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043473420260782346, + "loss": 3.1718, + "theoretical_loss": 3.950910816540187, + "tokens_seen": 459488256 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043472417251755265, + "loss": 3.1192, + "theoretical_loss": 3.9508492404979005, + "tokens_seen": 459553792 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004347141424272819, + "loss": 3.248, + "theoretical_loss": 3.950787675694562, + "tokens_seen": 459619328 + }, + { + "epoch": 1.05, + "learning_rate": 0.000434704112337011, + "loss": 2.9839, + "theoretical_loss": 3.950726122126519, + "tokens_seen": 459684864 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043469408224674024, + "loss": 3.1868, + "theoretical_loss": 3.9506645797901196, + "tokens_seen": 459750400 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004346840521564694, + "loss": 2.8167, + "theoretical_loss": 3.950603048681714, + "tokens_seen": 459815936 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004346740220661986, + "loss": 3.07, + "theoretical_loss": 3.950541528797652, + "tokens_seen": 459881472 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043466399197592784, + "loss": 2.9742, + "theoretical_loss": 3.9504800201342896, + "tokens_seen": 459947008 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043465396188565697, + "loss": 3.0656, + "theoretical_loss": 3.9504185226879804, + "tokens_seen": 460012544 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004346439317953862, + "loss": 3.0949, + "theoretical_loss": 3.950357036455081, + "tokens_seen": 460078080 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043463390170511533, + "loss": 2.9103, + "theoretical_loss": 3.950295561431951, + "tokens_seen": 460143616 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043462387161484457, + "loss": 3.0709, + "theoretical_loss": 3.95023409761495, + "tokens_seen": 460209152 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043461384152457375, + "loss": 2.99, + "theoretical_loss": 3.950172645000439, + "tokens_seen": 460274688 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043460381143430293, + "loss": 3.1795, + "theoretical_loss": 3.9501112035847834, + "tokens_seen": 460340224 + }, + { + "epoch": 1.05, + "objective/train/docs_used": 550910, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.578948736190796, + "objective/train/theoretical_loss": 3.950065129870103, + "objective/train/tokens_used": 480849376, + "theoretical_loss": 3.950065129870103, + "tokens_seen": 460389376 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004345937813440321, + "loss": 2.9974, + "theoretical_loss": 3.950049773364347, + "tokens_seen": 460405760 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004345837512537613, + "loss": 3.1578, + "theoretical_loss": 3.9499883543354977, + "tokens_seen": 460471296 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043457372116349047, + "loss": 3.0666, + "theoretical_loss": 3.9499269464946036, + "tokens_seen": 460536832 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004345636910732197, + "loss": 3.0071, + "theoretical_loss": 3.949865549838035, + "tokens_seen": 460602368 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043455366098294883, + "loss": 3.25, + "theoretical_loss": 3.9498041643621646, + "tokens_seen": 460667904 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043454363089267807, + "loss": 3.1709, + "theoretical_loss": 3.949742790063366, + "tokens_seen": 460733440 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043453360080240725, + "loss": 2.9825, + "theoretical_loss": 3.9496814269380143, + "tokens_seen": 460798976 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043452357071213643, + "loss": 2.7215, + "theoretical_loss": 3.9496200749824864, + "tokens_seen": 460864512 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004345135406218656, + "loss": 3.2174, + "theoretical_loss": 3.949558734193162, + "tokens_seen": 460930048 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004345035105315948, + "loss": 3.0671, + "theoretical_loss": 3.9494974045664213, + "tokens_seen": 460995584 + }, + { + "epoch": 1.05, + "learning_rate": 0.000434493480441324, + "loss": 2.8494, + "theoretical_loss": 3.949436086098646, + "tokens_seen": 461061120 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004344834503510532, + "loss": 3.0576, + "theoretical_loss": 3.9493747787862206, + "tokens_seen": 461126656 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043447342026078234, + "loss": 2.9423, + "theoretical_loss": 3.9493134826255303, + "tokens_seen": 461192192 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043446339017051157, + "loss": 3.0723, + "theoretical_loss": 3.9492521976129624, + "tokens_seen": 461257728 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004344533600802407, + "loss": 2.9127, + "theoretical_loss": 3.9491909237449065, + "tokens_seen": 461323264 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043444332998996993, + "loss": 2.8746, + "theoretical_loss": 3.9491296610177526, + "tokens_seen": 461388800 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004344332998996991, + "loss": 3.0022, + "theoretical_loss": 3.949068409427893, + "tokens_seen": 461454336 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004344232698094283, + "loss": 2.8165, + "theoretical_loss": 3.949007168971722, + "tokens_seen": 461519872 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004344132397191575, + "loss": 3.2636, + "theoretical_loss": 3.9489459396456343, + "tokens_seen": 461585408 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004344032096288867, + "loss": 3.1145, + "theoretical_loss": 3.9488847214460288, + "tokens_seen": 461650944 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043439317953861584, + "loss": 2.9202, + "theoretical_loss": 3.9488235143693036, + "tokens_seen": 461716480 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004343831494483451, + "loss": 2.8524, + "theoretical_loss": 3.94876231841186, + "tokens_seen": 461782016 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004343731193580742, + "loss": 3.0581, + "theoretical_loss": 3.948701133570099, + "tokens_seen": 461847552 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043436308926780344, + "loss": 3.0472, + "theoretical_loss": 3.9486399598404263, + "tokens_seen": 461913088 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004343530591775326, + "loss": 2.8361, + "theoretical_loss": 3.948578797219247, + "tokens_seen": 461978624 + }, + { + "epoch": 1.05, + "objective/train/docs_used": 552255, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.0387556552886963, + "objective/train/theoretical_loss": 3.9485329325411502, + "objective/train/tokens_used": 482487776, + "theoretical_loss": 3.9485329325411502, + "tokens_seen": 462027776 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004343430290872618, + "loss": 2.9454, + "theoretical_loss": 3.9485176457029683, + "tokens_seen": 462044160 + }, + { + "epoch": 1.05, + "learning_rate": 0.000434332998996991, + "loss": 2.9201, + "theoretical_loss": 3.948456505287999, + "tokens_seen": 462109696 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043432296890672016, + "loss": 2.7927, + "theoretical_loss": 3.9483953759707515, + "tokens_seen": 462175232 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043431293881644934, + "loss": 3.0389, + "theoretical_loss": 3.948334257747636, + "tokens_seen": 462240768 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004343029087261786, + "loss": 3.0607, + "theoretical_loss": 3.948273150615068, + "tokens_seen": 462306304 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004342928786359077, + "loss": 2.8376, + "theoretical_loss": 3.9482120545694626, + "tokens_seen": 462371840 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043428284854563694, + "loss": 3.1191, + "theoretical_loss": 3.948150969607237, + "tokens_seen": 462437376 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043427281845536607, + "loss": 3.0403, + "theoretical_loss": 3.9480898957248116, + "tokens_seen": 462502912 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004342627883650953, + "loss": 3.2992, + "theoretical_loss": 3.948028832918606, + "tokens_seen": 462568448 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004342527582748245, + "loss": 2.7688, + "theoretical_loss": 3.9479677811850427, + "tokens_seen": 462633984 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043424272818455367, + "loss": 2.912, + "theoretical_loss": 3.947906740520546, + "tokens_seen": 462699520 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043423269809428285, + "loss": 2.9298, + "theoretical_loss": 3.9478457109215412, + "tokens_seen": 462765056 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004342226680040121, + "loss": 2.8303, + "theoretical_loss": 3.947784692384457, + "tokens_seen": 462830592 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004342126379137412, + "loss": 3.0642, + "theoretical_loss": 3.9477236849057205, + "tokens_seen": 462896128 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043420260782347044, + "loss": 3.0432, + "theoretical_loss": 3.9476626884817634, + "tokens_seen": 462961664 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043419257773319957, + "loss": 2.8215, + "theoretical_loss": 3.9476017031090187, + "tokens_seen": 463027200 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004341825476429288, + "loss": 2.9692, + "theoretical_loss": 3.9475407287839195, + "tokens_seen": 463092736 + }, + { + "epoch": 1.05, + "learning_rate": 0.000434172517552658, + "loss": 3.0146, + "theoretical_loss": 3.947479765502902, + "tokens_seen": 463158272 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043416248746238717, + "loss": 3.0465, + "theoretical_loss": 3.947418813262403, + "tokens_seen": 463223808 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043415245737211635, + "loss": 3.0851, + "theoretical_loss": 3.947357872058862, + "tokens_seen": 463289344 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043414242728184553, + "loss": 2.7837, + "theoretical_loss": 3.9472969418887196, + "tokens_seen": 463354880 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004341323971915747, + "loss": 3.2644, + "theoretical_loss": 3.9472360227484176, + "tokens_seen": 463420416 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043412236710130395, + "loss": 3.228, + "theoretical_loss": 3.9471751146344003, + "tokens_seen": 463485952 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004341123370110331, + "loss": 2.8354, + "theoretical_loss": 3.947114217543113, + "tokens_seen": 463551488 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004341023069207623, + "loss": 2.8766, + "theoretical_loss": 3.947053331471003, + "tokens_seen": 463617024 + }, + { + "epoch": 1.05, + "objective/train/docs_used": 552845, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9544570446014404, + "objective/train/theoretical_loss": 3.94700767414612, + "objective/train/tokens_used": 484126176, + "theoretical_loss": 3.94700767414612, + "tokens_seen": 463666176 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043409227683049144, + "loss": 3.0834, + "theoretical_loss": 3.9469924564145202, + "tokens_seen": 463682560 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043408224674022067, + "loss": 2.8789, + "theoretical_loss": 3.9469315923701136, + "tokens_seen": 463748096 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043407221664994985, + "loss": 2.9113, + "theoretical_loss": 3.946870739334236, + "tokens_seen": 463813632 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043406218655967903, + "loss": 2.9886, + "theoretical_loss": 3.946809897303342, + "tokens_seen": 463879168 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004340521564694082, + "loss": 2.9726, + "theoretical_loss": 3.9467490662738856, + "tokens_seen": 463944704 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043404212637913745, + "loss": 2.9747, + "theoretical_loss": 3.9466882462423243, + "tokens_seen": 464010240 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004340320962888666, + "loss": 3.2831, + "theoretical_loss": 3.9466274372051178, + "tokens_seen": 464075776 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004340220661985958, + "loss": 2.8867, + "theoretical_loss": 3.9465666391587257, + "tokens_seen": 464141312 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043401203610832494, + "loss": 3.0543, + "theoretical_loss": 3.94650585209961, + "tokens_seen": 464206848 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004340020060180542, + "loss": 2.9371, + "theoretical_loss": 3.9464450760242347, + "tokens_seen": 464272384 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043399197592778336, + "loss": 3.0978, + "theoretical_loss": 3.946384310929065, + "tokens_seen": 464337920 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043398194583751254, + "loss": 2.9277, + "theoretical_loss": 3.9463235568105675, + "tokens_seen": 464403456 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004339719157472417, + "loss": 3.1595, + "theoretical_loss": 3.946262813665211, + "tokens_seen": 464468992 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004339618856569709, + "loss": 2.8856, + "theoretical_loss": 3.9462020814894663, + "tokens_seen": 464534528 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004339518555667001, + "loss": 2.9228, + "theoretical_loss": 3.946141360279804, + "tokens_seen": 464600064 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004339418254764293, + "loss": 2.8936, + "theoretical_loss": 3.9460806500326986, + "tokens_seen": 464665600 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004339317953861585, + "loss": 3.135, + "theoretical_loss": 3.9460199507446245, + "tokens_seen": 464731136 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004339217652958877, + "loss": 3.0263, + "theoretical_loss": 3.9459592624120594, + "tokens_seen": 464796672 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004339117352056169, + "loss": 3.0817, + "theoretical_loss": 3.94589858503148, + "tokens_seen": 464862208 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043390170511534604, + "loss": 2.9917, + "theoretical_loss": 3.945837918599368, + "tokens_seen": 464927744 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004338916750250753, + "loss": 3.0115, + "theoretical_loss": 3.9457772631122037, + "tokens_seen": 464993280 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004338816449348044, + "loss": 2.9553, + "theoretical_loss": 3.945716618566472, + "tokens_seen": 465058816 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043387161484453364, + "loss": 2.8438, + "theoretical_loss": 3.945655984958656, + "tokens_seen": 465124352 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004338615847542628, + "loss": 2.9346, + "theoretical_loss": 3.9455953622852427, + "tokens_seen": 465189888 + }, + { + "epoch": 1.05, + "learning_rate": 0.000433851554663992, + "loss": 2.9295, + "theoretical_loss": 3.9455347505427207, + "tokens_seen": 465255424 + }, + { + "epoch": 1.05, + "objective/train/docs_used": 553513, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9996495246887207, + "objective/train/theoretical_loss": 3.945489298907115, + "objective/train/tokens_used": 485764576, + "theoretical_loss": 3.945489298907115, + "tokens_seen": 465304576 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004338415245737212, + "loss": 2.9936, + "theoretical_loss": 3.9454741497275796, + "tokens_seen": 465320960 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043383149448345036, + "loss": 2.875, + "theoretical_loss": 3.945413559836311, + "tokens_seen": 465386496 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043382146439317954, + "loss": 3.1446, + "theoretical_loss": 3.9453529808654064, + "tokens_seen": 465452032 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004338114343029088, + "loss": 3.0625, + "theoretical_loss": 3.9452924128113622, + "tokens_seen": 465517568 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004338014042126379, + "loss": 3.0735, + "theoretical_loss": 3.945231855670674, + "tokens_seen": 465583104 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043379137412236714, + "loss": 3.1854, + "theoretical_loss": 3.9451713094398393, + "tokens_seen": 465648640 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043378134403209627, + "loss": 3.1381, + "theoretical_loss": 3.9451107741153577, + "tokens_seen": 465714176 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004337713139418255, + "loss": 3.0158, + "theoretical_loss": 3.9450502496937307, + "tokens_seen": 465779712 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004337612838515547, + "loss": 2.8256, + "theoretical_loss": 3.9449897361714603, + "tokens_seen": 465845248 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043375125376128387, + "loss": 2.8059, + "theoretical_loss": 3.944929233545051, + "tokens_seen": 465910784 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043374122367101305, + "loss": 2.9526, + "theoretical_loss": 3.9448687418110095, + "tokens_seen": 465976320 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004337311935807423, + "loss": 3.1095, + "theoretical_loss": 3.944808260965842, + "tokens_seen": 466041856 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004337211634904714, + "loss": 3.2617, + "theoretical_loss": 3.9447477910060584, + "tokens_seen": 466107392 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043371113340020064, + "loss": 3.0492, + "theoretical_loss": 3.94468733192817, + "tokens_seen": 466172928 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043370110330992977, + "loss": 3.0652, + "theoretical_loss": 3.944626883728688, + "tokens_seen": 466238464 + }, + { + "epoch": 1.05, + "learning_rate": 0.000433691073219659, + "loss": 3.1735, + "theoretical_loss": 3.9445664464041266, + "tokens_seen": 466304000 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004336810431293882, + "loss": 2.9657, + "theoretical_loss": 3.944506019951002, + "tokens_seen": 466369536 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043367101303911737, + "loss": 2.8674, + "theoretical_loss": 3.9444456043658302, + "tokens_seen": 466435072 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043366098294884655, + "loss": 3.032, + "theoretical_loss": 3.944385199645132, + "tokens_seen": 466500608 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043365095285857573, + "loss": 2.8132, + "theoretical_loss": 3.9443248057854254, + "tokens_seen": 466566144 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004336409227683049, + "loss": 2.8774, + "theoretical_loss": 3.9442644227832337, + "tokens_seen": 466631680 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043363089267803415, + "loss": 2.8377, + "theoretical_loss": 3.9442040506350806, + "tokens_seen": 466697216 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004336208625877633, + "loss": 3.2088, + "theoretical_loss": 3.9441436893374906, + "tokens_seen": 466762752 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004336108324974925, + "loss": 2.8835, + "theoretical_loss": 3.9440833388869914, + "tokens_seen": 466828288 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043360080240722164, + "loss": 2.9217, + "theoretical_loss": 3.9440229992801106, + "tokens_seen": 466893824 + }, + { + "epoch": 1.05, + "objective/train/docs_used": 555014, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.009627103805542, + "objective/train/theoretical_loss": 3.943977751688987, + "objective/train/tokens_used": 487402976, + "theoretical_loss": 3.943977751688987, + "tokens_seen": 466942976 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043359077231695087, + "loss": 2.8843, + "theoretical_loss": 3.943962670513378, + "tokens_seen": 466959360 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043358074222668005, + "loss": 2.7812, + "theoretical_loss": 3.9439023525833257, + "tokens_seen": 467024896 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043357071213640923, + "loss": 3.0704, + "theoretical_loss": 3.9438420454864875, + "tokens_seen": 467090432 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004335606820461384, + "loss": 3.2732, + "theoretical_loss": 3.9437817492193963, + "tokens_seen": 467155968 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043355065195586765, + "loss": 2.9273, + "theoretical_loss": 3.9437214637785902, + "tokens_seen": 467221504 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004335406218655968, + "loss": 3.0148, + "theoretical_loss": 3.943661189160607, + "tokens_seen": 467287040 + }, + { + "epoch": 1.05, + "learning_rate": 0.000433530591775326, + "loss": 3.1349, + "theoretical_loss": 3.9436009253619853, + "tokens_seen": 467352576 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043352056168505514, + "loss": 2.9502, + "theoretical_loss": 3.9435406723792665, + "tokens_seen": 467418112 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004335105315947844, + "loss": 2.8156, + "theoretical_loss": 3.9434804302089934, + "tokens_seen": 467483648 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043350050150451356, + "loss": 2.8378, + "theoretical_loss": 3.9434201988477113, + "tokens_seen": 467549184 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043349047141424274, + "loss": 3.1404, + "theoretical_loss": 3.943359978291965, + "tokens_seen": 467614720 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004334804413239719, + "loss": 2.8589, + "theoretical_loss": 3.943299768538302, + "tokens_seen": 467680256 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004334704112337011, + "loss": 3.0278, + "theoretical_loss": 3.943239569583272, + "tokens_seen": 467745792 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004334603811434303, + "loss": 3.1748, + "theoretical_loss": 3.943179381423425, + "tokens_seen": 467811328 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004334503510531595, + "loss": 2.818, + "theoretical_loss": 3.9431192040553142, + "tokens_seen": 467876864 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043344032096288864, + "loss": 2.8783, + "theoretical_loss": 3.9430590374754924, + "tokens_seen": 467942400 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004334302908726179, + "loss": 3.0593, + "theoretical_loss": 3.9429988816805155, + "tokens_seen": 468007936 + }, + { + "epoch": 1.05, + "learning_rate": 0.000433420260782347, + "loss": 2.9382, + "theoretical_loss": 3.9429387366669406, + "tokens_seen": 468073472 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043341023069207624, + "loss": 3.0675, + "theoretical_loss": 3.9428786024313256, + "tokens_seen": 468139008 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004334002006018054, + "loss": 2.8618, + "theoretical_loss": 3.9428184789702323, + "tokens_seen": 468204544 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004333901705115346, + "loss": 2.9609, + "theoretical_loss": 3.9427583662802204, + "tokens_seen": 468270080 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004333801404212638, + "loss": 2.8331, + "theoretical_loss": 3.9426982643578548, + "tokens_seen": 468335616 + }, + { + "epoch": 1.05, + "learning_rate": 0.000433370110330993, + "loss": 2.8806, + "theoretical_loss": 3.9426381731996996, + "tokens_seen": 468401152 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043336008024072215, + "loss": 2.8969, + "theoretical_loss": 3.9425780928023215, + "tokens_seen": 468466688 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004333500501504514, + "loss": 2.82, + "theoretical_loss": 3.9425180231622883, + "tokens_seen": 468532224 + }, + { + "epoch": 1.05, + "objective/train/docs_used": 555695, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4344749450683594, + "objective/train/theoretical_loss": 3.9424729779897074, + "objective/train/tokens_used": 489041376, + "theoretical_loss": 3.9424729779897074, + "tokens_seen": 468581376 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004333400200601805, + "loss": 2.8374, + "theoretical_loss": 3.94245796427617, + "tokens_seen": 468597760 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043332998996990974, + "loss": 2.963, + "theoretical_loss": 3.942397916140538, + "tokens_seen": 468663296 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004333199598796389, + "loss": 2.9556, + "theoretical_loss": 3.942337878751964, + "tokens_seen": 468728832 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004333099297893681, + "loss": 2.7379, + "theoretical_loss": 3.942277852107024, + "tokens_seen": 468794368 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004332998996990973, + "loss": 2.9639, + "theoretical_loss": 3.9422178362022917, + "tokens_seen": 468859904 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043328986960882647, + "loss": 3.1148, + "theoretical_loss": 3.9421578310343466, + "tokens_seen": 468925440 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043327983951855565, + "loss": 3.0438, + "theoretical_loss": 3.9420978365997668, + "tokens_seen": 468990976 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004332698094282849, + "loss": 2.9268, + "theoretical_loss": 3.942037852895133, + "tokens_seen": 469056512 + }, + { + "epoch": 1.05, + "learning_rate": 0.000433259779338014, + "loss": 2.9954, + "theoretical_loss": 3.9419778799170286, + "tokens_seen": 469122048 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043324974924774325, + "loss": 2.9419, + "theoretical_loss": 3.941917917662035, + "tokens_seen": 469187584 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043323971915747243, + "loss": 3.0639, + "theoretical_loss": 3.941857966126739, + "tokens_seen": 469253120 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004332296890672016, + "loss": 2.9824, + "theoretical_loss": 3.941798025307728, + "tokens_seen": 469318656 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004332196589769308, + "loss": 2.8335, + "theoretical_loss": 3.941738095201589, + "tokens_seen": 469384192 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043320962888665997, + "loss": 2.9499, + "theoretical_loss": 3.941678175804913, + "tokens_seen": 469449728 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043319959879638915, + "loss": 2.9987, + "theoretical_loss": 3.9416182671142908, + "tokens_seen": 469515264 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004331895687061184, + "loss": 3.0375, + "theoretical_loss": 3.9415583691263167, + "tokens_seen": 469580800 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043317953861584757, + "loss": 2.9472, + "theoretical_loss": 3.9414984818375842, + "tokens_seen": 469646336 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043316950852557675, + "loss": 3.2323, + "theoretical_loss": 3.9414386052446906, + "tokens_seen": 469711872 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043315947843530593, + "loss": 3.0547, + "theoretical_loss": 3.9413787393442328, + "tokens_seen": 469777408 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004331494483450351, + "loss": 2.8593, + "theoretical_loss": 3.9413188841328104, + "tokens_seen": 469842944 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043313941825476435, + "loss": 2.9321, + "theoretical_loss": 3.941259039607025, + "tokens_seen": 469908480 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004331293881644935, + "loss": 2.831, + "theoretical_loss": 3.9411992057634775, + "tokens_seen": 469974016 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004331193580742227, + "loss": 3.0224, + "theoretical_loss": 3.9411393825987737, + "tokens_seen": 470039552 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043310932798395184, + "loss": 2.9007, + "theoretical_loss": 3.941079570109518, + "tokens_seen": 470105088 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043309929789368107, + "loss": 2.9678, + "theoretical_loss": 3.9410197682923185, + "tokens_seen": 470170624 + }, + { + "epoch": 1.05, + "objective/train/docs_used": 556746, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.731928586959839, + "objective/train/theoretical_loss": 3.9409749239309146, + "objective/train/tokens_used": 490679776, + "theoretical_loss": 3.9409749239309146, + "tokens_seen": 470219776 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043308926780341025, + "loss": 2.95, + "theoretical_loss": 3.940959977143783, + "tokens_seen": 470236160 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043307923771313943, + "loss": 2.7916, + "theoretical_loss": 3.9409001966605217, + "tokens_seen": 470301696 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004330692076228686, + "loss": 3.0253, + "theoretical_loss": 3.9408404268391473, + "tokens_seen": 470367232 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043305917753259785, + "loss": 2.9868, + "theoretical_loss": 3.940780667676272, + "tokens_seen": 470432768 + }, + { + "epoch": 1.05, + "learning_rate": 0.000433049147442327, + "loss": 3.1419, + "theoretical_loss": 3.9407209191685117, + "tokens_seen": 470498304 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004330391173520562, + "loss": 2.9542, + "theoretical_loss": 3.940661181312482, + "tokens_seen": 470563840 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043302908726178534, + "loss": 3.0518, + "theoretical_loss": 3.9406014541048013, + "tokens_seen": 470629376 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004330190571715146, + "loss": 2.8218, + "theoretical_loss": 3.9405417375420893, + "tokens_seen": 470694912 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043300902708124376, + "loss": 3.094, + "theoretical_loss": 3.9404820316209666, + "tokens_seen": 470760448 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043299899699097294, + "loss": 2.9356, + "theoretical_loss": 3.940422336338056, + "tokens_seen": 470825984 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004329889669007021, + "loss": 3.0012, + "theoretical_loss": 3.940362651689982, + "tokens_seen": 470891520 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004329789368104313, + "loss": 3.1362, + "theoretical_loss": 3.940302977673369, + "tokens_seen": 470957056 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004329689067201605, + "loss": 3.0018, + "theoretical_loss": 3.940243314284846, + "tokens_seen": 471022592 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004329588766298897, + "loss": 2.8662, + "theoretical_loss": 3.940183661521041, + "tokens_seen": 471088128 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043294884653961884, + "loss": 3.1666, + "theoretical_loss": 3.940124019378583, + "tokens_seen": 471153664 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004329388164493481, + "loss": 3.1978, + "theoretical_loss": 3.940064387854106, + "tokens_seen": 471219200 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004329287863590772, + "loss": 2.7946, + "theoretical_loss": 3.9400047669442424, + "tokens_seen": 471284736 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043291875626880644, + "loss": 2.9157, + "theoretical_loss": 3.939945156645627, + "tokens_seen": 471350272 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004329087261785356, + "loss": 2.9845, + "theoretical_loss": 3.9398855569548963, + "tokens_seen": 471415808 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004328986960882648, + "loss": 2.9426, + "theoretical_loss": 3.939825967868688, + "tokens_seen": 471481344 + }, + { + "epoch": 1.05, + "learning_rate": 0.000432888665997994, + "loss": 2.7147, + "theoretical_loss": 3.9397663893836428, + "tokens_seen": 471546880 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004328786359077232, + "loss": 3.2359, + "theoretical_loss": 3.9397068214964, + "tokens_seen": 471612416 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043286860581745235, + "loss": 3.0412, + "theoretical_loss": 3.939647264203604, + "tokens_seen": 471677952 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004328585757271816, + "loss": 2.9235, + "theoretical_loss": 3.9395877175018965, + "tokens_seen": 471743488 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004328485456369107, + "loss": 3.1163, + "theoretical_loss": 3.9395281813879257, + "tokens_seen": 471809024 + }, + { + "epoch": 1.05, + "objective/train/docs_used": 557491, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.751948595046997, + "objective/train/theoretical_loss": 3.939483536248632, + "objective/train/tokens_used": 492318176, + "theoretical_loss": 3.939483536248632, + "tokens_seen": 471858176 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043283851554663994, + "loss": 2.8767, + "theoretical_loss": 3.9394686558583376, + "tokens_seen": 471874560 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004328284854563691, + "loss": 2.8283, + "theoretical_loss": 3.9394091409097807, + "tokens_seen": 471940096 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004328184553660983, + "loss": 2.7199, + "theoretical_loss": 3.939349636538905, + "tokens_seen": 472005632 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004328084252758275, + "loss": 3.1735, + "theoretical_loss": 3.9392901427423634, + "tokens_seen": 472071168 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043279839518555667, + "loss": 2.9966, + "theoretical_loss": 3.939230659516808, + "tokens_seen": 472136704 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043278836509528585, + "loss": 2.6941, + "theoretical_loss": 3.9391711868588946, + "tokens_seen": 472202240 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004327783350050151, + "loss": 3.0288, + "theoretical_loss": 3.9391117247652785, + "tokens_seen": 472267776 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004327683049147442, + "loss": 3.1079, + "theoretical_loss": 3.939052273232618, + "tokens_seen": 472333312 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043275827482447345, + "loss": 3.0203, + "theoretical_loss": 3.938992832257572, + "tokens_seen": 472398848 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043274824473420263, + "loss": 2.8275, + "theoretical_loss": 3.9389334018368025, + "tokens_seen": 472464384 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004327382146439318, + "loss": 3.2009, + "theoretical_loss": 3.938873981966971, + "tokens_seen": 472529920 + }, + { + "epoch": 1.05, + "learning_rate": 0.000432728184553661, + "loss": 2.8613, + "theoretical_loss": 3.938814572644741, + "tokens_seen": 472595456 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043271815446339017, + "loss": 2.9294, + "theoretical_loss": 3.938755173866779, + "tokens_seen": 472660992 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043270812437311935, + "loss": 3.0615, + "theoretical_loss": 3.9386957856297515, + "tokens_seen": 472726528 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004326980942828486, + "loss": 3.0866, + "theoretical_loss": 3.9386364079303267, + "tokens_seen": 472792064 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004326880641925777, + "loss": 2.9678, + "theoretical_loss": 3.938577040765175, + "tokens_seen": 472857600 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043267803410230695, + "loss": 3.0434, + "theoretical_loss": 3.9385176841309675, + "tokens_seen": 472923136 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004326680040120361, + "loss": 3.0813, + "theoretical_loss": 3.938458338024377, + "tokens_seen": 472988672 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004326579739217653, + "loss": 3.0351, + "theoretical_loss": 3.9383990024420794, + "tokens_seen": 473054208 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004326479438314945, + "loss": 3.0305, + "theoretical_loss": 3.938339677380749, + "tokens_seen": 473119744 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004326379137412237, + "loss": 3.0702, + "theoretical_loss": 3.938280362837064, + "tokens_seen": 473185280 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043262788365095286, + "loss": 2.8885, + "theoretical_loss": 3.938221058807704, + "tokens_seen": 473250816 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043261785356068204, + "loss": 3.0995, + "theoretical_loss": 3.9381617652893484, + "tokens_seen": 473316352 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004326078234704112, + "loss": 3.1313, + "theoretical_loss": 3.93810248227868, + "tokens_seen": 473381888 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043259779338014045, + "loss": 2.9592, + "theoretical_loss": 3.938043209772382, + "tokens_seen": 473447424 + }, + { + "epoch": 1.05, + "objective/train/docs_used": 558752, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.13558030128479, + "objective/train/theoretical_loss": 3.937998762284158, + "objective/train/tokens_used": 493956576, + "theoretical_loss": 3.937998762284158, + "tokens_seen": 473496576 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004325877632898696, + "loss": 2.9736, + "theoretical_loss": 3.9379839477671403, + "tokens_seen": 473512960 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004325777331995988, + "loss": 3.0554, + "theoretical_loss": 3.937924696259641, + "tokens_seen": 473578496 + }, + { + "epoch": 1.05, + "learning_rate": 0.000432567703109328, + "loss": 3.0045, + "theoretical_loss": 3.9378654552465715, + "tokens_seen": 473644032 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004325576730190572, + "loss": 3.0406, + "theoretical_loss": 3.937806224724622, + "tokens_seen": 473709568 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043254764292878636, + "loss": 2.9355, + "theoretical_loss": 3.9377470046904834, + "tokens_seen": 473775104 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043253761283851554, + "loss": 2.8813, + "theoretical_loss": 3.9376877951408487, + "tokens_seen": 473840640 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004325275827482447, + "loss": 2.823, + "theoretical_loss": 3.937628596072412, + "tokens_seen": 473906176 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043251755265797396, + "loss": 2.9766, + "theoretical_loss": 3.9375694074818677, + "tokens_seen": 473971712 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004325075225677031, + "loss": 2.8709, + "theoretical_loss": 3.9375102293659143, + "tokens_seen": 474037248 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004324974924774323, + "loss": 3.0939, + "theoretical_loss": 3.9374510617212497, + "tokens_seen": 474102784 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043248746238716145, + "loss": 3.0702, + "theoretical_loss": 3.937391904544574, + "tokens_seen": 474168320 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004324774322968907, + "loss": 3.1023, + "theoretical_loss": 3.937332757832589, + "tokens_seen": 474233856 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043246740220661986, + "loss": 2.914, + "theoretical_loss": 3.9372736215819977, + "tokens_seen": 474299392 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043245737211634904, + "loss": 2.7269, + "theoretical_loss": 3.9372144957895046, + "tokens_seen": 474364928 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004324473420260782, + "loss": 3.1261, + "theoretical_loss": 3.9371553804518156, + "tokens_seen": 474430464 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004324373119358074, + "loss": 3.0787, + "theoretical_loss": 3.9370962755656382, + "tokens_seen": 474496000 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043242728184553664, + "loss": 3.2062, + "theoretical_loss": 3.937037181127682, + "tokens_seen": 474561536 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004324172517552658, + "loss": 3.1947, + "theoretical_loss": 3.936978097134657, + "tokens_seen": 474627072 + }, + { + "epoch": 1.05, + "learning_rate": 0.000432407221664995, + "loss": 3.0234, + "theoretical_loss": 3.9369190235832754, + "tokens_seen": 474692608 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004323971915747242, + "loss": 2.9252, + "theoretical_loss": 3.936859960470251, + "tokens_seen": 474758144 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004323871614844534, + "loss": 2.8916, + "theoretical_loss": 3.936800907792298, + "tokens_seen": 474823680 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043237713139418255, + "loss": 3.1763, + "theoretical_loss": 3.9367418655461335, + "tokens_seen": 474889216 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004323671013039118, + "loss": 3.1412, + "theoretical_loss": 3.936682833728476, + "tokens_seen": 474954752 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004323570712136409, + "loss": 3.0111, + "theoretical_loss": 3.936623812336043, + "tokens_seen": 475020288 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043234704112337014, + "loss": 3.1496, + "theoretical_loss": 3.9365648013655576, + "tokens_seen": 475085824 + }, + { + "epoch": 1.05, + "objective/train/docs_used": 559487, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8737614154815674, + "objective/train/theoretical_loss": 3.936520549975124, + "objective/train/tokens_used": 495594976, + "theoretical_loss": 3.936520549975124, + "tokens_seen": 475134976 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004323370110330993, + "loss": 3.1738, + "theoretical_loss": 3.936505800813741, + "tokens_seen": 475151360 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004323269809428285, + "loss": 2.7831, + "theoretical_loss": 3.9364468106773174, + "tokens_seen": 475216896 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004323169508525577, + "loss": 2.8794, + "theoretical_loss": 3.936387830953013, + "tokens_seen": 475282432 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043230692076228687, + "loss": 3.0701, + "theoretical_loss": 3.936328861637553, + "tokens_seen": 475347968 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043229689067201605, + "loss": 2.9024, + "theoretical_loss": 3.936269902727667, + "tokens_seen": 475413504 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004322868605817453, + "loss": 3.3167, + "theoretical_loss": 3.9362109542200843, + "tokens_seen": 475479040 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004322768304914744, + "loss": 3.2355, + "theoretical_loss": 3.9361520161115364, + "tokens_seen": 475544576 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043226680040120365, + "loss": 3.1841, + "theoretical_loss": 3.9360930883987564, + "tokens_seen": 475610112 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043225677031093283, + "loss": 3.0284, + "theoretical_loss": 3.936034171078478, + "tokens_seen": 475675648 + }, + { + "epoch": 1.05, + "learning_rate": 0.000432246740220662, + "loss": 3.0582, + "theoretical_loss": 3.9359752641474373, + "tokens_seen": 475741184 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004322367101303912, + "loss": 3.0325, + "theoretical_loss": 3.9359163676023705, + "tokens_seen": 475806720 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043222668004012037, + "loss": 3.1134, + "theoretical_loss": 3.935857481440018, + "tokens_seen": 475872256 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043221664994984955, + "loss": 3.2472, + "theoretical_loss": 3.935798605657119, + "tokens_seen": 475937792 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004322066198595788, + "loss": 3.1808, + "theoretical_loss": 3.9357397402504146, + "tokens_seen": 476003328 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004321965897693079, + "loss": 3.0204, + "theoretical_loss": 3.935680885216649, + "tokens_seen": 476068864 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043218655967903715, + "loss": 3.1083, + "theoretical_loss": 3.9356220405525666, + "tokens_seen": 476134400 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004321765295887663, + "loss": 2.8384, + "theoretical_loss": 3.9355632062549124, + "tokens_seen": 476199936 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004321664994984955, + "loss": 3.0837, + "theoretical_loss": 3.9355043823204348, + "tokens_seen": 476265472 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004321564694082247, + "loss": 3.0754, + "theoretical_loss": 3.935445568745883, + "tokens_seen": 476331008 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004321464393179539, + "loss": 3.1409, + "theoretical_loss": 3.9353867655280066, + "tokens_seen": 476396544 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043213640922768306, + "loss": 3.2543, + "theoretical_loss": 3.935327972663558, + "tokens_seen": 476462080 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043212637913741224, + "loss": 2.7809, + "theoretical_loss": 3.935269190149291, + "tokens_seen": 476527616 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004321163490471414, + "loss": 2.9419, + "theoretical_loss": 3.93521041798196, + "tokens_seen": 476593152 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043210631895687065, + "loss": 2.8718, + "theoretical_loss": 3.935151656158321, + "tokens_seen": 476658688 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004320962888665998, + "loss": 3.0566, + "theoretical_loss": 3.9350929046751313, + "tokens_seen": 476724224 + }, + { + "epoch": 1.05, + "objective/train/docs_used": 560707, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.231940984725952, + "objective/train/theoretical_loss": 3.9350488478467103, + "objective/train/tokens_used": 497233376, + "theoretical_loss": 3.9350488478467103, + "tokens_seen": 476773376 + }, + { + "epoch": 1.05, + "learning_rate": 0.000432086258776329, + "loss": 3.0351, + "theoretical_loss": 3.9350341635291515, + "tokens_seen": 476789760 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004320762286860582, + "loss": 3.0426, + "theoretical_loss": 3.934975432717142, + "tokens_seen": 476855296 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004320661985957874, + "loss": 2.9293, + "theoretical_loss": 3.9349167122358644, + "tokens_seen": 476920832 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043205616850551656, + "loss": 3.0121, + "theoretical_loss": 3.934858002082082, + "tokens_seen": 476986368 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043204613841524574, + "loss": 3.1352, + "theoretical_loss": 3.9347993022525607, + "tokens_seen": 477051904 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004320361083249749, + "loss": 3.0302, + "theoretical_loss": 3.9347406127440663, + "tokens_seen": 477117440 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043202607823470416, + "loss": 2.9891, + "theoretical_loss": 3.9346819335533674, + "tokens_seen": 477182976 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004320160481444333, + "loss": 3.0205, + "theoretical_loss": 3.9346232646772332, + "tokens_seen": 477248512 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004320060180541625, + "loss": 2.7596, + "theoretical_loss": 3.934564606112435, + "tokens_seen": 477314048 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043199598796389165, + "loss": 3.0225, + "theoretical_loss": 3.934505957855744, + "tokens_seen": 477379584 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004319859578736209, + "loss": 2.9677, + "theoretical_loss": 3.934447319903935, + "tokens_seen": 477445120 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043197592778335006, + "loss": 2.7368, + "theoretical_loss": 3.934388692253783, + "tokens_seen": 477510656 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043196589769307924, + "loss": 3.0439, + "theoretical_loss": 3.934330074902065, + "tokens_seen": 477576192 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004319558676028084, + "loss": 2.9094, + "theoretical_loss": 3.9342714678455586, + "tokens_seen": 477641728 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004319458375125376, + "loss": 2.786, + "theoretical_loss": 3.9342128710810433, + "tokens_seen": 477707264 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004319358074222668, + "loss": 3.1414, + "theoretical_loss": 3.9341542846053006, + "tokens_seen": 477772800 + }, + { + "epoch": 1.05, + "learning_rate": 0.000431925777331996, + "loss": 2.915, + "theoretical_loss": 3.934095708415113, + "tokens_seen": 477838336 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043191574724172515, + "loss": 2.9227, + "theoretical_loss": 3.934037142507264, + "tokens_seen": 477903872 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004319057171514544, + "loss": 2.7711, + "theoretical_loss": 3.9339785868785397, + "tokens_seen": 477969408 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043189568706118357, + "loss": 2.8507, + "theoretical_loss": 3.9339200415257265, + "tokens_seen": 478034944 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043188565697091275, + "loss": 3.0298, + "theoretical_loss": 3.933861506445613, + "tokens_seen": 478100480 + }, + { + "epoch": 1.05, + "learning_rate": 0.00043187562688064193, + "loss": 2.8273, + "theoretical_loss": 3.933802981634988, + "tokens_seen": 478166016 + }, + { + "epoch": 1.05, + "learning_rate": 0.0004318655967903711, + "loss": 3.1108, + "theoretical_loss": 3.9337444670906434, + "tokens_seen": 478231552 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004318555667001003, + "loss": 2.9331, + "theoretical_loss": 3.9336859628093723, + "tokens_seen": 478297088 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004318455366098295, + "loss": 3.0098, + "theoretical_loss": 3.933627468787968, + "tokens_seen": 478362624 + }, + { + "epoch": 1.06, + "objective/train/docs_used": 561422, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4702494144439697, + "objective/train/theoretical_loss": 3.933583605003024, + "objective/train/tokens_used": 498871776, + "theoretical_loss": 3.933583605003024, + "tokens_seen": 478411776 + }, + { + "epoch": 1.06, + "learning_rate": 0.00043183550651955865, + "loss": 2.9758, + "theoretical_loss": 3.933568985023226, + "tokens_seen": 478428160 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004318254764292879, + "loss": 3.136, + "theoretical_loss": 3.933510511511943, + "tokens_seen": 478493696 + }, + { + "epoch": 1.06, + "learning_rate": 0.000431815446339017, + "loss": 3.188, + "theoretical_loss": 3.9334520482509183, + "tokens_seen": 478559232 + }, + { + "epoch": 1.06, + "learning_rate": 0.00043180541624874625, + "loss": 3.0016, + "theoretical_loss": 3.933393595236951, + "tokens_seen": 478624768 + }, + { + "epoch": 1.06, + "learning_rate": 0.00043179538615847543, + "loss": 2.8216, + "theoretical_loss": 3.9333351524668427, + "tokens_seen": 478690304 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004317853560682046, + "loss": 2.9534, + "theoretical_loss": 3.9332767199373957, + "tokens_seen": 478755840 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004317753259779338, + "loss": 3.0022, + "theoretical_loss": 3.9332182976454146, + "tokens_seen": 478821376 + }, + { + "epoch": 1.06, + "learning_rate": 0.00043176529588766303, + "loss": 3.1134, + "theoretical_loss": 3.9331598855877044, + "tokens_seen": 478886912 + }, + { + "epoch": 1.06, + "learning_rate": 0.00043175526579739215, + "loss": 2.8389, + "theoretical_loss": 3.9331014837610727, + "tokens_seen": 478952448 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004317452357071214, + "loss": 2.9801, + "theoretical_loss": 3.933043092162327, + "tokens_seen": 479017984 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004317352056168505, + "loss": 2.9627, + "theoretical_loss": 3.9329847107882787, + "tokens_seen": 479083520 + }, + { + "epoch": 1.06, + "learning_rate": 0.00043172517552657975, + "loss": 2.7128, + "theoretical_loss": 3.9329263396357375, + "tokens_seen": 479149056 + }, + { + "epoch": 1.06, + "learning_rate": 0.00043171514543630893, + "loss": 2.8601, + "theoretical_loss": 3.932867978701517, + "tokens_seen": 479214592 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004317051153460381, + "loss": 3.0035, + "theoretical_loss": 3.932809627982431, + "tokens_seen": 479280128 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004316950852557673, + "loss": 3.0842, + "theoretical_loss": 3.9327512874752952, + "tokens_seen": 479345664 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004316850551654965, + "loss": 3.1308, + "theoretical_loss": 3.9326929571769265, + "tokens_seen": 479411200 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004316750250752257, + "loss": 3.1854, + "theoretical_loss": 3.9326346370841434, + "tokens_seen": 479476736 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004316649949849549, + "loss": 2.9725, + "theoretical_loss": 3.932576327193766, + "tokens_seen": 479542272 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004316549648946841, + "loss": 2.7423, + "theoretical_loss": 3.9325180275026153, + "tokens_seen": 479607808 + }, + { + "epoch": 1.06, + "learning_rate": 0.00043164493480441326, + "loss": 3.0698, + "theoretical_loss": 3.9324597380075135, + "tokens_seen": 479673344 + }, + { + "epoch": 1.06, + "learning_rate": 0.00043163490471414244, + "loss": 2.9938, + "theoretical_loss": 3.932401458705286, + "tokens_seen": 479738880 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004316248746238716, + "loss": 3.025, + "theoretical_loss": 3.932343189592757, + "tokens_seen": 479804416 + }, + { + "epoch": 1.06, + "learning_rate": 0.00043161484453360085, + "loss": 3.0979, + "theoretical_loss": 3.9322849306667536, + "tokens_seen": 479869952 + }, + { + "epoch": 1.06, + "learning_rate": 0.00043160481444333, + "loss": 2.8941, + "theoretical_loss": 3.9322266819241056, + "tokens_seen": 479935488 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004315947843530592, + "loss": 3.0009, + "theoretical_loss": 3.9321684433616415, + "tokens_seen": 480001024 + }, + { + "epoch": 1.06, + "objective/train/docs_used": 562507, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.448958158493042, + "objective/train/theoretical_loss": 3.932124771118633, + "objective/train/tokens_used": 500510176, + "theoretical_loss": 3.932124771118633, + "tokens_seen": 480050176 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004315847542627884, + "loss": 2.906, + "theoretical_loss": 3.932110214976193, + "tokens_seen": 480066560 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004315747241725176, + "loss": 2.8742, + "theoretical_loss": 3.9320519967645926, + "tokens_seen": 480132096 + }, + { + "epoch": 1.06, + "learning_rate": 0.00043156469408224676, + "loss": 3.1216, + "theoretical_loss": 3.931993788723674, + "tokens_seen": 480197632 + }, + { + "epoch": 1.06, + "learning_rate": 0.00043155466399197594, + "loss": 2.9462, + "theoretical_loss": 3.9319355908502738, + "tokens_seen": 480263168 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004315446339017051, + "loss": 2.9604, + "theoretical_loss": 3.931877403141228, + "tokens_seen": 480328704 + }, + { + "epoch": 1.06, + "learning_rate": 0.00043153460381143436, + "loss": 3.1821, + "theoretical_loss": 3.9318192255933755, + "tokens_seen": 480394240 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004315245737211635, + "loss": 3.1779, + "theoretical_loss": 3.9317610582035556, + "tokens_seen": 480459776 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004315145436308927, + "loss": 3.0144, + "theoretical_loss": 3.9317029009686086, + "tokens_seen": 480525312 + }, + { + "epoch": 1.06, + "learning_rate": 0.00043150451354062185, + "loss": 3.1989, + "theoretical_loss": 3.9316447538853794, + "tokens_seen": 480590848 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004314944834503511, + "loss": 2.9107, + "theoretical_loss": 3.9315866169507094, + "tokens_seen": 480656384 + }, + { + "epoch": 1.06, + "learning_rate": 0.00043148445336008026, + "loss": 2.7994, + "theoretical_loss": 3.931528490161446, + "tokens_seen": 480721920 + }, + { + "epoch": 1.06, + "learning_rate": 0.00043147442326980944, + "loss": 2.9174, + "theoretical_loss": 3.9314703735144345, + "tokens_seen": 480787456 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004314643931795386, + "loss": 2.7913, + "theoretical_loss": 3.931412267006524, + "tokens_seen": 480852992 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004314543630892678, + "loss": 3.0289, + "theoretical_loss": 3.931354170634563, + "tokens_seen": 480918528 + }, + { + "epoch": 1.06, + "learning_rate": 0.000431444332998997, + "loss": 3.0211, + "theoretical_loss": 3.9312960843954046, + "tokens_seen": 480984064 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004314343029087262, + "loss": 2.9529, + "theoretical_loss": 3.9312380082858995, + "tokens_seen": 481049600 + }, + { + "epoch": 1.06, + "learning_rate": 0.00043142427281845535, + "loss": 3.0352, + "theoretical_loss": 3.931179942302902, + "tokens_seen": 481115136 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004314142427281846, + "loss": 2.9488, + "theoretical_loss": 3.9311218864432673, + "tokens_seen": 481180672 + }, + { + "epoch": 1.06, + "learning_rate": 0.00043140421263791377, + "loss": 2.9354, + "theoretical_loss": 3.9310638407038523, + "tokens_seen": 481246208 + }, + { + "epoch": 1.06, + "learning_rate": 0.00043139418254764295, + "loss": 2.8764, + "theoretical_loss": 3.931005805081515, + "tokens_seen": 481311744 + }, + { + "epoch": 1.06, + "learning_rate": 0.00043138415245737213, + "loss": 2.8272, + "theoretical_loss": 3.9309477795731143, + "tokens_seen": 481377280 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004313741223671013, + "loss": 2.9896, + "theoretical_loss": 3.930889764175512, + "tokens_seen": 481442816 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004313640922768305, + "loss": 2.9828, + "theoretical_loss": 3.930831758885569, + "tokens_seen": 481508352 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004313540621865597, + "loss": 3.0283, + "theoretical_loss": 3.9307737637001505, + "tokens_seen": 481573888 + }, + { + "epoch": 1.06, + "learning_rate": 0.00043134403209628885, + "loss": 2.915, + "theoretical_loss": 3.93071577861612, + "tokens_seen": 481639424 + }, + { + "epoch": 1.06, + "objective/train/docs_used": 563271, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.949291229248047, + "objective/train/theoretical_loss": 3.9306722964302496, + "objective/train/tokens_used": 502148576, + "theoretical_loss": 3.9306722964302496, + "tokens_seen": 481688576 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004313340020060181, + "loss": 3.0166, + "theoretical_loss": 3.930657803630346, + "tokens_seen": 481704960 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004313239719157472, + "loss": 2.8268, + "theoretical_loss": 3.9305998387396945, + "tokens_seen": 481770496 + }, + { + "epoch": 1.06, + "learning_rate": 0.00043131394182547645, + "loss": 3.1203, + "theoretical_loss": 3.9305418839410353, + "tokens_seen": 481836032 + }, + { + "epoch": 1.06, + "learning_rate": 0.00043130391173520563, + "loss": 3.1673, + "theoretical_loss": 3.930483939231239, + "tokens_seen": 481901568 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004312938816449348, + "loss": 2.9154, + "theoretical_loss": 3.930426004607178, + "tokens_seen": 481967104 + }, + { + "epoch": 1.06, + "learning_rate": 0.000431283851554664, + "loss": 2.7331, + "theoretical_loss": 3.930368080065726, + "tokens_seen": 482032640 + }, + { + "epoch": 1.06, + "learning_rate": 0.00043127382146439323, + "loss": 2.9744, + "theoretical_loss": 3.930310165603757, + "tokens_seen": 482098176 + }, + { + "epoch": 1.06, + "learning_rate": 0.00043126379137412236, + "loss": 2.959, + "theoretical_loss": 3.930252261218147, + "tokens_seen": 482163712 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004312537612838516, + "loss": 2.9013, + "theoretical_loss": 3.9301943669057744, + "tokens_seen": 482229248 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004312437311935807, + "loss": 2.9071, + "theoretical_loss": 3.9301364826635186, + "tokens_seen": 482294784 + }, + { + "epoch": 1.06, + "learning_rate": 0.00043123370110330995, + "loss": 2.7957, + "theoretical_loss": 3.930078608488259, + "tokens_seen": 482360320 + }, + { + "epoch": 1.06, + "learning_rate": 0.00043122367101303913, + "loss": 2.822, + "theoretical_loss": 3.9300207443768773, + "tokens_seen": 482425856 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004312136409227683, + "loss": 3.0268, + "theoretical_loss": 3.929962890326258, + "tokens_seen": 482491392 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004312036108324975, + "loss": 3.1638, + "theoretical_loss": 3.9299050463332845, + "tokens_seen": 482556928 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004311935807422267, + "loss": 3.1264, + "theoretical_loss": 3.929847212394843, + "tokens_seen": 482622464 + }, + { + "epoch": 1.06, + "learning_rate": 0.00043118355065195586, + "loss": 2.8405, + "theoretical_loss": 3.929789388507821, + "tokens_seen": 482688000 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004311735205616851, + "loss": 2.9131, + "theoretical_loss": 3.9297315746691073, + "tokens_seen": 482753536 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004311634904714142, + "loss": 2.8616, + "theoretical_loss": 3.9296737708755916, + "tokens_seen": 482819072 + }, + { + "epoch": 1.06, + "learning_rate": 0.00043115346038114346, + "loss": 3.1964, + "theoretical_loss": 3.9296159771241657, + "tokens_seen": 482884608 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004311434302908726, + "loss": 3.1841, + "theoretical_loss": 3.9295581934117223, + "tokens_seen": 482950144 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004311334002006018, + "loss": 2.7733, + "theoretical_loss": 3.9295004197351564, + "tokens_seen": 483015680 + }, + { + "epoch": 1.06, + "learning_rate": 0.000431123370110331, + "loss": 2.8998, + "theoretical_loss": 3.9294426560913625, + "tokens_seen": 483081216 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004311133400200602, + "loss": 3.0662, + "theoretical_loss": 3.9293849024772385, + "tokens_seen": 483146752 + }, + { + "epoch": 1.06, + "learning_rate": 0.00043110330992978936, + "loss": 3.1417, + "theoretical_loss": 3.9293271588896825, + "tokens_seen": 483212288 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004310932798395186, + "loss": 3.0213, + "theoretical_loss": 3.929269425325594, + "tokens_seen": 483277824 + }, + { + "epoch": 1.06, + "objective/train/docs_used": 567309, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9936575889587402, + "objective/train/theoretical_loss": 3.929226131728565, + "objective/train/tokens_used": 503786976, + "theoretical_loss": 3.929226131728565, + "tokens_seen": 483326976 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004310832497492477, + "loss": 2.9944, + "theoretical_loss": 3.929211701781875, + "tokens_seen": 483343360 + }, + { + "epoch": 1.06, + "learning_rate": 0.00043107321965897696, + "loss": 2.9511, + "theoretical_loss": 3.929153988255427, + "tokens_seen": 483408896 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004310631895687061, + "loss": 3.0354, + "theoretical_loss": 3.9290962847431548, + "tokens_seen": 483474432 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004310531594784353, + "loss": 2.9623, + "theoretical_loss": 3.9290385912419628, + "tokens_seen": 483539968 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004310431293881645, + "loss": 2.8225, + "theoretical_loss": 3.9289809077487585, + "tokens_seen": 483605504 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004310330992978937, + "loss": 3.083, + "theoretical_loss": 3.9289232342604494, + "tokens_seen": 483671040 + }, + { + "epoch": 1.06, + "learning_rate": 0.00043102306920762286, + "loss": 3.022, + "theoretical_loss": 3.9288655707739455, + "tokens_seen": 483736576 + }, + { + "epoch": 1.06, + "learning_rate": 0.00043101303911735205, + "loss": 2.8435, + "theoretical_loss": 3.928807917286157, + "tokens_seen": 483802112 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004310030090270812, + "loss": 2.8934, + "theoretical_loss": 3.928750273793997, + "tokens_seen": 483867648 + }, + { + "epoch": 1.06, + "learning_rate": 0.00043099297893681046, + "loss": 3.0615, + "theoretical_loss": 3.9286926402943774, + "tokens_seen": 483933184 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004309829488465396, + "loss": 2.8151, + "theoretical_loss": 3.928635016784215, + "tokens_seen": 483998720 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004309729187562688, + "loss": 3.0473, + "theoretical_loss": 3.9285774032604244, + "tokens_seen": 484064256 + }, + { + "epoch": 1.06, + "learning_rate": 0.00043096288866599795, + "loss": 3.172, + "theoretical_loss": 3.9285197997199246, + "tokens_seen": 484129792 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004309528585757272, + "loss": 2.8463, + "theoretical_loss": 3.928462206159634, + "tokens_seen": 484195328 + }, + { + "epoch": 1.06, + "learning_rate": 0.00043094282848545637, + "loss": 2.9347, + "theoretical_loss": 3.928404622576472, + "tokens_seen": 484260864 + }, + { + "epoch": 1.06, + "learning_rate": 0.00043093279839518555, + "loss": 2.9781, + "theoretical_loss": 3.928347048967362, + "tokens_seen": 484326400 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004309227683049148, + "loss": 3.0882, + "theoretical_loss": 3.928289485329227, + "tokens_seen": 484391936 + }, + { + "epoch": 1.06, + "learning_rate": 0.00043091273821464397, + "loss": 2.8604, + "theoretical_loss": 3.92823193165899, + "tokens_seen": 484457472 + }, + { + "epoch": 1.06, + "learning_rate": 0.00043090270812437315, + "loss": 2.9441, + "theoretical_loss": 3.928174387953579, + "tokens_seen": 484523008 + }, + { + "epoch": 1.06, + "learning_rate": 0.00043089267803410233, + "loss": 2.9715, + "theoretical_loss": 3.9281168542099194, + "tokens_seen": 484588544 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004308826479438315, + "loss": 3.1032, + "theoretical_loss": 3.9280593304249405, + "tokens_seen": 484654080 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004308726178535607, + "loss": 2.9442, + "theoretical_loss": 3.928001816595572, + "tokens_seen": 484719616 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004308625877632899, + "loss": 3.163, + "theoretical_loss": 3.9279443127187452, + "tokens_seen": 484785152 + }, + { + "epoch": 1.06, + "learning_rate": 0.00043085255767301905, + "loss": 3.1574, + "theoretical_loss": 3.9278868187913933, + "tokens_seen": 484850688 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004308425275827483, + "loss": 2.7887, + "theoretical_loss": 3.9278293348104496, + "tokens_seen": 484916224 + }, + { + "epoch": 1.06, + "objective/train/docs_used": 572265, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.1022114753723145, + "objective/train/theoretical_loss": 3.927786228350229, + "objective/train/tokens_used": 505425376, + "theoretical_loss": 3.927786228350229, + "tokens_seen": 484965376 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004308324974924774, + "loss": 3.0504, + "theoretical_loss": 3.9277718607728502, + "tokens_seen": 484981760 + }, + { + "epoch": 1.06, + "learning_rate": 0.00043082246740220665, + "loss": 2.8934, + "theoretical_loss": 3.9277143966755315, + "tokens_seen": 485047296 + }, + { + "epoch": 1.06, + "learning_rate": 0.00043081243731193583, + "loss": 2.9485, + "theoretical_loss": 3.927656942515431, + "tokens_seen": 485112832 + }, + { + "epoch": 1.06, + "learning_rate": 0.000430802407221665, + "loss": 3.1336, + "theoretical_loss": 3.927599498289489, + "tokens_seen": 485178368 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004307923771313942, + "loss": 2.9035, + "theoretical_loss": 3.927542063994646, + "tokens_seen": 485243904 + }, + { + "epoch": 1.06, + "learning_rate": 0.00043078234704112343, + "loss": 3.1617, + "theoretical_loss": 3.927484639627844, + "tokens_seen": 485309440 + }, + { + "epoch": 1.06, + "learning_rate": 0.00043077231695085256, + "loss": 2.9164, + "theoretical_loss": 3.927427225186027, + "tokens_seen": 485374976 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004307622868605818, + "loss": 3.2708, + "theoretical_loss": 3.927369820666139, + "tokens_seen": 485440512 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004307522567703109, + "loss": 3.16, + "theoretical_loss": 3.9273124260651278, + "tokens_seen": 485506048 + }, + { + "epoch": 1.06, + "learning_rate": 0.00043074222668004015, + "loss": 3.1773, + "theoretical_loss": 3.927255041379939, + "tokens_seen": 485571584 + }, + { + "epoch": 1.06, + "learning_rate": 0.00043073219658976933, + "loss": 2.9499, + "theoretical_loss": 3.9271976666075226, + "tokens_seen": 485637120 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004307221664994985, + "loss": 2.79, + "theoretical_loss": 3.9271403017448288, + "tokens_seen": 485702656 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004307121364092277, + "loss": 3.0427, + "theoretical_loss": 3.9270829467888095, + "tokens_seen": 485768192 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004307021063189569, + "loss": 3.1896, + "theoretical_loss": 3.9270256017364167, + "tokens_seen": 485833728 + }, + { + "epoch": 1.06, + "learning_rate": 0.00043069207622868606, + "loss": 3.217, + "theoretical_loss": 3.9269682665846064, + "tokens_seen": 485899264 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004306820461384153, + "loss": 3.0031, + "theoretical_loss": 3.9269109413303322, + "tokens_seen": 485964800 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004306720160481444, + "loss": 3.174, + "theoretical_loss": 3.9268536259705527, + "tokens_seen": 486030336 + }, + { + "epoch": 1.06, + "learning_rate": 0.00043066198595787366, + "loss": 2.8777, + "theoretical_loss": 3.926796320502225, + "tokens_seen": 486095872 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004306519558676028, + "loss": 3.1536, + "theoretical_loss": 3.926739024922311, + "tokens_seen": 486161408 + }, + { + "epoch": 1.06, + "learning_rate": 0.000430641925777332, + "loss": 2.9334, + "theoretical_loss": 3.926681739227769, + "tokens_seen": 486226944 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004306318956870612, + "loss": 2.8147, + "theoretical_loss": 3.926624463415563, + "tokens_seen": 486292480 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004306218655967904, + "loss": 2.9436, + "theoretical_loss": 3.9265671974826564, + "tokens_seen": 486358016 + }, + { + "epoch": 1.06, + "learning_rate": 0.00043061183550651956, + "loss": 2.9908, + "theoretical_loss": 3.9265099414260147, + "tokens_seen": 486423552 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004306018054162488, + "loss": 3.0091, + "theoretical_loss": 3.9264526952426033, + "tokens_seen": 486489088 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004305917753259779, + "loss": 2.9953, + "theoretical_loss": 3.926395458929391, + "tokens_seen": 486554624 + }, + { + "epoch": 1.06, + "objective/train/docs_used": 577157, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9272384643554688, + "objective/train/theoretical_loss": 3.9263525381699758, + "objective/train/tokens_used": 507063776, + "theoretical_loss": 3.9263525381699758, + "tokens_seen": 486603776 + }, + { + "epoch": 1.06, + "learning_rate": 0.00043058174523570716, + "loss": 3.0174, + "theoretical_loss": 3.926338232483346, + "tokens_seen": 486620160 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004305717151454363, + "loss": 3.1603, + "theoretical_loss": 3.9262810159014396, + "tokens_seen": 486685696 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004305616850551655, + "loss": 3.1405, + "theoretical_loss": 3.926223809180643, + "tokens_seen": 486751232 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004305516549648947, + "loss": 2.9448, + "theoretical_loss": 3.9261666123179295, + "tokens_seen": 486816768 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004305416248746239, + "loss": 3.2668, + "theoretical_loss": 3.9261094253102735, + "tokens_seen": 486882304 + }, + { + "epoch": 1.06, + "learning_rate": 0.00043053159478435306, + "loss": 2.8592, + "theoretical_loss": 3.926052248154651, + "tokens_seen": 486947840 + }, + { + "epoch": 1.06, + "learning_rate": 0.00043052156469408225, + "loss": 2.8944, + "theoretical_loss": 3.925995080848039, + "tokens_seen": 487013376 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004305115346038114, + "loss": 2.9189, + "theoretical_loss": 3.9259379233874157, + "tokens_seen": 487078912 + }, + { + "epoch": 1.06, + "learning_rate": 0.00043050150451354066, + "loss": 2.8615, + "theoretical_loss": 3.9258807757697607, + "tokens_seen": 487144448 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004304914744232698, + "loss": 3.1765, + "theoretical_loss": 3.9258236379920564, + "tokens_seen": 487209984 + }, + { + "epoch": 1.06, + "learning_rate": 0.000430481444332999, + "loss": 2.7061, + "theoretical_loss": 3.925766510051284, + "tokens_seen": 487275520 + }, + { + "epoch": 1.06, + "learning_rate": 0.00043047141424272815, + "loss": 2.9585, + "theoretical_loss": 3.9257093919444275, + "tokens_seen": 487341056 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004304613841524574, + "loss": 3.07, + "theoretical_loss": 3.925652283668472, + "tokens_seen": 487406592 + }, + { + "epoch": 1.06, + "learning_rate": 0.00043045135406218657, + "loss": 3.2781, + "theoretical_loss": 3.9255951852204047, + "tokens_seen": 487472128 + }, + { + "epoch": 1.06, + "learning_rate": 0.00043044132397191575, + "loss": 2.9378, + "theoretical_loss": 3.9255380965972124, + "tokens_seen": 487537664 + }, + { + "epoch": 1.06, + "learning_rate": 0.00043043129388164493, + "loss": 3.1473, + "theoretical_loss": 3.9254810177958848, + "tokens_seen": 487603200 + }, + { + "epoch": 1.06, + "learning_rate": 0.00043042126379137417, + "loss": 2.8296, + "theoretical_loss": 3.9254239488134117, + "tokens_seen": 487668736 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004304112337011033, + "loss": 3.0285, + "theoretical_loss": 3.925366889646786, + "tokens_seen": 487734272 + }, + { + "epoch": 1.06, + "learning_rate": 0.00043040120361083253, + "loss": 2.8856, + "theoretical_loss": 3.925309840292999, + "tokens_seen": 487799808 + }, + { + "epoch": 1.06, + "learning_rate": 0.00043039117352056165, + "loss": 2.8328, + "theoretical_loss": 3.925252800749047, + "tokens_seen": 487865344 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004303811434302909, + "loss": 2.9618, + "theoretical_loss": 3.925195771011924, + "tokens_seen": 487930880 + }, + { + "epoch": 1.06, + "learning_rate": 0.00043037111334002007, + "loss": 3.0661, + "theoretical_loss": 3.925138751078629, + "tokens_seen": 487996416 + }, + { + "epoch": 1.06, + "learning_rate": 0.00043036108324974925, + "loss": 3.176, + "theoretical_loss": 3.925081740946159, + "tokens_seen": 488061952 + }, + { + "epoch": 1.06, + "learning_rate": 0.00043035105315947843, + "loss": 3.0429, + "theoretical_loss": 3.9250247406115135, + "tokens_seen": 488127488 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004303410230692076, + "loss": 3.0888, + "theoretical_loss": 3.924967750071694, + "tokens_seen": 488193024 + }, + { + "epoch": 1.06, + "objective/train/docs_used": 582212, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.2292799949645996, + "objective/train/theoretical_loss": 3.9249250135928815, + "objective/train/tokens_used": 508702176, + "theoretical_loss": 3.9249250135928815, + "tokens_seen": 488242176 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004303309929789368, + "loss": 2.9761, + "theoretical_loss": 3.9249107693237035, + "tokens_seen": 488258560 + }, + { + "epoch": 1.06, + "learning_rate": 0.00043032096288866603, + "loss": 2.9192, + "theoretical_loss": 3.924853798364545, + "tokens_seen": 488324096 + }, + { + "epoch": 1.06, + "learning_rate": 0.00043031093279839516, + "loss": 3.129, + "theoretical_loss": 3.9247968371912227, + "tokens_seen": 488389632 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004303009027081244, + "loss": 3.078, + "theoretical_loss": 3.9247398858007445, + "tokens_seen": 488455168 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004302908726178535, + "loss": 3.0365, + "theoretical_loss": 3.924682944190117, + "tokens_seen": 488520704 + }, + { + "epoch": 1.06, + "learning_rate": 0.00043028084252758276, + "loss": 2.8859, + "theoretical_loss": 3.924626012356349, + "tokens_seen": 488586240 + }, + { + "epoch": 1.06, + "learning_rate": 0.00043027081243731194, + "loss": 3.0156, + "theoretical_loss": 3.924569090296451, + "tokens_seen": 488651776 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004302607823470411, + "loss": 3.0717, + "theoretical_loss": 3.9245121780074355, + "tokens_seen": 488717312 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004302507522567703, + "loss": 2.8483, + "theoretical_loss": 3.924455275486314, + "tokens_seen": 488782848 + }, + { + "epoch": 1.06, + "learning_rate": 0.00043024072216649953, + "loss": 2.7496, + "theoretical_loss": 3.9243983827301006, + "tokens_seen": 488848384 + }, + { + "epoch": 1.06, + "learning_rate": 0.00043023069207622866, + "loss": 3.1034, + "theoretical_loss": 3.924341499735812, + "tokens_seen": 488913920 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004302206619859579, + "loss": 2.8161, + "theoretical_loss": 3.924284626500464, + "tokens_seen": 488979456 + }, + { + "epoch": 1.06, + "learning_rate": 0.000430210631895687, + "loss": 3.0165, + "theoretical_loss": 3.9242277630210753, + "tokens_seen": 489044992 + }, + { + "epoch": 1.06, + "learning_rate": 0.00043020060180541626, + "loss": 3.1262, + "theoretical_loss": 3.9241709092946655, + "tokens_seen": 489110528 + }, + { + "epoch": 1.06, + "learning_rate": 0.00043019057171514544, + "loss": 3.0549, + "theoretical_loss": 3.924114065318255, + "tokens_seen": 489176064 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004301805416248746, + "loss": 3.0324, + "theoretical_loss": 3.9240572310888657, + "tokens_seen": 489241600 + }, + { + "epoch": 1.06, + "learning_rate": 0.00043017051153460386, + "loss": 2.9433, + "theoretical_loss": 3.924000406603521, + "tokens_seen": 489307136 + }, + { + "epoch": 1.06, + "learning_rate": 0.000430160481444333, + "loss": 2.8329, + "theoretical_loss": 3.9239435918592456, + "tokens_seen": 489372672 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004301504513540622, + "loss": 3.0254, + "theoretical_loss": 3.923886786853066, + "tokens_seen": 489438208 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004301404212637914, + "loss": 2.874, + "theoretical_loss": 3.9238299915820085, + "tokens_seen": 489503744 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004301303911735206, + "loss": 2.8958, + "theoretical_loss": 3.9237732060431023, + "tokens_seen": 489569280 + }, + { + "epoch": 1.06, + "learning_rate": 0.00043012036108324976, + "loss": 2.9936, + "theoretical_loss": 3.9237164302333776, + "tokens_seen": 489634816 + }, + { + "epoch": 1.06, + "learning_rate": 0.000430110330992979, + "loss": 3.0332, + "theoretical_loss": 3.923659664149865, + "tokens_seen": 489700352 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004301003009027081, + "loss": 3.1645, + "theoretical_loss": 3.923602907789597, + "tokens_seen": 489765888 + }, + { + "epoch": 1.06, + "learning_rate": 0.00043009027081243736, + "loss": 2.7748, + "theoretical_loss": 3.9235461611496083, + "tokens_seen": 489831424 + }, + { + "epoch": 1.06, + "objective/train/docs_used": 587295, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.1512694358825684, + "objective/train/theoretical_loss": 3.923503607546766, + "objective/train/tokens_used": 510340576, + "theoretical_loss": 3.923503607546766, + "tokens_seen": 489880576 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004300802407221665, + "loss": 2.8325, + "theoretical_loss": 3.9234894242269327, + "tokens_seen": 489896960 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004300702106318957, + "loss": 2.8625, + "theoretical_loss": 3.9234326970186073, + "tokens_seen": 489962496 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004300601805416249, + "loss": 3.0172, + "theoretical_loss": 3.92337597952167, + "tokens_seen": 490028032 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004300501504513541, + "loss": 3.0533, + "theoretical_loss": 3.9233192717331598, + "tokens_seen": 490093568 + }, + { + "epoch": 1.06, + "learning_rate": 0.00043004012036108326, + "loss": 2.9677, + "theoretical_loss": 3.923262573650116, + "tokens_seen": 490159104 + }, + { + "epoch": 1.06, + "learning_rate": 0.00043003009027081245, + "loss": 2.8643, + "theoretical_loss": 3.9232058852695815, + "tokens_seen": 490224640 + }, + { + "epoch": 1.06, + "learning_rate": 0.00043002006018054163, + "loss": 3.1406, + "theoretical_loss": 3.9231492065885982, + "tokens_seen": 490290176 + }, + { + "epoch": 1.06, + "learning_rate": 0.00043001003009027086, + "loss": 3.2728, + "theoretical_loss": 3.9230925376042114, + "tokens_seen": 490355712 + }, + { + "epoch": 1.06, + "learning_rate": 0.00043, + "loss": 3.1938, + "theoretical_loss": 3.923035878313465, + "tokens_seen": 490421248 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004299899699097292, + "loss": 2.9093, + "theoretical_loss": 3.9229792287134075, + "tokens_seen": 490486784 + }, + { + "epoch": 1.06, + "learning_rate": 0.00042997993981945835, + "loss": 2.8083, + "theoretical_loss": 3.922922588801086, + "tokens_seen": 490552320 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004299699097291876, + "loss": 3.3187, + "theoretical_loss": 3.9228659585735497, + "tokens_seen": 490617856 + }, + { + "epoch": 1.06, + "learning_rate": 0.00042995987963891677, + "loss": 2.8643, + "theoretical_loss": 3.92280933802785, + "tokens_seen": 490683392 + }, + { + "epoch": 1.06, + "learning_rate": 0.00042994984954864595, + "loss": 2.8438, + "theoretical_loss": 3.922752727161038, + "tokens_seen": 490748928 + }, + { + "epoch": 1.06, + "learning_rate": 0.00042993981945837513, + "loss": 3.1228, + "theoretical_loss": 3.922696125970168, + "tokens_seen": 490814464 + }, + { + "epoch": 1.06, + "learning_rate": 0.00042992978936810437, + "loss": 2.8619, + "theoretical_loss": 3.9226395344522933, + "tokens_seen": 490880000 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004299197592778335, + "loss": 3.0386, + "theoretical_loss": 3.9225829526044707, + "tokens_seen": 490945536 + }, + { + "epoch": 1.06, + "learning_rate": 0.00042990972918756273, + "loss": 2.8399, + "theoretical_loss": 3.922526380423757, + "tokens_seen": 491011072 + }, + { + "epoch": 1.06, + "learning_rate": 0.00042989969909729185, + "loss": 2.8468, + "theoretical_loss": 3.922469817907211, + "tokens_seen": 491076608 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004298896690070211, + "loss": 2.9284, + "theoretical_loss": 3.9224132650518913, + "tokens_seen": 491142144 + }, + { + "epoch": 1.06, + "learning_rate": 0.00042987963891675027, + "loss": 2.9865, + "theoretical_loss": 3.9223567218548596, + "tokens_seen": 491207680 + }, + { + "epoch": 1.06, + "learning_rate": 0.00042986960882647945, + "loss": 2.9417, + "theoretical_loss": 3.9223001883131783, + "tokens_seen": 491273216 + }, + { + "epoch": 1.06, + "learning_rate": 0.00042985957873620863, + "loss": 3.009, + "theoretical_loss": 3.9222436644239114, + "tokens_seen": 491338752 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004298495486459378, + "loss": 2.8589, + "theoretical_loss": 3.922187150184122, + "tokens_seen": 491404288 + }, + { + "epoch": 1.06, + "learning_rate": 0.000429839518555667, + "loss": 2.9139, + "theoretical_loss": 3.9221306455908778, + "tokens_seen": 491469824 + }, + { + "debugging/Self-BLEU-5": 0.5154921509631192, + "debugging/distinct-1-grams": 0.734842616282129, + "debugging/distinct-2-grams": 0.8846155258692667, + "debugging/entropy-1-grams": 6.056339442041576, + "debugging/entropy-2-grams": 7.145308721973038, + "debugging/length": 484.85, + "debugging/num_segments": 20, + "debugging/score": 0.003003109095205246, + "debugging/score_std": 0.004943536934620138, + "epoch": 1.06, + "objective/train/docs_used": 590236, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.2846457958221436, + "objective/train/theoretical_loss": 3.922088273474726, + "objective/train/tokens_used": 511978976, + "theoretical_loss": 3.922088273474726, + "tokens_seen": 491518976 + }, + { + "epoch": 1.06, + "learning_rate": 0.00042982948846539623, + "loss": 3.0361, + "theoretical_loss": 3.9220741506412464, + "tokens_seen": 491535360 + }, + { + "epoch": 1.06, + "learning_rate": 0.00042981945837512536, + "loss": 2.8973, + "theoretical_loss": 3.9220176653322953, + "tokens_seen": 491600896 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004298094282848546, + "loss": 2.7277, + "theoretical_loss": 3.9219611896610944, + "tokens_seen": 491666432 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004297993981945837, + "loss": 2.8928, + "theoretical_loss": 3.921904723624716, + "tokens_seen": 491731968 + }, + { + "epoch": 1.06, + "learning_rate": 0.00042978936810431296, + "loss": 2.8518, + "theoretical_loss": 3.9218482672202324, + "tokens_seen": 491797504 + }, + { + "epoch": 1.06, + "learning_rate": 0.00042977933801404214, + "loss": 3.1745, + "theoretical_loss": 3.9217918204447173, + "tokens_seen": 491863040 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004297693079237713, + "loss": 2.4701, + "theoretical_loss": 3.9217353832952453, + "tokens_seen": 491928576 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004297592778335005, + "loss": 3.0984, + "theoretical_loss": 3.921678955768893, + "tokens_seen": 491994112 + }, + { + "epoch": 1.06, + "learning_rate": 0.00042974924774322973, + "loss": 3.0063, + "theoretical_loss": 3.9216225378627385, + "tokens_seen": 492059648 + }, + { + "epoch": 1.06, + "learning_rate": 0.00042973921765295886, + "loss": 2.8113, + "theoretical_loss": 3.92156612957386, + "tokens_seen": 492125184 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004297291875626881, + "loss": 3.1234, + "theoretical_loss": 3.921509730899338, + "tokens_seen": 492190720 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004297191574724172, + "loss": 2.9805, + "theoretical_loss": 3.9214533418362545, + "tokens_seen": 492256256 + }, + { + "epoch": 1.06, + "learning_rate": 0.00042970912738214646, + "loss": 2.7724, + "theoretical_loss": 3.9213969623816913, + "tokens_seen": 492321792 + }, + { + "epoch": 1.06, + "learning_rate": 0.00042969909729187564, + "loss": 2.9657, + "theoretical_loss": 3.921340592532733, + "tokens_seen": 492387328 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004296890672016048, + "loss": 3.0349, + "theoretical_loss": 3.921284232286465, + "tokens_seen": 492452864 + }, + { + "epoch": 1.06, + "learning_rate": 0.000429679037111334, + "loss": 3.1185, + "theoretical_loss": 3.9212278816399735, + "tokens_seen": 492518400 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004296690070210632, + "loss": 3.1046, + "theoretical_loss": 3.9211715405903464, + "tokens_seen": 492583936 + }, + { + "epoch": 1.06, + "learning_rate": 0.00042965897693079236, + "loss": 2.7548, + "theoretical_loss": 3.9211152091346735, + "tokens_seen": 492649472 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004296489468405216, + "loss": 2.9028, + "theoretical_loss": 3.9210588872700436, + "tokens_seen": 492715008 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004296389167502507, + "loss": 2.9545, + "theoretical_loss": 3.9210025749935493, + "tokens_seen": 492780544 + }, + { + "epoch": 1.06, + "learning_rate": 0.00042962888665997996, + "loss": 2.8856, + "theoretical_loss": 3.9209462723022837, + "tokens_seen": 492846080 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004296188565697091, + "loss": 2.8481, + "theoretical_loss": 3.9208899791933414, + "tokens_seen": 492911616 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004296088264794383, + "loss": 3.013, + "theoretical_loss": 3.9208336956638163, + "tokens_seen": 492977152 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004295987963891675, + "loss": 2.8045, + "theoretical_loss": 3.920777421710807, + "tokens_seen": 493042688 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004295887662988967, + "loss": 2.8337, + "theoretical_loss": 3.9207211573314096, + "tokens_seen": 493108224 + }, + { + "epoch": 1.06, + "objective/train/docs_used": 590917, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5917956829071045, + "objective/train/theoretical_loss": 3.9206789653278, + "objective/train/tokens_used": 513617376, + "theoretical_loss": 3.9206789653278, + "tokens_seen": 493157376 + }, + { + "epoch": 1.06, + "learning_rate": 0.00042957873620862587, + "loss": 2.8503, + "theoretical_loss": 3.9206649025227245, + "tokens_seen": 493173760 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004295687061183551, + "loss": 3.2481, + "theoretical_loss": 3.9206086572818517, + "tokens_seen": 493239296 + }, + { + "epoch": 1.06, + "learning_rate": 0.00042955867602808423, + "loss": 3.1467, + "theoretical_loss": 3.920552421605894, + "tokens_seen": 493304832 + }, + { + "epoch": 1.06, + "learning_rate": 0.00042954864593781347, + "loss": 2.8639, + "theoretical_loss": 3.920496195491953, + "tokens_seen": 493370368 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004295386158475426, + "loss": 3.1696, + "theoretical_loss": 3.920439978937134, + "tokens_seen": 493435904 + }, + { + "epoch": 1.06, + "learning_rate": 0.00042952858575727183, + "loss": 3.0995, + "theoretical_loss": 3.9203837719385417, + "tokens_seen": 493501440 + }, + { + "epoch": 1.06, + "learning_rate": 0.000429518555667001, + "loss": 2.6516, + "theoretical_loss": 3.920327574493284, + "tokens_seen": 493566976 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004295085255767302, + "loss": 3.0273, + "theoretical_loss": 3.920271386598468, + "tokens_seen": 493632512 + }, + { + "epoch": 1.06, + "learning_rate": 0.00042949849548645937, + "loss": 2.7616, + "theoretical_loss": 3.9202152082512036, + "tokens_seen": 493698048 + }, + { + "epoch": 1.06, + "learning_rate": 0.00042948846539618855, + "loss": 2.9871, + "theoretical_loss": 3.9201590394486012, + "tokens_seen": 493763584 + }, + { + "epoch": 1.06, + "learning_rate": 0.00042947843530591773, + "loss": 2.8426, + "theoretical_loss": 3.9201028801877724, + "tokens_seen": 493829120 + }, + { + "epoch": 1.06, + "learning_rate": 0.00042946840521564697, + "loss": 2.9274, + "theoretical_loss": 3.920046730465831, + "tokens_seen": 493894656 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004294583751253761, + "loss": 2.8587, + "theoretical_loss": 3.919990590279891, + "tokens_seen": 493960192 + }, + { + "epoch": 1.06, + "learning_rate": 0.00042944834503510533, + "loss": 2.7168, + "theoretical_loss": 3.9199344596270675, + "tokens_seen": 494025728 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004294383149448345, + "loss": 2.8623, + "theoretical_loss": 3.9198783385044784, + "tokens_seen": 494091264 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004294282848545637, + "loss": 2.8781, + "theoretical_loss": 3.919822226909241, + "tokens_seen": 494156800 + }, + { + "epoch": 1.06, + "learning_rate": 0.00042941825476429293, + "loss": 3.0756, + "theoretical_loss": 3.919766124838475, + "tokens_seen": 494222336 + }, + { + "epoch": 1.06, + "learning_rate": 0.00042940822467402206, + "loss": 2.9693, + "theoretical_loss": 3.9197100322893013, + "tokens_seen": 494287872 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004293981945837513, + "loss": 2.8422, + "theoretical_loss": 3.919653949258841, + "tokens_seen": 494353408 + }, + { + "epoch": 1.06, + "learning_rate": 0.00042938816449348047, + "loss": 2.9434, + "theoretical_loss": 3.919597875744218, + "tokens_seen": 494418944 + }, + { + "epoch": 1.06, + "learning_rate": 0.00042937813440320965, + "loss": 2.9088, + "theoretical_loss": 3.9195418117425564, + "tokens_seen": 494484480 + }, + { + "epoch": 1.06, + "learning_rate": 0.00042936810431293883, + "loss": 2.8325, + "theoretical_loss": 3.919485757250982, + "tokens_seen": 494550016 + }, + { + "epoch": 1.06, + "learning_rate": 0.000429358074222668, + "loss": 2.8604, + "theoretical_loss": 3.9194297122666213, + "tokens_seen": 494615552 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004293480441323972, + "loss": 2.9683, + "theoretical_loss": 3.919373676786603, + "tokens_seen": 494681088 + }, + { + "epoch": 1.06, + "learning_rate": 0.00042933801404212643, + "loss": 3.1226, + "theoretical_loss": 3.919317650808056, + "tokens_seen": 494746624 + }, + { + "epoch": 1.06, + "objective/train/docs_used": 592366, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.896491527557373, + "objective/train/theoretical_loss": 3.91927563755776, + "objective/train/tokens_used": 515255776, + "theoretical_loss": 3.91927563755776, + "tokens_seen": 494795776 + }, + { + "epoch": 1.06, + "learning_rate": 0.00042932798395185556, + "loss": 3.0733, + "theoretical_loss": 3.9192616343281106, + "tokens_seen": 494812160 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004293179538615848, + "loss": 3.0851, + "theoretical_loss": 3.9192056273438998, + "tokens_seen": 494877696 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004293079237713139, + "loss": 3.1179, + "theoretical_loss": 3.919149629852556, + "tokens_seen": 494943232 + }, + { + "epoch": 1.06, + "learning_rate": 0.00042929789368104316, + "loss": 3.011, + "theoretical_loss": 3.919093641851214, + "tokens_seen": 495008768 + }, + { + "epoch": 1.06, + "learning_rate": 0.00042928786359077234, + "loss": 2.7185, + "theoretical_loss": 3.9190376633370088, + "tokens_seen": 495074304 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004292778335005015, + "loss": 3.0177, + "theoretical_loss": 3.918981694307077, + "tokens_seen": 495139840 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004292678034102307, + "loss": 2.9071, + "theoretical_loss": 3.9189257347585578, + "tokens_seen": 495205376 + }, + { + "epoch": 1.06, + "learning_rate": 0.00042925777331995993, + "loss": 3.0857, + "theoretical_loss": 3.9188697846885905, + "tokens_seen": 495270912 + }, + { + "epoch": 1.06, + "learning_rate": 0.00042924774322968906, + "loss": 3.1196, + "theoretical_loss": 3.9188138440943145, + "tokens_seen": 495336448 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004292377131394183, + "loss": 2.9395, + "theoretical_loss": 3.9187579129728727, + "tokens_seen": 495401984 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004292276830491474, + "loss": 3.0981, + "theoretical_loss": 3.9187019913214076, + "tokens_seen": 495467520 + }, + { + "epoch": 1.06, + "learning_rate": 0.00042921765295887666, + "loss": 2.8836, + "theoretical_loss": 3.918646079137064, + "tokens_seen": 495533056 + }, + { + "epoch": 1.06, + "learning_rate": 0.00042920762286860584, + "loss": 3.0769, + "theoretical_loss": 3.918590176416987, + "tokens_seen": 495598592 + }, + { + "epoch": 1.06, + "learning_rate": 0.000429197592778335, + "loss": 2.9373, + "theoretical_loss": 3.918534283158323, + "tokens_seen": 495664128 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004291875626880642, + "loss": 2.8688, + "theoretical_loss": 3.9184783993582215, + "tokens_seen": 495729664 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004291775325977934, + "loss": 2.866, + "theoretical_loss": 3.91842252501383, + "tokens_seen": 495795200 + }, + { + "epoch": 1.06, + "learning_rate": 0.00042916750250752256, + "loss": 3.0718, + "theoretical_loss": 3.9183666601222997, + "tokens_seen": 495860736 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004291574724172518, + "loss": 2.7853, + "theoretical_loss": 3.9183108046807833, + "tokens_seen": 495926272 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004291474423269809, + "loss": 2.8375, + "theoretical_loss": 3.918254958686432, + "tokens_seen": 495991808 + }, + { + "epoch": 1.06, + "learning_rate": 0.00042913741223671016, + "loss": 3.17, + "theoretical_loss": 3.918199122136401, + "tokens_seen": 496057344 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004291273821464393, + "loss": 2.9484, + "theoretical_loss": 3.9181432950278463, + "tokens_seen": 496122880 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004291173520561685, + "loss": 2.966, + "theoretical_loss": 3.9180874773579233, + "tokens_seen": 496188416 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004291073219658977, + "loss": 2.9595, + "theoretical_loss": 3.91803166912379, + "tokens_seen": 496253952 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004290972918756269, + "loss": 2.6767, + "theoretical_loss": 3.917975870322607, + "tokens_seen": 496319488 + }, + { + "epoch": 1.06, + "learning_rate": 0.00042908726178535607, + "loss": 2.8961, + "theoretical_loss": 3.917920080951533, + "tokens_seen": 496385024 + }, + { + "epoch": 1.06, + "objective/train/docs_used": 593160, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7651126384735107, + "objective/train/theoretical_loss": 3.9178782451100296, + "objective/train/tokens_used": 516894176, + "theoretical_loss": 3.9178782451100296, + "tokens_seen": 496434176 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004290772316950853, + "loss": 3.037, + "theoretical_loss": 3.9178643010077305, + "tokens_seen": 496450560 + }, + { + "epoch": 1.06, + "learning_rate": 0.00042906720160481443, + "loss": 2.8813, + "theoretical_loss": 3.9178085304883616, + "tokens_seen": 496516096 + }, + { + "epoch": 1.06, + "learning_rate": 0.00042905717151454367, + "loss": 2.8188, + "theoretical_loss": 3.9177527693905914, + "tokens_seen": 496581632 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004290471414242728, + "loss": 3.0604, + "theoretical_loss": 3.9176970177115837, + "tokens_seen": 496647168 + }, + { + "epoch": 1.06, + "learning_rate": 0.00042903711133400203, + "loss": 2.8691, + "theoretical_loss": 3.9176412754485064, + "tokens_seen": 496712704 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004290270812437312, + "loss": 2.9651, + "theoretical_loss": 3.917585542598527, + "tokens_seen": 496778240 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004290170511534604, + "loss": 3.1923, + "theoretical_loss": 3.9175298191588137, + "tokens_seen": 496843776 + }, + { + "epoch": 1.06, + "learning_rate": 0.00042900702106318957, + "loss": 2.8673, + "theoretical_loss": 3.9174741051265367, + "tokens_seen": 496909312 + }, + { + "epoch": 1.06, + "learning_rate": 0.00042899699097291875, + "loss": 2.9436, + "theoretical_loss": 3.9174184004988684, + "tokens_seen": 496974848 + }, + { + "epoch": 1.06, + "learning_rate": 0.00042898696088264793, + "loss": 3.2724, + "theoretical_loss": 3.9173627052729803, + "tokens_seen": 497040384 + }, + { + "epoch": 1.06, + "learning_rate": 0.00042897693079237717, + "loss": 3.1074, + "theoretical_loss": 3.9173070194460466, + "tokens_seen": 497105920 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004289669007021063, + "loss": 2.9554, + "theoretical_loss": 3.9172513430152427, + "tokens_seen": 497171456 + }, + { + "epoch": 1.06, + "learning_rate": 0.00042895687061183553, + "loss": 2.8133, + "theoretical_loss": 3.9171956759777453, + "tokens_seen": 497236992 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004289468405215647, + "loss": 2.9769, + "theoretical_loss": 3.9171400183307306, + "tokens_seen": 497302528 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004289368104312939, + "loss": 3.1201, + "theoretical_loss": 3.917084370071378, + "tokens_seen": 497368064 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004289267803410231, + "loss": 2.8267, + "theoretical_loss": 3.9170287311968677, + "tokens_seen": 497433600 + }, + { + "epoch": 1.06, + "learning_rate": 0.00042891675025075226, + "loss": 2.8601, + "theoretical_loss": 3.9169731017043805, + "tokens_seen": 497499136 + }, + { + "epoch": 1.06, + "learning_rate": 0.00042890672016048144, + "loss": 3.0935, + "theoretical_loss": 3.9169174815910983, + "tokens_seen": 497564672 + }, + { + "epoch": 1.06, + "learning_rate": 0.00042889669007021067, + "loss": 2.785, + "theoretical_loss": 3.916861870854206, + "tokens_seen": 497630208 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004288866599799398, + "loss": 3.1246, + "theoretical_loss": 3.9168062694908876, + "tokens_seen": 497695744 + }, + { + "epoch": 1.06, + "learning_rate": 0.00042887662988966903, + "loss": 3.036, + "theoretical_loss": 3.916750677498329, + "tokens_seen": 497761280 + }, + { + "epoch": 1.06, + "learning_rate": 0.00042886659979939816, + "loss": 3.0575, + "theoretical_loss": 3.916695094873718, + "tokens_seen": 497826816 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004288565697091274, + "loss": 2.9982, + "theoretical_loss": 3.9166395216142424, + "tokens_seen": 497892352 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004288465396188566, + "loss": 3.0071, + "theoretical_loss": 3.9165839577170924, + "tokens_seen": 497957888 + }, + { + "epoch": 1.06, + "learning_rate": 0.00042883650952858576, + "loss": 2.8608, + "theoretical_loss": 3.916528403179459, + "tokens_seen": 498023424 + }, + { + "epoch": 1.06, + "objective/train/docs_used": 593907, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9000375270843506, + "objective/train/theoretical_loss": 3.916486743416727, + "objective/train/tokens_used": 518532576, + "theoretical_loss": 3.916486743416727, + "tokens_seen": 498072576 + }, + { + "epoch": 1.06, + "learning_rate": 0.00042882647943831494, + "loss": 2.9289, + "theoretical_loss": 3.9164728579985333, + "tokens_seen": 498088960 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004288164493480441, + "loss": 2.7877, + "theoretical_loss": 3.91641732217151, + "tokens_seen": 498154496 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004288064192577733, + "loss": 2.9093, + "theoretical_loss": 3.9163617956955825, + "tokens_seen": 498220032 + }, + { + "epoch": 1.06, + "learning_rate": 0.00042879638916750254, + "loss": 2.9958, + "theoretical_loss": 3.916306278567947, + "tokens_seen": 498285568 + }, + { + "epoch": 1.06, + "learning_rate": 0.00042878635907723166, + "loss": 2.9177, + "theoretical_loss": 3.9162507707858003, + "tokens_seen": 498351104 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004287763289869609, + "loss": 2.8919, + "theoretical_loss": 3.916195272346341, + "tokens_seen": 498416640 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004287662988966901, + "loss": 2.8702, + "theoretical_loss": 3.916139783246768, + "tokens_seen": 498482176 + }, + { + "epoch": 1.06, + "learning_rate": 0.00042875626880641926, + "loss": 2.7663, + "theoretical_loss": 3.916084303484282, + "tokens_seen": 498547712 + }, + { + "epoch": 1.06, + "learning_rate": 0.00042874623871614844, + "loss": 2.8986, + "theoretical_loss": 3.9160288330560844, + "tokens_seen": 498613248 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004287362086258776, + "loss": 2.9018, + "theoretical_loss": 3.9159733719593786, + "tokens_seen": 498678784 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004287261785356068, + "loss": 2.907, + "theoretical_loss": 3.9159179201913688, + "tokens_seen": 498744320 + }, + { + "epoch": 1.06, + "learning_rate": 0.00042871614844533604, + "loss": 2.955, + "theoretical_loss": 3.9158624777492603, + "tokens_seen": 498809856 + }, + { + "epoch": 1.06, + "learning_rate": 0.00042870611835506517, + "loss": 3.0871, + "theoretical_loss": 3.9158070446302595, + "tokens_seen": 498875392 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004286960882647944, + "loss": 2.8137, + "theoretical_loss": 3.915751620831575, + "tokens_seen": 498940928 + }, + { + "epoch": 1.06, + "learning_rate": 0.00042868605817452353, + "loss": 2.955, + "theoretical_loss": 3.9156962063504146, + "tokens_seen": 499006464 + }, + { + "epoch": 1.06, + "learning_rate": 0.00042867602808425276, + "loss": 2.7694, + "theoretical_loss": 3.915640801183989, + "tokens_seen": 499072000 + }, + { + "epoch": 1.06, + "learning_rate": 0.000428665997993982, + "loss": 3.0629, + "theoretical_loss": 3.9155854053295105, + "tokens_seen": 499137536 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004286559679037111, + "loss": 2.9166, + "theoretical_loss": 3.91553001878419, + "tokens_seen": 499203072 + }, + { + "epoch": 1.06, + "learning_rate": 0.00042864593781344036, + "loss": 2.9496, + "theoretical_loss": 3.915474641545243, + "tokens_seen": 499268608 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004286359077231695, + "loss": 2.9411, + "theoretical_loss": 3.915419273609883, + "tokens_seen": 499334144 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004286258776328987, + "loss": 2.7409, + "theoretical_loss": 3.9153639149753277, + "tokens_seen": 499399680 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004286158475426279, + "loss": 2.7517, + "theoretical_loss": 3.915308565638793, + "tokens_seen": 499465216 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004286058174523571, + "loss": 2.8606, + "theoretical_loss": 3.9152532255974983, + "tokens_seen": 499530752 + }, + { + "epoch": 1.06, + "learning_rate": 0.00042859578736208627, + "loss": 3.058, + "theoretical_loss": 3.9151978948486637, + "tokens_seen": 499596288 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004285857572718155, + "loss": 3.086, + "theoretical_loss": 3.9151425733895096, + "tokens_seen": 499661824 + }, + { + "epoch": 1.06, + "objective/train/docs_used": 595290, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.1176106929779053, + "objective/train/theoretical_loss": 3.915101088389826, + "objective/train/tokens_used": 520170976, + "theoretical_loss": 3.915101088389826, + "tokens_seen": 499710976 + }, + { + "epoch": 1.06, + "learning_rate": 0.00042857572718154463, + "loss": 3.1203, + "theoretical_loss": 3.9150872612172583, + "tokens_seen": 499727360 + }, + { + "epoch": 1.06, + "learning_rate": 0.00042856569709127387, + "loss": 2.8166, + "theoretical_loss": 3.9150319583291333, + "tokens_seen": 499792896 + }, + { + "epoch": 1.06, + "learning_rate": 0.000428555667001003, + "loss": 3.111, + "theoretical_loss": 3.914976664722359, + "tokens_seen": 499858432 + }, + { + "epoch": 1.06, + "learning_rate": 0.00042854563691073223, + "loss": 3.1595, + "theoretical_loss": 3.914921380394162, + "tokens_seen": 499923968 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004285356068204614, + "loss": 2.9206, + "theoretical_loss": 3.914866105341768, + "tokens_seen": 499989504 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004285255767301906, + "loss": 2.7353, + "theoretical_loss": 3.914810839562406, + "tokens_seen": 500055040 + }, + { + "epoch": 1.06, + "learning_rate": 0.00042851554663991977, + "loss": 3.1139, + "theoretical_loss": 3.9147555830533047, + "tokens_seen": 500120576 + }, + { + "epoch": 1.06, + "learning_rate": 0.00042850551654964895, + "loss": 2.973, + "theoretical_loss": 3.9147003358116956, + "tokens_seen": 500186112 + }, + { + "epoch": 1.06, + "learning_rate": 0.00042849548645937813, + "loss": 2.8314, + "theoretical_loss": 3.9146450978348093, + "tokens_seen": 500251648 + }, + { + "epoch": 1.06, + "learning_rate": 0.00042848545636910737, + "loss": 2.9281, + "theoretical_loss": 3.9145898691198795, + "tokens_seen": 500317184 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004284754262788365, + "loss": 2.805, + "theoretical_loss": 3.91453464966414, + "tokens_seen": 500382720 + }, + { + "epoch": 1.06, + "learning_rate": 0.00042846539618856573, + "loss": 2.773, + "theoretical_loss": 3.9144794394648263, + "tokens_seen": 500448256 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004284553660982949, + "loss": 2.9605, + "theoretical_loss": 3.9144242385191745, + "tokens_seen": 500513792 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004284453360080241, + "loss": 2.9448, + "theoretical_loss": 3.9143690468244223, + "tokens_seen": 500579328 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004284353059177533, + "loss": 3.1425, + "theoretical_loss": 3.9143138643778093, + "tokens_seen": 500644864 + }, + { + "epoch": 1.06, + "learning_rate": 0.00042842527582748246, + "loss": 2.8884, + "theoretical_loss": 3.914258691176575, + "tokens_seen": 500710400 + }, + { + "epoch": 1.06, + "learning_rate": 0.00042841524573721164, + "loss": 3.1058, + "theoretical_loss": 3.9142035272179605, + "tokens_seen": 500775936 + }, + { + "epoch": 1.06, + "learning_rate": 0.00042840521564694087, + "loss": 2.7515, + "theoretical_loss": 3.9141483724992083, + "tokens_seen": 500841472 + }, + { + "epoch": 1.06, + "learning_rate": 0.00042839518555667, + "loss": 3.033, + "theoretical_loss": 3.914093227017562, + "tokens_seen": 500907008 + }, + { + "epoch": 1.06, + "learning_rate": 0.00042838515546639923, + "loss": 2.963, + "theoretical_loss": 3.9140380907702665, + "tokens_seen": 500972544 + }, + { + "epoch": 1.06, + "learning_rate": 0.00042837512537612836, + "loss": 3.0991, + "theoretical_loss": 3.9139829637545676, + "tokens_seen": 501038080 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004283650952858576, + "loss": 2.987, + "theoretical_loss": 3.913927845967712, + "tokens_seen": 501103616 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004283550651955868, + "loss": 3.0178, + "theoretical_loss": 3.9138727374069493, + "tokens_seen": 501169152 + }, + { + "epoch": 1.06, + "learning_rate": 0.00042834503510531596, + "loss": 3.0842, + "theoretical_loss": 3.9138176380695278, + "tokens_seen": 501234688 + }, + { + "epoch": 1.06, + "learning_rate": 0.00042833500501504514, + "loss": 3.0508, + "theoretical_loss": 3.9137625479526985, + "tokens_seen": 501300224 + }, + { + "epoch": 1.06, + "objective/train/docs_used": 595812, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.282951831817627, + "objective/train/theoretical_loss": 3.913721236414437, + "objective/train/tokens_used": 521809376, + "theoretical_loss": 3.913721236414437, + "tokens_seen": 501349376 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004283249749247743, + "loss": 3.1562, + "theoretical_loss": 3.9137074670537135, + "tokens_seen": 501365760 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004283149448345035, + "loss": 3.1112, + "theoretical_loss": 3.9136523953698257, + "tokens_seen": 501431296 + }, + { + "epoch": 1.06, + "learning_rate": 0.00042830491474423274, + "loss": 2.9509, + "theoretical_loss": 3.913597332898289, + "tokens_seen": 501496832 + }, + { + "epoch": 1.06, + "learning_rate": 0.00042829488465396186, + "loss": 2.7248, + "theoretical_loss": 3.9135422796363595, + "tokens_seen": 501562368 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004282848545636911, + "loss": 3.0696, + "theoretical_loss": 3.913487235581293, + "tokens_seen": 501627904 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004282748244734203, + "loss": 2.7844, + "theoretical_loss": 3.913432200730348, + "tokens_seen": 501693440 + }, + { + "epoch": 1.06, + "learning_rate": 0.00042826479438314946, + "loss": 2.8522, + "theoretical_loss": 3.913377175080783, + "tokens_seen": 501758976 + }, + { + "epoch": 1.06, + "learning_rate": 0.00042825476429287864, + "loss": 2.7476, + "theoretical_loss": 3.9133221586298577, + "tokens_seen": 501824512 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004282447342026078, + "loss": 2.8357, + "theoretical_loss": 3.913267151374834, + "tokens_seen": 501890048 + }, + { + "epoch": 1.06, + "learning_rate": 0.000428234704112337, + "loss": 2.9961, + "theoretical_loss": 3.913212153312974, + "tokens_seen": 501955584 + }, + { + "epoch": 1.06, + "learning_rate": 0.00042822467402206624, + "loss": 3.0573, + "theoretical_loss": 3.9131571644415413, + "tokens_seen": 502021120 + }, + { + "epoch": 1.06, + "learning_rate": 0.00042821464393179537, + "loss": 2.9952, + "theoretical_loss": 3.913102184757801, + "tokens_seen": 502086656 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004282046138415246, + "loss": 2.7377, + "theoretical_loss": 3.9130472142590187, + "tokens_seen": 502152192 + }, + { + "epoch": 1.06, + "learning_rate": 0.00042819458375125373, + "loss": 2.9165, + "theoretical_loss": 3.9129922529424617, + "tokens_seen": 502217728 + }, + { + "epoch": 1.06, + "learning_rate": 0.00042818455366098296, + "loss": 2.8041, + "theoretical_loss": 3.9129373008053983, + "tokens_seen": 502283264 + }, + { + "epoch": 1.06, + "learning_rate": 0.00042817452357071215, + "loss": 2.9187, + "theoretical_loss": 3.9128823578450977, + "tokens_seen": 502348800 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004281644934804413, + "loss": 2.8599, + "theoretical_loss": 3.9128274240588308, + "tokens_seen": 502414336 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004281544633901705, + "loss": 3.1846, + "theoretical_loss": 3.9127724994438693, + "tokens_seen": 502479872 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004281444332998997, + "loss": 2.9606, + "theoretical_loss": 3.9127175839974866, + "tokens_seen": 502545408 + }, + { + "epoch": 1.06, + "learning_rate": 0.00042813440320962887, + "loss": 2.8654, + "theoretical_loss": 3.912662677716956, + "tokens_seen": 502610944 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004281243731193581, + "loss": 3.0389, + "theoretical_loss": 3.9126077805995534, + "tokens_seen": 502676480 + }, + { + "epoch": 1.06, + "learning_rate": 0.00042811434302908723, + "loss": 3.0599, + "theoretical_loss": 3.912552892642555, + "tokens_seen": 502742016 + }, + { + "epoch": 1.06, + "learning_rate": 0.00042810431293881647, + "loss": 2.8393, + "theoretical_loss": 3.912498013843238, + "tokens_seen": 502807552 + }, + { + "epoch": 1.06, + "learning_rate": 0.00042809428284854565, + "loss": 2.9673, + "theoretical_loss": 3.9124431441988823, + "tokens_seen": 502873088 + }, + { + "epoch": 1.06, + "learning_rate": 0.00042808425275827483, + "loss": 2.9235, + "theoretical_loss": 3.9123882837067674, + "tokens_seen": 502938624 + }, + { + "epoch": 1.06, + "objective/train/docs_used": 597072, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4846396446228027, + "objective/train/theoretical_loss": 3.9123471443422035, + "objective/train/tokens_used": 523447776, + "theoretical_loss": 3.9123471443422035, + "tokens_seen": 502987776 + }, + { + "epoch": 1.06, + "learning_rate": 0.000428074222668004, + "loss": 2.9092, + "theoretical_loss": 3.912333432364174, + "tokens_seen": 503004160 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004280641925777332, + "loss": 2.7859, + "theoretical_loss": 3.912278590168385, + "tokens_seen": 503069696 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004280541624874624, + "loss": 2.85, + "theoretical_loss": 3.9122237571166827, + "tokens_seen": 503135232 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004280441323971916, + "loss": 2.9421, + "theoretical_loss": 3.912168933206353, + "tokens_seen": 503200768 + }, + { + "epoch": 1.06, + "learning_rate": 0.00042803410230692074, + "loss": 3.1732, + "theoretical_loss": 3.9121141184346806, + "tokens_seen": 503266304 + }, + { + "epoch": 1.06, + "learning_rate": 0.00042802407221664997, + "loss": 3.1852, + "theoretical_loss": 3.912059312798953, + "tokens_seen": 503331840 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004280140421263791, + "loss": 3.0069, + "theoretical_loss": 3.9120045162964594, + "tokens_seen": 503397376 + }, + { + "epoch": 1.06, + "learning_rate": 0.00042800401203610833, + "loss": 3.0059, + "theoretical_loss": 3.911949728924487, + "tokens_seen": 503462912 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004279939819458375, + "loss": 3.1234, + "theoretical_loss": 3.9118949506803267, + "tokens_seen": 503528448 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004279839518555667, + "loss": 2.9553, + "theoretical_loss": 3.9118401815612707, + "tokens_seen": 503593984 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004279739217652959, + "loss": 2.8518, + "theoretical_loss": 3.9117854215646117, + "tokens_seen": 503659520 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004279638916750251, + "loss": 3.1847, + "theoretical_loss": 3.9117306706876427, + "tokens_seen": 503725056 + }, + { + "epoch": 1.06, + "learning_rate": 0.00042795386158475424, + "loss": 2.9503, + "theoretical_loss": 3.9116759289276595, + "tokens_seen": 503790592 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004279438314944835, + "loss": 2.8428, + "theoretical_loss": 3.911621196281958, + "tokens_seen": 503856128 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004279338014042126, + "loss": 2.9084, + "theoretical_loss": 3.9115664727478356, + "tokens_seen": 503921664 + }, + { + "epoch": 1.06, + "learning_rate": 0.00042792377131394184, + "loss": 3.0378, + "theoretical_loss": 3.9115117583225905, + "tokens_seen": 503987200 + }, + { + "epoch": 1.06, + "learning_rate": 0.00042791374122367107, + "loss": 3.0881, + "theoretical_loss": 3.911457053003523, + "tokens_seen": 504052736 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004279037111334002, + "loss": 2.8481, + "theoretical_loss": 3.9114023567879332, + "tokens_seen": 504118272 + }, + { + "epoch": 1.06, + "learning_rate": 0.00042789368104312943, + "loss": 2.9654, + "theoretical_loss": 3.9113476696731233, + "tokens_seen": 504183808 + }, + { + "epoch": 1.06, + "learning_rate": 0.00042788365095285856, + "loss": 3.1006, + "theoretical_loss": 3.911292991656396, + "tokens_seen": 504249344 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004278736208625878, + "loss": 3.0438, + "theoretical_loss": 3.9112383227350564, + "tokens_seen": 504314880 + }, + { + "epoch": 1.06, + "learning_rate": 0.000427863590772317, + "loss": 3.0496, + "theoretical_loss": 3.911183662906409, + "tokens_seen": 504380416 + }, + { + "epoch": 1.06, + "learning_rate": 0.00042785356068204616, + "loss": 2.8254, + "theoretical_loss": 3.911129012167761, + "tokens_seen": 504445952 + }, + { + "epoch": 1.06, + "learning_rate": 0.00042784353059177534, + "loss": 2.6538, + "theoretical_loss": 3.911074370516419, + "tokens_seen": 504511488 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004278335005015045, + "loss": 2.9649, + "theoretical_loss": 3.911019737949693, + "tokens_seen": 504577024 + }, + { + "epoch": 1.06, + "objective/train/docs_used": 597784, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.1036291122436523, + "objective/train/theoretical_loss": 3.9109787694848093, + "objective/train/tokens_used": 525086176, + "theoretical_loss": 3.9109787694848093, + "tokens_seen": 504626176 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004278234704112337, + "loss": 3.0501, + "theoretical_loss": 3.9109651144648927, + "tokens_seen": 504642560 + }, + { + "epoch": 1.06, + "learning_rate": 0.00042781344032096294, + "loss": 3.2442, + "theoretical_loss": 3.910910500059329, + "tokens_seen": 504708096 + }, + { + "epoch": 1.06, + "learning_rate": 0.00042780341023069206, + "loss": 3.1424, + "theoretical_loss": 3.910855894730314, + "tokens_seen": 504773632 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004277933801404213, + "loss": 2.6353, + "theoretical_loss": 3.9108012984751612, + "tokens_seen": 504839168 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004277833500501505, + "loss": 3.0854, + "theoretical_loss": 3.9107467112911856, + "tokens_seen": 504904704 + }, + { + "epoch": 1.06, + "learning_rate": 0.00042777331995987966, + "loss": 3.162, + "theoretical_loss": 3.9106921331757016, + "tokens_seen": 504970240 + }, + { + "epoch": 1.06, + "learning_rate": 0.00042776328986960884, + "loss": 3.0313, + "theoretical_loss": 3.910637564126028, + "tokens_seen": 505035776 + }, + { + "epoch": 1.06, + "learning_rate": 0.000427753259779338, + "loss": 2.8933, + "theoretical_loss": 3.9105830041394816, + "tokens_seen": 505101312 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004277432296890672, + "loss": 2.9269, + "theoretical_loss": 3.910528453213381, + "tokens_seen": 505166848 + }, + { + "epoch": 1.06, + "learning_rate": 0.00042773319959879644, + "loss": 2.9692, + "theoretical_loss": 3.910473911345048, + "tokens_seen": 505232384 + }, + { + "epoch": 1.06, + "learning_rate": 0.00042772316950852557, + "loss": 2.987, + "theoretical_loss": 3.9104193785318024, + "tokens_seen": 505297920 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004277131394182548, + "loss": 2.7983, + "theoretical_loss": 3.9103648547709673, + "tokens_seen": 505363456 + }, + { + "epoch": 1.06, + "learning_rate": 0.00042770310932798393, + "loss": 2.9446, + "theoretical_loss": 3.910310340059867, + "tokens_seen": 505428992 + }, + { + "epoch": 1.06, + "learning_rate": 0.00042769307923771316, + "loss": 2.9765, + "theoretical_loss": 3.9102558343958256, + "tokens_seen": 505494528 + }, + { + "epoch": 1.06, + "learning_rate": 0.00042768304914744235, + "loss": 2.9479, + "theoretical_loss": 3.9102013377761695, + "tokens_seen": 505560064 + }, + { + "epoch": 1.06, + "learning_rate": 0.00042767301905717153, + "loss": 2.8593, + "theoretical_loss": 3.910146850198225, + "tokens_seen": 505625600 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004276629889669007, + "loss": 3.1793, + "theoretical_loss": 3.910092371659321, + "tokens_seen": 505691136 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004276529588766299, + "loss": 3.1078, + "theoretical_loss": 3.9100379021567875, + "tokens_seen": 505756672 + }, + { + "epoch": 1.06, + "learning_rate": 0.00042764292878635907, + "loss": 2.8893, + "theoretical_loss": 3.9099834416879533, + "tokens_seen": 505822208 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004276328986960883, + "loss": 3.161, + "theoretical_loss": 3.909928990250151, + "tokens_seen": 505887744 + }, + { + "epoch": 1.06, + "learning_rate": 0.00042762286860581743, + "loss": 2.8981, + "theoretical_loss": 3.9098745478407135, + "tokens_seen": 505953280 + }, + { + "epoch": 1.06, + "learning_rate": 0.00042761283851554667, + "loss": 3.054, + "theoretical_loss": 3.909820114456975, + "tokens_seen": 506018816 + }, + { + "epoch": 1.06, + "learning_rate": 0.00042760280842527585, + "loss": 2.8977, + "theoretical_loss": 3.909765690096269, + "tokens_seen": 506084352 + }, + { + "epoch": 1.06, + "learning_rate": 0.00042759277833500503, + "loss": 2.7607, + "theoretical_loss": 3.9097112747559337, + "tokens_seen": 506149888 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004275827482447342, + "loss": 2.8732, + "theoretical_loss": 3.909656868433305, + "tokens_seen": 506215424 + }, + { + "epoch": 1.06, + "objective/train/docs_used": 599198, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.1902496814727783, + "objective/train/theoretical_loss": 3.9096160696076026, + "objective/train/tokens_used": 526724576, + "theoretical_loss": 3.9096160696076026, + "tokens_seen": 506264576 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004275727181544634, + "loss": 2.8656, + "theoretical_loss": 3.9096024711257216, + "tokens_seen": 506280960 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004275626880641926, + "loss": 2.7504, + "theoretical_loss": 3.909548082830523, + "tokens_seen": 506346496 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004275526579739218, + "loss": 3.0306, + "theoretical_loss": 3.90949370354505, + "tokens_seen": 506412032 + }, + { + "epoch": 1.06, + "learning_rate": 0.00042754262788365094, + "loss": 3.0496, + "theoretical_loss": 3.909439333266645, + "tokens_seen": 506477568 + }, + { + "epoch": 1.06, + "learning_rate": 0.00042753259779338017, + "loss": 2.7544, + "theoretical_loss": 3.90938497199265, + "tokens_seen": 506543104 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004275225677031093, + "loss": 2.8873, + "theoretical_loss": 3.909330619720409, + "tokens_seen": 506608640 + }, + { + "epoch": 1.06, + "learning_rate": 0.00042751253761283853, + "loss": 3.0007, + "theoretical_loss": 3.909276276447268, + "tokens_seen": 506674176 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004275025075225677, + "loss": 2.8218, + "theoretical_loss": 3.909221942170573, + "tokens_seen": 506739712 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004274924774322969, + "loss": 2.8335, + "theoretical_loss": 3.9091676168876717, + "tokens_seen": 506805248 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004274824473420261, + "loss": 2.826, + "theoretical_loss": 3.9091133005959118, + "tokens_seen": 506870784 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004274724172517553, + "loss": 2.9372, + "theoretical_loss": 3.9090589932926436, + "tokens_seen": 506936320 + }, + { + "epoch": 1.06, + "learning_rate": 0.00042746238716148444, + "loss": 2.9096, + "theoretical_loss": 3.909004694975218, + "tokens_seen": 507001856 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004274523570712137, + "loss": 2.9741, + "theoretical_loss": 3.908950405640987, + "tokens_seen": 507067392 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004274423269809428, + "loss": 2.9571, + "theoretical_loss": 3.908896125287303, + "tokens_seen": 507132928 + }, + { + "epoch": 1.06, + "learning_rate": 0.00042743229689067204, + "loss": 2.7355, + "theoretical_loss": 3.908841853911521, + "tokens_seen": 507198464 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004274222668004012, + "loss": 3.0665, + "theoretical_loss": 3.9087875915109955, + "tokens_seen": 507264000 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004274122367101304, + "loss": 2.9467, + "theoretical_loss": 3.908733338083084, + "tokens_seen": 507329536 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004274022066198596, + "loss": 2.7414, + "theoretical_loss": 3.9086790936251425, + "tokens_seen": 507395072 + }, + { + "epoch": 1.06, + "learning_rate": 0.00042739217652958876, + "loss": 2.9773, + "theoretical_loss": 3.908624858134531, + "tokens_seen": 507460608 + }, + { + "epoch": 1.06, + "learning_rate": 0.00042738214643931794, + "loss": 2.893, + "theoretical_loss": 3.908570631608609, + "tokens_seen": 507526144 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004273721163490472, + "loss": 2.8147, + "theoretical_loss": 3.9085164140447377, + "tokens_seen": 507591680 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004273620862587763, + "loss": 2.8449, + "theoretical_loss": 3.908462205440278, + "tokens_seen": 507657216 + }, + { + "epoch": 1.06, + "learning_rate": 0.00042735205616850554, + "loss": 2.9502, + "theoretical_loss": 3.9084080057925936, + "tokens_seen": 507722752 + }, + { + "epoch": 1.06, + "learning_rate": 0.00042734202607823467, + "loss": 2.9411, + "theoretical_loss": 3.908353815099049, + "tokens_seen": 507788288 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004273319959879639, + "loss": 2.8535, + "theoretical_loss": 3.90829963335701, + "tokens_seen": 507853824 + }, + { + "epoch": 1.06, + "objective/train/docs_used": 599925, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.022632360458374, + "objective/train/theoretical_loss": 3.9082590029233213, + "objective/train/tokens_used": 528362976, + "theoretical_loss": 3.9082590029233213, + "tokens_seen": 507902976 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004273219658976931, + "loss": 3.1551, + "theoretical_loss": 3.908245460563842, + "tokens_seen": 507919360 + }, + { + "epoch": 1.06, + "learning_rate": 0.00042731193580742226, + "loss": 2.7927, + "theoretical_loss": 3.908191296716913, + "tokens_seen": 507984896 + }, + { + "epoch": 1.06, + "learning_rate": 0.00042730190571715145, + "loss": 2.7956, + "theoretical_loss": 3.908137141813592, + "tokens_seen": 508050432 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004272918756268807, + "loss": 2.7378, + "theoretical_loss": 3.9080829958512484, + "tokens_seen": 508115968 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004272818455366098, + "loss": 2.8576, + "theoretical_loss": 3.9080288588272536, + "tokens_seen": 508181504 + }, + { + "epoch": 1.06, + "learning_rate": 0.00042727181544633904, + "loss": 2.6358, + "theoretical_loss": 3.90797473073898, + "tokens_seen": 508247040 + }, + { + "epoch": 1.06, + "learning_rate": 0.00042726178535606817, + "loss": 2.9043, + "theoretical_loss": 3.9079206115837994, + "tokens_seen": 508312576 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004272517552657974, + "loss": 3.0203, + "theoretical_loss": 3.907866501359087, + "tokens_seen": 508378112 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004272417251755266, + "loss": 2.8236, + "theoretical_loss": 3.907812400062218, + "tokens_seen": 508443648 + }, + { + "epoch": 1.06, + "learning_rate": 0.00042723169508525577, + "loss": 2.8062, + "theoretical_loss": 3.9077583076905684, + "tokens_seen": 508509184 + }, + { + "epoch": 1.06, + "learning_rate": 0.00042722166499498495, + "loss": 2.9367, + "theoretical_loss": 3.907704224241517, + "tokens_seen": 508574720 + }, + { + "epoch": 1.06, + "learning_rate": 0.00042721163490471413, + "loss": 2.758, + "theoretical_loss": 3.907650149712442, + "tokens_seen": 508640256 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004272016048144433, + "loss": 2.8615, + "theoretical_loss": 3.9075960841007227, + "tokens_seen": 508705792 + }, + { + "epoch": 1.06, + "learning_rate": 0.00042719157472417255, + "loss": 2.8553, + "theoretical_loss": 3.9075420274037405, + "tokens_seen": 508771328 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004271815446339017, + "loss": 2.7945, + "theoretical_loss": 3.9074879796188773, + "tokens_seen": 508836864 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004271715145436309, + "loss": 2.718, + "theoretical_loss": 3.907433940743516, + "tokens_seen": 508902400 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004271614844533601, + "loss": 3.0734, + "theoretical_loss": 3.9073799107750413, + "tokens_seen": 508967936 + }, + { + "epoch": 1.06, + "learning_rate": 0.00042715145436308927, + "loss": 2.8651, + "theoretical_loss": 3.9073258897108385, + "tokens_seen": 509033472 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004271414242728185, + "loss": 2.684, + "theoretical_loss": 3.9072718775482933, + "tokens_seen": 509099008 + }, + { + "epoch": 1.06, + "learning_rate": 0.00042713139418254763, + "loss": 2.9408, + "theoretical_loss": 3.907217874284794, + "tokens_seen": 509164544 + }, + { + "epoch": 1.06, + "learning_rate": 0.00042712136409227687, + "loss": 2.7014, + "theoretical_loss": 3.9071638799177295, + "tokens_seen": 509230080 + }, + { + "epoch": 1.06, + "learning_rate": 0.00042711133400200605, + "loss": 3.0804, + "theoretical_loss": 3.907109894444489, + "tokens_seen": 509295616 + }, + { + "epoch": 1.06, + "learning_rate": 0.00042710130391173523, + "loss": 2.8745, + "theoretical_loss": 3.907055917862463, + "tokens_seen": 509361152 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004270912738214644, + "loss": 3.088, + "theoretical_loss": 3.9070019501690445, + "tokens_seen": 509426688 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004270812437311936, + "loss": 2.8545, + "theoretical_loss": 3.9069479913616254, + "tokens_seen": 509492224 + }, + { + "epoch": 1.06, + "objective/train/docs_used": 601205, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.895568370819092, + "objective/train/theoretical_loss": 3.906907528085932, + "objective/train/tokens_used": 530001376, + "theoretical_loss": 3.906907528085932, + "tokens_seen": 509541376 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004270712136409228, + "loss": 2.8726, + "theoretical_loss": 3.906894041437601, + "tokens_seen": 509557760 + }, + { + "epoch": 1.06, + "learning_rate": 0.000427061183550652, + "loss": 2.6773, + "theoretical_loss": 3.906840100394366, + "tokens_seen": 509623296 + }, + { + "epoch": 1.06, + "learning_rate": 0.00042705115346038114, + "loss": 2.8422, + "theoretical_loss": 3.9067861682293166, + "tokens_seen": 509688832 + }, + { + "epoch": 1.06, + "learning_rate": 0.00042704112337011037, + "loss": 2.8928, + "theoretical_loss": 3.9067322449398505, + "tokens_seen": 509754368 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004270310932798395, + "loss": 3.308, + "theoretical_loss": 3.9066783305233663, + "tokens_seen": 509819904 + }, + { + "epoch": 1.06, + "learning_rate": 0.00042702106318956873, + "loss": 3.0129, + "theoretical_loss": 3.906624424977263, + "tokens_seen": 509885440 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004270110330992979, + "loss": 3.0955, + "theoretical_loss": 3.9065705282989427, + "tokens_seen": 509950976 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004270010030090271, + "loss": 3.2208, + "theoretical_loss": 3.9065166404858056, + "tokens_seen": 510016512 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004269909729187563, + "loss": 2.9995, + "theoretical_loss": 3.906462761535256, + "tokens_seen": 510082048 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004269809428284855, + "loss": 2.6179, + "theoretical_loss": 3.9064088914446975, + "tokens_seen": 510147584 + }, + { + "epoch": 1.06, + "learning_rate": 0.00042697091273821464, + "loss": 2.9016, + "theoretical_loss": 3.9063550302115346, + "tokens_seen": 510213120 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004269608826479439, + "loss": 2.7884, + "theoretical_loss": 3.9063011778331744, + "tokens_seen": 510278656 + }, + { + "epoch": 1.06, + "learning_rate": 0.000426950852557673, + "loss": 3.0448, + "theoretical_loss": 3.9062473343070234, + "tokens_seen": 510344192 + }, + { + "epoch": 1.06, + "learning_rate": 0.00042694082246740224, + "loss": 3.076, + "theoretical_loss": 3.9061934996304912, + "tokens_seen": 510409728 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004269307923771314, + "loss": 2.9906, + "theoretical_loss": 3.906139673800986, + "tokens_seen": 510475264 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004269207622868606, + "loss": 3.0174, + "theoretical_loss": 3.9060858568159196, + "tokens_seen": 510540800 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004269107321965898, + "loss": 2.8787, + "theoretical_loss": 3.906032048672702, + "tokens_seen": 510606336 + }, + { + "epoch": 1.06, + "learning_rate": 0.00042690070210631896, + "loss": 2.8605, + "theoretical_loss": 3.9059782493687476, + "tokens_seen": 510671872 + }, + { + "epoch": 1.06, + "learning_rate": 0.00042689067201604814, + "loss": 3.0283, + "theoretical_loss": 3.9059244589014694, + "tokens_seen": 510737408 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004268806419257774, + "loss": 2.945, + "theoretical_loss": 3.9058706772682825, + "tokens_seen": 510802944 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004268706118355065, + "loss": 3.2505, + "theoretical_loss": 3.905816904466603, + "tokens_seen": 510868480 + }, + { + "epoch": 1.06, + "learning_rate": 0.00042686058174523574, + "loss": 3.0026, + "theoretical_loss": 3.905763140493848, + "tokens_seen": 510934016 + }, + { + "epoch": 1.06, + "learning_rate": 0.00042685055165496487, + "loss": 2.9411, + "theoretical_loss": 3.905709385347436, + "tokens_seen": 510999552 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004268405215646941, + "loss": 3.2739, + "theoretical_loss": 3.9056556390247863, + "tokens_seen": 511065088 + }, + { + "epoch": 1.06, + "learning_rate": 0.0004268304914744233, + "loss": 3.2656, + "theoretical_loss": 3.905601901523318, + "tokens_seen": 511130624 + }, + { + "epoch": 1.06, + "objective/train/docs_used": 601687, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9587132930755615, + "objective/train/theoretical_loss": 3.905561604184567, + "objective/train/tokens_used": 531639776, + "theoretical_loss": 3.905561604184567, + "tokens_seen": 511179776 + }, + { + "epoch": 1.06, + "learning_rate": 0.00042682046138415246, + "loss": 3.1384, + "theoretical_loss": 3.9055481728404544, + "tokens_seen": 511196160 + }, + { + "epoch": 1.06, + "learning_rate": 0.00042681043129388165, + "loss": 3.1338, + "theoretical_loss": 3.905494452973617, + "tokens_seen": 511261696 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004268004012036109, + "loss": 2.854, + "theoretical_loss": 3.905440741920229, + "tokens_seen": 511327232 + }, + { + "epoch": 1.07, + "learning_rate": 0.00042679037111334, + "loss": 2.9444, + "theoretical_loss": 3.9053870396777164, + "tokens_seen": 511392768 + }, + { + "epoch": 1.07, + "learning_rate": 0.00042678034102306924, + "loss": 2.9435, + "theoretical_loss": 3.905333346243504, + "tokens_seen": 511458304 + }, + { + "epoch": 1.07, + "learning_rate": 0.00042677031093279837, + "loss": 3.0561, + "theoretical_loss": 3.905279661615019, + "tokens_seen": 511523840 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004267602808425276, + "loss": 2.9755, + "theoretical_loss": 3.9052259857896887, + "tokens_seen": 511589376 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004267502507522568, + "loss": 2.9857, + "theoretical_loss": 3.9051723187649436, + "tokens_seen": 511654912 + }, + { + "epoch": 1.07, + "learning_rate": 0.00042674022066198597, + "loss": 3.0649, + "theoretical_loss": 3.905118660538213, + "tokens_seen": 511720448 + }, + { + "epoch": 1.07, + "learning_rate": 0.00042673019057171515, + "loss": 2.9349, + "theoretical_loss": 3.905065011106928, + "tokens_seen": 511785984 + }, + { + "epoch": 1.07, + "learning_rate": 0.00042672016048144433, + "loss": 3.0933, + "theoretical_loss": 3.9050113704685208, + "tokens_seen": 511851520 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004267101303911735, + "loss": 3.0626, + "theoretical_loss": 3.904957738620425, + "tokens_seen": 511917056 + }, + { + "epoch": 1.07, + "learning_rate": 0.00042670010030090275, + "loss": 3.0902, + "theoretical_loss": 3.9049041155600746, + "tokens_seen": 511982592 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004266900702106319, + "loss": 2.8052, + "theoretical_loss": 3.9048505012849053, + "tokens_seen": 512048128 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004266800401203611, + "loss": 2.9866, + "theoretical_loss": 3.9047968957923542, + "tokens_seen": 512113664 + }, + { + "epoch": 1.07, + "learning_rate": 0.00042667001003009024, + "loss": 3.2107, + "theoretical_loss": 3.9047432990798585, + "tokens_seen": 512179200 + }, + { + "epoch": 1.07, + "learning_rate": 0.00042665997993981947, + "loss": 2.9144, + "theoretical_loss": 3.9046897111448575, + "tokens_seen": 512244736 + }, + { + "epoch": 1.07, + "learning_rate": 0.00042664994984954865, + "loss": 2.8282, + "theoretical_loss": 3.90463613198479, + "tokens_seen": 512310272 + }, + { + "epoch": 1.07, + "learning_rate": 0.00042663991975927783, + "loss": 3.0063, + "theoretical_loss": 3.9045825615970973, + "tokens_seen": 512375808 + }, + { + "epoch": 1.07, + "learning_rate": 0.000426629889669007, + "loss": 2.9466, + "theoretical_loss": 3.904528999979222, + "tokens_seen": 512441344 + }, + { + "epoch": 1.07, + "learning_rate": 0.00042661985957873625, + "loss": 2.9841, + "theoretical_loss": 3.9044754471286063, + "tokens_seen": 512506880 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004266098294884654, + "loss": 2.7521, + "theoretical_loss": 3.9044219030426945, + "tokens_seen": 512572416 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004265997993981946, + "loss": 2.8133, + "theoretical_loss": 3.904368367718932, + "tokens_seen": 512637952 + }, + { + "epoch": 1.07, + "learning_rate": 0.00042658976930792374, + "loss": 2.9629, + "theoretical_loss": 3.904314841154765, + "tokens_seen": 512703488 + }, + { + "epoch": 1.07, + "learning_rate": 0.000426579739217653, + "loss": 3.0293, + "theoretical_loss": 3.9042613233476406, + "tokens_seen": 512769024 + }, + { + "epoch": 1.07, + "objective/train/docs_used": 602985, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.070340633392334, + "objective/train/theoretical_loss": 3.904221190737572, + "objective/train/tokens_used": 533278176, + "theoretical_loss": 3.904221190737572, + "tokens_seen": 512818176 + }, + { + "epoch": 1.07, + "learning_rate": 0.00042656970912738216, + "loss": 2.8697, + "theoretical_loss": 3.9042078142950074, + "tokens_seen": 512834560 + }, + { + "epoch": 1.07, + "learning_rate": 0.00042655967903711134, + "loss": 2.8872, + "theoretical_loss": 3.9041543139943147, + "tokens_seen": 512900096 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004265496489468405, + "loss": 3.0673, + "theoretical_loss": 3.9041008224430134, + "tokens_seen": 512965632 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004265396188565697, + "loss": 2.8721, + "theoretical_loss": 3.9040473396385544, + "tokens_seen": 513031168 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004265295887662989, + "loss": 2.8403, + "theoretical_loss": 3.903993865578391, + "tokens_seen": 513096704 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004265195586760281, + "loss": 2.8301, + "theoretical_loss": 3.9039404002599767, + "tokens_seen": 513162240 + }, + { + "epoch": 1.07, + "learning_rate": 0.00042650952858575724, + "loss": 2.9424, + "theoretical_loss": 3.903886943680766, + "tokens_seen": 513227776 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004264994984954865, + "loss": 3.0853, + "theoretical_loss": 3.903833495838215, + "tokens_seen": 513293312 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004264894684052156, + "loss": 3.214, + "theoretical_loss": 3.903780056729781, + "tokens_seen": 513358848 + }, + { + "epoch": 1.07, + "learning_rate": 0.00042647943831494484, + "loss": 3.1121, + "theoretical_loss": 3.9037266263529213, + "tokens_seen": 513424384 + }, + { + "epoch": 1.07, + "learning_rate": 0.000426469408224674, + "loss": 3.105, + "theoretical_loss": 3.903673204705096, + "tokens_seen": 513489920 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004264593781344032, + "loss": 2.6414, + "theoretical_loss": 3.903619791783764, + "tokens_seen": 513555456 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004264493480441324, + "loss": 3.0392, + "theoretical_loss": 3.903566387586387, + "tokens_seen": 513620992 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004264393179538616, + "loss": 2.8988, + "theoretical_loss": 3.9035129921104277, + "tokens_seen": 513686528 + }, + { + "epoch": 1.07, + "learning_rate": 0.00042642928786359075, + "loss": 2.9126, + "theoretical_loss": 3.9034596053533486, + "tokens_seen": 513752064 + }, + { + "epoch": 1.07, + "learning_rate": 0.00042641925777332, + "loss": 3.1472, + "theoretical_loss": 3.9034062273126144, + "tokens_seen": 513817600 + }, + { + "epoch": 1.07, + "learning_rate": 0.00042640922768304916, + "loss": 2.9302, + "theoretical_loss": 3.9033528579856904, + "tokens_seen": 513883136 + }, + { + "epoch": 1.07, + "learning_rate": 0.00042639919759277834, + "loss": 3.1095, + "theoretical_loss": 3.9032994973700434, + "tokens_seen": 513948672 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004263891675025076, + "loss": 3.0708, + "theoretical_loss": 3.9032461454631404, + "tokens_seen": 514014208 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004263791374122367, + "loss": 3.0222, + "theoretical_loss": 3.9031928022624514, + "tokens_seen": 514079744 + }, + { + "epoch": 1.07, + "learning_rate": 0.00042636910732196594, + "loss": 3.1103, + "theoretical_loss": 3.9031394677654445, + "tokens_seen": 514145280 + }, + { + "epoch": 1.07, + "learning_rate": 0.00042635907723169507, + "loss": 2.7347, + "theoretical_loss": 3.903086141969591, + "tokens_seen": 514210816 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004263490471414243, + "loss": 2.9696, + "theoretical_loss": 3.903032824872362, + "tokens_seen": 514276352 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004263390170511535, + "loss": 3.09, + "theoretical_loss": 3.902979516471232, + "tokens_seen": 514341888 + }, + { + "epoch": 1.07, + "learning_rate": 0.00042632898696088266, + "loss": 2.9927, + "theoretical_loss": 3.9029262167636736, + "tokens_seen": 514407424 + }, + { + "epoch": 1.07, + "objective/train/docs_used": 603651, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.3740158081054688, + "objective/train/theoretical_loss": 3.902886247686642, + "objective/train/tokens_used": 534916576, + "theoretical_loss": 3.902886247686642, + "tokens_seen": 514456576 + }, + { + "epoch": 1.07, + "learning_rate": 0.00042631895687061185, + "loss": 2.8779, + "theoretical_loss": 3.9028729257471615, + "tokens_seen": 514472960 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004263089267803411, + "loss": 3.0081, + "theoretical_loss": 3.902819643419173, + "tokens_seen": 514538496 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004262988966900702, + "loss": 3.0532, + "theoretical_loss": 3.9027663697771837, + "tokens_seen": 514604032 + }, + { + "epoch": 1.07, + "learning_rate": 0.00042628886659979944, + "loss": 3.1043, + "theoretical_loss": 3.9027131048186727, + "tokens_seen": 514669568 + }, + { + "epoch": 1.07, + "learning_rate": 0.00042627883650952857, + "loss": 2.8976, + "theoretical_loss": 3.902659848541119, + "tokens_seen": 514735104 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004262688064192578, + "loss": 3.0731, + "theoretical_loss": 3.9026066009420024, + "tokens_seen": 514800640 + }, + { + "epoch": 1.07, + "learning_rate": 0.000426258776328987, + "loss": 2.867, + "theoretical_loss": 3.902553362018805, + "tokens_seen": 514866176 + }, + { + "epoch": 1.07, + "learning_rate": 0.00042624874623871617, + "loss": 2.911, + "theoretical_loss": 3.902500131769008, + "tokens_seen": 514931712 + }, + { + "epoch": 1.07, + "learning_rate": 0.00042623871614844535, + "loss": 3.2478, + "theoretical_loss": 3.9024469101900956, + "tokens_seen": 514997248 + }, + { + "epoch": 1.07, + "learning_rate": 0.00042622868605817453, + "loss": 2.9697, + "theoretical_loss": 3.902393697279552, + "tokens_seen": 515062784 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004262186559679037, + "loss": 2.874, + "theoretical_loss": 3.9023404930348624, + "tokens_seen": 515128320 + }, + { + "epoch": 1.07, + "learning_rate": 0.00042620862587763295, + "loss": 2.9714, + "theoretical_loss": 3.9022872974535137, + "tokens_seen": 515193856 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004261985957873621, + "loss": 3.0392, + "theoretical_loss": 3.9022341105329934, + "tokens_seen": 515259392 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004261885656970913, + "loss": 2.9288, + "theoretical_loss": 3.90218093227079, + "tokens_seen": 515324928 + }, + { + "epoch": 1.07, + "learning_rate": 0.00042617853560682044, + "loss": 2.958, + "theoretical_loss": 3.902127762664393, + "tokens_seen": 515390464 + }, + { + "epoch": 1.07, + "learning_rate": 0.00042616850551654967, + "loss": 2.8225, + "theoretical_loss": 3.902074601711294, + "tokens_seen": 515456000 + }, + { + "epoch": 1.07, + "learning_rate": 0.00042615847542627885, + "loss": 2.7655, + "theoretical_loss": 3.9020214494089833, + "tokens_seen": 515521536 + }, + { + "epoch": 1.07, + "learning_rate": 0.00042614844533600803, + "loss": 2.8536, + "theoretical_loss": 3.901968305754955, + "tokens_seen": 515587072 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004261384152457372, + "loss": 2.8864, + "theoretical_loss": 3.901915170746702, + "tokens_seen": 515652608 + }, + { + "epoch": 1.07, + "learning_rate": 0.00042612838515546645, + "loss": 2.9468, + "theoretical_loss": 3.90186204438172, + "tokens_seen": 515718144 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004261183550651956, + "loss": 2.9706, + "theoretical_loss": 3.9018089266575045, + "tokens_seen": 515783680 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004261083249749248, + "loss": 2.9346, + "theoretical_loss": 3.9017558175715523, + "tokens_seen": 515849216 + }, + { + "epoch": 1.07, + "learning_rate": 0.00042609829488465394, + "loss": 2.8477, + "theoretical_loss": 3.9017027171213616, + "tokens_seen": 515914752 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004260882647943832, + "loss": 2.8322, + "theoretical_loss": 3.9016496253044317, + "tokens_seen": 515980288 + }, + { + "epoch": 1.07, + "learning_rate": 0.00042607823470411236, + "loss": 2.764, + "theoretical_loss": 3.901596542118263, + "tokens_seen": 516045824 + }, + { + "epoch": 1.07, + "objective/train/docs_used": 604730, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.402284622192383, + "objective/train/theoretical_loss": 3.901556735391069, + "objective/train/tokens_used": 536554976, + "theoretical_loss": 3.901556735391069, + "tokens_seen": 516094976 + }, + { + "epoch": 1.07, + "learning_rate": 0.00042606820461384154, + "loss": 3.1128, + "theoretical_loss": 3.901543467560355, + "tokens_seen": 516111360 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004260581745235707, + "loss": 2.903, + "theoretical_loss": 3.9014904016282124, + "tokens_seen": 516176896 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004260481444332999, + "loss": 3.158, + "theoretical_loss": 3.901437344319336, + "tokens_seen": 516242432 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004260381143430291, + "loss": 2.6293, + "theoretical_loss": 3.9013842956312317, + "tokens_seen": 516307968 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004260280842527583, + "loss": 2.7853, + "theoretical_loss": 3.901331255561405, + "tokens_seen": 516373504 + }, + { + "epoch": 1.07, + "learning_rate": 0.00042601805416248744, + "loss": 2.7866, + "theoretical_loss": 3.9012782241073602, + "tokens_seen": 516439040 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004260080240722167, + "loss": 3.2774, + "theoretical_loss": 3.9012252012666067, + "tokens_seen": 516504576 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004259979939819458, + "loss": 2.8639, + "theoretical_loss": 3.901172187036652, + "tokens_seen": 516570112 + }, + { + "epoch": 1.07, + "learning_rate": 0.00042598796389167504, + "loss": 2.9217, + "theoretical_loss": 3.901119181415006, + "tokens_seen": 516635648 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004259779338014042, + "loss": 2.8692, + "theoretical_loss": 3.901066184399179, + "tokens_seen": 516701184 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004259679037111334, + "loss": 2.8358, + "theoretical_loss": 3.9010131959866827, + "tokens_seen": 516766720 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004259578736208626, + "loss": 2.9191, + "theoretical_loss": 3.9009602161750294, + "tokens_seen": 516832256 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004259478435305918, + "loss": 2.8169, + "theoretical_loss": 3.9009072449617324, + "tokens_seen": 516897792 + }, + { + "epoch": 1.07, + "learning_rate": 0.00042593781344032095, + "loss": 3.0129, + "theoretical_loss": 3.900854282344307, + "tokens_seen": 516963328 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004259277833500502, + "loss": 3.0345, + "theoretical_loss": 3.9008013283202683, + "tokens_seen": 517028864 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004259177532597793, + "loss": 2.9059, + "theoretical_loss": 3.9007483828871337, + "tokens_seen": 517094400 + }, + { + "epoch": 1.07, + "learning_rate": 0.00042590772316950854, + "loss": 3.049, + "theoretical_loss": 3.9006954460424206, + "tokens_seen": 517159936 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004258976930792377, + "loss": 2.6156, + "theoretical_loss": 3.9006425177836475, + "tokens_seen": 517225472 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004258876629889669, + "loss": 2.9233, + "theoretical_loss": 3.9005895981083345, + "tokens_seen": 517291008 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004258776328986961, + "loss": 2.7387, + "theoretical_loss": 3.900536687014002, + "tokens_seen": 517356544 + }, + { + "epoch": 1.07, + "learning_rate": 0.00042586760280842527, + "loss": 2.8999, + "theoretical_loss": 3.9004837844981726, + "tokens_seen": 517422080 + }, + { + "epoch": 1.07, + "learning_rate": 0.00042585757271815445, + "loss": 2.5944, + "theoretical_loss": 3.9004308905583684, + "tokens_seen": 517487616 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004258475426278837, + "loss": 3.1158, + "theoretical_loss": 3.900378005192114, + "tokens_seen": 517553152 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004258375125376128, + "loss": 2.851, + "theoretical_loss": 3.900325128396934, + "tokens_seen": 517618688 + }, + { + "epoch": 1.07, + "learning_rate": 0.00042582748244734205, + "loss": 2.6032, + "theoretical_loss": 3.900272260170354, + "tokens_seen": 517684224 + }, + { + "epoch": 1.07, + "objective/train/docs_used": 605280, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.3959805965423584, + "objective/train/theoretical_loss": 3.900232614622076, + "objective/train/tokens_used": 538193376, + "theoretical_loss": 3.900232614622076, + "tokens_seen": 517733376 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004258174523570712, + "loss": 3.1391, + "theoretical_loss": 3.9002194005099016, + "tokens_seen": 517749760 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004258074222668004, + "loss": 2.8197, + "theoretical_loss": 3.900166549413105, + "tokens_seen": 517815296 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004257973921765296, + "loss": 3.117, + "theoretical_loss": 3.900113706877492, + "tokens_seen": 517880832 + }, + { + "epoch": 1.07, + "learning_rate": 0.00042578736208625877, + "loss": 2.9555, + "theoretical_loss": 3.9000608729005943, + "tokens_seen": 517946368 + }, + { + "epoch": 1.07, + "learning_rate": 0.00042577733199598795, + "loss": 2.7419, + "theoretical_loss": 3.9000080474799415, + "tokens_seen": 518011904 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004257673019057172, + "loss": 2.6787, + "theoretical_loss": 3.8999552306130667, + "tokens_seen": 518077440 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004257572718154463, + "loss": 2.8847, + "theoretical_loss": 3.8999024222975036, + "tokens_seen": 518142976 + }, + { + "epoch": 1.07, + "learning_rate": 0.00042574724172517555, + "loss": 2.898, + "theoretical_loss": 3.899849622530785, + "tokens_seen": 518208512 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004257372116349047, + "loss": 3.0798, + "theoretical_loss": 3.899796831310446, + "tokens_seen": 518274048 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004257271815446339, + "loss": 2.8054, + "theoretical_loss": 3.899744048634024, + "tokens_seen": 518339584 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004257171514543631, + "loss": 2.847, + "theoretical_loss": 3.899691274499056, + "tokens_seen": 518405120 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004257071213640923, + "loss": 2.9221, + "theoretical_loss": 3.8996385089030796, + "tokens_seen": 518470656 + }, + { + "epoch": 1.07, + "learning_rate": 0.00042569709127382145, + "loss": 2.92, + "theoretical_loss": 3.899585751843635, + "tokens_seen": 518536192 + }, + { + "epoch": 1.07, + "learning_rate": 0.00042568706118355064, + "loss": 2.9806, + "theoretical_loss": 3.899533003318261, + "tokens_seen": 518601728 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004256770310932798, + "loss": 2.9909, + "theoretical_loss": 3.8994802633245014, + "tokens_seen": 518667264 + }, + { + "epoch": 1.07, + "learning_rate": 0.00042566700100300905, + "loss": 2.8064, + "theoretical_loss": 3.899427531859896, + "tokens_seen": 518732800 + }, + { + "epoch": 1.07, + "learning_rate": 0.00042565697091273823, + "loss": 2.6601, + "theoretical_loss": 3.899374808921989, + "tokens_seen": 518798336 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004256469408224674, + "loss": 2.8185, + "theoretical_loss": 3.8993220945083253, + "tokens_seen": 518863872 + }, + { + "epoch": 1.07, + "learning_rate": 0.00042563691073219665, + "loss": 2.9848, + "theoretical_loss": 3.8992693886164504, + "tokens_seen": 518929408 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004256268806419258, + "loss": 3.0078, + "theoretical_loss": 3.8992166912439092, + "tokens_seen": 518994944 + }, + { + "epoch": 1.07, + "learning_rate": 0.000425616850551655, + "loss": 2.8503, + "theoretical_loss": 3.8991640023882512, + "tokens_seen": 519060480 + }, + { + "epoch": 1.07, + "learning_rate": 0.00042560682046138414, + "loss": 3.1139, + "theoretical_loss": 3.8991113220470233, + "tokens_seen": 519126016 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004255967903711134, + "loss": 2.8546, + "theoretical_loss": 3.8990586502177758, + "tokens_seen": 519191552 + }, + { + "epoch": 1.07, + "learning_rate": 0.00042558676028084256, + "loss": 2.8532, + "theoretical_loss": 3.8990059868980587, + "tokens_seen": 519257088 + }, + { + "epoch": 1.07, + "learning_rate": 0.00042557673019057174, + "loss": 2.891, + "theoretical_loss": 3.8989533320854237, + "tokens_seen": 519322624 + }, + { + "epoch": 1.07, + "objective/train/docs_used": 606732, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.3824806213378906, + "objective/train/theoretical_loss": 3.8989138465572477, + "objective/train/tokens_used": 539831776, + "theoretical_loss": 3.8989138465572477, + "tokens_seen": 519371776 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004255667001003009, + "loss": 2.845, + "theoretical_loss": 3.898900685777423, + "tokens_seen": 519388160 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004255566700100301, + "loss": 2.9526, + "theoretical_loss": 3.8988480479716108, + "tokens_seen": 519453696 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004255466399197593, + "loss": 3.05, + "theoretical_loss": 3.8987954186655402, + "tokens_seen": 519519232 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004255366098294885, + "loss": 2.9402, + "theoretical_loss": 3.8987427978567686, + "tokens_seen": 519584768 + }, + { + "epoch": 1.07, + "learning_rate": 0.00042552657973921764, + "loss": 2.8061, + "theoretical_loss": 3.898690185542852, + "tokens_seen": 519650304 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004255165496489469, + "loss": 2.8778, + "theoretical_loss": 3.8986375817213466, + "tokens_seen": 519715840 + }, + { + "epoch": 1.07, + "learning_rate": 0.000425506519558676, + "loss": 2.7645, + "theoretical_loss": 3.8985849863898125, + "tokens_seen": 519781376 + }, + { + "epoch": 1.07, + "learning_rate": 0.00042549648946840524, + "loss": 3.1105, + "theoretical_loss": 3.8985323995458083, + "tokens_seen": 519846912 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004254864593781344, + "loss": 2.9125, + "theoretical_loss": 3.8984798211868963, + "tokens_seen": 519912448 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004254764292878636, + "loss": 3.1081, + "theoretical_loss": 3.8984272513106357, + "tokens_seen": 519977984 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004254663991975928, + "loss": 3.0043, + "theoretical_loss": 3.89837468991459, + "tokens_seen": 520043520 + }, + { + "epoch": 1.07, + "learning_rate": 0.000425456369107322, + "loss": 2.9944, + "theoretical_loss": 3.898322136996324, + "tokens_seen": 520109056 + }, + { + "epoch": 1.07, + "learning_rate": 0.00042544633901705115, + "loss": 2.826, + "theoretical_loss": 3.898269592553401, + "tokens_seen": 520174592 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004254363089267804, + "loss": 2.5198, + "theoretical_loss": 3.8982170565833867, + "tokens_seen": 520240128 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004254262788365095, + "loss": 2.9318, + "theoretical_loss": 3.8981645290838483, + "tokens_seen": 520305664 + }, + { + "epoch": 1.07, + "learning_rate": 0.00042541624874623874, + "loss": 2.7986, + "theoretical_loss": 3.898112010052353, + "tokens_seen": 520371200 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004254062186559679, + "loss": 3.0719, + "theoretical_loss": 3.898059499486469, + "tokens_seen": 520436736 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004253961885656971, + "loss": 2.9799, + "theoretical_loss": 3.8980069973837677, + "tokens_seen": 520502272 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004253861584754263, + "loss": 2.9913, + "theoretical_loss": 3.8979545037418175, + "tokens_seen": 520567808 + }, + { + "epoch": 1.07, + "learning_rate": 0.00042537612838515547, + "loss": 2.8971, + "theoretical_loss": 3.897902018558192, + "tokens_seen": 520633344 + }, + { + "epoch": 1.07, + "learning_rate": 0.00042536609829488465, + "loss": 2.9909, + "theoretical_loss": 3.8978495418304626, + "tokens_seen": 520698880 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004253560682046139, + "loss": 3.0641, + "theoretical_loss": 3.8977970735562035, + "tokens_seen": 520764416 + }, + { + "epoch": 1.07, + "learning_rate": 0.000425346038114343, + "loss": 3.1792, + "theoretical_loss": 3.8977446137329887, + "tokens_seen": 520829952 + }, + { + "epoch": 1.07, + "learning_rate": 0.00042533600802407225, + "loss": 3.1222, + "theoretical_loss": 3.8976921623583944, + "tokens_seen": 520895488 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004253259779338014, + "loss": 2.9062, + "theoretical_loss": 3.8976397194299977, + "tokens_seen": 520961024 + }, + { + "epoch": 1.07, + "objective/train/docs_used": 607442, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.450615167617798, + "objective/train/theoretical_loss": 3.8976003927750593, + "objective/train/tokens_used": 541470176, + "theoretical_loss": 3.8976003927750593, + "tokens_seen": 521010176 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004253159478435306, + "loss": 2.8771, + "theoretical_loss": 3.8975872849453754, + "tokens_seen": 521026560 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004253059177532598, + "loss": 2.9847, + "theoretical_loss": 3.897534858902107, + "tokens_seen": 521092096 + }, + { + "epoch": 1.07, + "learning_rate": 0.00042529588766298897, + "loss": 2.7599, + "theoretical_loss": 3.897482441297771, + "tokens_seen": 521157632 + }, + { + "epoch": 1.07, + "learning_rate": 0.00042528585757271815, + "loss": 2.8196, + "theoretical_loss": 3.8974300321299484, + "tokens_seen": 521223168 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004252758274824474, + "loss": 2.7218, + "theoretical_loss": 3.897377631396222, + "tokens_seen": 521288704 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004252657973921765, + "loss": 2.7832, + "theoretical_loss": 3.8973252390941733, + "tokens_seen": 521354240 + }, + { + "epoch": 1.07, + "learning_rate": 0.00042525576730190575, + "loss": 3.1878, + "theoretical_loss": 3.8972728552213862, + "tokens_seen": 521419776 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004252457372116349, + "loss": 2.9531, + "theoretical_loss": 3.897220479775446, + "tokens_seen": 521485312 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004252357071213641, + "loss": 2.8667, + "theoretical_loss": 3.8971681127539366, + "tokens_seen": 521550848 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004252256770310933, + "loss": 2.84, + "theoretical_loss": 3.8971157541544468, + "tokens_seen": 521616384 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004252156469408225, + "loss": 2.7896, + "theoretical_loss": 3.8970634039745624, + "tokens_seen": 521681920 + }, + { + "epoch": 1.07, + "learning_rate": 0.00042520561685055165, + "loss": 3.1027, + "theoretical_loss": 3.897011062211873, + "tokens_seen": 521747456 + }, + { + "epoch": 1.07, + "learning_rate": 0.00042519558676028084, + "loss": 2.7281, + "theoretical_loss": 3.896958728863968, + "tokens_seen": 521812992 + }, + { + "epoch": 1.07, + "learning_rate": 0.00042518555667001, + "loss": 2.9021, + "theoretical_loss": 3.8969064039284387, + "tokens_seen": 521878528 + }, + { + "epoch": 1.07, + "learning_rate": 0.00042517552657973925, + "loss": 2.8982, + "theoretical_loss": 3.8968540874028754, + "tokens_seen": 521944064 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004251654964894684, + "loss": 3.0018, + "theoretical_loss": 3.8968017792848713, + "tokens_seen": 522009600 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004251554663991976, + "loss": 3.1168, + "theoretical_loss": 3.8967494795720197, + "tokens_seen": 522075136 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004251454363089268, + "loss": 2.9336, + "theoretical_loss": 3.896697188261916, + "tokens_seen": 522140672 + }, + { + "epoch": 1.07, + "learning_rate": 0.000425135406218656, + "loss": 2.8327, + "theoretical_loss": 3.8966449053521552, + "tokens_seen": 522206208 + }, + { + "epoch": 1.07, + "learning_rate": 0.00042512537612838516, + "loss": 2.992, + "theoretical_loss": 3.896592630840334, + "tokens_seen": 522271744 + }, + { + "epoch": 1.07, + "learning_rate": 0.00042511534603811434, + "loss": 2.643, + "theoretical_loss": 3.896540364724049, + "tokens_seen": 522337280 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004251053159478435, + "loss": 2.9115, + "theoretical_loss": 3.8964881070009003, + "tokens_seen": 522402816 + }, + { + "epoch": 1.07, + "learning_rate": 0.00042509528585757276, + "loss": 2.8193, + "theoretical_loss": 3.896435857668486, + "tokens_seen": 522468352 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004250852557673019, + "loss": 2.856, + "theoretical_loss": 3.8963836167244077, + "tokens_seen": 522533888 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004250752256770311, + "loss": 2.9309, + "theoretical_loss": 3.896331384166267, + "tokens_seen": 522599424 + }, + { + "epoch": 1.07, + "objective/train/docs_used": 608749, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.052105665206909, + "objective/train/theoretical_loss": 3.896292215249489, + "objective/train/tokens_used": 543108576, + "theoretical_loss": 3.896292215249489, + "tokens_seen": 522648576 + }, + { + "epoch": 1.07, + "learning_rate": 0.00042506519558676024, + "loss": 2.8582, + "theoretical_loss": 3.896279159991664, + "tokens_seen": 522664960 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004250551654964895, + "loss": 2.9196, + "theoretical_loss": 3.8962269441982063, + "tokens_seen": 522730496 + }, + { + "epoch": 1.07, + "learning_rate": 0.00042504513540621866, + "loss": 2.9007, + "theoretical_loss": 3.8961747367834945, + "tokens_seen": 522796032 + }, + { + "epoch": 1.07, + "learning_rate": 0.00042503510531594784, + "loss": 3.1226, + "theoretical_loss": 3.896122537745136, + "tokens_seen": 522861568 + }, + { + "epoch": 1.07, + "learning_rate": 0.000425025075225677, + "loss": 3.0625, + "theoretical_loss": 3.896070347080737, + "tokens_seen": 522927104 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004250150451354062, + "loss": 2.948, + "theoretical_loss": 3.896018164787905, + "tokens_seen": 522992640 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004250050150451354, + "loss": 2.7946, + "theoretical_loss": 3.895965990864247, + "tokens_seen": 523058176 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004249949849548646, + "loss": 2.8184, + "theoretical_loss": 3.895913825307374, + "tokens_seen": 523123712 + }, + { + "epoch": 1.07, + "learning_rate": 0.00042498495486459375, + "loss": 2.6486, + "theoretical_loss": 3.895861668114896, + "tokens_seen": 523189248 + }, + { + "epoch": 1.07, + "learning_rate": 0.000424974924774323, + "loss": 2.7146, + "theoretical_loss": 3.895809519284424, + "tokens_seen": 523254784 + }, + { + "epoch": 1.07, + "learning_rate": 0.00042496489468405216, + "loss": 3.0432, + "theoretical_loss": 3.8957573788135704, + "tokens_seen": 523320320 + }, + { + "epoch": 1.07, + "learning_rate": 0.00042495486459378135, + "loss": 2.756, + "theoretical_loss": 3.895705246699949, + "tokens_seen": 523385856 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004249448345035105, + "loss": 2.9122, + "theoretical_loss": 3.895653122941173, + "tokens_seen": 523451392 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004249348044132397, + "loss": 3.0265, + "theoretical_loss": 3.895601007534858, + "tokens_seen": 523516928 + }, + { + "epoch": 1.07, + "learning_rate": 0.00042492477432296894, + "loss": 3.0803, + "theoretical_loss": 3.8955489004786212, + "tokens_seen": 523582464 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004249147442326981, + "loss": 2.9926, + "theoretical_loss": 3.8954968017700793, + "tokens_seen": 523648000 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004249047141424273, + "loss": 2.8496, + "theoretical_loss": 3.89544471140685, + "tokens_seen": 523713536 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004248946840521565, + "loss": 2.8785, + "theoretical_loss": 3.8953926293865524, + "tokens_seen": 523779072 + }, + { + "epoch": 1.07, + "learning_rate": 0.00042488465396188567, + "loss": 3.0031, + "theoretical_loss": 3.8953405557068077, + "tokens_seen": 523844608 + }, + { + "epoch": 1.07, + "learning_rate": 0.00042487462387161485, + "loss": 2.9991, + "theoretical_loss": 3.895288490365236, + "tokens_seen": 523910144 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004248645937813441, + "loss": 2.8258, + "theoretical_loss": 3.8952364333594596, + "tokens_seen": 523975680 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004248545636910732, + "loss": 2.7998, + "theoretical_loss": 3.8951843846871013, + "tokens_seen": 524041216 + }, + { + "epoch": 1.07, + "learning_rate": 0.00042484453360080245, + "loss": 2.7548, + "theoretical_loss": 3.895132344345787, + "tokens_seen": 524106752 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004248345035105316, + "loss": 3.0234, + "theoretical_loss": 3.895080312333139, + "tokens_seen": 524172288 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004248244734202608, + "loss": 2.9753, + "theoretical_loss": 3.8950282886467846, + "tokens_seen": 524237824 + }, + { + "epoch": 1.07, + "objective/train/docs_used": 609143, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.270958662033081, + "objective/train/theoretical_loss": 3.894989276344722, + "objective/train/tokens_used": 544746976, + "theoretical_loss": 3.894989276344722, + "tokens_seen": 524286976 + }, + { + "epoch": 1.07, + "learning_rate": 0.00042481444332999, + "loss": 3.1384, + "theoretical_loss": 3.8949762732843514, + "tokens_seen": 524303360 + }, + { + "epoch": 1.07, + "learning_rate": 0.00042480441323971917, + "loss": 2.5421, + "theoretical_loss": 3.8949242662434664, + "tokens_seen": 524368896 + }, + { + "epoch": 1.07, + "learning_rate": 0.00042479438314944835, + "loss": 3.0288, + "theoretical_loss": 3.894872267521759, + "tokens_seen": 524434432 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004247843530591776, + "loss": 3.0604, + "theoretical_loss": 3.8948202771168585, + "tokens_seen": 524499968 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004247743229689067, + "loss": 2.8957, + "theoretical_loss": 3.894768295026396, + "tokens_seen": 524565504 + }, + { + "epoch": 1.07, + "learning_rate": 0.00042476429287863595, + "loss": 2.7608, + "theoretical_loss": 3.894716321248004, + "tokens_seen": 524631040 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004247542627883651, + "loss": 2.9537, + "theoretical_loss": 3.8946643557793146, + "tokens_seen": 524696576 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004247442326980943, + "loss": 2.9661, + "theoretical_loss": 3.8946123986179613, + "tokens_seen": 524762112 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004247342026078235, + "loss": 2.9881, + "theoretical_loss": 3.8945604497615802, + "tokens_seen": 524827648 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004247241725175527, + "loss": 2.8578, + "theoretical_loss": 3.8945085092078053, + "tokens_seen": 524893184 + }, + { + "epoch": 1.07, + "learning_rate": 0.00042471414242728186, + "loss": 2.9858, + "theoretical_loss": 3.8944565769542745, + "tokens_seen": 524958720 + }, + { + "epoch": 1.07, + "learning_rate": 0.00042470411233701104, + "loss": 2.8689, + "theoretical_loss": 3.8944046529986247, + "tokens_seen": 525024256 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004246940822467402, + "loss": 2.9482, + "theoretical_loss": 3.8943527373384947, + "tokens_seen": 525089792 + }, + { + "epoch": 1.07, + "learning_rate": 0.00042468405215646945, + "loss": 3.0764, + "theoretical_loss": 3.8943008299715247, + "tokens_seen": 525155328 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004246740220661986, + "loss": 3.077, + "theoretical_loss": 3.8942489308953547, + "tokens_seen": 525220864 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004246639919759278, + "loss": 2.7081, + "theoretical_loss": 3.894197040107626, + "tokens_seen": 525286400 + }, + { + "epoch": 1.07, + "learning_rate": 0.000424653961885657, + "loss": 2.9005, + "theoretical_loss": 3.8941451576059807, + "tokens_seen": 525351936 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004246439317953862, + "loss": 2.7815, + "theoretical_loss": 3.8940932833880635, + "tokens_seen": 525417472 + }, + { + "epoch": 1.07, + "learning_rate": 0.00042463390170511536, + "loss": 2.698, + "theoretical_loss": 3.8940414174515183, + "tokens_seen": 525483008 + }, + { + "epoch": 1.07, + "learning_rate": 0.00042462387161484454, + "loss": 2.7497, + "theoretical_loss": 3.8939895597939898, + "tokens_seen": 525548544 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004246138415245737, + "loss": 2.8141, + "theoretical_loss": 3.893937710413125, + "tokens_seen": 525614080 + }, + { + "epoch": 1.07, + "learning_rate": 0.00042460381143430296, + "loss": 2.9974, + "theoretical_loss": 3.893885869306571, + "tokens_seen": 525679616 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004245937813440321, + "loss": 2.6887, + "theoretical_loss": 3.8938340364719766, + "tokens_seen": 525745152 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004245837512537613, + "loss": 2.852, + "theoretical_loss": 3.89378221190699, + "tokens_seen": 525810688 + }, + { + "epoch": 1.07, + "learning_rate": 0.00042457372116349044, + "loss": 2.9421, + "theoretical_loss": 3.8937303956092615, + "tokens_seen": 525876224 + }, + { + "epoch": 1.07, + "objective/train/docs_used": 610436, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9925901889801025, + "objective/train/theoretical_loss": 3.893691538809941, + "objective/train/tokens_used": 546385376, + "theoretical_loss": 3.893691538809941, + "tokens_seen": 525925376 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004245636910732197, + "loss": 2.7874, + "theoretical_loss": 3.893678587576443, + "tokens_seen": 525941760 + }, + { + "epoch": 1.07, + "learning_rate": 0.00042455366098294886, + "loss": 2.9986, + "theoretical_loss": 3.8936267878061868, + "tokens_seen": 526007296 + }, + { + "epoch": 1.07, + "learning_rate": 0.00042454363089267804, + "loss": 2.7701, + "theoretical_loss": 3.8935749962961443, + "tokens_seen": 526072832 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004245336008024072, + "loss": 2.8532, + "theoretical_loss": 3.8935232130439714, + "tokens_seen": 526138368 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004245235707121364, + "loss": 2.8331, + "theoretical_loss": 3.893471438047322, + "tokens_seen": 526203904 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004245135406218656, + "loss": 3.127, + "theoretical_loss": 3.8934196713038522, + "tokens_seen": 526269440 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004245035105315948, + "loss": 3.0505, + "theoretical_loss": 3.8933679128112186, + "tokens_seen": 526334976 + }, + { + "epoch": 1.07, + "learning_rate": 0.00042449348044132395, + "loss": 3.023, + "theoretical_loss": 3.89331616256708, + "tokens_seen": 526400512 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004244834503510532, + "loss": 3.0169, + "theoretical_loss": 3.893264420569094, + "tokens_seen": 526466048 + }, + { + "epoch": 1.07, + "learning_rate": 0.00042447342026078236, + "loss": 2.9502, + "theoretical_loss": 3.893212686814921, + "tokens_seen": 526531584 + }, + { + "epoch": 1.07, + "learning_rate": 0.00042446339017051155, + "loss": 2.9393, + "theoretical_loss": 3.893160961302222, + "tokens_seen": 526597120 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004244533600802407, + "loss": 2.765, + "theoretical_loss": 3.893109244028658, + "tokens_seen": 526662656 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004244433299899699, + "loss": 3.2206, + "theoretical_loss": 3.8930575349918923, + "tokens_seen": 526728192 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004244332998996991, + "loss": 2.9727, + "theoretical_loss": 3.893005834189588, + "tokens_seen": 526793728 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004244232698094283, + "loss": 3.0495, + "theoretical_loss": 3.8929541416194096, + "tokens_seen": 526859264 + }, + { + "epoch": 1.07, + "learning_rate": 0.00042441323971915745, + "loss": 3.11, + "theoretical_loss": 3.8929024572790225, + "tokens_seen": 526924800 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004244032096288867, + "loss": 2.7881, + "theoretical_loss": 3.8928507811660937, + "tokens_seen": 526990336 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004243931795386158, + "loss": 3.0553, + "theoretical_loss": 3.8927991132782904, + "tokens_seen": 527055872 + }, + { + "epoch": 1.07, + "learning_rate": 0.00042438314944834505, + "loss": 3.0207, + "theoretical_loss": 3.8927474536132802, + "tokens_seen": 527121408 + }, + { + "epoch": 1.07, + "learning_rate": 0.00042437311935807423, + "loss": 2.9144, + "theoretical_loss": 3.8926958021687335, + "tokens_seen": 527186944 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004243630892678034, + "loss": 2.8823, + "theoretical_loss": 3.8926441589423195, + "tokens_seen": 527252480 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004243530591775326, + "loss": 3.1453, + "theoretical_loss": 3.89259252393171, + "tokens_seen": 527318016 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004243430290872618, + "loss": 2.7057, + "theoretical_loss": 3.8925408971345776, + "tokens_seen": 527383552 + }, + { + "epoch": 1.07, + "learning_rate": 0.00042433299899699095, + "loss": 2.645, + "theoretical_loss": 3.8924892785485943, + "tokens_seen": 527449088 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004243229689067202, + "loss": 2.9985, + "theoretical_loss": 3.8924376681714348, + "tokens_seen": 527514624 + }, + { + "epoch": 1.07, + "objective/train/docs_used": 611010, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8362579345703125, + "objective/train/theoretical_loss": 3.8923989657742073, + "objective/train/tokens_used": 548023776, + "theoretical_loss": 3.8923989657742073, + "tokens_seen": 527563776 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004243129388164493, + "loss": 2.9011, + "theoretical_loss": 3.892386066000774, + "tokens_seen": 527580160 + }, + { + "epoch": 1.07, + "learning_rate": 0.00042430290872617855, + "loss": 2.876, + "theoretical_loss": 3.892334472034287, + "tokens_seen": 527645696 + }, + { + "epoch": 1.07, + "learning_rate": 0.00042429287863590773, + "loss": 3.0023, + "theoretical_loss": 3.8922828862696526, + "tokens_seen": 527711232 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004242828485456369, + "loss": 2.8032, + "theoretical_loss": 3.8922313087045466, + "tokens_seen": 527776768 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004242728184553661, + "loss": 2.8308, + "theoretical_loss": 3.892179739336649, + "tokens_seen": 527842304 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004242627883650953, + "loss": 2.8076, + "theoretical_loss": 3.892128178163639, + "tokens_seen": 527907840 + }, + { + "epoch": 1.07, + "learning_rate": 0.00042425275827482446, + "loss": 3.0146, + "theoretical_loss": 3.892076625183197, + "tokens_seen": 527973376 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004242427281845537, + "loss": 2.9239, + "theoretical_loss": 3.8920250803930054, + "tokens_seen": 528038912 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004242326980942828, + "loss": 2.7142, + "theoretical_loss": 3.891973543790746, + "tokens_seen": 528104448 + }, + { + "epoch": 1.07, + "learning_rate": 0.00042422266800401206, + "loss": 2.8914, + "theoretical_loss": 3.891922015374103, + "tokens_seen": 528169984 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004242126379137412, + "loss": 2.6705, + "theoretical_loss": 3.8918704951407603, + "tokens_seen": 528235520 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004242026078234704, + "loss": 2.8523, + "theoretical_loss": 3.8918189830884033, + "tokens_seen": 528301056 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004241925777331996, + "loss": 2.9579, + "theoretical_loss": 3.8917674792147183, + "tokens_seen": 528366592 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004241825476429288, + "loss": 2.8322, + "theoretical_loss": 3.8917159835173933, + "tokens_seen": 528432128 + }, + { + "epoch": 1.07, + "learning_rate": 0.000424172517552658, + "loss": 2.8149, + "theoretical_loss": 3.891664495994115, + "tokens_seen": 528497664 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004241624874623872, + "loss": 3.0571, + "theoretical_loss": 3.891613016642574, + "tokens_seen": 528563200 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004241524573721164, + "loss": 3.1115, + "theoretical_loss": 3.8915615454604593, + "tokens_seen": 528628736 + }, + { + "epoch": 1.07, + "learning_rate": 0.00042414242728184556, + "loss": 2.8828, + "theoretical_loss": 3.8915100824454627, + "tokens_seen": 528694272 + }, + { + "epoch": 1.07, + "learning_rate": 0.00042413239719157474, + "loss": 3.0419, + "theoretical_loss": 3.8914586275952763, + "tokens_seen": 528759808 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004241223671013039, + "loss": 2.9529, + "theoretical_loss": 3.8914071809075916, + "tokens_seen": 528825344 + }, + { + "epoch": 1.07, + "learning_rate": 0.00042411233701103316, + "loss": 2.8425, + "theoretical_loss": 3.891355742380104, + "tokens_seen": 528890880 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004241023069207623, + "loss": 2.7841, + "theoretical_loss": 3.891304312010507, + "tokens_seen": 528956416 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004240922768304915, + "loss": 2.9775, + "theoretical_loss": 3.8912528897964975, + "tokens_seen": 529021952 + }, + { + "epoch": 1.07, + "learning_rate": 0.00042408224674022065, + "loss": 2.9298, + "theoretical_loss": 3.891201475735772, + "tokens_seen": 529087488 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004240722166499499, + "loss": 2.8611, + "theoretical_loss": 3.891150069826027, + "tokens_seen": 529153024 + }, + { + "epoch": 1.07, + "objective/train/docs_used": 612097, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.09621000289917, + "objective/train/theoretical_loss": 3.8911115207414153, + "objective/train/tokens_used": 549662176, + "theoretical_loss": 3.8911115207414153, + "tokens_seen": 529202176 + }, + { + "epoch": 1.07, + "learning_rate": 0.00042406218655967906, + "loss": 3.0195, + "theoretical_loss": 3.8910986720649614, + "tokens_seen": 529218560 + }, + { + "epoch": 1.07, + "learning_rate": 0.00042405215646940824, + "loss": 3.0257, + "theoretical_loss": 3.8910472824502755, + "tokens_seen": 529284096 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004240421263791374, + "loss": 3.2495, + "theoretical_loss": 3.890995900979669, + "tokens_seen": 529349632 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004240320962888666, + "loss": 2.8085, + "theoretical_loss": 3.890944527650843, + "tokens_seen": 529415168 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004240220661985958, + "loss": 2.9752, + "theoretical_loss": 3.8908931624615004, + "tokens_seen": 529480704 + }, + { + "epoch": 1.07, + "learning_rate": 0.000424012036108325, + "loss": 2.9726, + "theoretical_loss": 3.8908418054093437, + "tokens_seen": 529546240 + }, + { + "epoch": 1.07, + "learning_rate": 0.00042400200601805415, + "loss": 2.9715, + "theoretical_loss": 3.890790456492078, + "tokens_seen": 529611776 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004239919759277834, + "loss": 2.7099, + "theoretical_loss": 3.890739115707407, + "tokens_seen": 529677312 + }, + { + "epoch": 1.07, + "learning_rate": 0.00042398194583751256, + "loss": 2.9256, + "theoretical_loss": 3.8906877830530373, + "tokens_seen": 529742848 + }, + { + "epoch": 1.07, + "learning_rate": 0.00042397191574724175, + "loss": 2.8714, + "theoretical_loss": 3.8906364585266764, + "tokens_seen": 529808384 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004239618856569709, + "loss": 2.8072, + "theoretical_loss": 3.8905851421260316, + "tokens_seen": 529873920 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004239518555667001, + "loss": 2.8951, + "theoretical_loss": 3.890533833848811, + "tokens_seen": 529939456 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004239418254764293, + "loss": 2.8725, + "theoretical_loss": 3.8904825336927256, + "tokens_seen": 530004992 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004239317953861585, + "loss": 3.0388, + "theoretical_loss": 3.890431241655485, + "tokens_seen": 530070528 + }, + { + "epoch": 1.07, + "learning_rate": 0.00042392176529588765, + "loss": 2.9665, + "theoretical_loss": 3.890379957734801, + "tokens_seen": 530136064 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004239117352056169, + "loss": 2.9384, + "theoretical_loss": 3.890328681928387, + "tokens_seen": 530201600 + }, + { + "epoch": 1.07, + "learning_rate": 0.000423901705115346, + "loss": 3.113, + "theoretical_loss": 3.890277414233955, + "tokens_seen": 530267136 + }, + { + "epoch": 1.07, + "learning_rate": 0.00042389167502507525, + "loss": 2.801, + "theoretical_loss": 3.89022615464922, + "tokens_seen": 530332672 + }, + { + "epoch": 1.07, + "learning_rate": 0.00042388164493480443, + "loss": 2.8593, + "theoretical_loss": 3.8901749031718973, + "tokens_seen": 530398208 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004238716148445336, + "loss": 3.1283, + "theoretical_loss": 3.890123659799703, + "tokens_seen": 530463744 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004238615847542628, + "loss": 2.9641, + "theoretical_loss": 3.890072424530354, + "tokens_seen": 530529280 + }, + { + "epoch": 1.07, + "learning_rate": 0.000423851554663992, + "loss": 3.1211, + "theoretical_loss": 3.890021197361569, + "tokens_seen": 530594816 + }, + { + "epoch": 1.07, + "learning_rate": 0.00042384152457372115, + "loss": 3.0983, + "theoretical_loss": 3.889969978291066, + "tokens_seen": 530660352 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004238314944834504, + "loss": 3.0237, + "theoretical_loss": 3.8899187673165656, + "tokens_seen": 530725888 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004238214643931795, + "loss": 2.7202, + "theoretical_loss": 3.8898675644357885, + "tokens_seen": 530791424 + }, + { + "epoch": 1.07, + "objective/train/docs_used": 612816, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.779944658279419, + "objective/train/theoretical_loss": 3.889829167585341, + "objective/train/tokens_used": 551300576, + "theoretical_loss": 3.889829167585341, + "tokens_seen": 530840576 + }, + { + "epoch": 1.07, + "learning_rate": 0.00042381143430290875, + "loss": 2.7002, + "theoretical_loss": 3.8898163696464563, + "tokens_seen": 530856960 + }, + { + "epoch": 1.07, + "learning_rate": 0.00042380140421263793, + "loss": 2.8166, + "theoretical_loss": 3.889765182946292, + "tokens_seen": 530922496 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004237913741223671, + "loss": 3.0376, + "theoretical_loss": 3.8897140043330185, + "tokens_seen": 530988032 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004237813440320963, + "loss": 3.2238, + "theoretical_loss": 3.889662833804361, + "tokens_seen": 531053568 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004237713139418255, + "loss": 2.808, + "theoretical_loss": 3.889611671358044, + "tokens_seen": 531119104 + }, + { + "epoch": 1.07, + "learning_rate": 0.00042376128385155466, + "loss": 3.0963, + "theoretical_loss": 3.8895605169917955, + "tokens_seen": 531184640 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004237512537612839, + "loss": 2.9041, + "theoretical_loss": 3.8895093707033404, + "tokens_seen": 531250176 + }, + { + "epoch": 1.07, + "learning_rate": 0.000423741223671013, + "loss": 3.0183, + "theoretical_loss": 3.8894582324904086, + "tokens_seen": 531315712 + }, + { + "epoch": 1.07, + "learning_rate": 0.00042373119358074226, + "loss": 2.8208, + "theoretical_loss": 3.889407102350729, + "tokens_seen": 531381248 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004237211634904714, + "loss": 2.8103, + "theoretical_loss": 3.8893559802820317, + "tokens_seen": 531446784 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004237111334002006, + "loss": 2.868, + "theoretical_loss": 3.8893048662820466, + "tokens_seen": 531512320 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004237011033099298, + "loss": 2.7113, + "theoretical_loss": 3.8892537603485065, + "tokens_seen": 531577856 + }, + { + "epoch": 1.07, + "learning_rate": 0.000423691073219659, + "loss": 2.7521, + "theoretical_loss": 3.889202662479144, + "tokens_seen": 531643392 + }, + { + "epoch": 1.07, + "learning_rate": 0.00042368104312938816, + "loss": 2.9089, + "theoretical_loss": 3.8891515726716923, + "tokens_seen": 531708928 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004236710130391174, + "loss": 2.9575, + "theoretical_loss": 3.8891004909238873, + "tokens_seen": 531774464 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004236609829488465, + "loss": 2.8788, + "theoretical_loss": 3.889049417233463, + "tokens_seen": 531840000 + }, + { + "epoch": 1.07, + "learning_rate": 0.00042365095285857576, + "loss": 2.7697, + "theoretical_loss": 3.8889983515981568, + "tokens_seen": 531905536 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004236409227683049, + "loss": 3.0722, + "theoretical_loss": 3.888947294015705, + "tokens_seen": 531971072 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004236308926780341, + "loss": 3.1524, + "theoretical_loss": 3.8888962444838473, + "tokens_seen": 532036608 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004236208625877633, + "loss": 3.0272, + "theoretical_loss": 3.888845203000322, + "tokens_seen": 532102144 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004236108324974925, + "loss": 2.8684, + "theoretical_loss": 3.8887941695628694, + "tokens_seen": 532167680 + }, + { + "epoch": 1.07, + "learning_rate": 0.00042360080240722166, + "loss": 2.9734, + "theoretical_loss": 3.88874314416923, + "tokens_seen": 532233216 + }, + { + "epoch": 1.07, + "learning_rate": 0.00042359077231695085, + "loss": 2.8621, + "theoretical_loss": 3.888692126817147, + "tokens_seen": 532298752 + }, + { + "epoch": 1.07, + "learning_rate": 0.00042358074222668, + "loss": 2.6702, + "theoretical_loss": 3.8886411175043616, + "tokens_seen": 532364288 + }, + { + "epoch": 1.07, + "learning_rate": 0.00042357071213640926, + "loss": 2.8799, + "theoretical_loss": 3.8885901162286185, + "tokens_seen": 532429824 + }, + { + "epoch": 1.07, + "objective/train/docs_used": 614210, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.1336987018585205, + "objective/train/theoretical_loss": 3.888551870544763, + "objective/train/tokens_used": 552938976, + "theoretical_loss": 3.888551870544763, + "tokens_seen": 532478976 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004235606820461384, + "loss": 3.1439, + "theoretical_loss": 3.8885391229876625, + "tokens_seen": 532495360 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004235506519558676, + "loss": 2.9976, + "theoretical_loss": 3.8884881377792384, + "tokens_seen": 532560896 + }, + { + "epoch": 1.07, + "learning_rate": 0.00042354062186559675, + "loss": 3.0269, + "theoretical_loss": 3.888437160601093, + "tokens_seen": 532626432 + }, + { + "epoch": 1.07, + "learning_rate": 0.000423530591775326, + "loss": 2.83, + "theoretical_loss": 3.8883861914509743, + "tokens_seen": 532691968 + }, + { + "epoch": 1.07, + "learning_rate": 0.00042352056168505517, + "loss": 2.9997, + "theoretical_loss": 3.88833523032663, + "tokens_seen": 532757504 + }, + { + "epoch": 1.07, + "learning_rate": 0.00042351053159478435, + "loss": 3.0225, + "theoretical_loss": 3.888284277225809, + "tokens_seen": 532823040 + }, + { + "epoch": 1.07, + "learning_rate": 0.00042350050150451353, + "loss": 2.8851, + "theoretical_loss": 3.888233332146262, + "tokens_seen": 532888576 + }, + { + "epoch": 1.07, + "learning_rate": 0.00042349047141424276, + "loss": 2.8663, + "theoretical_loss": 3.8881823950857397, + "tokens_seen": 532954112 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004234804413239719, + "loss": 2.9248, + "theoretical_loss": 3.8881314660419934, + "tokens_seen": 533019648 + }, + { + "epoch": 1.07, + "learning_rate": 0.00042347041123370113, + "loss": 2.9781, + "theoretical_loss": 3.8880805450127776, + "tokens_seen": 533085184 + }, + { + "epoch": 1.07, + "learning_rate": 0.00042346038114343025, + "loss": 2.9781, + "theoretical_loss": 3.8880296319958445, + "tokens_seen": 533150720 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004234503510531595, + "loss": 2.7219, + "theoretical_loss": 3.8879787269889494, + "tokens_seen": 533216256 + }, + { + "epoch": 1.07, + "learning_rate": 0.00042344032096288867, + "loss": 3.0108, + "theoretical_loss": 3.887927829989848, + "tokens_seen": 533281792 + }, + { + "epoch": 1.07, + "learning_rate": 0.00042343029087261785, + "loss": 2.8635, + "theoretical_loss": 3.887876940996296, + "tokens_seen": 533347328 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004234202607823471, + "loss": 2.8422, + "theoretical_loss": 3.8878260600060512, + "tokens_seen": 533412864 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004234102306920762, + "loss": 3.0575, + "theoretical_loss": 3.8877751870168717, + "tokens_seen": 533478400 + }, + { + "epoch": 1.07, + "learning_rate": 0.00042340020060180545, + "loss": 2.9074, + "theoretical_loss": 3.8877243220265174, + "tokens_seen": 533543936 + }, + { + "epoch": 1.07, + "learning_rate": 0.00042339017051153463, + "loss": 3.0241, + "theoretical_loss": 3.887673465032748, + "tokens_seen": 533609472 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004233801404212638, + "loss": 2.865, + "theoretical_loss": 3.8876226160333234, + "tokens_seen": 533675008 + }, + { + "epoch": 1.07, + "learning_rate": 0.000423370110330993, + "loss": 3.0424, + "theoretical_loss": 3.8875717750260064, + "tokens_seen": 533740544 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004233600802407222, + "loss": 3.1135, + "theoretical_loss": 3.8875209420085595, + "tokens_seen": 533806080 + }, + { + "epoch": 1.07, + "learning_rate": 0.00042335005015045135, + "loss": 3.0429, + "theoretical_loss": 3.887470116978747, + "tokens_seen": 533871616 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004233400200601806, + "loss": 3.0547, + "theoretical_loss": 3.887419299934333, + "tokens_seen": 533937152 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004233299899699097, + "loss": 3.0215, + "theoretical_loss": 3.887368490873083, + "tokens_seen": 534002688 + }, + { + "epoch": 1.07, + "learning_rate": 0.00042331995987963895, + "loss": 3.0887, + "theoretical_loss": 3.887317689792763, + "tokens_seen": 534068224 + }, + { + "epoch": 1.07, + "objective/train/docs_used": 614920, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9131345748901367, + "objective/train/theoretical_loss": 3.8872795942186653, + "objective/train/tokens_used": 554577376, + "theoretical_loss": 3.8872795942186653, + "tokens_seen": 534117376 + }, + { + "epoch": 1.07, + "learning_rate": 0.00042330992978936813, + "loss": 2.9236, + "theoretical_loss": 3.887266896691141, + "tokens_seen": 534133760 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004232998996990973, + "loss": 2.948, + "theoretical_loss": 3.8872161115659845, + "tokens_seen": 534199296 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004232898696088265, + "loss": 2.7217, + "theoretical_loss": 3.8871653344150623, + "tokens_seen": 534264832 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004232798395185557, + "loss": 2.8982, + "theoretical_loss": 3.8871145652361454, + "tokens_seen": 534330368 + }, + { + "epoch": 1.07, + "learning_rate": 0.00042326980942828486, + "loss": 3.1815, + "theoretical_loss": 3.887063804027004, + "tokens_seen": 534395904 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004232597793380141, + "loss": 2.967, + "theoretical_loss": 3.88701305078541, + "tokens_seen": 534461440 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004232497492477432, + "loss": 3.0584, + "theoretical_loss": 3.886962305509136, + "tokens_seen": 534526976 + }, + { + "epoch": 1.07, + "learning_rate": 0.00042323971915747246, + "loss": 3.0766, + "theoretical_loss": 3.8869115681959556, + "tokens_seen": 534592512 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004232296890672016, + "loss": 3.0689, + "theoretical_loss": 3.886860838843643, + "tokens_seen": 534658048 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004232196589769308, + "loss": 2.9005, + "theoretical_loss": 3.886810117449974, + "tokens_seen": 534723584 + }, + { + "epoch": 1.07, + "learning_rate": 0.00042320962888666, + "loss": 2.9064, + "theoretical_loss": 3.8867594040127242, + "tokens_seen": 534789120 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004231995987963892, + "loss": 2.9348, + "theoretical_loss": 3.8867086985296715, + "tokens_seen": 534854656 + }, + { + "epoch": 1.07, + "learning_rate": 0.00042318956870611836, + "loss": 3.1647, + "theoretical_loss": 3.8866580009985934, + "tokens_seen": 534920192 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004231795386158476, + "loss": 2.8449, + "theoretical_loss": 3.8866073114172686, + "tokens_seen": 534985728 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004231695085255767, + "loss": 2.9154, + "theoretical_loss": 3.8865566297834775, + "tokens_seen": 535051264 + }, + { + "epoch": 1.07, + "learning_rate": 0.00042315947843530596, + "loss": 3.0347, + "theoretical_loss": 3.886505956095, + "tokens_seen": 535116800 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004231494483450351, + "loss": 2.9126, + "theoretical_loss": 3.886455290349619, + "tokens_seen": 535182336 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004231394182547643, + "loss": 3.0763, + "theoretical_loss": 3.8864046325451156, + "tokens_seen": 535247872 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004231293881644935, + "loss": 2.9425, + "theoretical_loss": 3.8863539826792737, + "tokens_seen": 535313408 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004231193580742227, + "loss": 2.9192, + "theoretical_loss": 3.886303340749878, + "tokens_seen": 535378944 + }, + { + "epoch": 1.07, + "learning_rate": 0.00042310932798395186, + "loss": 3.0836, + "theoretical_loss": 3.8862527067547123, + "tokens_seen": 535444480 + }, + { + "epoch": 1.07, + "learning_rate": 0.00042309929789368105, + "loss": 2.7298, + "theoretical_loss": 3.8862020806915645, + "tokens_seen": 535510016 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004230892678034102, + "loss": 3.1905, + "theoretical_loss": 3.88615146255822, + "tokens_seen": 535575552 + }, + { + "epoch": 1.07, + "learning_rate": 0.00042307923771313946, + "loss": 2.9487, + "theoretical_loss": 3.8861008523524676, + "tokens_seen": 535641088 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004230692076228686, + "loss": 2.7571, + "theoretical_loss": 3.8860502500720946, + "tokens_seen": 535706624 + }, + { + "epoch": 1.07, + "objective/train/docs_used": 616369, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9449539184570312, + "objective/train/theoretical_loss": 3.886012303561517, + "objective/train/tokens_used": 556215776, + "theoretical_loss": 3.886012303561517, + "tokens_seen": 535755776 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004230591775325978, + "loss": 2.9742, + "theoretical_loss": 3.885999655714893, + "tokens_seen": 535772160 + }, + { + "epoch": 1.07, + "learning_rate": 0.00042304914744232695, + "loss": 3.0579, + "theoretical_loss": 3.8859490692786505, + "tokens_seen": 535837696 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004230391173520562, + "loss": 3.0758, + "theoretical_loss": 3.8858984907611607, + "tokens_seen": 535903232 + }, + { + "epoch": 1.07, + "learning_rate": 0.00042302908726178537, + "loss": 2.8493, + "theoretical_loss": 3.8858479201602147, + "tokens_seen": 535968768 + }, + { + "epoch": 1.07, + "learning_rate": 0.00042301905717151455, + "loss": 3.0233, + "theoretical_loss": 3.8857973574736056, + "tokens_seen": 536034304 + }, + { + "epoch": 1.07, + "learning_rate": 0.00042300902708124373, + "loss": 2.9448, + "theoretical_loss": 3.8857468026991278, + "tokens_seen": 536099840 + }, + { + "epoch": 1.07, + "learning_rate": 0.00042299899699097297, + "loss": 2.9356, + "theoretical_loss": 3.885696255834576, + "tokens_seen": 536165376 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004229889669007021, + "loss": 2.8509, + "theoretical_loss": 3.885645716877746, + "tokens_seen": 536230912 + }, + { + "epoch": 1.07, + "learning_rate": 0.00042297893681043133, + "loss": 2.9117, + "theoretical_loss": 3.8855951858264346, + "tokens_seen": 536296448 + }, + { + "epoch": 1.07, + "learning_rate": 0.00042296890672016045, + "loss": 3.0141, + "theoretical_loss": 3.8855446626784396, + "tokens_seen": 536361984 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004229588766298897, + "loss": 2.8382, + "theoretical_loss": 3.8854941474315585, + "tokens_seen": 536427520 + }, + { + "epoch": 1.07, + "learning_rate": 0.00042294884653961887, + "loss": 2.84, + "theoretical_loss": 3.885443640083592, + "tokens_seen": 536493056 + }, + { + "epoch": 1.07, + "learning_rate": 0.00042293881644934805, + "loss": 3.0141, + "theoretical_loss": 3.8853931406323383, + "tokens_seen": 536558592 + }, + { + "epoch": 1.07, + "learning_rate": 0.00042292878635907723, + "loss": 3.0608, + "theoretical_loss": 3.8853426490756005, + "tokens_seen": 536624128 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004229187562688064, + "loss": 3.0681, + "theoretical_loss": 3.8852921654111796, + "tokens_seen": 536689664 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004229087261785356, + "loss": 3.1048, + "theoretical_loss": 3.8852416896368784, + "tokens_seen": 536755200 + }, + { + "epoch": 1.07, + "learning_rate": 0.00042289869608826483, + "loss": 2.8965, + "theoretical_loss": 3.8851912217505014, + "tokens_seen": 536820736 + }, + { + "epoch": 1.07, + "learning_rate": 0.00042288866599799396, + "loss": 3.0463, + "theoretical_loss": 3.885140761749852, + "tokens_seen": 536886272 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004228786359077232, + "loss": 2.8842, + "theoretical_loss": 3.8850903096327363, + "tokens_seen": 536951808 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004228686058174523, + "loss": 3.0815, + "theoretical_loss": 3.8850398653969607, + "tokens_seen": 537017344 + }, + { + "epoch": 1.07, + "learning_rate": 0.00042285857572718155, + "loss": 3.1825, + "theoretical_loss": 3.884989429040332, + "tokens_seen": 537082880 + }, + { + "epoch": 1.07, + "learning_rate": 0.00042284854563691074, + "loss": 2.6999, + "theoretical_loss": 3.8849390005606597, + "tokens_seen": 537148416 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004228385155466399, + "loss": 3.094, + "theoretical_loss": 3.884888579955751, + "tokens_seen": 537213952 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004228284854563691, + "loss": 2.8036, + "theoretical_loss": 3.8848381672234167, + "tokens_seen": 537279488 + }, + { + "epoch": 1.07, + "learning_rate": 0.00042281845536609833, + "loss": 3.1333, + "theoretical_loss": 3.8847877623614675, + "tokens_seen": 537345024 + }, + { + "epoch": 1.07, + "objective/train/docs_used": 617122, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5995001792907715, + "objective/train/theoretical_loss": 3.8847499638786287, + "objective/train/tokens_used": 557854176, + "theoretical_loss": 3.8847499638786287, + "tokens_seen": 537394176 + }, + { + "epoch": 1.07, + "learning_rate": 0.00042280842527582746, + "loss": 2.8701, + "theoretical_loss": 3.884737365367714, + "tokens_seen": 537410560 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004227983951855567, + "loss": 2.9849, + "theoretical_loss": 3.8846869762399705, + "tokens_seen": 537476096 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004227883650952858, + "loss": 3.0028, + "theoretical_loss": 3.8846365949760493, + "tokens_seen": 537541632 + }, + { + "epoch": 1.07, + "learning_rate": 0.00042277833500501506, + "loss": 2.748, + "theoretical_loss": 3.8845862215737643, + "tokens_seen": 537607168 + }, + { + "epoch": 1.07, + "learning_rate": 0.00042276830491474424, + "loss": 2.9641, + "theoretical_loss": 3.884535856030932, + "tokens_seen": 537672704 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004227582748244734, + "loss": 2.9048, + "theoretical_loss": 3.8844854983453665, + "tokens_seen": 537738240 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004227482447342026, + "loss": 2.9251, + "theoretical_loss": 3.884435148514886, + "tokens_seen": 537803776 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004227382146439318, + "loss": 2.9232, + "theoretical_loss": 3.884384806537308, + "tokens_seen": 537869312 + }, + { + "epoch": 1.07, + "learning_rate": 0.00042272818455366096, + "loss": 3.2003, + "theoretical_loss": 3.8843344724104507, + "tokens_seen": 537934848 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004227181544633902, + "loss": 3.0039, + "theoretical_loss": 3.884284146132134, + "tokens_seen": 538000384 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004227081243731193, + "loss": 2.8426, + "theoretical_loss": 3.8842338277001778, + "tokens_seen": 538065920 + }, + { + "epoch": 1.07, + "learning_rate": 0.00042269809428284856, + "loss": 2.8665, + "theoretical_loss": 3.884183517112404, + "tokens_seen": 538131456 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004226880641925777, + "loss": 2.8744, + "theoretical_loss": 3.884133214366634, + "tokens_seen": 538196992 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004226780341023069, + "loss": 3.132, + "theoretical_loss": 3.884082919460692, + "tokens_seen": 538262528 + }, + { + "epoch": 1.07, + "learning_rate": 0.00042266800401203616, + "loss": 2.7897, + "theoretical_loss": 3.8840326323923993, + "tokens_seen": 538328064 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004226579739217653, + "loss": 2.889, + "theoretical_loss": 3.883982353159583, + "tokens_seen": 538393600 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004226479438314945, + "loss": 2.858, + "theoretical_loss": 3.8839320817600678, + "tokens_seen": 538459136 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004226379137412237, + "loss": 2.9575, + "theoretical_loss": 3.8838818181916803, + "tokens_seen": 538524672 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004226278836509529, + "loss": 3.1264, + "theoretical_loss": 3.8838315624522473, + "tokens_seen": 538590208 + }, + { + "epoch": 1.07, + "learning_rate": 0.00042261785356068206, + "loss": 2.9071, + "theoretical_loss": 3.883781314539598, + "tokens_seen": 538655744 + }, + { + "epoch": 1.07, + "learning_rate": 0.00042260782347041125, + "loss": 2.7484, + "theoretical_loss": 3.8837310744515605, + "tokens_seen": 538721280 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004225977933801404, + "loss": 2.934, + "theoretical_loss": 3.8836808421859645, + "tokens_seen": 538786816 + }, + { + "epoch": 1.07, + "learning_rate": 0.00042258776328986966, + "loss": 3.0486, + "theoretical_loss": 3.883630617740642, + "tokens_seen": 538852352 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004225777331995988, + "loss": 2.9116, + "theoretical_loss": 3.883580401113423, + "tokens_seen": 538917888 + }, + { + "epoch": 1.07, + "learning_rate": 0.000422567703109328, + "loss": 2.8901, + "theoretical_loss": 3.8835301923021417, + "tokens_seen": 538983424 + }, + { + "epoch": 1.07, + "objective/train/docs_used": 618589, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.2161307334899902, + "objective/train/theoretical_loss": 3.8834925408215852, + "objective/train/tokens_used": 559492576, + "theoretical_loss": 3.8834925408215852, + "tokens_seen": 539032576 + }, + { + "epoch": 1.07, + "learning_rate": 0.00042255767301905715, + "loss": 2.9765, + "theoretical_loss": 3.8834799913046303, + "tokens_seen": 539048960 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004225476429287864, + "loss": 3.1059, + "theoretical_loss": 3.883429798118723, + "tokens_seen": 539114496 + }, + { + "epoch": 1.07, + "learning_rate": 0.00042253761283851557, + "loss": 2.8609, + "theoretical_loss": 3.8833796127422557, + "tokens_seen": 539180032 + }, + { + "epoch": 1.07, + "learning_rate": 0.00042252758274824475, + "loss": 3.0228, + "theoretical_loss": 3.8833294351730636, + "tokens_seen": 539245568 + }, + { + "epoch": 1.07, + "learning_rate": 0.00042251755265797393, + "loss": 2.9747, + "theoretical_loss": 3.8832792654089845, + "tokens_seen": 539311104 + }, + { + "epoch": 1.07, + "learning_rate": 0.00042250752256770317, + "loss": 3.0639, + "theoretical_loss": 3.8832291034478548, + "tokens_seen": 539376640 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004224974924774323, + "loss": 3.0354, + "theoretical_loss": 3.8831789492875135, + "tokens_seen": 539442176 + }, + { + "epoch": 1.07, + "learning_rate": 0.00042248746238716153, + "loss": 2.953, + "theoretical_loss": 3.8831288029258006, + "tokens_seen": 539507712 + }, + { + "epoch": 1.07, + "learning_rate": 0.00042247743229689065, + "loss": 2.6457, + "theoretical_loss": 3.8830786643605553, + "tokens_seen": 539573248 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004224674022066199, + "loss": 3.2025, + "theoretical_loss": 3.88302853358962, + "tokens_seen": 539638784 + }, + { + "epoch": 1.07, + "learning_rate": 0.00042245737211634907, + "loss": 2.9132, + "theoretical_loss": 3.8829784106108356, + "tokens_seen": 539704320 + }, + { + "epoch": 1.07, + "learning_rate": 0.00042244734202607825, + "loss": 2.6975, + "theoretical_loss": 3.882928295422045, + "tokens_seen": 539769856 + }, + { + "epoch": 1.07, + "learning_rate": 0.00042243731193580743, + "loss": 2.6874, + "theoretical_loss": 3.8828781880210927, + "tokens_seen": 539835392 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004224272818455366, + "loss": 2.9888, + "theoretical_loss": 3.882828088405823, + "tokens_seen": 539900928 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004224172517552658, + "loss": 2.7301, + "theoretical_loss": 3.8827779965740805, + "tokens_seen": 539966464 + }, + { + "epoch": 1.07, + "learning_rate": 0.00042240722166499503, + "loss": 2.7876, + "theoretical_loss": 3.882727912523712, + "tokens_seen": 540032000 + }, + { + "epoch": 1.07, + "learning_rate": 0.00042239719157472416, + "loss": 2.7523, + "theoretical_loss": 3.882677836252565, + "tokens_seen": 540097536 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004223871614844534, + "loss": 3.0484, + "theoretical_loss": 3.8826277677584873, + "tokens_seen": 540163072 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004223771313941825, + "loss": 3.0711, + "theoretical_loss": 3.882577707039328, + "tokens_seen": 540228608 + }, + { + "epoch": 1.07, + "learning_rate": 0.00042236710130391176, + "loss": 2.949, + "theoretical_loss": 3.8825276540929354, + "tokens_seen": 540294144 + }, + { + "epoch": 1.07, + "learning_rate": 0.00042235707121364094, + "loss": 2.7796, + "theoretical_loss": 3.8824776089171618, + "tokens_seen": 540359680 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004223470411233701, + "loss": 2.7581, + "theoretical_loss": 3.8824275715098575, + "tokens_seen": 540425216 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004223370110330993, + "loss": 2.9913, + "theoretical_loss": 3.882377541868875, + "tokens_seen": 540490752 + }, + { + "epoch": 1.07, + "learning_rate": 0.00042232698094282853, + "loss": 3.1344, + "theoretical_loss": 3.882327519992068, + "tokens_seen": 540556288 + }, + { + "epoch": 1.07, + "learning_rate": 0.00042231695085255766, + "loss": 2.8887, + "theoretical_loss": 3.88227750587729, + "tokens_seen": 540621824 + }, + { + "epoch": 1.07, + "objective/train/docs_used": 619158, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8451883792877197, + "objective/train/theoretical_loss": 3.8822400003837476, + "objective/train/tokens_used": 561130976, + "theoretical_loss": 3.8822400003837476, + "tokens_seen": 540670976 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004223069207622869, + "loss": 2.9941, + "theoretical_loss": 3.8822274995223958, + "tokens_seen": 540687360 + }, + { + "epoch": 1.07, + "learning_rate": 0.000422296890672016, + "loss": 2.7622, + "theoretical_loss": 3.8821775009252413, + "tokens_seen": 540752896 + }, + { + "epoch": 1.07, + "learning_rate": 0.00042228686058174526, + "loss": 2.8625, + "theoretical_loss": 3.8821275100836825, + "tokens_seen": 540818432 + }, + { + "epoch": 1.07, + "learning_rate": 0.00042227683049147444, + "loss": 2.6665, + "theoretical_loss": 3.8820775269955776, + "tokens_seen": 540883968 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004222668004012036, + "loss": 2.8965, + "theoretical_loss": 3.8820275516587843, + "tokens_seen": 540949504 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004222567703109328, + "loss": 2.7812, + "theoretical_loss": 3.8819775840711612, + "tokens_seen": 541015040 + }, + { + "epoch": 1.07, + "learning_rate": 0.000422246740220662, + "loss": 2.7666, + "theoretical_loss": 3.8819276242305696, + "tokens_seen": 541080576 + }, + { + "epoch": 1.07, + "learning_rate": 0.00042223671013039116, + "loss": 2.5857, + "theoretical_loss": 3.881877672134869, + "tokens_seen": 541146112 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004222266800401204, + "loss": 2.9337, + "theoretical_loss": 3.8818277277819213, + "tokens_seen": 541211648 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004222166499498495, + "loss": 2.8628, + "theoretical_loss": 3.88177779116959, + "tokens_seen": 541277184 + }, + { + "epoch": 1.07, + "learning_rate": 0.00042220661985957876, + "loss": 2.5178, + "theoretical_loss": 3.881727862295737, + "tokens_seen": 541342720 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004221965897693079, + "loss": 2.8533, + "theoretical_loss": 3.8816779411582276, + "tokens_seen": 541408256 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004221865596790371, + "loss": 2.8706, + "theoretical_loss": 3.881628027754926, + "tokens_seen": 541473792 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004221765295887663, + "loss": 2.9555, + "theoretical_loss": 3.8815781220836985, + "tokens_seen": 541539328 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004221664994984955, + "loss": 2.957, + "theoretical_loss": 3.8815282241424116, + "tokens_seen": 541604864 + }, + { + "epoch": 1.07, + "learning_rate": 0.00042215646940822467, + "loss": 2.5968, + "theoretical_loss": 3.881478333928933, + "tokens_seen": 541670400 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004221464393179539, + "loss": 2.8318, + "theoretical_loss": 3.8814284514411312, + "tokens_seen": 541735936 + }, + { + "epoch": 1.07, + "learning_rate": 0.00042213640922768303, + "loss": 2.748, + "theoretical_loss": 3.881378576676876, + "tokens_seen": 541801472 + }, + { + "epoch": 1.07, + "learning_rate": 0.00042212637913741226, + "loss": 2.952, + "theoretical_loss": 3.8813287096340363, + "tokens_seen": 541867008 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004221163490471414, + "loss": 2.8788, + "theoretical_loss": 3.881278850310484, + "tokens_seen": 541932544 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004221063189568706, + "loss": 3.0883, + "theoretical_loss": 3.88122899870409, + "tokens_seen": 541998080 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004220962888665998, + "loss": 2.8509, + "theoretical_loss": 3.8811791548127275, + "tokens_seen": 542063616 + }, + { + "epoch": 1.07, + "learning_rate": 0.000422086258776329, + "loss": 2.943, + "theoretical_loss": 3.8811293186342706, + "tokens_seen": 542129152 + }, + { + "epoch": 1.07, + "learning_rate": 0.00042207622868605817, + "loss": 2.9274, + "theoretical_loss": 3.881079490166593, + "tokens_seen": 542194688 + }, + { + "epoch": 1.07, + "learning_rate": 0.00042206619859578735, + "loss": 3.0352, + "theoretical_loss": 3.881029669407569, + "tokens_seen": 542260224 + }, + { + "epoch": 1.07, + "objective/train/docs_used": 620361, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.204775810241699, + "objective/train/theoretical_loss": 3.8809923088958285, + "objective/train/tokens_used": 562769376, + "theoretical_loss": 3.8809923088958285, + "tokens_seen": 542309376 + }, + { + "epoch": 1.07, + "learning_rate": 0.00042205616850551653, + "loss": 3.0893, + "theoretical_loss": 3.8809798563550766, + "tokens_seen": 542325760 + }, + { + "epoch": 1.07, + "learning_rate": 0.00042204613841524577, + "loss": 3.1817, + "theoretical_loss": 3.880930051006991, + "tokens_seen": 542391296 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004220361083249749, + "loss": 2.8584, + "theoretical_loss": 3.88088025336119, + "tokens_seen": 542456832 + }, + { + "epoch": 1.07, + "learning_rate": 0.00042202607823470413, + "loss": 2.8222, + "theoretical_loss": 3.8808304634155535, + "tokens_seen": 542522368 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004220160481444333, + "loss": 2.9794, + "theoretical_loss": 3.8807806811679595, + "tokens_seen": 542587904 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004220060180541625, + "loss": 2.7943, + "theoretical_loss": 3.8807309066162885, + "tokens_seen": 542653440 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004219959879638917, + "loss": 3.0174, + "theoretical_loss": 3.8806811397584218, + "tokens_seen": 542718976 + }, + { + "epoch": 1.07, + "learning_rate": 0.00042198595787362085, + "loss": 3.0142, + "theoretical_loss": 3.8806313805922414, + "tokens_seen": 542784512 + }, + { + "epoch": 1.07, + "learning_rate": 0.00042197592778335004, + "loss": 2.6437, + "theoretical_loss": 3.88058162911563, + "tokens_seen": 542850048 + }, + { + "epoch": 1.07, + "learning_rate": 0.00042196589769307927, + "loss": 3.1293, + "theoretical_loss": 3.8805318853264708, + "tokens_seen": 542915584 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004219558676028084, + "loss": 3.0661, + "theoretical_loss": 3.8804821492226482, + "tokens_seen": 542981120 + }, + { + "epoch": 1.07, + "learning_rate": 0.00042194583751253763, + "loss": 2.8919, + "theoretical_loss": 3.880432420802048, + "tokens_seen": 543046656 + }, + { + "epoch": 1.07, + "learning_rate": 0.00042193580742226676, + "loss": 2.8511, + "theoretical_loss": 3.880382700062556, + "tokens_seen": 543112192 + }, + { + "epoch": 1.07, + "learning_rate": 0.000421925777331996, + "loss": 2.9383, + "theoretical_loss": 3.880332987002059, + "tokens_seen": 543177728 + }, + { + "epoch": 1.07, + "learning_rate": 0.00042191574724172523, + "loss": 2.8971, + "theoretical_loss": 3.8802832816184445, + "tokens_seen": 543243264 + }, + { + "epoch": 1.07, + "learning_rate": 0.00042190571715145436, + "loss": 2.8064, + "theoretical_loss": 3.880233583909602, + "tokens_seen": 543308800 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004218956870611836, + "loss": 2.9946, + "theoretical_loss": 3.8801838938734203, + "tokens_seen": 543374336 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004218856569709127, + "loss": 2.9893, + "theoretical_loss": 3.8801342115077895, + "tokens_seen": 543439872 + }, + { + "epoch": 1.07, + "learning_rate": 0.00042187562688064196, + "loss": 3.1243, + "theoretical_loss": 3.8800845368106014, + "tokens_seen": 543505408 + }, + { + "epoch": 1.07, + "learning_rate": 0.00042186559679037114, + "loss": 3.0212, + "theoretical_loss": 3.880034869779747, + "tokens_seen": 543570944 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004218555667001003, + "loss": 3.2732, + "theoretical_loss": 3.8799852104131194, + "tokens_seen": 543636480 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004218455366098295, + "loss": 2.9189, + "theoretical_loss": 3.879935558708613, + "tokens_seen": 543702016 + }, + { + "epoch": 1.07, + "learning_rate": 0.00042183550651955873, + "loss": 3.024, + "theoretical_loss": 3.879885914664121, + "tokens_seen": 543767552 + }, + { + "epoch": 1.07, + "learning_rate": 0.00042182547642928786, + "loss": 3.0686, + "theoretical_loss": 3.8798362782775393, + "tokens_seen": 543833088 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004218154463390171, + "loss": 2.9981, + "theoretical_loss": 3.879786649546764, + "tokens_seen": 543898624 + }, + { + "epoch": 1.07, + "objective/train/docs_used": 620805, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7509706020355225, + "objective/train/theoretical_loss": 3.8797494330215407, + "objective/train/tokens_used": 564407776, + "theoretical_loss": 3.8797494330215407, + "tokens_seen": 543947776 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004218054162487462, + "loss": 2.807, + "theoretical_loss": 3.879737028469692, + "tokens_seen": 543964160 + }, + { + "epoch": 1.07, + "learning_rate": 0.00042179538615847546, + "loss": 2.9816, + "theoretical_loss": 3.8796874150442213, + "tokens_seen": 544029696 + }, + { + "epoch": 1.07, + "learning_rate": 0.00042178535606820464, + "loss": 2.8923, + "theoretical_loss": 3.8796378092682495, + "tokens_seen": 544095232 + }, + { + "epoch": 1.07, + "learning_rate": 0.0004217753259779338, + "loss": 2.6996, + "theoretical_loss": 3.8795882111396773, + "tokens_seen": 544160768 + }, + { + "epoch": 1.07, + "learning_rate": 0.000421765295887663, + "loss": 2.9286, + "theoretical_loss": 3.8795386206564038, + "tokens_seen": 544226304 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004217552657973922, + "loss": 2.9942, + "theoretical_loss": 3.8794890378163314, + "tokens_seen": 544291840 + }, + { + "epoch": 1.08, + "learning_rate": 0.00042174523570712136, + "loss": 3.0129, + "theoretical_loss": 3.879439462617361, + "tokens_seen": 544357376 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004217352056168506, + "loss": 2.9842, + "theoretical_loss": 3.8793898950573955, + "tokens_seen": 544422912 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004217251755265797, + "loss": 3.0301, + "theoretical_loss": 3.8793403351343385, + "tokens_seen": 544488448 + }, + { + "epoch": 1.08, + "learning_rate": 0.00042171514543630896, + "loss": 2.7943, + "theoretical_loss": 3.8792907828460947, + "tokens_seen": 544553984 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004217051153460381, + "loss": 2.812, + "theoretical_loss": 3.8792412381905685, + "tokens_seen": 544619520 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004216950852557673, + "loss": 2.8875, + "theoretical_loss": 3.879191701165667, + "tokens_seen": 544685056 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004216850551654965, + "loss": 2.8014, + "theoretical_loss": 3.8791421717692964, + "tokens_seen": 544750592 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004216750250752257, + "loss": 2.9776, + "theoretical_loss": 3.8790926499993645, + "tokens_seen": 544816128 + }, + { + "epoch": 1.08, + "learning_rate": 0.00042166499498495487, + "loss": 3.0308, + "theoretical_loss": 3.87904313585378, + "tokens_seen": 544881664 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004216549648946841, + "loss": 2.9902, + "theoretical_loss": 3.878993629330452, + "tokens_seen": 544947200 + }, + { + "epoch": 1.08, + "learning_rate": 0.00042164493480441323, + "loss": 2.8341, + "theoretical_loss": 3.878944130427291, + "tokens_seen": 545012736 + }, + { + "epoch": 1.08, + "learning_rate": 0.00042163490471414246, + "loss": 2.7959, + "theoretical_loss": 3.878894639142208, + "tokens_seen": 545078272 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004216248746238716, + "loss": 3.1664, + "theoretical_loss": 3.8788451554731145, + "tokens_seen": 545143808 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004216148445336008, + "loss": 2.8896, + "theoretical_loss": 3.878795679417923, + "tokens_seen": 545209344 + }, + { + "epoch": 1.08, + "learning_rate": 0.00042160481444333, + "loss": 3.0888, + "theoretical_loss": 3.8787462109745476, + "tokens_seen": 545274880 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004215947843530592, + "loss": 3.0511, + "theoretical_loss": 3.878696750140902, + "tokens_seen": 545340416 + }, + { + "epoch": 1.08, + "learning_rate": 0.00042158475426278837, + "loss": 2.7537, + "theoretical_loss": 3.8786472969149015, + "tokens_seen": 545405952 + }, + { + "epoch": 1.08, + "learning_rate": 0.00042157472417251755, + "loss": 2.9606, + "theoretical_loss": 3.8785978512944626, + "tokens_seen": 545471488 + }, + { + "epoch": 1.08, + "learning_rate": 0.00042156469408224673, + "loss": 2.8147, + "theoretical_loss": 3.878548413277501, + "tokens_seen": 545537024 + }, + { + "epoch": 1.08, + "objective/train/docs_used": 621470, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.958339214324951, + "objective/train/theoretical_loss": 3.8785113397533095, + "objective/train/tokens_used": 566046176, + "theoretical_loss": 3.8785113397533095, + "tokens_seen": 545586176 + }, + { + "epoch": 1.08, + "learning_rate": 0.00042155466399197597, + "loss": 2.9194, + "theoretical_loss": 3.878498982861935, + "tokens_seen": 545602560 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004215446339017051, + "loss": 2.8887, + "theoretical_loss": 3.8784495600456825, + "tokens_seen": 545668096 + }, + { + "epoch": 1.08, + "learning_rate": 0.00042153460381143433, + "loss": 3.0382, + "theoretical_loss": 3.8784001448266636, + "tokens_seen": 545733632 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004215245737211635, + "loss": 2.8951, + "theoretical_loss": 3.8783507372027977, + "tokens_seen": 545799168 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004215145436308927, + "loss": 2.7978, + "theoretical_loss": 3.8783013371720054, + "tokens_seen": 545864704 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004215045135406219, + "loss": 3.1933, + "theoretical_loss": 3.8782519447322086, + "tokens_seen": 545930240 + }, + { + "epoch": 1.08, + "learning_rate": 0.00042149448345035105, + "loss": 2.7269, + "theoretical_loss": 3.8782025598813297, + "tokens_seen": 545995776 + }, + { + "epoch": 1.08, + "learning_rate": 0.00042148445336008024, + "loss": 3.0344, + "theoretical_loss": 3.878153182617292, + "tokens_seen": 546061312 + }, + { + "epoch": 1.08, + "learning_rate": 0.00042147442326980947, + "loss": 2.9469, + "theoretical_loss": 3.8781038129380203, + "tokens_seen": 546126848 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004214643931795386, + "loss": 2.9681, + "theoretical_loss": 3.8780544508414394, + "tokens_seen": 546192384 + }, + { + "epoch": 1.08, + "learning_rate": 0.00042145436308926783, + "loss": 2.7063, + "theoretical_loss": 3.8780050963254746, + "tokens_seen": 546257920 + }, + { + "epoch": 1.08, + "learning_rate": 0.00042144433299899696, + "loss": 2.8749, + "theoretical_loss": 3.877955749388052, + "tokens_seen": 546323456 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004214343029087262, + "loss": 2.9719, + "theoretical_loss": 3.8779064100270997, + "tokens_seen": 546388992 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004214242728184554, + "loss": 2.941, + "theoretical_loss": 3.877857078240546, + "tokens_seen": 546454528 + }, + { + "epoch": 1.08, + "learning_rate": 0.00042141424272818456, + "loss": 3.0534, + "theoretical_loss": 3.8778077540263194, + "tokens_seen": 546520064 + }, + { + "epoch": 1.08, + "learning_rate": 0.00042140421263791374, + "loss": 2.7779, + "theoretical_loss": 3.877758437382351, + "tokens_seen": 546585600 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004213941825476429, + "loss": 2.7718, + "theoretical_loss": 3.877709128306569, + "tokens_seen": 546651136 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004213841524573721, + "loss": 2.7369, + "theoretical_loss": 3.8776598267969073, + "tokens_seen": 546716672 + }, + { + "epoch": 1.08, + "learning_rate": 0.00042137412236710134, + "loss": 2.6951, + "theoretical_loss": 3.877610532851297, + "tokens_seen": 546782208 + }, + { + "epoch": 1.08, + "learning_rate": 0.00042136409227683046, + "loss": 3.0101, + "theoretical_loss": 3.8775612464676716, + "tokens_seen": 546847744 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004213540621865597, + "loss": 2.9433, + "theoretical_loss": 3.8775119676439647, + "tokens_seen": 546913280 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004213440320962889, + "loss": 2.6857, + "theoretical_loss": 3.877462696378111, + "tokens_seen": 546978816 + }, + { + "epoch": 1.08, + "learning_rate": 0.00042133400200601806, + "loss": 2.8557, + "theoretical_loss": 3.877413432668046, + "tokens_seen": 547044352 + }, + { + "epoch": 1.08, + "learning_rate": 0.00042132397191574724, + "loss": 3.0713, + "theoretical_loss": 3.8773641765117066, + "tokens_seen": 547109888 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004213139418254764, + "loss": 3.0662, + "theoretical_loss": 3.8773149279070296, + "tokens_seen": 547175424 + }, + { + "epoch": 1.08, + "objective/train/docs_used": 622904, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.055859327316284, + "objective/train/theoretical_loss": 3.8772779964080595, + "objective/train/tokens_used": 567684576, + "theoretical_loss": 3.8772779964080595, + "tokens_seen": 547224576 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004213039117352056, + "loss": 2.6809, + "theoretical_loss": 3.8772656868519526, + "tokens_seen": 547240960 + }, + { + "epoch": 1.08, + "learning_rate": 0.00042129388164493484, + "loss": 3.0167, + "theoretical_loss": 3.877216453344415, + "tokens_seen": 547306496 + }, + { + "epoch": 1.08, + "learning_rate": 0.00042128385155466397, + "loss": 2.8911, + "theoretical_loss": 3.8771672273823556, + "tokens_seen": 547372032 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004212738214643932, + "loss": 3.198, + "theoretical_loss": 3.8771180089637154, + "tokens_seen": 547437568 + }, + { + "epoch": 1.08, + "learning_rate": 0.00042126379137412233, + "loss": 2.8962, + "theoretical_loss": 3.8770687980864356, + "tokens_seen": 547503104 + }, + { + "epoch": 1.08, + "learning_rate": 0.00042125376128385156, + "loss": 3.0193, + "theoretical_loss": 3.8770195947484583, + "tokens_seen": 547568640 + }, + { + "epoch": 1.08, + "learning_rate": 0.00042124373119358075, + "loss": 2.8844, + "theoretical_loss": 3.8769703989477255, + "tokens_seen": 547634176 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004212337011033099, + "loss": 2.9321, + "theoretical_loss": 3.8769212106821813, + "tokens_seen": 547699712 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004212236710130391, + "loss": 2.9915, + "theoretical_loss": 3.8768720299497703, + "tokens_seen": 547765248 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004212136409227683, + "loss": 2.6956, + "theoretical_loss": 3.8768228567484377, + "tokens_seen": 547830784 + }, + { + "epoch": 1.08, + "learning_rate": 0.00042120361083249747, + "loss": 2.4523, + "theoretical_loss": 3.8767736910761297, + "tokens_seen": 547896320 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004211935807422267, + "loss": 3.2263, + "theoretical_loss": 3.8767245329307927, + "tokens_seen": 547961856 + }, + { + "epoch": 1.08, + "learning_rate": 0.00042118355065195583, + "loss": 2.8253, + "theoretical_loss": 3.8766753823103746, + "tokens_seen": 548027392 + }, + { + "epoch": 1.08, + "learning_rate": 0.00042117352056168507, + "loss": 2.6039, + "theoretical_loss": 3.8766262392128237, + "tokens_seen": 548092928 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004211634904714143, + "loss": 2.9644, + "theoretical_loss": 3.876577103636089, + "tokens_seen": 548158464 + }, + { + "epoch": 1.08, + "learning_rate": 0.00042115346038114343, + "loss": 2.9272, + "theoretical_loss": 3.8765279755781217, + "tokens_seen": 548224000 + }, + { + "epoch": 1.08, + "learning_rate": 0.00042114343029087266, + "loss": 2.8769, + "theoretical_loss": 3.876478855036872, + "tokens_seen": 548289536 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004211334002006018, + "loss": 3.018, + "theoretical_loss": 3.876429742010291, + "tokens_seen": 548355072 + }, + { + "epoch": 1.08, + "learning_rate": 0.00042112337011033103, + "loss": 3.134, + "theoretical_loss": 3.8763806364963314, + "tokens_seen": 548420608 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004211133400200602, + "loss": 2.9645, + "theoretical_loss": 3.876331538492947, + "tokens_seen": 548486144 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004211033099297894, + "loss": 2.9324, + "theoretical_loss": 3.8762824479980917, + "tokens_seen": 548551680 + }, + { + "epoch": 1.08, + "learning_rate": 0.00042109327983951857, + "loss": 2.7315, + "theoretical_loss": 3.8762333650097203, + "tokens_seen": 548617216 + }, + { + "epoch": 1.08, + "learning_rate": 0.00042108324974924775, + "loss": 2.7846, + "theoretical_loss": 3.8761842895257885, + "tokens_seen": 548682752 + }, + { + "epoch": 1.08, + "learning_rate": 0.00042107321965897693, + "loss": 2.798, + "theoretical_loss": 3.8761352215442524, + "tokens_seen": 548748288 + }, + { + "epoch": 1.08, + "learning_rate": 0.00042106318956870617, + "loss": 2.8967, + "theoretical_loss": 3.8760861610630695, + "tokens_seen": 548813824 + }, + { + "epoch": 1.08, + "objective/train/docs_used": 623446, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.478111743927002, + "objective/train/theoretical_loss": 3.876049370623061, + "objective/train/tokens_used": 569322976, + "theoretical_loss": 3.876049370623061, + "tokens_seen": 548862976 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004210531594784353, + "loss": 3.0132, + "theoretical_loss": 3.8760371080801983, + "tokens_seen": 548879360 + }, + { + "epoch": 1.08, + "learning_rate": 0.00042104312938816453, + "loss": 2.8714, + "theoretical_loss": 3.8759880625935974, + "tokens_seen": 548944896 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004210330992978937, + "loss": 3.1069, + "theoretical_loss": 3.8759390246012266, + "tokens_seen": 549010432 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004210230692076229, + "loss": 2.8727, + "theoretical_loss": 3.8758899941010463, + "tokens_seen": 549075968 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004210130391173521, + "loss": 2.7593, + "theoretical_loss": 3.8758409710910176, + "tokens_seen": 549141504 + }, + { + "epoch": 1.08, + "learning_rate": 0.00042100300902708125, + "loss": 2.9946, + "theoretical_loss": 3.8757919555691025, + "tokens_seen": 549207040 + }, + { + "epoch": 1.08, + "learning_rate": 0.00042099297893681044, + "loss": 2.9204, + "theoretical_loss": 3.875742947533264, + "tokens_seen": 549272576 + }, + { + "epoch": 1.08, + "learning_rate": 0.00042098294884653967, + "loss": 2.8915, + "theoretical_loss": 3.8756939469814666, + "tokens_seen": 549338112 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004209729187562688, + "loss": 2.9225, + "theoretical_loss": 3.8756449539116735, + "tokens_seen": 549403648 + }, + { + "epoch": 1.08, + "learning_rate": 0.00042096288866599803, + "loss": 3.0075, + "theoretical_loss": 3.8755959683218504, + "tokens_seen": 549469184 + }, + { + "epoch": 1.08, + "learning_rate": 0.00042095285857572716, + "loss": 2.6968, + "theoretical_loss": 3.8755469902099633, + "tokens_seen": 549534720 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004209428284854564, + "loss": 2.8839, + "theoretical_loss": 3.875498019573979, + "tokens_seen": 549600256 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004209327983951856, + "loss": 3.1297, + "theoretical_loss": 3.875449056411866, + "tokens_seen": 549665792 + }, + { + "epoch": 1.08, + "learning_rate": 0.00042092276830491476, + "loss": 2.8652, + "theoretical_loss": 3.875400100721592, + "tokens_seen": 549731328 + }, + { + "epoch": 1.08, + "learning_rate": 0.00042091273821464394, + "loss": 3.0584, + "theoretical_loss": 3.8753511525011257, + "tokens_seen": 549796864 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004209027081243731, + "loss": 2.7467, + "theoretical_loss": 3.875302211748439, + "tokens_seen": 549862400 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004208926780341023, + "loss": 3.218, + "theoretical_loss": 3.8752532784615004, + "tokens_seen": 549927936 + }, + { + "epoch": 1.08, + "learning_rate": 0.00042088264794383154, + "loss": 2.906, + "theoretical_loss": 3.8752043526382827, + "tokens_seen": 549993472 + }, + { + "epoch": 1.08, + "learning_rate": 0.00042087261785356066, + "loss": 2.7583, + "theoretical_loss": 3.875155434276759, + "tokens_seen": 550059008 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004208625877632899, + "loss": 2.6835, + "theoretical_loss": 3.8751065233749005, + "tokens_seen": 550124544 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004208525576730191, + "loss": 2.6696, + "theoretical_loss": 3.8750576199306837, + "tokens_seen": 550190080 + }, + { + "epoch": 1.08, + "learning_rate": 0.00042084252758274826, + "loss": 3.2374, + "theoretical_loss": 3.8750087239420807, + "tokens_seen": 550255616 + }, + { + "epoch": 1.08, + "learning_rate": 0.00042083249749247744, + "loss": 2.8243, + "theoretical_loss": 3.8749598354070693, + "tokens_seen": 550321152 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004208224674022066, + "loss": 2.954, + "theoretical_loss": 3.8749109543236253, + "tokens_seen": 550386688 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004208124373119358, + "loss": 2.8086, + "theoretical_loss": 3.8748620806897254, + "tokens_seen": 550452224 + }, + { + "epoch": 1.08, + "objective/train/docs_used": 624875, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.574239730834961, + "objective/train/theoretical_loss": 3.8748254303518475, + "objective/train/tokens_used": 570961376, + "theoretical_loss": 3.8748254303518475, + "tokens_seen": 550501376 + }, + { + "epoch": 1.08, + "learning_rate": 0.00042080240722166504, + "loss": 3.106, + "theoretical_loss": 3.874813214503348, + "tokens_seen": 550517760 + }, + { + "epoch": 1.08, + "learning_rate": 0.00042079237713139417, + "loss": 2.974, + "theoretical_loss": 3.8747643557624714, + "tokens_seen": 550583296 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004207823470411234, + "loss": 2.9065, + "theoretical_loss": 3.874715504465075, + "tokens_seen": 550648832 + }, + { + "epoch": 1.08, + "learning_rate": 0.00042077231695085253, + "loss": 2.9074, + "theoretical_loss": 3.87466666060914, + "tokens_seen": 550714368 + }, + { + "epoch": 1.08, + "learning_rate": 0.00042076228686058176, + "loss": 2.8743, + "theoretical_loss": 3.8746178241926468, + "tokens_seen": 550779904 + }, + { + "epoch": 1.08, + "learning_rate": 0.00042075225677031095, + "loss": 3.0312, + "theoretical_loss": 3.8745689952135773, + "tokens_seen": 550845440 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004207422266800401, + "loss": 2.8656, + "theoretical_loss": 3.8745201736699144, + "tokens_seen": 550910976 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004207321965897693, + "loss": 3.0798, + "theoretical_loss": 3.8744713595596414, + "tokens_seen": 550976512 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004207221664994985, + "loss": 3.0155, + "theoretical_loss": 3.874422552880743, + "tokens_seen": 551042048 + }, + { + "epoch": 1.08, + "learning_rate": 0.00042071213640922767, + "loss": 2.8655, + "theoretical_loss": 3.8743737536312035, + "tokens_seen": 551107584 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004207021063189569, + "loss": 2.895, + "theoretical_loss": 3.87432496180901, + "tokens_seen": 551173120 + }, + { + "epoch": 1.08, + "learning_rate": 0.00042069207622868603, + "loss": 2.878, + "theoretical_loss": 3.8742761774121472, + "tokens_seen": 551238656 + }, + { + "epoch": 1.08, + "learning_rate": 0.00042068204613841527, + "loss": 2.9071, + "theoretical_loss": 3.874227400438604, + "tokens_seen": 551304192 + }, + { + "epoch": 1.08, + "learning_rate": 0.00042067201604814445, + "loss": 2.8582, + "theoretical_loss": 3.8741786308863677, + "tokens_seen": 551369728 + }, + { + "epoch": 1.08, + "learning_rate": 0.00042066198595787363, + "loss": 2.8426, + "theoretical_loss": 3.8741298687534282, + "tokens_seen": 551435264 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004206519558676028, + "loss": 2.9456, + "theoretical_loss": 3.874081114037775, + "tokens_seen": 551500800 + }, + { + "epoch": 1.08, + "learning_rate": 0.000420641925777332, + "loss": 2.6779, + "theoretical_loss": 3.874032366737398, + "tokens_seen": 551566336 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004206318956870612, + "loss": 2.8382, + "theoretical_loss": 3.873983626850289, + "tokens_seen": 551631872 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004206218655967904, + "loss": 2.7917, + "theoretical_loss": 3.8739348943744396, + "tokens_seen": 551697408 + }, + { + "epoch": 1.08, + "learning_rate": 0.00042061183550651954, + "loss": 2.7561, + "theoretical_loss": 3.8738861693078435, + "tokens_seen": 551762944 + }, + { + "epoch": 1.08, + "learning_rate": 0.00042060180541624877, + "loss": 3.085, + "theoretical_loss": 3.8738374516484937, + "tokens_seen": 551828480 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004205917753259779, + "loss": 2.8885, + "theoretical_loss": 3.8737887413943852, + "tokens_seen": 551894016 + }, + { + "epoch": 1.08, + "learning_rate": 0.00042058174523570713, + "loss": 2.8237, + "theoretical_loss": 3.8737400385435126, + "tokens_seen": 551959552 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004205717151454363, + "loss": 3.0146, + "theoretical_loss": 3.8736913430938724, + "tokens_seen": 552025088 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004205616850551655, + "loss": 2.8244, + "theoretical_loss": 3.8736426550434615, + "tokens_seen": 552090624 + }, + { + "epoch": 1.08, + "objective/train/docs_used": 625640, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8349080085754395, + "objective/train/theoretical_loss": 3.8736061438601923, + "objective/train/tokens_used": 572599776, + "theoretical_loss": 3.8736061438601923, + "tokens_seen": 552139776 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004205516549648947, + "loss": 2.9482, + "theoretical_loss": 3.8735939743902765, + "tokens_seen": 552156160 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004205416248746239, + "loss": 2.7414, + "theoretical_loss": 3.873545301132317, + "tokens_seen": 552221696 + }, + { + "epoch": 1.08, + "learning_rate": 0.00042053159478435304, + "loss": 2.9545, + "theoretical_loss": 3.873496635267581, + "tokens_seen": 552287232 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004205215646940823, + "loss": 2.6246, + "theoretical_loss": 3.8734479767940693, + "tokens_seen": 552352768 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004205115346038114, + "loss": 2.9388, + "theoretical_loss": 3.8733993257097827, + "tokens_seen": 552418304 + }, + { + "epoch": 1.08, + "learning_rate": 0.00042050150451354064, + "loss": 3.0344, + "theoretical_loss": 3.8733506820127213, + "tokens_seen": 552483840 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004204914744232698, + "loss": 2.99, + "theoretical_loss": 3.8733020457008887, + "tokens_seen": 552549376 + }, + { + "epoch": 1.08, + "learning_rate": 0.000420481444332999, + "loss": 3.006, + "theoretical_loss": 3.8732534167722874, + "tokens_seen": 552614912 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004204714142427282, + "loss": 2.7293, + "theoretical_loss": 3.873204795224921, + "tokens_seen": 552680448 + }, + { + "epoch": 1.08, + "learning_rate": 0.00042046138415245736, + "loss": 2.6337, + "theoretical_loss": 3.873156181056795, + "tokens_seen": 552745984 + }, + { + "epoch": 1.08, + "learning_rate": 0.00042045135406218654, + "loss": 3.1811, + "theoretical_loss": 3.8731075742659136, + "tokens_seen": 552811520 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004204413239719158, + "loss": 3.0877, + "theoretical_loss": 3.8730589748502835, + "tokens_seen": 552877056 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004204312938816449, + "loss": 2.8474, + "theoretical_loss": 3.8730103828079114, + "tokens_seen": 552942592 + }, + { + "epoch": 1.08, + "learning_rate": 0.00042042126379137414, + "loss": 3.0734, + "theoretical_loss": 3.8729617981368047, + "tokens_seen": 553008128 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004204112337011033, + "loss": 3.238, + "theoretical_loss": 3.872913220834973, + "tokens_seen": 553073664 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004204012036108325, + "loss": 2.9764, + "theoretical_loss": 3.872864650900424, + "tokens_seen": 553139200 + }, + { + "epoch": 1.08, + "learning_rate": 0.00042039117352056174, + "loss": 2.9151, + "theoretical_loss": 3.8728160883311684, + "tokens_seen": 553204736 + }, + { + "epoch": 1.08, + "learning_rate": 0.00042038114343029086, + "loss": 3.0027, + "theoretical_loss": 3.8727675331252174, + "tokens_seen": 553270272 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004203711133400201, + "loss": 2.6286, + "theoretical_loss": 3.872718985280582, + "tokens_seen": 553335808 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004203610832497493, + "loss": 3.0266, + "theoretical_loss": 3.8726704447952747, + "tokens_seen": 553401344 + }, + { + "epoch": 1.08, + "learning_rate": 0.00042035105315947846, + "loss": 3.1303, + "theoretical_loss": 3.8726219116673084, + "tokens_seen": 553466880 + }, + { + "epoch": 1.08, + "learning_rate": 0.00042034102306920764, + "loss": 2.82, + "theoretical_loss": 3.8725733858946967, + "tokens_seen": 553532416 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004203309929789368, + "loss": 2.9129, + "theoretical_loss": 3.8725248674754553, + "tokens_seen": 553597952 + }, + { + "epoch": 1.08, + "learning_rate": 0.000420320962888666, + "loss": 2.728, + "theoretical_loss": 3.8724763564075984, + "tokens_seen": 553663488 + }, + { + "epoch": 1.08, + "learning_rate": 0.00042031093279839524, + "loss": 2.947, + "theoretical_loss": 3.8724278526891425, + "tokens_seen": 553729024 + }, + { + "epoch": 1.08, + "objective/train/docs_used": 626970, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.11159610748291, + "objective/train/theoretical_loss": 3.872391479722152, + "objective/train/tokens_used": 574238176, + "theoretical_loss": 3.872391479722152, + "tokens_seen": 553778176 + }, + { + "epoch": 1.08, + "learning_rate": 0.00042030090270812437, + "loss": 2.9388, + "theoretical_loss": 3.872379356318105, + "tokens_seen": 553794560 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004202908726178536, + "loss": 2.8643, + "theoretical_loss": 3.8723308672925025, + "tokens_seen": 553860096 + }, + { + "epoch": 1.08, + "learning_rate": 0.00042028084252758273, + "loss": 2.8689, + "theoretical_loss": 3.8722823856103554, + "tokens_seen": 553925632 + }, + { + "epoch": 1.08, + "learning_rate": 0.00042027081243731196, + "loss": 3.1274, + "theoretical_loss": 3.8722339112696815, + "tokens_seen": 553991168 + }, + { + "epoch": 1.08, + "learning_rate": 0.00042026078234704115, + "loss": 2.9725, + "theoretical_loss": 3.8721854442685, + "tokens_seen": 554056704 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004202507522567703, + "loss": 2.8734, + "theoretical_loss": 3.8721369846048344, + "tokens_seen": 554122240 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004202407221664995, + "loss": 2.7302, + "theoretical_loss": 3.8720885322767042, + "tokens_seen": 554187776 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004202306920762287, + "loss": 2.8491, + "theoretical_loss": 3.872040087282132, + "tokens_seen": 554253312 + }, + { + "epoch": 1.08, + "learning_rate": 0.00042022066198595787, + "loss": 2.9801, + "theoretical_loss": 3.8719916496191407, + "tokens_seen": 554318848 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004202106318956871, + "loss": 2.8109, + "theoretical_loss": 3.871943219285755, + "tokens_seen": 554384384 + }, + { + "epoch": 1.08, + "learning_rate": 0.00042020060180541623, + "loss": 3.0374, + "theoretical_loss": 3.8718947962799986, + "tokens_seen": 554449920 + }, + { + "epoch": 1.08, + "learning_rate": 0.00042019057171514547, + "loss": 2.6717, + "theoretical_loss": 3.871846380599897, + "tokens_seen": 554515456 + }, + { + "epoch": 1.08, + "learning_rate": 0.00042018054162487465, + "loss": 2.6651, + "theoretical_loss": 3.8717979722434777, + "tokens_seen": 554580992 + }, + { + "epoch": 1.08, + "learning_rate": 0.00042017051153460383, + "loss": 2.9567, + "theoretical_loss": 3.871749571208766, + "tokens_seen": 554646528 + }, + { + "epoch": 1.08, + "learning_rate": 0.000420160481444333, + "loss": 2.9258, + "theoretical_loss": 3.87170117749379, + "tokens_seen": 554712064 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004201504513540622, + "loss": 2.8143, + "theoretical_loss": 3.8716527910965786, + "tokens_seen": 554777600 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004201404212637914, + "loss": 3.085, + "theoretical_loss": 3.8716044120151603, + "tokens_seen": 554843136 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004201303911735206, + "loss": 2.8709, + "theoretical_loss": 3.8715560402475657, + "tokens_seen": 554908672 + }, + { + "epoch": 1.08, + "learning_rate": 0.00042012036108324974, + "loss": 3.0459, + "theoretical_loss": 3.8715076757918254, + "tokens_seen": 554974208 + }, + { + "epoch": 1.08, + "learning_rate": 0.00042011033099297897, + "loss": 3.0664, + "theoretical_loss": 3.8714593186459703, + "tokens_seen": 555039744 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004201003009027081, + "loss": 2.8809, + "theoretical_loss": 3.871410968808034, + "tokens_seen": 555105280 + }, + { + "epoch": 1.08, + "learning_rate": 0.00042009027081243733, + "loss": 2.9343, + "theoretical_loss": 3.8713626262760474, + "tokens_seen": 555170816 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004200802407221665, + "loss": 2.6519, + "theoretical_loss": 3.871314291048046, + "tokens_seen": 555236352 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004200702106318957, + "loss": 2.856, + "theoretical_loss": 3.871265963122064, + "tokens_seen": 555301888 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004200601805416249, + "loss": 2.5503, + "theoretical_loss": 3.8712176424961373, + "tokens_seen": 555367424 + }, + { + "epoch": 1.08, + "objective/train/docs_used": 627698, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.411449432373047, + "objective/train/theoretical_loss": 3.871181406816171, + "objective/train/tokens_used": 575876576, + "theoretical_loss": 3.871181406816171, + "tokens_seen": 555416576 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004200501504513541, + "loss": 3.0096, + "theoretical_loss": 3.8711693291683003, + "tokens_seen": 555432960 + }, + { + "epoch": 1.08, + "learning_rate": 0.00042004012036108324, + "loss": 3.0007, + "theoretical_loss": 3.871121023136591, + "tokens_seen": 555498496 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004200300902708125, + "loss": 2.7156, + "theoretical_loss": 3.871072724399047, + "tokens_seen": 555564032 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004200200601805416, + "loss": 2.8436, + "theoretical_loss": 3.871024432953706, + "tokens_seen": 555629568 + }, + { + "epoch": 1.08, + "learning_rate": 0.00042001003009027084, + "loss": 2.6881, + "theoretical_loss": 3.870976148798608, + "tokens_seen": 555695104 + }, + { + "epoch": 1.08, + "learning_rate": 0.00042, + "loss": 3.0327, + "theoretical_loss": 3.8709278719317917, + "tokens_seen": 555760640 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004199899699097292, + "loss": 2.7813, + "theoretical_loss": 3.8708796023512986, + "tokens_seen": 555826176 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004199799398194584, + "loss": 2.7957, + "theoretical_loss": 3.8708313400551697, + "tokens_seen": 555891712 + }, + { + "epoch": 1.08, + "learning_rate": 0.00041996990972918756, + "loss": 2.955, + "theoretical_loss": 3.8707830850414475, + "tokens_seen": 555957248 + }, + { + "epoch": 1.08, + "learning_rate": 0.00041995987963891674, + "loss": 2.7701, + "theoretical_loss": 3.8707348373081745, + "tokens_seen": 556022784 + }, + { + "epoch": 1.08, + "learning_rate": 0.000419949849548646, + "loss": 2.7127, + "theoretical_loss": 3.870686596853395, + "tokens_seen": 556088320 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004199398194583751, + "loss": 2.9101, + "theoretical_loss": 3.870638363675152, + "tokens_seen": 556153856 + }, + { + "epoch": 1.08, + "learning_rate": 0.00041992978936810434, + "loss": 3.0043, + "theoretical_loss": 3.870590137771492, + "tokens_seen": 556219392 + }, + { + "epoch": 1.08, + "learning_rate": 0.00041991975927783347, + "loss": 2.8558, + "theoretical_loss": 3.870541919140461, + "tokens_seen": 556284928 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004199097291875627, + "loss": 2.9357, + "theoretical_loss": 3.8704937077801045, + "tokens_seen": 556350464 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004198996990972919, + "loss": 3.0304, + "theoretical_loss": 3.8704455036884706, + "tokens_seen": 556416000 + }, + { + "epoch": 1.08, + "learning_rate": 0.00041988966900702106, + "loss": 2.5689, + "theoretical_loss": 3.8703973068636075, + "tokens_seen": 556481536 + }, + { + "epoch": 1.08, + "learning_rate": 0.00041987963891675025, + "loss": 2.8771, + "theoretical_loss": 3.870349117303564, + "tokens_seen": 556547072 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004198696088264795, + "loss": 2.7484, + "theoretical_loss": 3.87030093500639, + "tokens_seen": 556612608 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004198595787362086, + "loss": 2.9133, + "theoretical_loss": 3.8702527599701355, + "tokens_seen": 556678144 + }, + { + "epoch": 1.08, + "learning_rate": 0.00041984954864593784, + "loss": 2.7249, + "theoretical_loss": 3.8702045921928523, + "tokens_seen": 556743680 + }, + { + "epoch": 1.08, + "learning_rate": 0.00041983951855566697, + "loss": 2.9768, + "theoretical_loss": 3.8701564316725916, + "tokens_seen": 556809216 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004198294884653962, + "loss": 2.7145, + "theoretical_loss": 3.8701082784074075, + "tokens_seen": 556874752 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004198194583751254, + "loss": 3.0215, + "theoretical_loss": 3.870060132395351, + "tokens_seen": 556940288 + }, + { + "epoch": 1.08, + "learning_rate": 0.00041980942828485457, + "loss": 2.9044, + "theoretical_loss": 3.8700119936344786, + "tokens_seen": 557005824 + }, + { + "epoch": 1.08, + "objective/train/docs_used": 628817, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7914021015167236, + "objective/train/theoretical_loss": 3.8699758943212434, + "objective/train/tokens_used": 577514976, + "theoretical_loss": 3.8699758943212434, + "tokens_seen": 557054976 + }, + { + "epoch": 1.08, + "learning_rate": 0.00041979939819458375, + "loss": 2.962, + "theoretical_loss": 3.8699638621228445, + "tokens_seen": 557071360 + }, + { + "epoch": 1.08, + "learning_rate": 0.00041978936810431293, + "loss": 2.7353, + "theoretical_loss": 3.8699157378585043, + "tokens_seen": 557136896 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004197793380140421, + "loss": 2.9422, + "theoretical_loss": 3.869867620839514, + "tokens_seen": 557202432 + }, + { + "epoch": 1.08, + "learning_rate": 0.00041976930792377135, + "loss": 3.1109, + "theoretical_loss": 3.8698195110639317, + "tokens_seen": 557267968 + }, + { + "epoch": 1.08, + "learning_rate": 0.00041975927783350047, + "loss": 2.8179, + "theoretical_loss": 3.8697714085298145, + "tokens_seen": 557333504 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004197492477432297, + "loss": 2.9445, + "theoretical_loss": 3.8697233132352222, + "tokens_seen": 557399040 + }, + { + "epoch": 1.08, + "learning_rate": 0.00041973921765295883, + "loss": 2.9836, + "theoretical_loss": 3.869675225178213, + "tokens_seen": 557464576 + }, + { + "epoch": 1.08, + "learning_rate": 0.00041972918756268807, + "loss": 2.9379, + "theoretical_loss": 3.8696271443568477, + "tokens_seen": 557530112 + }, + { + "epoch": 1.08, + "learning_rate": 0.00041971915747241725, + "loss": 2.803, + "theoretical_loss": 3.869579070769187, + "tokens_seen": 557595648 + }, + { + "epoch": 1.08, + "learning_rate": 0.00041970912738214643, + "loss": 3.052, + "theoretical_loss": 3.8695310044132927, + "tokens_seen": 557661184 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004196990972918756, + "loss": 2.9134, + "theoretical_loss": 3.8694829452872277, + "tokens_seen": 557726720 + }, + { + "epoch": 1.08, + "learning_rate": 0.00041968906720160485, + "loss": 2.7973, + "theoretical_loss": 3.8694348933890543, + "tokens_seen": 557792256 + }, + { + "epoch": 1.08, + "learning_rate": 0.000419679037111334, + "loss": 3.0926, + "theoretical_loss": 3.8693868487168372, + "tokens_seen": 557857792 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004196690070210632, + "loss": 3.0218, + "theoretical_loss": 3.8693388112686407, + "tokens_seen": 557923328 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004196589769307924, + "loss": 2.981, + "theoretical_loss": 3.8692907810425305, + "tokens_seen": 557988864 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004196489468405216, + "loss": 3.0689, + "theoretical_loss": 3.869242758036573, + "tokens_seen": 558054400 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004196389167502508, + "loss": 2.6322, + "theoretical_loss": 3.8691947422488333, + "tokens_seen": 558119936 + }, + { + "epoch": 1.08, + "learning_rate": 0.00041962888665997994, + "loss": 2.9541, + "theoretical_loss": 3.8691467336773817, + "tokens_seen": 558185472 + }, + { + "epoch": 1.08, + "learning_rate": 0.00041961885656970917, + "loss": 2.9968, + "theoretical_loss": 3.8690987323202846, + "tokens_seen": 558251008 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004196088264794383, + "loss": 3.0707, + "theoretical_loss": 3.8690507381756123, + "tokens_seen": 558316544 + }, + { + "epoch": 1.08, + "learning_rate": 0.00041959879638916753, + "loss": 2.7787, + "theoretical_loss": 3.8690027512414344, + "tokens_seen": 558382080 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004195887662988967, + "loss": 2.7311, + "theoretical_loss": 3.868954771515821, + "tokens_seen": 558447616 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004195787362086259, + "loss": 2.9757, + "theoretical_loss": 3.8689067989968438, + "tokens_seen": 558513152 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004195687061183551, + "loss": 3.019, + "theoretical_loss": 3.868858833682575, + "tokens_seen": 558578688 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004195586760280843, + "loss": 2.808, + "theoretical_loss": 3.868810875571088, + "tokens_seen": 558644224 + }, + { + "epoch": 1.08, + "objective/train/docs_used": 629246, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.687079429626465, + "objective/train/theoretical_loss": 3.8687749117131385, + "objective/train/tokens_used": 579153376, + "theoretical_loss": 3.8687749117131385, + "tokens_seen": 558693376 + }, + { + "epoch": 1.08, + "learning_rate": 0.00041954864593781344, + "loss": 2.8716, + "theoretical_loss": 3.868762924660455, + "tokens_seen": 558709760 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004195386158475427, + "loss": 2.9467, + "theoretical_loss": 3.8687149809487518, + "tokens_seen": 558775296 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004195285857572718, + "loss": 2.692, + "theoretical_loss": 3.8686670444340527, + "tokens_seen": 558840832 + }, + { + "epoch": 1.08, + "learning_rate": 0.00041951855566700104, + "loss": 2.7968, + "theoretical_loss": 3.8686191151144333, + "tokens_seen": 558906368 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004195085255767302, + "loss": 2.6929, + "theoretical_loss": 3.868571192987971, + "tokens_seen": 558971904 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004194984954864594, + "loss": 3.0387, + "theoretical_loss": 3.8685232780527423, + "tokens_seen": 559037440 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004194884653961886, + "loss": 2.7125, + "theoretical_loss": 3.868475370306826, + "tokens_seen": 559102976 + }, + { + "epoch": 1.08, + "learning_rate": 0.00041947843530591776, + "loss": 2.8212, + "theoretical_loss": 3.8684274697483003, + "tokens_seen": 559168512 + }, + { + "epoch": 1.08, + "learning_rate": 0.00041946840521564694, + "loss": 2.8875, + "theoretical_loss": 3.8683795763752444, + "tokens_seen": 559234048 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004194583751253762, + "loss": 3.2464, + "theoretical_loss": 3.8683316901857396, + "tokens_seen": 559299584 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004194483450351053, + "loss": 2.8747, + "theoretical_loss": 3.868283811177866, + "tokens_seen": 559365120 + }, + { + "epoch": 1.08, + "learning_rate": 0.00041943831494483454, + "loss": 2.6532, + "theoretical_loss": 3.868235939349706, + "tokens_seen": 559430656 + }, + { + "epoch": 1.08, + "learning_rate": 0.00041942828485456367, + "loss": 2.9353, + "theoretical_loss": 3.868188074699341, + "tokens_seen": 559496192 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004194182547642929, + "loss": 2.8988, + "theoretical_loss": 3.8681402172248553, + "tokens_seen": 559561728 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004194082246740221, + "loss": 2.664, + "theoretical_loss": 3.8680923669243326, + "tokens_seen": 559627264 + }, + { + "epoch": 1.08, + "learning_rate": 0.00041939819458375126, + "loss": 2.8466, + "theoretical_loss": 3.8680445237958567, + "tokens_seen": 559692800 + }, + { + "epoch": 1.08, + "learning_rate": 0.00041938816449348045, + "loss": 3.101, + "theoretical_loss": 3.8679966878375143, + "tokens_seen": 559758336 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004193781344032097, + "loss": 2.7125, + "theoretical_loss": 3.867948859047391, + "tokens_seen": 559823872 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004193681043129388, + "loss": 2.7661, + "theoretical_loss": 3.8679010374235734, + "tokens_seen": 559889408 + }, + { + "epoch": 1.08, + "learning_rate": 0.00041935807422266804, + "loss": 2.7567, + "theoretical_loss": 3.8678532229641496, + "tokens_seen": 559954944 + }, + { + "epoch": 1.08, + "learning_rate": 0.00041934804413239717, + "loss": 2.8698, + "theoretical_loss": 3.867805415667208, + "tokens_seen": 560020480 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004193380140421264, + "loss": 2.8603, + "theoretical_loss": 3.867757615530837, + "tokens_seen": 560086016 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004193279839518556, + "loss": 2.8624, + "theoretical_loss": 3.8677098225531266, + "tokens_seen": 560151552 + }, + { + "epoch": 1.08, + "learning_rate": 0.00041931795386158477, + "loss": 3.0192, + "theoretical_loss": 3.867662036732168, + "tokens_seen": 560217088 + }, + { + "epoch": 1.08, + "learning_rate": 0.00041930792377131395, + "loss": 2.552, + "theoretical_loss": 3.8676142580660517, + "tokens_seen": 560282624 + }, + { + "epoch": 1.08, + "objective/train/docs_used": 630481, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8848719596862793, + "objective/train/theoretical_loss": 3.867578428760682, + "objective/train/tokens_used": 580791776, + "theoretical_loss": 3.867578428760682, + "tokens_seen": 560331776 + }, + { + "epoch": 1.08, + "learning_rate": 0.00041929789368104313, + "loss": 2.9827, + "theoretical_loss": 3.8675664865528705, + "tokens_seen": 560348160 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004192878635907723, + "loss": 2.9074, + "theoretical_loss": 3.8675187221907166, + "tokens_seen": 560413696 + }, + { + "epoch": 1.08, + "learning_rate": 0.00041927783350050155, + "loss": 2.7092, + "theoretical_loss": 3.867470964977683, + "tokens_seen": 560479232 + }, + { + "epoch": 1.08, + "learning_rate": 0.00041926780341023067, + "loss": 2.9471, + "theoretical_loss": 3.867423214911865, + "tokens_seen": 560544768 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004192577733199599, + "loss": 2.9313, + "theoretical_loss": 3.867375471991357, + "tokens_seen": 560610304 + }, + { + "epoch": 1.08, + "learning_rate": 0.00041924774322968904, + "loss": 2.7035, + "theoretical_loss": 3.8673277362142544, + "tokens_seen": 560675840 + }, + { + "epoch": 1.08, + "learning_rate": 0.00041923771313941827, + "loss": 2.9026, + "theoretical_loss": 3.867280007578654, + "tokens_seen": 560741376 + }, + { + "epoch": 1.08, + "learning_rate": 0.00041922768304914745, + "loss": 2.7299, + "theoretical_loss": 3.867232286082653, + "tokens_seen": 560806912 + }, + { + "epoch": 1.08, + "learning_rate": 0.00041921765295887663, + "loss": 3.0076, + "theoretical_loss": 3.867184571724349, + "tokens_seen": 560872448 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004192076228686058, + "loss": 2.8349, + "theoretical_loss": 3.867136864501841, + "tokens_seen": 560937984 + }, + { + "epoch": 1.08, + "learning_rate": 0.00041919759277833505, + "loss": 3.0804, + "theoretical_loss": 3.867089164413228, + "tokens_seen": 561003520 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004191875626880642, + "loss": 2.8924, + "theoretical_loss": 3.8670414714566093, + "tokens_seen": 561069056 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004191775325977934, + "loss": 2.9753, + "theoretical_loss": 3.866993785630087, + "tokens_seen": 561134592 + }, + { + "epoch": 1.08, + "learning_rate": 0.00041916750250752254, + "loss": 2.901, + "theoretical_loss": 3.8669461069317617, + "tokens_seen": 561200128 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004191574724172518, + "loss": 2.9177, + "theoretical_loss": 3.866898435359736, + "tokens_seen": 561265664 + }, + { + "epoch": 1.08, + "learning_rate": 0.00041914744232698095, + "loss": 2.7428, + "theoretical_loss": 3.866850770912113, + "tokens_seen": 561331200 + }, + { + "epoch": 1.08, + "learning_rate": 0.00041913741223671014, + "loss": 2.6984, + "theoretical_loss": 3.8668031135869962, + "tokens_seen": 561396736 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004191273821464393, + "loss": 3.0854, + "theoretical_loss": 3.86675546338249, + "tokens_seen": 561462272 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004191173520561685, + "loss": 2.815, + "theoretical_loss": 3.866707820296699, + "tokens_seen": 561527808 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004191073219658977, + "loss": 2.8582, + "theoretical_loss": 3.8666601843277304, + "tokens_seen": 561593344 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004190972918756269, + "loss": 2.9834, + "theoretical_loss": 3.866612555473689, + "tokens_seen": 561658880 + }, + { + "epoch": 1.08, + "learning_rate": 0.00041908726178535604, + "loss": 2.9236, + "theoretical_loss": 3.8665649337326835, + "tokens_seen": 561724416 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004190772316950853, + "loss": 2.8714, + "theoretical_loss": 3.8665173191028215, + "tokens_seen": 561789952 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004190672016048144, + "loss": 2.7921, + "theoretical_loss": 3.8664697115822118, + "tokens_seen": 561855488 + }, + { + "epoch": 1.08, + "learning_rate": 0.00041905717151454364, + "loss": 2.8708, + "theoretical_loss": 3.866422111168964, + "tokens_seen": 561921024 + }, + { + "epoch": 1.08, + "objective/train/docs_used": 630917, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.474799871444702, + "objective/train/theoretical_loss": 3.866386415522097, + "objective/train/tokens_used": 582430176, + "theoretical_loss": 3.866386415522097, + "tokens_seen": 561970176 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004190471414242728, + "loss": 2.7912, + "theoretical_loss": 3.8663745178611872, + "tokens_seen": 561986560 + }, + { + "epoch": 1.08, + "learning_rate": 0.000419037111334002, + "loss": 2.9873, + "theoretical_loss": 3.866326931656994, + "tokens_seen": 562052096 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004190270812437312, + "loss": 2.8812, + "theoretical_loss": 3.8662793525544954, + "tokens_seen": 562117632 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004190170511534604, + "loss": 2.6993, + "theoretical_loss": 3.8662317805518036, + "tokens_seen": 562183168 + }, + { + "epoch": 1.08, + "learning_rate": 0.00041900702106318954, + "loss": 3.0518, + "theoretical_loss": 3.8661842156470314, + "tokens_seen": 562248704 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004189969909729188, + "loss": 2.944, + "theoretical_loss": 3.866136657838293, + "tokens_seen": 562314240 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004189869608826479, + "loss": 2.7422, + "theoretical_loss": 3.866089107123703, + "tokens_seen": 562379776 + }, + { + "epoch": 1.08, + "learning_rate": 0.00041897693079237714, + "loss": 2.8368, + "theoretical_loss": 3.8660415635013767, + "tokens_seen": 562445312 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004189669007021063, + "loss": 2.9691, + "theoretical_loss": 3.86599402696943, + "tokens_seen": 562510848 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004189568706118355, + "loss": 2.8235, + "theoretical_loss": 3.8659464975259787, + "tokens_seen": 562576384 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004189468405215647, + "loss": 2.9171, + "theoretical_loss": 3.865898975169142, + "tokens_seen": 562641920 + }, + { + "epoch": 1.08, + "learning_rate": 0.00041893681043129387, + "loss": 3.0027, + "theoretical_loss": 3.865851459897037, + "tokens_seen": 562707456 + }, + { + "epoch": 1.08, + "learning_rate": 0.00041892678034102305, + "loss": 2.9641, + "theoretical_loss": 3.865803951707782, + "tokens_seen": 562772992 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004189167502507523, + "loss": 2.9334, + "theoretical_loss": 3.8657564505994975, + "tokens_seen": 562838528 + }, + { + "epoch": 1.08, + "learning_rate": 0.00041890672016048146, + "loss": 2.9198, + "theoretical_loss": 3.8657089565703036, + "tokens_seen": 562904064 + }, + { + "epoch": 1.08, + "learning_rate": 0.00041889669007021065, + "loss": 3.1526, + "theoretical_loss": 3.865661469618321, + "tokens_seen": 562969600 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004188866599799399, + "loss": 3.0167, + "theoretical_loss": 3.8656139897416715, + "tokens_seen": 563035136 + }, + { + "epoch": 1.08, + "learning_rate": 0.000418876629889669, + "loss": 2.8326, + "theoretical_loss": 3.865566516938478, + "tokens_seen": 563100672 + }, + { + "epoch": 1.08, + "learning_rate": 0.00041886659979939824, + "loss": 2.9995, + "theoretical_loss": 3.8655190512068627, + "tokens_seen": 563166208 + }, + { + "epoch": 1.08, + "learning_rate": 0.00041885656970912737, + "loss": 2.9358, + "theoretical_loss": 3.8654715925449503, + "tokens_seen": 563231744 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004188465396188566, + "loss": 2.7895, + "theoretical_loss": 3.8654241409508656, + "tokens_seen": 563297280 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004188365095285858, + "loss": 2.9822, + "theoretical_loss": 3.865376696422733, + "tokens_seen": 563362816 + }, + { + "epoch": 1.08, + "learning_rate": 0.00041882647943831497, + "loss": 3.0189, + "theoretical_loss": 3.865329258958679, + "tokens_seen": 563428352 + }, + { + "epoch": 1.08, + "learning_rate": 0.00041881644934804415, + "loss": 2.7797, + "theoretical_loss": 3.8652818285568307, + "tokens_seen": 563493888 + }, + { + "epoch": 1.08, + "learning_rate": 0.00041880641925777333, + "loss": 2.861, + "theoretical_loss": 3.8652344052153147, + "tokens_seen": 563559424 + }, + { + "epoch": 1.08, + "objective/train/docs_used": 632338, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9370810985565186, + "objective/train/theoretical_loss": 3.865198842341395, + "objective/train/tokens_used": 584068576, + "theoretical_loss": 3.865198842341395, + "tokens_seen": 563608576 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004187963891675025, + "loss": 2.935, + "theoretical_loss": 3.86518698893226, + "tokens_seen": 563624960 + }, + { + "epoch": 1.08, + "learning_rate": 0.00041878635907723175, + "loss": 2.9378, + "theoretical_loss": 3.8651395797057946, + "tokens_seen": 563690496 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004187763289869609, + "loss": 2.7841, + "theoretical_loss": 3.8650921775340494, + "tokens_seen": 563756032 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004187662988966901, + "loss": 2.921, + "theoretical_loss": 3.8650447824151533, + "tokens_seen": 563821568 + }, + { + "epoch": 1.08, + "learning_rate": 0.00041875626880641924, + "loss": 2.959, + "theoretical_loss": 3.864997394347238, + "tokens_seen": 563887104 + }, + { + "epoch": 1.08, + "learning_rate": 0.00041874623871614847, + "loss": 2.6183, + "theoretical_loss": 3.864950013328435, + "tokens_seen": 563952640 + }, + { + "epoch": 1.08, + "learning_rate": 0.00041873620862587765, + "loss": 2.9683, + "theoretical_loss": 3.864902639356877, + "tokens_seen": 564018176 + }, + { + "epoch": 1.08, + "learning_rate": 0.00041872617853560683, + "loss": 3.0088, + "theoretical_loss": 3.864855272430697, + "tokens_seen": 564083712 + }, + { + "epoch": 1.08, + "learning_rate": 0.000418716148445336, + "loss": 3.0662, + "theoretical_loss": 3.864807912548029, + "tokens_seen": 564149248 + }, + { + "epoch": 1.08, + "learning_rate": 0.00041870611835506525, + "loss": 2.9283, + "theoretical_loss": 3.8647605597070074, + "tokens_seen": 564214784 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004186960882647944, + "loss": 2.8571, + "theoretical_loss": 3.864713213905768, + "tokens_seen": 564280320 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004186860581745236, + "loss": 2.82, + "theoretical_loss": 3.864665875142446, + "tokens_seen": 564345856 + }, + { + "epoch": 1.08, + "learning_rate": 0.00041867602808425274, + "loss": 3.119, + "theoretical_loss": 3.8646185434151787, + "tokens_seen": 564411392 + }, + { + "epoch": 1.08, + "learning_rate": 0.000418665997993982, + "loss": 3.0035, + "theoretical_loss": 3.864571218722103, + "tokens_seen": 564476928 + }, + { + "epoch": 1.08, + "learning_rate": 0.00041865596790371115, + "loss": 2.8305, + "theoretical_loss": 3.8645239010613572, + "tokens_seen": 564542464 + }, + { + "epoch": 1.08, + "learning_rate": 0.00041864593781344034, + "loss": 2.8717, + "theoretical_loss": 3.8644765904310803, + "tokens_seen": 564608000 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004186359077231695, + "loss": 2.7008, + "theoretical_loss": 3.864429286829412, + "tokens_seen": 564673536 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004186258776328987, + "loss": 2.8845, + "theoretical_loss": 3.8643819902544925, + "tokens_seen": 564739072 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004186158475426279, + "loss": 2.9494, + "theoretical_loss": 3.864334700704462, + "tokens_seen": 564804608 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004186058174523571, + "loss": 2.9526, + "theoretical_loss": 3.8642874181774634, + "tokens_seen": 564870144 + }, + { + "epoch": 1.08, + "learning_rate": 0.00041859578736208624, + "loss": 2.8049, + "theoretical_loss": 3.8642401426716377, + "tokens_seen": 564935680 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004185857572718155, + "loss": 2.703, + "theoretical_loss": 3.8641928741851292, + "tokens_seen": 565001216 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004185757271815446, + "loss": 3.0939, + "theoretical_loss": 3.864145612716081, + "tokens_seen": 565066752 + }, + { + "epoch": 1.08, + "learning_rate": 0.00041856569709127384, + "loss": 2.748, + "theoretical_loss": 3.8640983582626385, + "tokens_seen": 565132288 + }, + { + "epoch": 1.08, + "learning_rate": 0.000418555667001003, + "loss": 3.0962, + "theoretical_loss": 3.8640511108229454, + "tokens_seen": 565197824 + }, + { + "epoch": 1.08, + "objective/train/docs_used": 633117, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.0730414390563965, + "objective/train/theoretical_loss": 3.864015679844834, + "objective/train/tokens_used": 585706976, + "theoretical_loss": 3.864015679844834, + "tokens_seen": 565246976 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004185456369107322, + "loss": 3.0086, + "theoretical_loss": 3.8640038703951483, + "tokens_seen": 565263360 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004185356068204614, + "loss": 2.9326, + "theoretical_loss": 3.8639566369773943, + "tokens_seen": 565328896 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004185255767301906, + "loss": 2.6394, + "theoretical_loss": 3.86390941056783, + "tokens_seen": 565394432 + }, + { + "epoch": 1.08, + "learning_rate": 0.00041851554663991974, + "loss": 2.9686, + "theoretical_loss": 3.8638621911646043, + "tokens_seen": 565459968 + }, + { + "epoch": 1.08, + "learning_rate": 0.000418505516549649, + "loss": 2.9378, + "theoretical_loss": 3.863814978765865, + "tokens_seen": 565525504 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004184954864593781, + "loss": 2.7113, + "theoretical_loss": 3.8637677733697617, + "tokens_seen": 565591040 + }, + { + "epoch": 1.08, + "learning_rate": 0.00041848545636910734, + "loss": 2.9256, + "theoretical_loss": 3.8637205749744448, + "tokens_seen": 565656576 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004184754262788365, + "loss": 2.8471, + "theoretical_loss": 3.8636733835780652, + "tokens_seen": 565722112 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004184653961885657, + "loss": 2.896, + "theoretical_loss": 3.863626199178774, + "tokens_seen": 565787648 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004184553660982949, + "loss": 2.755, + "theoretical_loss": 3.863579021774724, + "tokens_seen": 565853184 + }, + { + "epoch": 1.08, + "learning_rate": 0.00041844533600802407, + "loss": 2.9088, + "theoretical_loss": 3.8635318513640677, + "tokens_seen": 565918720 + }, + { + "epoch": 1.08, + "learning_rate": 0.00041843530591775325, + "loss": 2.8796, + "theoretical_loss": 3.863484687944959, + "tokens_seen": 565984256 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004184252758274825, + "loss": 2.6829, + "theoretical_loss": 3.863437531515552, + "tokens_seen": 566049792 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004184152457372116, + "loss": 3.0297, + "theoretical_loss": 3.8633903820740016, + "tokens_seen": 566115328 + }, + { + "epoch": 1.08, + "learning_rate": 0.00041840521564694085, + "loss": 2.7752, + "theoretical_loss": 3.863343239618464, + "tokens_seen": 566180864 + }, + { + "epoch": 1.08, + "learning_rate": 0.00041839518555666997, + "loss": 3.0822, + "theoretical_loss": 3.863296104147096, + "tokens_seen": 566246400 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004183851554663992, + "loss": 2.9419, + "theoretical_loss": 3.8632489756580535, + "tokens_seen": 566311936 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004183751253761284, + "loss": 3.0283, + "theoretical_loss": 3.8632018541494952, + "tokens_seen": 566377472 + }, + { + "epoch": 1.08, + "learning_rate": 0.00041836509528585757, + "loss": 2.9068, + "theoretical_loss": 3.8631547396195796, + "tokens_seen": 566443008 + }, + { + "epoch": 1.08, + "learning_rate": 0.00041835506519558675, + "loss": 3.0821, + "theoretical_loss": 3.863107632066466, + "tokens_seen": 566508544 + }, + { + "epoch": 1.08, + "learning_rate": 0.000418345035105316, + "loss": 2.8347, + "theoretical_loss": 3.8630605314883137, + "tokens_seen": 566574080 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004183350050150451, + "loss": 2.8839, + "theoretical_loss": 3.863013437883284, + "tokens_seen": 566639616 + }, + { + "epoch": 1.08, + "learning_rate": 0.00041832497492477435, + "loss": 2.713, + "theoretical_loss": 3.862966351249538, + "tokens_seen": 566705152 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004183149448345035, + "loss": 2.9915, + "theoretical_loss": 3.8629192715852376, + "tokens_seen": 566770688 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004183049147442327, + "loss": 2.868, + "theoretical_loss": 3.862872198888546, + "tokens_seen": 566836224 + }, + { + "epoch": 1.08, + "objective/train/docs_used": 634194, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.527719020843506, + "objective/train/theoretical_loss": 3.8628368989374158, + "objective/train/tokens_used": 587345376, + "theoretical_loss": 3.8628368989374158, + "tokens_seen": 566885376 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004182948846539619, + "loss": 2.473, + "theoretical_loss": 3.862825133157626, + "tokens_seen": 566901760 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004182848545636911, + "loss": 2.9303, + "theoretical_loss": 3.8627780743906426, + "tokens_seen": 566967296 + }, + { + "epoch": 1.08, + "learning_rate": 0.00041827482447342025, + "loss": 2.9108, + "theoretical_loss": 3.862731022585759, + "tokens_seen": 567032832 + }, + { + "epoch": 1.08, + "learning_rate": 0.00041826479438314944, + "loss": 2.8899, + "theoretical_loss": 3.862683977741143, + "tokens_seen": 567098368 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004182547642928786, + "loss": 2.7323, + "theoretical_loss": 3.8626369398549585, + "tokens_seen": 567163904 + }, + { + "epoch": 1.08, + "learning_rate": 0.00041824473420260785, + "loss": 2.9033, + "theoretical_loss": 3.862589908925374, + "tokens_seen": 567229440 + }, + { + "epoch": 1.08, + "learning_rate": 0.000418234704112337, + "loss": 3.1807, + "theoretical_loss": 3.8625428849505563, + "tokens_seen": 567294976 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004182246740220662, + "loss": 2.795, + "theoretical_loss": 3.862495867928674, + "tokens_seen": 567360512 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004182146439317954, + "loss": 3.0679, + "theoretical_loss": 3.8624488578578964, + "tokens_seen": 567426048 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004182046138415246, + "loss": 2.8983, + "theoretical_loss": 3.8624018547363925, + "tokens_seen": 567491584 + }, + { + "epoch": 1.08, + "learning_rate": 0.00041819458375125376, + "loss": 2.9737, + "theoretical_loss": 3.862354858562333, + "tokens_seen": 567557120 + }, + { + "epoch": 1.08, + "learning_rate": 0.00041818455366098294, + "loss": 2.8067, + "theoretical_loss": 3.862307869333889, + "tokens_seen": 567622656 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004181745235707121, + "loss": 2.8586, + "theoretical_loss": 3.8622608870492323, + "tokens_seen": 567688192 + }, + { + "epoch": 1.08, + "learning_rate": 0.00041816449348044136, + "loss": 2.8927, + "theoretical_loss": 3.862213911706535, + "tokens_seen": 567753728 + }, + { + "epoch": 1.08, + "learning_rate": 0.00041815446339017054, + "loss": 2.7804, + "theoretical_loss": 3.862166943303971, + "tokens_seen": 567819264 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004181444332998997, + "loss": 2.6391, + "theoretical_loss": 3.862119981839713, + "tokens_seen": 567884800 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004181344032096289, + "loss": 2.6862, + "theoretical_loss": 3.8620730273119364, + "tokens_seen": 567950336 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004181243731193581, + "loss": 2.8688, + "theoretical_loss": 3.862026079718816, + "tokens_seen": 568015872 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004181143430290873, + "loss": 3.0498, + "theoretical_loss": 3.8619791390585285, + "tokens_seen": 568081408 + }, + { + "epoch": 1.08, + "learning_rate": 0.00041810431293881644, + "loss": 2.8153, + "theoretical_loss": 3.8619322053292495, + "tokens_seen": 568146944 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004180942828485457, + "loss": 2.723, + "theoretical_loss": 3.8618852785291566, + "tokens_seen": 568212480 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004180842527582748, + "loss": 2.8977, + "theoretical_loss": 3.8618383586564278, + "tokens_seen": 568278016 + }, + { + "epoch": 1.08, + "learning_rate": 0.00041807422266800404, + "loss": 2.8766, + "theoretical_loss": 3.861791445709242, + "tokens_seen": 568343552 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004180641925777332, + "loss": 2.8196, + "theoretical_loss": 3.8617445396857786, + "tokens_seen": 568409088 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004180541624874624, + "loss": 2.5105, + "theoretical_loss": 3.861697640584217, + "tokens_seen": 568474624 + }, + { + "epoch": 1.08, + "objective/train/docs_used": 634895, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.1014013290405273, + "objective/train/theoretical_loss": 3.8616624707994496, + "objective/train/tokens_used": 588983776, + "theoretical_loss": 3.8616624707994496, + "tokens_seen": 568523776 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004180441323971916, + "loss": 2.6694, + "theoretical_loss": 3.861650748402738, + "tokens_seen": 568540160 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004180341023069208, + "loss": 2.9858, + "theoretical_loss": 3.861603863139524, + "tokens_seen": 568605696 + }, + { + "epoch": 1.08, + "learning_rate": 0.00041802407221664994, + "loss": 2.4696, + "theoretical_loss": 3.861556984792756, + "tokens_seen": 568671232 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004180140421263792, + "loss": 2.7752, + "theoretical_loss": 3.861510113360618, + "tokens_seen": 568736768 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004180040120361083, + "loss": 2.6513, + "theoretical_loss": 3.861463248841292, + "tokens_seen": 568802304 + }, + { + "epoch": 1.08, + "learning_rate": 0.00041799398194583754, + "loss": 2.6531, + "theoretical_loss": 3.861416391232963, + "tokens_seen": 568867840 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004179839518555667, + "loss": 2.8723, + "theoretical_loss": 3.8613695405338158, + "tokens_seen": 568933376 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004179739217652959, + "loss": 2.6643, + "theoretical_loss": 3.861322696742036, + "tokens_seen": 568998912 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004179638916750251, + "loss": 2.8317, + "theoretical_loss": 3.8612758598558097, + "tokens_seen": 569064448 + }, + { + "epoch": 1.08, + "learning_rate": 0.00041795386158475427, + "loss": 2.9949, + "theoretical_loss": 3.8612290298733236, + "tokens_seen": 569129984 + }, + { + "epoch": 1.08, + "learning_rate": 0.00041794383149448345, + "loss": 2.7817, + "theoretical_loss": 3.8611822067927655, + "tokens_seen": 569195520 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004179338014042127, + "loss": 2.8349, + "theoretical_loss": 3.861135390612324, + "tokens_seen": 569261056 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004179237713139418, + "loss": 2.6289, + "theoretical_loss": 3.8610885813301876, + "tokens_seen": 569326592 + }, + { + "epoch": 1.08, + "learning_rate": 0.00041791374122367105, + "loss": 2.9583, + "theoretical_loss": 3.8610417789445464, + "tokens_seen": 569392128 + }, + { + "epoch": 1.08, + "learning_rate": 0.00041790371113340017, + "loss": 2.8696, + "theoretical_loss": 3.86099498345359, + "tokens_seen": 569457664 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004178936810431294, + "loss": 2.7377, + "theoretical_loss": 3.86094819485551, + "tokens_seen": 569523200 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004178836509528586, + "loss": 2.5355, + "theoretical_loss": 3.8609014131484978, + "tokens_seen": 569588736 + }, + { + "epoch": 1.08, + "learning_rate": 0.00041787362086258777, + "loss": 2.7731, + "theoretical_loss": 3.860854638330746, + "tokens_seen": 569654272 + }, + { + "epoch": 1.08, + "learning_rate": 0.00041786359077231695, + "loss": 2.8211, + "theoretical_loss": 3.860807870400447, + "tokens_seen": 569719808 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004178535606820462, + "loss": 2.9783, + "theoretical_loss": 3.8607611093557956, + "tokens_seen": 569785344 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004178435305917753, + "loss": 2.6345, + "theoretical_loss": 3.860714355194986, + "tokens_seen": 569850880 + }, + { + "epoch": 1.08, + "learning_rate": 0.00041783350050150455, + "loss": 2.885, + "theoretical_loss": 3.8606676079162128, + "tokens_seen": 569916416 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004178234704112337, + "loss": 3.0496, + "theoretical_loss": 3.860620867517672, + "tokens_seen": 569981952 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004178134403209629, + "loss": 2.9936, + "theoretical_loss": 3.86057413399756, + "tokens_seen": 570047488 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004178034102306921, + "loss": 2.5621, + "theoretical_loss": 3.8605274073540743, + "tokens_seen": 570113024 + }, + { + "epoch": 1.08, + "objective/train/docs_used": 635428, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.758199691772461, + "objective/train/theoretical_loss": 3.860492366883161, + "objective/train/tokens_used": 590622176, + "theoretical_loss": 3.860492366883161, + "tokens_seen": 570162176 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004177933801404213, + "loss": 2.7609, + "theoretical_loss": 3.8604806875854116, + "tokens_seen": 570178560 + }, + { + "epoch": 1.08, + "learning_rate": 0.00041778335005015045, + "loss": 2.6497, + "theoretical_loss": 3.8604339746897725, + "tokens_seen": 570244096 + }, + { + "epoch": 1.08, + "learning_rate": 0.00041777331995987964, + "loss": 2.9305, + "theoretical_loss": 3.860387268665354, + "tokens_seen": 570309632 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004177632898696088, + "loss": 2.5576, + "theoretical_loss": 3.860340569510357, + "tokens_seen": 570375168 + }, + { + "epoch": 1.08, + "learning_rate": 0.00041775325977933805, + "loss": 2.7111, + "theoretical_loss": 3.8602938772229827, + "tokens_seen": 570440704 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004177432296890672, + "loss": 2.9168, + "theoretical_loss": 3.8602471918014305, + "tokens_seen": 570506240 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004177331995987964, + "loss": 2.7029, + "theoretical_loss": 3.8602005132439037, + "tokens_seen": 570571776 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004177231695085256, + "loss": 2.7222, + "theoretical_loss": 3.8601538415486045, + "tokens_seen": 570637312 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004177131394182548, + "loss": 2.7994, + "theoretical_loss": 3.8601071767137363, + "tokens_seen": 570702848 + }, + { + "epoch": 1.08, + "learning_rate": 0.00041770310932798396, + "loss": 2.7614, + "theoretical_loss": 3.8600605187375026, + "tokens_seen": 570768384 + }, + { + "epoch": 1.08, + "learning_rate": 0.00041769307923771314, + "loss": 2.6008, + "theoretical_loss": 3.860013867618109, + "tokens_seen": 570833920 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004176830491474423, + "loss": 2.9842, + "theoretical_loss": 3.8599672233537596, + "tokens_seen": 570899456 + }, + { + "epoch": 1.08, + "learning_rate": 0.00041767301905717156, + "loss": 2.601, + "theoretical_loss": 3.8599205859426604, + "tokens_seen": 570964992 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004176629889669007, + "loss": 2.7662, + "theoretical_loss": 3.8598739553830193, + "tokens_seen": 571030528 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004176529588766299, + "loss": 2.9515, + "theoretical_loss": 3.859827331673042, + "tokens_seen": 571096064 + }, + { + "epoch": 1.08, + "learning_rate": 0.00041764292878635904, + "loss": 2.8044, + "theoretical_loss": 3.8597807148109378, + "tokens_seen": 571161600 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004176328986960883, + "loss": 2.6835, + "theoretical_loss": 3.8597341047949145, + "tokens_seen": 571227136 + }, + { + "epoch": 1.08, + "learning_rate": 0.00041762286860581746, + "loss": 2.8642, + "theoretical_loss": 3.8596875016231817, + "tokens_seen": 571292672 + }, + { + "epoch": 1.08, + "learning_rate": 0.00041761283851554664, + "loss": 2.9252, + "theoretical_loss": 3.8596409052939498, + "tokens_seen": 571358208 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004176028084252758, + "loss": 2.2926, + "theoretical_loss": 3.8595943158054284, + "tokens_seen": 571423744 + }, + { + "epoch": 1.08, + "learning_rate": 0.000417592778335005, + "loss": 2.3754, + "theoretical_loss": 3.85954773315583, + "tokens_seen": 571489280 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004175827482447342, + "loss": 3.0418, + "theoretical_loss": 3.8595011573433657, + "tokens_seen": 571554816 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004175727181544634, + "loss": 2.8298, + "theoretical_loss": 3.8594545883662494, + "tokens_seen": 571620352 + }, + { + "epoch": 1.08, + "learning_rate": 0.00041756268806419255, + "loss": 2.6487, + "theoretical_loss": 3.8594080262226935, + "tokens_seen": 571685888 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004175526579739218, + "loss": 2.6435, + "theoretical_loss": 3.859361470910912, + "tokens_seen": 571751424 + }, + { + "epoch": 1.08, + "objective/train/docs_used": 638980, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.1796209812164307, + "objective/train/theoretical_loss": 3.859326558909354, + "objective/train/tokens_used": 592260576, + "theoretical_loss": 3.859326558909354, + "tokens_seen": 571800576 + }, + { + "epoch": 1.08, + "learning_rate": 0.00041754262788365096, + "loss": 2.9137, + "theoretical_loss": 3.8593149224291197, + "tokens_seen": 571816960 + }, + { + "epoch": 1.08, + "learning_rate": 0.00041753259779338015, + "loss": 2.54, + "theoretical_loss": 3.8592683807755326, + "tokens_seen": 571882496 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004175225677031093, + "loss": 2.74, + "theoretical_loss": 3.8592218459483663, + "tokens_seen": 571948032 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004175125376128385, + "loss": 2.6262, + "theoretical_loss": 3.8591753179458372, + "tokens_seen": 572013568 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004175025075225677, + "loss": 2.8701, + "theoretical_loss": 3.859128796766163, + "tokens_seen": 572079104 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004174924774322969, + "loss": 2.6111, + "theoretical_loss": 3.859082282407562, + "tokens_seen": 572144640 + }, + { + "epoch": 1.08, + "learning_rate": 0.00041748244734202605, + "loss": 2.7199, + "theoretical_loss": 3.8590357748682527, + "tokens_seen": 572210176 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004174724172517553, + "loss": 2.4907, + "theoretical_loss": 3.858989274146454, + "tokens_seen": 572275712 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004174623871614844, + "loss": 2.8167, + "theoretical_loss": 3.858942780240387, + "tokens_seen": 572341248 + }, + { + "epoch": 1.08, + "learning_rate": 0.00041745235707121365, + "loss": 2.6303, + "theoretical_loss": 3.858896293148272, + "tokens_seen": 572406784 + }, + { + "epoch": 1.08, + "learning_rate": 0.00041744232698094283, + "loss": 3.005, + "theoretical_loss": 3.85884981286833, + "tokens_seen": 572472320 + }, + { + "epoch": 1.08, + "learning_rate": 0.000417432296890672, + "loss": 2.8463, + "theoretical_loss": 3.858803339398783, + "tokens_seen": 572537856 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004174222668004012, + "loss": 3.0947, + "theoretical_loss": 3.858756872737855, + "tokens_seen": 572603392 + }, + { + "epoch": 1.08, + "learning_rate": 0.00041741223671013037, + "loss": 3.0426, + "theoretical_loss": 3.8587104128837675, + "tokens_seen": 572668928 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004174022066198596, + "loss": 2.8641, + "theoretical_loss": 3.8586639598347463, + "tokens_seen": 572734464 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004173921765295888, + "loss": 2.8205, + "theoretical_loss": 3.8586175135890155, + "tokens_seen": 572800000 + }, + { + "epoch": 1.08, + "learning_rate": 0.00041738214643931797, + "loss": 2.9435, + "theoretical_loss": 3.8585710741448, + "tokens_seen": 572865536 + }, + { + "epoch": 1.08, + "learning_rate": 0.00041737211634904715, + "loss": 2.9395, + "theoretical_loss": 3.8585246415003267, + "tokens_seen": 572931072 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004173620862587764, + "loss": 2.7106, + "theoretical_loss": 3.8584782156538218, + "tokens_seen": 572996608 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004173520561685055, + "loss": 2.8422, + "theoretical_loss": 3.858431796603513, + "tokens_seen": 573062144 + }, + { + "epoch": 1.08, + "learning_rate": 0.00041734202607823475, + "loss": 2.4501, + "theoretical_loss": 3.8583853843476277, + "tokens_seen": 573127680 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004173319959879639, + "loss": 2.7858, + "theoretical_loss": 3.8583389788843956, + "tokens_seen": 573193216 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004173219658976931, + "loss": 2.5905, + "theoretical_loss": 3.858292580212045, + "tokens_seen": 573258752 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004173119358074223, + "loss": 2.659, + "theoretical_loss": 3.8582461883288075, + "tokens_seen": 573324288 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004173019057171515, + "loss": 2.6405, + "theoretical_loss": 3.858199803232913, + "tokens_seen": 573389824 + }, + { + "epoch": 1.08, + "objective/train/docs_used": 641212, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7385621070861816, + "objective/train/theoretical_loss": 3.8581650188641214, + "objective/train/tokens_used": 593504736, + "theoretical_loss": 3.8581650188641214, + "tokens_seen": 573438976 + }, + { + "epoch": 1.08, + "learning_rate": 0.00041729187562688065, + "loss": 2.822, + "theoretical_loss": 3.858153424922592, + "tokens_seen": 573455360 + }, + { + "epoch": 1.08, + "learning_rate": 0.00041728184553660984, + "loss": 2.5345, + "theoretical_loss": 3.858107053396078, + "tokens_seen": 573520896 + }, + { + "epoch": 1.08, + "learning_rate": 0.000417271815446339, + "loss": 2.6229, + "theoretical_loss": 3.858060688651603, + "tokens_seen": 573586432 + }, + { + "epoch": 1.08, + "learning_rate": 0.00041726178535606825, + "loss": 2.5662, + "theoretical_loss": 3.8580143306874, + "tokens_seen": 573651968 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004172517552657974, + "loss": 2.9459, + "theoretical_loss": 3.857967979501704, + "tokens_seen": 573717504 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004172417251755266, + "loss": 2.9108, + "theoretical_loss": 3.857921635092749, + "tokens_seen": 573783040 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004172316950852558, + "loss": 2.7796, + "theoretical_loss": 3.8578752974587704, + "tokens_seen": 573848576 + }, + { + "epoch": 1.08, + "learning_rate": 0.000417221664994985, + "loss": 2.9508, + "theoretical_loss": 3.857828966598005, + "tokens_seen": 573914112 + }, + { + "epoch": 1.08, + "learning_rate": 0.00041721163490471416, + "loss": 2.4994, + "theoretical_loss": 3.857782642508688, + "tokens_seen": 573979648 + }, + { + "epoch": 1.08, + "learning_rate": 0.00041720160481444334, + "loss": 2.8715, + "theoretical_loss": 3.857736325189058, + "tokens_seen": 574045184 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004171915747241725, + "loss": 2.666, + "theoretical_loss": 3.8576900146373525, + "tokens_seen": 574110720 + }, + { + "epoch": 1.08, + "learning_rate": 0.00041718154463390176, + "loss": 2.6323, + "theoretical_loss": 3.8576437108518102, + "tokens_seen": 574176256 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004171715145436309, + "loss": 2.7769, + "theoretical_loss": 3.8575974138306703, + "tokens_seen": 574241792 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004171614844533601, + "loss": 2.8701, + "theoretical_loss": 3.857551123572174, + "tokens_seen": 574307328 + }, + { + "epoch": 1.08, + "learning_rate": 0.00041715145436308924, + "loss": 2.7098, + "theoretical_loss": 3.8575048400745597, + "tokens_seen": 574372864 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004171414242728185, + "loss": 2.9271, + "theoretical_loss": 3.8574585633360705, + "tokens_seen": 574438400 + }, + { + "epoch": 1.08, + "learning_rate": 0.00041713139418254766, + "loss": 2.7082, + "theoretical_loss": 3.8574122933549475, + "tokens_seen": 574503936 + }, + { + "epoch": 1.08, + "learning_rate": 0.00041712136409227684, + "loss": 2.8392, + "theoretical_loss": 3.857366030129434, + "tokens_seen": 574569472 + }, + { + "epoch": 1.08, + "learning_rate": 0.000417111334002006, + "loss": 2.5627, + "theoretical_loss": 3.857319773657772, + "tokens_seen": 574635008 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004171013039117352, + "loss": 2.5853, + "theoretical_loss": 3.8572735239382068, + "tokens_seen": 574700544 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004170912738214644, + "loss": 2.8761, + "theoretical_loss": 3.8572272809689823, + "tokens_seen": 574766080 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004170812437311936, + "loss": 2.9132, + "theoretical_loss": 3.857181044748344, + "tokens_seen": 574831616 + }, + { + "epoch": 1.08, + "learning_rate": 0.00041707121364092275, + "loss": 2.6828, + "theoretical_loss": 3.857134815274538, + "tokens_seen": 574897152 + }, + { + "epoch": 1.08, + "learning_rate": 0.000417061183550652, + "loss": 2.5846, + "theoretical_loss": 3.85708859254581, + "tokens_seen": 574962688 + }, + { + "epoch": 1.08, + "learning_rate": 0.00041705115346038116, + "loss": 2.9014, + "theoretical_loss": 3.8570423765604076, + "tokens_seen": 575028224 + }, + { + "epoch": 1.08, + "objective/train/docs_used": 641212, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5287535190582275, + "objective/train/theoretical_loss": 3.8570077189956096, + "objective/train/tokens_used": 593504736, + "theoretical_loss": 3.8570077189956096, + "tokens_seen": 575077376 + }, + { + "epoch": 1.08, + "learning_rate": 0.00041704112337011035, + "loss": 2.5591, + "theoretical_loss": 3.856996167316579, + "tokens_seen": 575093760 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004170310932798395, + "loss": 2.623, + "theoretical_loss": 3.8569499648125727, + "tokens_seen": 575159296 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004170210631895687, + "loss": 2.9332, + "theoretical_loss": 3.8569037690466375, + "tokens_seen": 575224832 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004170110330992979, + "loss": 2.7544, + "theoretical_loss": 3.8568575800170235, + "tokens_seen": 575290368 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004170010030090271, + "loss": 2.7303, + "theoretical_loss": 3.856811397721981, + "tokens_seen": 575355904 + }, + { + "epoch": 1.08, + "learning_rate": 0.00041699097291875625, + "loss": 2.5582, + "theoretical_loss": 3.856765222159762, + "tokens_seen": 575421440 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004169809428284855, + "loss": 2.8197, + "theoretical_loss": 3.856719053328616, + "tokens_seen": 575486976 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004169709127382146, + "loss": 2.7173, + "theoretical_loss": 3.8566728912267982, + "tokens_seen": 575552512 + }, + { + "epoch": 1.08, + "learning_rate": 0.00041696088264794385, + "loss": 2.8001, + "theoretical_loss": 3.85662673585256, + "tokens_seen": 575618048 + }, + { + "epoch": 1.08, + "learning_rate": 0.00041695085255767303, + "loss": 2.9218, + "theoretical_loss": 3.8565805872041556, + "tokens_seen": 575683584 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004169408224674022, + "loss": 2.7323, + "theoretical_loss": 3.8565344452798396, + "tokens_seen": 575749120 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004169307923771314, + "loss": 2.8589, + "theoretical_loss": 3.856488310077866, + "tokens_seen": 575814656 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004169207622868606, + "loss": 2.8005, + "theoretical_loss": 3.8564421815964924, + "tokens_seen": 575880192 + }, + { + "epoch": 1.08, + "learning_rate": 0.00041691073219658975, + "loss": 2.7526, + "theoretical_loss": 3.856396059833974, + "tokens_seen": 575945728 + }, + { + "epoch": 1.08, + "learning_rate": 0.000416900702106319, + "loss": 2.6452, + "theoretical_loss": 3.856349944788567, + "tokens_seen": 576011264 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004168906720160481, + "loss": 2.8848, + "theoretical_loss": 3.8563038364585314, + "tokens_seen": 576076800 + }, + { + "epoch": 1.08, + "learning_rate": 0.00041688064192577735, + "loss": 2.548, + "theoretical_loss": 3.856257734842123, + "tokens_seen": 576142336 + }, + { + "epoch": 1.08, + "learning_rate": 0.00041687061183550653, + "loss": 2.8584, + "theoretical_loss": 3.856211639937602, + "tokens_seen": 576207872 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004168605817452357, + "loss": 3.0388, + "theoretical_loss": 3.856165551743228, + "tokens_seen": 576273408 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004168505516549649, + "loss": 2.7249, + "theoretical_loss": 3.8561194702572603, + "tokens_seen": 576338944 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004168405215646941, + "loss": 2.5879, + "theoretical_loss": 3.856073395477962, + "tokens_seen": 576404480 + }, + { + "epoch": 1.08, + "learning_rate": 0.00041683049147442326, + "loss": 3.0491, + "theoretical_loss": 3.856027327403592, + "tokens_seen": 576470016 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004168204613841525, + "loss": 2.7849, + "theoretical_loss": 3.8559812660324138, + "tokens_seen": 576535552 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004168104312938816, + "loss": 2.5804, + "theoretical_loss": 3.8559352113626906, + "tokens_seen": 576601088 + }, + { + "epoch": 1.08, + "learning_rate": 0.00041680040120361085, + "loss": 2.7666, + "theoretical_loss": 3.855889163392685, + "tokens_seen": 576666624 + }, + { + "epoch": 1.08, + "objective/train/docs_used": 641212, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.898465633392334, + "objective/train/theoretical_loss": 3.8558546318108267, + "objective/train/tokens_used": 593504736, + "theoretical_loss": 3.8558546318108267, + "tokens_seen": 576715776 + }, + { + "epoch": 1.08, + "learning_rate": 0.00041679037111334, + "loss": 2.7956, + "theoretical_loss": 3.855843122120662, + "tokens_seen": 576732160 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004167803410230692, + "loss": 2.9043, + "theoretical_loss": 3.8557970875448855, + "tokens_seen": 576797696 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004167703109327984, + "loss": 2.7581, + "theoretical_loss": 3.8557510596636217, + "tokens_seen": 576863232 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004167602808425276, + "loss": 2.6658, + "theoretical_loss": 3.8557050384751363, + "tokens_seen": 576928768 + }, + { + "epoch": 1.08, + "learning_rate": 0.00041675025075225676, + "loss": 2.7947, + "theoretical_loss": 3.855659023977696, + "tokens_seen": 576994304 + }, + { + "epoch": 1.08, + "learning_rate": 0.000416740220661986, + "loss": 3.0682, + "theoretical_loss": 3.8556130161695688, + "tokens_seen": 577059840 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004167301905717151, + "loss": 2.7337, + "theoretical_loss": 3.8555670150490213, + "tokens_seen": 577125376 + }, + { + "epoch": 1.08, + "learning_rate": 0.00041672016048144436, + "loss": 2.8836, + "theoretical_loss": 3.855521020614324, + "tokens_seen": 577190912 + }, + { + "epoch": 1.08, + "learning_rate": 0.0004167101303911735, + "loss": 2.6244, + "theoretical_loss": 3.8554750328637444, + "tokens_seen": 577256448 + }, + { + "epoch": 1.09, + "learning_rate": 0.0004167001003009027, + "loss": 2.6494, + "theoretical_loss": 3.8554290517955536, + "tokens_seen": 577321984 + }, + { + "epoch": 1.09, + "learning_rate": 0.0004166900702106319, + "loss": 2.845, + "theoretical_loss": 3.855383077408022, + "tokens_seen": 577387520 + }, + { + "epoch": 1.09, + "learning_rate": 0.0004166800401203611, + "loss": 2.8374, + "theoretical_loss": 3.8553371096994207, + "tokens_seen": 577453056 + }, + { + "epoch": 1.09, + "learning_rate": 0.00041667001003009026, + "loss": 2.6816, + "theoretical_loss": 3.8552911486680217, + "tokens_seen": 577518592 + }, + { + "epoch": 1.09, + "learning_rate": 0.00041665997993981944, + "loss": 2.7496, + "theoretical_loss": 3.855245194312097, + "tokens_seen": 577584128 + }, + { + "epoch": 1.09, + "learning_rate": 0.0004166499498495487, + "loss": 2.6757, + "theoretical_loss": 3.8551992466299208, + "tokens_seen": 577649664 + }, + { + "epoch": 1.09, + "learning_rate": 0.00041663991975927786, + "loss": 2.7504, + "theoretical_loss": 3.8551533056197664, + "tokens_seen": 577715200 + }, + { + "epoch": 1.09, + "learning_rate": 0.00041662988966900704, + "loss": 2.8373, + "theoretical_loss": 3.8551073712799075, + "tokens_seen": 577780736 + }, + { + "epoch": 1.09, + "learning_rate": 0.0004166198595787362, + "loss": 2.8251, + "theoretical_loss": 3.85506144360862, + "tokens_seen": 577846272 + }, + { + "epoch": 1.09, + "learning_rate": 0.0004166098294884654, + "loss": 2.8219, + "theoretical_loss": 3.8550155226041802, + "tokens_seen": 577911808 + }, + { + "epoch": 1.09, + "learning_rate": 0.0004165997993981946, + "loss": 2.8431, + "theoretical_loss": 3.854969608264863, + "tokens_seen": 577977344 + }, + { + "epoch": 1.09, + "learning_rate": 0.0004165897693079238, + "loss": 2.9726, + "theoretical_loss": 3.854923700588947, + "tokens_seen": 578042880 + }, + { + "epoch": 1.09, + "learning_rate": 0.00041657973921765295, + "loss": 2.8628, + "theoretical_loss": 3.8548777995747088, + "tokens_seen": 578108416 + }, + { + "epoch": 1.09, + "learning_rate": 0.0004165697091273822, + "loss": 3.1248, + "theoretical_loss": 3.8548319052204265, + "tokens_seen": 578173952 + }, + { + "epoch": 1.09, + "learning_rate": 0.00041655967903711136, + "loss": 2.6876, + "theoretical_loss": 3.8547860175243795, + "tokens_seen": 578239488 + }, + { + "epoch": 1.09, + "learning_rate": 0.00041654964894684055, + "loss": 2.6237, + "theoretical_loss": 3.854740136484848, + "tokens_seen": 578305024 + }, + { + "epoch": 1.09, + "objective/train/docs_used": 641212, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9665675163269043, + "objective/train/theoretical_loss": 3.854705730072502, + "objective/train/tokens_used": 593504736, + "theoretical_loss": 3.854705730072502, + "tokens_seen": 578354176 + }, + { + "epoch": 1.09, + "learning_rate": 0.0004165396188565697, + "loss": 2.8729, + "theoretical_loss": 3.854694262100111, + "tokens_seen": 578370560 + }, + { + "epoch": 1.09, + "learning_rate": 0.0004165295887662989, + "loss": 2.616, + "theoretical_loss": 3.85464839436845, + "tokens_seen": 578436096 + }, + { + "epoch": 1.09, + "learning_rate": 0.0004165195586760281, + "loss": 2.9423, + "theoretical_loss": 3.854602533288147, + "tokens_seen": 578501632 + }, + { + "epoch": 1.09, + "learning_rate": 0.0004165095285857573, + "loss": 2.4722, + "theoretical_loss": 3.8545566788574828, + "tokens_seen": 578567168 + }, + { + "epoch": 1.09, + "learning_rate": 0.00041649949849548645, + "loss": 2.7993, + "theoretical_loss": 3.854510831074742, + "tokens_seen": 578632704 + }, + { + "epoch": 1.09, + "learning_rate": 0.0004164894684052157, + "loss": 2.8733, + "theoretical_loss": 3.8544649899382053, + "tokens_seen": 578698240 + }, + { + "epoch": 1.09, + "learning_rate": 0.0004164794383149448, + "loss": 2.8087, + "theoretical_loss": 3.85441915544616, + "tokens_seen": 578763776 + }, + { + "epoch": 1.09, + "learning_rate": 0.00041646940822467405, + "loss": 2.9179, + "theoretical_loss": 3.854373327596888, + "tokens_seen": 578829312 + }, + { + "epoch": 1.09, + "learning_rate": 0.00041645937813440323, + "loss": 2.9232, + "theoretical_loss": 3.854327506388677, + "tokens_seen": 578894848 + }, + { + "epoch": 1.09, + "learning_rate": 0.0004164493480441324, + "loss": 2.8144, + "theoretical_loss": 3.854281691819811, + "tokens_seen": 578960384 + }, + { + "epoch": 1.09, + "learning_rate": 0.0004164393179538616, + "loss": 2.7681, + "theoretical_loss": 3.8542358838885775, + "tokens_seen": 579025920 + }, + { + "epoch": 1.09, + "learning_rate": 0.0004164292878635908, + "loss": 2.7656, + "theoretical_loss": 3.854190082593264, + "tokens_seen": 579091456 + }, + { + "epoch": 1.09, + "learning_rate": 0.00041641925777331995, + "loss": 2.6538, + "theoretical_loss": 3.854144287932158, + "tokens_seen": 579156992 + }, + { + "epoch": 1.09, + "learning_rate": 0.0004164092276830492, + "loss": 2.8346, + "theoretical_loss": 3.8540984999035475, + "tokens_seen": 579222528 + }, + { + "epoch": 1.09, + "learning_rate": 0.0004163991975927783, + "loss": 2.5503, + "theoretical_loss": 3.8540527185057223, + "tokens_seen": 579288064 + }, + { + "epoch": 1.09, + "learning_rate": 0.00041638916750250755, + "loss": 2.8447, + "theoretical_loss": 3.854006943736972, + "tokens_seen": 579353600 + }, + { + "epoch": 1.09, + "learning_rate": 0.00041637913741223673, + "loss": 2.8478, + "theoretical_loss": 3.853961175595587, + "tokens_seen": 579419136 + }, + { + "epoch": 1.09, + "learning_rate": 0.0004163691073219659, + "loss": 2.7447, + "theoretical_loss": 3.8539154140798586, + "tokens_seen": 579484672 + }, + { + "epoch": 1.09, + "learning_rate": 0.0004163590772316951, + "loss": 2.7067, + "theoretical_loss": 3.853869659188078, + "tokens_seen": 579550208 + }, + { + "epoch": 1.09, + "learning_rate": 0.0004163490471414243, + "loss": 2.8598, + "theoretical_loss": 3.8538239109185377, + "tokens_seen": 579615744 + }, + { + "epoch": 1.09, + "learning_rate": 0.00041633901705115346, + "loss": 2.5542, + "theoretical_loss": 3.8537781692695305, + "tokens_seen": 579681280 + }, + { + "epoch": 1.09, + "learning_rate": 0.0004163289869608827, + "loss": 2.9101, + "theoretical_loss": 3.8537324342393506, + "tokens_seen": 579746816 + }, + { + "epoch": 1.09, + "learning_rate": 0.0004163189568706118, + "loss": 2.7138, + "theoretical_loss": 3.8536867058262914, + "tokens_seen": 579812352 + }, + { + "epoch": 1.09, + "learning_rate": 0.00041630892678034105, + "loss": 2.9372, + "theoretical_loss": 3.8536409840286483, + "tokens_seen": 579877888 + }, + { + "epoch": 1.09, + "learning_rate": 0.0004162988966900702, + "loss": 2.6821, + "theoretical_loss": 3.8535952688447166, + "tokens_seen": 579943424 + }, + { + "epoch": 1.09, + "objective/train/docs_used": 641212, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7990593910217285, + "objective/train/theoretical_loss": 3.8535609867959906, + "objective/train/tokens_used": 593504736, + "theoretical_loss": 3.8535609867959906, + "tokens_seen": 579992576 + }, + { + "epoch": 1.09, + "learning_rate": 0.0004162888665997994, + "loss": 2.7535, + "theoretical_loss": 3.853549560272792, + "tokens_seen": 580008960 + }, + { + "epoch": 1.09, + "learning_rate": 0.0004162788365095286, + "loss": 2.7862, + "theoretical_loss": 3.8535038583111723, + "tokens_seen": 580074496 + }, + { + "epoch": 1.09, + "learning_rate": 0.0004162688064192578, + "loss": 2.8232, + "theoretical_loss": 3.8534581629581535, + "tokens_seen": 580140032 + }, + { + "epoch": 1.09, + "learning_rate": 0.00041625877632898696, + "loss": 2.8708, + "theoretical_loss": 3.8534124742120346, + "tokens_seen": 580205568 + }, + { + "epoch": 1.09, + "learning_rate": 0.0004162487462387162, + "loss": 2.7073, + "theoretical_loss": 3.853366792071114, + "tokens_seen": 580271104 + }, + { + "epoch": 1.09, + "learning_rate": 0.0004162387161484453, + "loss": 2.4495, + "theoretical_loss": 3.8533211165336905, + "tokens_seen": 580336640 + }, + { + "epoch": 1.09, + "learning_rate": 0.00041622868605817456, + "loss": 2.9621, + "theoretical_loss": 3.8532754475980644, + "tokens_seen": 580402176 + }, + { + "epoch": 1.09, + "learning_rate": 0.0004162186559679037, + "loss": 2.8054, + "theoretical_loss": 3.8532297852625366, + "tokens_seen": 580467712 + }, + { + "epoch": 1.09, + "learning_rate": 0.0004162086258776329, + "loss": 2.8751, + "theoretical_loss": 3.8531841295254075, + "tokens_seen": 580533248 + }, + { + "epoch": 1.09, + "learning_rate": 0.0004161985957873621, + "loss": 2.7955, + "theoretical_loss": 3.8531384803849793, + "tokens_seen": 580598784 + }, + { + "epoch": 1.09, + "learning_rate": 0.0004161885656970913, + "loss": 2.9094, + "theoretical_loss": 3.8530928378395544, + "tokens_seen": 580664320 + }, + { + "epoch": 1.09, + "learning_rate": 0.00041617853560682046, + "loss": 2.7033, + "theoretical_loss": 3.8530472018874358, + "tokens_seen": 580729856 + }, + { + "epoch": 1.09, + "learning_rate": 0.00041616850551654964, + "loss": 2.7345, + "theoretical_loss": 3.8530015725269267, + "tokens_seen": 580795392 + }, + { + "epoch": 1.09, + "learning_rate": 0.0004161584754262788, + "loss": 2.8526, + "theoretical_loss": 3.852955949756332, + "tokens_seen": 580860928 + }, + { + "epoch": 1.09, + "learning_rate": 0.00041614844533600806, + "loss": 2.8981, + "theoretical_loss": 3.8529103335739565, + "tokens_seen": 580926464 + }, + { + "epoch": 1.09, + "learning_rate": 0.0004161384152457372, + "loss": 3.1064, + "theoretical_loss": 3.8528647239781053, + "tokens_seen": 580992000 + }, + { + "epoch": 1.09, + "learning_rate": 0.0004161283851554664, + "loss": 2.7506, + "theoretical_loss": 3.852819120967085, + "tokens_seen": 581057536 + }, + { + "epoch": 1.09, + "learning_rate": 0.00041611835506519555, + "loss": 2.8473, + "theoretical_loss": 3.8527735245392023, + "tokens_seen": 581123072 + }, + { + "epoch": 1.09, + "learning_rate": 0.0004161083249749248, + "loss": 2.7136, + "theoretical_loss": 3.852727934692765, + "tokens_seen": 581188608 + }, + { + "epoch": 1.09, + "learning_rate": 0.00041609829488465397, + "loss": 2.5776, + "theoretical_loss": 3.852682351426081, + "tokens_seen": 581254144 + }, + { + "epoch": 1.09, + "learning_rate": 0.00041608826479438315, + "loss": 2.5465, + "theoretical_loss": 3.8526367747374577, + "tokens_seen": 581319680 + }, + { + "epoch": 1.09, + "learning_rate": 0.00041607823470411233, + "loss": 2.8932, + "theoretical_loss": 3.852591204625206, + "tokens_seen": 581385216 + }, + { + "epoch": 1.09, + "learning_rate": 0.00041606820461384156, + "loss": 2.8596, + "theoretical_loss": 3.8525456410876355, + "tokens_seen": 581450752 + }, + { + "epoch": 1.09, + "learning_rate": 0.0004160581745235707, + "loss": 2.5327, + "theoretical_loss": 3.8525000841230566, + "tokens_seen": 581516288 + }, + { + "epoch": 1.09, + "learning_rate": 0.0004160481444332999, + "loss": 2.9714, + "theoretical_loss": 3.85245453372978, + "tokens_seen": 581581824 + }, + { + "epoch": 1.09, + "objective/train/docs_used": 641212, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.1994011402130127, + "objective/train/theoretical_loss": 3.8524203752462247, + "objective/train/tokens_used": 593504736, + "theoretical_loss": 3.8524203752462247, + "tokens_seen": 581630976 + }, + { + "epoch": 1.09, + "learning_rate": 0.00041603811434302905, + "loss": 2.7316, + "theoretical_loss": 3.8524089899061185, + "tokens_seen": 581647360 + }, + { + "epoch": 1.09, + "learning_rate": 0.0004160280842527583, + "loss": 2.8617, + "theoretical_loss": 3.8523634526503834, + "tokens_seen": 581712896 + }, + { + "epoch": 1.09, + "learning_rate": 0.00041601805416248747, + "loss": 2.7499, + "theoretical_loss": 3.8523179219608883, + "tokens_seen": 581778432 + }, + { + "epoch": 1.09, + "learning_rate": 0.00041600802407221665, + "loss": 2.5885, + "theoretical_loss": 3.8522723978359474, + "tokens_seen": 581843968 + }, + { + "epoch": 1.09, + "learning_rate": 0.00041599799398194583, + "loss": 2.7726, + "theoretical_loss": 3.852226880273874, + "tokens_seen": 581909504 + }, + { + "epoch": 1.09, + "learning_rate": 0.000415987963891675, + "loss": 2.7785, + "theoretical_loss": 3.8521813692729836, + "tokens_seen": 581975040 + }, + { + "epoch": 1.09, + "learning_rate": 0.0004159779338014042, + "loss": 2.6607, + "theoretical_loss": 3.852135864831591, + "tokens_seen": 582040576 + }, + { + "epoch": 1.09, + "learning_rate": 0.00041596790371113343, + "loss": 2.622, + "theoretical_loss": 3.8520903669480138, + "tokens_seen": 582106112 + }, + { + "epoch": 1.09, + "learning_rate": 0.00041595787362086256, + "loss": 2.7967, + "theoretical_loss": 3.8520448756205674, + "tokens_seen": 582171648 + }, + { + "epoch": 1.09, + "learning_rate": 0.0004159478435305918, + "loss": 2.9222, + "theoretical_loss": 3.85199939084757, + "tokens_seen": 582237184 + }, + { + "epoch": 1.09, + "learning_rate": 0.0004159378134403209, + "loss": 2.6956, + "theoretical_loss": 3.8519539126273394, + "tokens_seen": 582302720 + }, + { + "epoch": 1.09, + "learning_rate": 0.00041592778335005015, + "loss": 2.7274, + "theoretical_loss": 3.8519084409581943, + "tokens_seen": 582368256 + }, + { + "epoch": 1.09, + "learning_rate": 0.00041591775325977934, + "loss": 2.6459, + "theoretical_loss": 3.8518629758384537, + "tokens_seen": 582433792 + }, + { + "epoch": 1.09, + "learning_rate": 0.0004159077231695085, + "loss": 2.7572, + "theoretical_loss": 3.8518175172664377, + "tokens_seen": 582499328 + }, + { + "epoch": 1.09, + "learning_rate": 0.00041589769307923775, + "loss": 2.7572, + "theoretical_loss": 3.851772065240467, + "tokens_seen": 582564864 + }, + { + "epoch": 1.09, + "learning_rate": 0.00041588766298896693, + "loss": 2.8565, + "theoretical_loss": 3.851726619758862, + "tokens_seen": 582630400 + }, + { + "epoch": 1.09, + "learning_rate": 0.0004158776328986961, + "loss": 2.7034, + "theoretical_loss": 3.851681180819945, + "tokens_seen": 582695936 + }, + { + "epoch": 1.09, + "learning_rate": 0.0004158676028084253, + "loss": 2.7775, + "theoretical_loss": 3.851635748422039, + "tokens_seen": 582761472 + }, + { + "epoch": 1.09, + "learning_rate": 0.0004158575727181545, + "loss": 2.9109, + "theoretical_loss": 3.8515903225634656, + "tokens_seen": 582827008 + }, + { + "epoch": 1.09, + "learning_rate": 0.00041584754262788366, + "loss": 2.7499, + "theoretical_loss": 3.851544903242549, + "tokens_seen": 582892544 + }, + { + "epoch": 1.09, + "learning_rate": 0.0004158375125376129, + "loss": 2.854, + "theoretical_loss": 3.8514994904576136, + "tokens_seen": 582958080 + }, + { + "epoch": 1.09, + "learning_rate": 0.000415827482447342, + "loss": 3.1417, + "theoretical_loss": 3.851454084206985, + "tokens_seen": 583023616 + }, + { + "epoch": 1.09, + "learning_rate": 0.00041581745235707126, + "loss": 2.7627, + "theoretical_loss": 3.8514086844889865, + "tokens_seen": 583089152 + }, + { + "epoch": 1.09, + "learning_rate": 0.0004158074222668004, + "loss": 2.6031, + "theoretical_loss": 3.851363291301946, + "tokens_seen": 583154688 + }, + { + "epoch": 1.09, + "learning_rate": 0.0004157973921765296, + "loss": 2.9589, + "theoretical_loss": 3.85131790464419, + "tokens_seen": 583220224 + }, + { + "epoch": 1.09, + "objective/train/docs_used": 641212, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.419975757598877, + "objective/train/theoretical_loss": 3.8512838689347095, + "objective/train/tokens_used": 593504736, + "theoretical_loss": 3.8512838689347095, + "tokens_seen": 583269376 + }, + { + "epoch": 1.09, + "learning_rate": 0.0004157873620862588, + "loss": 2.8707, + "theoretical_loss": 3.8512725245140453, + "tokens_seen": 583285760 + }, + { + "epoch": 1.09, + "learning_rate": 0.000415777331995988, + "loss": 2.8639, + "theoretical_loss": 3.85122715090984, + "tokens_seen": 583351296 + }, + { + "epoch": 1.09, + "learning_rate": 0.00041576730190571716, + "loss": 2.7196, + "theoretical_loss": 3.8511817838299023, + "tokens_seen": 583416832 + }, + { + "epoch": 1.09, + "learning_rate": 0.0004157572718154464, + "loss": 2.828, + "theoretical_loss": 3.8511364232725622, + "tokens_seen": 583482368 + }, + { + "epoch": 1.09, + "learning_rate": 0.0004157472417251755, + "loss": 2.8964, + "theoretical_loss": 3.8510910692361486, + "tokens_seen": 583547904 + }, + { + "epoch": 1.09, + "learning_rate": 0.00041573721163490476, + "loss": 2.6145, + "theoretical_loss": 3.851045721718992, + "tokens_seen": 583613440 + }, + { + "epoch": 1.09, + "learning_rate": 0.0004157271815446339, + "loss": 2.6236, + "theoretical_loss": 3.851000380719424, + "tokens_seen": 583678976 + }, + { + "epoch": 1.09, + "learning_rate": 0.0004157171514543631, + "loss": 2.8353, + "theoretical_loss": 3.850955046235776, + "tokens_seen": 583744512 + }, + { + "epoch": 1.09, + "learning_rate": 0.0004157071213640923, + "loss": 2.9949, + "theoretical_loss": 3.85090971826638, + "tokens_seen": 583810048 + }, + { + "epoch": 1.09, + "learning_rate": 0.0004156970912738215, + "loss": 2.61, + "theoretical_loss": 3.850864396809569, + "tokens_seen": 583875584 + }, + { + "epoch": 1.09, + "learning_rate": 0.00041568706118355066, + "loss": 2.7655, + "theoretical_loss": 3.8508190818636763, + "tokens_seen": 583941120 + }, + { + "epoch": 1.09, + "learning_rate": 0.00041567703109327984, + "loss": 2.8568, + "theoretical_loss": 3.8507737734270355, + "tokens_seen": 584006656 + }, + { + "epoch": 1.09, + "learning_rate": 0.000415667001003009, + "loss": 2.6818, + "theoretical_loss": 3.850728471497982, + "tokens_seen": 584072192 + }, + { + "epoch": 1.09, + "learning_rate": 0.00041565697091273826, + "loss": 2.8308, + "theoretical_loss": 3.8506831760748517, + "tokens_seen": 584137728 + }, + { + "epoch": 1.09, + "learning_rate": 0.0004156469408224674, + "loss": 2.9406, + "theoretical_loss": 3.8506378871559788, + "tokens_seen": 584203264 + }, + { + "epoch": 1.09, + "learning_rate": 0.0004156369107321966, + "loss": 2.736, + "theoretical_loss": 3.850592604739701, + "tokens_seen": 584268800 + }, + { + "epoch": 1.09, + "learning_rate": 0.00041562688064192575, + "loss": 2.8512, + "theoretical_loss": 3.850547328824356, + "tokens_seen": 584334336 + }, + { + "epoch": 1.09, + "learning_rate": 0.000415616850551655, + "loss": 2.5336, + "theoretical_loss": 3.8505020594082797, + "tokens_seen": 584399872 + }, + { + "epoch": 1.09, + "learning_rate": 0.00041560682046138417, + "loss": 2.9136, + "theoretical_loss": 3.850456796489812, + "tokens_seen": 584465408 + }, + { + "epoch": 1.09, + "learning_rate": 0.00041559679037111335, + "loss": 2.5841, + "theoretical_loss": 3.8504115400672916, + "tokens_seen": 584530944 + }, + { + "epoch": 1.09, + "learning_rate": 0.00041558676028084253, + "loss": 2.7805, + "theoretical_loss": 3.850366290139057, + "tokens_seen": 584596480 + }, + { + "epoch": 1.09, + "learning_rate": 0.00041557673019057176, + "loss": 2.685, + "theoretical_loss": 3.85032104670345, + "tokens_seen": 584662016 + }, + { + "epoch": 1.09, + "learning_rate": 0.0004155667001003009, + "loss": 2.9191, + "theoretical_loss": 3.85027580975881, + "tokens_seen": 584727552 + }, + { + "epoch": 1.09, + "learning_rate": 0.0004155566700100301, + "loss": 3.0622, + "theoretical_loss": 3.8502305793034797, + "tokens_seen": 584793088 + }, + { + "epoch": 1.09, + "learning_rate": 0.00041554663991975925, + "loss": 2.9408, + "theoretical_loss": 3.8501853553357996, + "tokens_seen": 584858624 + }, + { + "epoch": 1.09, + "objective/train/docs_used": 641212, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.363860607147217, + "objective/train/theoretical_loss": 3.850151441616564, + "objective/train/tokens_used": 593504736, + "theoretical_loss": 3.850151441616564, + "tokens_seen": 584907776 + }, + { + "epoch": 1.09, + "learning_rate": 0.0004155366098294885, + "loss": 2.6347, + "theoretical_loss": 3.850140137854114, + "tokens_seen": 584924160 + }, + { + "epoch": 1.09, + "learning_rate": 0.00041552657973921767, + "loss": 2.8598, + "theoretical_loss": 3.850094926856765, + "tokens_seen": 584989696 + }, + { + "epoch": 1.09, + "learning_rate": 0.00041551654964894685, + "loss": 2.8034, + "theoretical_loss": 3.8500497223420966, + "tokens_seen": 585055232 + }, + { + "epoch": 1.09, + "learning_rate": 0.00041550651955867603, + "loss": 2.6854, + "theoretical_loss": 3.8500045243084537, + "tokens_seen": 585120768 + }, + { + "epoch": 1.09, + "learning_rate": 0.0004154964894684052, + "loss": 2.7721, + "theoretical_loss": 3.849959332754181, + "tokens_seen": 585186304 + }, + { + "epoch": 1.09, + "learning_rate": 0.0004154864593781344, + "loss": 3.0961, + "theoretical_loss": 3.849914147677624, + "tokens_seen": 585251840 + }, + { + "epoch": 1.09, + "learning_rate": 0.00041547642928786363, + "loss": 2.9701, + "theoretical_loss": 3.849868969077129, + "tokens_seen": 585317376 + }, + { + "epoch": 1.09, + "learning_rate": 0.00041546639919759276, + "loss": 2.7004, + "theoretical_loss": 3.8498237969510436, + "tokens_seen": 585382912 + }, + { + "epoch": 1.09, + "learning_rate": 0.000415456369107322, + "loss": 2.7834, + "theoretical_loss": 3.849778631297715, + "tokens_seen": 585448448 + }, + { + "epoch": 1.09, + "learning_rate": 0.0004154463390170511, + "loss": 2.7596, + "theoretical_loss": 3.8497334721154903, + "tokens_seen": 585513984 + }, + { + "epoch": 1.09, + "learning_rate": 0.00041543630892678035, + "loss": 2.7112, + "theoretical_loss": 3.84968831940272, + "tokens_seen": 585579520 + }, + { + "epoch": 1.09, + "learning_rate": 0.00041542627883650954, + "loss": 2.9759, + "theoretical_loss": 3.8496431731577516, + "tokens_seen": 585645056 + }, + { + "epoch": 1.09, + "learning_rate": 0.0004154162487462387, + "loss": 2.7964, + "theoretical_loss": 3.849598033378936, + "tokens_seen": 585710592 + }, + { + "epoch": 1.09, + "learning_rate": 0.0004154062186559679, + "loss": 2.8015, + "theoretical_loss": 3.849552900064624, + "tokens_seen": 585776128 + }, + { + "epoch": 1.09, + "learning_rate": 0.00041539618856569713, + "loss": 3.0477, + "theoretical_loss": 3.8495077732131655, + "tokens_seen": 585841664 + }, + { + "epoch": 1.09, + "learning_rate": 0.00041538615847542626, + "loss": 2.8499, + "theoretical_loss": 3.8494626528229134, + "tokens_seen": 585907200 + }, + { + "epoch": 1.09, + "learning_rate": 0.0004153761283851555, + "loss": 2.6039, + "theoretical_loss": 3.8494175388922196, + "tokens_seen": 585972736 + }, + { + "epoch": 1.09, + "learning_rate": 0.0004153660982948846, + "loss": 2.5178, + "theoretical_loss": 3.849372431419437, + "tokens_seen": 586038272 + }, + { + "epoch": 1.09, + "learning_rate": 0.00041535606820461386, + "loss": 2.827, + "theoretical_loss": 3.8493273304029194, + "tokens_seen": 586103808 + }, + { + "epoch": 1.09, + "learning_rate": 0.00041534603811434304, + "loss": 2.8356, + "theoretical_loss": 3.8492822358410206, + "tokens_seen": 586169344 + }, + { + "epoch": 1.09, + "learning_rate": 0.0004153360080240722, + "loss": 2.7122, + "theoretical_loss": 3.8492371477320955, + "tokens_seen": 586234880 + }, + { + "epoch": 1.09, + "learning_rate": 0.0004153259779338014, + "loss": 2.8285, + "theoretical_loss": 3.8491920660744996, + "tokens_seen": 586300416 + }, + { + "epoch": 1.09, + "learning_rate": 0.0004153159478435306, + "loss": 2.9381, + "theoretical_loss": 3.849146990866589, + "tokens_seen": 586365952 + }, + { + "epoch": 1.09, + "learning_rate": 0.00041530591775325976, + "loss": 2.871, + "theoretical_loss": 3.8491019221067195, + "tokens_seen": 586431488 + }, + { + "epoch": 1.09, + "learning_rate": 0.000415295887662989, + "loss": 2.9606, + "theoretical_loss": 3.849056859793249, + "tokens_seen": 586497024 + }, + { + "epoch": 1.09, + "objective/train/docs_used": 641212, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.706367015838623, + "objective/train/theoretical_loss": 3.8490230672876073, + "objective/train/tokens_used": 593504736, + "theoretical_loss": 3.8490230672876073, + "tokens_seen": 586546176 + }, + { + "epoch": 1.09, + "learning_rate": 0.0004152858575727181, + "loss": 2.7225, + "theoretical_loss": 3.8490118039245353, + "tokens_seen": 586562560 + }, + { + "epoch": 1.09, + "learning_rate": 0.00041527582748244736, + "loss": 2.711, + "theoretical_loss": 3.848966754498936, + "tokens_seen": 586628096 + }, + { + "epoch": 1.09, + "learning_rate": 0.0004152657973921765, + "loss": 2.9355, + "theoretical_loss": 3.848921711514811, + "tokens_seen": 586693632 + }, + { + "epoch": 1.09, + "learning_rate": 0.0004152557673019057, + "loss": 2.9359, + "theoretical_loss": 3.848876674970519, + "tokens_seen": 586759168 + }, + { + "epoch": 1.09, + "learning_rate": 0.0004152457372116349, + "loss": 2.8788, + "theoretical_loss": 3.848831644864421, + "tokens_seen": 586824704 + }, + { + "epoch": 1.09, + "learning_rate": 0.0004152357071213641, + "loss": 2.892, + "theoretical_loss": 3.8487866211948774, + "tokens_seen": 586890240 + }, + { + "epoch": 1.09, + "learning_rate": 0.00041522567703109327, + "loss": 2.8509, + "theoretical_loss": 3.8487416039602493, + "tokens_seen": 586955776 + }, + { + "epoch": 1.09, + "learning_rate": 0.0004152156469408225, + "loss": 2.6335, + "theoretical_loss": 3.8486965931588992, + "tokens_seen": 587021312 + }, + { + "epoch": 1.09, + "learning_rate": 0.00041520561685055163, + "loss": 2.764, + "theoretical_loss": 3.8486515887891892, + "tokens_seen": 587086848 + }, + { + "epoch": 1.09, + "learning_rate": 0.00041519558676028086, + "loss": 2.7661, + "theoretical_loss": 3.8486065908494824, + "tokens_seen": 587152384 + }, + { + "epoch": 1.09, + "learning_rate": 0.00041518555667001, + "loss": 2.8219, + "theoretical_loss": 3.8485615993381432, + "tokens_seen": 587217920 + }, + { + "epoch": 1.09, + "learning_rate": 0.0004151755265797392, + "loss": 2.6805, + "theoretical_loss": 3.8485166142535356, + "tokens_seen": 587283456 + }, + { + "epoch": 1.09, + "learning_rate": 0.0004151654964894684, + "loss": 2.7319, + "theoretical_loss": 3.8484716355940245, + "tokens_seen": 587348992 + }, + { + "epoch": 1.09, + "learning_rate": 0.0004151554663991976, + "loss": 2.8708, + "theoretical_loss": 3.8484266633579756, + "tokens_seen": 587414528 + }, + { + "epoch": 1.09, + "learning_rate": 0.0004151454363089268, + "loss": 2.8686, + "theoretical_loss": 3.848381697543755, + "tokens_seen": 587480064 + }, + { + "epoch": 1.09, + "learning_rate": 0.00041513540621865595, + "loss": 2.865, + "theoretical_loss": 3.8483367381497295, + "tokens_seen": 587545600 + }, + { + "epoch": 1.09, + "learning_rate": 0.0004151253761283852, + "loss": 2.8516, + "theoretical_loss": 3.8482917851742666, + "tokens_seen": 587611136 + }, + { + "epoch": 1.09, + "learning_rate": 0.00041511534603811437, + "loss": 2.8969, + "theoretical_loss": 3.848246838615734, + "tokens_seen": 587676672 + }, + { + "epoch": 1.09, + "learning_rate": 0.00041510531594784355, + "loss": 2.7833, + "theoretical_loss": 3.8482018984724995, + "tokens_seen": 587742208 + }, + { + "epoch": 1.09, + "learning_rate": 0.00041509528585757273, + "loss": 2.8398, + "theoretical_loss": 3.8481569647429343, + "tokens_seen": 587807744 + }, + { + "epoch": 1.09, + "learning_rate": 0.00041508525576730196, + "loss": 2.6893, + "theoretical_loss": 3.8481120374254063, + "tokens_seen": 587873280 + }, + { + "epoch": 1.09, + "learning_rate": 0.0004150752256770311, + "loss": 2.9593, + "theoretical_loss": 3.848067116518287, + "tokens_seen": 587938816 + }, + { + "epoch": 1.09, + "learning_rate": 0.0004150651955867603, + "loss": 2.9808, + "theoretical_loss": 3.848022202019946, + "tokens_seen": 588004352 + }, + { + "epoch": 1.09, + "learning_rate": 0.00041505516549648945, + "loss": 2.7058, + "theoretical_loss": 3.847977293928756, + "tokens_seen": 588069888 + }, + { + "epoch": 1.09, + "learning_rate": 0.0004150451354062187, + "loss": 2.6618, + "theoretical_loss": 3.8479323922430893, + "tokens_seen": 588135424 + }, + { + "epoch": 1.09, + "objective/train/docs_used": 641212, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.562694787979126, + "objective/train/theoretical_loss": 3.847898720181484, + "objective/train/tokens_used": 593504736, + "theoretical_loss": 3.847898720181484, + "tokens_seen": 588184576 + }, + { + "epoch": 1.09, + "learning_rate": 0.00041503510531594787, + "loss": 2.3911, + "theoretical_loss": 3.8478874969613175, + "tokens_seen": 588200960 + }, + { + "epoch": 1.09, + "learning_rate": 0.00041502507522567705, + "loss": 2.5121, + "theoretical_loss": 3.847842608081815, + "tokens_seen": 588266496 + }, + { + "epoch": 1.09, + "learning_rate": 0.00041501504513540623, + "loss": 2.7847, + "theoretical_loss": 3.8477977256029554, + "tokens_seen": 588332032 + }, + { + "epoch": 1.09, + "learning_rate": 0.0004150050150451354, + "loss": 2.5692, + "theoretical_loss": 3.847752849523112, + "tokens_seen": 588397568 + }, + { + "epoch": 1.09, + "learning_rate": 0.0004149949849548646, + "loss": 2.977, + "theoretical_loss": 3.847707979840662, + "tokens_seen": 588463104 + }, + { + "epoch": 1.09, + "learning_rate": 0.00041498495486459383, + "loss": 2.8128, + "theoretical_loss": 3.8476631165539796, + "tokens_seen": 588528640 + }, + { + "epoch": 1.09, + "learning_rate": 0.00041497492477432296, + "loss": 2.7995, + "theoretical_loss": 3.8476182596614414, + "tokens_seen": 588594176 + }, + { + "epoch": 1.09, + "learning_rate": 0.0004149648946840522, + "loss": 2.771, + "theoretical_loss": 3.8475734091614253, + "tokens_seen": 588659712 + }, + { + "epoch": 1.09, + "learning_rate": 0.0004149548645937813, + "loss": 2.8067, + "theoretical_loss": 3.847528565052307, + "tokens_seen": 588725248 + }, + { + "epoch": 1.09, + "learning_rate": 0.00041494483450351055, + "loss": 2.6756, + "theoretical_loss": 3.847483727332466, + "tokens_seen": 588790784 + }, + { + "epoch": 1.09, + "learning_rate": 0.00041493480441323974, + "loss": 2.6244, + "theoretical_loss": 3.8474388960002797, + "tokens_seen": 588856320 + }, + { + "epoch": 1.09, + "learning_rate": 0.0004149247743229689, + "loss": 2.6471, + "theoretical_loss": 3.8473940710541283, + "tokens_seen": 588921856 + }, + { + "epoch": 1.09, + "learning_rate": 0.0004149147442326981, + "loss": 2.4718, + "theoretical_loss": 3.847349252492392, + "tokens_seen": 588987392 + }, + { + "epoch": 1.09, + "learning_rate": 0.00041490471414242733, + "loss": 2.9592, + "theoretical_loss": 3.84730444031345, + "tokens_seen": 589052928 + }, + { + "epoch": 1.09, + "learning_rate": 0.00041489468405215646, + "loss": 2.5508, + "theoretical_loss": 3.847259634515684, + "tokens_seen": 589118464 + }, + { + "epoch": 1.09, + "learning_rate": 0.0004148846539618857, + "loss": 3.1104, + "theoretical_loss": 3.847214835097476, + "tokens_seen": 589184000 + }, + { + "epoch": 1.09, + "learning_rate": 0.0004148746238716148, + "loss": 2.821, + "theoretical_loss": 3.8471700420572077, + "tokens_seen": 589249536 + }, + { + "epoch": 1.09, + "learning_rate": 0.00041486459378134406, + "loss": 2.8675, + "theoretical_loss": 3.8471252553932618, + "tokens_seen": 589315072 + }, + { + "epoch": 1.09, + "learning_rate": 0.00041485456369107324, + "loss": 2.7168, + "theoretical_loss": 3.847080475104022, + "tokens_seen": 589380608 + }, + { + "epoch": 1.09, + "learning_rate": 0.0004148445336008024, + "loss": 2.5669, + "theoretical_loss": 3.847035701187872, + "tokens_seen": 589446144 + }, + { + "epoch": 1.09, + "learning_rate": 0.0004148345035105316, + "loss": 2.7023, + "theoretical_loss": 3.8469909336431964, + "tokens_seen": 589511680 + }, + { + "epoch": 1.09, + "learning_rate": 0.0004148244734202608, + "loss": 2.8024, + "theoretical_loss": 3.846946172468381, + "tokens_seen": 589577216 + }, + { + "epoch": 1.09, + "learning_rate": 0.00041481444332998996, + "loss": 2.839, + "theoretical_loss": 3.846901417661811, + "tokens_seen": 589642752 + }, + { + "epoch": 1.09, + "learning_rate": 0.0004148044132397192, + "loss": 2.5471, + "theoretical_loss": 3.846856669221872, + "tokens_seen": 589708288 + }, + { + "epoch": 1.09, + "learning_rate": 0.0004147943831494483, + "loss": 2.6021, + "theoretical_loss": 3.846811927146952, + "tokens_seen": 589773824 + }, + { + "epoch": 1.09, + "objective/train/docs_used": 641212, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.017899990081787, + "objective/train/theoretical_loss": 3.846778374766836, + "objective/train/tokens_used": 593504736, + "theoretical_loss": 3.846778374766836, + "tokens_seen": 589822976 + }, + { + "epoch": 1.09, + "learning_rate": 0.00041478435305917756, + "loss": 2.9278, + "theoretical_loss": 3.846767191435438, + "tokens_seen": 589839360 + }, + { + "epoch": 1.09, + "learning_rate": 0.0004147743229689067, + "loss": 2.6644, + "theoretical_loss": 3.8467224620857183, + "tokens_seen": 589904896 + }, + { + "epoch": 1.09, + "learning_rate": 0.0004147642928786359, + "loss": 2.7847, + "theoretical_loss": 3.846677739096182, + "tokens_seen": 589970432 + }, + { + "epoch": 1.09, + "learning_rate": 0.0004147542627883651, + "loss": 2.4443, + "theoretical_loss": 3.846633022465218, + "tokens_seen": 590035968 + }, + { + "epoch": 1.09, + "learning_rate": 0.0004147442326980943, + "loss": 2.6099, + "theoretical_loss": 3.846588312191215, + "tokens_seen": 590101504 + }, + { + "epoch": 1.09, + "learning_rate": 0.00041473420260782347, + "loss": 2.8192, + "theoretical_loss": 3.8465436082725653, + "tokens_seen": 590167040 + }, + { + "epoch": 1.09, + "learning_rate": 0.0004147241725175527, + "loss": 2.6361, + "theoretical_loss": 3.846498910707659, + "tokens_seen": 590232576 + }, + { + "epoch": 1.09, + "learning_rate": 0.00041471414242728183, + "loss": 2.7931, + "theoretical_loss": 3.8464542194948876, + "tokens_seen": 590298112 + }, + { + "epoch": 1.09, + "learning_rate": 0.00041470411233701106, + "loss": 2.8275, + "theoretical_loss": 3.8464095346326435, + "tokens_seen": 590363648 + }, + { + "epoch": 1.09, + "learning_rate": 0.0004146940822467402, + "loss": 2.5265, + "theoretical_loss": 3.8463648561193198, + "tokens_seen": 590429184 + }, + { + "epoch": 1.09, + "learning_rate": 0.0004146840521564694, + "loss": 2.4411, + "theoretical_loss": 3.84632018395331, + "tokens_seen": 590494720 + }, + { + "epoch": 1.09, + "learning_rate": 0.0004146740220661986, + "loss": 2.6881, + "theoretical_loss": 3.846275518133007, + "tokens_seen": 590560256 + }, + { + "epoch": 1.09, + "learning_rate": 0.0004146639919759278, + "loss": 2.9586, + "theoretical_loss": 3.846230858656806, + "tokens_seen": 590625792 + }, + { + "epoch": 1.09, + "learning_rate": 0.00041465396188565697, + "loss": 2.8721, + "theoretical_loss": 3.846186205523102, + "tokens_seen": 590691328 + }, + { + "epoch": 1.09, + "learning_rate": 0.00041464393179538615, + "loss": 2.5559, + "theoretical_loss": 3.846141558730291, + "tokens_seen": 590756864 + }, + { + "epoch": 1.09, + "learning_rate": 0.00041463390170511533, + "loss": 2.877, + "theoretical_loss": 3.8460969182767695, + "tokens_seen": 590822400 + }, + { + "epoch": 1.09, + "learning_rate": 0.00041462387161484457, + "loss": 2.8501, + "theoretical_loss": 3.8460522841609333, + "tokens_seen": 590887936 + }, + { + "epoch": 1.09, + "learning_rate": 0.0004146138415245737, + "loss": 2.7009, + "theoretical_loss": 3.8460076563811807, + "tokens_seen": 590953472 + }, + { + "epoch": 1.09, + "learning_rate": 0.00041460381143430293, + "loss": 2.6128, + "theoretical_loss": 3.8459630349359104, + "tokens_seen": 591019008 + }, + { + "epoch": 1.09, + "learning_rate": 0.0004145937813440321, + "loss": 2.4929, + "theoretical_loss": 3.8459184198235192, + "tokens_seen": 591084544 + }, + { + "epoch": 1.09, + "learning_rate": 0.0004145837512537613, + "loss": 2.8232, + "theoretical_loss": 3.8458738110424076, + "tokens_seen": 591150080 + }, + { + "epoch": 1.09, + "learning_rate": 0.0004145737211634905, + "loss": 2.6952, + "theoretical_loss": 3.845829208590975, + "tokens_seen": 591215616 + }, + { + "epoch": 1.09, + "learning_rate": 0.00041456369107321965, + "loss": 2.8109, + "theoretical_loss": 3.8457846124676225, + "tokens_seen": 591281152 + }, + { + "epoch": 1.09, + "learning_rate": 0.00041455366098294884, + "loss": 3.0057, + "theoretical_loss": 3.8457400226707494, + "tokens_seen": 591346688 + }, + { + "epoch": 1.09, + "learning_rate": 0.00041454363089267807, + "loss": 2.8575, + "theoretical_loss": 3.8456954391987592, + "tokens_seen": 591412224 + }, + { + "epoch": 1.09, + "objective/train/docs_used": 641212, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5051016807556152, + "objective/train/theoretical_loss": 3.845662005744509, + "objective/train/tokens_used": 593504736, + "theoretical_loss": 3.845662005744509, + "tokens_seen": 591461376 + }, + { + "epoch": 1.09, + "learning_rate": 0.0004145336008024072, + "loss": 2.763, + "theoretical_loss": 3.8456508620500527, + "tokens_seen": 591477760 + }, + { + "epoch": 1.09, + "learning_rate": 0.00041452357071213643, + "loss": 2.7731, + "theoretical_loss": 3.8456062912230324, + "tokens_seen": 591543296 + }, + { + "epoch": 1.09, + "learning_rate": 0.00041451354062186556, + "loss": 2.8782, + "theoretical_loss": 3.8455617267161024, + "tokens_seen": 591608832 + }, + { + "epoch": 1.09, + "learning_rate": 0.0004145035105315948, + "loss": 2.9828, + "theoretical_loss": 3.8455171685276666, + "tokens_seen": 591674368 + }, + { + "epoch": 1.09, + "learning_rate": 0.000414493480441324, + "loss": 2.6916, + "theoretical_loss": 3.845472616656129, + "tokens_seen": 591739904 + }, + { + "epoch": 1.09, + "learning_rate": 0.00041448345035105316, + "loss": 2.3959, + "theoretical_loss": 3.845428071099895, + "tokens_seen": 591805440 + }, + { + "epoch": 1.09, + "learning_rate": 0.00041447342026078234, + "loss": 2.7328, + "theoretical_loss": 3.8453835318573697, + "tokens_seen": 591870976 + }, + { + "epoch": 1.09, + "learning_rate": 0.0004144633901705115, + "loss": 2.8305, + "theoretical_loss": 3.8453389989269593, + "tokens_seen": 591936512 + }, + { + "epoch": 1.09, + "learning_rate": 0.0004144533600802407, + "loss": 2.7196, + "theoretical_loss": 3.8452944723070708, + "tokens_seen": 592002048 + }, + { + "epoch": 1.09, + "learning_rate": 0.00041444332998996994, + "loss": 2.6286, + "theoretical_loss": 3.845249951996111, + "tokens_seen": 592067584 + }, + { + "epoch": 1.09, + "learning_rate": 0.00041443329989969906, + "loss": 2.4818, + "theoretical_loss": 3.8452054379924894, + "tokens_seen": 592133120 + }, + { + "epoch": 1.09, + "learning_rate": 0.0004144232698094283, + "loss": 2.6504, + "theoretical_loss": 3.845160930294613, + "tokens_seen": 592198656 + }, + { + "epoch": 1.09, + "learning_rate": 0.00041441323971915753, + "loss": 2.715, + "theoretical_loss": 3.845116428900891, + "tokens_seen": 592264192 + }, + { + "epoch": 1.09, + "learning_rate": 0.00041440320962888666, + "loss": 2.7967, + "theoretical_loss": 3.8450719338097326, + "tokens_seen": 592329728 + }, + { + "epoch": 1.09, + "learning_rate": 0.0004143931795386159, + "loss": 3.0511, + "theoretical_loss": 3.8450274450195496, + "tokens_seen": 592395264 + }, + { + "epoch": 1.09, + "learning_rate": 0.000414383149448345, + "loss": 2.8589, + "theoretical_loss": 3.8449829625287517, + "tokens_seen": 592460800 + }, + { + "epoch": 1.09, + "learning_rate": 0.00041437311935807426, + "loss": 2.8515, + "theoretical_loss": 3.8449384863357503, + "tokens_seen": 592526336 + }, + { + "epoch": 1.09, + "learning_rate": 0.00041436308926780344, + "loss": 3.0156, + "theoretical_loss": 3.8448940164389573, + "tokens_seen": 592591872 + }, + { + "epoch": 1.09, + "learning_rate": 0.0004143530591775326, + "loss": 2.5779, + "theoretical_loss": 3.844849552836786, + "tokens_seen": 592657408 + }, + { + "epoch": 1.09, + "learning_rate": 0.0004143430290872618, + "loss": 2.5526, + "theoretical_loss": 3.844805095527648, + "tokens_seen": 592722944 + }, + { + "epoch": 1.09, + "learning_rate": 0.000414332998996991, + "loss": 2.5768, + "theoretical_loss": 3.844760644509959, + "tokens_seen": 592788480 + }, + { + "epoch": 1.09, + "learning_rate": 0.00041432296890672016, + "loss": 2.8752, + "theoretical_loss": 3.8447161997821313, + "tokens_seen": 592854016 + }, + { + "epoch": 1.09, + "learning_rate": 0.0004143129388164494, + "loss": 2.502, + "theoretical_loss": 3.844671761342581, + "tokens_seen": 592919552 + }, + { + "epoch": 1.09, + "learning_rate": 0.0004143029087261785, + "loss": 2.7409, + "theoretical_loss": 3.8446273291897226, + "tokens_seen": 592985088 + }, + { + "epoch": 1.09, + "learning_rate": 0.00041429287863590776, + "loss": 2.8724, + "theoretical_loss": 3.844582903321973, + "tokens_seen": 593050624 + }, + { + "epoch": 1.09, + "objective/train/docs_used": 641212, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6726303100585938, + "objective/train/theoretical_loss": 3.8445495880448104, + "objective/train/tokens_used": 593504736, + "theoretical_loss": 3.8445495880448104, + "tokens_seen": 593099776 + }, + { + "epoch": 1.09, + "learning_rate": 0.0004142828485456369, + "loss": 2.8083, + "theoretical_loss": 3.844538483737748, + "tokens_seen": 593116160 + }, + { + "epoch": 1.09, + "learning_rate": 0.0004142728184553661, + "loss": 3.0371, + "theoretical_loss": 3.8444940704354655, + "tokens_seen": 593181696 + }, + { + "epoch": 1.09, + "learning_rate": 0.0004142627883650953, + "loss": 2.7278, + "theoretical_loss": 3.844449663413542, + "tokens_seen": 593247232 + }, + { + "epoch": 1.09, + "learning_rate": 0.0004142527582748245, + "loss": 2.7245, + "theoretical_loss": 3.844405262670397, + "tokens_seen": 593312768 + }, + { + "epoch": 1.09, + "learning_rate": 0.00041424272818455367, + "loss": 2.728, + "theoretical_loss": 3.8443608682044488, + "tokens_seen": 593378304 + }, + { + "epoch": 1.09, + "learning_rate": 0.0004142326980942829, + "loss": 2.7423, + "theoretical_loss": 3.8443164800141165, + "tokens_seen": 593443840 + }, + { + "epoch": 1.09, + "learning_rate": 0.00041422266800401203, + "loss": 2.8193, + "theoretical_loss": 3.8442720980978207, + "tokens_seen": 593509376 + }, + { + "epoch": 2.0, + "learning_rate": 0.00041421263791374126, + "loss": 3.4325, + "theoretical_loss": 3.844217322819618, + "tokens_seen": 593590272 + }, + { + "epoch": 2.0, + "learning_rate": 0.0004142026078234704, + "loss": 2.9414, + "theoretical_loss": 3.844172954916165, + "tokens_seen": 593655808 + }, + { + "epoch": 2.0, + "learning_rate": 0.0004141925777331996, + "loss": 2.9461, + "theoretical_loss": 3.844128593281643, + "tokens_seen": 593721344 + }, + { + "epoch": 2.0, + "learning_rate": 0.0004141825476429288, + "loss": 2.8163, + "theoretical_loss": 3.8440842379144717, + "tokens_seen": 593786880 + }, + { + "epoch": 2.0, + "learning_rate": 0.000414172517552658, + "loss": 3.198, + "theoretical_loss": 3.844039888813076, + "tokens_seen": 593852416 + }, + { + "epoch": 2.0, + "learning_rate": 0.00041416248746238717, + "loss": 3.0478, + "theoretical_loss": 3.8439955459758792, + "tokens_seen": 593917952 + }, + { + "epoch": 2.0, + "learning_rate": 0.00041415245737211635, + "loss": 2.9957, + "theoretical_loss": 3.8439512094013057, + "tokens_seen": 593983488 + }, + { + "epoch": 2.0, + "learning_rate": 0.00041414242728184553, + "loss": 3.0462, + "theoretical_loss": 3.843906879087779, + "tokens_seen": 594049024 + }, + { + "epoch": 2.0, + "learning_rate": 0.00041413239719157477, + "loss": 2.8059, + "theoretical_loss": 3.8438625550337253, + "tokens_seen": 594114560 + }, + { + "epoch": 2.0, + "learning_rate": 0.0004141223671013039, + "loss": 2.8339, + "theoretical_loss": 3.84381823723757, + "tokens_seen": 594180096 + }, + { + "epoch": 2.0, + "learning_rate": 0.00041411233701103313, + "loss": 3.0042, + "theoretical_loss": 3.8437739256977403, + "tokens_seen": 594245632 + }, + { + "epoch": 2.0, + "learning_rate": 0.0004141023069207623, + "loss": 2.868, + "theoretical_loss": 3.8437296204126623, + "tokens_seen": 594311168 + }, + { + "epoch": 2.0, + "learning_rate": 0.0004140922768304915, + "loss": 3.0221, + "theoretical_loss": 3.843685321380763, + "tokens_seen": 594376704 + }, + { + "epoch": 2.0, + "learning_rate": 0.0004140822467402207, + "loss": 2.856, + "theoretical_loss": 3.8436410286004725, + "tokens_seen": 594442240 + }, + { + "epoch": 2.0, + "learning_rate": 0.00041407221664994985, + "loss": 2.9139, + "theoretical_loss": 3.8435967420702175, + "tokens_seen": 594507776 + }, + { + "epoch": 2.0, + "learning_rate": 0.00041406218655967904, + "loss": 2.8862, + "theoretical_loss": 3.8435524617884287, + "tokens_seen": 594573312 + }, + { + "epoch": 2.0, + "learning_rate": 0.00041405215646940827, + "loss": 2.977, + "theoretical_loss": 3.8435081877535344, + "tokens_seen": 594638848 + }, + { + "epoch": 2.0, + "learning_rate": 0.0004140421263791374, + "loss": 2.78, + "theoretical_loss": 3.843463919963966, + "tokens_seen": 594704384 + }, + { + "epoch": 2.0, + "objective/train/docs_used": 708692, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.850222587585449, + "objective/train/theoretical_loss": 3.843441788410689, + "objective/train/tokens_used": 615197152, + "theoretical_loss": 3.843441788410689, + "tokens_seen": 594737152 + }, + { + "epoch": 2.0, + "learning_rate": 0.00041403209628886663, + "loss": 2.9877, + "theoretical_loss": 3.8434196584181546, + "tokens_seen": 594769920 + }, + { + "epoch": 2.0, + "learning_rate": 0.00041402206619859576, + "loss": 2.9706, + "theoretical_loss": 3.8433754031145315, + "tokens_seen": 594835456 + }, + { + "epoch": 2.0, + "learning_rate": 0.000414012036108325, + "loss": 2.9933, + "theoretical_loss": 3.8433311540515285, + "tokens_seen": 594900992 + }, + { + "epoch": 2.0, + "learning_rate": 0.0004140020060180542, + "loss": 3.0618, + "theoretical_loss": 3.8432869112275787, + "tokens_seen": 594966528 + }, + { + "epoch": 2.0, + "learning_rate": 0.00041399197592778336, + "loss": 3.0248, + "theoretical_loss": 3.8432426746411146, + "tokens_seen": 595032064 + }, + { + "epoch": 2.0, + "learning_rate": 0.00041398194583751254, + "loss": 2.9282, + "theoretical_loss": 3.8431984442905707, + "tokens_seen": 595097600 + }, + { + "epoch": 2.0, + "learning_rate": 0.0004139719157472417, + "loss": 2.9484, + "theoretical_loss": 3.8431542201743807, + "tokens_seen": 595163136 + }, + { + "epoch": 2.0, + "learning_rate": 0.0004139618856569709, + "loss": 2.8406, + "theoretical_loss": 3.8431100022909797, + "tokens_seen": 595228672 + }, + { + "epoch": 2.0, + "learning_rate": 0.00041395185556670014, + "loss": 2.7116, + "theoretical_loss": 3.8430657906388035, + "tokens_seen": 595294208 + }, + { + "epoch": 2.0, + "learning_rate": 0.00041394182547642926, + "loss": 2.8288, + "theoretical_loss": 3.843021585216288, + "tokens_seen": 595359744 + }, + { + "epoch": 2.0, + "learning_rate": 0.0004139317953861585, + "loss": 2.886, + "theoretical_loss": 3.84297738602187, + "tokens_seen": 595425280 + }, + { + "epoch": 2.0, + "learning_rate": 0.0004139217652958877, + "loss": 2.9879, + "theoretical_loss": 3.8429331930539865, + "tokens_seen": 595490816 + }, + { + "epoch": 2.0, + "learning_rate": 0.00041391173520561686, + "loss": 2.8221, + "theoretical_loss": 3.8428890063110748, + "tokens_seen": 595556352 + }, + { + "epoch": 2.0, + "learning_rate": 0.00041390170511534604, + "loss": 2.8475, + "theoretical_loss": 3.8428448257915733, + "tokens_seen": 595621888 + }, + { + "epoch": 2.0, + "learning_rate": 0.0004138916750250752, + "loss": 3.1417, + "theoretical_loss": 3.8428006514939215, + "tokens_seen": 595687424 + }, + { + "epoch": 2.0, + "learning_rate": 0.0004138816449348044, + "loss": 2.7316, + "theoretical_loss": 3.8427564834165584, + "tokens_seen": 595752960 + }, + { + "epoch": 2.0, + "learning_rate": 0.00041387161484453364, + "loss": 2.8183, + "theoretical_loss": 3.842712321557924, + "tokens_seen": 595818496 + }, + { + "epoch": 2.0, + "learning_rate": 0.00041386158475426277, + "loss": 2.9654, + "theoretical_loss": 3.842668165916459, + "tokens_seen": 595884032 + }, + { + "epoch": 2.0, + "learning_rate": 0.000413851554663992, + "loss": 2.7973, + "theoretical_loss": 3.8426240164906043, + "tokens_seen": 595949568 + }, + { + "epoch": 2.0, + "learning_rate": 0.00041384152457372113, + "loss": 3.0307, + "theoretical_loss": 3.842579873278801, + "tokens_seen": 596015104 + }, + { + "epoch": 2.0, + "learning_rate": 0.00041383149448345036, + "loss": 2.7348, + "theoretical_loss": 3.842535736279493, + "tokens_seen": 596080640 + }, + { + "epoch": 2.0, + "learning_rate": 0.00041382146439317954, + "loss": 2.9633, + "theoretical_loss": 3.842491605491122, + "tokens_seen": 596146176 + }, + { + "epoch": 2.0, + "learning_rate": 0.0004138114343029087, + "loss": 2.9599, + "theoretical_loss": 3.842447480912131, + "tokens_seen": 596211712 + }, + { + "epoch": 2.0, + "learning_rate": 0.0004138014042126379, + "loss": 2.7724, + "theoretical_loss": 3.8424033625409644, + "tokens_seen": 596277248 + }, + { + "epoch": 2.0, + "learning_rate": 0.0004137913741223671, + "loss": 3.0336, + "theoretical_loss": 3.8423592503760666, + "tokens_seen": 596342784 + }, + { + "epoch": 2.0, + "objective/train/docs_used": 713945, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.0268349647521973, + "objective/train/theoretical_loss": 3.8423371966204827, + "objective/train/tokens_used": 616835552, + "theoretical_loss": 3.8423371966204827, + "tokens_seen": 596375552 + }, + { + "epoch": 2.0, + "learning_rate": 0.00041378134403209627, + "loss": 3.0419, + "theoretical_loss": 3.842315144415883, + "tokens_seen": 596408320 + }, + { + "epoch": 2.0, + "learning_rate": 0.0004137713139418255, + "loss": 2.9421, + "theoretical_loss": 3.842271044658859, + "tokens_seen": 596473856 + }, + { + "epoch": 2.0, + "learning_rate": 0.00041376128385155463, + "loss": 2.99, + "theoretical_loss": 3.84222695110344, + "tokens_seen": 596539392 + }, + { + "epoch": 2.0, + "learning_rate": 0.00041375125376128387, + "loss": 2.9369, + "theoretical_loss": 3.842182863748074, + "tokens_seen": 596604928 + }, + { + "epoch": 2.0, + "learning_rate": 0.00041374122367101305, + "loss": 3.0614, + "theoretical_loss": 3.8421387825912072, + "tokens_seen": 596670464 + }, + { + "epoch": 2.0, + "learning_rate": 0.00041373119358074223, + "loss": 3.1159, + "theoretical_loss": 3.842094707631288, + "tokens_seen": 596736000 + }, + { + "epoch": 2.0, + "learning_rate": 0.0004137211634904714, + "loss": 2.7884, + "theoretical_loss": 3.8420506388667652, + "tokens_seen": 596801536 + }, + { + "epoch": 2.0, + "learning_rate": 0.0004137111334002006, + "loss": 2.977, + "theoretical_loss": 3.842006576296087, + "tokens_seen": 596867072 + }, + { + "epoch": 2.0, + "learning_rate": 0.00041370110330992977, + "loss": 2.9942, + "theoretical_loss": 3.841962519917703, + "tokens_seen": 596932608 + }, + { + "epoch": 2.0, + "learning_rate": 0.000413691073219659, + "loss": 2.86, + "theoretical_loss": 3.8419184697300635, + "tokens_seen": 596998144 + }, + { + "epoch": 2.0, + "learning_rate": 0.00041368104312938813, + "loss": 2.8966, + "theoretical_loss": 3.841874425731619, + "tokens_seen": 597063680 + }, + { + "epoch": 2.0, + "learning_rate": 0.00041367101303911737, + "loss": 2.7813, + "theoretical_loss": 3.841830387920821, + "tokens_seen": 597129216 + }, + { + "epoch": 2.0, + "learning_rate": 0.00041366098294884655, + "loss": 2.6166, + "theoretical_loss": 3.8417863562961205, + "tokens_seen": 597194752 + }, + { + "epoch": 2.0, + "learning_rate": 0.00041365095285857573, + "loss": 2.9736, + "theoretical_loss": 3.841742330855971, + "tokens_seen": 597260288 + }, + { + "epoch": 2.0, + "learning_rate": 0.00041364092276830497, + "loss": 3.0773, + "theoretical_loss": 3.841698311598824, + "tokens_seen": 597325824 + }, + { + "epoch": 2.0, + "learning_rate": 0.0004136308926780341, + "loss": 3.1391, + "theoretical_loss": 3.8416542985231343, + "tokens_seen": 597391360 + }, + { + "epoch": 2.0, + "learning_rate": 0.00041362086258776333, + "loss": 2.9423, + "theoretical_loss": 3.841610291627355, + "tokens_seen": 597456896 + }, + { + "epoch": 2.0, + "learning_rate": 0.0004136108324974925, + "loss": 2.874, + "theoretical_loss": 3.841566290909941, + "tokens_seen": 597522432 + }, + { + "epoch": 2.0, + "learning_rate": 0.0004136008024072217, + "loss": 2.9495, + "theoretical_loss": 3.8415222963693467, + "tokens_seen": 597587968 + }, + { + "epoch": 2.0, + "learning_rate": 0.0004135907723169509, + "loss": 3.039, + "theoretical_loss": 3.8414783080040285, + "tokens_seen": 597653504 + }, + { + "epoch": 2.0, + "learning_rate": 0.00041358074222668005, + "loss": 2.8841, + "theoretical_loss": 3.8414343258124424, + "tokens_seen": 597719040 + }, + { + "epoch": 2.0, + "learning_rate": 0.00041357071213640924, + "loss": 2.803, + "theoretical_loss": 3.841390349793045, + "tokens_seen": 597784576 + }, + { + "epoch": 2.0, + "learning_rate": 0.00041356068204613847, + "loss": 2.9264, + "theoretical_loss": 3.8413463799442935, + "tokens_seen": 597850112 + }, + { + "epoch": 2.0, + "learning_rate": 0.0004135506519558676, + "loss": 3.0903, + "theoretical_loss": 3.8413024162646465, + "tokens_seen": 597915648 + }, + { + "epoch": 2.0, + "learning_rate": 0.00041354062186559683, + "loss": 2.7137, + "theoretical_loss": 3.8412584587525616, + "tokens_seen": 597981184 + }, + { + "epoch": 2.0, + "objective/train/docs_used": 718818, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.732727289199829, + "objective/train/theoretical_loss": 3.841236482308873, + "objective/train/tokens_used": 618473952, + "theoretical_loss": 3.841236482308873, + "tokens_seen": 598013952 + }, + { + "epoch": 2.0, + "learning_rate": 0.00041353059177532596, + "loss": 3.0269, + "theoretical_loss": 3.841214507406498, + "tokens_seen": 598046720 + }, + { + "epoch": 2.0, + "learning_rate": 0.0004135205616850552, + "loss": 2.9763, + "theoretical_loss": 3.8411705622249155, + "tokens_seen": 598112256 + }, + { + "epoch": 2.0, + "learning_rate": 0.0004135105315947844, + "loss": 3.026, + "theoretical_loss": 3.841126623206274, + "tokens_seen": 598177792 + }, + { + "epoch": 2.0, + "learning_rate": 0.00041350050150451356, + "loss": 3.0752, + "theoretical_loss": 3.841082690349034, + "tokens_seen": 598243328 + }, + { + "epoch": 2.0, + "learning_rate": 0.00041349047141424274, + "loss": 2.9446, + "theoretical_loss": 3.8410387636516568, + "tokens_seen": 598308864 + }, + { + "epoch": 2.0, + "learning_rate": 0.0004134804413239719, + "loss": 2.8162, + "theoretical_loss": 3.840994843112604, + "tokens_seen": 598374400 + }, + { + "epoch": 2.0, + "learning_rate": 0.0004134704112337011, + "loss": 2.8233, + "theoretical_loss": 3.8409509287303383, + "tokens_seen": 598439936 + }, + { + "epoch": 2.0, + "learning_rate": 0.00041346038114343034, + "loss": 2.9554, + "theoretical_loss": 3.840907020503322, + "tokens_seen": 598505472 + }, + { + "epoch": 2.0, + "learning_rate": 0.00041345035105315946, + "loss": 2.7912, + "theoretical_loss": 3.840863118430019, + "tokens_seen": 598571008 + }, + { + "epoch": 2.0, + "learning_rate": 0.0004134403209628887, + "loss": 2.9135, + "theoretical_loss": 3.840819222508893, + "tokens_seen": 598636544 + }, + { + "epoch": 2.0, + "learning_rate": 0.0004134302908726179, + "loss": 3.072, + "theoretical_loss": 3.840775332738408, + "tokens_seen": 598702080 + }, + { + "epoch": 2.0, + "learning_rate": 0.00041342026078234706, + "loss": 2.8755, + "theoretical_loss": 3.84073144911703, + "tokens_seen": 598767616 + }, + { + "epoch": 2.0, + "learning_rate": 0.00041341023069207624, + "loss": 2.984, + "theoretical_loss": 3.8406875716432243, + "tokens_seen": 598833152 + }, + { + "epoch": 2.0, + "learning_rate": 0.0004134002006018054, + "loss": 3.0003, + "theoretical_loss": 3.8406437003154568, + "tokens_seen": 598898688 + }, + { + "epoch": 2.0, + "learning_rate": 0.0004133901705115346, + "loss": 2.9602, + "theoretical_loss": 3.840599835132194, + "tokens_seen": 598964224 + }, + { + "epoch": 2.0, + "learning_rate": 0.00041338014042126384, + "loss": 2.85, + "theoretical_loss": 3.840555976091904, + "tokens_seen": 599029760 + }, + { + "epoch": 2.0, + "learning_rate": 0.00041337011033099297, + "loss": 2.9622, + "theoretical_loss": 3.840512123193054, + "tokens_seen": 599095296 + }, + { + "epoch": 2.0, + "learning_rate": 0.0004133600802407222, + "loss": 2.9202, + "theoretical_loss": 3.8404682764341125, + "tokens_seen": 599160832 + }, + { + "epoch": 2.0, + "learning_rate": 0.00041335005015045133, + "loss": 2.9157, + "theoretical_loss": 3.840424435813548, + "tokens_seen": 599226368 + }, + { + "epoch": 2.0, + "learning_rate": 0.00041334002006018056, + "loss": 2.9191, + "theoretical_loss": 3.84038060132983, + "tokens_seen": 599291904 + }, + { + "epoch": 2.0, + "learning_rate": 0.00041332998996990975, + "loss": 2.898, + "theoretical_loss": 3.8403367729814297, + "tokens_seen": 599357440 + }, + { + "epoch": 2.0, + "learning_rate": 0.0004133199598796389, + "loss": 2.9767, + "theoretical_loss": 3.840292950766816, + "tokens_seen": 599422976 + }, + { + "epoch": 2.0, + "learning_rate": 0.0004133099297893681, + "loss": 2.8749, + "theoretical_loss": 3.840249134684461, + "tokens_seen": 599488512 + }, + { + "epoch": 2.0, + "learning_rate": 0.0004132998996990973, + "loss": 2.9314, + "theoretical_loss": 3.8402053247328363, + "tokens_seen": 599554048 + }, + { + "epoch": 2.0, + "learning_rate": 0.00041328986960882647, + "loss": 2.8579, + "theoretical_loss": 3.8401615209104136, + "tokens_seen": 599619584 + }, + { + "epoch": 2.0, + "objective/train/docs_used": 723725, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7808403968811035, + "objective/train/theoretical_loss": 3.8401396212971757, + "objective/train/tokens_used": 620112352, + "theoretical_loss": 3.8401396212971757, + "tokens_seen": 599652352 + }, + { + "epoch": 2.0, + "learning_rate": 0.0004132798395185557, + "loss": 2.7878, + "theoretical_loss": 3.8401177232156654, + "tokens_seen": 599685120 + }, + { + "epoch": 2.0, + "learning_rate": 0.00041326980942828483, + "loss": 2.812, + "theoretical_loss": 3.8400739316470665, + "tokens_seen": 599750656 + }, + { + "epoch": 2.0, + "learning_rate": 0.00041325977933801407, + "loss": 3.0098, + "theoretical_loss": 3.8400301462030892, + "tokens_seen": 599816192 + }, + { + "epoch": 2.0, + "learning_rate": 0.00041324974924774325, + "loss": 2.9269, + "theoretical_loss": 3.8399863668822087, + "tokens_seen": 599881728 + }, + { + "epoch": 2.0, + "learning_rate": 0.00041323971915747243, + "loss": 2.8626, + "theoretical_loss": 3.8399425936828995, + "tokens_seen": 599947264 + }, + { + "epoch": 2.0, + "learning_rate": 0.0004132296890672016, + "loss": 2.9165, + "theoretical_loss": 3.839898826603637, + "tokens_seen": 600012800 + }, + { + "epoch": 2.0, + "learning_rate": 0.0004132196589769308, + "loss": 2.9539, + "theoretical_loss": 3.839855065642898, + "tokens_seen": 600078336 + }, + { + "epoch": 2.0, + "learning_rate": 0.00041320962888665997, + "loss": 2.9778, + "theoretical_loss": 3.839811310799158, + "tokens_seen": 600143872 + }, + { + "epoch": 2.0, + "learning_rate": 0.0004131995987963892, + "loss": 2.8436, + "theoretical_loss": 3.839767562070895, + "tokens_seen": 600209408 + }, + { + "epoch": 2.0, + "learning_rate": 0.00041318956870611833, + "loss": 2.8844, + "theoretical_loss": 3.839723819456586, + "tokens_seen": 600274944 + }, + { + "epoch": 2.0, + "learning_rate": 0.00041317953861584757, + "loss": 3.0573, + "theoretical_loss": 3.8396800829547093, + "tokens_seen": 600340480 + }, + { + "epoch": 2.0, + "learning_rate": 0.0004131695085255767, + "loss": 2.857, + "theoretical_loss": 3.839636352563744, + "tokens_seen": 600406016 + }, + { + "epoch": 2.0, + "learning_rate": 0.00041315947843530593, + "loss": 2.8885, + "theoretical_loss": 3.83959262828217, + "tokens_seen": 600471552 + }, + { + "epoch": 2.0, + "learning_rate": 0.0004131494483450351, + "loss": 2.8889, + "theoretical_loss": 3.8395489101084657, + "tokens_seen": 600537088 + }, + { + "epoch": 2.0, + "learning_rate": 0.0004131394182547643, + "loss": 2.8557, + "theoretical_loss": 3.8395051980411123, + "tokens_seen": 600602624 + }, + { + "epoch": 2.0, + "learning_rate": 0.0004131293881644935, + "loss": 2.9485, + "theoretical_loss": 3.8394614920785903, + "tokens_seen": 600668160 + }, + { + "epoch": 2.0, + "learning_rate": 0.0004131193580742227, + "loss": 2.8408, + "theoretical_loss": 3.8394177922193817, + "tokens_seen": 600733696 + }, + { + "epoch": 2.0, + "learning_rate": 0.00041310932798395184, + "loss": 2.8704, + "theoretical_loss": 3.8393740984619686, + "tokens_seen": 600799232 + }, + { + "epoch": 2.0, + "learning_rate": 0.0004130992978936811, + "loss": 2.8666, + "theoretical_loss": 3.8393304108048323, + "tokens_seen": 600864768 + }, + { + "epoch": 2.0, + "learning_rate": 0.0004130892678034102, + "loss": 2.9289, + "theoretical_loss": 3.839286729246458, + "tokens_seen": 600930304 + }, + { + "epoch": 2.0, + "learning_rate": 0.00041307923771313944, + "loss": 2.8433, + "theoretical_loss": 3.839243053785327, + "tokens_seen": 600995840 + }, + { + "epoch": 2.0, + "learning_rate": 0.0004130692076228686, + "loss": 2.8201, + "theoretical_loss": 3.8391993844199255, + "tokens_seen": 601061376 + }, + { + "epoch": 2.0, + "learning_rate": 0.0004130591775325978, + "loss": 2.9049, + "theoretical_loss": 3.839155721148737, + "tokens_seen": 601126912 + }, + { + "epoch": 2.0, + "learning_rate": 0.000413049147442327, + "loss": 2.8023, + "theoretical_loss": 3.8391120639702474, + "tokens_seen": 601192448 + }, + { + "epoch": 2.0, + "learning_rate": 0.00041303911735205616, + "loss": 2.9873, + "theoretical_loss": 3.839068412882942, + "tokens_seen": 601257984 + }, + { + "epoch": 2.0, + "objective/train/docs_used": 728779, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.1727378368377686, + "objective/train/theoretical_loss": 3.8390465896230106, + "objective/train/tokens_used": 621750752, + "theoretical_loss": 3.8390465896230106, + "tokens_seen": 601290752 + }, + { + "epoch": 2.0, + "learning_rate": 0.00041302908726178534, + "loss": 2.9188, + "theoretical_loss": 3.8390247678853076, + "tokens_seen": 601323520 + }, + { + "epoch": 2.0, + "learning_rate": 0.0004130190571715146, + "loss": 2.8634, + "theoretical_loss": 3.8389811289758304, + "tokens_seen": 601389056 + }, + { + "epoch": 2.0, + "learning_rate": 0.0004130090270812437, + "loss": 2.6309, + "theoretical_loss": 3.838937496152999, + "tokens_seen": 601454592 + }, + { + "epoch": 2.0, + "learning_rate": 0.00041299899699097294, + "loss": 2.9133, + "theoretical_loss": 3.8388938694153003, + "tokens_seen": 601520128 + }, + { + "epoch": 2.0, + "learning_rate": 0.00041298896690070207, + "loss": 2.9708, + "theoretical_loss": 3.838850248761223, + "tokens_seen": 601585664 + }, + { + "epoch": 2.0, + "learning_rate": 0.0004129789368104313, + "loss": 2.7942, + "theoretical_loss": 3.8388066341892566, + "tokens_seen": 601651200 + }, + { + "epoch": 2.0, + "learning_rate": 0.0004129689067201605, + "loss": 2.8799, + "theoretical_loss": 3.838763025697891, + "tokens_seen": 601716736 + }, + { + "epoch": 2.0, + "learning_rate": 0.00041295887662988966, + "loss": 2.881, + "theoretical_loss": 3.8387194232856148, + "tokens_seen": 601782272 + }, + { + "epoch": 2.0, + "learning_rate": 0.00041294884653961884, + "loss": 2.9561, + "theoretical_loss": 3.8386758269509205, + "tokens_seen": 601847808 + }, + { + "epoch": 2.0, + "learning_rate": 0.0004129388164493481, + "loss": 2.7078, + "theoretical_loss": 3.8386322366922982, + "tokens_seen": 601913344 + }, + { + "epoch": 2.0, + "learning_rate": 0.0004129287863590772, + "loss": 2.9614, + "theoretical_loss": 3.83858865250824, + "tokens_seen": 601978880 + }, + { + "epoch": 2.0, + "learning_rate": 0.00041291875626880644, + "loss": 2.8647, + "theoretical_loss": 3.8385450743972376, + "tokens_seen": 602044416 + }, + { + "epoch": 2.0, + "learning_rate": 0.0004129087261785356, + "loss": 2.8494, + "theoretical_loss": 3.8385015023577846, + "tokens_seen": 602109952 + }, + { + "epoch": 2.0, + "learning_rate": 0.0004128986960882648, + "loss": 2.8509, + "theoretical_loss": 3.8384579363883744, + "tokens_seen": 602175488 + }, + { + "epoch": 2.0, + "learning_rate": 0.00041288866599799404, + "loss": 2.8159, + "theoretical_loss": 3.8384143764875, + "tokens_seen": 602241024 + }, + { + "epoch": 2.0, + "learning_rate": 0.00041287863590772317, + "loss": 3.0505, + "theoretical_loss": 3.8383708226536575, + "tokens_seen": 602306560 + }, + { + "epoch": 2.0, + "learning_rate": 0.0004128686058174524, + "loss": 2.7001, + "theoretical_loss": 3.83832727488534, + "tokens_seen": 602372096 + }, + { + "epoch": 2.0, + "learning_rate": 0.00041285857572718153, + "loss": 2.7911, + "theoretical_loss": 3.838283733181044, + "tokens_seen": 602437632 + }, + { + "epoch": 2.0, + "learning_rate": 0.00041284854563691076, + "loss": 3.0129, + "theoretical_loss": 3.8382401975392653, + "tokens_seen": 602503168 + }, + { + "epoch": 2.0, + "learning_rate": 0.00041283851554663995, + "loss": 2.7819, + "theoretical_loss": 3.8381966679585005, + "tokens_seen": 602568704 + }, + { + "epoch": 2.0, + "learning_rate": 0.0004128284854563691, + "loss": 2.8773, + "theoretical_loss": 3.838153144437247, + "tokens_seen": 602634240 + }, + { + "epoch": 2.0, + "learning_rate": 0.0004128184553660983, + "loss": 2.819, + "theoretical_loss": 3.838109626974002, + "tokens_seen": 602699776 + }, + { + "epoch": 2.0, + "learning_rate": 0.0004128084252758275, + "loss": 3.0012, + "theoretical_loss": 3.8380661155672646, + "tokens_seen": 602765312 + }, + { + "epoch": 2.0, + "learning_rate": 0.00041279839518555667, + "loss": 2.8227, + "theoretical_loss": 3.838022610215532, + "tokens_seen": 602830848 + }, + { + "epoch": 2.0, + "learning_rate": 0.0004127883650952859, + "loss": 2.6656, + "theoretical_loss": 3.837979110917305, + "tokens_seen": 602896384 + }, + { + "epoch": 2.0, + "objective/train/docs_used": 731087, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8379440307617188, + "objective/train/theoretical_loss": 3.837957363537787, + "objective/train/tokens_used": 623389152, + "theoretical_loss": 3.837957363537787, + "tokens_seen": 602929152 + }, + { + "epoch": 2.0, + "learning_rate": 0.00041277833500501503, + "loss": 2.9037, + "theoretical_loss": 3.837935617671082, + "tokens_seen": 602961920 + }, + { + "epoch": 2.0, + "learning_rate": 0.00041276830491474427, + "loss": 2.9334, + "theoretical_loss": 3.837892130475365, + "tokens_seen": 603027456 + }, + { + "epoch": 2.0, + "learning_rate": 0.00041275827482447345, + "loss": 2.8944, + "theoretical_loss": 3.837848649328653, + "tokens_seen": 603092992 + }, + { + "epoch": 2.0, + "learning_rate": 0.00041274824473420263, + "loss": 2.8149, + "theoretical_loss": 3.8378051742294494, + "tokens_seen": 603158528 + }, + { + "epoch": 2.0, + "learning_rate": 0.0004127382146439318, + "loss": 2.8712, + "theoretical_loss": 3.8377617051762547, + "tokens_seen": 603224064 + }, + { + "epoch": 2.0, + "learning_rate": 0.000412728184553661, + "loss": 2.8573, + "theoretical_loss": 3.8377182421675715, + "tokens_seen": 603289600 + }, + { + "epoch": 2.0, + "learning_rate": 0.00041271815446339017, + "loss": 2.8153, + "theoretical_loss": 3.8376747852019033, + "tokens_seen": 603355136 + }, + { + "epoch": 2.0, + "learning_rate": 0.0004127081243731194, + "loss": 2.7117, + "theoretical_loss": 3.837631334277753, + "tokens_seen": 603420672 + }, + { + "epoch": 2.0, + "learning_rate": 0.00041269809428284854, + "loss": 2.8129, + "theoretical_loss": 3.8375878893936255, + "tokens_seen": 603486208 + }, + { + "epoch": 2.0, + "learning_rate": 0.00041268806419257777, + "loss": 2.9922, + "theoretical_loss": 3.837544450548025, + "tokens_seen": 603551744 + }, + { + "epoch": 2.0, + "learning_rate": 0.0004126780341023069, + "loss": 2.7937, + "theoretical_loss": 3.8375010177394566, + "tokens_seen": 603617280 + }, + { + "epoch": 2.0, + "learning_rate": 0.00041266800401203613, + "loss": 2.6681, + "theoretical_loss": 3.8374575909664257, + "tokens_seen": 603682816 + }, + { + "epoch": 2.0, + "learning_rate": 0.0004126579739217653, + "loss": 2.8324, + "theoretical_loss": 3.837414170227439, + "tokens_seen": 603748352 + }, + { + "epoch": 2.0, + "learning_rate": 0.0004126479438314945, + "loss": 3.0201, + "theoretical_loss": 3.8373707555210035, + "tokens_seen": 603813888 + }, + { + "epoch": 2.0, + "learning_rate": 0.0004126379137412237, + "loss": 2.7794, + "theoretical_loss": 3.8373273468456253, + "tokens_seen": 603879424 + }, + { + "epoch": 2.0, + "learning_rate": 0.0004126278836509529, + "loss": 2.9055, + "theoretical_loss": 3.8372839441998137, + "tokens_seen": 603944960 + }, + { + "epoch": 2.0, + "learning_rate": 0.00041261785356068204, + "loss": 2.978, + "theoretical_loss": 3.8372405475820757, + "tokens_seen": 604010496 + }, + { + "epoch": 2.0, + "learning_rate": 0.0004126078234704113, + "loss": 2.9492, + "theoretical_loss": 3.8371971569909205, + "tokens_seen": 604076032 + }, + { + "epoch": 2.0, + "learning_rate": 0.0004125977933801404, + "loss": 2.8896, + "theoretical_loss": 3.837153772424858, + "tokens_seen": 604141568 + }, + { + "epoch": 2.0, + "learning_rate": 0.00041258776328986964, + "loss": 2.9618, + "theoretical_loss": 3.837110393882398, + "tokens_seen": 604207104 + }, + { + "epoch": 2.0, + "learning_rate": 0.0004125777331995988, + "loss": 2.9102, + "theoretical_loss": 3.83706702136205, + "tokens_seen": 604272640 + }, + { + "epoch": 2.0, + "learning_rate": 0.000412567703109328, + "loss": 3.0573, + "theoretical_loss": 3.8370236548623264, + "tokens_seen": 604338176 + }, + { + "epoch": 2.0, + "learning_rate": 0.0004125576730190572, + "loss": 2.7037, + "theoretical_loss": 3.8369802943817377, + "tokens_seen": 604403712 + }, + { + "epoch": 2.0, + "learning_rate": 0.00041254764292878636, + "loss": 2.8876, + "theoretical_loss": 3.8369369399187963, + "tokens_seen": 604469248 + }, + { + "epoch": 2.0, + "learning_rate": 0.00041253761283851554, + "loss": 2.9452, + "theoretical_loss": 3.836893591472015, + "tokens_seen": 604534784 + }, + { + "epoch": 2.0, + "objective/train/docs_used": 736119, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.0697572231292725, + "objective/train/theoretical_loss": 3.83687191950422, + "objective/train/tokens_used": 625027552, + "theoretical_loss": 3.83687191950422, + "tokens_seen": 604567552 + }, + { + "epoch": 2.0, + "learning_rate": 0.0004125275827482448, + "loss": 3.0517, + "theoretical_loss": 3.836850249039906, + "tokens_seen": 604600320 + }, + { + "epoch": 2.0, + "learning_rate": 0.0004125175526579739, + "loss": 2.8523, + "theoretical_loss": 3.8368069126209843, + "tokens_seen": 604665856 + }, + { + "epoch": 2.0, + "learning_rate": 0.00041250752256770314, + "loss": 2.9029, + "theoretical_loss": 3.836763582213763, + "tokens_seen": 604731392 + }, + { + "epoch": 2.0, + "learning_rate": 0.00041249749247743227, + "loss": 2.6393, + "theoretical_loss": 3.8367202578167574, + "tokens_seen": 604796928 + }, + { + "epoch": 2.0, + "learning_rate": 0.0004124874623871615, + "loss": 2.7526, + "theoretical_loss": 3.8366769394284823, + "tokens_seen": 604862464 + }, + { + "epoch": 2.0, + "learning_rate": 0.0004124774322968907, + "loss": 2.9766, + "theoretical_loss": 3.836633627047453, + "tokens_seen": 604928000 + }, + { + "epoch": 2.0, + "learning_rate": 0.00041246740220661986, + "loss": 2.8323, + "theoretical_loss": 3.8365903206721867, + "tokens_seen": 604993536 + }, + { + "epoch": 2.0, + "learning_rate": 0.00041245737211634904, + "loss": 2.823, + "theoretical_loss": 3.8365470203012, + "tokens_seen": 605059072 + }, + { + "epoch": 2.0, + "learning_rate": 0.0004124473420260783, + "loss": 2.9371, + "theoretical_loss": 3.83650372593301, + "tokens_seen": 605124608 + }, + { + "epoch": 2.0, + "learning_rate": 0.0004124373119358074, + "loss": 2.8824, + "theoretical_loss": 3.836460437566134, + "tokens_seen": 605190144 + }, + { + "epoch": 2.0, + "learning_rate": 0.00041242728184553664, + "loss": 2.9946, + "theoretical_loss": 3.8364171551990918, + "tokens_seen": 605255680 + }, + { + "epoch": 2.0, + "learning_rate": 0.00041241725175526577, + "loss": 3.0648, + "theoretical_loss": 3.836373878830401, + "tokens_seen": 605321216 + }, + { + "epoch": 2.0, + "learning_rate": 0.000412407221664995, + "loss": 2.8366, + "theoretical_loss": 3.8363306084585815, + "tokens_seen": 605386752 + }, + { + "epoch": 2.0, + "learning_rate": 0.0004123971915747242, + "loss": 2.9835, + "theoretical_loss": 3.836287344082153, + "tokens_seen": 605452288 + }, + { + "epoch": 2.0, + "learning_rate": 0.00041238716148445337, + "loss": 2.8837, + "theoretical_loss": 3.8362440856996365, + "tokens_seen": 605517824 + }, + { + "epoch": 2.0, + "learning_rate": 0.00041237713139418255, + "loss": 2.726, + "theoretical_loss": 3.8362008333095527, + "tokens_seen": 605583360 + }, + { + "epoch": 2.0, + "learning_rate": 0.00041236710130391173, + "loss": 2.8451, + "theoretical_loss": 3.8361575869104225, + "tokens_seen": 605648896 + }, + { + "epoch": 2.0, + "learning_rate": 0.0004123570712136409, + "loss": 2.7774, + "theoretical_loss": 3.836114346500769, + "tokens_seen": 605714432 + }, + { + "epoch": 2.0, + "learning_rate": 0.00041234704112337015, + "loss": 2.7372, + "theoretical_loss": 3.8360711120791144, + "tokens_seen": 605779968 + }, + { + "epoch": 2.0, + "learning_rate": 0.00041233701103309927, + "loss": 2.8797, + "theoretical_loss": 3.8360278836439816, + "tokens_seen": 605845504 + }, + { + "epoch": 2.0, + "learning_rate": 0.0004123269809428285, + "loss": 3.046, + "theoretical_loss": 3.835984661193895, + "tokens_seen": 605911040 + }, + { + "epoch": 2.0, + "learning_rate": 0.00041231695085255763, + "loss": 2.7871, + "theoretical_loss": 3.8359414447273776, + "tokens_seen": 605976576 + }, + { + "epoch": 2.0, + "learning_rate": 0.00041230692076228687, + "loss": 2.8635, + "theoretical_loss": 3.835898234242954, + "tokens_seen": 606042112 + }, + { + "epoch": 2.0, + "learning_rate": 0.00041229689067201605, + "loss": 2.9083, + "theoretical_loss": 3.8358550297391507, + "tokens_seen": 606107648 + }, + { + "epoch": 2.0, + "learning_rate": 0.00041228686058174523, + "loss": 2.666, + "theoretical_loss": 3.8358118312144924, + "tokens_seen": 606173184 + }, + { + "epoch": 2.0, + "objective/train/docs_used": 740966, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.874371290206909, + "objective/train/theoretical_loss": 3.8357902341938823, + "objective/train/tokens_used": 626665952, + "theoretical_loss": 3.8357902341938823, + "tokens_seen": 606205952 + }, + { + "epoch": 2.0, + "learning_rate": 0.0004122768304914744, + "loss": 2.7719, + "theoretical_loss": 3.835768638667506, + "tokens_seen": 606238720 + }, + { + "epoch": 2.0, + "learning_rate": 0.00041226680040120365, + "loss": 2.8218, + "theoretical_loss": 3.8357254520967174, + "tokens_seen": 606304256 + }, + { + "epoch": 2.0, + "learning_rate": 0.0004122567703109328, + "loss": 2.8802, + "theoretical_loss": 3.835682271500655, + "tokens_seen": 606369792 + }, + { + "epoch": 2.0, + "learning_rate": 0.000412246740220662, + "loss": 3.0319, + "theoretical_loss": 3.8356390968778453, + "tokens_seen": 606435328 + }, + { + "epoch": 2.0, + "learning_rate": 0.00041223671013039114, + "loss": 2.865, + "theoretical_loss": 3.8355959282268177, + "tokens_seen": 606500864 + }, + { + "epoch": 2.0, + "learning_rate": 0.0004122266800401204, + "loss": 2.9013, + "theoretical_loss": 3.8355527655461, + "tokens_seen": 606566400 + }, + { + "epoch": 2.0, + "learning_rate": 0.00041221664994984955, + "loss": 2.9206, + "theoretical_loss": 3.835509608834223, + "tokens_seen": 606631936 + }, + { + "epoch": 2.0, + "learning_rate": 0.00041220661985957874, + "loss": 2.9424, + "theoretical_loss": 3.8354664580897153, + "tokens_seen": 606697472 + }, + { + "epoch": 2.0, + "learning_rate": 0.0004121965897693079, + "loss": 2.8802, + "theoretical_loss": 3.835423313311108, + "tokens_seen": 606763008 + }, + { + "epoch": 2.0, + "learning_rate": 0.0004121865596790371, + "loss": 2.9102, + "theoretical_loss": 3.8353801744969322, + "tokens_seen": 606828544 + }, + { + "epoch": 2.0, + "learning_rate": 0.0004121765295887663, + "loss": 2.8268, + "theoretical_loss": 3.835337041645719, + "tokens_seen": 606894080 + }, + { + "epoch": 2.0, + "learning_rate": 0.0004121664994984955, + "loss": 2.9799, + "theoretical_loss": 3.8352939147559995, + "tokens_seen": 606959616 + }, + { + "epoch": 2.0, + "learning_rate": 0.0004121564694082247, + "loss": 2.7371, + "theoretical_loss": 3.835250793826308, + "tokens_seen": 607025152 + }, + { + "epoch": 2.0, + "learning_rate": 0.0004121464393179539, + "loss": 2.7789, + "theoretical_loss": 3.835207678855176, + "tokens_seen": 607090688 + }, + { + "epoch": 2.0, + "learning_rate": 0.0004121364092276831, + "loss": 2.9266, + "theoretical_loss": 3.835164569841138, + "tokens_seen": 607156224 + }, + { + "epoch": 2.0, + "learning_rate": 0.00041212637913741224, + "loss": 2.6294, + "theoretical_loss": 3.8351214667827276, + "tokens_seen": 607221760 + }, + { + "epoch": 2.0, + "learning_rate": 0.0004121163490471415, + "loss": 2.7727, + "theoretical_loss": 3.835078369678479, + "tokens_seen": 607287296 + }, + { + "epoch": 2.0, + "learning_rate": 0.0004121063189568706, + "loss": 2.9085, + "theoretical_loss": 3.835035278526928, + "tokens_seen": 607352832 + }, + { + "epoch": 2.0, + "learning_rate": 0.00041209628886659984, + "loss": 2.8811, + "theoretical_loss": 3.8349921933266105, + "tokens_seen": 607418368 + }, + { + "epoch": 2.0, + "learning_rate": 0.000412086258776329, + "loss": 2.8147, + "theoretical_loss": 3.8349491140760614, + "tokens_seen": 607483904 + }, + { + "epoch": 2.0, + "learning_rate": 0.0004120762286860582, + "loss": 2.9682, + "theoretical_loss": 3.8349060407738182, + "tokens_seen": 607549440 + }, + { + "epoch": 2.0, + "learning_rate": 0.0004120661985957874, + "loss": 2.9455, + "theoretical_loss": 3.834862973418418, + "tokens_seen": 607614976 + }, + { + "epoch": 2.0, + "learning_rate": 0.00041205616850551656, + "loss": 2.921, + "theoretical_loss": 3.834819912008398, + "tokens_seen": 607680512 + }, + { + "epoch": 2.0, + "learning_rate": 0.00041204613841524574, + "loss": 2.913, + "theoretical_loss": 3.8347768565422973, + "tokens_seen": 607746048 + }, + { + "epoch": 2.0, + "learning_rate": 0.000412036108324975, + "loss": 2.8299, + "theoretical_loss": 3.8347338070186536, + "tokens_seen": 607811584 + }, + { + "epoch": 2.0, + "objective/train/docs_used": 746024, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.745020627975464, + "objective/train/theoretical_loss": 3.834712284484797, + "objective/train/tokens_used": 628304352, + "theoretical_loss": 3.834712284484797, + "tokens_seen": 607844352 + }, + { + "epoch": 2.0, + "learning_rate": 0.0004120260782347041, + "loss": 2.9376, + "theoretical_loss": 3.834690763436007, + "tokens_seen": 607877120 + }, + { + "epoch": 2.0, + "learning_rate": 0.00041201604814443334, + "loss": 2.8025, + "theoretical_loss": 3.8346477257928964, + "tokens_seen": 607942656 + }, + { + "epoch": 2.0, + "learning_rate": 0.00041200601805416247, + "loss": 2.9184, + "theoretical_loss": 3.8346046940878624, + "tokens_seen": 608008192 + }, + { + "epoch": 2.0, + "learning_rate": 0.0004119959879638917, + "loss": 2.8953, + "theoretical_loss": 3.834561668319447, + "tokens_seen": 608073728 + }, + { + "epoch": 2.0, + "learning_rate": 0.0004119859578736209, + "loss": 2.7333, + "theoretical_loss": 3.834518648486189, + "tokens_seen": 608139264 + }, + { + "epoch": 2.0, + "learning_rate": 0.00041197592778335006, + "loss": 2.8579, + "theoretical_loss": 3.8344756345866324, + "tokens_seen": 608204800 + }, + { + "epoch": 2.0, + "learning_rate": 0.00041196589769307924, + "loss": 2.8816, + "theoretical_loss": 3.834432626619318, + "tokens_seen": 608270336 + }, + { + "epoch": 2.0, + "learning_rate": 0.0004119558676028085, + "loss": 2.8811, + "theoretical_loss": 3.8343896245827898, + "tokens_seen": 608335872 + }, + { + "epoch": 2.0, + "learning_rate": 0.0004119458375125376, + "loss": 2.997, + "theoretical_loss": 3.8343466284755907, + "tokens_seen": 608401408 + }, + { + "epoch": 2.0, + "learning_rate": 0.00041193580742226684, + "loss": 2.8126, + "theoretical_loss": 3.8343036382962645, + "tokens_seen": 608466944 + }, + { + "epoch": 2.0, + "learning_rate": 0.00041192577733199597, + "loss": 2.9976, + "theoretical_loss": 3.834260654043356, + "tokens_seen": 608532480 + }, + { + "epoch": 2.0, + "learning_rate": 0.0004119157472417252, + "loss": 2.8142, + "theoretical_loss": 3.8342176757154087, + "tokens_seen": 608598016 + }, + { + "epoch": 2.0, + "learning_rate": 0.0004119057171514544, + "loss": 2.7509, + "theoretical_loss": 3.8341747033109703, + "tokens_seen": 608663552 + }, + { + "epoch": 2.0, + "learning_rate": 0.00041189568706118357, + "loss": 2.6745, + "theoretical_loss": 3.8341317368285845, + "tokens_seen": 608729088 + }, + { + "epoch": 2.0, + "learning_rate": 0.00041188565697091275, + "loss": 2.8161, + "theoretical_loss": 3.8340887762667992, + "tokens_seen": 608794624 + }, + { + "epoch": 2.0, + "learning_rate": 0.00041187562688064193, + "loss": 2.9308, + "theoretical_loss": 3.8340458216241613, + "tokens_seen": 608860160 + }, + { + "epoch": 2.0, + "learning_rate": 0.0004118655967903711, + "loss": 2.8721, + "theoretical_loss": 3.834002872899217, + "tokens_seen": 608925696 + }, + { + "epoch": 2.0, + "learning_rate": 0.00041185556670010035, + "loss": 2.7833, + "theoretical_loss": 3.833959930090516, + "tokens_seen": 608991232 + }, + { + "epoch": 2.0, + "learning_rate": 0.00041184553660982947, + "loss": 2.8491, + "theoretical_loss": 3.8339169931966053, + "tokens_seen": 609056768 + }, + { + "epoch": 2.0, + "learning_rate": 0.0004118355065195587, + "loss": 2.8048, + "theoretical_loss": 3.8338740622160343, + "tokens_seen": 609122304 + }, + { + "epoch": 2.0, + "learning_rate": 0.00041182547642928783, + "loss": 2.8229, + "theoretical_loss": 3.833831137147353, + "tokens_seen": 609187840 + }, + { + "epoch": 2.0, + "learning_rate": 0.00041181544633901707, + "loss": 3.0615, + "theoretical_loss": 3.8337882179891114, + "tokens_seen": 609253376 + }, + { + "epoch": 2.0, + "learning_rate": 0.00041180541624874625, + "loss": 2.6738, + "theoretical_loss": 3.83374530473986, + "tokens_seen": 609318912 + }, + { + "epoch": 2.0, + "learning_rate": 0.00041179538615847543, + "loss": 2.706, + "theoretical_loss": 3.833702397398149, + "tokens_seen": 609384448 + }, + { + "epoch": 2.0, + "learning_rate": 0.0004117853560682046, + "loss": 2.7287, + "theoretical_loss": 3.8336594959625314, + "tokens_seen": 609449984 + }, + { + "epoch": 2.0, + "objective/train/docs_used": 751044, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.0871493816375732, + "objective/train/theoretical_loss": 3.833638047459054, + "objective/train/tokens_used": 629942752, + "theoretical_loss": 3.833638047459054, + "tokens_seen": 609482752 + }, + { + "epoch": 2.0, + "learning_rate": 0.00041177532597793385, + "loss": 3.0586, + "theoretical_loss": 3.833616600431558, + "tokens_seen": 609515520 + }, + { + "epoch": 2.0, + "learning_rate": 0.000411765295887663, + "loss": 2.8549, + "theoretical_loss": 3.8335737108037815, + "tokens_seen": 609581056 + }, + { + "epoch": 2.0, + "learning_rate": 0.0004117552657973922, + "loss": 2.872, + "theoretical_loss": 3.833530827077756, + "tokens_seen": 609646592 + }, + { + "epoch": 2.0, + "learning_rate": 0.00041174523570712134, + "loss": 2.8627, + "theoretical_loss": 3.833487949252034, + "tokens_seen": 609712128 + }, + { + "epoch": 2.0, + "learning_rate": 0.0004117352056168506, + "loss": 2.8217, + "theoretical_loss": 3.8334450773251705, + "tokens_seen": 609777664 + }, + { + "epoch": 2.0, + "learning_rate": 0.00041172517552657975, + "loss": 2.99, + "theoretical_loss": 3.833402211295719, + "tokens_seen": 609843200 + }, + { + "epoch": 2.0, + "learning_rate": 0.00041171514543630894, + "loss": 2.7994, + "theoretical_loss": 3.833359351162236, + "tokens_seen": 609908736 + }, + { + "epoch": 2.0, + "learning_rate": 0.0004117051153460381, + "loss": 2.9186, + "theoretical_loss": 3.8333164969232767, + "tokens_seen": 609974272 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004116950852557673, + "loss": 2.9245, + "theoretical_loss": 3.8332736485773964, + "tokens_seen": 610039808 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004116850551654965, + "loss": 2.7503, + "theoretical_loss": 3.8332308061231526, + "tokens_seen": 610105344 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004116750250752257, + "loss": 2.7762, + "theoretical_loss": 3.833187969559102, + "tokens_seen": 610170880 + }, + { + "epoch": 2.01, + "learning_rate": 0.00041166499498495484, + "loss": 3.0806, + "theoretical_loss": 3.833145138883803, + "tokens_seen": 610236416 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004116549648946841, + "loss": 2.5722, + "theoretical_loss": 3.8331023140958127, + "tokens_seen": 610301952 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004116449348044132, + "loss": 2.9739, + "theoretical_loss": 3.833059495193691, + "tokens_seen": 610367488 + }, + { + "epoch": 2.01, + "learning_rate": 0.00041163490471414244, + "loss": 2.908, + "theoretical_loss": 3.833016682175996, + "tokens_seen": 610433024 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004116248746238716, + "loss": 2.9373, + "theoretical_loss": 3.8329738750412883, + "tokens_seen": 610498560 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004116148445336008, + "loss": 2.7929, + "theoretical_loss": 3.832931073788127, + "tokens_seen": 610564096 + }, + { + "epoch": 2.01, + "learning_rate": 0.00041160481444333, + "loss": 2.9314, + "theoretical_loss": 3.8328882784150746, + "tokens_seen": 610629632 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004115947843530592, + "loss": 2.9051, + "theoretical_loss": 3.8328454889206904, + "tokens_seen": 610695168 + }, + { + "epoch": 2.01, + "learning_rate": 0.00041158475426278834, + "loss": 2.8858, + "theoretical_loss": 3.8328027053035374, + "tokens_seen": 610760704 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004115747241725176, + "loss": 3.0172, + "theoretical_loss": 3.8327599275621775, + "tokens_seen": 610826240 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004115646940822467, + "loss": 2.8714, + "theoretical_loss": 3.832717155695173, + "tokens_seen": 610891776 + }, + { + "epoch": 2.01, + "learning_rate": 0.00041155466399197594, + "loss": 2.7776, + "theoretical_loss": 3.832674389701088, + "tokens_seen": 610957312 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004115446339017051, + "loss": 2.892, + "theoretical_loss": 3.8326316295784855, + "tokens_seen": 611022848 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004115346038114343, + "loss": 2.9538, + "theoretical_loss": 3.8325888753259303, + "tokens_seen": 611088384 + }, + { + "epoch": 2.01, + "objective/train/docs_used": 756149, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.102915048599243, + "objective/train/theoretical_loss": 3.8325675004004713, + "objective/train/tokens_used": 631581152, + "theoretical_loss": 3.8325675004004713, + "tokens_seen": 611121152 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004115245737211635, + "loss": 2.9014, + "theoretical_loss": 3.8325461269419865, + "tokens_seen": 611153920 + }, + { + "epoch": 2.01, + "learning_rate": 0.00041151454363089267, + "loss": 2.8073, + "theoretical_loss": 3.8325033844252197, + "tokens_seen": 611219456 + }, + { + "epoch": 2.01, + "learning_rate": 0.00041150451354062185, + "loss": 2.8775, + "theoretical_loss": 3.8324606477741963, + "tokens_seen": 611284992 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004114944834503511, + "loss": 2.8357, + "theoretical_loss": 3.832417916987482, + "tokens_seen": 611350528 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004114844533600802, + "loss": 2.7828, + "theoretical_loss": 3.832375192063643, + "tokens_seen": 611416064 + }, + { + "epoch": 2.01, + "learning_rate": 0.00041147442326980944, + "loss": 2.8968, + "theoretical_loss": 3.832332473001247, + "tokens_seen": 611481600 + }, + { + "epoch": 2.01, + "learning_rate": 0.00041146439317953857, + "loss": 2.957, + "theoretical_loss": 3.832289759798863, + "tokens_seen": 611547136 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004114543630892678, + "loss": 2.8542, + "theoretical_loss": 3.832247052455058, + "tokens_seen": 611612672 + }, + { + "epoch": 2.01, + "learning_rate": 0.000411444332998997, + "loss": 2.9705, + "theoretical_loss": 3.832204350968401, + "tokens_seen": 611678208 + }, + { + "epoch": 2.01, + "learning_rate": 0.00041143430290872617, + "loss": 2.7964, + "theoretical_loss": 3.832161655337462, + "tokens_seen": 611743744 + }, + { + "epoch": 2.01, + "learning_rate": 0.00041142427281845535, + "loss": 2.6554, + "theoretical_loss": 3.832118965560809, + "tokens_seen": 611809280 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004114142427281846, + "loss": 2.8301, + "theoretical_loss": 3.832076281637014, + "tokens_seen": 611874816 + }, + { + "epoch": 2.01, + "learning_rate": 0.00041140421263791377, + "loss": 2.6453, + "theoretical_loss": 3.8320336035646476, + "tokens_seen": 611940352 + }, + { + "epoch": 2.01, + "learning_rate": 0.00041139418254764295, + "loss": 2.846, + "theoretical_loss": 3.831990931342281, + "tokens_seen": 612005888 + }, + { + "epoch": 2.01, + "learning_rate": 0.00041138415245737213, + "loss": 2.8807, + "theoretical_loss": 3.831948264968485, + "tokens_seen": 612071424 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004113741223671013, + "loss": 2.842, + "theoretical_loss": 3.8319056044418334, + "tokens_seen": 612136960 + }, + { + "epoch": 2.01, + "learning_rate": 0.00041136409227683055, + "loss": 2.9488, + "theoretical_loss": 3.8318629497608985, + "tokens_seen": 612202496 + }, + { + "epoch": 2.01, + "learning_rate": 0.00041135406218655967, + "loss": 2.8403, + "theoretical_loss": 3.831820300924253, + "tokens_seen": 612268032 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004113440320962889, + "loss": 2.8262, + "theoretical_loss": 3.8317776579304716, + "tokens_seen": 612333568 + }, + { + "epoch": 2.01, + "learning_rate": 0.00041133400200601803, + "loss": 2.9601, + "theoretical_loss": 3.8317350207781278, + "tokens_seen": 612399104 + }, + { + "epoch": 2.01, + "learning_rate": 0.00041132397191574727, + "loss": 2.8308, + "theoretical_loss": 3.831692389465797, + "tokens_seen": 612464640 + }, + { + "epoch": 2.01, + "learning_rate": 0.00041131394182547645, + "loss": 2.7944, + "theoretical_loss": 3.831649763992054, + "tokens_seen": 612530176 + }, + { + "epoch": 2.01, + "learning_rate": 0.00041130391173520563, + "loss": 2.942, + "theoretical_loss": 3.831607144355476, + "tokens_seen": 612595712 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004112938816449348, + "loss": 2.8861, + "theoretical_loss": 3.8315645305546377, + "tokens_seen": 612661248 + }, + { + "epoch": 2.01, + "learning_rate": 0.00041128385155466405, + "loss": 2.8315, + "theoretical_loss": 3.8315219225881165, + "tokens_seen": 612726784 + }, + { + "epoch": 2.01, + "objective/train/docs_used": 761270, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.108196973800659, + "objective/train/theoretical_loss": 3.83150062079228, + "objective/train/tokens_used": 633219552, + "theoretical_loss": 3.83150062079228, + "tokens_seen": 612759552 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004112738214643932, + "loss": 2.84, + "theoretical_loss": 3.8314793204544895, + "tokens_seen": 612792320 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004112637913741224, + "loss": 2.7863, + "theoretical_loss": 3.831436724152335, + "tokens_seen": 612857856 + }, + { + "epoch": 2.01, + "learning_rate": 0.00041125376128385154, + "loss": 2.9543, + "theoretical_loss": 3.831394133680231, + "tokens_seen": 612923392 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004112437311935808, + "loss": 2.9146, + "theoretical_loss": 3.8313515490367562, + "tokens_seen": 612988928 + }, + { + "epoch": 2.01, + "learning_rate": 0.00041123370110330995, + "loss": 2.8663, + "theoretical_loss": 3.8313089702204906, + "tokens_seen": 613054464 + }, + { + "epoch": 2.01, + "learning_rate": 0.00041122367101303914, + "loss": 2.9834, + "theoretical_loss": 3.831266397230013, + "tokens_seen": 613120000 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004112136409227683, + "loss": 2.9004, + "theoretical_loss": 3.8312238300639048, + "tokens_seen": 613185536 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004112036108324975, + "loss": 2.9223, + "theoretical_loss": 3.8311812687207456, + "tokens_seen": 613251072 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004111935807422267, + "loss": 2.9421, + "theoretical_loss": 3.8311387131991173, + "tokens_seen": 613316608 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004111835506519559, + "loss": 2.9294, + "theoretical_loss": 3.831096163497602, + "tokens_seen": 613382144 + }, + { + "epoch": 2.01, + "learning_rate": 0.00041117352056168504, + "loss": 2.7985, + "theoretical_loss": 3.8310536196147815, + "tokens_seen": 613447680 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004111634904714143, + "loss": 2.8329, + "theoretical_loss": 3.831011081549239, + "tokens_seen": 613513216 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004111534603811434, + "loss": 3.1471, + "theoretical_loss": 3.8309685492995573, + "tokens_seen": 613578752 + }, + { + "epoch": 2.01, + "learning_rate": 0.00041114343029087264, + "loss": 2.9241, + "theoretical_loss": 3.830926022864321, + "tokens_seen": 613644288 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004111334002006018, + "loss": 2.7, + "theoretical_loss": 3.8308835022421137, + "tokens_seen": 613709824 + }, + { + "epoch": 2.01, + "learning_rate": 0.000411123370110331, + "loss": 2.9448, + "theoretical_loss": 3.83084098743152, + "tokens_seen": 613775360 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004111133400200602, + "loss": 2.6292, + "theoretical_loss": 3.8307984784311255, + "tokens_seen": 613840896 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004111033099297894, + "loss": 2.8419, + "theoretical_loss": 3.8307559752395166, + "tokens_seen": 613906432 + }, + { + "epoch": 2.01, + "learning_rate": 0.00041109327983951854, + "loss": 3.0578, + "theoretical_loss": 3.8307134778552783, + "tokens_seen": 613971968 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004110832497492478, + "loss": 2.6516, + "theoretical_loss": 3.8306709862769983, + "tokens_seen": 614037504 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004110732196589769, + "loss": 2.8264, + "theoretical_loss": 3.830628500503263, + "tokens_seen": 614103040 + }, + { + "epoch": 2.01, + "learning_rate": 0.00041106318956870614, + "loss": 2.635, + "theoretical_loss": 3.830586020532661, + "tokens_seen": 614168576 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004110531594784353, + "loss": 2.9087, + "theoretical_loss": 3.830543546363781, + "tokens_seen": 614234112 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004110431293881645, + "loss": 2.8193, + "theoretical_loss": 3.8305010779952102, + "tokens_seen": 614299648 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004110330992978937, + "loss": 2.8163, + "theoretical_loss": 3.8304586154255387, + "tokens_seen": 614365184 + }, + { + "epoch": 2.01, + "objective/train/docs_used": 766282, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.869065046310425, + "objective/train/theoretical_loss": 3.8304373863148493, + "objective/train/tokens_used": 634857952, + "theoretical_loss": 3.8304373863148493, + "tokens_seen": 614397952 + }, + { + "epoch": 2.01, + "learning_rate": 0.00041102306920762287, + "loss": 2.9004, + "theoretical_loss": 3.8304161586533563, + "tokens_seen": 614430720 + }, + { + "epoch": 2.01, + "learning_rate": 0.00041101303911735205, + "loss": 2.8674, + "theoretical_loss": 3.830373707677253, + "tokens_seen": 614496256 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004110030090270813, + "loss": 2.8783, + "theoretical_loss": 3.83033126249582, + "tokens_seen": 614561792 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004109929789368104, + "loss": 3.0358, + "theoretical_loss": 3.8302888231076473, + "tokens_seen": 614627328 + }, + { + "epoch": 2.01, + "learning_rate": 0.00041098294884653965, + "loss": 2.8569, + "theoretical_loss": 3.830246389511328, + "tokens_seen": 614692864 + }, + { + "epoch": 2.01, + "learning_rate": 0.00041097291875626877, + "loss": 2.7641, + "theoretical_loss": 3.8302039617054526, + "tokens_seen": 614758400 + }, + { + "epoch": 2.01, + "learning_rate": 0.000410962888665998, + "loss": 2.9641, + "theoretical_loss": 3.830161539688616, + "tokens_seen": 614823936 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004109528585757272, + "loss": 2.8592, + "theoretical_loss": 3.8301191234594096, + "tokens_seen": 614889472 + }, + { + "epoch": 2.01, + "learning_rate": 0.00041094282848545637, + "loss": 2.8512, + "theoretical_loss": 3.830076713016428, + "tokens_seen": 614955008 + }, + { + "epoch": 2.01, + "learning_rate": 0.00041093279839518555, + "loss": 2.6787, + "theoretical_loss": 3.8300343083582646, + "tokens_seen": 615020544 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004109227683049148, + "loss": 2.7377, + "theoretical_loss": 3.829991909483515, + "tokens_seen": 615086080 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004109127382146439, + "loss": 3.0285, + "theoretical_loss": 3.8299495163907733, + "tokens_seen": 615151616 + }, + { + "epoch": 2.01, + "learning_rate": 0.00041090270812437315, + "loss": 2.7276, + "theoretical_loss": 3.829907129078636, + "tokens_seen": 615217152 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004108926780341023, + "loss": 2.821, + "theoretical_loss": 3.829864747545699, + "tokens_seen": 615282688 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004108826479438315, + "loss": 2.8471, + "theoretical_loss": 3.8298223717905584, + "tokens_seen": 615348224 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004108726178535607, + "loss": 2.6164, + "theoretical_loss": 3.8297800018118116, + "tokens_seen": 615413760 + }, + { + "epoch": 2.01, + "learning_rate": 0.00041086258776328987, + "loss": 2.7788, + "theoretical_loss": 3.8297376376080567, + "tokens_seen": 615479296 + }, + { + "epoch": 2.01, + "learning_rate": 0.00041085255767301905, + "loss": 2.6422, + "theoretical_loss": 3.829695279177891, + "tokens_seen": 615544832 + }, + { + "epoch": 2.01, + "learning_rate": 0.00041084252758274823, + "loss": 2.924, + "theoretical_loss": 3.8296529265199135, + "tokens_seen": 615610368 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004108324974924774, + "loss": 2.9178, + "theoretical_loss": 3.8296105796327233, + "tokens_seen": 615675904 + }, + { + "epoch": 2.01, + "learning_rate": 0.00041082246740220665, + "loss": 2.8156, + "theoretical_loss": 3.82956823851492, + "tokens_seen": 615741440 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004108124373119358, + "loss": 2.8831, + "theoretical_loss": 3.829525903165103, + "tokens_seen": 615806976 + }, + { + "epoch": 2.01, + "learning_rate": 0.000410802407221665, + "loss": 2.7111, + "theoretical_loss": 3.8294835735818733, + "tokens_seen": 615872512 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004107923771313942, + "loss": 2.9392, + "theoretical_loss": 3.8294412497638324, + "tokens_seen": 615938048 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004107823470411234, + "loss": 2.6815, + "theoretical_loss": 3.829398931709581, + "tokens_seen": 616003584 + }, + { + "epoch": 2.01, + "objective/train/docs_used": 771277, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5148863792419434, + "objective/train/theoretical_loss": 3.829377774843439, + "objective/train/tokens_used": 636496352, + "theoretical_loss": 3.829377774843439, + "tokens_seen": 616036352 + }, + { + "epoch": 2.01, + "learning_rate": 0.00041077231695085256, + "loss": 2.7967, + "theoretical_loss": 3.8293566194177213, + "tokens_seen": 616069120 + }, + { + "epoch": 2.01, + "learning_rate": 0.00041076228686058174, + "loss": 2.6174, + "theoretical_loss": 3.829314312886856, + "tokens_seen": 616134656 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004107522567703109, + "loss": 2.7027, + "theoretical_loss": 3.829272012115588, + "tokens_seen": 616200192 + }, + { + "epoch": 2.01, + "learning_rate": 0.00041074222668004015, + "loss": 2.5431, + "theoretical_loss": 3.8292297171025202, + "tokens_seen": 616265728 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004107321965897693, + "loss": 2.7535, + "theoretical_loss": 3.8291874278462577, + "tokens_seen": 616331264 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004107221664994985, + "loss": 2.8045, + "theoretical_loss": 3.8291451443454037, + "tokens_seen": 616396800 + }, + { + "epoch": 2.01, + "learning_rate": 0.00041071213640922764, + "loss": 2.6862, + "theoretical_loss": 3.8291028665985642, + "tokens_seen": 616462336 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004107021063189569, + "loss": 2.6272, + "theoretical_loss": 3.8290605946043437, + "tokens_seen": 616527872 + }, + { + "epoch": 2.01, + "learning_rate": 0.00041069207622868606, + "loss": 2.736, + "theoretical_loss": 3.829018328361349, + "tokens_seen": 616593408 + }, + { + "epoch": 2.01, + "learning_rate": 0.00041068204613841524, + "loss": 2.9518, + "theoretical_loss": 3.8289760678681852, + "tokens_seen": 616658944 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004106720160481444, + "loss": 2.8972, + "theoretical_loss": 3.8289338131234603, + "tokens_seen": 616724480 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004106619859578736, + "loss": 2.8132, + "theoretical_loss": 3.8288915641257812, + "tokens_seen": 616790016 + }, + { + "epoch": 2.01, + "learning_rate": 0.00041065195586760284, + "loss": 2.9286, + "theoretical_loss": 3.828849320873756, + "tokens_seen": 616855552 + }, + { + "epoch": 2.01, + "learning_rate": 0.000410641925777332, + "loss": 2.6631, + "theoretical_loss": 3.828807083365992, + "tokens_seen": 616921088 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004106318956870612, + "loss": 2.8104, + "theoretical_loss": 3.8287648516010995, + "tokens_seen": 616986624 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004106218655967904, + "loss": 2.8351, + "theoretical_loss": 3.8287226255776865, + "tokens_seen": 617052160 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004106118355065196, + "loss": 2.757, + "theoretical_loss": 3.8286804052943637, + "tokens_seen": 617117696 + }, + { + "epoch": 2.01, + "learning_rate": 0.00041060180541624874, + "loss": 2.9234, + "theoretical_loss": 3.828638190749741, + "tokens_seen": 617183232 + }, + { + "epoch": 2.01, + "learning_rate": 0.000410591775325978, + "loss": 2.6066, + "theoretical_loss": 3.8285959819424296, + "tokens_seen": 617248768 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004105817452357071, + "loss": 2.9455, + "theoretical_loss": 3.828553778871039, + "tokens_seen": 617314304 + }, + { + "epoch": 2.01, + "learning_rate": 0.00041057171514543634, + "loss": 2.924, + "theoretical_loss": 3.8285115815341833, + "tokens_seen": 617379840 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004105616850551655, + "loss": 2.7455, + "theoretical_loss": 3.828469389930473, + "tokens_seen": 617445376 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004105516549648947, + "loss": 2.7622, + "theoretical_loss": 3.8284272040585208, + "tokens_seen": 617510912 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004105416248746239, + "loss": 2.8468, + "theoretical_loss": 3.8283850239169412, + "tokens_seen": 617576448 + }, + { + "epoch": 2.01, + "learning_rate": 0.00041053159478435307, + "loss": 2.74, + "theoretical_loss": 3.8283428495043466, + "tokens_seen": 617641984 + }, + { + "epoch": 2.01, + "objective/train/docs_used": 772734, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.1529695987701416, + "objective/train/theoretical_loss": 3.8283217644459855, + "objective/train/tokens_used": 638134752, + "theoretical_loss": 3.8283217644459855, + "tokens_seen": 617674752 + }, + { + "epoch": 2.01, + "learning_rate": 0.00041052156469408225, + "loss": 2.6459, + "theoretical_loss": 3.8283006808193516, + "tokens_seen": 617707520 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004105115346038115, + "loss": 3.0692, + "theoretical_loss": 3.8282585178605704, + "tokens_seen": 617773056 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004105015045135406, + "loss": 2.944, + "theoretical_loss": 3.828216360626619, + "tokens_seen": 617838592 + }, + { + "epoch": 2.01, + "learning_rate": 0.00041049147442326985, + "loss": 2.9468, + "theoretical_loss": 3.8281742091161117, + "tokens_seen": 617904128 + }, + { + "epoch": 2.01, + "learning_rate": 0.00041048144433299897, + "loss": 2.9138, + "theoretical_loss": 3.828132063327666, + "tokens_seen": 617969664 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004104714142427282, + "loss": 2.7093, + "theoretical_loss": 3.8280899232598973, + "tokens_seen": 618035200 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004104613841524574, + "loss": 2.9611, + "theoretical_loss": 3.8280477889114226, + "tokens_seen": 618100736 + }, + { + "epoch": 2.01, + "learning_rate": 0.00041045135406218657, + "loss": 2.8029, + "theoretical_loss": 3.82800566028086, + "tokens_seen": 618166272 + }, + { + "epoch": 2.01, + "learning_rate": 0.00041044132397191575, + "loss": 2.9444, + "theoretical_loss": 3.827963537366827, + "tokens_seen": 618231808 + }, + { + "epoch": 2.01, + "learning_rate": 0.000410431293881645, + "loss": 2.8183, + "theoretical_loss": 3.827921420167942, + "tokens_seen": 618297344 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004104212637913741, + "loss": 2.9094, + "theoretical_loss": 3.827879308682825, + "tokens_seen": 618362880 + }, + { + "epoch": 2.01, + "learning_rate": 0.00041041123370110335, + "loss": 2.7128, + "theoretical_loss": 3.8278372029100938, + "tokens_seen": 618428416 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004104012036108325, + "loss": 2.8372, + "theoretical_loss": 3.8277951028483694, + "tokens_seen": 618493952 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004103911735205617, + "loss": 3.0235, + "theoretical_loss": 3.8277530084962716, + "tokens_seen": 618559488 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004103811434302909, + "loss": 2.7758, + "theoretical_loss": 3.827710919852422, + "tokens_seen": 618625024 + }, + { + "epoch": 2.01, + "learning_rate": 0.00041037111334002007, + "loss": 2.7514, + "theoretical_loss": 3.827668836915441, + "tokens_seen": 618690560 + }, + { + "epoch": 2.01, + "learning_rate": 0.00041036108324974925, + "loss": 2.6761, + "theoretical_loss": 3.8276267596839513, + "tokens_seen": 618756096 + }, + { + "epoch": 2.01, + "learning_rate": 0.00041035105315947844, + "loss": 2.8231, + "theoretical_loss": 3.8275846881565743, + "tokens_seen": 618821632 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004103410230692076, + "loss": 2.8471, + "theoretical_loss": 3.827542622331933, + "tokens_seen": 618887168 + }, + { + "epoch": 2.01, + "learning_rate": 0.00041033099297893685, + "loss": 2.7828, + "theoretical_loss": 3.827500562208651, + "tokens_seen": 618952704 + }, + { + "epoch": 2.01, + "learning_rate": 0.000410320962888666, + "loss": 3.0159, + "theoretical_loss": 3.8274585077853525, + "tokens_seen": 619018240 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004103109327983952, + "loss": 2.8519, + "theoretical_loss": 3.82741645906066, + "tokens_seen": 619083776 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004103009027081244, + "loss": 2.7824, + "theoretical_loss": 3.8273744160331997, + "tokens_seen": 619149312 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004102908726178536, + "loss": 2.8297, + "theoretical_loss": 3.8273323787015965, + "tokens_seen": 619214848 + }, + { + "epoch": 2.01, + "learning_rate": 0.00041028084252758276, + "loss": 2.9612, + "theoretical_loss": 3.8272903470644755, + "tokens_seen": 619280384 + }, + { + "epoch": 2.01, + "objective/train/docs_used": 773070, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.051236152648926, + "objective/train/theoretical_loss": 3.827269333380917, + "objective/train/tokens_used": 639773152, + "theoretical_loss": 3.827269333380917, + "tokens_seen": 619313152 + }, + { + "epoch": 2.01, + "learning_rate": 0.00041027081243731194, + "loss": 2.9915, + "theoretical_loss": 3.8272483211204635, + "tokens_seen": 619345920 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004102607823470411, + "loss": 2.84, + "theoretical_loss": 3.8272063008681867, + "tokens_seen": 619411456 + }, + { + "epoch": 2.01, + "learning_rate": 0.00041025075225677035, + "loss": 2.7795, + "theoretical_loss": 3.827164286306272, + "tokens_seen": 619476992 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004102407221664995, + "loss": 2.9163, + "theoretical_loss": 3.8271222774333467, + "tokens_seen": 619542528 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004102306920762287, + "loss": 2.7408, + "theoretical_loss": 3.8270802742480394, + "tokens_seen": 619608064 + }, + { + "epoch": 2.01, + "learning_rate": 0.00041022066198595784, + "loss": 2.7077, + "theoretical_loss": 3.8270382767489783, + "tokens_seen": 619673600 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004102106318956871, + "loss": 2.7924, + "theoretical_loss": 3.8269962849347934, + "tokens_seen": 619739136 + }, + { + "epoch": 2.01, + "learning_rate": 0.00041020060180541626, + "loss": 2.9734, + "theoretical_loss": 3.8269542988041123, + "tokens_seen": 619804672 + }, + { + "epoch": 2.01, + "learning_rate": 0.00041019057171514544, + "loss": 2.8563, + "theoretical_loss": 3.826912318355566, + "tokens_seen": 619870208 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004101805416248746, + "loss": 2.7356, + "theoretical_loss": 3.826870343587785, + "tokens_seen": 619935744 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004101705115346038, + "loss": 2.94, + "theoretical_loss": 3.8268283744993994, + "tokens_seen": 620001280 + }, + { + "epoch": 2.01, + "learning_rate": 0.000410160481444333, + "loss": 3.0616, + "theoretical_loss": 3.8267864110890413, + "tokens_seen": 620066816 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004101504513540622, + "loss": 2.8621, + "theoretical_loss": 3.8267444533553423, + "tokens_seen": 620132352 + }, + { + "epoch": 2.01, + "learning_rate": 0.00041014042126379135, + "loss": 2.7912, + "theoretical_loss": 3.8267025012969347, + "tokens_seen": 620197888 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004101303911735206, + "loss": 2.8802, + "theoretical_loss": 3.8266605549124515, + "tokens_seen": 620263424 + }, + { + "epoch": 2.01, + "learning_rate": 0.00041012036108324976, + "loss": 2.7613, + "theoretical_loss": 3.8266186142005254, + "tokens_seen": 620328960 + }, + { + "epoch": 2.01, + "learning_rate": 0.00041011033099297894, + "loss": 2.8051, + "theoretical_loss": 3.82657667915979, + "tokens_seen": 620394496 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004101003009027081, + "loss": 2.8462, + "theoretical_loss": 3.8265347497888804, + "tokens_seen": 620460032 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004100902708124373, + "loss": 2.8898, + "theoretical_loss": 3.8264928260864304, + "tokens_seen": 620525568 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004100802407221665, + "loss": 2.881, + "theoretical_loss": 3.8264509080510756, + "tokens_seen": 620591104 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004100702106318957, + "loss": 2.6456, + "theoretical_loss": 3.8264089956814513, + "tokens_seen": 620656640 + }, + { + "epoch": 2.01, + "learning_rate": 0.00041006018054162485, + "loss": 2.9374, + "theoretical_loss": 3.826367088976194, + "tokens_seen": 620722176 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004100501504513541, + "loss": 2.9263, + "theoretical_loss": 3.8263251879339393, + "tokens_seen": 620787712 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004100401203610832, + "loss": 2.9678, + "theoretical_loss": 3.8262832925533257, + "tokens_seen": 620853248 + }, + { + "epoch": 2.01, + "learning_rate": 0.00041003009027081245, + "loss": 2.7197, + "theoretical_loss": 3.826241402832989, + "tokens_seen": 620918784 + }, + { + "epoch": 2.01, + "objective/train/docs_used": 774439, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7738590240478516, + "objective/train/theoretical_loss": 3.8262204600949996, + "objective/train/tokens_used": 641411552, + "theoretical_loss": 3.8262204600949996, + "tokens_seen": 620951552 + }, + { + "epoch": 2.01, + "learning_rate": 0.00041002006018054163, + "loss": 2.8329, + "theoretical_loss": 3.8261995187715687, + "tokens_seen": 620984320 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004100100300902708, + "loss": 2.9793, + "theoretical_loss": 3.8261576403677027, + "tokens_seen": 621049856 + }, + { + "epoch": 2.01, + "learning_rate": 0.00041, + "loss": 2.8844, + "theoretical_loss": 3.8261157676200295, + "tokens_seen": 621115392 + }, + { + "epoch": 2.01, + "learning_rate": 0.00040998996990972917, + "loss": 2.8916, + "theoretical_loss": 3.826073900527189, + "tokens_seen": 621180928 + }, + { + "epoch": 2.01, + "learning_rate": 0.00040997993981945835, + "loss": 2.8177, + "theoretical_loss": 3.8260320390878206, + "tokens_seen": 621246464 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004099699097291876, + "loss": 2.5451, + "theoretical_loss": 3.8259901833005645, + "tokens_seen": 621312000 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004099598796389167, + "loss": 2.862, + "theoretical_loss": 3.825948333164062, + "tokens_seen": 621377536 + }, + { + "epoch": 2.01, + "learning_rate": 0.00040994984954864595, + "loss": 2.7519, + "theoretical_loss": 3.8259064886769547, + "tokens_seen": 621443072 + }, + { + "epoch": 2.01, + "learning_rate": 0.00040993981945837513, + "loss": 2.8865, + "theoretical_loss": 3.825864649837883, + "tokens_seen": 621508608 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004099297893681043, + "loss": 2.8783, + "theoretical_loss": 3.825822816645491, + "tokens_seen": 621574144 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004099197592778335, + "loss": 3.0324, + "theoretical_loss": 3.8257809890984196, + "tokens_seen": 621639680 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004099097291875627, + "loss": 2.9964, + "theoretical_loss": 3.8257391671953127, + "tokens_seen": 621705216 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004098996990972919, + "loss": 2.9455, + "theoretical_loss": 3.825697350934814, + "tokens_seen": 621770752 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004098896690070211, + "loss": 2.9301, + "theoretical_loss": 3.8256555403155668, + "tokens_seen": 621836288 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004098796389167503, + "loss": 2.7822, + "theoretical_loss": 3.8256137353362174, + "tokens_seen": 621901824 + }, + { + "epoch": 2.01, + "learning_rate": 0.00040986960882647945, + "loss": 2.723, + "theoretical_loss": 3.8255719359954083, + "tokens_seen": 621967360 + }, + { + "epoch": 2.01, + "learning_rate": 0.00040985957873620864, + "loss": 2.733, + "theoretical_loss": 3.8255301422917873, + "tokens_seen": 622032896 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004098495486459378, + "loss": 2.935, + "theoretical_loss": 3.8254883542239995, + "tokens_seen": 622098432 + }, + { + "epoch": 2.01, + "learning_rate": 0.00040983951855566705, + "loss": 2.8432, + "theoretical_loss": 3.8254465717906907, + "tokens_seen": 622163968 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004098294884653962, + "loss": 2.7378, + "theoretical_loss": 3.8254047949905083, + "tokens_seen": 622229504 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004098194583751254, + "loss": 2.7812, + "theoretical_loss": 3.8253630238221, + "tokens_seen": 622295040 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004098094282848546, + "loss": 2.9145, + "theoretical_loss": 3.825321258284113, + "tokens_seen": 622360576 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004097993981945838, + "loss": 3.0614, + "theoretical_loss": 3.825279498375196, + "tokens_seen": 622426112 + }, + { + "epoch": 2.01, + "learning_rate": 0.00040978936810431296, + "loss": 2.9549, + "theoretical_loss": 3.8252377440939975, + "tokens_seen": 622491648 + }, + { + "epoch": 2.01, + "learning_rate": 0.00040977933801404214, + "loss": 2.8137, + "theoretical_loss": 3.825195995439167, + "tokens_seen": 622557184 + }, + { + "epoch": 2.01, + "objective/train/docs_used": 775020, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.917367696762085, + "objective/train/theoretical_loss": 3.825175123221218, + "objective/train/tokens_used": 643049952, + "theoretical_loss": 3.825175123221218, + "tokens_seen": 622589952 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004097693079237713, + "loss": 2.9835, + "theoretical_loss": 3.8251542524093542, + "tokens_seen": 622622720 + }, + { + "epoch": 2.01, + "learning_rate": 0.00040975927783350055, + "loss": 2.8027, + "theoretical_loss": 3.825112515003209, + "tokens_seen": 622688256 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004097492477432297, + "loss": 3.0546, + "theoretical_loss": 3.825070783219382, + "tokens_seen": 622753792 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004097392176529589, + "loss": 2.7498, + "theoretical_loss": 3.825029057056524, + "tokens_seen": 622819328 + }, + { + "epoch": 2.01, + "learning_rate": 0.00040972918756268804, + "loss": 2.7878, + "theoretical_loss": 3.8249873365132876, + "tokens_seen": 622884864 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004097191574724173, + "loss": 2.6325, + "theoretical_loss": 3.8249456215883235, + "tokens_seen": 622950400 + }, + { + "epoch": 2.01, + "learning_rate": 0.00040970912738214646, + "loss": 2.837, + "theoretical_loss": 3.824903912280285, + "tokens_seen": 623015936 + }, + { + "epoch": 2.01, + "learning_rate": 0.00040969909729187564, + "loss": 2.8094, + "theoretical_loss": 3.8248622085878257, + "tokens_seen": 623081472 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004096890672016048, + "loss": 3.0344, + "theoretical_loss": 3.824820510509597, + "tokens_seen": 623147008 + }, + { + "epoch": 2.01, + "learning_rate": 0.000409679037111334, + "loss": 3.0147, + "theoretical_loss": 3.824778818044255, + "tokens_seen": 623212544 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004096690070210632, + "loss": 2.8037, + "theoretical_loss": 3.824737131190452, + "tokens_seen": 623278080 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004096589769307924, + "loss": 2.8736, + "theoretical_loss": 3.8246954499468444, + "tokens_seen": 623343616 + }, + { + "epoch": 2.01, + "learning_rate": 0.00040964894684052155, + "loss": 2.7638, + "theoretical_loss": 3.824653774312087, + "tokens_seen": 623409152 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004096389167502508, + "loss": 3.0273, + "theoretical_loss": 3.824612104284835, + "tokens_seen": 623474688 + }, + { + "epoch": 2.01, + "learning_rate": 0.00040962888665997996, + "loss": 2.8466, + "theoretical_loss": 3.824570439863746, + "tokens_seen": 623540224 + }, + { + "epoch": 2.01, + "learning_rate": 0.00040961885656970914, + "loss": 3.077, + "theoretical_loss": 3.8245287810474746, + "tokens_seen": 623605760 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004096088264794383, + "loss": 2.9727, + "theoretical_loss": 3.8244871278346793, + "tokens_seen": 623671296 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004095987963891675, + "loss": 2.7729, + "theoretical_loss": 3.8244454802240173, + "tokens_seen": 623736832 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004095887662988967, + "loss": 2.7011, + "theoretical_loss": 3.8244038382141463, + "tokens_seen": 623802368 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004095787362086259, + "loss": 3.0728, + "theoretical_loss": 3.824362201803726, + "tokens_seen": 623867904 + }, + { + "epoch": 2.01, + "learning_rate": 0.00040956870611835505, + "loss": 2.7587, + "theoretical_loss": 3.8243205709914143, + "tokens_seen": 623933440 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004095586760280843, + "loss": 2.7799, + "theoretical_loss": 3.8242789457758706, + "tokens_seen": 623998976 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004095486459378134, + "loss": 2.6693, + "theoretical_loss": 3.824237326155756, + "tokens_seen": 624064512 + }, + { + "epoch": 2.01, + "learning_rate": 0.00040953861584754265, + "loss": 2.8832, + "theoretical_loss": 3.824195712129729, + "tokens_seen": 624130048 + }, + { + "epoch": 2.01, + "learning_rate": 0.00040952858575727183, + "loss": 2.7424, + "theoretical_loss": 3.824154103696452, + "tokens_seen": 624195584 + }, + { + "epoch": 2.01, + "objective/train/docs_used": 776436, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9514684677124023, + "objective/train/theoretical_loss": 3.8241333015766763, + "objective/train/tokens_used": 644688352, + "theoretical_loss": 3.8241333015766763, + "tokens_seen": 624228352 + }, + { + "epoch": 2.01, + "learning_rate": 0.000409518555667001, + "loss": 2.7882, + "theoretical_loss": 3.8241125008545853, + "tokens_seen": 624261120 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004095085255767302, + "loss": 2.756, + "theoretical_loss": 3.8240709036027916, + "tokens_seen": 624326656 + }, + { + "epoch": 2.01, + "learning_rate": 0.00040949849548645937, + "loss": 2.9293, + "theoretical_loss": 3.8240293119397317, + "tokens_seen": 624392192 + }, + { + "epoch": 2.01, + "learning_rate": 0.00040948846539618855, + "loss": 2.7802, + "theoretical_loss": 3.823987725864069, + "tokens_seen": 624457728 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004094784353059178, + "loss": 2.8242, + "theoretical_loss": 3.8239461453744674, + "tokens_seen": 624523264 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004094684052156469, + "loss": 2.8764, + "theoretical_loss": 3.8239045704695895, + "tokens_seen": 624588800 + }, + { + "epoch": 2.01, + "learning_rate": 0.00040945837512537615, + "loss": 2.6924, + "theoretical_loss": 3.8238630011480996, + "tokens_seen": 624654336 + }, + { + "epoch": 2.01, + "learning_rate": 0.00040944834503510533, + "loss": 3.0381, + "theoretical_loss": 3.823821437408662, + "tokens_seen": 624719872 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004094383149448345, + "loss": 2.821, + "theoretical_loss": 3.8237798792499422, + "tokens_seen": 624785408 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004094282848545637, + "loss": 2.6496, + "theoretical_loss": 3.823738326670605, + "tokens_seen": 624850944 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004094182547642929, + "loss": 2.7156, + "theoretical_loss": 3.8236967796693166, + "tokens_seen": 624916480 + }, + { + "epoch": 2.01, + "learning_rate": 0.00040940822467402206, + "loss": 2.9485, + "theoretical_loss": 3.8236552382447435, + "tokens_seen": 624982016 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004093981945837513, + "loss": 2.8782, + "theoretical_loss": 3.8236137023955523, + "tokens_seen": 625047552 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004093881644934804, + "loss": 2.8737, + "theoretical_loss": 3.8235721721204103, + "tokens_seen": 625113088 + }, + { + "epoch": 2.01, + "learning_rate": 0.00040937813440320965, + "loss": 2.7295, + "theoretical_loss": 3.823530647417985, + "tokens_seen": 625178624 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004093681043129388, + "loss": 2.9309, + "theoretical_loss": 3.823489128286945, + "tokens_seen": 625244160 + }, + { + "epoch": 2.01, + "learning_rate": 0.000409358074222668, + "loss": 2.9655, + "theoretical_loss": 3.8234476147259584, + "tokens_seen": 625309696 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004093480441323972, + "loss": 2.8918, + "theoretical_loss": 3.8234061067336946, + "tokens_seen": 625375232 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004093380140421264, + "loss": 2.7062, + "theoretical_loss": 3.823364604308823, + "tokens_seen": 625440768 + }, + { + "epoch": 2.01, + "learning_rate": 0.00040932798395185556, + "loss": 2.7036, + "theoretical_loss": 3.823323107450014, + "tokens_seen": 625506304 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004093179538615848, + "loss": 2.9356, + "theoretical_loss": 3.8232816161559375, + "tokens_seen": 625571840 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004093079237713139, + "loss": 2.5131, + "theoretical_loss": 3.8232401304252646, + "tokens_seen": 625637376 + }, + { + "epoch": 2.01, + "learning_rate": 0.00040929789368104316, + "loss": 2.9493, + "theoretical_loss": 3.823198650256667, + "tokens_seen": 625702912 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004092878635907723, + "loss": 3.0427, + "theoretical_loss": 3.8231571756488165, + "tokens_seen": 625768448 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004092778335005015, + "loss": 2.8405, + "theoretical_loss": 3.823115706600385, + "tokens_seen": 625833984 + }, + { + "epoch": 2.01, + "objective/train/docs_used": 777260, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8199868202209473, + "objective/train/theoretical_loss": 3.8230949741605365, + "objective/train/tokens_used": 646326752, + "theoretical_loss": 3.8230949741605365, + "tokens_seen": 625866752 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004092678034102307, + "loss": 2.7164, + "theoretical_loss": 3.823074243110045, + "tokens_seen": 625899520 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004092577733199599, + "loss": 2.6414, + "theoretical_loss": 3.82303278517647, + "tokens_seen": 625965056 + }, + { + "epoch": 2.01, + "learning_rate": 0.00040924774322968906, + "loss": 2.8903, + "theoretical_loss": 3.8229913327983343, + "tokens_seen": 626030592 + }, + { + "epoch": 2.01, + "learning_rate": 0.00040923771313941824, + "loss": 2.7473, + "theoretical_loss": 3.822949885974311, + "tokens_seen": 626096128 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004092276830491474, + "loss": 2.7383, + "theoretical_loss": 3.8229084447030757, + "tokens_seen": 626161664 + }, + { + "epoch": 2.01, + "learning_rate": 0.00040921765295887666, + "loss": 3.0065, + "theoretical_loss": 3.8228670089833026, + "tokens_seen": 626227200 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004092076228686058, + "loss": 2.6521, + "theoretical_loss": 3.822825578813667, + "tokens_seen": 626292736 + }, + { + "epoch": 2.01, + "learning_rate": 0.000409197592778335, + "loss": 2.8301, + "theoretical_loss": 3.8227841541928456, + "tokens_seen": 626358272 + }, + { + "epoch": 2.01, + "learning_rate": 0.00040918756268806415, + "loss": 2.7502, + "theoretical_loss": 3.8227427351195145, + "tokens_seen": 626423808 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004091775325977934, + "loss": 2.9458, + "theoretical_loss": 3.82270132159235, + "tokens_seen": 626489344 + }, + { + "epoch": 2.01, + "learning_rate": 0.00040916750250752257, + "loss": 2.6212, + "theoretical_loss": 3.82265991361003, + "tokens_seen": 626554880 + }, + { + "epoch": 2.01, + "learning_rate": 0.00040915747241725175, + "loss": 2.9773, + "theoretical_loss": 3.8226185111712327, + "tokens_seen": 626620416 + }, + { + "epoch": 2.01, + "learning_rate": 0.000409147442326981, + "loss": 2.9269, + "theoretical_loss": 3.822577114274635, + "tokens_seen": 626685952 + }, + { + "epoch": 2.01, + "learning_rate": 0.00040913741223671016, + "loss": 3.0878, + "theoretical_loss": 3.822535722918916, + "tokens_seen": 626751488 + }, + { + "epoch": 2.01, + "learning_rate": 0.00040912738214643934, + "loss": 2.9192, + "theoretical_loss": 3.8224943371027558, + "tokens_seen": 626817024 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004091173520561685, + "loss": 2.4529, + "theoretical_loss": 3.822452956824833, + "tokens_seen": 626882560 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004091073219658977, + "loss": 2.703, + "theoretical_loss": 3.822411582083827, + "tokens_seen": 626948096 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004090972918756269, + "loss": 2.8391, + "theoretical_loss": 3.82237021287842, + "tokens_seen": 627013632 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004090872617853561, + "loss": 2.7474, + "theoretical_loss": 3.8223288492072918, + "tokens_seen": 627079168 + }, + { + "epoch": 2.01, + "learning_rate": 0.00040907723169508525, + "loss": 2.6742, + "theoretical_loss": 3.8222874910691234, + "tokens_seen": 627144704 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004090672016048145, + "loss": 2.6279, + "theoretical_loss": 3.8222461384625976, + "tokens_seen": 627210240 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004090571715145436, + "loss": 2.8529, + "theoretical_loss": 3.822204791386396, + "tokens_seen": 627275776 + }, + { + "epoch": 2.01, + "learning_rate": 0.00040904714142427285, + "loss": 2.8604, + "theoretical_loss": 3.822163449839202, + "tokens_seen": 627341312 + }, + { + "epoch": 2.01, + "learning_rate": 0.00040903711133400203, + "loss": 2.97, + "theoretical_loss": 3.822122113819698, + "tokens_seen": 627406848 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004090270812437312, + "loss": 2.714, + "theoretical_loss": 3.8220807833265678, + "tokens_seen": 627472384 + }, + { + "epoch": 2.01, + "objective/train/docs_used": 778767, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.658456563949585, + "objective/train/theoretical_loss": 3.8220601201519817, + "objective/train/tokens_used": 647965152, + "theoretical_loss": 3.8220601201519817, + "tokens_seen": 627505152 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004090170511534604, + "loss": 2.9567, + "theoretical_loss": 3.8220394583584953, + "tokens_seen": 627537920 + }, + { + "epoch": 2.01, + "learning_rate": 0.00040900702106318957, + "loss": 2.6408, + "theoretical_loss": 3.821998138914166, + "tokens_seen": 627603456 + }, + { + "epoch": 2.01, + "learning_rate": 0.00040899699097291875, + "loss": 2.8496, + "theoretical_loss": 3.821956824992264, + "tokens_seen": 627668992 + }, + { + "epoch": 2.01, + "learning_rate": 0.000408986960882648, + "loss": 2.7743, + "theoretical_loss": 3.821915516591475, + "tokens_seen": 627734528 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004089769307923771, + "loss": 2.6965, + "theoretical_loss": 3.8218742137104846, + "tokens_seen": 627800064 + }, + { + "epoch": 2.01, + "learning_rate": 0.00040896690070210635, + "loss": 2.8615, + "theoretical_loss": 3.8218329163479794, + "tokens_seen": 627865600 + }, + { + "epoch": 2.01, + "learning_rate": 0.00040895687061183553, + "loss": 2.5942, + "theoretical_loss": 3.821791624502646, + "tokens_seen": 627931136 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004089468405215647, + "loss": 2.8906, + "theoretical_loss": 3.821750338173172, + "tokens_seen": 627996672 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004089368104312939, + "loss": 2.9984, + "theoretical_loss": 3.8217090573582446, + "tokens_seen": 628062208 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004089267803410231, + "loss": 2.8825, + "theoretical_loss": 3.821667782056552, + "tokens_seen": 628127744 + }, + { + "epoch": 2.01, + "learning_rate": 0.00040891675025075226, + "loss": 2.8492, + "theoretical_loss": 3.821626512266784, + "tokens_seen": 628193280 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004089067201604815, + "loss": 2.7001, + "theoretical_loss": 3.821585247987627, + "tokens_seen": 628258816 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004088966900702106, + "loss": 2.8747, + "theoretical_loss": 3.8215439892177727, + "tokens_seen": 628324352 + }, + { + "epoch": 2.01, + "learning_rate": 0.00040888665997993985, + "loss": 2.9203, + "theoretical_loss": 3.82150273595591, + "tokens_seen": 628389888 + }, + { + "epoch": 2.01, + "learning_rate": 0.000408876629889669, + "loss": 2.8152, + "theoretical_loss": 3.8214614882007303, + "tokens_seen": 628455424 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004088665997993982, + "loss": 2.818, + "theoretical_loss": 3.8214202459509234, + "tokens_seen": 628520960 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004088565697091274, + "loss": 2.9598, + "theoretical_loss": 3.8213790092051805, + "tokens_seen": 628586496 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004088465396188566, + "loss": 2.8876, + "theoretical_loss": 3.821337777962194, + "tokens_seen": 628652032 + }, + { + "epoch": 2.01, + "learning_rate": 0.00040883650952858576, + "loss": 2.8699, + "theoretical_loss": 3.8212965522206552, + "tokens_seen": 628717568 + }, + { + "epoch": 2.01, + "learning_rate": 0.000408826479438315, + "loss": 2.866, + "theoretical_loss": 3.821255331979258, + "tokens_seen": 628783104 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004088164493480441, + "loss": 3.1374, + "theoretical_loss": 3.8212141172366945, + "tokens_seen": 628848640 + }, + { + "epoch": 2.01, + "learning_rate": 0.00040880641925777336, + "loss": 2.736, + "theoretical_loss": 3.8211729079916577, + "tokens_seen": 628914176 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004087963891675025, + "loss": 2.7657, + "theoretical_loss": 3.821131704242843, + "tokens_seen": 628979712 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004087863590772317, + "loss": 2.918, + "theoretical_loss": 3.821090505988944, + "tokens_seen": 629045248 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004087763289869609, + "loss": 2.7973, + "theoretical_loss": 3.821049313228655, + "tokens_seen": 629110784 + }, + { + "epoch": 2.01, + "objective/train/docs_used": 779594, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.082977056503296, + "objective/train/theoretical_loss": 3.821028718908207, + "objective/train/tokens_used": 649603552, + "theoretical_loss": 3.821028718908207, + "tokens_seen": 629143552 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004087662988966901, + "loss": 2.8122, + "theoretical_loss": 3.821008125960672, + "tokens_seen": 629176320 + }, + { + "epoch": 2.01, + "learning_rate": 0.00040875626880641926, + "loss": 2.943, + "theoretical_loss": 3.8209669441836907, + "tokens_seen": 629241856 + }, + { + "epoch": 2.01, + "learning_rate": 0.00040874623871614844, + "loss": 2.8359, + "theoretical_loss": 3.8209257678964073, + "tokens_seen": 629307392 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004087362086258776, + "loss": 2.7174, + "theoretical_loss": 3.8208845970975176, + "tokens_seen": 629372928 + }, + { + "epoch": 2.01, + "learning_rate": 0.00040872617853560686, + "loss": 2.9775, + "theoretical_loss": 3.82084343178572, + "tokens_seen": 629438464 + }, + { + "epoch": 2.01, + "learning_rate": 0.000408716148445336, + "loss": 2.5463, + "theoretical_loss": 3.8208022719597112, + "tokens_seen": 629504000 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004087061183550652, + "loss": 2.9226, + "theoretical_loss": 3.820761117618189, + "tokens_seen": 629569536 + }, + { + "epoch": 2.01, + "learning_rate": 0.00040869608826479435, + "loss": 2.774, + "theoretical_loss": 3.8207199687598523, + "tokens_seen": 629635072 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004086860581745236, + "loss": 2.56, + "theoretical_loss": 3.8206788253834, + "tokens_seen": 629700608 + }, + { + "epoch": 2.01, + "learning_rate": 0.00040867602808425277, + "loss": 2.6942, + "theoretical_loss": 3.8206376874875305, + "tokens_seen": 629766144 + }, + { + "epoch": 2.01, + "learning_rate": 0.00040866599799398195, + "loss": 2.6446, + "theoretical_loss": 3.8205965550709444, + "tokens_seen": 629831680 + }, + { + "epoch": 2.01, + "learning_rate": 0.00040865596790371113, + "loss": 2.6938, + "theoretical_loss": 3.8205554281323417, + "tokens_seen": 629897216 + }, + { + "epoch": 2.01, + "learning_rate": 0.00040864593781344036, + "loss": 2.8075, + "theoretical_loss": 3.8205143066704226, + "tokens_seen": 629962752 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004086359077231695, + "loss": 2.742, + "theoretical_loss": 3.820473190683889, + "tokens_seen": 630028288 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004086258776328987, + "loss": 2.8813, + "theoretical_loss": 3.8204320801714413, + "tokens_seen": 630093824 + }, + { + "epoch": 2.01, + "learning_rate": 0.00040861584754262785, + "loss": 2.9127, + "theoretical_loss": 3.820390975131782, + "tokens_seen": 630159360 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004086058174523571, + "loss": 2.9052, + "theoretical_loss": 3.8203498755636143, + "tokens_seen": 630224896 + }, + { + "epoch": 2.01, + "learning_rate": 0.00040859578736208627, + "loss": 2.7453, + "theoretical_loss": 3.8203087814656396, + "tokens_seen": 630290432 + }, + { + "epoch": 2.01, + "learning_rate": 0.00040858575727181545, + "loss": 2.8833, + "theoretical_loss": 3.820267692836562, + "tokens_seen": 630355968 + }, + { + "epoch": 2.01, + "learning_rate": 0.00040857572718154463, + "loss": 2.9168, + "theoretical_loss": 3.8202266096750854, + "tokens_seen": 630421504 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004085656970912738, + "loss": 2.7973, + "theoretical_loss": 3.8201855319799134, + "tokens_seen": 630487040 + }, + { + "epoch": 2.01, + "learning_rate": 0.000408555667001003, + "loss": 2.6587, + "theoretical_loss": 3.8201444597497507, + "tokens_seen": 630552576 + }, + { + "epoch": 2.01, + "learning_rate": 0.00040854563691073223, + "loss": 3.037, + "theoretical_loss": 3.8201033929833024, + "tokens_seen": 630618112 + }, + { + "epoch": 2.01, + "learning_rate": 0.00040853560682046136, + "loss": 2.8116, + "theoretical_loss": 3.8200623316792743, + "tokens_seen": 630683648 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004085255767301906, + "loss": 2.8274, + "theoretical_loss": 3.820021275836372, + "tokens_seen": 630749184 + }, + { + "epoch": 2.01, + "objective/train/docs_used": 780765, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5284388065338135, + "objective/train/theoretical_loss": 3.820000749962438, + "objective/train/tokens_used": 651241952, + "theoretical_loss": 3.820000749962438, + "tokens_seen": 630781952 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004085155466399197, + "loss": 2.7464, + "theoretical_loss": 3.8199802254533015, + "tokens_seen": 630814720 + }, + { + "epoch": 2.01, + "learning_rate": 0.00040850551654964895, + "loss": 2.7889, + "theoretical_loss": 3.8199391805287704, + "tokens_seen": 630880256 + }, + { + "epoch": 2.01, + "learning_rate": 0.00040849548645937814, + "loss": 2.7819, + "theoretical_loss": 3.8198981410614854, + "tokens_seen": 630945792 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004084854563691073, + "loss": 2.7129, + "theoretical_loss": 3.819857107050155, + "tokens_seen": 631011328 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004084754262788365, + "loss": 2.8668, + "theoretical_loss": 3.819816078493486, + "tokens_seen": 631076864 + }, + { + "epoch": 2.01, + "learning_rate": 0.00040846539618856573, + "loss": 2.7094, + "theoretical_loss": 3.819775055390188, + "tokens_seen": 631142400 + }, + { + "epoch": 2.01, + "learning_rate": 0.00040845536609829486, + "loss": 3.036, + "theoretical_loss": 3.8197340377389697, + "tokens_seen": 631207936 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004084453360080241, + "loss": 2.8579, + "theoretical_loss": 3.81969302553854, + "tokens_seen": 631273472 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004084353059177532, + "loss": 2.7545, + "theoretical_loss": 3.81965201878761, + "tokens_seen": 631339008 + }, + { + "epoch": 2.01, + "learning_rate": 0.00040842527582748246, + "loss": 2.9343, + "theoretical_loss": 3.819611017484889, + "tokens_seen": 631404544 + }, + { + "epoch": 2.01, + "learning_rate": 0.00040841524573721164, + "loss": 2.9302, + "theoretical_loss": 3.8195700216290884, + "tokens_seen": 631470080 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004084052156469408, + "loss": 2.6115, + "theoretical_loss": 3.8195290312189187, + "tokens_seen": 631535616 + }, + { + "epoch": 2.01, + "learning_rate": 0.00040839518555667005, + "loss": 2.9017, + "theoretical_loss": 3.8194880462530922, + "tokens_seen": 631601152 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004083851554663992, + "loss": 2.8711, + "theoretical_loss": 3.819447066730321, + "tokens_seen": 631666688 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004083751253761284, + "loss": 2.7859, + "theoretical_loss": 3.8194060926493165, + "tokens_seen": 631732224 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004083650952858576, + "loss": 2.9598, + "theoretical_loss": 3.819365124008793, + "tokens_seen": 631797760 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004083550651955868, + "loss": 2.6389, + "theoretical_loss": 3.8193241608074637, + "tokens_seen": 631863296 + }, + { + "epoch": 2.01, + "learning_rate": 0.00040834503510531596, + "loss": 2.9403, + "theoretical_loss": 3.8192832030440416, + "tokens_seen": 631928832 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004083350050150452, + "loss": 2.9505, + "theoretical_loss": 3.819242250717242, + "tokens_seen": 631994368 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004083249749247743, + "loss": 2.7696, + "theoretical_loss": 3.819201303825779, + "tokens_seen": 632059904 + }, + { + "epoch": 2.01, + "learning_rate": 0.00040831494483450356, + "loss": 2.9617, + "theoretical_loss": 3.8191603623683674, + "tokens_seen": 632125440 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004083049147442327, + "loss": 2.7968, + "theoretical_loss": 3.819119426343724, + "tokens_seen": 632190976 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004082948846539619, + "loss": 2.6314, + "theoretical_loss": 3.8190784957505635, + "tokens_seen": 632256512 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004082848545636911, + "loss": 2.9328, + "theoretical_loss": 3.819037570587603, + "tokens_seen": 632322048 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004082748244734203, + "loss": 2.8324, + "theoretical_loss": 3.8189966508535598, + "tokens_seen": 632387584 + }, + { + "epoch": 2.01, + "objective/train/docs_used": 781551, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.776937246322632, + "objective/train/theoretical_loss": 3.8189761930219808, + "objective/train/tokens_used": 652880352, + "theoretical_loss": 3.8189761930219808, + "tokens_seen": 632420352 + }, + { + "epoch": 2.01, + "learning_rate": 0.00040826479438314946, + "loss": 2.8414, + "theoretical_loss": 3.8189557365471503, + "tokens_seen": 632453120 + }, + { + "epoch": 2.01, + "learning_rate": 0.00040825476429287864, + "loss": 3.0336, + "theoretical_loss": 3.8189148276670926, + "tokens_seen": 632518656 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004082447342026078, + "loss": 2.9431, + "theoretical_loss": 3.818873924212105, + "tokens_seen": 632584192 + }, + { + "epoch": 2.01, + "learning_rate": 0.00040823470411233706, + "loss": 2.8801, + "theoretical_loss": 3.818833026180906, + "tokens_seen": 632649728 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004082246740220662, + "loss": 2.7998, + "theoretical_loss": 3.8187921335722157, + "tokens_seen": 632715264 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004082146439317954, + "loss": 2.8741, + "theoretical_loss": 3.8187512463847515, + "tokens_seen": 632780800 + }, + { + "epoch": 2.01, + "learning_rate": 0.00040820461384152455, + "loss": 2.6709, + "theoretical_loss": 3.8187103646172353, + "tokens_seen": 632846336 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004081945837512538, + "loss": 2.939, + "theoretical_loss": 3.8186694882683865, + "tokens_seen": 632911872 + }, + { + "epoch": 2.01, + "learning_rate": 0.00040818455366098297, + "loss": 2.9783, + "theoretical_loss": 3.818628617336926, + "tokens_seen": 632977408 + }, + { + "epoch": 2.01, + "learning_rate": 0.00040817452357071215, + "loss": 2.9095, + "theoretical_loss": 3.8185877518215756, + "tokens_seen": 633042944 + }, + { + "epoch": 2.01, + "learning_rate": 0.00040816449348044133, + "loss": 2.9708, + "theoretical_loss": 3.818546891721056, + "tokens_seen": 633108480 + }, + { + "epoch": 2.01, + "learning_rate": 0.00040815446339017056, + "loss": 2.9867, + "theoretical_loss": 3.81850603703409, + "tokens_seen": 633174016 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004081444332998997, + "loss": 2.8313, + "theoretical_loss": 3.8184651877594007, + "tokens_seen": 633239552 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004081344032096289, + "loss": 2.9659, + "theoretical_loss": 3.8184243438957095, + "tokens_seen": 633305088 + }, + { + "epoch": 2.01, + "learning_rate": 0.00040812437311935805, + "loss": 2.8979, + "theoretical_loss": 3.8183835054417408, + "tokens_seen": 633370624 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004081143430290873, + "loss": 2.7882, + "theoretical_loss": 3.818342672396219, + "tokens_seen": 633436160 + }, + { + "epoch": 2.01, + "learning_rate": 0.00040810431293881647, + "loss": 2.8737, + "theoretical_loss": 3.818301844757867, + "tokens_seen": 633501696 + }, + { + "epoch": 2.01, + "learning_rate": 0.00040809428284854565, + "loss": 2.8172, + "theoretical_loss": 3.8182610225254106, + "tokens_seen": 633567232 + }, + { + "epoch": 2.01, + "learning_rate": 0.00040808425275827483, + "loss": 2.8925, + "theoretical_loss": 3.8182202056975747, + "tokens_seen": 633632768 + }, + { + "epoch": 2.01, + "learning_rate": 0.000408074222668004, + "loss": 2.6774, + "theoretical_loss": 3.8181793942730846, + "tokens_seen": 633698304 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004080641925777332, + "loss": 2.7061, + "theoretical_loss": 3.8181385882506667, + "tokens_seen": 633763840 + }, + { + "epoch": 2.01, + "learning_rate": 0.00040805416248746243, + "loss": 3.0322, + "theoretical_loss": 3.8180977876290467, + "tokens_seen": 633829376 + }, + { + "epoch": 2.01, + "learning_rate": 0.00040804413239719156, + "loss": 2.8356, + "theoretical_loss": 3.8180569924069525, + "tokens_seen": 633894912 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004080341023069208, + "loss": 2.8216, + "theoretical_loss": 3.8180162025831104, + "tokens_seen": 633960448 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004080240722166499, + "loss": 2.7041, + "theoretical_loss": 3.817975418156249, + "tokens_seen": 634025984 + }, + { + "epoch": 2.01, + "objective/train/docs_used": 783036, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.2487502098083496, + "objective/train/theoretical_loss": 3.8179550279662884, + "objective/train/tokens_used": 654518752, + "theoretical_loss": 3.8179550279662884, + "tokens_seen": 634058752 + }, + { + "epoch": 2.01, + "learning_rate": 0.00040801404212637915, + "loss": 2.9766, + "theoretical_loss": 3.817934639125096, + "tokens_seen": 634091520 + }, + { + "epoch": 2.01, + "learning_rate": 0.00040800401203610834, + "loss": 2.7949, + "theoretical_loss": 3.8178938654883803, + "tokens_seen": 634157056 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004079939819458375, + "loss": 2.6729, + "theoretical_loss": 3.8178530972448304, + "tokens_seen": 634222592 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004079839518555667, + "loss": 2.7617, + "theoretical_loss": 3.8178123343931762, + "tokens_seen": 634288128 + }, + { + "epoch": 2.01, + "learning_rate": 0.00040797392176529593, + "loss": 2.6544, + "theoretical_loss": 3.8177715769321474, + "tokens_seen": 634353664 + }, + { + "epoch": 2.01, + "learning_rate": 0.00040796389167502506, + "loss": 2.9967, + "theoretical_loss": 3.817730824860474, + "tokens_seen": 634419200 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004079538615847543, + "loss": 2.7682, + "theoretical_loss": 3.817690078176888, + "tokens_seen": 634484736 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004079438314944834, + "loss": 2.7531, + "theoretical_loss": 3.817649336880119, + "tokens_seen": 634550272 + }, + { + "epoch": 2.01, + "learning_rate": 0.00040793380140421266, + "loss": 2.7709, + "theoretical_loss": 3.8176086009689, + "tokens_seen": 634615808 + }, + { + "epoch": 2.01, + "learning_rate": 0.00040792377131394184, + "loss": 2.7165, + "theoretical_loss": 3.817567870441962, + "tokens_seen": 634681344 + }, + { + "epoch": 2.01, + "learning_rate": 0.000407913741223671, + "loss": 2.8137, + "theoretical_loss": 3.8175271452980377, + "tokens_seen": 634746880 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004079037111334002, + "loss": 2.9933, + "theoretical_loss": 3.8174864255358605, + "tokens_seen": 634812416 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004078936810431294, + "loss": 2.8765, + "theoretical_loss": 3.8174457111541633, + "tokens_seen": 634877952 + }, + { + "epoch": 2.01, + "learning_rate": 0.00040788365095285856, + "loss": 2.8943, + "theoretical_loss": 3.8174050021516797, + "tokens_seen": 634943488 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004078736208625878, + "loss": 2.979, + "theoretical_loss": 3.8173642985271443, + "tokens_seen": 635009024 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004078635907723169, + "loss": 2.7522, + "theoretical_loss": 3.817323600279291, + "tokens_seen": 635074560 + }, + { + "epoch": 2.01, + "learning_rate": 0.00040785356068204616, + "loss": 2.7643, + "theoretical_loss": 3.817282907406857, + "tokens_seen": 635140096 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004078435305917753, + "loss": 2.9688, + "theoretical_loss": 3.8172422199085743, + "tokens_seen": 635205632 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004078335005015045, + "loss": 2.7879, + "theoretical_loss": 3.817201537783182, + "tokens_seen": 635271168 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004078234704112337, + "loss": 2.8632, + "theoretical_loss": 3.8171608610294143, + "tokens_seen": 635336704 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004078134403209629, + "loss": 2.8936, + "theoretical_loss": 3.8171201896460083, + "tokens_seen": 635402240 + }, + { + "epoch": 2.01, + "learning_rate": 0.00040780341023069207, + "loss": 2.8438, + "theoretical_loss": 3.817079523631703, + "tokens_seen": 635467776 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004077933801404213, + "loss": 2.9047, + "theoretical_loss": 3.8170388629852336, + "tokens_seen": 635533312 + }, + { + "epoch": 2.01, + "learning_rate": 0.00040778335005015043, + "loss": 2.8102, + "theoretical_loss": 3.8169982077053395, + "tokens_seen": 635598848 + }, + { + "epoch": 2.01, + "learning_rate": 0.00040777331995987966, + "loss": 2.7813, + "theoretical_loss": 3.816957557790759, + "tokens_seen": 635664384 + }, + { + "epoch": 2.01, + "objective/train/docs_used": 783857, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6842525005340576, + "objective/train/theoretical_loss": 3.8169372348450663, + "objective/train/tokens_used": 656157152, + "theoretical_loss": 3.8169372348450663, + "tokens_seen": 635697152 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004077632898696088, + "loss": 2.8004, + "theoretical_loss": 3.81691691324023, + "tokens_seen": 635729920 + }, + { + "epoch": 2.01, + "learning_rate": 0.000407753259779338, + "loss": 2.8361, + "theoretical_loss": 3.8168762740524933, + "tokens_seen": 635795456 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004077432296890672, + "loss": 3.1555, + "theoretical_loss": 3.8168356402262877, + "tokens_seen": 635860992 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004077331995987964, + "loss": 2.8418, + "theoretical_loss": 3.816795011760354, + "tokens_seen": 635926528 + }, + { + "epoch": 2.01, + "learning_rate": 0.00040772316950852557, + "loss": 2.7288, + "theoretical_loss": 3.816754388653432, + "tokens_seen": 635992064 + }, + { + "epoch": 2.01, + "learning_rate": 0.00040771313941825475, + "loss": 2.8514, + "theoretical_loss": 3.816713770904263, + "tokens_seen": 636057600 + }, + { + "epoch": 2.01, + "learning_rate": 0.00040770310932798393, + "loss": 2.8342, + "theoretical_loss": 3.8166731585115894, + "tokens_seen": 636123136 + }, + { + "epoch": 2.01, + "learning_rate": 0.00040769307923771317, + "loss": 2.6405, + "theoretical_loss": 3.816632551474152, + "tokens_seen": 636188672 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004076830491474423, + "loss": 2.8957, + "theoretical_loss": 3.816591949790693, + "tokens_seen": 636254208 + }, + { + "epoch": 2.01, + "learning_rate": 0.00040767301905717153, + "loss": 2.6612, + "theoretical_loss": 3.816551353459955, + "tokens_seen": 636319744 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004076629889669007, + "loss": 3.0216, + "theoretical_loss": 3.816510762480683, + "tokens_seen": 636385280 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004076529588766299, + "loss": 2.8275, + "theoretical_loss": 3.8164701768516185, + "tokens_seen": 636450816 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004076429287863591, + "loss": 2.9587, + "theoretical_loss": 3.816429596571506, + "tokens_seen": 636516352 + }, + { + "epoch": 2.01, + "learning_rate": 0.00040763289869608825, + "loss": 2.7802, + "theoretical_loss": 3.81638902163909, + "tokens_seen": 636581888 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004076228686058175, + "loss": 2.867, + "theoretical_loss": 3.8163484520531155, + "tokens_seen": 636647424 + }, + { + "epoch": 2.01, + "learning_rate": 0.00040761283851554667, + "loss": 2.5608, + "theoretical_loss": 3.8163078878123278, + "tokens_seen": 636712960 + }, + { + "epoch": 2.01, + "learning_rate": 0.00040760280842527585, + "loss": 2.7673, + "theoretical_loss": 3.8162673289154725, + "tokens_seen": 636778496 + }, + { + "epoch": 2.01, + "learning_rate": 0.00040759277833500503, + "loss": 2.8313, + "theoretical_loss": 3.8162267753612964, + "tokens_seen": 636844032 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004075827482447342, + "loss": 2.911, + "theoretical_loss": 3.8161862271485445, + "tokens_seen": 636909568 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004075727181544634, + "loss": 2.8002, + "theoretical_loss": 3.8161456842759645, + "tokens_seen": 636975104 + }, + { + "epoch": 2.01, + "learning_rate": 0.00040756268806419263, + "loss": 2.8853, + "theoretical_loss": 3.816105146742304, + "tokens_seen": 637040640 + }, + { + "epoch": 2.01, + "learning_rate": 0.00040755265797392176, + "loss": 2.6506, + "theoretical_loss": 3.816064614546311, + "tokens_seen": 637106176 + }, + { + "epoch": 2.01, + "learning_rate": 0.000407542627883651, + "loss": 3.0013, + "theoretical_loss": 3.816024087686733, + "tokens_seen": 637171712 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004075325977933801, + "loss": 2.8045, + "theoretical_loss": 3.81598356616232, + "tokens_seen": 637237248 + }, + { + "epoch": 2.01, + "learning_rate": 0.00040752256770310935, + "loss": 2.9861, + "theoretical_loss": 3.8159430499718194, + "tokens_seen": 637302784 + }, + { + "epoch": 2.01, + "objective/train/docs_used": 785359, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.857738494873047, + "objective/train/theoretical_loss": 3.8159227938763953, + "objective/train/tokens_used": 657795552, + "theoretical_loss": 3.8159227938763953, + "tokens_seen": 637335552 + }, + { + "epoch": 2.01, + "learning_rate": 0.00040751253761283854, + "loss": 2.8626, + "theoretical_loss": 3.815902539113981, + "tokens_seen": 637368320 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004075025075225677, + "loss": 2.8335, + "theoretical_loss": 3.815862033587556, + "tokens_seen": 637433856 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004074924774322969, + "loss": 2.9154, + "theoretical_loss": 3.8158215333912935, + "tokens_seen": 637499392 + }, + { + "epoch": 2.01, + "learning_rate": 0.00040748244734202613, + "loss": 2.9701, + "theoretical_loss": 3.8157810385239443, + "tokens_seen": 637564928 + }, + { + "epoch": 2.01, + "learning_rate": 0.00040747241725175526, + "loss": 2.7581, + "theoretical_loss": 3.81574054898426, + "tokens_seen": 637630464 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004074623871614845, + "loss": 3.1263, + "theoretical_loss": 3.8157000647709927, + "tokens_seen": 637696000 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004074523570712136, + "loss": 2.6475, + "theoretical_loss": 3.8156595858828934, + "tokens_seen": 637761536 + }, + { + "epoch": 2.01, + "learning_rate": 0.00040744232698094286, + "loss": 2.9252, + "theoretical_loss": 3.815619112318715, + "tokens_seen": 637827072 + }, + { + "epoch": 2.01, + "learning_rate": 0.00040743229689067204, + "loss": 2.8951, + "theoretical_loss": 3.8155786440772106, + "tokens_seen": 637892608 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004074222668004012, + "loss": 3.0481, + "theoretical_loss": 3.8155381811571325, + "tokens_seen": 637958144 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004074122367101304, + "loss": 2.9415, + "theoretical_loss": 3.8154977235572356, + "tokens_seen": 638023680 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004074022066198596, + "loss": 2.789, + "theoretical_loss": 3.8154572712762733, + "tokens_seen": 638089216 + }, + { + "epoch": 2.01, + "learning_rate": 0.00040739217652958876, + "loss": 2.7131, + "theoretical_loss": 3.815416824313001, + "tokens_seen": 638154752 + }, + { + "epoch": 2.01, + "learning_rate": 0.000407382146439318, + "loss": 2.7745, + "theoretical_loss": 3.815376382666172, + "tokens_seen": 638220288 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004073721163490471, + "loss": 2.9315, + "theoretical_loss": 3.8153359463345433, + "tokens_seen": 638285824 + }, + { + "epoch": 2.01, + "learning_rate": 0.00040736208625877636, + "loss": 2.6523, + "theoretical_loss": 3.8152955153168704, + "tokens_seen": 638351360 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004073520561685055, + "loss": 3.1046, + "theoretical_loss": 3.8152550896119086, + "tokens_seen": 638416896 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004073420260782347, + "loss": 2.9757, + "theoretical_loss": 3.815214669218416, + "tokens_seen": 638482432 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004073319959879639, + "loss": 3.0314, + "theoretical_loss": 3.8151742541351483, + "tokens_seen": 638547968 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004073219658976931, + "loss": 3.0917, + "theoretical_loss": 3.8151338443608633, + "tokens_seen": 638613504 + }, + { + "epoch": 2.01, + "learning_rate": 0.00040731193580742227, + "loss": 2.6262, + "theoretical_loss": 3.8150934398943193, + "tokens_seen": 638679040 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004073019057171515, + "loss": 2.7144, + "theoretical_loss": 3.8150530407342744, + "tokens_seen": 638744576 + }, + { + "epoch": 2.01, + "learning_rate": 0.00040729187562688063, + "loss": 2.8772, + "theoretical_loss": 3.815012646879487, + "tokens_seen": 638810112 + }, + { + "epoch": 2.01, + "learning_rate": 0.00040728184553660986, + "loss": 2.7468, + "theoretical_loss": 3.814972258328717, + "tokens_seen": 638875648 + }, + { + "epoch": 2.01, + "learning_rate": 0.000407271815446339, + "loss": 2.6933, + "theoretical_loss": 3.8149318750807235, + "tokens_seen": 638941184 + }, + { + "epoch": 2.01, + "objective/train/docs_used": 786054, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.3852269649505615, + "objective/train/theoretical_loss": 3.814911685444881, + "objective/train/tokens_used": 659433952, + "theoretical_loss": 3.814911685444881, + "tokens_seen": 638973952 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004072617853560682, + "loss": 2.5543, + "theoretical_loss": 3.8148914971342665, + "tokens_seen": 639006720 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004072517552657974, + "loss": 2.8611, + "theoretical_loss": 3.814851124488106, + "tokens_seen": 639072256 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004072417251755266, + "loss": 2.774, + "theoretical_loss": 3.8148107571410037, + "tokens_seen": 639137792 + }, + { + "epoch": 2.01, + "learning_rate": 0.00040723169508525577, + "loss": 2.7506, + "theoretical_loss": 3.8147703950917204, + "tokens_seen": 639203328 + }, + { + "epoch": 2.01, + "learning_rate": 0.00040722166499498495, + "loss": 2.7369, + "theoretical_loss": 3.8147300383390172, + "tokens_seen": 639268864 + }, + { + "epoch": 2.01, + "learning_rate": 0.00040721163490471413, + "loss": 2.4588, + "theoretical_loss": 3.814689686881657, + "tokens_seen": 639334400 + }, + { + "epoch": 2.01, + "learning_rate": 0.00040720160481444337, + "loss": 2.7529, + "theoretical_loss": 3.814649340718402, + "tokens_seen": 639399936 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004071915747241725, + "loss": 2.7639, + "theoretical_loss": 3.814608999848015, + "tokens_seen": 639465472 + }, + { + "epoch": 2.01, + "learning_rate": 0.00040718154463390173, + "loss": 2.9076, + "theoretical_loss": 3.814568664269259, + "tokens_seen": 639531008 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004071715145436309, + "loss": 2.7979, + "theoretical_loss": 3.8145283339808986, + "tokens_seen": 639596544 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004071614844533601, + "loss": 2.7578, + "theoretical_loss": 3.814488008981697, + "tokens_seen": 639662080 + }, + { + "epoch": 2.01, + "learning_rate": 0.00040715145436308927, + "loss": 3.0341, + "theoretical_loss": 3.8144476892704198, + "tokens_seen": 639727616 + }, + { + "epoch": 2.01, + "learning_rate": 0.00040714142427281845, + "loss": 2.897, + "theoretical_loss": 3.8144073748458305, + "tokens_seen": 639793152 + }, + { + "epoch": 2.01, + "learning_rate": 0.00040713139418254763, + "loss": 2.9447, + "theoretical_loss": 3.814367065706696, + "tokens_seen": 639858688 + }, + { + "epoch": 2.01, + "learning_rate": 0.00040712136409227687, + "loss": 2.8507, + "theoretical_loss": 3.814326761851781, + "tokens_seen": 639924224 + }, + { + "epoch": 2.01, + "learning_rate": 0.000407111334002006, + "loss": 2.7963, + "theoretical_loss": 3.8142864632798523, + "tokens_seen": 639989760 + }, + { + "epoch": 2.01, + "learning_rate": 0.00040710130391173523, + "loss": 2.8708, + "theoretical_loss": 3.8142461699896764, + "tokens_seen": 640055296 + }, + { + "epoch": 2.01, + "learning_rate": 0.00040709127382146436, + "loss": 2.8233, + "theoretical_loss": 3.8142058819800204, + "tokens_seen": 640120832 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004070812437311936, + "loss": 3.0831, + "theoretical_loss": 3.8141655992496517, + "tokens_seen": 640186368 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004070712136409228, + "loss": 2.7594, + "theoretical_loss": 3.814125321797338, + "tokens_seen": 640251904 + }, + { + "epoch": 2.01, + "learning_rate": 0.00040706118355065196, + "loss": 2.5152, + "theoretical_loss": 3.814085049621848, + "tokens_seen": 640317440 + }, + { + "epoch": 2.01, + "learning_rate": 0.00040705115346038114, + "loss": 2.7952, + "theoretical_loss": 3.81404478272195, + "tokens_seen": 640382976 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004070411233701103, + "loss": 2.7167, + "theoretical_loss": 3.8140045210964133, + "tokens_seen": 640448512 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004070310932798395, + "loss": 2.8056, + "theoretical_loss": 3.8139642647440075, + "tokens_seen": 640514048 + }, + { + "epoch": 2.01, + "learning_rate": 0.00040702106318956874, + "loss": 2.6948, + "theoretical_loss": 3.8139240136635024, + "tokens_seen": 640579584 + }, + { + "epoch": 2.01, + "objective/train/docs_used": 786648, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7099239826202393, + "objective/train/theoretical_loss": 3.8139038900998283, + "objective/train/tokens_used": 661072352, + "theoretical_loss": 3.8139038900998283, + "tokens_seen": 640612352 + }, + { + "epoch": 2.01, + "learning_rate": 0.00040701103309929786, + "loss": 2.7378, + "theoretical_loss": 3.8138837678536683, + "tokens_seen": 640645120 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004070010030090271, + "loss": 2.8674, + "theoretical_loss": 3.813843527313276, + "tokens_seen": 640710656 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004069909729187563, + "loss": 2.786, + "theoretical_loss": 3.813803292041097, + "tokens_seen": 640776192 + }, + { + "epoch": 2.01, + "learning_rate": 0.00040698094282848546, + "loss": 2.9161, + "theoretical_loss": 3.8137630620359024, + "tokens_seen": 640841728 + }, + { + "epoch": 2.01, + "learning_rate": 0.00040697091273821464, + "loss": 2.8332, + "theoretical_loss": 3.8137228372964644, + "tokens_seen": 640907264 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004069608826479438, + "loss": 2.754, + "theoretical_loss": 3.8136826178215557, + "tokens_seen": 640972800 + }, + { + "epoch": 2.01, + "learning_rate": 0.000406950852557673, + "loss": 2.7697, + "theoretical_loss": 3.8136424036099488, + "tokens_seen": 641038336 + }, + { + "epoch": 2.01, + "learning_rate": 0.00040694082246740224, + "loss": 2.7057, + "theoretical_loss": 3.8136021946604166, + "tokens_seen": 641103872 + }, + { + "epoch": 2.01, + "learning_rate": 0.00040693079237713137, + "loss": 2.8738, + "theoretical_loss": 3.813561990971734, + "tokens_seen": 641169408 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004069207622868606, + "loss": 2.8105, + "theoretical_loss": 3.8135217925426734, + "tokens_seen": 641234944 + }, + { + "epoch": 2.01, + "learning_rate": 0.00040691073219658973, + "loss": 2.8888, + "theoretical_loss": 3.81348159937201, + "tokens_seen": 641300480 + }, + { + "epoch": 2.01, + "learning_rate": 0.00040690070210631896, + "loss": 2.8901, + "theoretical_loss": 3.81344141145852, + "tokens_seen": 641366016 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004068906720160482, + "loss": 2.7657, + "theoretical_loss": 3.813401228800976, + "tokens_seen": 641431552 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004068806419257773, + "loss": 2.6503, + "theoretical_loss": 3.8133610513981555, + "tokens_seen": 641497088 + }, + { + "epoch": 2.01, + "learning_rate": 0.00040687061183550656, + "loss": 2.725, + "theoretical_loss": 3.8133208792488347, + "tokens_seen": 641562624 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004068605817452357, + "loss": 2.6084, + "theoretical_loss": 3.8132807123517893, + "tokens_seen": 641628160 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004068505516549649, + "loss": 2.8921, + "theoretical_loss": 3.813240550705797, + "tokens_seen": 641693696 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004068405215646941, + "loss": 2.7129, + "theoretical_loss": 3.813200394309635, + "tokens_seen": 641759232 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004068304914744233, + "loss": 2.7996, + "theoretical_loss": 3.8131602431620797, + "tokens_seen": 641824768 + }, + { + "epoch": 2.01, + "learning_rate": 0.00040682046138415247, + "loss": 2.9959, + "theoretical_loss": 3.8131200972619115, + "tokens_seen": 641890304 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004068104312938817, + "loss": 2.6639, + "theoretical_loss": 3.8130799566079077, + "tokens_seen": 641955840 + }, + { + "epoch": 2.01, + "learning_rate": 0.00040680040120361083, + "loss": 2.751, + "theoretical_loss": 3.813039821198847, + "tokens_seen": 642021376 + }, + { + "epoch": 2.01, + "learning_rate": 0.00040679037111334006, + "loss": 2.7787, + "theoretical_loss": 3.8129996910335096, + "tokens_seen": 642086912 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004067803410230692, + "loss": 2.8948, + "theoretical_loss": 3.8129595661106745, + "tokens_seen": 642152448 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004067703109327984, + "loss": 2.8215, + "theoretical_loss": 3.8129194464291225, + "tokens_seen": 642217984 + }, + { + "epoch": 2.01, + "objective/train/docs_used": 788043, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.803264856338501, + "objective/train/theoretical_loss": 3.8128993885534466, + "objective/train/tokens_used": 662710752, + "theoretical_loss": 3.8128993885534466, + "tokens_seen": 642250752 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004067602808425276, + "loss": 2.762, + "theoretical_loss": 3.812879331987634, + "tokens_seen": 642283520 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004067502507522568, + "loss": 2.7921, + "theoretical_loss": 3.8128392227849903, + "tokens_seen": 642349056 + }, + { + "epoch": 2.01, + "learning_rate": 0.00040674022066198597, + "loss": 2.869, + "theoretical_loss": 3.8127991188199726, + "tokens_seen": 642414592 + }, + { + "epoch": 2.01, + "learning_rate": 0.00040673019057171515, + "loss": 2.7139, + "theoretical_loss": 3.812759020091362, + "tokens_seen": 642480128 + }, + { + "epoch": 2.01, + "learning_rate": 0.00040672016048144433, + "loss": 2.9997, + "theoretical_loss": 3.812718926597942, + "tokens_seen": 642545664 + }, + { + "epoch": 2.01, + "learning_rate": 0.00040671013039117357, + "loss": 2.9864, + "theoretical_loss": 3.8126788383384946, + "tokens_seen": 642611200 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004067001003009027, + "loss": 2.9218, + "theoretical_loss": 3.812638755311803, + "tokens_seen": 642676736 + }, + { + "epoch": 2.01, + "learning_rate": 0.00040669007021063193, + "loss": 2.8599, + "theoretical_loss": 3.812598677516651, + "tokens_seen": 642742272 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004066800401203611, + "loss": 2.8809, + "theoretical_loss": 3.812558604951822, + "tokens_seen": 642807808 + }, + { + "epoch": 2.01, + "learning_rate": 0.0004066700100300903, + "loss": 2.9374, + "theoretical_loss": 3.8125185376161, + "tokens_seen": 642873344 + }, + { + "epoch": 2.01, + "learning_rate": 0.00040665997993981947, + "loss": 2.9225, + "theoretical_loss": 3.8124784755082706, + "tokens_seen": 642938880 + }, + { + "epoch": 2.01, + "learning_rate": 0.00040664994984954865, + "loss": 2.8897, + "theoretical_loss": 3.8124384186271176, + "tokens_seen": 643004416 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040663991975927783, + "loss": 2.67, + "theoretical_loss": 3.812398366971428, + "tokens_seen": 643069952 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040662988966900707, + "loss": 2.7487, + "theoretical_loss": 3.8123583205399862, + "tokens_seen": 643135488 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004066198595787362, + "loss": 2.8362, + "theoretical_loss": 3.8123182793315804, + "tokens_seen": 643201024 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040660982948846543, + "loss": 3.058, + "theoretical_loss": 3.8122782433449953, + "tokens_seen": 643266560 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040659979939819456, + "loss": 2.9504, + "theoretical_loss": 3.812238212579019, + "tokens_seen": 643332096 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004065897693079238, + "loss": 2.9792, + "theoretical_loss": 3.8121981870324393, + "tokens_seen": 643397632 + }, + { + "epoch": 2.02, + "learning_rate": 0.000406579739217653, + "loss": 2.7157, + "theoretical_loss": 3.8121581667040436, + "tokens_seen": 643463168 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040656970912738216, + "loss": 2.6707, + "theoretical_loss": 3.81211815159262, + "tokens_seen": 643528704 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040655967903711134, + "loss": 2.7079, + "theoretical_loss": 3.8120781416969587, + "tokens_seen": 643594240 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004065496489468405, + "loss": 2.9709, + "theoretical_loss": 3.812038137015847, + "tokens_seen": 643659776 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004065396188565697, + "loss": 2.7942, + "theoretical_loss": 3.811998137548075, + "tokens_seen": 643725312 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040652958876629894, + "loss": 3.1165, + "theoretical_loss": 3.8119581432924337, + "tokens_seen": 643790848 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040651955867602806, + "loss": 2.6032, + "theoretical_loss": 3.811918154247712, + "tokens_seen": 643856384 + }, + { + "epoch": 2.02, + "objective/train/docs_used": 789663, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.804586172103882, + "objective/train/theoretical_loss": 3.8118981616790686, + "objective/train/tokens_used": 664349152, + "theoretical_loss": 3.8118981616790686, + "tokens_seen": 643889152 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004065095285857573, + "loss": 2.8536, + "theoretical_loss": 3.811878170412702, + "tokens_seen": 643921920 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004064994984954865, + "loss": 2.9554, + "theoretical_loss": 3.811838191786193, + "tokens_seen": 643987456 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040648946840521566, + "loss": 2.9039, + "theoretical_loss": 3.8117982183669787, + "tokens_seen": 644052992 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040647943831494484, + "loss": 2.6216, + "theoretical_loss": 3.81175825015385, + "tokens_seen": 644118528 + }, + { + "epoch": 2.02, + "learning_rate": 0.000406469408224674, + "loss": 2.8063, + "theoretical_loss": 3.8117182871455997, + "tokens_seen": 644184064 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004064593781344032, + "loss": 2.8755, + "theoretical_loss": 3.81167832934102, + "tokens_seen": 644249600 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040644934804413244, + "loss": 2.7333, + "theoretical_loss": 3.811638376738904, + "tokens_seen": 644315136 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040643931795386157, + "loss": 2.8455, + "theoretical_loss": 3.811598429338046, + "tokens_seen": 644380672 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004064292878635908, + "loss": 3.0777, + "theoretical_loss": 3.8115584871372397, + "tokens_seen": 644446208 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040641925777331993, + "loss": 2.8559, + "theoretical_loss": 3.811518550135279, + "tokens_seen": 644511744 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040640922768304916, + "loss": 3.0454, + "theoretical_loss": 3.811478618330959, + "tokens_seen": 644577280 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040639919759277834, + "loss": 2.5906, + "theoretical_loss": 3.8114386917230756, + "tokens_seen": 644642816 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004063891675025075, + "loss": 2.681, + "theoretical_loss": 3.8113987703104235, + "tokens_seen": 644708352 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004063891675025075, + "loss": 2.8988, + "theoretical_loss": 3.811358854091799, + "tokens_seen": 644773888 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004063791374122367, + "loss": 2.8432, + "theoretical_loss": 3.8113189430659986, + "tokens_seen": 644839424 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004063691073219659, + "loss": 2.8233, + "theoretical_loss": 3.811279037231819, + "tokens_seen": 644904960 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040635907723169507, + "loss": 2.7678, + "theoretical_loss": 3.8112391365880574, + "tokens_seen": 644970496 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004063490471414243, + "loss": 2.9558, + "theoretical_loss": 3.811199241133511, + "tokens_seen": 645036032 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040633901705115343, + "loss": 2.9838, + "theoretical_loss": 3.811159350866978, + "tokens_seen": 645101568 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040632898696088267, + "loss": 2.69, + "theoretical_loss": 3.8111194657872582, + "tokens_seen": 645167104 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040631895687061185, + "loss": 2.763, + "theoretical_loss": 3.8110795858931485, + "tokens_seen": 645232640 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040630892678034103, + "loss": 2.7282, + "theoretical_loss": 3.811039711183448, + "tokens_seen": 645298176 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004062988966900702, + "loss": 2.7764, + "theoretical_loss": 3.810999841656958, + "tokens_seen": 645363712 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004062888665997994, + "loss": 2.7889, + "theoretical_loss": 3.8109599773124776, + "tokens_seen": 645429248 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040627883650952857, + "loss": 2.6931, + "theoretical_loss": 3.810920118148807, + "tokens_seen": 645494784 + }, + { + "epoch": 2.02, + "objective/train/docs_used": 790384, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9371323585510254, + "objective/train/theoretical_loss": 3.810900190509401, + "objective/train/tokens_used": 665987552, + "theoretical_loss": 3.810900190509401, + "tokens_seen": 645527552 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004062688064192578, + "loss": 2.9117, + "theoretical_loss": 3.810880264164747, + "tokens_seen": 645560320 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040625877632898693, + "loss": 2.9301, + "theoretical_loss": 3.8108404153590993, + "tokens_seen": 645625856 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040624874623871617, + "loss": 2.7162, + "theoretical_loss": 3.810800571730665, + "tokens_seen": 645691392 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004062387161484453, + "loss": 2.9238, + "theoretical_loss": 3.8107607332782463, + "tokens_seen": 645756928 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040622868605817453, + "loss": 3.0003, + "theoretical_loss": 3.8107209000006455, + "tokens_seen": 645822464 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004062186559679037, + "loss": 3.068, + "theoretical_loss": 3.8106810718966657, + "tokens_seen": 645888000 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004062086258776329, + "loss": 3.066, + "theoretical_loss": 3.8106412489651103, + "tokens_seen": 645953536 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004061985957873621, + "loss": 2.8349, + "theoretical_loss": 3.810601431204782, + "tokens_seen": 646019072 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040618856569709126, + "loss": 2.8544, + "theoretical_loss": 3.810561618614485, + "tokens_seen": 646084608 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040617853560682044, + "loss": 2.753, + "theoretical_loss": 3.810521811193024, + "tokens_seen": 646150144 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040616850551654967, + "loss": 2.7691, + "theoretical_loss": 3.8104820089392035, + "tokens_seen": 646215680 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004061584754262788, + "loss": 2.8669, + "theoretical_loss": 3.8104422118518295, + "tokens_seen": 646281216 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040614844533600804, + "loss": 2.9435, + "theoretical_loss": 3.810402419929707, + "tokens_seen": 646346752 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040613841524573727, + "loss": 2.8705, + "theoretical_loss": 3.810362633171642, + "tokens_seen": 646412288 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004061283851554664, + "loss": 2.8442, + "theoretical_loss": 3.81032285157644, + "tokens_seen": 646477824 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040611835506519563, + "loss": 2.6825, + "theoretical_loss": 3.8102830751429093, + "tokens_seen": 646543360 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040610832497492476, + "loss": 2.8439, + "theoretical_loss": 3.810243303869856, + "tokens_seen": 646608896 + }, + { + "epoch": 2.02, + "learning_rate": 0.000406098294884654, + "loss": 2.5503, + "theoretical_loss": 3.810203537756088, + "tokens_seen": 646674432 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004060882647943832, + "loss": 2.5302, + "theoretical_loss": 3.8101637768004144, + "tokens_seen": 646739968 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040607823470411236, + "loss": 2.8447, + "theoretical_loss": 3.8101240210016414, + "tokens_seen": 646805504 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040606820461384154, + "loss": 2.9075, + "theoretical_loss": 3.810084270358579, + "tokens_seen": 646871040 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004060581745235707, + "loss": 2.8127, + "theoretical_loss": 3.810044524870036, + "tokens_seen": 646936576 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004060481444332999, + "loss": 2.7271, + "theoretical_loss": 3.810004784534823, + "tokens_seen": 647002112 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040603811434302914, + "loss": 2.7557, + "theoretical_loss": 3.8099650493517476, + "tokens_seen": 647067648 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040602808425275826, + "loss": 2.7964, + "theoretical_loss": 3.809925319319623, + "tokens_seen": 647133184 + }, + { + "epoch": 2.02, + "objective/train/docs_used": 791083, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.1197621822357178, + "objective/train/theoretical_loss": 3.8099054562347945, + "objective/train/tokens_used": 667625952, + "theoretical_loss": 3.8099054562347945, + "tokens_seen": 647165952 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004060180541624875, + "loss": 2.8243, + "theoretical_loss": 3.809885594437258, + "tokens_seen": 647198720 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004060080240722167, + "loss": 2.7903, + "theoretical_loss": 3.8098458747034636, + "tokens_seen": 647264256 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040599799398194586, + "loss": 2.6526, + "theoretical_loss": 3.809806160117053, + "tokens_seen": 647329792 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040598796389167504, + "loss": 2.8791, + "theoretical_loss": 3.8097664506768365, + "tokens_seen": 647395328 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004059779338014042, + "loss": 2.6126, + "theoretical_loss": 3.809726746381627, + "tokens_seen": 647460864 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004059679037111334, + "loss": 2.7736, + "theoretical_loss": 3.8096870472302378, + "tokens_seen": 647526400 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040595787362086264, + "loss": 2.7677, + "theoretical_loss": 3.8096473532214805, + "tokens_seen": 647591936 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040594784353059177, + "loss": 2.7009, + "theoretical_loss": 3.8096076643541696, + "tokens_seen": 647657472 + }, + { + "epoch": 2.02, + "learning_rate": 0.000405937813440321, + "loss": 2.745, + "theoretical_loss": 3.8095679806271194, + "tokens_seen": 647723008 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040592778335005013, + "loss": 2.7262, + "theoretical_loss": 3.8095283020391433, + "tokens_seen": 647788544 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040591775325977936, + "loss": 2.8404, + "theoretical_loss": 3.8094886285890563, + "tokens_seen": 647854080 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040590772316950854, + "loss": 2.7365, + "theoretical_loss": 3.8094489602756734, + "tokens_seen": 647919616 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004058976930792377, + "loss": 2.9187, + "theoretical_loss": 3.809409297097811, + "tokens_seen": 647985152 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004058876629889669, + "loss": 2.6985, + "theoretical_loss": 3.809369639054283, + "tokens_seen": 648050688 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004058776328986961, + "loss": 2.5365, + "theoretical_loss": 3.809329986143907, + "tokens_seen": 648116224 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040586760280842527, + "loss": 2.8878, + "theoretical_loss": 3.8092903383654995, + "tokens_seen": 648181760 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004058575727181545, + "loss": 2.6767, + "theoretical_loss": 3.8092506957178776, + "tokens_seen": 648247296 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040584754262788363, + "loss": 2.8827, + "theoretical_loss": 3.809211058199858, + "tokens_seen": 648312832 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040583751253761287, + "loss": 2.7196, + "theoretical_loss": 3.80917142581026, + "tokens_seen": 648378368 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040582748244734205, + "loss": 3.0595, + "theoretical_loss": 3.8091317985478996, + "tokens_seen": 648443904 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040581745235707123, + "loss": 2.8787, + "theoretical_loss": 3.8090921764115975, + "tokens_seen": 648509440 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004058074222668004, + "loss": 2.9666, + "theoretical_loss": 3.809052559400172, + "tokens_seen": 648574976 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004057973921765296, + "loss": 2.8784, + "theoretical_loss": 3.809012947512442, + "tokens_seen": 648640512 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040578736208625877, + "loss": 2.7818, + "theoretical_loss": 3.8089733407472273, + "tokens_seen": 648706048 + }, + { + "epoch": 2.02, + "learning_rate": 0.000405777331995988, + "loss": 3.0789, + "theoretical_loss": 3.8089337391033484, + "tokens_seen": 648771584 + }, + { + "epoch": 2.02, + "objective/train/docs_used": 792264, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.787963628768921, + "objective/train/theoretical_loss": 3.8089139402015415, + "objective/train/tokens_used": 669264352, + "theoretical_loss": 3.8089139402015415, + "tokens_seen": 648804352 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040576730190571713, + "loss": 2.8683, + "theoretical_loss": 3.808894142579626, + "tokens_seen": 648837120 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040575727181544637, + "loss": 2.8342, + "theoretical_loss": 3.808854551174881, + "tokens_seen": 648902656 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004057472417251755, + "loss": 2.7637, + "theoretical_loss": 3.8088149648879344, + "tokens_seen": 648968192 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040573721163490473, + "loss": 2.8181, + "theoretical_loss": 3.8087753837176077, + "tokens_seen": 649033728 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004057271815446339, + "loss": 2.8324, + "theoretical_loss": 3.8087358076627242, + "tokens_seen": 649099264 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004057171514543631, + "loss": 2.9637, + "theoretical_loss": 3.8086962367221053, + "tokens_seen": 649164800 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004057071213640923, + "loss": 2.9783, + "theoretical_loss": 3.808656670894574, + "tokens_seen": 649230336 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040569709127382146, + "loss": 3.0008, + "theoretical_loss": 3.8086171101789543, + "tokens_seen": 649295872 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040568706118355064, + "loss": 2.7163, + "theoretical_loss": 3.808577554574069, + "tokens_seen": 649361408 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004056770310932799, + "loss": 2.6197, + "theoretical_loss": 3.808538004078743, + "tokens_seen": 649426944 + }, + { + "epoch": 2.02, + "learning_rate": 0.000405667001003009, + "loss": 2.8005, + "theoretical_loss": 3.8084984586918, + "tokens_seen": 649492480 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040565697091273824, + "loss": 2.9181, + "theoretical_loss": 3.808458918412065, + "tokens_seen": 649558016 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004056469408224674, + "loss": 2.7293, + "theoretical_loss": 3.8084193832383635, + "tokens_seen": 649623552 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004056369107321966, + "loss": 2.7755, + "theoretical_loss": 3.8083798531695208, + "tokens_seen": 649689088 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004056268806419258, + "loss": 2.9192, + "theoretical_loss": 3.8083403282043635, + "tokens_seen": 649754624 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040561685055165496, + "loss": 2.5435, + "theoretical_loss": 3.808300808341717, + "tokens_seen": 649820160 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040560682046138414, + "loss": 2.7715, + "theoretical_loss": 3.8082612935804097, + "tokens_seen": 649885696 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004055967903711134, + "loss": 2.7429, + "theoretical_loss": 3.808221783919267, + "tokens_seen": 649951232 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004055867602808425, + "loss": 2.8494, + "theoretical_loss": 3.8081822793571174, + "tokens_seen": 650016768 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040557673019057174, + "loss": 2.5832, + "theoretical_loss": 3.8081427798927887, + "tokens_seen": 650082304 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040556670010030087, + "loss": 2.797, + "theoretical_loss": 3.808103285525109, + "tokens_seen": 650147840 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004055566700100301, + "loss": 2.9601, + "theoretical_loss": 3.8080637962529074, + "tokens_seen": 650213376 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004055466399197593, + "loss": 3.0431, + "theoretical_loss": 3.808024312075013, + "tokens_seen": 650278912 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040553660982948846, + "loss": 2.8572, + "theoretical_loss": 3.807984832990255, + "tokens_seen": 650344448 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040552657973921764, + "loss": 2.8196, + "theoretical_loss": 3.807945358997463, + "tokens_seen": 650409984 + }, + { + "epoch": 2.02, + "objective/train/docs_used": 793032, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8657760620117188, + "objective/train/theoretical_loss": 3.8079256239101893, + "objective/train/tokens_used": 670902752, + "theoretical_loss": 3.8079256239101893, + "tokens_seen": 650442752 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004055165496489469, + "loss": 2.8388, + "theoretical_loss": 3.8079058900954683, + "tokens_seen": 650475520 + }, + { + "epoch": 2.02, + "learning_rate": 0.000405506519558676, + "loss": 2.9388, + "theoretical_loss": 3.807866426283101, + "tokens_seen": 650541056 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040549648946840524, + "loss": 2.7149, + "theoretical_loss": 3.8078269675591914, + "tokens_seen": 650606592 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040548645937813437, + "loss": 2.6655, + "theoretical_loss": 3.8077875139225723, + "tokens_seen": 650672128 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004054764292878636, + "loss": 2.708, + "theoretical_loss": 3.807748065372075, + "tokens_seen": 650737664 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004054663991975928, + "loss": 3.0241, + "theoretical_loss": 3.807708621906531, + "tokens_seen": 650803200 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040545636910732197, + "loss": 2.9069, + "theoretical_loss": 3.807669183524774, + "tokens_seen": 650868736 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040544633901705115, + "loss": 2.7804, + "theoretical_loss": 3.8076297502256358, + "tokens_seen": 650934272 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040543630892678033, + "loss": 3.0921, + "theoretical_loss": 3.8075903220079503, + "tokens_seen": 650999808 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004054262788365095, + "loss": 2.917, + "theoretical_loss": 3.807550898870552, + "tokens_seen": 651065344 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040541624874623874, + "loss": 2.6225, + "theoretical_loss": 3.8075114808122734, + "tokens_seen": 651130880 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040540621865596787, + "loss": 2.7534, + "theoretical_loss": 3.807472067831951, + "tokens_seen": 651196416 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004053961885656971, + "loss": 2.6766, + "theoretical_loss": 3.807432659928418, + "tokens_seen": 651261952 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004053861584754263, + "loss": 2.7737, + "theoretical_loss": 3.8073932571005105, + "tokens_seen": 651327488 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040537612838515547, + "loss": 2.8894, + "theoretical_loss": 3.807353859347064, + "tokens_seen": 651393024 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004053660982948847, + "loss": 2.7208, + "theoretical_loss": 3.8073144666669148, + "tokens_seen": 651458560 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040535606820461383, + "loss": 2.686, + "theoretical_loss": 3.8072750790588987, + "tokens_seen": 651524096 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040534603811434307, + "loss": 2.7685, + "theoretical_loss": 3.807235696521853, + "tokens_seen": 651589632 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040533600802407225, + "loss": 2.694, + "theoretical_loss": 3.807196319054615, + "tokens_seen": 651655168 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040532597793380143, + "loss": 2.8357, + "theoretical_loss": 3.807156946656022, + "tokens_seen": 651720704 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004053159478435306, + "loss": 2.7878, + "theoretical_loss": 3.8071175793249123, + "tokens_seen": 651786240 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004053059177532598, + "loss": 2.9112, + "theoretical_loss": 3.8070782170601234, + "tokens_seen": 651851776 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040529588766298897, + "loss": 2.8191, + "theoretical_loss": 3.807038859860495, + "tokens_seen": 651917312 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004052858575727182, + "loss": 2.8678, + "theoretical_loss": 3.806999507724866, + "tokens_seen": 651982848 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040527582748244733, + "loss": 2.8658, + "theoretical_loss": 3.8069601606520758, + "tokens_seen": 652048384 + }, + { + "epoch": 2.02, + "objective/train/docs_used": 794457, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.615185499191284, + "objective/train/theoretical_loss": 3.8069404890138827, + "objective/train/tokens_used": 672541152, + "theoretical_loss": 3.8069404890138827, + "tokens_seen": 652081152 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040526579739217657, + "loss": 2.8081, + "theoretical_loss": 3.8069208186409647, + "tokens_seen": 652113920 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004052557673019057, + "loss": 3.0599, + "theoretical_loss": 3.8068814816903718, + "tokens_seen": 652179456 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040524573721163493, + "loss": 2.475, + "theoretical_loss": 3.806842149799139, + "tokens_seen": 652244992 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004052357071213641, + "loss": 2.8371, + "theoretical_loss": 3.806802822966106, + "tokens_seen": 652310528 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004052256770310933, + "loss": 2.9591, + "theoretical_loss": 3.806763501190116, + "tokens_seen": 652376064 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004052156469408225, + "loss": 3.0144, + "theoretical_loss": 3.8067241844700095, + "tokens_seen": 652441600 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040520561685055166, + "loss": 2.9075, + "theoretical_loss": 3.8066848728046287, + "tokens_seen": 652507136 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040519558676028084, + "loss": 2.8039, + "theoretical_loss": 3.8066455661928167, + "tokens_seen": 652572672 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004051855566700101, + "loss": 2.9402, + "theoretical_loss": 3.806606264633416, + "tokens_seen": 652638208 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004051755265797392, + "loss": 2.6916, + "theoretical_loss": 3.8065669681252707, + "tokens_seen": 652703744 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040516549648946844, + "loss": 2.9478, + "theoretical_loss": 3.8065276766672236, + "tokens_seen": 652769280 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004051554663991976, + "loss": 2.8893, + "theoretical_loss": 3.806488390258119, + "tokens_seen": 652834816 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004051454363089268, + "loss": 2.8495, + "theoretical_loss": 3.8064491088968015, + "tokens_seen": 652900352 + }, + { + "epoch": 2.02, + "learning_rate": 0.000405135406218656, + "loss": 2.9459, + "theoretical_loss": 3.8064098325821156, + "tokens_seen": 652965888 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040512537612838516, + "loss": 2.7826, + "theoretical_loss": 3.8063705613129075, + "tokens_seen": 653031424 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040511534603811434, + "loss": 2.928, + "theoretical_loss": 3.8063312950880217, + "tokens_seen": 653096960 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004051053159478436, + "loss": 2.9083, + "theoretical_loss": 3.806292033906305, + "tokens_seen": 653162496 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004050952858575727, + "loss": 2.9771, + "theoretical_loss": 3.806252777766603, + "tokens_seen": 653228032 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040508525576730194, + "loss": 2.8971, + "theoretical_loss": 3.806213526667763, + "tokens_seen": 653293568 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040507522567703107, + "loss": 2.8588, + "theoretical_loss": 3.806174280608632, + "tokens_seen": 653359104 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004050651955867603, + "loss": 2.9839, + "theoretical_loss": 3.8061350395880567, + "tokens_seen": 653424640 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004050551654964895, + "loss": 2.6543, + "theoretical_loss": 3.8060958036048866, + "tokens_seen": 653490176 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040504513540621866, + "loss": 2.8108, + "theoretical_loss": 3.806056572657969, + "tokens_seen": 653555712 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040503510531594784, + "loss": 2.7504, + "theoretical_loss": 3.8060173467461524, + "tokens_seen": 653621248 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004050250752256771, + "loss": 2.9447, + "theoretical_loss": 3.805978125868286, + "tokens_seen": 653686784 + }, + { + "epoch": 2.02, + "objective/train/docs_used": 795253, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.0313971042633057, + "objective/train/theoretical_loss": 3.805958517316725, + "objective/train/tokens_used": 674179552, + "theoretical_loss": 3.805958517316725, + "tokens_seen": 653719552 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004050150451354062, + "loss": 2.9876, + "theoretical_loss": 3.8059389100232197, + "tokens_seen": 653752320 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040500501504513544, + "loss": 2.7331, + "theoretical_loss": 3.805899699209802, + "tokens_seen": 653817856 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040499498495486457, + "loss": 2.6985, + "theoretical_loss": 3.8058604934268843, + "tokens_seen": 653883392 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004049849548645938, + "loss": 2.6745, + "theoretical_loss": 3.8058212926733175, + "tokens_seen": 653948928 + }, + { + "epoch": 2.02, + "learning_rate": 0.000404974924774323, + "loss": 2.5625, + "theoretical_loss": 3.8057820969479508, + "tokens_seen": 654014464 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040496489468405217, + "loss": 2.6322, + "theoretical_loss": 3.805742906249636, + "tokens_seen": 654080000 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040495486459378135, + "loss": 2.7656, + "theoretical_loss": 3.8057037205772257, + "tokens_seen": 654145536 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040494483450351053, + "loss": 2.8471, + "theoretical_loss": 3.8056645399295714, + "tokens_seen": 654211072 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004049348044132397, + "loss": 2.807, + "theoretical_loss": 3.805625364305526, + "tokens_seen": 654276608 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040492477432296894, + "loss": 2.897, + "theoretical_loss": 3.805586193703941, + "tokens_seen": 654342144 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040491474423269807, + "loss": 2.5623, + "theoretical_loss": 3.8055470281236707, + "tokens_seen": 654407680 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004049047141424273, + "loss": 2.6036, + "theoretical_loss": 3.805507867563568, + "tokens_seen": 654473216 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040489468405215643, + "loss": 2.7559, + "theoretical_loss": 3.805468712022488, + "tokens_seen": 654538752 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040488465396188567, + "loss": 2.7901, + "theoretical_loss": 3.8054295614992832, + "tokens_seen": 654604288 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040487462387161485, + "loss": 2.9805, + "theoretical_loss": 3.80539041599281, + "tokens_seen": 654669824 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040486459378134403, + "loss": 2.7287, + "theoretical_loss": 3.805351275501922, + "tokens_seen": 654735360 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004048545636910732, + "loss": 2.7279, + "theoretical_loss": 3.8053121400254755, + "tokens_seen": 654800896 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040484453360080245, + "loss": 3.0278, + "theoretical_loss": 3.8052730095623266, + "tokens_seen": 654866432 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004048345035105316, + "loss": 2.7637, + "theoretical_loss": 3.8052338841113302, + "tokens_seen": 654931968 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004048244734202608, + "loss": 2.7719, + "theoretical_loss": 3.805194763671344, + "tokens_seen": 654997504 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040481444332998994, + "loss": 2.857, + "theoretical_loss": 3.8051556482412248, + "tokens_seen": 655063040 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040480441323971917, + "loss": 2.822, + "theoretical_loss": 3.8051165378198295, + "tokens_seen": 655128576 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040479438314944835, + "loss": 2.7325, + "theoretical_loss": 3.8050774324060157, + "tokens_seen": 655194112 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040478435305917753, + "loss": 3.0475, + "theoretical_loss": 3.8050383319986425, + "tokens_seen": 655259648 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004047743229689067, + "loss": 2.7759, + "theoretical_loss": 3.804999236596567, + "tokens_seen": 655325184 + }, + { + "debugging/Self-BLEU-5": 0.4859850452541583, + "debugging/distinct-1-grams": 0.7709278301586879, + "debugging/distinct-2-grams": 0.9620726312090557, + "debugging/entropy-1-grams": 5.7746798218568784, + "debugging/entropy-2-grams": 6.637753508493868, + "debugging/length": 600.7777777777778, + "debugging/num_segments": 9, + "debugging/score": 0.006362688036581238, + "debugging/score_std": 0.004646295055099869, + "epoch": 2.02, + "objective/train/docs_used": 796377, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9904468059539795, + "objective/train/theoretical_loss": 3.8049796907721594, + "objective/train/tokens_used": 675817952, + "theoretical_loss": 3.8049796907721594, + "tokens_seen": 655357952 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004047642928786359, + "loss": 2.8813, + "theoretical_loss": 3.8049601461986486, + "tokens_seen": 655390720 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004047542627883651, + "loss": 2.9686, + "theoretical_loss": 3.8049210608037463, + "tokens_seen": 655456256 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004047442326980943, + "loss": 2.8707, + "theoretical_loss": 3.8048819804107197, + "tokens_seen": 655521792 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040473420260782344, + "loss": 2.6262, + "theoretical_loss": 3.804842905018429, + "tokens_seen": 655587328 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004047241725175527, + "loss": 2.9797, + "theoretical_loss": 3.8048038346257345, + "tokens_seen": 655652864 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004047141424272818, + "loss": 2.845, + "theoretical_loss": 3.8047647692314968, + "tokens_seen": 655718400 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040470411233701104, + "loss": 2.7958, + "theoretical_loss": 3.8047257088345763, + "tokens_seen": 655783936 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004046940822467402, + "loss": 2.9501, + "theoretical_loss": 3.8046866534338353, + "tokens_seen": 655849472 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004046840521564694, + "loss": 2.7035, + "theoretical_loss": 3.804647603028135, + "tokens_seen": 655915008 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004046740220661986, + "loss": 2.6593, + "theoretical_loss": 3.804608557616338, + "tokens_seen": 655980544 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004046639919759278, + "loss": 2.9606, + "theoretical_loss": 3.8045695171973066, + "tokens_seen": 656046080 + }, + { + "epoch": 2.02, + "learning_rate": 0.000404653961885657, + "loss": 2.9506, + "theoretical_loss": 3.8045304817699037, + "tokens_seen": 656111616 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004046439317953862, + "loss": 2.6727, + "theoretical_loss": 3.804491451332993, + "tokens_seen": 656177152 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040463390170511536, + "loss": 2.5689, + "theoretical_loss": 3.8044524258854375, + "tokens_seen": 656242688 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040462387161484454, + "loss": 2.7187, + "theoretical_loss": 3.8044134054261014, + "tokens_seen": 656308224 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004046138415245738, + "loss": 2.7893, + "theoretical_loss": 3.80437438995385, + "tokens_seen": 656373760 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004046038114343029, + "loss": 2.8434, + "theoretical_loss": 3.8043353794675463, + "tokens_seen": 656439296 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040459378134403214, + "loss": 2.9646, + "theoretical_loss": 3.804296373966057, + "tokens_seen": 656504832 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040458375125376127, + "loss": 2.9982, + "theoretical_loss": 3.8042573734482463, + "tokens_seen": 656570368 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004045737211634905, + "loss": 2.9142, + "theoretical_loss": 3.8042183779129823, + "tokens_seen": 656635904 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004045636910732197, + "loss": 2.8449, + "theoretical_loss": 3.8041793873591283, + "tokens_seen": 656701440 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040455366098294886, + "loss": 2.8761, + "theoretical_loss": 3.8041404017855536, + "tokens_seen": 656766976 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040454363089267804, + "loss": 2.7735, + "theoretical_loss": 3.804101421191124, + "tokens_seen": 656832512 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004045336008024073, + "loss": 2.761, + "theoretical_loss": 3.804062445574706, + "tokens_seen": 656898048 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004045235707121364, + "loss": 2.6464, + "theoretical_loss": 3.8040234749351693, + "tokens_seen": 656963584 + }, + { + "epoch": 2.02, + "objective/train/docs_used": 797121, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.958272695541382, + "objective/train/theoretical_loss": 3.804003991481377, + "objective/train/tokens_used": 677456352, + "theoretical_loss": 3.804003991481377, + "tokens_seen": 656996352 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040451354062186564, + "loss": 2.8226, + "theoretical_loss": 3.8039845092713804, + "tokens_seen": 657029120 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040450351053159477, + "loss": 2.9393, + "theoretical_loss": 3.8039455485822087, + "tokens_seen": 657094656 + }, + { + "epoch": 2.02, + "learning_rate": 0.000404493480441324, + "loss": 2.8633, + "theoretical_loss": 3.8039065928665226, + "tokens_seen": 657160192 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004044834503510532, + "loss": 2.9581, + "theoretical_loss": 3.8038676421231914, + "tokens_seen": 657225728 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040447342026078237, + "loss": 2.7917, + "theoretical_loss": 3.8038286963510846, + "tokens_seen": 657291264 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040446339017051155, + "loss": 2.6505, + "theoretical_loss": 3.8037897555490723, + "tokens_seen": 657356800 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040445336008024073, + "loss": 3.0437, + "theoretical_loss": 3.803750819716025, + "tokens_seen": 657422336 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004044433299899699, + "loss": 2.9178, + "theoretical_loss": 3.8037118888508132, + "tokens_seen": 657487872 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040443329989969915, + "loss": 2.7833, + "theoretical_loss": 3.803672962952308, + "tokens_seen": 657553408 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040442326980942827, + "loss": 2.8205, + "theoretical_loss": 3.803634042019381, + "tokens_seen": 657618944 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004044132397191575, + "loss": 2.6532, + "theoretical_loss": 3.8035951260509036, + "tokens_seen": 657684480 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040440320962888663, + "loss": 2.9728, + "theoretical_loss": 3.8035562150457487, + "tokens_seen": 657750016 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040439317953861587, + "loss": 2.8446, + "theoretical_loss": 3.803517309002788, + "tokens_seen": 657815552 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040438314944834505, + "loss": 2.6177, + "theoretical_loss": 3.8034784079208945, + "tokens_seen": 657881088 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040437311935807423, + "loss": 2.8787, + "theoretical_loss": 3.803439511798943, + "tokens_seen": 657946624 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004043630892678034, + "loss": 3.0163, + "theoretical_loss": 3.8034006206358044, + "tokens_seen": 658012160 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040435305917753265, + "loss": 2.7094, + "theoretical_loss": 3.803361734430355, + "tokens_seen": 658077696 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004043430290872618, + "loss": 2.9232, + "theoretical_loss": 3.8033228531814682, + "tokens_seen": 658143232 + }, + { + "epoch": 2.02, + "learning_rate": 0.000404332998996991, + "loss": 2.9224, + "theoretical_loss": 3.803283976888019, + "tokens_seen": 658208768 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040432296890672014, + "loss": 2.6132, + "theoretical_loss": 3.803245105548883, + "tokens_seen": 658274304 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040431293881644937, + "loss": 2.7078, + "theoretical_loss": 3.803206239162935, + "tokens_seen": 658339840 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040430290872617855, + "loss": 2.6425, + "theoretical_loss": 3.803167377729051, + "tokens_seen": 658405376 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040429287863590773, + "loss": 2.7553, + "theoretical_loss": 3.803128521246107, + "tokens_seen": 658470912 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004042828485456369, + "loss": 2.9433, + "theoretical_loss": 3.80308966971298, + "tokens_seen": 658536448 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004042728184553661, + "loss": 2.9494, + "theoretical_loss": 3.8030508231285465, + "tokens_seen": 658601984 + }, + { + "epoch": 2.02, + "objective/train/docs_used": 798701, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.928380250930786, + "objective/train/theoretical_loss": 3.8030314016917397, + "objective/train/tokens_used": 679094752, + "theoretical_loss": 3.8030314016917397, + "tokens_seen": 658634752 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004042627883650953, + "loss": 3.0227, + "theoretical_loss": 3.803011981491685, + "tokens_seen": 658667520 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004042527582748245, + "loss": 2.8351, + "theoretical_loss": 3.802973144801272, + "tokens_seen": 658733056 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040424272818455364, + "loss": 2.837, + "theoretical_loss": 3.802934313056186, + "tokens_seen": 658798592 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004042326980942829, + "loss": 2.9081, + "theoretical_loss": 3.802895486255305, + "tokens_seen": 658864128 + }, + { + "epoch": 2.02, + "learning_rate": 0.000404222668004012, + "loss": 2.8801, + "theoretical_loss": 3.802856664397508, + "tokens_seen": 658929664 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040421263791374124, + "loss": 2.8748, + "theoretical_loss": 3.8028178474816743, + "tokens_seen": 658995200 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004042026078234704, + "loss": 2.9186, + "theoretical_loss": 3.802779035506684, + "tokens_seen": 659060736 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004041925777331996, + "loss": 2.8017, + "theoretical_loss": 3.802740228471416, + "tokens_seen": 659126272 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004041825476429288, + "loss": 2.7335, + "theoretical_loss": 3.8027014263747505, + "tokens_seen": 659191808 + }, + { + "epoch": 2.02, + "learning_rate": 0.000404172517552658, + "loss": 2.7138, + "theoretical_loss": 3.802662629215569, + "tokens_seen": 659257344 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040416248746238714, + "loss": 2.843, + "theoretical_loss": 3.802623836992752, + "tokens_seen": 659322880 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004041524573721164, + "loss": 2.9381, + "theoretical_loss": 3.8025850497051805, + "tokens_seen": 659388416 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004041424272818455, + "loss": 3.0615, + "theoretical_loss": 3.802546267351737, + "tokens_seen": 659453952 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040413239719157474, + "loss": 2.6555, + "theoretical_loss": 3.802507489931303, + "tokens_seen": 659519488 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004041223671013039, + "loss": 2.6922, + "theoretical_loss": 3.802468717442761, + "tokens_seen": 659585024 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004041123370110331, + "loss": 2.8747, + "theoretical_loss": 3.802429949884994, + "tokens_seen": 659650560 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004041023069207623, + "loss": 2.6937, + "theoretical_loss": 3.8023911872568847, + "tokens_seen": 659716096 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040409227683049147, + "loss": 2.9162, + "theoretical_loss": 3.8023524295573177, + "tokens_seen": 659781632 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040408224674022065, + "loss": 2.7625, + "theoretical_loss": 3.8023136767851753, + "tokens_seen": 659847168 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004040722166499499, + "loss": 2.3571, + "theoretical_loss": 3.802274928939343, + "tokens_seen": 659912704 + }, + { + "epoch": 2.02, + "learning_rate": 0.000404062186559679, + "loss": 2.8614, + "theoretical_loss": 3.802236186018705, + "tokens_seen": 659978240 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040405215646940824, + "loss": 2.8778, + "theoretical_loss": 3.802197448022147, + "tokens_seen": 660043776 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040404212637913737, + "loss": 2.7498, + "theoretical_loss": 3.802158714948553, + "tokens_seen": 660109312 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004040320962888666, + "loss": 2.893, + "theoretical_loss": 3.8021199867968094, + "tokens_seen": 660174848 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004040220661985958, + "loss": 2.9762, + "theoretical_loss": 3.8020812635658023, + "tokens_seen": 660240384 + }, + { + "epoch": 2.02, + "objective/train/docs_used": 799454, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4462287425994873, + "objective/train/theoretical_loss": 3.8020619037952272, + "objective/train/tokens_used": 680733152, + "theoretical_loss": 3.8020619037952272, + "tokens_seen": 660273152 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040401203610832497, + "loss": 2.8877, + "theoretical_loss": 3.802042545254418, + "tokens_seen": 660305920 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040400200601805415, + "loss": 2.8215, + "theoretical_loss": 3.802003831861544, + "tokens_seen": 660371456 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004039919759277834, + "loss": 2.798, + "theoretical_loss": 3.8019651233860663, + "tokens_seen": 660436992 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004039819458375125, + "loss": 2.9349, + "theoretical_loss": 3.8019264198268736, + "tokens_seen": 660502528 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040397191574724175, + "loss": 2.7946, + "theoretical_loss": 3.801887721182853, + "tokens_seen": 660568064 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004039618856569709, + "loss": 2.9562, + "theoretical_loss": 3.8018490274528918, + "tokens_seen": 660633600 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004039518555667001, + "loss": 2.9122, + "theoretical_loss": 3.801810338635881, + "tokens_seen": 660699136 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004039418254764293, + "loss": 2.9185, + "theoretical_loss": 3.8017716547307074, + "tokens_seen": 660764672 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040393179538615847, + "loss": 2.7965, + "theoretical_loss": 3.801732975736262, + "tokens_seen": 660830208 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040392176529588765, + "loss": 3.0356, + "theoretical_loss": 3.801694301651433, + "tokens_seen": 660895744 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040391173520561683, + "loss": 2.8722, + "theoretical_loss": 3.8016556324751116, + "tokens_seen": 660961280 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040390170511534607, + "loss": 2.9781, + "theoretical_loss": 3.801616968206188, + "tokens_seen": 661026816 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040389167502507525, + "loss": 2.853, + "theoretical_loss": 3.8015783088435526, + "tokens_seen": 661092352 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040388164493480443, + "loss": 3.0213, + "theoretical_loss": 3.8015396543860964, + "tokens_seen": 661157888 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004038716148445336, + "loss": 2.7481, + "theoretical_loss": 3.8015010048327116, + "tokens_seen": 661223424 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040386158475426285, + "loss": 2.7774, + "theoretical_loss": 3.8014623601822892, + "tokens_seen": 661288960 + }, + { + "epoch": 2.02, + "learning_rate": 0.000403851554663992, + "loss": 2.7178, + "theoretical_loss": 3.8014237204337222, + "tokens_seen": 661354496 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004038415245737212, + "loss": 2.7672, + "theoretical_loss": 3.8013850855859026, + "tokens_seen": 661420032 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040383149448345034, + "loss": 2.8802, + "theoretical_loss": 3.801346455637724, + "tokens_seen": 661485568 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040382146439317957, + "loss": 2.8351, + "theoretical_loss": 3.801307830588079, + "tokens_seen": 661551104 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040381143430290875, + "loss": 2.8868, + "theoretical_loss": 3.801269210435862, + "tokens_seen": 661616640 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040380140421263794, + "loss": 2.7866, + "theoretical_loss": 3.801230595179966, + "tokens_seen": 661682176 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004037913741223671, + "loss": 2.7692, + "theoretical_loss": 3.801191984819286, + "tokens_seen": 661747712 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004037813440320963, + "loss": 2.7309, + "theoretical_loss": 3.8011533793527166, + "tokens_seen": 661813248 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004037713139418255, + "loss": 2.7817, + "theoretical_loss": 3.8011147787791533, + "tokens_seen": 661878784 + }, + { + "epoch": 2.02, + "objective/train/docs_used": 800678, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.2656052112579346, + "objective/train/theoretical_loss": 3.8010954803269037, + "objective/train/tokens_used": 682371552, + "theoretical_loss": 3.8010954803269037, + "tokens_seen": 661911552 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004037612838515547, + "loss": 2.8415, + "theoretical_loss": 3.801076183097491, + "tokens_seen": 661944320 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040375125376128384, + "loss": 2.8146, + "theoretical_loss": 3.801037592306626, + "tokens_seen": 662009856 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004037412236710131, + "loss": 2.8478, + "theoretical_loss": 3.800999006405454, + "tokens_seen": 662075392 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004037311935807422, + "loss": 2.8708, + "theoretical_loss": 3.8009604253928715, + "tokens_seen": 662140928 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040372116349047144, + "loss": 2.9692, + "theoretical_loss": 3.800921849267776, + "tokens_seen": 662206464 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004037111334002006, + "loss": 3.0543, + "theoretical_loss": 3.8008832780290636, + "tokens_seen": 662272000 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004037011033099298, + "loss": 2.8201, + "theoretical_loss": 3.8008447116756336, + "tokens_seen": 662337536 + }, + { + "epoch": 2.02, + "learning_rate": 0.000403691073219659, + "loss": 2.8622, + "theoretical_loss": 3.800806150206382, + "tokens_seen": 662403072 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004036810431293882, + "loss": 2.8962, + "theoretical_loss": 3.800767593620209, + "tokens_seen": 662468608 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040367101303911734, + "loss": 3.0683, + "theoretical_loss": 3.800729041916012, + "tokens_seen": 662534144 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004036609829488466, + "loss": 3.0585, + "theoretical_loss": 3.8006904950926907, + "tokens_seen": 662599680 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004036509528585757, + "loss": 3.0067, + "theoretical_loss": 3.800651953149144, + "tokens_seen": 662665216 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040364092276830494, + "loss": 3.0874, + "theoretical_loss": 3.800613416084272, + "tokens_seen": 662730752 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004036308926780341, + "loss": 2.905, + "theoretical_loss": 3.8005748838969744, + "tokens_seen": 662796288 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004036208625877633, + "loss": 2.9418, + "theoretical_loss": 3.8005363565861527, + "tokens_seen": 662861824 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004036108324974925, + "loss": 2.7549, + "theoretical_loss": 3.800497834150706, + "tokens_seen": 662927360 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040360080240722167, + "loss": 2.8905, + "theoretical_loss": 3.800459316589537, + "tokens_seen": 662992896 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040359077231695085, + "loss": 2.752, + "theoretical_loss": 3.800420803901546, + "tokens_seen": 663058432 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004035807422266801, + "loss": 2.7383, + "theoretical_loss": 3.8003822960856364, + "tokens_seen": 663123968 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004035707121364092, + "loss": 2.9399, + "theoretical_loss": 3.8003437931407094, + "tokens_seen": 663189504 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040356068204613844, + "loss": 2.9639, + "theoretical_loss": 3.8003052950656673, + "tokens_seen": 663255040 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040355065195586757, + "loss": 2.9999, + "theoretical_loss": 3.800266801859414, + "tokens_seen": 663320576 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004035406218655968, + "loss": 2.8012, + "theoretical_loss": 3.8002283135208517, + "tokens_seen": 663386112 + }, + { + "epoch": 2.02, + "learning_rate": 0.000403530591775326, + "loss": 2.7796, + "theoretical_loss": 3.800189830048885, + "tokens_seen": 663451648 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040352056168505517, + "loss": 2.8371, + "theoretical_loss": 3.800151351442418, + "tokens_seen": 663517184 + }, + { + "epoch": 2.02, + "objective/train/docs_used": 801382, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9103448390960693, + "objective/train/theoretical_loss": 3.800132113963404, + "objective/train/tokens_used": 684009952, + "theoretical_loss": 3.800132113963404, + "tokens_seen": 663549952 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040351053159478435, + "loss": 2.8541, + "theoretical_loss": 3.8001128777003546, + "tokens_seen": 663582720 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004035005015045136, + "loss": 2.8874, + "theoretical_loss": 3.800074408821599, + "tokens_seen": 663648256 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004034904714142427, + "loss": 2.8394, + "theoretical_loss": 3.8000359448050576, + "tokens_seen": 663713792 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040348044132397195, + "loss": 2.8841, + "theoretical_loss": 3.7999974856496346, + "tokens_seen": 663779328 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004034704112337011, + "loss": 2.6108, + "theoretical_loss": 3.799959031354237, + "tokens_seen": 663844864 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004034603811434303, + "loss": 2.8604, + "theoretical_loss": 3.7999205819177693, + "tokens_seen": 663910400 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004034503510531595, + "loss": 2.6844, + "theoretical_loss": 3.79988213733914, + "tokens_seen": 663975936 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040344032096288867, + "loss": 2.7968, + "theoretical_loss": 3.7998436976172543, + "tokens_seen": 664041472 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040343029087261785, + "loss": 2.923, + "theoretical_loss": 3.79980526275102, + "tokens_seen": 664107008 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040342026078234703, + "loss": 2.7212, + "theoretical_loss": 3.799766832739345, + "tokens_seen": 664172544 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004034102306920762, + "loss": 2.8013, + "theoretical_loss": 3.799728407581137, + "tokens_seen": 664238080 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040340020060180545, + "loss": 2.8651, + "theoretical_loss": 3.799689987275304, + "tokens_seen": 664303616 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004033901705115346, + "loss": 3.1115, + "theoretical_loss": 3.799651571820755, + "tokens_seen": 664369152 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004033801404212638, + "loss": 2.7985, + "theoretical_loss": 3.7996131612163984, + "tokens_seen": 664434688 + }, + { + "epoch": 2.02, + "learning_rate": 0.000403370110330993, + "loss": 2.779, + "theoretical_loss": 3.799574755461144, + "tokens_seen": 664500224 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004033600802407222, + "loss": 3.0251, + "theoretical_loss": 3.7995363545539016, + "tokens_seen": 664565760 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040335005015045136, + "loss": 2.9, + "theoretical_loss": 3.799497958493581, + "tokens_seen": 664631296 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040334002006018054, + "loss": 2.8359, + "theoretical_loss": 3.7994595672790927, + "tokens_seen": 664696832 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004033299899699097, + "loss": 2.8659, + "theoretical_loss": 3.7994211809093468, + "tokens_seen": 664762368 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040331995987963895, + "loss": 3.0194, + "theoretical_loss": 3.799382799383255, + "tokens_seen": 664827904 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004033099297893681, + "loss": 2.9732, + "theoretical_loss": 3.7993444226997295, + "tokens_seen": 664893440 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004032998996990973, + "loss": 2.7585, + "theoretical_loss": 3.7993060508576804, + "tokens_seen": 664958976 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040328986960882644, + "loss": 2.8927, + "theoretical_loss": 3.799267683856021, + "tokens_seen": 665024512 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004032798395185557, + "loss": 2.9975, + "theoretical_loss": 3.799229321693664, + "tokens_seen": 665090048 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040326980942828486, + "loss": 2.7975, + "theoretical_loss": 3.7991909643695214, + "tokens_seen": 665155584 + }, + { + "epoch": 2.02, + "objective/train/docs_used": 802714, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.3324520587921143, + "objective/train/theoretical_loss": 3.7991717875214412, + "objective/train/tokens_used": 685648352, + "theoretical_loss": 3.7991717875214412, + "tokens_seen": 665188352 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040325977933801404, + "loss": 2.7329, + "theoretical_loss": 3.7991526118825067, + "tokens_seen": 665221120 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004032497492477432, + "loss": 2.9358, + "theoretical_loss": 3.799114264231534, + "tokens_seen": 665286656 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004032397191574724, + "loss": 2.9407, + "theoretical_loss": 3.7990759214155156, + "tokens_seen": 665352192 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004032296890672016, + "loss": 2.8405, + "theoretical_loss": 3.799037583433368, + "tokens_seen": 665417728 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004032196589769308, + "loss": 3.0254, + "theoretical_loss": 3.7989992502840044, + "tokens_seen": 665483264 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040320962888665995, + "loss": 2.6699, + "theoretical_loss": 3.79896092196634, + "tokens_seen": 665548800 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004031995987963892, + "loss": 2.8213, + "theoretical_loss": 3.7989225984792903, + "tokens_seen": 665614336 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040318956870611836, + "loss": 2.8812, + "theoretical_loss": 3.7988842798217703, + "tokens_seen": 665679872 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040317953861584754, + "loss": 2.8593, + "theoretical_loss": 3.7988459659926965, + "tokens_seen": 665745408 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004031695085255767, + "loss": 2.8102, + "theoretical_loss": 3.798807656990986, + "tokens_seen": 665810944 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004031594784353059, + "loss": 3.0188, + "theoretical_loss": 3.798769352815554, + "tokens_seen": 665876480 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040314944834503514, + "loss": 2.7791, + "theoretical_loss": 3.7987310534653194, + "tokens_seen": 665942016 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004031394182547643, + "loss": 2.8322, + "theoretical_loss": 3.7986927589391977, + "tokens_seen": 666007552 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004031293881644935, + "loss": 2.9257, + "theoretical_loss": 3.798654469236107, + "tokens_seen": 666073088 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004031193580742227, + "loss": 2.8127, + "theoretical_loss": 3.798616184354967, + "tokens_seen": 666138624 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040310932798395187, + "loss": 2.847, + "theoretical_loss": 3.798577904294694, + "tokens_seen": 666204160 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040309929789368105, + "loss": 3.0944, + "theoretical_loss": 3.7985396290542086, + "tokens_seen": 666269696 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004030892678034103, + "loss": 2.6814, + "theoretical_loss": 3.798501358632429, + "tokens_seen": 666335232 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004030792377131394, + "loss": 2.9891, + "theoretical_loss": 3.798463093028275, + "tokens_seen": 666400768 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040306920762286864, + "loss": 2.7525, + "theoretical_loss": 3.798424832240666, + "tokens_seen": 666466304 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040305917753259777, + "loss": 3.0883, + "theoretical_loss": 3.7983865762685225, + "tokens_seen": 666531840 + }, + { + "epoch": 2.02, + "learning_rate": 0.000403049147442327, + "loss": 2.946, + "theoretical_loss": 3.7983483251107657, + "tokens_seen": 666597376 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004030391173520562, + "loss": 2.8419, + "theoretical_loss": 3.798310078766315, + "tokens_seen": 666662912 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040302908726178537, + "loss": 2.8564, + "theoretical_loss": 3.7982718372340933, + "tokens_seen": 666728448 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040301905717151455, + "loss": 2.7861, + "theoretical_loss": 3.7982336005130213, + "tokens_seen": 666793984 + }, + { + "epoch": 2.02, + "objective/train/docs_used": 803703, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.0033750534057617, + "objective/train/theoretical_loss": 3.798214483956329, + "objective/train/tokens_used": 687286752, + "theoretical_loss": 3.798214483956329, + "tokens_seen": 666826752 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004030090270812438, + "loss": 2.9464, + "theoretical_loss": 3.7981953686020207, + "tokens_seen": 666859520 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004029989969909729, + "loss": 3.1007, + "theoretical_loss": 3.798157141500014, + "tokens_seen": 666925056 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040298896690070215, + "loss": 2.8264, + "theoretical_loss": 3.7981189192059244, + "tokens_seen": 666990592 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004029789368104313, + "loss": 2.8476, + "theoretical_loss": 3.7980807017186744, + "tokens_seen": 667056128 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004029689067201605, + "loss": 3.1105, + "theoretical_loss": 3.7980424890371873, + "tokens_seen": 667121664 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004029588766298897, + "loss": 2.6239, + "theoretical_loss": 3.7980042811603867, + "tokens_seen": 667187200 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040294884653961887, + "loss": 2.8954, + "theoretical_loss": 3.797966078087197, + "tokens_seen": 667252736 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040293881644934805, + "loss": 3.0282, + "theoretical_loss": 3.797927879816542, + "tokens_seen": 667318272 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040292878635907723, + "loss": 3.0588, + "theoretical_loss": 3.7978896863473466, + "tokens_seen": 667383808 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004029187562688064, + "loss": 2.8539, + "theoretical_loss": 3.7978514976785362, + "tokens_seen": 667449344 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040290872617853565, + "loss": 2.8734, + "theoretical_loss": 3.797813313809036, + "tokens_seen": 667514880 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004028986960882648, + "loss": 2.8289, + "theoretical_loss": 3.7977751347377717, + "tokens_seen": 667580416 + }, + { + "epoch": 2.02, + "learning_rate": 0.000402888665997994, + "loss": 2.7012, + "theoretical_loss": 3.7977369604636695, + "tokens_seen": 667645952 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004028786359077232, + "loss": 2.9201, + "theoretical_loss": 3.7976987909856557, + "tokens_seen": 667711488 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004028686058174524, + "loss": 2.8353, + "theoretical_loss": 3.7976606263026573, + "tokens_seen": 667777024 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040285857572718156, + "loss": 3.0119, + "theoretical_loss": 3.7976224664136007, + "tokens_seen": 667842560 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040284854563691074, + "loss": 2.9295, + "theoretical_loss": 3.7975843113174146, + "tokens_seen": 667908096 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004028385155466399, + "loss": 2.653, + "theoretical_loss": 3.797546161013026, + "tokens_seen": 667973632 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040282848545636915, + "loss": 2.9208, + "theoretical_loss": 3.7975080154993632, + "tokens_seen": 668039168 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004028184553660983, + "loss": 2.9167, + "theoretical_loss": 3.7974698747753552, + "tokens_seen": 668104704 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004028084252758275, + "loss": 2.961, + "theoretical_loss": 3.7974317388399297, + "tokens_seen": 668170240 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040279839518555664, + "loss": 2.9478, + "theoretical_loss": 3.797393607692017, + "tokens_seen": 668235776 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004027883650952859, + "loss": 2.8548, + "theoretical_loss": 3.7973554813305466, + "tokens_seen": 668301312 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040277833500501506, + "loss": 2.9796, + "theoretical_loss": 3.797317359754448, + "tokens_seen": 668366848 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040276830491474424, + "loss": 2.7677, + "theoretical_loss": 3.797279242962651, + "tokens_seen": 668432384 + }, + { + "epoch": 2.02, + "objective/train/docs_used": 804287, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6115474700927734, + "objective/train/theoretical_loss": 3.7972601863605315, + "objective/train/tokens_used": 688925152, + "theoretical_loss": 3.7972601863605315, + "tokens_seen": 668465152 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004027582748244734, + "loss": 2.9188, + "theoretical_loss": 3.797241130954087, + "tokens_seen": 668497920 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004027482447342026, + "loss": 2.7482, + "theoretical_loss": 3.7972030237276866, + "tokens_seen": 668563456 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004027382146439318, + "loss": 2.6248, + "theoretical_loss": 3.7971649212823806, + "tokens_seen": 668628992 + }, + { + "epoch": 2.02, + "learning_rate": 0.000402728184553661, + "loss": 2.8523, + "theoretical_loss": 3.7971268236171016, + "tokens_seen": 668694528 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040271815446339015, + "loss": 2.7225, + "theoretical_loss": 3.7970887307307812, + "tokens_seen": 668760064 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004027081243731194, + "loss": 2.7711, + "theoretical_loss": 3.7970506426223514, + "tokens_seen": 668825600 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040269809428284856, + "loss": 3.0768, + "theoretical_loss": 3.797012559290745, + "tokens_seen": 668891136 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040268806419257774, + "loss": 3.112, + "theoretical_loss": 3.796974480734894, + "tokens_seen": 668956672 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004026780341023069, + "loss": 2.8936, + "theoretical_loss": 3.7969364069537344, + "tokens_seen": 669022208 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004026680040120361, + "loss": 2.8176, + "theoretical_loss": 3.796898337946197, + "tokens_seen": 669087744 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004026579739217653, + "loss": 2.8419, + "theoretical_loss": 3.796860273711217, + "tokens_seen": 669153280 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004026479438314945, + "loss": 2.9462, + "theoretical_loss": 3.7968222142477295, + "tokens_seen": 669218816 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040263791374122365, + "loss": 2.8592, + "theoretical_loss": 3.796784159554668, + "tokens_seen": 669284352 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004026278836509529, + "loss": 2.7023, + "theoretical_loss": 3.796746109630967, + "tokens_seen": 669349888 + }, + { + "epoch": 2.02, + "learning_rate": 0.000402617853560682, + "loss": 2.7926, + "theoretical_loss": 3.796708064475564, + "tokens_seen": 669415424 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040260782347041125, + "loss": 2.7279, + "theoretical_loss": 3.796670024087393, + "tokens_seen": 669480960 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040259779338014043, + "loss": 3.0721, + "theoretical_loss": 3.796631988465391, + "tokens_seen": 669546496 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004025877632898696, + "loss": 2.7945, + "theoretical_loss": 3.796593957608494, + "tokens_seen": 669612032 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004025777331995988, + "loss": 2.9095, + "theoretical_loss": 3.7965559315156385, + "tokens_seen": 669677568 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040256770310932797, + "loss": 2.977, + "theoretical_loss": 3.796517910185762, + "tokens_seen": 669743104 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040255767301905715, + "loss": 2.8426, + "theoretical_loss": 3.7964798936178017, + "tokens_seen": 669808640 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004025476429287864, + "loss": 2.8808, + "theoretical_loss": 3.7964418818106953, + "tokens_seen": 669874176 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004025376128385155, + "loss": 2.8402, + "theoretical_loss": 3.7964038747633815, + "tokens_seen": 669939712 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040252758274824475, + "loss": 2.9317, + "theoretical_loss": 3.796365872474798, + "tokens_seen": 670005248 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040251755265797393, + "loss": 2.8884, + "theoretical_loss": 3.7963278749438842, + "tokens_seen": 670070784 + }, + { + "epoch": 2.02, + "objective/train/docs_used": 805654, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.389000654220581, + "objective/train/theoretical_loss": 3.796308877962222, + "objective/train/tokens_used": 690563552, + "theoretical_loss": 3.796308877962222, + "tokens_seen": 670103552 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004025075225677031, + "loss": 3.0705, + "theoretical_loss": 3.796289882169579, + "tokens_seen": 670136320 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004024974924774323, + "loss": 2.9853, + "theoretical_loss": 3.7962518941508216, + "tokens_seen": 670201856 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004024874623871615, + "loss": 2.7051, + "theoretical_loss": 3.796213910886552, + "tokens_seen": 670267392 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040247743229689066, + "loss": 2.6793, + "theoretical_loss": 3.79617593237571, + "tokens_seen": 670332928 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004024674022066199, + "loss": 2.7152, + "theoretical_loss": 3.796137958617237, + "tokens_seen": 670398464 + }, + { + "epoch": 2.02, + "learning_rate": 0.000402457372116349, + "loss": 2.7582, + "theoretical_loss": 3.796099989610073, + "tokens_seen": 670464000 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040244734202607825, + "loss": 3.1524, + "theoretical_loss": 3.7960620253531596, + "tokens_seen": 670529536 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004024373119358074, + "loss": 3.0175, + "theoretical_loss": 3.7960240658454385, + "tokens_seen": 670595072 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004024272818455366, + "loss": 2.637, + "theoretical_loss": 3.795986111085851, + "tokens_seen": 670660608 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004024172517552658, + "loss": 2.8622, + "theoretical_loss": 3.7959481610733397, + "tokens_seen": 670726144 + }, + { + "epoch": 2.02, + "learning_rate": 0.000402407221664995, + "loss": 2.5987, + "theoretical_loss": 3.7959102158068463, + "tokens_seen": 670791680 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004023971915747242, + "loss": 3.1051, + "theoretical_loss": 3.795872275285315, + "tokens_seen": 670857216 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004023871614844534, + "loss": 2.5387, + "theoretical_loss": 3.7958343395076883, + "tokens_seen": 670922752 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004023771313941826, + "loss": 2.6301, + "theoretical_loss": 3.7957964084729094, + "tokens_seen": 670988288 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040236710130391176, + "loss": 3.0434, + "theoretical_loss": 3.795758482179923, + "tokens_seen": 671053824 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040235707121364094, + "loss": 2.9399, + "theoretical_loss": 3.7957205606276725, + "tokens_seen": 671119360 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004023470411233701, + "loss": 2.9368, + "theoretical_loss": 3.795682643815103, + "tokens_seen": 671184896 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040233701103309935, + "loss": 3.0092, + "theoretical_loss": 3.7956447317411595, + "tokens_seen": 671250432 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004023269809428285, + "loss": 2.6606, + "theoretical_loss": 3.7956068244047865, + "tokens_seen": 671315968 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004023169508525577, + "loss": 2.905, + "theoretical_loss": 3.79556892180493, + "tokens_seen": 671381504 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040230692076228684, + "loss": 2.768, + "theoretical_loss": 3.795531023940536, + "tokens_seen": 671447040 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004022968906720161, + "loss": 3.1211, + "theoretical_loss": 3.7954931308105513, + "tokens_seen": 671512576 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040228686058174526, + "loss": 3.0504, + "theoretical_loss": 3.7954552424139214, + "tokens_seen": 671578112 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040227683049147444, + "loss": 2.8887, + "theoretical_loss": 3.795417358749594, + "tokens_seen": 671643648 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004022668004012036, + "loss": 2.6697, + "theoretical_loss": 3.795379479816516, + "tokens_seen": 671709184 + }, + { + "epoch": 2.02, + "objective/train/docs_used": 806421, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.0061473846435547, + "objective/train/theoretical_loss": 3.7953605421238663, + "objective/train/tokens_used": 692201952, + "theoretical_loss": 3.7953605421238663, + "tokens_seen": 671741952 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004022567703109328, + "loss": 2.9267, + "theoretical_loss": 3.7953416056136344, + "tokens_seen": 671774720 + }, + { + "epoch": 2.02, + "learning_rate": 0.000402246740220662, + "loss": 2.9145, + "theoretical_loss": 3.795303736139899, + "tokens_seen": 671840256 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004022367101303912, + "loss": 2.9675, + "theoretical_loss": 3.795265871394256, + "tokens_seen": 671905792 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040222668004012035, + "loss": 3.1682, + "theoretical_loss": 3.795228011375655, + "tokens_seen": 671971328 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004022166499498496, + "loss": 2.7738, + "theoretical_loss": 3.795190156083045, + "tokens_seen": 672036864 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040220661985957876, + "loss": 2.8969, + "theoretical_loss": 3.7951523055153755, + "tokens_seen": 672102400 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040219658976930794, + "loss": 2.9553, + "theoretical_loss": 3.795114459671595, + "tokens_seen": 672167936 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004021865596790371, + "loss": 2.8685, + "theoretical_loss": 3.795076618550654, + "tokens_seen": 672233472 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004021765295887663, + "loss": 2.8885, + "theoretical_loss": 3.795038782151504, + "tokens_seen": 672299008 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004021664994984955, + "loss": 2.9009, + "theoretical_loss": 3.795000950473094, + "tokens_seen": 672364544 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004021564694082247, + "loss": 3.0639, + "theoretical_loss": 3.7949631235143753, + "tokens_seen": 672430080 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040214643931795385, + "loss": 2.7665, + "theoretical_loss": 3.7949253012743, + "tokens_seen": 672495616 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004021364092276831, + "loss": 2.8488, + "theoretical_loss": 3.7948874837518183, + "tokens_seen": 672561152 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004021263791374122, + "loss": 2.8415, + "theoretical_loss": 3.794849670945884, + "tokens_seen": 672626688 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040211634904714145, + "loss": 2.9815, + "theoretical_loss": 3.7948118628554477, + "tokens_seen": 672692224 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040210631895687063, + "loss": 2.9568, + "theoretical_loss": 3.794774059479463, + "tokens_seen": 672757760 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004020962888665998, + "loss": 2.7326, + "theoretical_loss": 3.7947362608168826, + "tokens_seen": 672823296 + }, + { + "epoch": 2.02, + "learning_rate": 0.000402086258776329, + "loss": 3.0833, + "theoretical_loss": 3.79469846686666, + "tokens_seen": 672888832 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040207622868605817, + "loss": 3.0286, + "theoretical_loss": 3.794660677627748, + "tokens_seen": 672954368 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040206619859578735, + "loss": 2.8386, + "theoretical_loss": 3.7946228930991017, + "tokens_seen": 673019904 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004020561685055166, + "loss": 2.9388, + "theoretical_loss": 3.794585113279675, + "tokens_seen": 673085440 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004020461384152457, + "loss": 3.028, + "theoretical_loss": 3.7945473381684227, + "tokens_seen": 673150976 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040203610832497495, + "loss": 2.7476, + "theoretical_loss": 3.7945095677642993, + "tokens_seen": 673216512 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040202607823470413, + "loss": 2.9742, + "theoretical_loss": 3.7944718020662602, + "tokens_seen": 673282048 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004020160481444333, + "loss": 3.0702, + "theoretical_loss": 3.794434041073261, + "tokens_seen": 673347584 + }, + { + "epoch": 2.02, + "objective/train/docs_used": 807207, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9605650901794434, + "objective/train/theoretical_loss": 3.794415162340826, + "objective/train/tokens_used": 693840352, + "theoretical_loss": 3.794415162340826, + "tokens_seen": 673380352 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004020060180541625, + "loss": 2.7631, + "theoretical_loss": 3.7943962847842587, + "tokens_seen": 673413120 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004019959879638917, + "loss": 3.1873, + "theoretical_loss": 3.794358533198208, + "tokens_seen": 673478656 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040198595787362086, + "loss": 3.0331, + "theoretical_loss": 3.7943207863140667, + "tokens_seen": 673544192 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004019759277833501, + "loss": 2.8757, + "theoretical_loss": 3.7942830441307915, + "tokens_seen": 673609728 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004019658976930792, + "loss": 2.8293, + "theoretical_loss": 3.794245306647339, + "tokens_seen": 673675264 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040195586760280845, + "loss": 2.7213, + "theoretical_loss": 3.7942075738626677, + "tokens_seen": 673740800 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004019458375125376, + "loss": 2.8015, + "theoretical_loss": 3.794169845775736, + "tokens_seen": 673806336 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004019358074222668, + "loss": 2.7856, + "theoretical_loss": 3.7941321223855002, + "tokens_seen": 673871872 + }, + { + "epoch": 2.02, + "learning_rate": 0.000401925777331996, + "loss": 2.8947, + "theoretical_loss": 3.794094403690921, + "tokens_seen": 673937408 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004019157472417252, + "loss": 2.7252, + "theoretical_loss": 3.794056689690956, + "tokens_seen": 674002944 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040190571715145436, + "loss": 2.9241, + "theoretical_loss": 3.794018980384566, + "tokens_seen": 674068480 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004018956870611836, + "loss": 2.9409, + "theoretical_loss": 3.7939812757707094, + "tokens_seen": 674134016 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004018856569709127, + "loss": 2.9654, + "theoretical_loss": 3.7939435758483464, + "tokens_seen": 674199552 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040187562688064196, + "loss": 2.6226, + "theoretical_loss": 3.793905880616437, + "tokens_seen": 674265088 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004018655967903711, + "loss": 2.7045, + "theoretical_loss": 3.793868190073943, + "tokens_seen": 674330624 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004018555667001003, + "loss": 2.8452, + "theoretical_loss": 3.793830504219824, + "tokens_seen": 674396160 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004018455366098295, + "loss": 2.8149, + "theoretical_loss": 3.793792823053042, + "tokens_seen": 674461696 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004018355065195587, + "loss": 2.997, + "theoretical_loss": 3.7937551465725585, + "tokens_seen": 674527232 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040182547642928786, + "loss": 2.7987, + "theoretical_loss": 3.7937174747773357, + "tokens_seen": 674592768 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040181544633901704, + "loss": 2.7863, + "theoretical_loss": 3.7936798076663347, + "tokens_seen": 674658304 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004018054162487462, + "loss": 2.7427, + "theoretical_loss": 3.79364214523852, + "tokens_seen": 674723840 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040179538615847546, + "loss": 2.8898, + "theoretical_loss": 3.7936044874928534, + "tokens_seen": 674789376 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004017853560682046, + "loss": 2.7465, + "theoretical_loss": 3.7935668344282982, + "tokens_seen": 674854912 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004017753259779338, + "loss": 2.7848, + "theoretical_loss": 3.793529186043818, + "tokens_seen": 674920448 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040176529588766295, + "loss": 2.7028, + "theoretical_loss": 3.7934915423383764, + "tokens_seen": 674985984 + }, + { + "epoch": 2.02, + "objective/train/docs_used": 807950, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.584026336669922, + "objective/train/theoretical_loss": 3.7934727222399722, + "objective/train/tokens_used": 695478752, + "theoretical_loss": 3.7934727222399722, + "tokens_seen": 675018752 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004017552657973922, + "loss": 2.759, + "theoretical_loss": 3.793453903310939, + "tokens_seen": 675051520 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040174523570712137, + "loss": 2.9461, + "theoretical_loss": 3.793416268960469, + "tokens_seen": 675117056 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040173520561685055, + "loss": 2.9082, + "theoretical_loss": 3.7933786392859323, + "tokens_seen": 675182592 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040172517552657973, + "loss": 2.8488, + "theoretical_loss": 3.793341014286293, + "tokens_seen": 675248128 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040171514543630896, + "loss": 3.0354, + "theoretical_loss": 3.7933033939605183, + "tokens_seen": 675313664 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004017051153460381, + "loss": 2.7168, + "theoretical_loss": 3.793265778307572, + "tokens_seen": 675379200 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004016950852557673, + "loss": 2.8791, + "theoretical_loss": 3.7932281673264225, + "tokens_seen": 675444736 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040168505516549645, + "loss": 2.5665, + "theoretical_loss": 3.7931905610160355, + "tokens_seen": 675510272 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004016750250752257, + "loss": 3.0928, + "theoretical_loss": 3.793152959375377, + "tokens_seen": 675575808 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040166499498495487, + "loss": 2.8882, + "theoretical_loss": 3.7931153624034155, + "tokens_seen": 675641344 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040165496489468405, + "loss": 2.9232, + "theoretical_loss": 3.793077770099118, + "tokens_seen": 675706880 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004016449348044133, + "loss": 3.0883, + "theoretical_loss": 3.7930401824614526, + "tokens_seen": 675772416 + }, + { + "epoch": 2.02, + "learning_rate": 0.0004016349047141424, + "loss": 2.8628, + "theoretical_loss": 3.7930025994893866, + "tokens_seen": 675837952 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040162487462387165, + "loss": 3.0159, + "theoretical_loss": 3.79296502118189, + "tokens_seen": 675903488 + }, + { + "epoch": 2.02, + "learning_rate": 0.00040161484453360083, + "loss": 2.916, + "theoretical_loss": 3.792927447537931, + "tokens_seen": 675969024 + }, + { + "epoch": 2.03, + "learning_rate": 0.00040160481444333, + "loss": 2.9463, + "theoretical_loss": 3.792889878556479, + "tokens_seen": 676034560 + }, + { + "epoch": 2.03, + "learning_rate": 0.0004015947843530592, + "loss": 2.9628, + "theoretical_loss": 3.7928523142365025, + "tokens_seen": 676100096 + }, + { + "epoch": 2.03, + "learning_rate": 0.00040158475426278837, + "loss": 3.0279, + "theoretical_loss": 3.7928147545769724, + "tokens_seen": 676165632 + }, + { + "epoch": 2.03, + "learning_rate": 0.00040157472417251755, + "loss": 2.4866, + "theoretical_loss": 3.792777199576859, + "tokens_seen": 676231168 + }, + { + "epoch": 2.03, + "learning_rate": 0.0004015646940822468, + "loss": 2.6683, + "theoretical_loss": 3.792739649235132, + "tokens_seen": 676296704 + }, + { + "epoch": 2.03, + "learning_rate": 0.0004015546639919759, + "loss": 2.8666, + "theoretical_loss": 3.792702103550763, + "tokens_seen": 676362240 + }, + { + "epoch": 2.03, + "learning_rate": 0.00040154463390170515, + "loss": 3.0335, + "theoretical_loss": 3.792664562522723, + "tokens_seen": 676427776 + }, + { + "epoch": 2.03, + "learning_rate": 0.00040153460381143433, + "loss": 3.1476, + "theoretical_loss": 3.7926270261499826, + "tokens_seen": 676493312 + }, + { + "epoch": 2.03, + "learning_rate": 0.0004015245737211635, + "loss": 2.7187, + "theoretical_loss": 3.792589494431515, + "tokens_seen": 676558848 + }, + { + "epoch": 2.03, + "learning_rate": 0.0004015145436308927, + "loss": 2.9815, + "theoretical_loss": 3.792551967366291, + "tokens_seen": 676624384 + }, + { + "epoch": 2.03, + "objective/train/docs_used": 809057, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.946103096008301, + "objective/train/theoretical_loss": 3.7925332055783247, + "objective/train/tokens_used": 697117152, + "theoretical_loss": 3.7925332055783247, + "tokens_seen": 676657152 + }, + { + "epoch": 2.03, + "learning_rate": 0.0004015045135406219, + "loss": 3.0284, + "theoretical_loss": 3.7925144449532837, + "tokens_seen": 676689920 + }, + { + "epoch": 2.03, + "learning_rate": 0.00040149448345035106, + "loss": 2.8247, + "theoretical_loss": 3.7924769271914665, + "tokens_seen": 676755456 + }, + { + "epoch": 2.03, + "learning_rate": 0.0004014844533600803, + "loss": 2.9618, + "theoretical_loss": 3.7924394140798117, + "tokens_seen": 676820992 + }, + { + "epoch": 2.03, + "learning_rate": 0.0004014744232698094, + "loss": 2.795, + "theoretical_loss": 3.792401905617293, + "tokens_seen": 676886528 + }, + { + "epoch": 2.03, + "learning_rate": 0.00040146439317953865, + "loss": 3.0326, + "theoretical_loss": 3.792364401802884, + "tokens_seen": 676952064 + }, + { + "epoch": 2.03, + "learning_rate": 0.0004014543630892678, + "loss": 3.1664, + "theoretical_loss": 3.7923269026355593, + "tokens_seen": 677017600 + }, + { + "epoch": 2.03, + "learning_rate": 0.000401444332998997, + "loss": 2.8116, + "theoretical_loss": 3.7922894081142924, + "tokens_seen": 677083136 + }, + { + "epoch": 2.03, + "learning_rate": 0.0004014343029087262, + "loss": 2.6791, + "theoretical_loss": 3.792251918238059, + "tokens_seen": 677148672 + }, + { + "epoch": 2.03, + "learning_rate": 0.0004014242728184554, + "loss": 2.8246, + "theoretical_loss": 3.7922144330058334, + "tokens_seen": 677214208 + }, + { + "epoch": 2.03, + "learning_rate": 0.00040141424272818456, + "loss": 2.7107, + "theoretical_loss": 3.7921769524165923, + "tokens_seen": 677279744 + }, + { + "epoch": 2.03, + "learning_rate": 0.0004014042126379138, + "loss": 2.9204, + "theoretical_loss": 3.7921394764693095, + "tokens_seen": 677345280 + }, + { + "epoch": 2.03, + "learning_rate": 0.0004013941825476429, + "loss": 2.6977, + "theoretical_loss": 3.7921020051629624, + "tokens_seen": 677410816 + }, + { + "epoch": 2.03, + "learning_rate": 0.00040138415245737216, + "loss": 2.658, + "theoretical_loss": 3.7920645384965272, + "tokens_seen": 677476352 + }, + { + "epoch": 2.03, + "learning_rate": 0.0004013741223671013, + "loss": 2.928, + "theoretical_loss": 3.7920270764689805, + "tokens_seen": 677541888 + }, + { + "epoch": 2.03, + "learning_rate": 0.0004013640922768305, + "loss": 2.8152, + "theoretical_loss": 3.791989619079299, + "tokens_seen": 677607424 + }, + { + "epoch": 2.03, + "learning_rate": 0.0004013540621865597, + "loss": 2.8409, + "theoretical_loss": 3.7919521663264604, + "tokens_seen": 677672960 + }, + { + "epoch": 2.03, + "learning_rate": 0.0004013440320962889, + "loss": 3.0376, + "theoretical_loss": 3.7919147182094424, + "tokens_seen": 677738496 + }, + { + "epoch": 2.03, + "learning_rate": 0.00040133400200601806, + "loss": 2.7009, + "theoretical_loss": 3.7918772747272227, + "tokens_seen": 677804032 + }, + { + "epoch": 2.03, + "learning_rate": 0.00040132397191574724, + "loss": 2.8787, + "theoretical_loss": 3.79183983587878, + "tokens_seen": 677869568 + }, + { + "epoch": 2.03, + "learning_rate": 0.0004013139418254764, + "loss": 2.9441, + "theoretical_loss": 3.7918024016630922, + "tokens_seen": 677935104 + }, + { + "epoch": 2.03, + "learning_rate": 0.00040130391173520566, + "loss": 2.6351, + "theoretical_loss": 3.7917649720791395, + "tokens_seen": 678000640 + }, + { + "epoch": 2.03, + "learning_rate": 0.0004012938816449348, + "loss": 2.7128, + "theoretical_loss": 3.7917275471259, + "tokens_seen": 678066176 + }, + { + "epoch": 2.03, + "learning_rate": 0.000401283851554664, + "loss": 2.9271, + "theoretical_loss": 3.791690126802354, + "tokens_seen": 678131712 + }, + { + "epoch": 2.03, + "learning_rate": 0.00040127382146439315, + "loss": 2.9554, + "theoretical_loss": 3.7916527111074814, + "tokens_seen": 678197248 + }, + { + "epoch": 2.03, + "learning_rate": 0.0004012637913741224, + "loss": 2.9009, + "theoretical_loss": 3.791615300040262, + "tokens_seen": 678262784 + }, + { + "epoch": 2.03, + "objective/train/docs_used": 809644, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9782016277313232, + "objective/train/theoretical_loss": 3.791596596241704, + "objective/train/tokens_used": 698755552, + "theoretical_loss": 3.791596596241704, + "tokens_seen": 678295552 + }, + { + "epoch": 2.03, + "learning_rate": 0.00040125376128385157, + "loss": 2.9855, + "theoretical_loss": 3.7915778935996767, + "tokens_seen": 678328320 + }, + { + "epoch": 2.03, + "learning_rate": 0.00040124373119358075, + "loss": 3.0623, + "theoretical_loss": 3.7915404917847066, + "tokens_seen": 678393856 + }, + { + "epoch": 2.03, + "learning_rate": 0.00040123370110330993, + "loss": 2.8576, + "theoretical_loss": 3.791503094594333, + "tokens_seen": 678459392 + }, + { + "epoch": 2.03, + "learning_rate": 0.00040122367101303916, + "loss": 2.8939, + "theoretical_loss": 3.791465702027537, + "tokens_seen": 678524928 + }, + { + "epoch": 2.03, + "learning_rate": 0.0004012136409227683, + "loss": 2.9493, + "theoretical_loss": 3.7914283140833005, + "tokens_seen": 678590464 + }, + { + "epoch": 2.03, + "learning_rate": 0.0004012036108324975, + "loss": 2.5703, + "theoretical_loss": 3.791390930760606, + "tokens_seen": 678656000 + }, + { + "epoch": 2.03, + "learning_rate": 0.00040119358074222665, + "loss": 2.9312, + "theoretical_loss": 3.791353552058436, + "tokens_seen": 678721536 + }, + { + "epoch": 2.03, + "learning_rate": 0.0004011835506519559, + "loss": 2.6408, + "theoretical_loss": 3.791316177975773, + "tokens_seen": 678787072 + }, + { + "epoch": 2.03, + "learning_rate": 0.00040117352056168507, + "loss": 2.7553, + "theoretical_loss": 3.7912788085116005, + "tokens_seen": 678852608 + }, + { + "epoch": 2.03, + "learning_rate": 0.00040116349047141425, + "loss": 2.8303, + "theoretical_loss": 3.7912414436649016, + "tokens_seen": 678918144 + }, + { + "epoch": 2.03, + "learning_rate": 0.00040115346038114343, + "loss": 3.0389, + "theoretical_loss": 3.7912040834346605, + "tokens_seen": 678983680 + }, + { + "epoch": 2.03, + "learning_rate": 0.0004011434302908726, + "loss": 2.7881, + "theoretical_loss": 3.7911667278198617, + "tokens_seen": 679049216 + }, + { + "epoch": 2.03, + "learning_rate": 0.0004011334002006018, + "loss": 2.6978, + "theoretical_loss": 3.791129376819489, + "tokens_seen": 679114752 + }, + { + "epoch": 2.03, + "learning_rate": 0.00040112337011033103, + "loss": 2.9489, + "theoretical_loss": 3.7910920304325275, + "tokens_seen": 679180288 + }, + { + "epoch": 2.03, + "learning_rate": 0.00040111334002006016, + "loss": 2.7048, + "theoretical_loss": 3.7910546886579617, + "tokens_seen": 679245824 + }, + { + "epoch": 2.03, + "learning_rate": 0.0004011033099297894, + "loss": 2.9393, + "theoretical_loss": 3.791017351494778, + "tokens_seen": 679311360 + }, + { + "epoch": 2.03, + "learning_rate": 0.0004010932798395185, + "loss": 2.9715, + "theoretical_loss": 3.790980018941961, + "tokens_seen": 679376896 + }, + { + "epoch": 2.03, + "learning_rate": 0.00040108324974924775, + "loss": 2.8581, + "theoretical_loss": 3.790942690998498, + "tokens_seen": 679442432 + }, + { + "epoch": 2.03, + "learning_rate": 0.00040107321965897693, + "loss": 2.5724, + "theoretical_loss": 3.790905367663374, + "tokens_seen": 679507968 + }, + { + "epoch": 2.03, + "learning_rate": 0.0004010631895687061, + "loss": 2.8063, + "theoretical_loss": 3.790868048935577, + "tokens_seen": 679573504 + }, + { + "epoch": 2.03, + "learning_rate": 0.0004010531594784353, + "loss": 3.0307, + "theoretical_loss": 3.7908307348140933, + "tokens_seen": 679639040 + }, + { + "epoch": 2.03, + "learning_rate": 0.00040104312938816453, + "loss": 2.8895, + "theoretical_loss": 3.7907934252979105, + "tokens_seen": 679704576 + }, + { + "epoch": 2.03, + "learning_rate": 0.00040103309929789366, + "loss": 2.9646, + "theoretical_loss": 3.790756120386016, + "tokens_seen": 679770112 + }, + { + "epoch": 2.03, + "learning_rate": 0.0004010230692076229, + "loss": 2.8548, + "theoretical_loss": 3.7907188200773985, + "tokens_seen": 679835648 + }, + { + "epoch": 2.03, + "learning_rate": 0.000401013039117352, + "loss": 2.9081, + "theoretical_loss": 3.7906815243710454, + "tokens_seen": 679901184 + }, + { + "epoch": 2.03, + "objective/train/docs_used": 811111, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.683133363723755, + "objective/train/theoretical_loss": 3.7906628782434018, + "objective/train/tokens_used": 700393952, + "theoretical_loss": 3.7906628782434018, + "tokens_seen": 679933952 + }, + { + "epoch": 2.03, + "learning_rate": 0.00040100300902708126, + "loss": 2.9855, + "theoretical_loss": 3.7906442332659456, + "tokens_seen": 679966720 + }, + { + "epoch": 2.03, + "learning_rate": 0.00040099297893681044, + "loss": 2.7769, + "theoretical_loss": 3.790606946761088, + "tokens_seen": 680032256 + }, + { + "epoch": 2.03, + "learning_rate": 0.0004009829488465396, + "loss": 2.7631, + "theoretical_loss": 3.790569664855462, + "tokens_seen": 680097792 + }, + { + "epoch": 2.03, + "learning_rate": 0.0004009729187562688, + "loss": 3.0075, + "theoretical_loss": 3.7905323875480574, + "tokens_seen": 680163328 + }, + { + "epoch": 2.03, + "learning_rate": 0.000400962888665998, + "loss": 2.6681, + "theoretical_loss": 3.7904951148378636, + "tokens_seen": 680228864 + }, + { + "epoch": 2.03, + "learning_rate": 0.00040095285857572716, + "loss": 3.1031, + "theoretical_loss": 3.7904578467238714, + "tokens_seen": 680294400 + }, + { + "epoch": 2.03, + "learning_rate": 0.0004009428284854564, + "loss": 2.8202, + "theoretical_loss": 3.790420583205071, + "tokens_seen": 680359936 + }, + { + "epoch": 2.03, + "learning_rate": 0.0004009327983951855, + "loss": 2.8716, + "theoretical_loss": 3.7903833242804534, + "tokens_seen": 680425472 + }, + { + "epoch": 2.03, + "learning_rate": 0.00040092276830491476, + "loss": 2.6202, + "theoretical_loss": 3.790346069949009, + "tokens_seen": 680491008 + }, + { + "epoch": 2.03, + "learning_rate": 0.0004009127382146439, + "loss": 2.8913, + "theoretical_loss": 3.790308820209731, + "tokens_seen": 680556544 + }, + { + "epoch": 2.03, + "learning_rate": 0.0004009027081243731, + "loss": 3.0475, + "theoretical_loss": 3.7902715750616087, + "tokens_seen": 680622080 + }, + { + "epoch": 2.03, + "learning_rate": 0.00040089267803410236, + "loss": 2.9008, + "theoretical_loss": 3.7902343345036367, + "tokens_seen": 680687616 + }, + { + "epoch": 2.03, + "learning_rate": 0.0004008826479438315, + "loss": 2.9714, + "theoretical_loss": 3.790197098534806, + "tokens_seen": 680753152 + }, + { + "epoch": 2.03, + "learning_rate": 0.0004008726178535607, + "loss": 2.8291, + "theoretical_loss": 3.7901598671541104, + "tokens_seen": 680818688 + }, + { + "epoch": 2.03, + "learning_rate": 0.0004008625877632899, + "loss": 3.0035, + "theoretical_loss": 3.7901226403605417, + "tokens_seen": 680884224 + }, + { + "epoch": 2.03, + "learning_rate": 0.0004008525576730191, + "loss": 2.8466, + "theoretical_loss": 3.7900854181530943, + "tokens_seen": 680949760 + }, + { + "epoch": 2.03, + "learning_rate": 0.00040084252758274826, + "loss": 3.0043, + "theoretical_loss": 3.7900482005307614, + "tokens_seen": 681015296 + }, + { + "epoch": 2.03, + "learning_rate": 0.00040083249749247744, + "loss": 2.7097, + "theoretical_loss": 3.7900109874925376, + "tokens_seen": 681080832 + }, + { + "epoch": 2.03, + "learning_rate": 0.0004008224674022066, + "loss": 2.713, + "theoretical_loss": 3.7899737790374166, + "tokens_seen": 681146368 + }, + { + "epoch": 2.03, + "learning_rate": 0.00040081243731193586, + "loss": 3.0035, + "theoretical_loss": 3.7899365751643934, + "tokens_seen": 681211904 + }, + { + "epoch": 2.03, + "learning_rate": 0.000400802407221665, + "loss": 2.9085, + "theoretical_loss": 3.7898993758724635, + "tokens_seen": 681277440 + }, + { + "epoch": 2.03, + "learning_rate": 0.0004007923771313942, + "loss": 2.908, + "theoretical_loss": 3.789862181160621, + "tokens_seen": 681342976 + }, + { + "epoch": 2.03, + "learning_rate": 0.00040078234704112335, + "loss": 2.8204, + "theoretical_loss": 3.7898249910278627, + "tokens_seen": 681408512 + }, + { + "epoch": 2.03, + "learning_rate": 0.0004007723169508526, + "loss": 2.8935, + "theoretical_loss": 3.7897878054731837, + "tokens_seen": 681474048 + }, + { + "epoch": 2.03, + "learning_rate": 0.00040076228686058177, + "loss": 3.0723, + "theoretical_loss": 3.7897506244955808, + "tokens_seen": 681539584 + }, + { + "epoch": 2.03, + "objective/train/docs_used": 811509, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8229663372039795, + "objective/train/theoretical_loss": 3.789732035722869, + "objective/train/tokens_used": 702032352, + "theoretical_loss": 3.789732035722869, + "tokens_seen": 681572352 + }, + { + "epoch": 2.03, + "learning_rate": 0.00040075225677031095, + "loss": 2.7856, + "theoretical_loss": 3.7897134480940506, + "tokens_seen": 681605120 + }, + { + "epoch": 2.03, + "learning_rate": 0.00040074222668004013, + "loss": 2.6028, + "theoretical_loss": 3.7896762762675893, + "tokens_seen": 681670656 + }, + { + "epoch": 2.03, + "learning_rate": 0.00040073219658976936, + "loss": 2.8811, + "theoretical_loss": 3.7896391090151944, + "tokens_seen": 681736192 + }, + { + "epoch": 2.03, + "learning_rate": 0.0004007221664994985, + "loss": 2.9482, + "theoretical_loss": 3.789601946335864, + "tokens_seen": 681801728 + }, + { + "epoch": 2.03, + "learning_rate": 0.0004007121364092277, + "loss": 2.8387, + "theoretical_loss": 3.7895647882285957, + "tokens_seen": 681867264 + }, + { + "epoch": 2.03, + "learning_rate": 0.00040070210631895685, + "loss": 2.8092, + "theoretical_loss": 3.789527634692387, + "tokens_seen": 681932800 + }, + { + "epoch": 2.03, + "learning_rate": 0.0004006920762286861, + "loss": 2.9132, + "theoretical_loss": 3.7894904857262373, + "tokens_seen": 681998336 + }, + { + "epoch": 2.03, + "learning_rate": 0.00040068204613841527, + "loss": 3.1188, + "theoretical_loss": 3.7894533413291445, + "tokens_seen": 682063872 + }, + { + "epoch": 2.03, + "learning_rate": 0.00040067201604814445, + "loss": 3.0301, + "theoretical_loss": 3.7894162015001087, + "tokens_seen": 682129408 + }, + { + "epoch": 2.03, + "learning_rate": 0.00040066198595787363, + "loss": 2.6942, + "theoretical_loss": 3.789379066238128, + "tokens_seen": 682194944 + }, + { + "epoch": 2.03, + "learning_rate": 0.0004006519558676028, + "loss": 3.0303, + "theoretical_loss": 3.789341935542204, + "tokens_seen": 682260480 + }, + { + "epoch": 2.03, + "learning_rate": 0.000400641925777332, + "loss": 2.9624, + "theoretical_loss": 3.789304809411335, + "tokens_seen": 682326016 + }, + { + "epoch": 2.03, + "learning_rate": 0.00040063189568706123, + "loss": 2.893, + "theoretical_loss": 3.789267687844522, + "tokens_seen": 682391552 + }, + { + "epoch": 2.03, + "learning_rate": 0.00040062186559679036, + "loss": 2.8456, + "theoretical_loss": 3.7892305708407656, + "tokens_seen": 682457088 + }, + { + "epoch": 2.03, + "learning_rate": 0.0004006118355065196, + "loss": 2.7705, + "theoretical_loss": 3.789193458399067, + "tokens_seen": 682522624 + }, + { + "epoch": 2.03, + "learning_rate": 0.0004006018054162487, + "loss": 3.0149, + "theoretical_loss": 3.789156350518428, + "tokens_seen": 682588160 + }, + { + "epoch": 2.03, + "learning_rate": 0.00040059177532597795, + "loss": 2.8807, + "theoretical_loss": 3.7891192471978488, + "tokens_seen": 682653696 + }, + { + "epoch": 2.03, + "learning_rate": 0.00040058174523570713, + "loss": 2.9405, + "theoretical_loss": 3.789082148436332, + "tokens_seen": 682719232 + }, + { + "epoch": 2.03, + "learning_rate": 0.0004005717151454363, + "loss": 2.8325, + "theoretical_loss": 3.7890450542328806, + "tokens_seen": 682784768 + }, + { + "epoch": 2.03, + "learning_rate": 0.0004005616850551655, + "loss": 2.9391, + "theoretical_loss": 3.7890079645864967, + "tokens_seen": 682850304 + }, + { + "epoch": 2.03, + "learning_rate": 0.00040055165496489473, + "loss": 3.1444, + "theoretical_loss": 3.7889708794961825, + "tokens_seen": 682915840 + }, + { + "epoch": 2.03, + "learning_rate": 0.00040054162487462386, + "loss": 2.9871, + "theoretical_loss": 3.788933798960942, + "tokens_seen": 682981376 + }, + { + "epoch": 2.03, + "learning_rate": 0.0004005315947843531, + "loss": 2.8059, + "theoretical_loss": 3.7888967229797785, + "tokens_seen": 683046912 + }, + { + "epoch": 2.03, + "learning_rate": 0.0004005215646940822, + "loss": 2.5791, + "theoretical_loss": 3.7888596515516952, + "tokens_seen": 683112448 + }, + { + "epoch": 2.03, + "learning_rate": 0.00040051153460381146, + "loss": 3.0333, + "theoretical_loss": 3.7888225846756978, + "tokens_seen": 683177984 + }, + { + "epoch": 2.03, + "objective/train/docs_used": 812647, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.369586229324341, + "objective/train/theoretical_loss": 3.7888040529444194, + "objective/train/tokens_used": 703670752, + "theoretical_loss": 3.7888040529444194, + "tokens_seen": 683210752 + }, + { + "epoch": 2.03, + "learning_rate": 0.00040050150451354064, + "loss": 2.8012, + "theoretical_loss": 3.788785522350789, + "tokens_seen": 683243520 + }, + { + "epoch": 2.03, + "learning_rate": 0.0004004914744232698, + "loss": 3.2464, + "theoretical_loss": 3.7887484645759746, + "tokens_seen": 683309056 + }, + { + "epoch": 2.03, + "learning_rate": 0.000400481444332999, + "loss": 2.9848, + "theoretical_loss": 3.788711411350259, + "tokens_seen": 683374592 + }, + { + "epoch": 2.03, + "learning_rate": 0.0004004714142427282, + "loss": 2.8967, + "theoretical_loss": 3.7886743626726487, + "tokens_seen": 683440128 + }, + { + "epoch": 2.03, + "learning_rate": 0.00040046138415245736, + "loss": 2.9512, + "theoretical_loss": 3.788637318542148, + "tokens_seen": 683505664 + }, + { + "epoch": 2.03, + "learning_rate": 0.0004004513540621866, + "loss": 2.9431, + "theoretical_loss": 3.7886002789577633, + "tokens_seen": 683571200 + }, + { + "epoch": 2.03, + "learning_rate": 0.0004004413239719157, + "loss": 2.8512, + "theoretical_loss": 3.788563243918502, + "tokens_seen": 683636736 + }, + { + "epoch": 2.03, + "learning_rate": 0.00040043129388164496, + "loss": 2.7822, + "theoretical_loss": 3.7885262134233697, + "tokens_seen": 683702272 + }, + { + "epoch": 2.03, + "learning_rate": 0.0004004212637913741, + "loss": 2.977, + "theoretical_loss": 3.7884891874713738, + "tokens_seen": 683767808 + }, + { + "epoch": 2.03, + "learning_rate": 0.0004004112337011033, + "loss": 2.7165, + "theoretical_loss": 3.7884521660615205, + "tokens_seen": 683833344 + }, + { + "epoch": 2.03, + "learning_rate": 0.0004004012036108325, + "loss": 2.6355, + "theoretical_loss": 3.7884151491928186, + "tokens_seen": 683898880 + }, + { + "epoch": 2.03, + "learning_rate": 0.0004003911735205617, + "loss": 2.8597, + "theoretical_loss": 3.788378136864276, + "tokens_seen": 683964416 + }, + { + "epoch": 2.03, + "learning_rate": 0.00040038114343029087, + "loss": 2.9315, + "theoretical_loss": 3.7883411290749005, + "tokens_seen": 684029952 + }, + { + "epoch": 2.03, + "learning_rate": 0.0004003711133400201, + "loss": 2.7077, + "theoretical_loss": 3.7883041258237, + "tokens_seen": 684095488 + }, + { + "epoch": 2.03, + "learning_rate": 0.00040036108324974923, + "loss": 2.912, + "theoretical_loss": 3.7882671271096844, + "tokens_seen": 684161024 + }, + { + "epoch": 2.03, + "learning_rate": 0.00040035105315947846, + "loss": 2.9169, + "theoretical_loss": 3.7882301329318624, + "tokens_seen": 684226560 + }, + { + "epoch": 2.03, + "learning_rate": 0.0004003410230692076, + "loss": 2.8847, + "theoretical_loss": 3.788193143289243, + "tokens_seen": 684292096 + }, + { + "epoch": 2.03, + "learning_rate": 0.0004003309929789368, + "loss": 2.8739, + "theoretical_loss": 3.7881561581808363, + "tokens_seen": 684357632 + }, + { + "epoch": 2.03, + "learning_rate": 0.000400320962888666, + "loss": 2.8555, + "theoretical_loss": 3.788119177605653, + "tokens_seen": 684423168 + }, + { + "epoch": 2.03, + "learning_rate": 0.0004003109327983952, + "loss": 2.7767, + "theoretical_loss": 3.7880822015627023, + "tokens_seen": 684488704 + }, + { + "epoch": 2.03, + "learning_rate": 0.00040030090270812437, + "loss": 2.8128, + "theoretical_loss": 3.788045230050996, + "tokens_seen": 684554240 + }, + { + "epoch": 2.03, + "learning_rate": 0.00040029087261785355, + "loss": 2.9026, + "theoretical_loss": 3.788008263069544, + "tokens_seen": 684619776 + }, + { + "epoch": 2.03, + "learning_rate": 0.00040028084252758273, + "loss": 2.7174, + "theoretical_loss": 3.7879713006173583, + "tokens_seen": 684685312 + }, + { + "epoch": 2.03, + "learning_rate": 0.00040027081243731197, + "loss": 2.9723, + "theoretical_loss": 3.787934342693451, + "tokens_seen": 684750848 + }, + { + "epoch": 2.03, + "learning_rate": 0.0004002607823470411, + "loss": 3.017, + "theoretical_loss": 3.787897389296833, + "tokens_seen": 684816384 + }, + { + "epoch": 2.03, + "objective/train/docs_used": 812952, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.962841510772705, + "objective/train/theoretical_loss": 3.787878914295949, + "objective/train/tokens_used": 705309152, + "theoretical_loss": 3.787878914295949, + "tokens_seen": 684849152 + }, + { + "epoch": 2.03, + "learning_rate": 0.00040025075225677033, + "loss": 2.9822, + "theoretical_loss": 3.787860440426517, + "tokens_seen": 684881920 + }, + { + "epoch": 2.03, + "learning_rate": 0.00040024072216649946, + "loss": 3.1501, + "theoretical_loss": 3.787823496081515, + "tokens_seen": 684947456 + }, + { + "epoch": 2.03, + "learning_rate": 0.0004002306920762287, + "loss": 3.0707, + "theoretical_loss": 3.787786556260841, + "tokens_seen": 685012992 + }, + { + "epoch": 2.03, + "learning_rate": 0.00040022066198595787, + "loss": 2.8825, + "theoretical_loss": 3.7877496209635075, + "tokens_seen": 685078528 + }, + { + "epoch": 2.03, + "learning_rate": 0.00040021063189568705, + "loss": 3.1118, + "theoretical_loss": 3.7877126901885276, + "tokens_seen": 685144064 + }, + { + "epoch": 2.03, + "learning_rate": 0.00040020060180541623, + "loss": 3.1401, + "theoretical_loss": 3.7876757639349163, + "tokens_seen": 685209600 + }, + { + "epoch": 2.03, + "learning_rate": 0.00040019057171514547, + "loss": 3.1658, + "theoretical_loss": 3.787638842201686, + "tokens_seen": 685275136 + }, + { + "epoch": 2.03, + "learning_rate": 0.0004001805416248746, + "loss": 3.1845, + "theoretical_loss": 3.7876019249878525, + "tokens_seen": 685340672 + }, + { + "epoch": 2.03, + "learning_rate": 0.00040017051153460383, + "loss": 2.8411, + "theoretical_loss": 3.78756501229243, + "tokens_seen": 685406208 + }, + { + "epoch": 2.03, + "learning_rate": 0.00040016048144433296, + "loss": 3.0347, + "theoretical_loss": 3.7875281041144335, + "tokens_seen": 685471744 + }, + { + "epoch": 2.03, + "learning_rate": 0.0004001504513540622, + "loss": 3.0841, + "theoretical_loss": 3.787491200452878, + "tokens_seen": 685537280 + }, + { + "epoch": 2.03, + "learning_rate": 0.00040014042126379143, + "loss": 2.7852, + "theoretical_loss": 3.78745430130678, + "tokens_seen": 685602816 + }, + { + "epoch": 2.03, + "learning_rate": 0.00040013039117352056, + "loss": 2.7576, + "theoretical_loss": 3.7874174066751545, + "tokens_seen": 685668352 + }, + { + "epoch": 2.03, + "learning_rate": 0.0004001203610832498, + "loss": 3.0064, + "theoretical_loss": 3.7873805165570182, + "tokens_seen": 685733888 + }, + { + "epoch": 2.03, + "learning_rate": 0.0004001103309929789, + "loss": 2.9897, + "theoretical_loss": 3.7873436309513884, + "tokens_seen": 685799424 + }, + { + "epoch": 2.03, + "learning_rate": 0.00040010030090270815, + "loss": 3.0036, + "theoretical_loss": 3.7873067498572808, + "tokens_seen": 685864960 + }, + { + "epoch": 2.03, + "learning_rate": 0.00040009027081243733, + "loss": 2.9462, + "theoretical_loss": 3.787269873273713, + "tokens_seen": 685930496 + }, + { + "epoch": 2.03, + "learning_rate": 0.0004000802407221665, + "loss": 3.017, + "theoretical_loss": 3.787233001199703, + "tokens_seen": 685996032 + }, + { + "epoch": 2.03, + "learning_rate": 0.0004000702106318957, + "loss": 3.1799, + "theoretical_loss": 3.7871961336342674, + "tokens_seen": 686061568 + }, + { + "epoch": 2.03, + "learning_rate": 0.00040006018054162493, + "loss": 3.0584, + "theoretical_loss": 3.787159270576425, + "tokens_seen": 686127104 + }, + { + "epoch": 2.03, + "learning_rate": 0.00040005015045135406, + "loss": 2.837, + "theoretical_loss": 3.787122412025195, + "tokens_seen": 686192640 + }, + { + "epoch": 2.03, + "learning_rate": 0.0004000401203610833, + "loss": 3.0278, + "theoretical_loss": 3.7870855579795952, + "tokens_seen": 686258176 + }, + { + "epoch": 2.03, + "learning_rate": 0.0004000300902708124, + "loss": 2.9139, + "theoretical_loss": 3.787048708438644, + "tokens_seen": 686323712 + }, + { + "epoch": 2.03, + "learning_rate": 0.00040002006018054166, + "loss": 3.0253, + "theoretical_loss": 3.7870118634013625, + "tokens_seen": 686389248 + }, + { + "epoch": 2.03, + "learning_rate": 0.00040001003009027084, + "loss": 2.9288, + "theoretical_loss": 3.786975022866769, + "tokens_seen": 686454784 + }, + { + "epoch": 2.03, + "objective/train/docs_used": 812952, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6860439777374268, + "objective/train/theoretical_loss": 3.786956604287674, + "objective/train/tokens_used": 706947552, + "theoretical_loss": 3.786956604287674, + "tokens_seen": 686487552 + }, + { + "epoch": 2.03, + "learning_rate": 0.0004, + "loss": 3.0279, + "theoretical_loss": 3.786938186833884, + "tokens_seen": 686520320 + }, + { + "epoch": 2.03, + "learning_rate": 0.0003999899699097292, + "loss": 3.0499, + "theoretical_loss": 3.7869013553017274, + "tokens_seen": 686585856 + }, + { + "epoch": 2.03, + "learning_rate": 0.0003999799398194584, + "loss": 3.1523, + "theoretical_loss": 3.7868645282693203, + "tokens_seen": 686651392 + }, + { + "epoch": 2.03, + "learning_rate": 0.00039996990972918756, + "loss": 2.8675, + "theoretical_loss": 3.786827705735683, + "tokens_seen": 686716928 + }, + { + "epoch": 2.03, + "learning_rate": 0.0003999598796389168, + "loss": 2.8969, + "theoretical_loss": 3.7867908876998366, + "tokens_seen": 686782464 + }, + { + "epoch": 2.03, + "learning_rate": 0.0003999498495486459, + "loss": 3.0232, + "theoretical_loss": 3.786754074160803, + "tokens_seen": 686848000 + }, + { + "epoch": 2.03, + "learning_rate": 0.00039993981945837516, + "loss": 3.1544, + "theoretical_loss": 3.7867172651176038, + "tokens_seen": 686913536 + }, + { + "epoch": 2.03, + "learning_rate": 0.0003999297893681043, + "loss": 3.3209, + "theoretical_loss": 3.7866804605692614, + "tokens_seen": 686979072 + }, + { + "epoch": 2.03, + "learning_rate": 0.0003999197592778335, + "loss": 2.9485, + "theoretical_loss": 3.786643660514798, + "tokens_seen": 687044608 + }, + { + "epoch": 2.03, + "learning_rate": 0.0003999097291875627, + "loss": 2.9027, + "theoretical_loss": 3.786606864953236, + "tokens_seen": 687110144 + }, + { + "epoch": 2.03, + "learning_rate": 0.0003998996990972919, + "loss": 3.0381, + "theoretical_loss": 3.786570073883599, + "tokens_seen": 687175680 + }, + { + "epoch": 2.03, + "learning_rate": 0.00039988966900702107, + "loss": 3.0156, + "theoretical_loss": 3.78653328730491, + "tokens_seen": 687241216 + }, + { + "epoch": 2.03, + "learning_rate": 0.0003998796389167503, + "loss": 3.2087, + "theoretical_loss": 3.7864965052161925, + "tokens_seen": 687306752 + }, + { + "epoch": 2.03, + "learning_rate": 0.00039986960882647943, + "loss": 3.0047, + "theoretical_loss": 3.7864597276164704, + "tokens_seen": 687372288 + }, + { + "epoch": 2.03, + "learning_rate": 0.00039985957873620866, + "loss": 3.0319, + "theoretical_loss": 3.7864229545047685, + "tokens_seen": 687437824 + }, + { + "epoch": 2.03, + "learning_rate": 0.0003998495486459378, + "loss": 3.2716, + "theoretical_loss": 3.7863861858801107, + "tokens_seen": 687503360 + }, + { + "epoch": 2.03, + "learning_rate": 0.000399839518555667, + "loss": 2.8685, + "theoretical_loss": 3.786349421741522, + "tokens_seen": 687568896 + }, + { + "epoch": 2.03, + "learning_rate": 0.0003998294884653962, + "loss": 2.9713, + "theoretical_loss": 3.7863126620880276, + "tokens_seen": 687634432 + }, + { + "epoch": 2.03, + "learning_rate": 0.0003998194583751254, + "loss": 2.9644, + "theoretical_loss": 3.786275906918653, + "tokens_seen": 687699968 + }, + { + "epoch": 2.03, + "learning_rate": 0.00039980942828485457, + "loss": 2.9327, + "theoretical_loss": 3.7862391562324245, + "tokens_seen": 687765504 + }, + { + "epoch": 2.03, + "learning_rate": 0.00039979939819458375, + "loss": 2.8463, + "theoretical_loss": 3.786202410028367, + "tokens_seen": 687831040 + }, + { + "epoch": 2.03, + "learning_rate": 0.00039978936810431293, + "loss": 3.209, + "theoretical_loss": 3.7861656683055074, + "tokens_seen": 687896576 + }, + { + "epoch": 2.03, + "learning_rate": 0.00039977933801404217, + "loss": 3.247, + "theoretical_loss": 3.7861289310628727, + "tokens_seen": 687962112 + }, + { + "epoch": 2.03, + "learning_rate": 0.0003997693079237713, + "loss": 3.0825, + "theoretical_loss": 3.7860921982994897, + "tokens_seen": 688027648 + }, + { + "epoch": 2.03, + "learning_rate": 0.00039975927783350053, + "loss": 2.9732, + "theoretical_loss": 3.786055470014386, + "tokens_seen": 688093184 + }, + { + "epoch": 2.03, + "objective/train/docs_used": 813708, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.2107160091400146, + "objective/train/theoretical_loss": 3.786037107550884, + "objective/train/tokens_used": 708585952, + "theoretical_loss": 3.786037107550884, + "tokens_seen": 688125952 + }, + { + "epoch": 2.03, + "learning_rate": 0.00039974924774322966, + "loss": 3.1854, + "theoretical_loss": 3.7860187462065875, + "tokens_seen": 688158720 + }, + { + "epoch": 2.03, + "learning_rate": 0.0003997392176529589, + "loss": 3.036, + "theoretical_loss": 3.7859820268751245, + "tokens_seen": 688224256 + }, + { + "epoch": 2.03, + "learning_rate": 0.00039972918756268807, + "loss": 2.9536, + "theoretical_loss": 3.785945312019024, + "tokens_seen": 688289792 + }, + { + "epoch": 2.03, + "learning_rate": 0.00039971915747241725, + "loss": 2.7898, + "theoretical_loss": 3.7859086016373134, + "tokens_seen": 688355328 + }, + { + "epoch": 2.03, + "learning_rate": 0.00039970912738214643, + "loss": 2.9671, + "theoretical_loss": 3.785871895729024, + "tokens_seen": 688420864 + }, + { + "epoch": 2.03, + "learning_rate": 0.00039969909729187567, + "loss": 2.9539, + "theoretical_loss": 3.785835194293183, + "tokens_seen": 688486400 + }, + { + "epoch": 2.03, + "learning_rate": 0.0003996890672016048, + "loss": 2.9139, + "theoretical_loss": 3.7857984973288206, + "tokens_seen": 688551936 + }, + { + "epoch": 2.03, + "learning_rate": 0.00039967903711133403, + "loss": 3.1913, + "theoretical_loss": 3.7857618048349657, + "tokens_seen": 688617472 + }, + { + "epoch": 2.03, + "learning_rate": 0.00039966900702106316, + "loss": 3.1764, + "theoretical_loss": 3.7857251168106494, + "tokens_seen": 688683008 + }, + { + "epoch": 2.03, + "learning_rate": 0.0003996589769307924, + "loss": 2.9506, + "theoretical_loss": 3.7856884332549017, + "tokens_seen": 688748544 + }, + { + "epoch": 2.03, + "learning_rate": 0.0003996489468405216, + "loss": 3.1994, + "theoretical_loss": 3.7856517541667523, + "tokens_seen": 688814080 + }, + { + "epoch": 2.03, + "learning_rate": 0.00039963891675025076, + "loss": 3.0963, + "theoretical_loss": 3.7856150795452335, + "tokens_seen": 688879616 + }, + { + "epoch": 2.03, + "learning_rate": 0.00039962888665997994, + "loss": 2.9912, + "theoretical_loss": 3.7855784093893754, + "tokens_seen": 688945152 + }, + { + "epoch": 2.03, + "learning_rate": 0.0003996188565697091, + "loss": 3.1565, + "theoretical_loss": 3.78554174369821, + "tokens_seen": 689010688 + }, + { + "epoch": 2.03, + "learning_rate": 0.0003996088264794383, + "loss": 2.9481, + "theoretical_loss": 3.7855050824707694, + "tokens_seen": 689076224 + }, + { + "epoch": 2.03, + "learning_rate": 0.00039959879638916754, + "loss": 2.9037, + "theoretical_loss": 3.7854684257060853, + "tokens_seen": 689141760 + }, + { + "epoch": 2.03, + "learning_rate": 0.00039958876629889666, + "loss": 3.0422, + "theoretical_loss": 3.7854317734031904, + "tokens_seen": 689207296 + }, + { + "epoch": 2.03, + "learning_rate": 0.0003995787362086259, + "loss": 3.0962, + "theoretical_loss": 3.7853951255611165, + "tokens_seen": 689272832 + }, + { + "epoch": 2.03, + "learning_rate": 0.0003995687061183551, + "loss": 2.9324, + "theoretical_loss": 3.785358482178898, + "tokens_seen": 689338368 + }, + { + "epoch": 2.03, + "learning_rate": 0.00039955867602808426, + "loss": 3.0348, + "theoretical_loss": 3.7853218432555678, + "tokens_seen": 689403904 + }, + { + "epoch": 2.03, + "learning_rate": 0.00039954864593781344, + "loss": 2.9487, + "theoretical_loss": 3.7852852087901594, + "tokens_seen": 689469440 + }, + { + "epoch": 2.03, + "learning_rate": 0.0003995386158475426, + "loss": 3.1092, + "theoretical_loss": 3.7852485787817063, + "tokens_seen": 689534976 + }, + { + "epoch": 2.03, + "learning_rate": 0.0003995285857572718, + "loss": 2.9527, + "theoretical_loss": 3.7852119532292434, + "tokens_seen": 689600512 + }, + { + "epoch": 2.03, + "learning_rate": 0.00039951855566700104, + "loss": 3.163, + "theoretical_loss": 3.785175332131805, + "tokens_seen": 689666048 + }, + { + "epoch": 2.03, + "learning_rate": 0.00039950852557673017, + "loss": 2.9854, + "theoretical_loss": 3.7851387154884257, + "tokens_seen": 689731584 + }, + { + "epoch": 2.03, + "objective/train/docs_used": 815148, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.4033315181732178, + "objective/train/theoretical_loss": 3.7851204088367068, + "objective/train/tokens_used": 710224352, + "theoretical_loss": 3.7851204088367068, + "tokens_seen": 689764352 + }, + { + "epoch": 2.03, + "learning_rate": 0.0003994984954864594, + "loss": 3.257, + "theoretical_loss": 3.785102103298141, + "tokens_seen": 689797120 + }, + { + "epoch": 2.03, + "learning_rate": 0.00039948846539618853, + "loss": 3.0081, + "theoretical_loss": 3.7850654955599863, + "tokens_seen": 689862656 + }, + { + "epoch": 2.03, + "learning_rate": 0.00039947843530591776, + "loss": 2.9746, + "theoretical_loss": 3.7850288922729973, + "tokens_seen": 689928192 + }, + { + "epoch": 2.03, + "learning_rate": 0.00039946840521564694, + "loss": 2.9742, + "theoretical_loss": 3.78499229343621, + "tokens_seen": 689993728 + }, + { + "epoch": 2.03, + "learning_rate": 0.0003994583751253761, + "loss": 3.0874, + "theoretical_loss": 3.784955699048661, + "tokens_seen": 690059264 + }, + { + "epoch": 2.03, + "learning_rate": 0.0003994483450351053, + "loss": 3.1328, + "theoretical_loss": 3.784919109109386, + "tokens_seen": 690124800 + }, + { + "epoch": 2.03, + "learning_rate": 0.0003994383149448345, + "loss": 2.89, + "theoretical_loss": 3.784882523617423, + "tokens_seen": 690190336 + }, + { + "epoch": 2.03, + "learning_rate": 0.00039942828485456367, + "loss": 2.852, + "theoretical_loss": 3.784845942571809, + "tokens_seen": 690255872 + }, + { + "epoch": 2.03, + "learning_rate": 0.0003994182547642929, + "loss": 3.2457, + "theoretical_loss": 3.784809365971581, + "tokens_seen": 690321408 + }, + { + "epoch": 2.03, + "learning_rate": 0.00039940822467402203, + "loss": 2.9565, + "theoretical_loss": 3.784772793815777, + "tokens_seen": 690386944 + }, + { + "epoch": 2.03, + "learning_rate": 0.00039939819458375127, + "loss": 2.902, + "theoretical_loss": 3.784736226103436, + "tokens_seen": 690452480 + }, + { + "epoch": 2.03, + "learning_rate": 0.0003993881644934805, + "loss": 3.042, + "theoretical_loss": 3.784699662833596, + "tokens_seen": 690518016 + }, + { + "epoch": 2.03, + "learning_rate": 0.00039937813440320963, + "loss": 3.0018, + "theoretical_loss": 3.784663104005295, + "tokens_seen": 690583552 + }, + { + "epoch": 2.03, + "learning_rate": 0.00039936810431293886, + "loss": 2.8559, + "theoretical_loss": 3.784626549617573, + "tokens_seen": 690649088 + }, + { + "epoch": 2.03, + "learning_rate": 0.000399358074222668, + "loss": 3.0771, + "theoretical_loss": 3.7845899996694685, + "tokens_seen": 690714624 + }, + { + "epoch": 2.03, + "learning_rate": 0.0003993480441323972, + "loss": 3.186, + "theoretical_loss": 3.7845534541600223, + "tokens_seen": 690780160 + }, + { + "epoch": 2.03, + "learning_rate": 0.0003993380140421264, + "loss": 2.9598, + "theoretical_loss": 3.7845169130882734, + "tokens_seen": 690845696 + }, + { + "epoch": 2.03, + "learning_rate": 0.0003993279839518556, + "loss": 3.0254, + "theoretical_loss": 3.7844803764532617, + "tokens_seen": 690911232 + }, + { + "epoch": 2.03, + "learning_rate": 0.00039931795386158477, + "loss": 2.7558, + "theoretical_loss": 3.784443844254029, + "tokens_seen": 690976768 + }, + { + "epoch": 2.03, + "learning_rate": 0.00039930792377131395, + "loss": 3.0946, + "theoretical_loss": 3.784407316489615, + "tokens_seen": 691042304 + }, + { + "epoch": 2.03, + "learning_rate": 0.00039929789368104313, + "loss": 2.9042, + "theoretical_loss": 3.784370793159061, + "tokens_seen": 691107840 + }, + { + "epoch": 2.03, + "learning_rate": 0.00039928786359077237, + "loss": 3.007, + "theoretical_loss": 3.784334274261409, + "tokens_seen": 691173376 + }, + { + "epoch": 2.03, + "learning_rate": 0.0003992778335005015, + "loss": 3.001, + "theoretical_loss": 3.7842977597957006, + "tokens_seen": 691238912 + }, + { + "epoch": 2.03, + "learning_rate": 0.00039926780341023073, + "loss": 3.1521, + "theoretical_loss": 3.784261249760977, + "tokens_seen": 691304448 + }, + { + "epoch": 2.03, + "learning_rate": 0.00039925777331995986, + "loss": 3.0055, + "theoretical_loss": 3.784224744156282, + "tokens_seen": 691369984 + }, + { + "epoch": 2.03, + "objective/train/docs_used": 815781, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.672840118408203, + "objective/train/theoretical_loss": 3.7842064930148953, + "objective/train/tokens_used": 711862752, + "theoretical_loss": 3.7842064930148953, + "tokens_seen": 691402752 + }, + { + "epoch": 2.03, + "learning_rate": 0.0003992477432296891, + "loss": 3.1546, + "theoretical_loss": 3.784188242980657, + "tokens_seen": 691435520 + }, + { + "epoch": 2.03, + "learning_rate": 0.00039923771313941827, + "loss": 2.9453, + "theoretical_loss": 3.784151746233145, + "tokens_seen": 691501056 + }, + { + "epoch": 2.03, + "learning_rate": 0.00039922768304914745, + "loss": 2.872, + "theoretical_loss": 3.78411525391279, + "tokens_seen": 691566592 + }, + { + "epoch": 2.03, + "learning_rate": 0.00039921765295887663, + "loss": 3.0222, + "theoretical_loss": 3.7840787660186344, + "tokens_seen": 691632128 + }, + { + "epoch": 2.03, + "learning_rate": 0.00039920762286860587, + "loss": 2.9783, + "theoretical_loss": 3.7840422825497235, + "tokens_seen": 691697664 + }, + { + "epoch": 2.03, + "learning_rate": 0.000399197592778335, + "loss": 2.9824, + "theoretical_loss": 3.7840058035050994, + "tokens_seen": 691763200 + }, + { + "epoch": 2.03, + "learning_rate": 0.00039918756268806423, + "loss": 3.1181, + "theoretical_loss": 3.7839693288838085, + "tokens_seen": 691828736 + }, + { + "epoch": 2.03, + "learning_rate": 0.00039917753259779336, + "loss": 3.0017, + "theoretical_loss": 3.783932858684895, + "tokens_seen": 691894272 + }, + { + "epoch": 2.03, + "learning_rate": 0.0003991675025075226, + "loss": 2.8385, + "theoretical_loss": 3.783896392907402, + "tokens_seen": 691959808 + }, + { + "epoch": 2.03, + "learning_rate": 0.0003991574724172518, + "loss": 3.1507, + "theoretical_loss": 3.7838599315503774, + "tokens_seen": 692025344 + }, + { + "epoch": 2.03, + "learning_rate": 0.00039914744232698096, + "loss": 2.9381, + "theoretical_loss": 3.7838234746128654, + "tokens_seen": 692090880 + }, + { + "epoch": 2.03, + "learning_rate": 0.00039913741223671014, + "loss": 3.2148, + "theoretical_loss": 3.7837870220939127, + "tokens_seen": 692156416 + }, + { + "epoch": 2.03, + "learning_rate": 0.0003991273821464393, + "loss": 3.0144, + "theoretical_loss": 3.783750573992565, + "tokens_seen": 692221952 + }, + { + "epoch": 2.03, + "learning_rate": 0.0003991173520561685, + "loss": 3.1139, + "theoretical_loss": 3.7837141303078683, + "tokens_seen": 692287488 + }, + { + "epoch": 2.03, + "learning_rate": 0.00039910732196589774, + "loss": 3.0469, + "theoretical_loss": 3.7836776910388696, + "tokens_seen": 692353024 + }, + { + "epoch": 2.03, + "learning_rate": 0.00039909729187562686, + "loss": 2.8595, + "theoretical_loss": 3.783641256184617, + "tokens_seen": 692418560 + }, + { + "epoch": 2.03, + "learning_rate": 0.0003990872617853561, + "loss": 3.0581, + "theoretical_loss": 3.7836048257441566, + "tokens_seen": 692484096 + }, + { + "epoch": 2.03, + "learning_rate": 0.0003990772316950853, + "loss": 3.0968, + "theoretical_loss": 3.7835683997165366, + "tokens_seen": 692549632 + }, + { + "epoch": 2.03, + "learning_rate": 0.00039906720160481446, + "loss": 2.6969, + "theoretical_loss": 3.7835319781008048, + "tokens_seen": 692615168 + }, + { + "epoch": 2.03, + "learning_rate": 0.00039905717151454364, + "loss": 2.973, + "theoretical_loss": 3.78349556089601, + "tokens_seen": 692680704 + }, + { + "epoch": 2.03, + "learning_rate": 0.0003990471414242728, + "loss": 2.8064, + "theoretical_loss": 3.7834591481011994, + "tokens_seen": 692746240 + }, + { + "epoch": 2.03, + "learning_rate": 0.000399037111334002, + "loss": 3.1452, + "theoretical_loss": 3.783422739715424, + "tokens_seen": 692811776 + }, + { + "epoch": 2.03, + "learning_rate": 0.00039902708124373124, + "loss": 3.2073, + "theoretical_loss": 3.7833863357377306, + "tokens_seen": 692877312 + }, + { + "epoch": 2.03, + "learning_rate": 0.00039901705115346037, + "loss": 2.7702, + "theoretical_loss": 3.7833499361671703, + "tokens_seen": 692942848 + }, + { + "epoch": 2.03, + "learning_rate": 0.0003990070210631896, + "loss": 2.927, + "theoretical_loss": 3.7833135410027925, + "tokens_seen": 693008384 + }, + { + "epoch": 2.03, + "objective/train/docs_used": 817009, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7722556591033936, + "objective/train/theoretical_loss": 3.7832953450726245, + "objective/train/tokens_used": 713501152, + "theoretical_loss": 3.7832953450726245, + "tokens_seen": 693041152 + }, + { + "epoch": 2.03, + "learning_rate": 0.00039899699097291873, + "loss": 3.0062, + "theoretical_loss": 3.783277150243646, + "tokens_seen": 693073920 + }, + { + "epoch": 2.03, + "learning_rate": 0.00039898696088264796, + "loss": 3.0559, + "theoretical_loss": 3.7832407638887826, + "tokens_seen": 693139456 + }, + { + "epoch": 2.03, + "learning_rate": 0.00039897693079237714, + "loss": 3.0706, + "theoretical_loss": 3.783204381937253, + "tokens_seen": 693204992 + }, + { + "epoch": 2.03, + "learning_rate": 0.0003989669007021063, + "loss": 2.9877, + "theoretical_loss": 3.7831680043881066, + "tokens_seen": 693270528 + }, + { + "epoch": 2.03, + "learning_rate": 0.0003989568706118355, + "loss": 2.8072, + "theoretical_loss": 3.7831316312403964, + "tokens_seen": 693336064 + }, + { + "epoch": 2.03, + "learning_rate": 0.0003989468405215647, + "loss": 2.7801, + "theoretical_loss": 3.7830952624931724, + "tokens_seen": 693401600 + }, + { + "epoch": 2.03, + "learning_rate": 0.00039893681043129387, + "loss": 3.0159, + "theoretical_loss": 3.7830588981454873, + "tokens_seen": 693467136 + }, + { + "epoch": 2.03, + "learning_rate": 0.0003989267803410231, + "loss": 2.9373, + "theoretical_loss": 3.783022538196393, + "tokens_seen": 693532672 + }, + { + "epoch": 2.03, + "learning_rate": 0.00039891675025075223, + "loss": 2.9548, + "theoretical_loss": 3.782986182644941, + "tokens_seen": 693598208 + }, + { + "epoch": 2.03, + "learning_rate": 0.00039890672016048147, + "loss": 3.2316, + "theoretical_loss": 3.7829498314901855, + "tokens_seen": 693663744 + }, + { + "epoch": 2.03, + "learning_rate": 0.00039889669007021065, + "loss": 3.1027, + "theoretical_loss": 3.7829134847311785, + "tokens_seen": 693729280 + }, + { + "epoch": 2.03, + "learning_rate": 0.00039888665997993983, + "loss": 3.2061, + "theoretical_loss": 3.782877142366974, + "tokens_seen": 693794816 + }, + { + "epoch": 2.03, + "learning_rate": 0.000398876629889669, + "loss": 3.1499, + "theoretical_loss": 3.782840804396624, + "tokens_seen": 693860352 + }, + { + "epoch": 2.03, + "learning_rate": 0.0003988665997993982, + "loss": 3.1008, + "theoretical_loss": 3.782804470819184, + "tokens_seen": 693925888 + }, + { + "epoch": 2.03, + "learning_rate": 0.00039885656970912737, + "loss": 3.0239, + "theoretical_loss": 3.782768141633708, + "tokens_seen": 693991424 + }, + { + "epoch": 2.03, + "learning_rate": 0.0003988465396188566, + "loss": 3.1596, + "theoretical_loss": 3.7827318168392488, + "tokens_seen": 694056960 + }, + { + "epoch": 2.03, + "learning_rate": 0.00039883650952858573, + "loss": 2.9416, + "theoretical_loss": 3.782695496434863, + "tokens_seen": 694122496 + }, + { + "epoch": 2.03, + "learning_rate": 0.00039882647943831497, + "loss": 2.9495, + "theoretical_loss": 3.782659180419605, + "tokens_seen": 694188032 + }, + { + "epoch": 2.03, + "learning_rate": 0.0003988164493480441, + "loss": 2.9938, + "theoretical_loss": 3.7826228687925294, + "tokens_seen": 694253568 + }, + { + "epoch": 2.03, + "learning_rate": 0.00039880641925777333, + "loss": 2.9565, + "theoretical_loss": 3.7825865615526926, + "tokens_seen": 694319104 + }, + { + "epoch": 2.03, + "learning_rate": 0.0003987963891675025, + "loss": 3.0452, + "theoretical_loss": 3.7825502586991506, + "tokens_seen": 694384640 + }, + { + "epoch": 2.03, + "learning_rate": 0.0003987863590772317, + "loss": 2.9098, + "theoretical_loss": 3.782513960230959, + "tokens_seen": 694450176 + }, + { + "epoch": 2.03, + "learning_rate": 0.0003987763289869609, + "loss": 2.9125, + "theoretical_loss": 3.7824776661471744, + "tokens_seen": 694515712 + }, + { + "epoch": 2.03, + "learning_rate": 0.00039876629889669006, + "loss": 2.9881, + "theoretical_loss": 3.7824413764468536, + "tokens_seen": 694581248 + }, + { + "epoch": 2.03, + "learning_rate": 0.00039875626880641924, + "loss": 3.0315, + "theoretical_loss": 3.7824050911290543, + "tokens_seen": 694646784 + }, + { + "epoch": 2.03, + "objective/train/docs_used": 817800, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.937741279602051, + "objective/train/theoretical_loss": 3.782386950113305, + "objective/train/tokens_used": 715139552, + "theoretical_loss": 3.782386950113305, + "tokens_seen": 694679552 + }, + { + "epoch": 2.03, + "learning_rate": 0.00039874623871614847, + "loss": 2.7712, + "theoretical_loss": 3.782368810192833, + "tokens_seen": 694712320 + }, + { + "epoch": 2.03, + "learning_rate": 0.0003987362086258776, + "loss": 3.0905, + "theoretical_loss": 3.782332533637248, + "tokens_seen": 694777856 + }, + { + "epoch": 2.03, + "learning_rate": 0.00039872617853560683, + "loss": 3.1504, + "theoretical_loss": 3.7822962614613567, + "tokens_seen": 694843392 + }, + { + "epoch": 2.03, + "learning_rate": 0.000398716148445336, + "loss": 3.1779, + "theoretical_loss": 3.7822599936642174, + "tokens_seen": 694908928 + }, + { + "epoch": 2.03, + "learning_rate": 0.0003987061183550652, + "loss": 2.9834, + "theoretical_loss": 3.7822237302448887, + "tokens_seen": 694974464 + }, + { + "epoch": 2.03, + "learning_rate": 0.0003986960882647944, + "loss": 2.9804, + "theoretical_loss": 3.78218747120243, + "tokens_seen": 695040000 + }, + { + "epoch": 2.03, + "learning_rate": 0.00039868605817452356, + "loss": 2.9168, + "theoretical_loss": 3.7821512165358993, + "tokens_seen": 695105536 + }, + { + "epoch": 2.03, + "learning_rate": 0.00039867602808425274, + "loss": 3.0235, + "theoretical_loss": 3.782114966244357, + "tokens_seen": 695171072 + }, + { + "epoch": 2.03, + "learning_rate": 0.000398665997993982, + "loss": 2.9155, + "theoretical_loss": 3.7820787203268624, + "tokens_seen": 695236608 + }, + { + "epoch": 2.03, + "learning_rate": 0.0003986559679037111, + "loss": 3.0621, + "theoretical_loss": 3.7820424787824747, + "tokens_seen": 695302144 + }, + { + "epoch": 2.03, + "learning_rate": 0.00039864593781344034, + "loss": 2.9857, + "theoretical_loss": 3.7820062416102553, + "tokens_seen": 695367680 + }, + { + "epoch": 2.03, + "learning_rate": 0.0003986359077231695, + "loss": 2.9637, + "theoretical_loss": 3.7819700088092647, + "tokens_seen": 695433216 + }, + { + "epoch": 2.03, + "learning_rate": 0.0003986258776328987, + "loss": 2.8773, + "theoretical_loss": 3.781933780378563, + "tokens_seen": 695498752 + }, + { + "epoch": 2.03, + "learning_rate": 0.00039861584754262794, + "loss": 2.8766, + "theoretical_loss": 3.781897556317212, + "tokens_seen": 695564288 + }, + { + "epoch": 2.03, + "learning_rate": 0.00039860581745235706, + "loss": 2.9318, + "theoretical_loss": 3.781861336624273, + "tokens_seen": 695629824 + }, + { + "epoch": 2.03, + "learning_rate": 0.0003985957873620863, + "loss": 2.9717, + "theoretical_loss": 3.7818251212988074, + "tokens_seen": 695695360 + }, + { + "epoch": 2.03, + "learning_rate": 0.0003985857572718155, + "loss": 2.9584, + "theoretical_loss": 3.781788910339877, + "tokens_seen": 695760896 + }, + { + "epoch": 2.03, + "learning_rate": 0.00039857572718154466, + "loss": 2.743, + "theoretical_loss": 3.781752703746545, + "tokens_seen": 695826432 + }, + { + "epoch": 2.03, + "learning_rate": 0.00039856569709127384, + "loss": 3.0222, + "theoretical_loss": 3.7817165015178733, + "tokens_seen": 695891968 + }, + { + "epoch": 2.03, + "learning_rate": 0.000398555667001003, + "loss": 3.1952, + "theoretical_loss": 3.7816803036529247, + "tokens_seen": 695957504 + }, + { + "epoch": 2.03, + "learning_rate": 0.0003985456369107322, + "loss": 2.9163, + "theoretical_loss": 3.781644110150763, + "tokens_seen": 696023040 + }, + { + "epoch": 2.03, + "learning_rate": 0.00039853560682046144, + "loss": 3.0045, + "theoretical_loss": 3.781607921010451, + "tokens_seen": 696088576 + }, + { + "epoch": 2.03, + "learning_rate": 0.00039852557673019057, + "loss": 2.875, + "theoretical_loss": 3.781571736231053, + "tokens_seen": 696154112 + }, + { + "epoch": 2.03, + "learning_rate": 0.0003985155466399198, + "loss": 3.1498, + "theoretical_loss": 3.7815355558116326, + "tokens_seen": 696219648 + }, + { + "epoch": 2.03, + "learning_rate": 0.00039850551654964893, + "loss": 3.0813, + "theoretical_loss": 3.7814993797512546, + "tokens_seen": 696285184 + }, + { + "epoch": 2.03, + "objective/train/docs_used": 823060, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.890718460083008, + "objective/train/theoretical_loss": 3.7814812933554136, + "objective/train/tokens_used": 716777952, + "theoretical_loss": 3.7814812933554136, + "tokens_seen": 696317952 + }, + { + "epoch": 2.03, + "learning_rate": 0.00039849548645937816, + "loss": 2.9396, + "theoretical_loss": 3.781463208048983, + "tokens_seen": 696350720 + }, + { + "epoch": 2.03, + "learning_rate": 0.00039848545636910734, + "loss": 2.9903, + "theoretical_loss": 3.781427040703883, + "tokens_seen": 696416256 + }, + { + "epoch": 2.03, + "learning_rate": 0.0003984754262788365, + "loss": 2.9655, + "theoretical_loss": 3.781390877715019, + "tokens_seen": 696481792 + }, + { + "epoch": 2.03, + "learning_rate": 0.0003984653961885657, + "loss": 2.9875, + "theoretical_loss": 3.781354719081458, + "tokens_seen": 696547328 + }, + { + "epoch": 2.03, + "learning_rate": 0.0003984553660982949, + "loss": 2.7848, + "theoretical_loss": 3.7813185648022647, + "tokens_seen": 696612864 + }, + { + "epoch": 2.03, + "learning_rate": 0.00039844533600802407, + "loss": 2.7708, + "theoretical_loss": 3.7812824148765056, + "tokens_seen": 696678400 + }, + { + "epoch": 2.03, + "learning_rate": 0.0003984353059177533, + "loss": 2.9602, + "theoretical_loss": 3.781246269303247, + "tokens_seen": 696743936 + }, + { + "epoch": 2.03, + "learning_rate": 0.00039842527582748243, + "loss": 2.9896, + "theoretical_loss": 3.7812101280815553, + "tokens_seen": 696809472 + }, + { + "epoch": 2.03, + "learning_rate": 0.00039841524573721167, + "loss": 2.9217, + "theoretical_loss": 3.7811739912104976, + "tokens_seen": 696875008 + }, + { + "epoch": 2.03, + "learning_rate": 0.00039840521564694085, + "loss": 2.9529, + "theoretical_loss": 3.781137858689141, + "tokens_seen": 696940544 + }, + { + "epoch": 2.03, + "learning_rate": 0.00039839518555667003, + "loss": 3.0855, + "theoretical_loss": 3.781101730516553, + "tokens_seen": 697006080 + }, + { + "epoch": 2.03, + "learning_rate": 0.0003983851554663992, + "loss": 3.0658, + "theoretical_loss": 3.7810656066918016, + "tokens_seen": 697071616 + }, + { + "epoch": 2.03, + "learning_rate": 0.0003983751253761284, + "loss": 2.76, + "theoretical_loss": 3.7810294872139547, + "tokens_seen": 697137152 + }, + { + "epoch": 2.03, + "learning_rate": 0.00039836509528585757, + "loss": 2.8135, + "theoretical_loss": 3.7809933720820803, + "tokens_seen": 697202688 + }, + { + "epoch": 2.03, + "learning_rate": 0.0003983550651955868, + "loss": 3.0346, + "theoretical_loss": 3.7809572612952476, + "tokens_seen": 697268224 + }, + { + "epoch": 2.03, + "learning_rate": 0.00039834503510531593, + "loss": 2.9071, + "theoretical_loss": 3.7809211548525257, + "tokens_seen": 697333760 + }, + { + "epoch": 2.03, + "learning_rate": 0.00039833500501504517, + "loss": 3.0486, + "theoretical_loss": 3.7808850527529825, + "tokens_seen": 697399296 + }, + { + "epoch": 2.03, + "learning_rate": 0.0003983249749247743, + "loss": 2.8335, + "theoretical_loss": 3.7808489549956894, + "tokens_seen": 697464832 + }, + { + "epoch": 2.03, + "learning_rate": 0.00039831494483450353, + "loss": 3.1494, + "theoretical_loss": 3.7808128615797143, + "tokens_seen": 697530368 + }, + { + "epoch": 2.03, + "learning_rate": 0.0003983049147442327, + "loss": 2.9519, + "theoretical_loss": 3.780776772504129, + "tokens_seen": 697595904 + }, + { + "epoch": 2.03, + "learning_rate": 0.0003982948846539619, + "loss": 2.9174, + "theoretical_loss": 3.7807406877680023, + "tokens_seen": 697661440 + }, + { + "epoch": 2.03, + "learning_rate": 0.0003982848545636911, + "loss": 2.8904, + "theoretical_loss": 3.7807046073704065, + "tokens_seen": 697726976 + }, + { + "epoch": 2.03, + "learning_rate": 0.00039827482447342026, + "loss": 2.9245, + "theoretical_loss": 3.780668531310411, + "tokens_seen": 697792512 + }, + { + "epoch": 2.03, + "learning_rate": 0.00039826479438314944, + "loss": 3.0444, + "theoretical_loss": 3.7806324595870877, + "tokens_seen": 697858048 + }, + { + "epoch": 2.03, + "learning_rate": 0.00039825476429287867, + "loss": 2.9878, + "theoretical_loss": 3.7805963921995085, + "tokens_seen": 697923584 + }, + { + "epoch": 2.03, + "objective/train/docs_used": 828245, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.0669054985046387, + "objective/train/theoretical_loss": 3.7805783601313325, + "objective/train/tokens_used": 718416352, + "theoretical_loss": 3.7805783601313325, + "tokens_seen": 697956352 + }, + { + "epoch": 2.03, + "learning_rate": 0.0003982447342026078, + "loss": 3.0409, + "theoretical_loss": 3.7805603291467444, + "tokens_seen": 697989120 + }, + { + "epoch": 2.03, + "learning_rate": 0.00039823470411233703, + "loss": 3.0377, + "theoretical_loss": 3.7805242704278683, + "tokens_seen": 698054656 + }, + { + "epoch": 2.03, + "learning_rate": 0.0003982246740220662, + "loss": 2.7453, + "theoretical_loss": 3.7804882160419515, + "tokens_seen": 698120192 + }, + { + "epoch": 2.03, + "learning_rate": 0.0003982146439317954, + "loss": 3.0153, + "theoretical_loss": 3.780452165988067, + "tokens_seen": 698185728 + }, + { + "epoch": 2.03, + "learning_rate": 0.0003982046138415246, + "loss": 2.937, + "theoretical_loss": 3.780416120265289, + "tokens_seen": 698251264 + }, + { + "epoch": 2.03, + "learning_rate": 0.00039819458375125376, + "loss": 2.9108, + "theoretical_loss": 3.780380078872689, + "tokens_seen": 698316800 + }, + { + "epoch": 2.03, + "learning_rate": 0.00039818455366098294, + "loss": 2.7331, + "theoretical_loss": 3.7803440418093417, + "tokens_seen": 698382336 + }, + { + "epoch": 2.03, + "learning_rate": 0.0003981745235707122, + "loss": 3.0641, + "theoretical_loss": 3.78030800907432, + "tokens_seen": 698447872 + }, + { + "epoch": 2.03, + "learning_rate": 0.0003981644934804413, + "loss": 2.9069, + "theoretical_loss": 3.780271980666699, + "tokens_seen": 698513408 + }, + { + "epoch": 2.03, + "learning_rate": 0.00039815446339017054, + "loss": 2.8881, + "theoretical_loss": 3.780235956585552, + "tokens_seen": 698578944 + }, + { + "epoch": 2.03, + "learning_rate": 0.00039814443329989966, + "loss": 3.1442, + "theoretical_loss": 3.7801999368299537, + "tokens_seen": 698644480 + }, + { + "epoch": 2.03, + "learning_rate": 0.0003981344032096289, + "loss": 2.9341, + "theoretical_loss": 3.78016392139898, + "tokens_seen": 698710016 + }, + { + "epoch": 2.03, + "learning_rate": 0.0003981243731193581, + "loss": 2.6563, + "theoretical_loss": 3.7801279102917063, + "tokens_seen": 698775552 + }, + { + "epoch": 2.03, + "learning_rate": 0.00039811434302908726, + "loss": 3.1938, + "theoretical_loss": 3.780091903507206, + "tokens_seen": 698841088 + }, + { + "epoch": 2.03, + "learning_rate": 0.00039810431293881644, + "loss": 3.0582, + "theoretical_loss": 3.7800559010445567, + "tokens_seen": 698906624 + }, + { + "epoch": 2.03, + "learning_rate": 0.0003980942828485457, + "loss": 2.7882, + "theoretical_loss": 3.7800199029028345, + "tokens_seen": 698972160 + }, + { + "epoch": 2.03, + "learning_rate": 0.0003980842527582748, + "loss": 2.9789, + "theoretical_loss": 3.779983909081115, + "tokens_seen": 699037696 + }, + { + "epoch": 2.03, + "learning_rate": 0.00039807422266800404, + "loss": 2.9775, + "theoretical_loss": 3.7799479195784746, + "tokens_seen": 699103232 + }, + { + "epoch": 2.03, + "learning_rate": 0.00039806419257773317, + "loss": 3.027, + "theoretical_loss": 3.779911934393991, + "tokens_seen": 699168768 + }, + { + "epoch": 2.03, + "learning_rate": 0.0003980541624874624, + "loss": 3.0297, + "theoretical_loss": 3.7798759535267417, + "tokens_seen": 699234304 + }, + { + "epoch": 2.03, + "learning_rate": 0.0003980441323971916, + "loss": 2.949, + "theoretical_loss": 3.779839976975803, + "tokens_seen": 699299840 + }, + { + "epoch": 2.03, + "learning_rate": 0.00039803410230692077, + "loss": 2.8488, + "theoretical_loss": 3.779804004740253, + "tokens_seen": 699365376 + }, + { + "epoch": 2.03, + "learning_rate": 0.00039802407221664995, + "loss": 2.9867, + "theoretical_loss": 3.7797680368191706, + "tokens_seen": 699430912 + }, + { + "epoch": 2.03, + "learning_rate": 0.00039801404212637913, + "loss": 2.9055, + "theoretical_loss": 3.7797320732116337, + "tokens_seen": 699496448 + }, + { + "epoch": 2.03, + "learning_rate": 0.0003980040120361083, + "loss": 3.1272, + "theoretical_loss": 3.7796961139167204, + "tokens_seen": 699561984 + }, + { + "epoch": 2.03, + "objective/train/docs_used": 833268, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7306745052337646, + "objective/train/theoretical_loss": 3.7796781358862104, + "objective/train/tokens_used": 720054752, + "theoretical_loss": 3.7796781358862104, + "tokens_seen": 699594752 + }, + { + "epoch": 2.03, + "learning_rate": 0.00039799398194583754, + "loss": 2.9271, + "theoretical_loss": 3.7796601589335106, + "tokens_seen": 699627520 + }, + { + "epoch": 2.03, + "learning_rate": 0.00039798395185556667, + "loss": 2.8494, + "theoretical_loss": 3.7796242082610823, + "tokens_seen": 699693056 + }, + { + "epoch": 2.03, + "learning_rate": 0.0003979739217652959, + "loss": 2.9823, + "theoretical_loss": 3.7795882618985157, + "tokens_seen": 699758592 + }, + { + "epoch": 2.03, + "learning_rate": 0.00039796389167502503, + "loss": 2.9894, + "theoretical_loss": 3.779552319844891, + "tokens_seen": 699824128 + }, + { + "epoch": 2.03, + "learning_rate": 0.00039795386158475427, + "loss": 2.8972, + "theoretical_loss": 3.779516382099287, + "tokens_seen": 699889664 + }, + { + "epoch": 2.03, + "learning_rate": 0.00039794383149448345, + "loss": 2.8741, + "theoretical_loss": 3.7794804486607845, + "tokens_seen": 699955200 + }, + { + "epoch": 2.03, + "learning_rate": 0.00039793380140421263, + "loss": 2.907, + "theoretical_loss": 3.779444519528465, + "tokens_seen": 700020736 + }, + { + "epoch": 2.03, + "learning_rate": 0.0003979237713139418, + "loss": 2.9134, + "theoretical_loss": 3.779408594701408, + "tokens_seen": 700086272 + }, + { + "epoch": 2.03, + "learning_rate": 0.00039791374122367105, + "loss": 2.7774, + "theoretical_loss": 3.779372674178696, + "tokens_seen": 700151808 + }, + { + "epoch": 2.03, + "learning_rate": 0.0003979037111334002, + "loss": 3.0367, + "theoretical_loss": 3.7793367579594097, + "tokens_seen": 700217344 + }, + { + "epoch": 2.03, + "learning_rate": 0.0003978936810431294, + "loss": 2.8712, + "theoretical_loss": 3.7793008460426307, + "tokens_seen": 700282880 + }, + { + "epoch": 2.03, + "learning_rate": 0.0003978836509528586, + "loss": 2.9051, + "theoretical_loss": 3.779264938427441, + "tokens_seen": 700348416 + }, + { + "epoch": 2.03, + "learning_rate": 0.00039787362086258777, + "loss": 3.1493, + "theoretical_loss": 3.7792290351129236, + "tokens_seen": 700413952 + }, + { + "epoch": 2.03, + "learning_rate": 0.000397863590772317, + "loss": 2.8041, + "theoretical_loss": 3.7791931360981605, + "tokens_seen": 700479488 + }, + { + "epoch": 2.03, + "learning_rate": 0.00039785356068204613, + "loss": 3.0465, + "theoretical_loss": 3.7791572413822343, + "tokens_seen": 700545024 + }, + { + "epoch": 2.03, + "learning_rate": 0.00039784353059177537, + "loss": 2.8724, + "theoretical_loss": 3.7791213509642287, + "tokens_seen": 700610560 + }, + { + "epoch": 2.03, + "learning_rate": 0.0003978335005015045, + "loss": 2.8372, + "theoretical_loss": 3.7790854648432273, + "tokens_seen": 700676096 + }, + { + "epoch": 2.03, + "learning_rate": 0.00039782347041123373, + "loss": 3.0213, + "theoretical_loss": 3.779049583018313, + "tokens_seen": 700741632 + }, + { + "epoch": 2.03, + "learning_rate": 0.0003978134403209629, + "loss": 3.0222, + "theoretical_loss": 3.7790137054885697, + "tokens_seen": 700807168 + }, + { + "epoch": 2.03, + "learning_rate": 0.0003978034102306921, + "loss": 2.9475, + "theoretical_loss": 3.7789778322530827, + "tokens_seen": 700872704 + }, + { + "epoch": 2.03, + "learning_rate": 0.0003977933801404213, + "loss": 2.7373, + "theoretical_loss": 3.7789419633109365, + "tokens_seen": 700938240 + }, + { + "epoch": 2.03, + "learning_rate": 0.00039778335005015046, + "loss": 2.9236, + "theoretical_loss": 3.7789060986612144, + "tokens_seen": 701003776 + }, + { + "epoch": 2.03, + "learning_rate": 0.00039777331995987964, + "loss": 2.8003, + "theoretical_loss": 3.7788702383030035, + "tokens_seen": 701069312 + }, + { + "epoch": 2.03, + "learning_rate": 0.00039776328986960887, + "loss": 2.7636, + "theoretical_loss": 3.7788343822353876, + "tokens_seen": 701134848 + }, + { + "epoch": 2.03, + "learning_rate": 0.000397753259779338, + "loss": 2.7646, + "theoretical_loss": 3.778798530457453, + "tokens_seen": 701200384 + }, + { + "epoch": 2.03, + "objective/train/docs_used": 838443, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.2617812156677246, + "objective/train/theoretical_loss": 3.7787806061768308, + "objective/train/tokens_used": 721693152, + "theoretical_loss": 3.7787806061768308, + "tokens_seen": 701233152 + }, + { + "epoch": 2.03, + "learning_rate": 0.00039774322968906723, + "loss": 2.84, + "theoretical_loss": 3.7787626829682863, + "tokens_seen": 701265920 + }, + { + "epoch": 2.03, + "learning_rate": 0.0003977331995987964, + "loss": 2.831, + "theoretical_loss": 3.778726839766972, + "tokens_seen": 701331456 + }, + { + "epoch": 2.03, + "learning_rate": 0.0003977231695085256, + "loss": 2.8498, + "theoretical_loss": 3.7786910008525987, + "tokens_seen": 701396992 + }, + { + "epoch": 2.03, + "learning_rate": 0.0003977131394182548, + "loss": 2.8381, + "theoretical_loss": 3.778655166224252, + "tokens_seen": 701462528 + }, + { + "epoch": 2.03, + "learning_rate": 0.00039770310932798396, + "loss": 2.9054, + "theoretical_loss": 3.778619335881019, + "tokens_seen": 701528064 + }, + { + "epoch": 2.03, + "learning_rate": 0.00039769307923771314, + "loss": 3.1357, + "theoretical_loss": 3.7785835098219867, + "tokens_seen": 701593600 + }, + { + "epoch": 2.03, + "learning_rate": 0.0003976830491474424, + "loss": 2.7488, + "theoretical_loss": 3.7785476880462436, + "tokens_seen": 701659136 + }, + { + "epoch": 2.03, + "learning_rate": 0.0003976730190571715, + "loss": 3.1839, + "theoretical_loss": 3.778511870552877, + "tokens_seen": 701724672 + }, + { + "epoch": 2.03, + "learning_rate": 0.00039766298896690074, + "loss": 3.026, + "theoretical_loss": 3.778476057340975, + "tokens_seen": 701790208 + }, + { + "epoch": 2.03, + "learning_rate": 0.00039765295887662987, + "loss": 2.8715, + "theoretical_loss": 3.778440248409627, + "tokens_seen": 701855744 + }, + { + "epoch": 2.03, + "learning_rate": 0.0003976429287863591, + "loss": 2.9057, + "theoretical_loss": 3.7784044437579207, + "tokens_seen": 701921280 + }, + { + "epoch": 2.03, + "learning_rate": 0.0003976328986960883, + "loss": 2.9479, + "theoretical_loss": 3.7783686433849457, + "tokens_seen": 701986816 + }, + { + "epoch": 2.03, + "learning_rate": 0.00039762286860581746, + "loss": 2.8235, + "theoretical_loss": 3.778332847289791, + "tokens_seen": 702052352 + }, + { + "epoch": 2.03, + "learning_rate": 0.00039761283851554664, + "loss": 2.9134, + "theoretical_loss": 3.7782970554715467, + "tokens_seen": 702117888 + }, + { + "epoch": 2.03, + "learning_rate": 0.0003976028084252759, + "loss": 2.9239, + "theoretical_loss": 3.7782612679293015, + "tokens_seen": 702183424 + }, + { + "epoch": 2.03, + "learning_rate": 0.000397592778335005, + "loss": 2.7292, + "theoretical_loss": 3.778225484662147, + "tokens_seen": 702248960 + }, + { + "epoch": 2.03, + "learning_rate": 0.00039758274824473424, + "loss": 2.8728, + "theoretical_loss": 3.7781897056691722, + "tokens_seen": 702314496 + }, + { + "epoch": 2.03, + "learning_rate": 0.00039757271815446337, + "loss": 3.129, + "theoretical_loss": 3.778153930949469, + "tokens_seen": 702380032 + }, + { + "epoch": 2.03, + "learning_rate": 0.0003975626880641926, + "loss": 2.8147, + "theoretical_loss": 3.7781181605021277, + "tokens_seen": 702445568 + }, + { + "epoch": 2.03, + "learning_rate": 0.0003975526579739218, + "loss": 3.1122, + "theoretical_loss": 3.7780823943262396, + "tokens_seen": 702511104 + }, + { + "epoch": 2.03, + "learning_rate": 0.00039754262788365097, + "loss": 2.9351, + "theoretical_loss": 3.7780466324208968, + "tokens_seen": 702576640 + }, + { + "epoch": 2.03, + "learning_rate": 0.00039753259779338015, + "loss": 2.5674, + "theoretical_loss": 3.77801087478519, + "tokens_seen": 702642176 + }, + { + "epoch": 2.03, + "learning_rate": 0.00039752256770310933, + "loss": 3.0548, + "theoretical_loss": 3.7779751214182125, + "tokens_seen": 702707712 + }, + { + "epoch": 2.03, + "learning_rate": 0.0003975125376128385, + "loss": 2.8329, + "theoretical_loss": 3.777939372319056, + "tokens_seen": 702773248 + }, + { + "epoch": 2.03, + "learning_rate": 0.00039750250752256774, + "loss": 2.9937, + "theoretical_loss": 3.777903627486813, + "tokens_seen": 702838784 + }, + { + "epoch": 2.03, + "objective/train/docs_used": 839055, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7638256549835205, + "objective/train/theoretical_loss": 3.777885756670501, + "objective/train/tokens_used": 723331552, + "theoretical_loss": 3.777885756670501, + "tokens_seen": 702871552 + }, + { + "epoch": 2.03, + "learning_rate": 0.00039749247743229687, + "loss": 2.9961, + "theoretical_loss": 3.7778678869205766, + "tokens_seen": 702904320 + }, + { + "epoch": 2.03, + "learning_rate": 0.0003974824473420261, + "loss": 2.8954, + "theoretical_loss": 3.7778321506194406, + "tokens_seen": 702969856 + }, + { + "epoch": 2.03, + "learning_rate": 0.00039747241725175523, + "loss": 2.8431, + "theoretical_loss": 3.7777964185824975, + "tokens_seen": 703035392 + }, + { + "epoch": 2.03, + "learning_rate": 0.00039746238716148447, + "loss": 2.9681, + "theoretical_loss": 3.7777606908088415, + "tokens_seen": 703100928 + }, + { + "epoch": 2.03, + "learning_rate": 0.00039745235707121365, + "loss": 3.0087, + "theoretical_loss": 3.7777249672975666, + "tokens_seen": 703166464 + }, + { + "epoch": 2.03, + "learning_rate": 0.00039744232698094283, + "loss": 3.2902, + "theoretical_loss": 3.7776892480477673, + "tokens_seen": 703232000 + }, + { + "epoch": 2.03, + "learning_rate": 0.000397432296890672, + "loss": 3.011, + "theoretical_loss": 3.7776535330585377, + "tokens_seen": 703297536 + }, + { + "epoch": 2.03, + "learning_rate": 0.00039742226680040125, + "loss": 2.8944, + "theoretical_loss": 3.777617822328973, + "tokens_seen": 703363072 + }, + { + "epoch": 2.03, + "learning_rate": 0.0003974122367101304, + "loss": 2.918, + "theoretical_loss": 3.7775821158581686, + "tokens_seen": 703428608 + }, + { + "epoch": 2.03, + "learning_rate": 0.0003974022066198596, + "loss": 2.7816, + "theoretical_loss": 3.777546413645219, + "tokens_seen": 703494144 + }, + { + "epoch": 2.03, + "learning_rate": 0.00039739217652958874, + "loss": 3.0054, + "theoretical_loss": 3.777510715689221, + "tokens_seen": 703559680 + }, + { + "epoch": 2.03, + "learning_rate": 0.00039738214643931797, + "loss": 2.9605, + "theoretical_loss": 3.77747502198927, + "tokens_seen": 703625216 + }, + { + "epoch": 2.03, + "learning_rate": 0.00039737211634904715, + "loss": 2.8161, + "theoretical_loss": 3.7774393325444615, + "tokens_seen": 703690752 + }, + { + "epoch": 2.03, + "learning_rate": 0.00039736208625877633, + "loss": 2.9456, + "theoretical_loss": 3.7774036473538937, + "tokens_seen": 703756288 + }, + { + "epoch": 2.03, + "learning_rate": 0.0003973520561685055, + "loss": 3.0258, + "theoretical_loss": 3.777367966416662, + "tokens_seen": 703821824 + }, + { + "epoch": 2.03, + "learning_rate": 0.0003973420260782347, + "loss": 2.979, + "theoretical_loss": 3.777332289731864, + "tokens_seen": 703887360 + }, + { + "epoch": 2.03, + "learning_rate": 0.0003973319959879639, + "loss": 2.9257, + "theoretical_loss": 3.777296617298597, + "tokens_seen": 703952896 + }, + { + "epoch": 2.03, + "learning_rate": 0.0003973219658976931, + "loss": 2.7951, + "theoretical_loss": 3.7772609491159588, + "tokens_seen": 704018432 + }, + { + "epoch": 2.03, + "learning_rate": 0.00039731193580742224, + "loss": 2.9469, + "theoretical_loss": 3.7772252851830466, + "tokens_seen": 704083968 + }, + { + "epoch": 2.03, + "learning_rate": 0.0003973019057171515, + "loss": 3.1696, + "theoretical_loss": 3.777189625498959, + "tokens_seen": 704149504 + }, + { + "epoch": 2.03, + "learning_rate": 0.0003972918756268806, + "loss": 2.9136, + "theoretical_loss": 3.7771539700627956, + "tokens_seen": 704215040 + }, + { + "epoch": 2.03, + "learning_rate": 0.00039728184553660984, + "loss": 2.8302, + "theoretical_loss": 3.7771183188736535, + "tokens_seen": 704280576 + }, + { + "epoch": 2.03, + "learning_rate": 0.000397271815446339, + "loss": 2.9683, + "theoretical_loss": 3.777082671930632, + "tokens_seen": 704346112 + }, + { + "epoch": 2.03, + "learning_rate": 0.0003972617853560682, + "loss": 2.8791, + "theoretical_loss": 3.77704702923283, + "tokens_seen": 704411648 + }, + { + "epoch": 2.03, + "learning_rate": 0.0003972517552657974, + "loss": 2.969, + "theoretical_loss": 3.7770113907793483, + "tokens_seen": 704477184 + }, + { + "epoch": 2.03, + "objective/train/docs_used": 840230, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.1280670166015625, + "objective/train/theoretical_loss": 3.776993573143946, + "objective/train/tokens_used": 724969952, + "theoretical_loss": 3.776993573143946, + "tokens_seen": 704509952 + }, + { + "epoch": 2.03, + "learning_rate": 0.0003972417251755266, + "loss": 2.7157, + "theoretical_loss": 3.776975756569286, + "tokens_seen": 704542720 + }, + { + "epoch": 2.03, + "learning_rate": 0.00039723169508525574, + "loss": 2.8356, + "theoretical_loss": 3.776940126601743, + "tokens_seen": 704608256 + }, + { + "epoch": 2.03, + "learning_rate": 0.000397221664994985, + "loss": 2.8503, + "theoretical_loss": 3.7769045008758204, + "tokens_seen": 704673792 + }, + { + "epoch": 2.03, + "learning_rate": 0.0003972116349047141, + "loss": 2.9024, + "theoretical_loss": 3.7768688793906184, + "tokens_seen": 704739328 + }, + { + "epoch": 2.03, + "learning_rate": 0.00039720160481444334, + "loss": 3.0062, + "theoretical_loss": 3.776833262145237, + "tokens_seen": 704804864 + }, + { + "epoch": 2.03, + "learning_rate": 0.0003971915747241725, + "loss": 3.1082, + "theoretical_loss": 3.776797649138779, + "tokens_seen": 704870400 + }, + { + "epoch": 2.03, + "learning_rate": 0.0003971815446339017, + "loss": 2.9018, + "theoretical_loss": 3.776762040370345, + "tokens_seen": 704935936 + }, + { + "epoch": 2.03, + "learning_rate": 0.0003971715145436309, + "loss": 2.7978, + "theoretical_loss": 3.776726435839037, + "tokens_seen": 705001472 + }, + { + "epoch": 2.03, + "learning_rate": 0.00039716148445336007, + "loss": 2.8779, + "theoretical_loss": 3.776690835543956, + "tokens_seen": 705067008 + }, + { + "epoch": 2.03, + "learning_rate": 0.00039715145436308925, + "loss": 2.9387, + "theoretical_loss": 3.7766552394842057, + "tokens_seen": 705132544 + }, + { + "epoch": 2.03, + "learning_rate": 0.0003971414242728185, + "loss": 2.6552, + "theoretical_loss": 3.7766196476588885, + "tokens_seen": 705198080 + }, + { + "epoch": 2.03, + "learning_rate": 0.00039713139418254766, + "loss": 3.0808, + "theoretical_loss": 3.7765840600671066, + "tokens_seen": 705263616 + }, + { + "epoch": 2.03, + "learning_rate": 0.00039712136409227684, + "loss": 2.9143, + "theoretical_loss": 3.7765484767079633, + "tokens_seen": 705329152 + }, + { + "epoch": 2.03, + "learning_rate": 0.0003971113340020061, + "loss": 3.0465, + "theoretical_loss": 3.776512897580562, + "tokens_seen": 705394688 + }, + { + "epoch": 2.03, + "learning_rate": 0.0003971013039117352, + "loss": 2.8223, + "theoretical_loss": 3.7764773226840065, + "tokens_seen": 705460224 + }, + { + "epoch": 2.03, + "learning_rate": 0.00039709127382146444, + "loss": 2.892, + "theoretical_loss": 3.7764417520174005, + "tokens_seen": 705525760 + }, + { + "epoch": 2.03, + "learning_rate": 0.00039708124373119357, + "loss": 2.7686, + "theoretical_loss": 3.7764061855798485, + "tokens_seen": 705591296 + }, + { + "epoch": 2.03, + "learning_rate": 0.0003970712136409228, + "loss": 2.8557, + "theoretical_loss": 3.776370623370455, + "tokens_seen": 705656832 + }, + { + "epoch": 2.03, + "learning_rate": 0.000397061183550652, + "loss": 2.9014, + "theoretical_loss": 3.7763350653883236, + "tokens_seen": 705722368 + }, + { + "epoch": 2.03, + "learning_rate": 0.00039705115346038117, + "loss": 2.8048, + "theoretical_loss": 3.776299511632561, + "tokens_seen": 705787904 + }, + { + "epoch": 2.03, + "learning_rate": 0.00039704112337011035, + "loss": 2.9953, + "theoretical_loss": 3.7762639621022718, + "tokens_seen": 705853440 + }, + { + "epoch": 2.03, + "learning_rate": 0.00039703109327983953, + "loss": 3.0841, + "theoretical_loss": 3.7762284167965614, + "tokens_seen": 705918976 + }, + { + "epoch": 2.03, + "learning_rate": 0.0003970210631895687, + "loss": 2.9048, + "theoretical_loss": 3.7761928757145355, + "tokens_seen": 705984512 + }, + { + "epoch": 2.03, + "learning_rate": 0.00039701103309929794, + "loss": 2.7224, + "theoretical_loss": 3.7761573388553007, + "tokens_seen": 706050048 + }, + { + "epoch": 2.03, + "learning_rate": 0.00039700100300902707, + "loss": 2.8874, + "theoretical_loss": 3.776121806217963, + "tokens_seen": 706115584 + }, + { + "epoch": 2.03, + "objective/train/docs_used": 840818, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.0678188800811768, + "objective/train/theoretical_loss": 3.7761040414822267, + "objective/train/tokens_used": 726608352, + "theoretical_loss": 3.7761040414822267, + "tokens_seen": 706148352 + }, + { + "epoch": 2.03, + "learning_rate": 0.0003969909729187563, + "loss": 2.97, + "theoretical_loss": 3.776086277801629, + "tokens_seen": 706181120 + }, + { + "epoch": 2.03, + "learning_rate": 0.00039698094282848543, + "loss": 2.8323, + "theoretical_loss": 3.776050753605406, + "tokens_seen": 706246656 + }, + { + "epoch": 2.03, + "learning_rate": 0.00039697091273821467, + "loss": 2.9463, + "theoretical_loss": 3.776015233628401, + "tokens_seen": 706312192 + }, + { + "epoch": 2.03, + "learning_rate": 0.00039696088264794385, + "loss": 3.0658, + "theoretical_loss": 3.7759797178697214, + "tokens_seen": 706377728 + }, + { + "epoch": 2.03, + "learning_rate": 0.00039695085255767303, + "loss": 3.0671, + "theoretical_loss": 3.775944206328475, + "tokens_seen": 706443264 + }, + { + "epoch": 2.03, + "learning_rate": 0.0003969408224674022, + "loss": 2.9199, + "theoretical_loss": 3.77590869900377, + "tokens_seen": 706508800 + }, + { + "epoch": 2.03, + "learning_rate": 0.00039693079237713145, + "loss": 2.8948, + "theoretical_loss": 3.7758731958947145, + "tokens_seen": 706574336 + }, + { + "epoch": 2.03, + "learning_rate": 0.0003969207622868606, + "loss": 2.8665, + "theoretical_loss": 3.7758376970004166, + "tokens_seen": 706639872 + }, + { + "epoch": 2.03, + "learning_rate": 0.0003969107321965898, + "loss": 2.8945, + "theoretical_loss": 3.775802202319986, + "tokens_seen": 706705408 + }, + { + "epoch": 2.03, + "learning_rate": 0.00039690070210631894, + "loss": 3.097, + "theoretical_loss": 3.7757667118525315, + "tokens_seen": 706770944 + }, + { + "epoch": 2.03, + "learning_rate": 0.00039689067201604817, + "loss": 2.9045, + "theoretical_loss": 3.7757312255971622, + "tokens_seen": 706836480 + }, + { + "epoch": 2.03, + "learning_rate": 0.00039688064192577735, + "loss": 2.7058, + "theoretical_loss": 3.775695743552988, + "tokens_seen": 706902016 + }, + { + "epoch": 2.03, + "learning_rate": 0.00039687061183550653, + "loss": 3.0764, + "theoretical_loss": 3.775660265719118, + "tokens_seen": 706967552 + }, + { + "epoch": 2.03, + "learning_rate": 0.0003968605817452357, + "loss": 2.9457, + "theoretical_loss": 3.7756247920946633, + "tokens_seen": 707033088 + }, + { + "epoch": 2.03, + "learning_rate": 0.0003968505516549649, + "loss": 2.9319, + "theoretical_loss": 3.7755893226787345, + "tokens_seen": 707098624 + }, + { + "epoch": 2.03, + "learning_rate": 0.0003968405215646941, + "loss": 2.9837, + "theoretical_loss": 3.7755538574704417, + "tokens_seen": 707164160 + }, + { + "epoch": 2.03, + "learning_rate": 0.0003968304914744233, + "loss": 3.0032, + "theoretical_loss": 3.7755183964688968, + "tokens_seen": 707229696 + }, + { + "epoch": 2.03, + "learning_rate": 0.00039682046138415244, + "loss": 2.7542, + "theoretical_loss": 3.7754829396732097, + "tokens_seen": 707295232 + }, + { + "epoch": 2.03, + "learning_rate": 0.0003968104312938817, + "loss": 2.9074, + "theoretical_loss": 3.7754474870824923, + "tokens_seen": 707360768 + }, + { + "epoch": 2.03, + "learning_rate": 0.0003968004012036108, + "loss": 2.716, + "theoretical_loss": 3.7754120386958574, + "tokens_seen": 707426304 + }, + { + "epoch": 2.03, + "learning_rate": 0.00039679037111334004, + "loss": 2.7301, + "theoretical_loss": 3.775376594512416, + "tokens_seen": 707491840 + }, + { + "epoch": 2.03, + "learning_rate": 0.0003967803410230692, + "loss": 3.0049, + "theoretical_loss": 3.7753411545312803, + "tokens_seen": 707557376 + }, + { + "epoch": 2.03, + "learning_rate": 0.0003967703109327984, + "loss": 3.0008, + "theoretical_loss": 3.7753057187515644, + "tokens_seen": 707622912 + }, + { + "epoch": 2.03, + "learning_rate": 0.0003967602808425276, + "loss": 3.1812, + "theoretical_loss": 3.7752702871723796, + "tokens_seen": 707688448 + }, + { + "epoch": 2.03, + "learning_rate": 0.0003967502507522568, + "loss": 2.9954, + "theoretical_loss": 3.77523485979284, + "tokens_seen": 707753984 + }, + { + "epoch": 2.03, + "objective/train/docs_used": 842171, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9438395500183105, + "objective/train/theoretical_loss": 3.77521714767766, + "objective/train/tokens_used": 728246752, + "theoretical_loss": 3.77521714767766, + "tokens_seen": 707786752 + }, + { + "epoch": 2.03, + "learning_rate": 0.00039674022066198594, + "loss": 2.8499, + "theoretical_loss": 3.7751994366120583, + "tokens_seen": 707819520 + }, + { + "epoch": 2.03, + "learning_rate": 0.0003967301905717152, + "loss": 2.9356, + "theoretical_loss": 3.7751640176291485, + "tokens_seen": 707885056 + }, + { + "epoch": 2.03, + "learning_rate": 0.0003967201604814443, + "loss": 3.1708, + "theoretical_loss": 3.7751286028432247, + "tokens_seen": 707950592 + }, + { + "epoch": 2.03, + "learning_rate": 0.00039671013039117354, + "loss": 2.9933, + "theoretical_loss": 3.7750931922534017, + "tokens_seen": 708016128 + }, + { + "epoch": 2.03, + "learning_rate": 0.0003967001003009027, + "loss": 2.985, + "theoretical_loss": 3.7750577858587926, + "tokens_seen": 708081664 + }, + { + "epoch": 2.03, + "learning_rate": 0.0003966900702106319, + "loss": 2.7421, + "theoretical_loss": 3.775022383658513, + "tokens_seen": 708147200 + }, + { + "epoch": 2.03, + "learning_rate": 0.0003966800401203611, + "loss": 3.0504, + "theoretical_loss": 3.774986985651678, + "tokens_seen": 708212736 + }, + { + "epoch": 2.03, + "learning_rate": 0.00039667001003009027, + "loss": 2.8545, + "theoretical_loss": 3.774951591837402, + "tokens_seen": 708278272 + }, + { + "epoch": 2.03, + "learning_rate": 0.00039665997993981945, + "loss": 2.7481, + "theoretical_loss": 3.7749162022148024, + "tokens_seen": 708343808 + }, + { + "epoch": 2.03, + "learning_rate": 0.0003966499498495487, + "loss": 2.7234, + "theoretical_loss": 3.774880816782993, + "tokens_seen": 708409344 + }, + { + "epoch": 2.03, + "learning_rate": 0.0003966399197592778, + "loss": 2.9445, + "theoretical_loss": 3.774845435541091, + "tokens_seen": 708474880 + }, + { + "epoch": 2.03, + "learning_rate": 0.00039662988966900704, + "loss": 2.8104, + "theoretical_loss": 3.7748100584882134, + "tokens_seen": 708540416 + }, + { + "epoch": 2.03, + "learning_rate": 0.00039661985957873617, + "loss": 2.9856, + "theoretical_loss": 3.774774685623475, + "tokens_seen": 708605952 + }, + { + "epoch": 2.03, + "learning_rate": 0.0003966098294884654, + "loss": 2.8665, + "theoretical_loss": 3.774739316945994, + "tokens_seen": 708671488 + }, + { + "epoch": 2.03, + "learning_rate": 0.0003965997993981946, + "loss": 2.8147, + "theoretical_loss": 3.774703952454888, + "tokens_seen": 708737024 + }, + { + "epoch": 2.03, + "learning_rate": 0.00039658976930792377, + "loss": 3.0254, + "theoretical_loss": 3.7746685921492737, + "tokens_seen": 708802560 + }, + { + "epoch": 2.03, + "learning_rate": 0.00039657973921765295, + "loss": 3.0298, + "theoretical_loss": 3.7746332360282677, + "tokens_seen": 708868096 + }, + { + "epoch": 2.03, + "learning_rate": 0.0003965697091273822, + "loss": 3.0365, + "theoretical_loss": 3.77459788409099, + "tokens_seen": 708933632 + }, + { + "epoch": 2.03, + "learning_rate": 0.0003965596790371113, + "loss": 3.0917, + "theoretical_loss": 3.7745625363365582, + "tokens_seen": 708999168 + }, + { + "epoch": 2.04, + "learning_rate": 0.00039654964894684055, + "loss": 3.0597, + "theoretical_loss": 3.7745271927640904, + "tokens_seen": 709064704 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003965396188565697, + "loss": 2.8429, + "theoretical_loss": 3.7744918533727057, + "tokens_seen": 709130240 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003965295887662989, + "loss": 2.9297, + "theoretical_loss": 3.7744565181615233, + "tokens_seen": 709195776 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003965195586760281, + "loss": 2.9209, + "theoretical_loss": 3.7744211871296613, + "tokens_seen": 709261312 + }, + { + "epoch": 2.04, + "learning_rate": 0.00039650952858575727, + "loss": 2.7967, + "theoretical_loss": 3.774385860276241, + "tokens_seen": 709326848 + }, + { + "epoch": 2.04, + "learning_rate": 0.00039649949849548645, + "loss": 3.0973, + "theoretical_loss": 3.7743505376003816, + "tokens_seen": 709392384 + }, + { + "epoch": 2.04, + "objective/train/docs_used": 842767, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9267566204071045, + "objective/train/theoretical_loss": 3.774332877828762, + "objective/train/tokens_used": 729885152, + "theoretical_loss": 3.774332877828762, + "tokens_seen": 709425152 + }, + { + "epoch": 2.04, + "learning_rate": 0.00039648946840521563, + "loss": 2.8282, + "theoretical_loss": 3.7743152191012026, + "tokens_seen": 709457920 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003964794383149448, + "loss": 2.88, + "theoretical_loss": 3.7742799047778255, + "tokens_seen": 709523456 + }, + { + "epoch": 2.04, + "learning_rate": 0.00039646940822467405, + "loss": 2.8465, + "theoretical_loss": 3.77424459462937, + "tokens_seen": 709588992 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003964593781344032, + "loss": 2.949, + "theoretical_loss": 3.774209288654957, + "tokens_seen": 709654528 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003964493480441324, + "loss": 2.9575, + "theoretical_loss": 3.7741739868537088, + "tokens_seen": 709720064 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003964393179538616, + "loss": 2.8422, + "theoretical_loss": 3.7741386892247455, + "tokens_seen": 709785600 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003964292878635908, + "loss": 3.0997, + "theoretical_loss": 3.77410339576719, + "tokens_seen": 709851136 + }, + { + "epoch": 2.04, + "learning_rate": 0.00039641925777331996, + "loss": 2.8329, + "theoretical_loss": 3.774068106480163, + "tokens_seen": 709916672 + }, + { + "epoch": 2.04, + "learning_rate": 0.00039640922768304914, + "loss": 2.9717, + "theoretical_loss": 3.774032821362788, + "tokens_seen": 709982208 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003963991975927783, + "loss": 2.9319, + "theoretical_loss": 3.773997540414187, + "tokens_seen": 710047744 + }, + { + "epoch": 2.04, + "learning_rate": 0.00039638916750250755, + "loss": 2.8197, + "theoretical_loss": 3.773962263633482, + "tokens_seen": 710113280 + }, + { + "epoch": 2.04, + "learning_rate": 0.00039637913741223673, + "loss": 2.7053, + "theoretical_loss": 3.7739269910197972, + "tokens_seen": 710178816 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003963691073219659, + "loss": 2.9991, + "theoretical_loss": 3.7738917225722557, + "tokens_seen": 710244352 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003963590772316951, + "loss": 2.995, + "theoretical_loss": 3.7738564582899805, + "tokens_seen": 710309888 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003963490471414243, + "loss": 3.0643, + "theoretical_loss": 3.7738211981720964, + "tokens_seen": 710375424 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003963390170511535, + "loss": 3.0085, + "theoretical_loss": 3.7737859422177262, + "tokens_seen": 710440960 + }, + { + "epoch": 2.04, + "learning_rate": 0.00039632898696088264, + "loss": 2.9135, + "theoretical_loss": 3.773750690425995, + "tokens_seen": 710506496 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003963189568706119, + "loss": 2.8237, + "theoretical_loss": 3.7737154427960276, + "tokens_seen": 710572032 + }, + { + "epoch": 2.04, + "learning_rate": 0.000396308926780341, + "loss": 2.9719, + "theoretical_loss": 3.7736801993269484, + "tokens_seen": 710637568 + }, + { + "epoch": 2.04, + "learning_rate": 0.00039629889669007024, + "loss": 2.8327, + "theoretical_loss": 3.7736449600178834, + "tokens_seen": 710703104 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003962888665997994, + "loss": 2.8824, + "theoretical_loss": 3.773609724867957, + "tokens_seen": 710768640 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003962788365095286, + "loss": 2.8354, + "theoretical_loss": 3.7735744938762954, + "tokens_seen": 710834176 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003962688064192578, + "loss": 3.2813, + "theoretical_loss": 3.7735392670420254, + "tokens_seen": 710899712 + }, + { + "epoch": 2.04, + "learning_rate": 0.000396258776328987, + "loss": 2.8482, + "theoretical_loss": 3.7735040443642713, + "tokens_seen": 710965248 + }, + { + "epoch": 2.04, + "learning_rate": 0.00039624874623871614, + "loss": 2.9067, + "theoretical_loss": 3.773468825842161, + "tokens_seen": 711030784 + }, + { + "epoch": 2.04, + "objective/train/docs_used": 844300, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.3437087535858154, + "objective/train/theoretical_loss": 3.7734512181391993, + "objective/train/tokens_used": 731523552, + "theoretical_loss": 3.7734512181391993, + "tokens_seen": 711063552 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003962387161484454, + "loss": 3.113, + "theoretical_loss": 3.773433611474821, + "tokens_seen": 711096320 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003962286860581745, + "loss": 2.9326, + "theoretical_loss": 3.773398401261378, + "tokens_seen": 711161856 + }, + { + "epoch": 2.04, + "learning_rate": 0.00039621865596790374, + "loss": 3.1375, + "theoretical_loss": 3.7733631952009596, + "tokens_seen": 711227392 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003962086258776329, + "loss": 2.7491, + "theoretical_loss": 3.773327993292693, + "tokens_seen": 711292928 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003961985957873621, + "loss": 2.9599, + "theoretical_loss": 3.7732927955357063, + "tokens_seen": 711358464 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003961885656970913, + "loss": 2.7884, + "theoretical_loss": 3.773257601929128, + "tokens_seen": 711424000 + }, + { + "epoch": 2.04, + "learning_rate": 0.00039617853560682047, + "loss": 2.8457, + "theoretical_loss": 3.7732224124720855, + "tokens_seen": 711489536 + }, + { + "epoch": 2.04, + "learning_rate": 0.00039616850551654965, + "loss": 2.9076, + "theoretical_loss": 3.7731872271637075, + "tokens_seen": 711555072 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003961584754262789, + "loss": 2.8034, + "theoretical_loss": 3.7731520460031236, + "tokens_seen": 711620608 + }, + { + "epoch": 2.04, + "learning_rate": 0.000396148445336008, + "loss": 2.7777, + "theoretical_loss": 3.7731168689894625, + "tokens_seen": 711686144 + }, + { + "epoch": 2.04, + "learning_rate": 0.00039613841524573724, + "loss": 2.9276, + "theoretical_loss": 3.7730816961218534, + "tokens_seen": 711751680 + }, + { + "epoch": 2.04, + "learning_rate": 0.00039612838515546637, + "loss": 2.7608, + "theoretical_loss": 3.7730465273994263, + "tokens_seen": 711817216 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003961183550651956, + "loss": 2.8722, + "theoretical_loss": 3.773011362821311, + "tokens_seen": 711882752 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003961083249749248, + "loss": 2.7365, + "theoretical_loss": 3.772976202386637, + "tokens_seen": 711948288 + }, + { + "epoch": 2.04, + "learning_rate": 0.00039609829488465397, + "loss": 2.8973, + "theoretical_loss": 3.772941046094536, + "tokens_seen": 712013824 + }, + { + "epoch": 2.04, + "learning_rate": 0.00039608826479438315, + "loss": 2.7796, + "theoretical_loss": 3.772905893944138, + "tokens_seen": 712079360 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003960782347041124, + "loss": 2.7532, + "theoretical_loss": 3.7728707459345734, + "tokens_seen": 712144896 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003960682046138415, + "loss": 2.973, + "theoretical_loss": 3.7728356020649745, + "tokens_seen": 712210432 + }, + { + "epoch": 2.04, + "learning_rate": 0.00039605817452357075, + "loss": 3.0538, + "theoretical_loss": 3.772800462334472, + "tokens_seen": 712275968 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003960481444332999, + "loss": 2.759, + "theoretical_loss": 3.7727653267421983, + "tokens_seen": 712341504 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003960381143430291, + "loss": 2.8805, + "theoretical_loss": 3.772730195287285, + "tokens_seen": 712407040 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003960280842527583, + "loss": 2.8023, + "theoretical_loss": 3.772695067968865, + "tokens_seen": 712472576 + }, + { + "epoch": 2.04, + "learning_rate": 0.00039601805416248747, + "loss": 2.9331, + "theoretical_loss": 3.7726599447860694, + "tokens_seen": 712538112 + }, + { + "epoch": 2.04, + "learning_rate": 0.00039600802407221665, + "loss": 2.9116, + "theoretical_loss": 3.772624825738032, + "tokens_seen": 712603648 + }, + { + "epoch": 2.04, + "learning_rate": 0.00039599799398194583, + "loss": 3.1991, + "theoretical_loss": 3.772589710823886, + "tokens_seen": 712669184 + }, + { + "epoch": 2.04, + "objective/train/docs_used": 844668, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.137606143951416, + "objective/train/theoretical_loss": 3.772572154916751, + "objective/train/tokens_used": 733161952, + "theoretical_loss": 3.772572154916751, + "tokens_seen": 712701952 + }, + { + "epoch": 2.04, + "learning_rate": 0.000395987963891675, + "loss": 3.0682, + "theoretical_loss": 3.772554600042764, + "tokens_seen": 712734720 + }, + { + "epoch": 2.04, + "learning_rate": 0.00039597793380140425, + "loss": 2.923, + "theoretical_loss": 3.7725194933938004, + "tokens_seen": 712800256 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003959679037111334, + "loss": 3.009, + "theoretical_loss": 3.7724843908761283, + "tokens_seen": 712865792 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003959578736208626, + "loss": 2.9241, + "theoretical_loss": 3.772449292488883, + "tokens_seen": 712931328 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003959478435305918, + "loss": 2.7148, + "theoretical_loss": 3.772414198231197, + "tokens_seen": 712996864 + }, + { + "epoch": 2.04, + "learning_rate": 0.000395937813440321, + "loss": 2.6896, + "theoretical_loss": 3.7723791081022062, + "tokens_seen": 713062400 + }, + { + "epoch": 2.04, + "learning_rate": 0.00039592778335005016, + "loss": 2.7487, + "theoretical_loss": 3.7723440221010454, + "tokens_seen": 713127936 + }, + { + "epoch": 2.04, + "learning_rate": 0.00039591775325977934, + "loss": 2.9316, + "theoretical_loss": 3.7723089402268495, + "tokens_seen": 713193472 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003959077231695085, + "loss": 2.8535, + "theoretical_loss": 3.772273862478754, + "tokens_seen": 713259008 + }, + { + "epoch": 2.04, + "learning_rate": 0.00039589769307923775, + "loss": 2.7874, + "theoretical_loss": 3.7722387888558946, + "tokens_seen": 713324544 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003958876629889669, + "loss": 2.7757, + "theoretical_loss": 3.7722037193574067, + "tokens_seen": 713390080 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003958776328986961, + "loss": 2.6952, + "theoretical_loss": 3.7721686539824275, + "tokens_seen": 713455616 + }, + { + "epoch": 2.04, + "learning_rate": 0.00039586760280842524, + "loss": 2.8004, + "theoretical_loss": 3.7721335927300927, + "tokens_seen": 713521152 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003958575727181545, + "loss": 2.8383, + "theoretical_loss": 3.772098535599539, + "tokens_seen": 713586688 + }, + { + "epoch": 2.04, + "learning_rate": 0.00039584754262788366, + "loss": 3.1454, + "theoretical_loss": 3.772063482589904, + "tokens_seen": 713652224 + }, + { + "epoch": 2.04, + "learning_rate": 0.00039583751253761284, + "loss": 2.7299, + "theoretical_loss": 3.7720284337003243, + "tokens_seen": 713717760 + }, + { + "epoch": 2.04, + "learning_rate": 0.000395827482447342, + "loss": 2.9994, + "theoretical_loss": 3.7719933889299377, + "tokens_seen": 713783296 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003958174523570712, + "loss": 2.7725, + "theoretical_loss": 3.771958348277882, + "tokens_seen": 713848832 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003958074222668004, + "loss": 2.7891, + "theoretical_loss": 3.7719233117432944, + "tokens_seen": 713914368 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003957973921765296, + "loss": 2.8777, + "theoretical_loss": 3.771888279325314, + "tokens_seen": 713979904 + }, + { + "epoch": 2.04, + "learning_rate": 0.00039578736208625875, + "loss": 2.9531, + "theoretical_loss": 3.771853251023079, + "tokens_seen": 714045440 + }, + { + "epoch": 2.04, + "learning_rate": 0.000395777331995988, + "loss": 2.9526, + "theoretical_loss": 3.7718182268357285, + "tokens_seen": 714110976 + }, + { + "epoch": 2.04, + "learning_rate": 0.00039576730190571716, + "loss": 2.879, + "theoretical_loss": 3.7717832067624015, + "tokens_seen": 714176512 + }, + { + "epoch": 2.04, + "learning_rate": 0.00039575727181544634, + "loss": 2.9837, + "theoretical_loss": 3.771748190802237, + "tokens_seen": 714242048 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003957472417251755, + "loss": 2.878, + "theoretical_loss": 3.771713178954375, + "tokens_seen": 714307584 + }, + { + "epoch": 2.04, + "objective/train/docs_used": 846114, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8042542934417725, + "objective/train/theoretical_loss": 3.771695674572288, + "objective/train/tokens_used": 734800352, + "theoretical_loss": 3.771695674572288, + "tokens_seen": 714340352 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003957372116349047, + "loss": 2.8705, + "theoretical_loss": 3.7716781712179546, + "tokens_seen": 714373120 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003957271815446339, + "loss": 2.8951, + "theoretical_loss": 3.7716431675921167, + "tokens_seen": 714438656 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003957171514543631, + "loss": 3.003, + "theoretical_loss": 3.7716081680760007, + "tokens_seen": 714504192 + }, + { + "epoch": 2.04, + "learning_rate": 0.00039570712136409225, + "loss": 2.734, + "theoretical_loss": 3.7715731726687487, + "tokens_seen": 714569728 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003956970912738215, + "loss": 2.9267, + "theoretical_loss": 3.7715381813694995, + "tokens_seen": 714635264 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003956870611835506, + "loss": 2.9854, + "theoretical_loss": 3.771503194177396, + "tokens_seen": 714700800 + }, + { + "epoch": 2.04, + "learning_rate": 0.00039567703109327985, + "loss": 2.7856, + "theoretical_loss": 3.771468211091579, + "tokens_seen": 714766336 + }, + { + "epoch": 2.04, + "learning_rate": 0.00039566700100300903, + "loss": 2.7446, + "theoretical_loss": 3.77143323211119, + "tokens_seen": 714831872 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003956569709127382, + "loss": 2.6806, + "theoretical_loss": 3.7713982572353704, + "tokens_seen": 714897408 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003956469408224674, + "loss": 2.8043, + "theoretical_loss": 3.771363286463263, + "tokens_seen": 714962944 + }, + { + "epoch": 2.04, + "learning_rate": 0.00039563691073219657, + "loss": 3.0227, + "theoretical_loss": 3.77132831979401, + "tokens_seen": 715028480 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003956268806419258, + "loss": 2.8135, + "theoretical_loss": 3.7712933572267544, + "tokens_seen": 715094016 + }, + { + "epoch": 2.04, + "learning_rate": 0.000395616850551655, + "loss": 2.5729, + "theoretical_loss": 3.7712583987606387, + "tokens_seen": 715159552 + }, + { + "epoch": 2.04, + "learning_rate": 0.00039560682046138417, + "loss": 2.8661, + "theoretical_loss": 3.7712234443948063, + "tokens_seen": 715225088 + }, + { + "epoch": 2.04, + "learning_rate": 0.00039559679037111335, + "loss": 2.8424, + "theoretical_loss": 3.7711884941284004, + "tokens_seen": 715290624 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003955867602808426, + "loss": 2.8453, + "theoretical_loss": 3.771153547960565, + "tokens_seen": 715356160 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003955767301905717, + "loss": 3.137, + "theoretical_loss": 3.771118605890444, + "tokens_seen": 715421696 + }, + { + "epoch": 2.04, + "learning_rate": 0.00039556670010030095, + "loss": 3.0339, + "theoretical_loss": 3.771083667917181, + "tokens_seen": 715487232 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003955566700100301, + "loss": 2.7859, + "theoretical_loss": 3.7710487340399217, + "tokens_seen": 715552768 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003955466399197593, + "loss": 2.7065, + "theoretical_loss": 3.7710138042578096, + "tokens_seen": 715618304 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003955366098294885, + "loss": 2.9122, + "theoretical_loss": 3.7709788785699905, + "tokens_seen": 715683840 + }, + { + "epoch": 2.04, + "learning_rate": 0.00039552657973921767, + "loss": 2.8986, + "theoretical_loss": 3.770943956975609, + "tokens_seen": 715749376 + }, + { + "epoch": 2.04, + "learning_rate": 0.00039551654964894685, + "loss": 2.7145, + "theoretical_loss": 3.770909039473811, + "tokens_seen": 715814912 + }, + { + "epoch": 2.04, + "learning_rate": 0.00039550651955867603, + "loss": 2.7466, + "theoretical_loss": 3.770874126063742, + "tokens_seen": 715880448 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003954964894684052, + "loss": 2.7592, + "theoretical_loss": 3.770839216744548, + "tokens_seen": 715945984 + }, + { + "epoch": 2.04, + "objective/train/docs_used": 846880, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.995128631591797, + "objective/train/theoretical_loss": 3.7708217636187626, + "objective/train/tokens_used": 736438752, + "theoretical_loss": 3.7708217636187626, + "tokens_seen": 715978752 + }, + { + "epoch": 2.04, + "learning_rate": 0.00039548645937813445, + "loss": 2.6837, + "theoretical_loss": 3.770804311515376, + "tokens_seen": 716011520 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003954764292878636, + "loss": 3.0525, + "theoretical_loss": 3.7707694103753715, + "tokens_seen": 716077056 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003954663991975928, + "loss": 2.9757, + "theoretical_loss": 3.770734513323682, + "tokens_seen": 716142592 + }, + { + "epoch": 2.04, + "learning_rate": 0.000395456369107322, + "loss": 2.7955, + "theoretical_loss": 3.770699620359454, + "tokens_seen": 716208128 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003954463390170512, + "loss": 2.8557, + "theoretical_loss": 3.770664731481835, + "tokens_seen": 716273664 + }, + { + "epoch": 2.04, + "learning_rate": 0.00039543630892678036, + "loss": 2.8646, + "theoretical_loss": 3.7706298466899724, + "tokens_seen": 716339200 + }, + { + "epoch": 2.04, + "learning_rate": 0.00039542627883650954, + "loss": 2.9426, + "theoretical_loss": 3.7705949659830145, + "tokens_seen": 716404736 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003954162487462387, + "loss": 2.7996, + "theoretical_loss": 3.7705600893601083, + "tokens_seen": 716470272 + }, + { + "epoch": 2.04, + "learning_rate": 0.00039540621865596795, + "loss": 3.0203, + "theoretical_loss": 3.7705252168204035, + "tokens_seen": 716535808 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003953961885656971, + "loss": 2.9499, + "theoretical_loss": 3.770490348363048, + "tokens_seen": 716601344 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003953861584754263, + "loss": 3.0872, + "theoretical_loss": 3.77045548398719, + "tokens_seen": 716666880 + }, + { + "epoch": 2.04, + "learning_rate": 0.00039537612838515544, + "loss": 2.6637, + "theoretical_loss": 3.7704206236919795, + "tokens_seen": 716732416 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003953660982948847, + "loss": 2.8815, + "theoretical_loss": 3.770385767476566, + "tokens_seen": 716797952 + }, + { + "epoch": 2.04, + "learning_rate": 0.00039535606820461386, + "loss": 2.943, + "theoretical_loss": 3.7703509153400976, + "tokens_seen": 716863488 + }, + { + "epoch": 2.04, + "learning_rate": 0.00039534603811434304, + "loss": 2.8796, + "theoretical_loss": 3.7703160672817257, + "tokens_seen": 716929024 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003953360080240722, + "loss": 2.9777, + "theoretical_loss": 3.7702812233006, + "tokens_seen": 716994560 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003953259779338014, + "loss": 2.858, + "theoretical_loss": 3.77024638339587, + "tokens_seen": 717060096 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003953159478435306, + "loss": 2.8236, + "theoretical_loss": 3.7702115475666877, + "tokens_seen": 717125632 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003953059177532598, + "loss": 2.9503, + "theoretical_loss": 3.7701767158122026, + "tokens_seen": 717191168 + }, + { + "epoch": 2.04, + "learning_rate": 0.00039529588766298895, + "loss": 2.8212, + "theoretical_loss": 3.7701418881315667, + "tokens_seen": 717256704 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003952858575727182, + "loss": 2.9965, + "theoretical_loss": 3.7701070645239314, + "tokens_seen": 717322240 + }, + { + "epoch": 2.04, + "learning_rate": 0.00039527582748244736, + "loss": 2.8431, + "theoretical_loss": 3.7700722449884476, + "tokens_seen": 717387776 + }, + { + "epoch": 2.04, + "learning_rate": 0.00039526579739217654, + "loss": 2.8866, + "theoretical_loss": 3.7700374295242676, + "tokens_seen": 717453312 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003952557673019057, + "loss": 2.9852, + "theoretical_loss": 3.770002618130544, + "tokens_seen": 717518848 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003952457372116349, + "loss": 2.7195, + "theoretical_loss": 3.769967810806428, + "tokens_seen": 717584384 + }, + { + "epoch": 2.04, + "objective/train/docs_used": 848381, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.755312204360962, + "objective/train/theoretical_loss": 3.7699504086702085, + "objective/train/tokens_used": 738077152, + "theoretical_loss": 3.7699504086702085, + "tokens_seen": 717617152 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003952357071213641, + "loss": 2.9789, + "theoretical_loss": 3.7699330075510735, + "tokens_seen": 717649920 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003952256770310933, + "loss": 2.678, + "theoretical_loss": 3.7698982083636325, + "tokens_seen": 717715456 + }, + { + "epoch": 2.04, + "learning_rate": 0.00039521564694082245, + "loss": 2.9702, + "theoretical_loss": 3.7698634132432587, + "tokens_seen": 717780992 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003952056168505517, + "loss": 2.7129, + "theoretical_loss": 3.7698286221891055, + "tokens_seen": 717846528 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003951955867602808, + "loss": 2.9426, + "theoretical_loss": 3.769793835200326, + "tokens_seen": 717912064 + }, + { + "epoch": 2.04, + "learning_rate": 0.00039518555667001005, + "loss": 2.8837, + "theoretical_loss": 3.7697590522760747, + "tokens_seen": 717977600 + }, + { + "epoch": 2.04, + "learning_rate": 0.00039517552657973923, + "loss": 2.89, + "theoretical_loss": 3.7697242734155054, + "tokens_seen": 718043136 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003951654964894684, + "loss": 2.9438, + "theoretical_loss": 3.7696894986177725, + "tokens_seen": 718108672 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003951554663991976, + "loss": 2.692, + "theoretical_loss": 3.7696547278820307, + "tokens_seen": 718174208 + }, + { + "epoch": 2.04, + "learning_rate": 0.00039514543630892677, + "loss": 3.0629, + "theoretical_loss": 3.769619961207435, + "tokens_seen": 718239744 + }, + { + "epoch": 2.04, + "learning_rate": 0.00039513540621865595, + "loss": 2.6968, + "theoretical_loss": 3.7695851985931412, + "tokens_seen": 718305280 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003951253761283852, + "loss": 2.6478, + "theoretical_loss": 3.7695504400383033, + "tokens_seen": 718370816 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003951153460381143, + "loss": 2.6335, + "theoretical_loss": 3.7695156855420784, + "tokens_seen": 718436352 + }, + { + "epoch": 2.04, + "learning_rate": 0.00039510531594784355, + "loss": 2.9399, + "theoretical_loss": 3.7694809351036214, + "tokens_seen": 718501888 + }, + { + "epoch": 2.04, + "learning_rate": 0.00039509528585757273, + "loss": 2.8495, + "theoretical_loss": 3.769446188722089, + "tokens_seen": 718567424 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003950852557673019, + "loss": 2.9849, + "theoretical_loss": 3.7694114463966377, + "tokens_seen": 718632960 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003950752256770311, + "loss": 2.7779, + "theoretical_loss": 3.7693767081264236, + "tokens_seen": 718698496 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003950651955867603, + "loss": 2.9114, + "theoretical_loss": 3.7693419739106044, + "tokens_seen": 718764032 + }, + { + "epoch": 2.04, + "learning_rate": 0.00039505516549648946, + "loss": 2.8417, + "theoretical_loss": 3.7693072437483366, + "tokens_seen": 718829568 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003950451354062187, + "loss": 2.8272, + "theoretical_loss": 3.7692725176387785, + "tokens_seen": 718895104 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003950351053159478, + "loss": 2.6431, + "theoretical_loss": 3.7692377955810867, + "tokens_seen": 718960640 + }, + { + "epoch": 2.04, + "learning_rate": 0.00039502507522567705, + "loss": 2.7929, + "theoretical_loss": 3.76920307757442, + "tokens_seen": 719026176 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003950150451354062, + "loss": 2.8803, + "theoretical_loss": 3.7691683636179367, + "tokens_seen": 719091712 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003950050150451354, + "loss": 2.8331, + "theoretical_loss": 3.7691336537107945, + "tokens_seen": 719157248 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003949949849548646, + "loss": 2.8321, + "theoretical_loss": 3.769098947852152, + "tokens_seen": 719222784 + }, + { + "epoch": 2.04, + "objective/train/docs_used": 849012, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8266098499298096, + "objective/train/theoretical_loss": 3.7690815964407562, + "objective/train/tokens_used": 739715552, + "theoretical_loss": 3.7690815964407562, + "tokens_seen": 719255552 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003949849548645938, + "loss": 2.8435, + "theoretical_loss": 3.7690642460411694, + "tokens_seen": 719288320 + }, + { + "epoch": 2.04, + "learning_rate": 0.00039497492477432296, + "loss": 2.9216, + "theoretical_loss": 3.769029548277005, + "tokens_seen": 719353856 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003949648946840522, + "loss": 2.851, + "theoretical_loss": 3.768994854558818, + "tokens_seen": 719419392 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003949548645937813, + "loss": 2.8108, + "theoretical_loss": 3.768960164885769, + "tokens_seen": 719484928 + }, + { + "epoch": 2.04, + "learning_rate": 0.00039494483450351056, + "loss": 2.9195, + "theoretical_loss": 3.768925479257017, + "tokens_seen": 719550464 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003949348044132397, + "loss": 2.8193, + "theoretical_loss": 3.768890797671723, + "tokens_seen": 719616000 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003949247743229689, + "loss": 2.7336, + "theoretical_loss": 3.7688561201290467, + "tokens_seen": 719681536 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003949147442326981, + "loss": 2.9406, + "theoretical_loss": 3.7688214466281496, + "tokens_seen": 719747072 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003949047141424273, + "loss": 2.7792, + "theoretical_loss": 3.7687867771681924, + "tokens_seen": 719812608 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003948946840521565, + "loss": 2.9954, + "theoretical_loss": 3.768752111748336, + "tokens_seen": 719878144 + }, + { + "epoch": 2.04, + "learning_rate": 0.00039488465396188564, + "loss": 3.0322, + "theoretical_loss": 3.768717450367742, + "tokens_seen": 719943680 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003948746238716149, + "loss": 2.8552, + "theoretical_loss": 3.7686827930255724, + "tokens_seen": 720009216 + }, + { + "epoch": 2.04, + "learning_rate": 0.00039486459378134406, + "loss": 2.9811, + "theoretical_loss": 3.7686481397209883, + "tokens_seen": 720074752 + }, + { + "epoch": 2.04, + "learning_rate": 0.00039485456369107324, + "loss": 2.9801, + "theoretical_loss": 3.768613490453153, + "tokens_seen": 720140288 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003948445336008024, + "loss": 2.9666, + "theoretical_loss": 3.7685788452212288, + "tokens_seen": 720205824 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003948345035105316, + "loss": 2.6315, + "theoretical_loss": 3.768544204024378, + "tokens_seen": 720271360 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003948244734202608, + "loss": 2.6003, + "theoretical_loss": 3.7685095668617636, + "tokens_seen": 720336896 + }, + { + "epoch": 2.04, + "learning_rate": 0.00039481444332999, + "loss": 2.9072, + "theoretical_loss": 3.7684749337325485, + "tokens_seen": 720402432 + }, + { + "epoch": 2.04, + "learning_rate": 0.00039480441323971915, + "loss": 2.9633, + "theoretical_loss": 3.768440304635897, + "tokens_seen": 720467968 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003947943831494484, + "loss": 2.8997, + "theoretical_loss": 3.7684056795709724, + "tokens_seen": 720533504 + }, + { + "epoch": 2.04, + "learning_rate": 0.00039478435305917756, + "loss": 2.8266, + "theoretical_loss": 3.768371058536939, + "tokens_seen": 720599040 + }, + { + "epoch": 2.04, + "learning_rate": 0.00039477432296890674, + "loss": 2.9913, + "theoretical_loss": 3.76833644153296, + "tokens_seen": 720664576 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003947642928786359, + "loss": 2.8822, + "theoretical_loss": 3.768301828558201, + "tokens_seen": 720730112 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003947542627883651, + "loss": 2.872, + "theoretical_loss": 3.7682672196118254, + "tokens_seen": 720795648 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003947442326980943, + "loss": 2.8944, + "theoretical_loss": 3.768232614693, + "tokens_seen": 720861184 + }, + { + "epoch": 2.04, + "objective/train/docs_used": 850658, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.295666217803955, + "objective/train/theoretical_loss": 3.768215313743657, + "objective/train/tokens_used": 741353952, + "theoretical_loss": 3.768215313743657, + "tokens_seen": 720893952 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003947342026078235, + "loss": 2.4079, + "theoretical_loss": 3.7681980138008884, + "tokens_seen": 720926720 + }, + { + "epoch": 2.04, + "learning_rate": 0.00039472417251755265, + "loss": 3.0534, + "theoretical_loss": 3.768163416934657, + "tokens_seen": 720992256 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003947141424272819, + "loss": 2.7413, + "theoretical_loss": 3.7681288240934707, + "tokens_seen": 721057792 + }, + { + "epoch": 2.04, + "learning_rate": 0.000394704112337011, + "loss": 2.9531, + "theoretical_loss": 3.768094235276496, + "tokens_seen": 721123328 + }, + { + "epoch": 2.04, + "learning_rate": 0.00039469408224674025, + "loss": 2.8847, + "theoretical_loss": 3.7680596504828987, + "tokens_seen": 721188864 + }, + { + "epoch": 2.04, + "learning_rate": 0.00039468405215646943, + "loss": 2.7946, + "theoretical_loss": 3.7680250697118467, + "tokens_seen": 721254400 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003946740220661986, + "loss": 2.9812, + "theoretical_loss": 3.7679904929625043, + "tokens_seen": 721319936 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003946639919759278, + "loss": 2.6334, + "theoretical_loss": 3.7679559202340407, + "tokens_seen": 721385472 + }, + { + "epoch": 2.04, + "learning_rate": 0.00039465396188565697, + "loss": 3.0705, + "theoretical_loss": 3.7679213515256214, + "tokens_seen": 721451008 + }, + { + "epoch": 2.04, + "learning_rate": 0.00039464393179538615, + "loss": 2.9367, + "theoretical_loss": 3.7678867868364145, + "tokens_seen": 721516544 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003946339017051154, + "loss": 2.859, + "theoretical_loss": 3.767852226165588, + "tokens_seen": 721582080 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003946238716148445, + "loss": 2.6695, + "theoretical_loss": 3.7678176695123096, + "tokens_seen": 721647616 + }, + { + "epoch": 2.04, + "learning_rate": 0.00039461384152457375, + "loss": 2.6397, + "theoretical_loss": 3.7677831168757474, + "tokens_seen": 721713152 + }, + { + "epoch": 2.04, + "learning_rate": 0.00039460381143430293, + "loss": 2.8705, + "theoretical_loss": 3.7677485682550698, + "tokens_seen": 721778688 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003945937813440321, + "loss": 2.848, + "theoretical_loss": 3.7677140236494457, + "tokens_seen": 721844224 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003945837512537613, + "loss": 2.8654, + "theoretical_loss": 3.767679483058044, + "tokens_seen": 721909760 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003945737211634905, + "loss": 2.7036, + "theoretical_loss": 3.7676449464800337, + "tokens_seen": 721975296 + }, + { + "epoch": 2.04, + "learning_rate": 0.00039456369107321966, + "loss": 2.7538, + "theoretical_loss": 3.7676104139145843, + "tokens_seen": 722040832 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003945536609829489, + "loss": 2.9453, + "theoretical_loss": 3.7675758853608654, + "tokens_seen": 722106368 + }, + { + "epoch": 2.04, + "learning_rate": 0.000394543630892678, + "loss": 2.9468, + "theoretical_loss": 3.767541360818047, + "tokens_seen": 722171904 + }, + { + "epoch": 2.04, + "learning_rate": 0.00039453360080240725, + "loss": 2.9775, + "theoretical_loss": 3.7675068402852996, + "tokens_seen": 722237440 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003945235707121364, + "loss": 3.0682, + "theoretical_loss": 3.7674723237617926, + "tokens_seen": 722302976 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003945135406218656, + "loss": 2.7909, + "theoretical_loss": 3.767437811246698, + "tokens_seen": 722368512 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003945035105315948, + "loss": 2.8995, + "theoretical_loss": 3.7674033027391856, + "tokens_seen": 722434048 + }, + { + "epoch": 2.04, + "learning_rate": 0.000394493480441324, + "loss": 2.6872, + "theoretical_loss": 3.7673687982384276, + "tokens_seen": 722499584 + }, + { + "epoch": 2.04, + "objective/train/docs_used": 851365, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.842928171157837, + "objective/train/theoretical_loss": 3.767351547490322, + "objective/train/tokens_used": 742992352, + "theoretical_loss": 3.767351547490322, + "tokens_seen": 722532352 + }, + { + "epoch": 2.04, + "learning_rate": 0.00039448345035105316, + "loss": 2.8114, + "theoretical_loss": 3.767334297743594, + "tokens_seen": 722565120 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003944734202607824, + "loss": 2.9769, + "theoretical_loss": 3.767299801253858, + "tokens_seen": 722630656 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003944633901705115, + "loss": 2.7042, + "theoretical_loss": 3.7672653087683905, + "tokens_seen": 722696192 + }, + { + "epoch": 2.04, + "learning_rate": 0.00039445336008024076, + "loss": 2.8248, + "theoretical_loss": 3.767230820286364, + "tokens_seen": 722761728 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003944433299899699, + "loss": 3.0078, + "theoretical_loss": 3.76719633580695, + "tokens_seen": 722827264 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003944332998996991, + "loss": 2.6326, + "theoretical_loss": 3.7671618553293227, + "tokens_seen": 722892800 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003944232698094283, + "loss": 2.8485, + "theoretical_loss": 3.7671273788526536, + "tokens_seen": 722958336 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003944132397191575, + "loss": 2.6747, + "theoretical_loss": 3.767092906376117, + "tokens_seen": 723023872 + }, + { + "epoch": 2.04, + "learning_rate": 0.00039440320962888666, + "loss": 2.8055, + "theoretical_loss": 3.7670584378988856, + "tokens_seen": 723089408 + }, + { + "epoch": 2.04, + "learning_rate": 0.00039439317953861584, + "loss": 2.9246, + "theoretical_loss": 3.767023973420133, + "tokens_seen": 723154944 + }, + { + "epoch": 2.04, + "learning_rate": 0.000394383149448345, + "loss": 2.8103, + "theoretical_loss": 3.766989512939033, + "tokens_seen": 723220480 + }, + { + "epoch": 2.04, + "learning_rate": 0.00039437311935807426, + "loss": 2.8357, + "theoretical_loss": 3.7669550564547594, + "tokens_seen": 723286016 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003943630892678034, + "loss": 2.8102, + "theoretical_loss": 3.7669206039664878, + "tokens_seen": 723351552 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003943530591775326, + "loss": 2.5936, + "theoretical_loss": 3.766886155473392, + "tokens_seen": 723417088 + }, + { + "epoch": 2.04, + "learning_rate": 0.00039434302908726175, + "loss": 3.0184, + "theoretical_loss": 3.766851710974646, + "tokens_seen": 723482624 + }, + { + "epoch": 2.04, + "learning_rate": 0.000394332998996991, + "loss": 2.7742, + "theoretical_loss": 3.766817270469427, + "tokens_seen": 723548160 + }, + { + "epoch": 2.04, + "learning_rate": 0.00039432296890672017, + "loss": 2.7539, + "theoretical_loss": 3.766782833956908, + "tokens_seen": 723613696 + }, + { + "epoch": 2.04, + "learning_rate": 0.00039431293881644935, + "loss": 2.7714, + "theoretical_loss": 3.766748401436266, + "tokens_seen": 723679232 + }, + { + "epoch": 2.04, + "learning_rate": 0.00039430290872617853, + "loss": 2.9656, + "theoretical_loss": 3.7667139729066768, + "tokens_seen": 723744768 + }, + { + "epoch": 2.04, + "learning_rate": 0.00039429287863590776, + "loss": 2.8626, + "theoretical_loss": 3.766679548367316, + "tokens_seen": 723810304 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003942828485456369, + "loss": 2.9391, + "theoretical_loss": 3.76664512781736, + "tokens_seen": 723875840 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003942728184553661, + "loss": 2.8405, + "theoretical_loss": 3.766610711255985, + "tokens_seen": 723941376 + }, + { + "epoch": 2.04, + "learning_rate": 0.00039426278836509525, + "loss": 2.9499, + "theoretical_loss": 3.7665762986823683, + "tokens_seen": 724006912 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003942527582748245, + "loss": 2.621, + "theoretical_loss": 3.766541890095687, + "tokens_seen": 724072448 + }, + { + "epoch": 2.04, + "learning_rate": 0.00039424272818455367, + "loss": 2.8388, + "theoretical_loss": 3.7665074854951186, + "tokens_seen": 724137984 + }, + { + "epoch": 2.04, + "objective/train/docs_used": 852859, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8854544162750244, + "objective/train/theoretical_loss": 3.766490284689369, + "objective/train/tokens_used": 744630752, + "theoretical_loss": 3.766490284689369, + "tokens_seen": 724170752 + }, + { + "epoch": 2.04, + "learning_rate": 0.00039423269809428285, + "loss": 2.9438, + "theoretical_loss": 3.76647308487984, + "tokens_seen": 724203520 + }, + { + "epoch": 2.04, + "learning_rate": 0.00039422266800401203, + "loss": 2.8946, + "theoretical_loss": 3.7664386882490293, + "tokens_seen": 724269056 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003942126379137412, + "loss": 3.0176, + "theoretical_loss": 3.7664042956018644, + "tokens_seen": 724334592 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003942026078234704, + "loss": 2.879, + "theoretical_loss": 3.7663699069375234, + "tokens_seen": 724400128 + }, + { + "epoch": 2.04, + "learning_rate": 0.00039419257773319963, + "loss": 2.8402, + "theoretical_loss": 3.7663355222551855, + "tokens_seen": 724465664 + }, + { + "epoch": 2.04, + "learning_rate": 0.00039418254764292876, + "loss": 2.8686, + "theoretical_loss": 3.766301141554029, + "tokens_seen": 724531200 + }, + { + "epoch": 2.04, + "learning_rate": 0.000394172517552658, + "loss": 2.8314, + "theoretical_loss": 3.7662667648332326, + "tokens_seen": 724596736 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003941624874623871, + "loss": 2.7861, + "theoretical_loss": 3.7662323920919762, + "tokens_seen": 724662272 + }, + { + "epoch": 2.04, + "learning_rate": 0.00039415245737211635, + "loss": 2.9663, + "theoretical_loss": 3.7661980233294385, + "tokens_seen": 724727808 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003941424272818456, + "loss": 2.8261, + "theoretical_loss": 3.7661636585447997, + "tokens_seen": 724793344 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003941323971915747, + "loss": 2.8967, + "theoretical_loss": 3.7661292977372405, + "tokens_seen": 724858880 + }, + { + "epoch": 2.04, + "learning_rate": 0.00039412236710130395, + "loss": 2.8327, + "theoretical_loss": 3.7660949409059397, + "tokens_seen": 724924416 + }, + { + "epoch": 2.04, + "learning_rate": 0.00039411233701103313, + "loss": 2.9669, + "theoretical_loss": 3.7660605880500784, + "tokens_seen": 724989952 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003941023069207623, + "loss": 2.6587, + "theoretical_loss": 3.7660262391688373, + "tokens_seen": 725055488 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003940922768304915, + "loss": 2.7309, + "theoretical_loss": 3.7659918942613975, + "tokens_seen": 725121024 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003940822467402207, + "loss": 2.7392, + "theoretical_loss": 3.76595755332694, + "tokens_seen": 725186560 + }, + { + "epoch": 2.04, + "learning_rate": 0.00039407221664994986, + "loss": 2.8362, + "theoretical_loss": 3.765923216364646, + "tokens_seen": 725252096 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003940621865596791, + "loss": 2.746, + "theoretical_loss": 3.765888883373698, + "tokens_seen": 725317632 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003940521564694082, + "loss": 2.7861, + "theoretical_loss": 3.765854554353277, + "tokens_seen": 725383168 + }, + { + "epoch": 2.04, + "learning_rate": 0.00039404212637913745, + "loss": 2.9553, + "theoretical_loss": 3.765820229302565, + "tokens_seen": 725448704 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003940320962888666, + "loss": 2.954, + "theoretical_loss": 3.765785908220746, + "tokens_seen": 725514240 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003940220661985958, + "loss": 2.7014, + "theoretical_loss": 3.7657515911070005, + "tokens_seen": 725579776 + }, + { + "epoch": 2.04, + "learning_rate": 0.000394012036108325, + "loss": 2.7269, + "theoretical_loss": 3.765717277960513, + "tokens_seen": 725645312 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003940020060180542, + "loss": 2.8501, + "theoretical_loss": 3.7656829687804656, + "tokens_seen": 725710848 + }, + { + "epoch": 2.04, + "learning_rate": 0.00039399197592778336, + "loss": 2.6812, + "theoretical_loss": 3.7656486635660427, + "tokens_seen": 725776384 + }, + { + "epoch": 2.04, + "objective/train/docs_used": 853613, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.0343728065490723, + "objective/train/theoretical_loss": 3.765631512445685, + "objective/train/tokens_used": 746269152, + "theoretical_loss": 3.765631512445685, + "tokens_seen": 725809152 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003939819458375126, + "loss": 2.9185, + "theoretical_loss": 3.765614362316427, + "tokens_seen": 725841920 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003939719157472417, + "loss": 2.9212, + "theoretical_loss": 3.765580065030803, + "tokens_seen": 725907456 + }, + { + "epoch": 2.04, + "learning_rate": 0.00039396188565697096, + "loss": 2.9167, + "theoretical_loss": 3.7655457717083545, + "tokens_seen": 725972992 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003939518555667001, + "loss": 2.8155, + "theoretical_loss": 3.7655114823482654, + "tokens_seen": 726038528 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003939418254764293, + "loss": 2.7203, + "theoretical_loss": 3.7654771969497203, + "tokens_seen": 726104064 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003939317953861585, + "loss": 2.7505, + "theoretical_loss": 3.765442915511905, + "tokens_seen": 726169600 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003939217652958877, + "loss": 2.9916, + "theoretical_loss": 3.765408638034004, + "tokens_seen": 726235136 + }, + { + "epoch": 2.04, + "learning_rate": 0.00039391173520561686, + "loss": 2.7676, + "theoretical_loss": 3.7653743645152025, + "tokens_seen": 726300672 + }, + { + "epoch": 2.04, + "learning_rate": 0.00039390170511534604, + "loss": 2.635, + "theoretical_loss": 3.765340094954686, + "tokens_seen": 726366208 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003938916750250752, + "loss": 2.8757, + "theoretical_loss": 3.7653058293516404, + "tokens_seen": 726431744 + }, + { + "epoch": 2.04, + "learning_rate": 0.00039388164493480446, + "loss": 2.8451, + "theoretical_loss": 3.7652715677052515, + "tokens_seen": 726497280 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003938716148445336, + "loss": 2.8985, + "theoretical_loss": 3.765237310014706, + "tokens_seen": 726562816 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003938615847542628, + "loss": 2.8408, + "theoretical_loss": 3.76520305627919, + "tokens_seen": 726628352 + }, + { + "epoch": 2.04, + "learning_rate": 0.00039385155466399195, + "loss": 2.9088, + "theoretical_loss": 3.76516880649789, + "tokens_seen": 726693888 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003938415245737212, + "loss": 3.0388, + "theoretical_loss": 3.765134560669994, + "tokens_seen": 726759424 + }, + { + "epoch": 2.04, + "learning_rate": 0.00039383149448345037, + "loss": 2.9202, + "theoretical_loss": 3.7651003187946888, + "tokens_seen": 726824960 + }, + { + "epoch": 2.04, + "learning_rate": 0.00039382146439317955, + "loss": 2.8556, + "theoretical_loss": 3.7650660808711613, + "tokens_seen": 726890496 + }, + { + "epoch": 2.04, + "learning_rate": 0.00039381143430290873, + "loss": 2.7686, + "theoretical_loss": 3.7650318468985993, + "tokens_seen": 726956032 + }, + { + "epoch": 2.04, + "learning_rate": 0.00039380140421263796, + "loss": 2.7862, + "theoretical_loss": 3.764997616876191, + "tokens_seen": 727021568 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003937913741223671, + "loss": 3.0449, + "theoretical_loss": 3.7649633908031257, + "tokens_seen": 727087104 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003937813440320963, + "loss": 2.6458, + "theoretical_loss": 3.7649291686785897, + "tokens_seen": 727152640 + }, + { + "epoch": 2.04, + "learning_rate": 0.00039377131394182545, + "loss": 2.7193, + "theoretical_loss": 3.7648949505017733, + "tokens_seen": 727218176 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003937612838515547, + "loss": 2.8087, + "theoretical_loss": 3.7648607362718645, + "tokens_seen": 727283712 + }, + { + "epoch": 2.04, + "learning_rate": 0.00039375125376128387, + "loss": 2.97, + "theoretical_loss": 3.7648265259880525, + "tokens_seen": 727349248 + }, + { + "epoch": 2.04, + "learning_rate": 0.00039374122367101305, + "loss": 3.2369, + "theoretical_loss": 3.764792319649527, + "tokens_seen": 727414784 + }, + { + "epoch": 2.04, + "objective/train/docs_used": 854233, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.945094108581543, + "objective/train/theoretical_loss": 3.764775217959494, + "objective/train/tokens_used": 747907552, + "theoretical_loss": 3.764775217959494, + "tokens_seen": 727447552 + }, + { + "epoch": 2.04, + "learning_rate": 0.00039373119358074223, + "loss": 2.8022, + "theoretical_loss": 3.7647581172554783, + "tokens_seen": 727480320 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003937211634904714, + "loss": 2.9182, + "theoretical_loss": 3.764723918805095, + "tokens_seen": 727545856 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003937111334002006, + "loss": 2.775, + "theoretical_loss": 3.7646897242975674, + "tokens_seen": 727611392 + }, + { + "epoch": 2.04, + "learning_rate": 0.00039370110330992983, + "loss": 3.0035, + "theoretical_loss": 3.7646555337320873, + "tokens_seen": 727676928 + }, + { + "epoch": 2.04, + "learning_rate": 0.00039369107321965896, + "loss": 2.8621, + "theoretical_loss": 3.764621347107843, + "tokens_seen": 727742464 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003936810431293882, + "loss": 3.0658, + "theoretical_loss": 3.764587164424027, + "tokens_seen": 727808000 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003936710130391173, + "loss": 2.7817, + "theoretical_loss": 3.76455298567983, + "tokens_seen": 727873536 + }, + { + "epoch": 2.04, + "learning_rate": 0.00039366098294884655, + "loss": 2.9779, + "theoretical_loss": 3.764518810874443, + "tokens_seen": 727939072 + }, + { + "epoch": 2.04, + "learning_rate": 0.00039365095285857573, + "loss": 2.8654, + "theoretical_loss": 3.7644846400070575, + "tokens_seen": 728004608 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003936409227683049, + "loss": 2.7282, + "theoretical_loss": 3.7644504730768653, + "tokens_seen": 728070144 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003936308926780341, + "loss": 2.7755, + "theoretical_loss": 3.764416310083059, + "tokens_seen": 728135680 + }, + { + "epoch": 2.04, + "learning_rate": 0.00039362086258776333, + "loss": 2.9102, + "theoretical_loss": 3.7643821510248303, + "tokens_seen": 728201216 + }, + { + "epoch": 2.04, + "learning_rate": 0.00039361083249749246, + "loss": 2.9127, + "theoretical_loss": 3.764347995901372, + "tokens_seen": 728266752 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003936008024072217, + "loss": 2.9826, + "theoretical_loss": 3.764313844711876, + "tokens_seen": 728332288 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003935907723169508, + "loss": 2.7889, + "theoretical_loss": 3.7642796974555366, + "tokens_seen": 728397824 + }, + { + "epoch": 2.04, + "learning_rate": 0.00039358074222668006, + "loss": 2.7005, + "theoretical_loss": 3.764245554131546, + "tokens_seen": 728463360 + }, + { + "epoch": 2.04, + "learning_rate": 0.00039357071213640924, + "loss": 3.0278, + "theoretical_loss": 3.7642114147390977, + "tokens_seen": 728528896 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003935606820461384, + "loss": 2.8777, + "theoretical_loss": 3.764177279277386, + "tokens_seen": 728594432 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003935506519558676, + "loss": 3.0676, + "theoretical_loss": 3.764143147745605, + "tokens_seen": 728659968 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003935406218655968, + "loss": 2.8788, + "theoretical_loss": 3.7641090201429477, + "tokens_seen": 728725504 + }, + { + "epoch": 2.04, + "learning_rate": 0.00039353059177532596, + "loss": 2.7074, + "theoretical_loss": 3.764074896468609, + "tokens_seen": 728791040 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003935205616850552, + "loss": 2.8258, + "theoretical_loss": 3.7640407767217843, + "tokens_seen": 728856576 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003935105315947843, + "loss": 2.9152, + "theoretical_loss": 3.764006660901667, + "tokens_seen": 728922112 + }, + { + "epoch": 2.04, + "learning_rate": 0.00039350050150451356, + "loss": 2.9142, + "theoretical_loss": 3.7639725490074536, + "tokens_seen": 728987648 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003934904714142427, + "loss": 2.8464, + "theoretical_loss": 3.7639384410383387, + "tokens_seen": 729053184 + }, + { + "epoch": 2.04, + "objective/train/docs_used": 855726, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.673104763031006, + "objective/train/theoretical_loss": 3.763921388525442, + "objective/train/tokens_used": 749545952, + "theoretical_loss": 3.763921388525442, + "tokens_seen": 729085952 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003934804413239719, + "loss": 2.9341, + "theoretical_loss": 3.763904336993518, + "tokens_seen": 729118720 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003934704112337011, + "loss": 2.7971, + "theoretical_loss": 3.7638702368721875, + "tokens_seen": 729184256 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003934603811434303, + "loss": 2.9572, + "theoretical_loss": 3.7638361406735434, + "tokens_seen": 729249792 + }, + { + "epoch": 2.04, + "learning_rate": 0.00039345035105315946, + "loss": 2.9707, + "theoretical_loss": 3.7638020483967813, + "tokens_seen": 729315328 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003934403209628887, + "loss": 2.6327, + "theoretical_loss": 3.7637679600410987, + "tokens_seen": 729380864 + }, + { + "epoch": 2.04, + "learning_rate": 0.00039343029087261783, + "loss": 2.9909, + "theoretical_loss": 3.7637338756056913, + "tokens_seen": 729446400 + }, + { + "epoch": 2.04, + "learning_rate": 0.00039342026078234706, + "loss": 2.7681, + "theoretical_loss": 3.7636997950897566, + "tokens_seen": 729511936 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003934102306920762, + "loss": 2.8921, + "theoretical_loss": 3.763665718492492, + "tokens_seen": 729577472 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003934002006018054, + "loss": 2.948, + "theoretical_loss": 3.7636316458130956, + "tokens_seen": 729643008 + }, + { + "epoch": 2.04, + "learning_rate": 0.00039339017051153466, + "loss": 2.8759, + "theoretical_loss": 3.7635975770507635, + "tokens_seen": 729708544 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003933801404212638, + "loss": 2.7037, + "theoretical_loss": 3.763563512204695, + "tokens_seen": 729774080 + }, + { + "epoch": 2.04, + "learning_rate": 0.000393370110330993, + "loss": 2.8733, + "theoretical_loss": 3.7635294512740876, + "tokens_seen": 729839616 + }, + { + "epoch": 2.04, + "learning_rate": 0.00039336008024072215, + "loss": 2.7322, + "theoretical_loss": 3.7634953942581397, + "tokens_seen": 729905152 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003933500501504514, + "loss": 2.8427, + "theoretical_loss": 3.7634613411560505, + "tokens_seen": 729970688 + }, + { + "epoch": 2.04, + "learning_rate": 0.00039334002006018057, + "loss": 2.8633, + "theoretical_loss": 3.7634272919670186, + "tokens_seen": 730036224 + }, + { + "epoch": 2.04, + "learning_rate": 0.00039332998996990975, + "loss": 2.7698, + "theoretical_loss": 3.763393246690243, + "tokens_seen": 730101760 + }, + { + "epoch": 2.04, + "learning_rate": 0.00039331995987963893, + "loss": 2.883, + "theoretical_loss": 3.7633592053249236, + "tokens_seen": 730167296 + }, + { + "epoch": 2.04, + "learning_rate": 0.00039330992978936816, + "loss": 2.8142, + "theoretical_loss": 3.763325167870259, + "tokens_seen": 730232832 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003932998996990973, + "loss": 3.0139, + "theoretical_loss": 3.76329113432545, + "tokens_seen": 730298368 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003932898696088265, + "loss": 2.8159, + "theoretical_loss": 3.7632571046896963, + "tokens_seen": 730363904 + }, + { + "epoch": 2.04, + "learning_rate": 0.00039327983951855565, + "loss": 2.8133, + "theoretical_loss": 3.763223078962198, + "tokens_seen": 730429440 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003932698094282849, + "loss": 2.9151, + "theoretical_loss": 3.763189057142156, + "tokens_seen": 730494976 + }, + { + "epoch": 2.04, + "learning_rate": 0.00039325977933801407, + "loss": 2.6749, + "theoretical_loss": 3.7631550392287707, + "tokens_seen": 730560512 + }, + { + "epoch": 2.04, + "learning_rate": 0.00039324974924774325, + "loss": 2.9721, + "theoretical_loss": 3.7631210252212437, + "tokens_seen": 730626048 + }, + { + "epoch": 2.04, + "learning_rate": 0.00039323971915747243, + "loss": 2.8593, + "theoretical_loss": 3.763087015118776, + "tokens_seen": 730691584 + }, + { + "epoch": 2.04, + "objective/train/docs_used": 856181, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.754777431488037, + "objective/train/theoretical_loss": 3.7630700115316893, + "objective/train/tokens_used": 751184352, + "theoretical_loss": 3.7630700115316893, + "tokens_seen": 730724352 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003932296890672016, + "loss": 2.8694, + "theoretical_loss": 3.7630530089205685, + "tokens_seen": 730757120 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003932196589769308, + "loss": 2.8381, + "theoretical_loss": 3.7630190066258233, + "tokens_seen": 730822656 + }, + { + "epoch": 2.04, + "learning_rate": 0.00039320962888666003, + "loss": 2.8851, + "theoretical_loss": 3.762985008233743, + "tokens_seen": 730888192 + }, + { + "epoch": 2.04, + "learning_rate": 0.00039319959879638916, + "loss": 3.0075, + "theoretical_loss": 3.762951013743529, + "tokens_seen": 730953728 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003931895687061184, + "loss": 3.1236, + "theoretical_loss": 3.762917023154384, + "tokens_seen": 731019264 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003931795386158475, + "loss": 2.8348, + "theoretical_loss": 3.762883036465511, + "tokens_seen": 731084800 + }, + { + "epoch": 2.04, + "learning_rate": 0.00039316950852557675, + "loss": 2.8395, + "theoretical_loss": 3.7628490536761117, + "tokens_seen": 731150336 + }, + { + "epoch": 2.04, + "learning_rate": 0.00039315947843530593, + "loss": 2.8552, + "theoretical_loss": 3.7628150747853906, + "tokens_seen": 731215872 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003931494483450351, + "loss": 2.9167, + "theoretical_loss": 3.7627810997925506, + "tokens_seen": 731281408 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003931394182547643, + "loss": 3.2027, + "theoretical_loss": 3.7627471286967946, + "tokens_seen": 731346944 + }, + { + "epoch": 2.04, + "learning_rate": 0.00039312938816449353, + "loss": 2.9124, + "theoretical_loss": 3.7627131614973277, + "tokens_seen": 731412480 + }, + { + "epoch": 2.04, + "learning_rate": 0.00039311935807422266, + "loss": 2.8646, + "theoretical_loss": 3.762679198193353, + "tokens_seen": 731478016 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003931093279839519, + "loss": 2.9585, + "theoretical_loss": 3.762645238784075, + "tokens_seen": 731543552 + }, + { + "epoch": 2.04, + "learning_rate": 0.000393099297893681, + "loss": 2.7172, + "theoretical_loss": 3.7626112832686984, + "tokens_seen": 731609088 + }, + { + "epoch": 2.04, + "learning_rate": 0.00039308926780341026, + "loss": 2.8292, + "theoretical_loss": 3.762577331646428, + "tokens_seen": 731674624 + }, + { + "epoch": 2.04, + "learning_rate": 0.00039307923771313944, + "loss": 2.8594, + "theoretical_loss": 3.762543383916469, + "tokens_seen": 731740160 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003930692076228686, + "loss": 2.9352, + "theoretical_loss": 3.7625094400780257, + "tokens_seen": 731805696 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003930591775325978, + "loss": 2.8496, + "theoretical_loss": 3.7624755001303045, + "tokens_seen": 731871232 + }, + { + "epoch": 2.04, + "learning_rate": 0.000393049147442327, + "loss": 2.9329, + "theoretical_loss": 3.7624415640725104, + "tokens_seen": 731936768 + }, + { + "epoch": 2.04, + "learning_rate": 0.00039303911735205616, + "loss": 2.7525, + "theoretical_loss": 3.7624076319038506, + "tokens_seen": 732002304 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003930290872617854, + "loss": 2.702, + "theoretical_loss": 3.7623737036235294, + "tokens_seen": 732067840 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003930190571715145, + "loss": 2.7606, + "theoretical_loss": 3.7623397792307545, + "tokens_seen": 732133376 + }, + { + "epoch": 2.04, + "learning_rate": 0.00039300902708124376, + "loss": 2.7868, + "theoretical_loss": 3.762305858724732, + "tokens_seen": 732198912 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003929989969909729, + "loss": 3.0959, + "theoretical_loss": 3.7622719421046695, + "tokens_seen": 732264448 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003929889669007021, + "loss": 3.0838, + "theoretical_loss": 3.7622380293697733, + "tokens_seen": 732329984 + }, + { + "epoch": 2.04, + "objective/train/docs_used": 857755, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.856901168823242, + "objective/train/theoretical_loss": 3.762221074459015, + "objective/train/tokens_used": 752822752, + "theoretical_loss": 3.762221074459015, + "tokens_seen": 732362752 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003929789368104313, + "loss": 2.7062, + "theoretical_loss": 3.762204120519251, + "tokens_seen": 732395520 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003929689067201605, + "loss": 2.7705, + "theoretical_loss": 3.76217021555231, + "tokens_seen": 732461056 + }, + { + "epoch": 2.04, + "learning_rate": 0.00039295887662988967, + "loss": 2.9732, + "theoretical_loss": 3.7621363144681585, + "tokens_seen": 732526592 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003929488465396189, + "loss": 2.9889, + "theoretical_loss": 3.7621024172660045, + "tokens_seen": 732592128 + }, + { + "epoch": 2.04, + "learning_rate": 0.00039293881644934803, + "loss": 2.7959, + "theoretical_loss": 3.7620685239450555, + "tokens_seen": 732657664 + }, + { + "epoch": 2.04, + "learning_rate": 0.00039292878635907726, + "loss": 2.9595, + "theoretical_loss": 3.762034634504521, + "tokens_seen": 732723200 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003929187562688064, + "loss": 2.746, + "theoretical_loss": 3.762000748943609, + "tokens_seen": 732788736 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003929087261785356, + "loss": 2.9863, + "theoretical_loss": 3.7619668672615285, + "tokens_seen": 732854272 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003928986960882648, + "loss": 2.6103, + "theoretical_loss": 3.761932989457489, + "tokens_seen": 732919808 + }, + { + "epoch": 2.04, + "learning_rate": 0.000392888665997994, + "loss": 2.9042, + "theoretical_loss": 3.7618991155307, + "tokens_seen": 732985344 + }, + { + "epoch": 2.04, + "learning_rate": 0.00039287863590772317, + "loss": 2.9509, + "theoretical_loss": 3.761865245480371, + "tokens_seen": 733050880 + }, + { + "epoch": 2.04, + "learning_rate": 0.00039286860581745235, + "loss": 2.9416, + "theoretical_loss": 3.7618313793057117, + "tokens_seen": 733116416 + }, + { + "epoch": 2.04, + "learning_rate": 0.00039285857572718153, + "loss": 2.9895, + "theoretical_loss": 3.761797517005933, + "tokens_seen": 733181952 + }, + { + "epoch": 2.04, + "learning_rate": 0.00039284854563691077, + "loss": 2.6667, + "theoretical_loss": 3.7617636585802434, + "tokens_seen": 733247488 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003928385155466399, + "loss": 2.8332, + "theoretical_loss": 3.7617298040278557, + "tokens_seen": 733313024 + }, + { + "epoch": 2.04, + "learning_rate": 0.00039282848545636913, + "loss": 2.8072, + "theoretical_loss": 3.761695953347979, + "tokens_seen": 733378560 + }, + { + "epoch": 2.04, + "learning_rate": 0.00039281845536609825, + "loss": 2.8533, + "theoretical_loss": 3.7616621065398257, + "tokens_seen": 733444096 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003928084252758275, + "loss": 2.8354, + "theoretical_loss": 3.761628263602606, + "tokens_seen": 733509632 + }, + { + "epoch": 2.04, + "learning_rate": 0.00039279839518555667, + "loss": 2.695, + "theoretical_loss": 3.7615944245355317, + "tokens_seen": 733575168 + }, + { + "epoch": 2.04, + "learning_rate": 0.00039278836509528585, + "loss": 2.7853, + "theoretical_loss": 3.7615605893378152, + "tokens_seen": 733640704 + }, + { + "epoch": 2.04, + "learning_rate": 0.00039277833500501503, + "loss": 2.8961, + "theoretical_loss": 3.7615267580086673, + "tokens_seen": 733706240 + }, + { + "epoch": 2.04, + "learning_rate": 0.00039276830491474427, + "loss": 3.0298, + "theoretical_loss": 3.761492930547301, + "tokens_seen": 733771776 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003927582748244734, + "loss": 2.914, + "theoretical_loss": 3.7614591069529286, + "tokens_seen": 733837312 + }, + { + "epoch": 2.04, + "learning_rate": 0.00039274824473420263, + "loss": 2.8187, + "theoretical_loss": 3.761425287224763, + "tokens_seen": 733902848 + }, + { + "epoch": 2.04, + "learning_rate": 0.00039273821464393176, + "loss": 2.7914, + "theoretical_loss": 3.7613914713620167, + "tokens_seen": 733968384 + }, + { + "epoch": 2.04, + "objective/train/docs_used": 858482, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.96612548828125, + "objective/train/theoretical_loss": 3.7613745648799295, + "objective/train/tokens_used": 754461152, + "theoretical_loss": 3.7613745648799295, + "tokens_seen": 734001152 + }, + { + "epoch": 2.04, + "learning_rate": 0.000392728184553661, + "loss": 2.6728, + "theoretical_loss": 3.7613576593639024, + "tokens_seen": 734033920 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003927181544633902, + "loss": 2.9194, + "theoretical_loss": 3.7613238512296348, + "tokens_seen": 734099456 + }, + { + "epoch": 2.04, + "learning_rate": 0.00039270812437311936, + "loss": 2.7902, + "theoretical_loss": 3.7612900469584263, + "tokens_seen": 734164992 + }, + { + "epoch": 2.04, + "learning_rate": 0.00039269809428284854, + "loss": 3.0611, + "theoretical_loss": 3.7612562465494905, + "tokens_seen": 734230528 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003926880641925777, + "loss": 2.6482, + "theoretical_loss": 3.7612224500020424, + "tokens_seen": 734296064 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003926780341023069, + "loss": 2.7174, + "theoretical_loss": 3.7611886573152957, + "tokens_seen": 734361600 + }, + { + "epoch": 2.04, + "learning_rate": 0.00039266800401203613, + "loss": 2.6784, + "theoretical_loss": 3.7611548684884655, + "tokens_seen": 734427136 + }, + { + "epoch": 2.04, + "learning_rate": 0.00039265797392176526, + "loss": 2.886, + "theoretical_loss": 3.7611210835207656, + "tokens_seen": 734492672 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003926479438314945, + "loss": 2.9471, + "theoretical_loss": 3.7610873024114113, + "tokens_seen": 734558208 + }, + { + "epoch": 2.04, + "learning_rate": 0.00039263791374122373, + "loss": 2.8858, + "theoretical_loss": 3.7610535251596184, + "tokens_seen": 734623744 + }, + { + "epoch": 2.04, + "learning_rate": 0.00039262788365095286, + "loss": 2.685, + "theoretical_loss": 3.7610197517646022, + "tokens_seen": 734689280 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003926178535606821, + "loss": 2.6936, + "theoretical_loss": 3.760985982225577, + "tokens_seen": 734754816 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003926078234704112, + "loss": 2.6473, + "theoretical_loss": 3.76095221654176, + "tokens_seen": 734820352 + }, + { + "epoch": 2.04, + "learning_rate": 0.00039259779338014046, + "loss": 2.7335, + "theoretical_loss": 3.7609184547123675, + "tokens_seen": 734885888 + }, + { + "epoch": 2.04, + "learning_rate": 0.00039258776328986964, + "loss": 2.9301, + "theoretical_loss": 3.760884696736615, + "tokens_seen": 734951424 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003925777331995988, + "loss": 2.7825, + "theoretical_loss": 3.7608509426137195, + "tokens_seen": 735016960 + }, + { + "epoch": 2.04, + "learning_rate": 0.000392567703109328, + "loss": 2.8205, + "theoretical_loss": 3.7608171923428975, + "tokens_seen": 735082496 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003925576730190572, + "loss": 2.8997, + "theoretical_loss": 3.7607834459233667, + "tokens_seen": 735148032 + }, + { + "epoch": 2.04, + "learning_rate": 0.00039254764292878636, + "loss": 2.6886, + "theoretical_loss": 3.7607497033543433, + "tokens_seen": 735213568 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003925376128385156, + "loss": 2.9114, + "theoretical_loss": 3.7607159646350454, + "tokens_seen": 735279104 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003925275827482447, + "loss": 2.9142, + "theoretical_loss": 3.760682229764691, + "tokens_seen": 735344640 + }, + { + "epoch": 2.04, + "learning_rate": 0.00039251755265797396, + "loss": 3.1446, + "theoretical_loss": 3.7606484987424973, + "tokens_seen": 735410176 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003925075225677031, + "loss": 2.8202, + "theoretical_loss": 3.760614771567683, + "tokens_seen": 735475712 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003924974924774323, + "loss": 2.784, + "theoretical_loss": 3.760581048239466, + "tokens_seen": 735541248 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003924874623871615, + "loss": 2.8307, + "theoretical_loss": 3.760547328757066, + "tokens_seen": 735606784 + }, + { + "epoch": 2.04, + "objective/train/docs_used": 859784, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6183910369873047, + "objective/train/theoretical_loss": 3.7605304704578026, + "objective/train/tokens_used": 756099552, + "theoretical_loss": 3.7605304704578026, + "tokens_seen": 735639552 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003924774322968907, + "loss": 2.844, + "theoretical_loss": 3.7605136131197003, + "tokens_seen": 735672320 + }, + { + "epoch": 2.04, + "learning_rate": 0.00039246740220661987, + "loss": 3.1915, + "theoretical_loss": 3.7604799013265895, + "tokens_seen": 735737856 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003924573721163491, + "loss": 3.0552, + "theoretical_loss": 3.760446193376952, + "tokens_seen": 735803392 + }, + { + "epoch": 2.04, + "learning_rate": 0.00039244734202607823, + "loss": 2.871, + "theoretical_loss": 3.7604124892700073, + "tokens_seen": 735868928 + }, + { + "epoch": 2.04, + "learning_rate": 0.00039243731193580746, + "loss": 2.6401, + "theoretical_loss": 3.7603787890049762, + "tokens_seen": 735934464 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003924272818455366, + "loss": 2.9156, + "theoretical_loss": 3.760345092581077, + "tokens_seen": 736000000 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003924172517552658, + "loss": 2.7434, + "theoretical_loss": 3.7603113999975317, + "tokens_seen": 736065536 + }, + { + "epoch": 2.04, + "learning_rate": 0.000392407221664995, + "loss": 2.6808, + "theoretical_loss": 3.7602777112535595, + "tokens_seen": 736131072 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003923971915747242, + "loss": 2.8149, + "theoretical_loss": 3.7602440263483814, + "tokens_seen": 736196608 + }, + { + "epoch": 2.04, + "learning_rate": 0.00039238716148445337, + "loss": 2.8053, + "theoretical_loss": 3.7602103452812186, + "tokens_seen": 736262144 + }, + { + "epoch": 2.04, + "learning_rate": 0.00039237713139418255, + "loss": 2.7973, + "theoretical_loss": 3.760176668051291, + "tokens_seen": 736327680 + }, + { + "epoch": 2.04, + "learning_rate": 0.00039236710130391173, + "loss": 2.9008, + "theoretical_loss": 3.760142994657822, + "tokens_seen": 736393216 + }, + { + "epoch": 2.04, + "learning_rate": 0.00039235707121364097, + "loss": 3.034, + "theoretical_loss": 3.760109325100032, + "tokens_seen": 736458752 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003923470411233701, + "loss": 2.8678, + "theoretical_loss": 3.7600756593771427, + "tokens_seen": 736524288 + }, + { + "epoch": 2.04, + "learning_rate": 0.00039233701103309933, + "loss": 2.8121, + "theoretical_loss": 3.7600419974883765, + "tokens_seen": 736589824 + }, + { + "epoch": 2.04, + "learning_rate": 0.00039232698094282846, + "loss": 2.7452, + "theoretical_loss": 3.7600083394329555, + "tokens_seen": 736655360 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003923169508525577, + "loss": 2.7862, + "theoretical_loss": 3.7599746852101026, + "tokens_seen": 736720896 + }, + { + "epoch": 2.04, + "learning_rate": 0.00039230692076228687, + "loss": 2.8344, + "theoretical_loss": 3.7599410348190396, + "tokens_seen": 736786432 + }, + { + "epoch": 2.04, + "learning_rate": 0.00039229689067201605, + "loss": 2.8182, + "theoretical_loss": 3.7599073882589904, + "tokens_seen": 736851968 + }, + { + "epoch": 2.04, + "learning_rate": 0.00039228686058174523, + "loss": 2.7019, + "theoretical_loss": 3.759873745529178, + "tokens_seen": 736917504 + }, + { + "epoch": 2.04, + "learning_rate": 0.00039227683049147447, + "loss": 2.7842, + "theoretical_loss": 3.7598401066288254, + "tokens_seen": 736983040 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003922668004012036, + "loss": 2.854, + "theoretical_loss": 3.759806471557156, + "tokens_seen": 737048576 + }, + { + "epoch": 2.04, + "learning_rate": 0.00039225677031093283, + "loss": 2.7951, + "theoretical_loss": 3.7597728403133948, + "tokens_seen": 737114112 + }, + { + "epoch": 2.04, + "learning_rate": 0.00039224674022066196, + "loss": 3.0258, + "theoretical_loss": 3.759739212896765, + "tokens_seen": 737179648 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003922367101303912, + "loss": 2.6118, + "theoretical_loss": 3.759705589306491, + "tokens_seen": 737245184 + }, + { + "epoch": 2.04, + "objective/train/docs_used": 860487, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.353097438812256, + "objective/train/theoretical_loss": 3.759688778945995, + "objective/train/tokens_used": 757737952, + "theoretical_loss": 3.759688778945995, + "tokens_seen": 737277952 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003922266800401204, + "loss": 2.6294, + "theoretical_loss": 3.759671969541797, + "tokens_seen": 737310720 + }, + { + "epoch": 2.04, + "learning_rate": 0.00039221664994984956, + "loss": 2.9801, + "theoretical_loss": 3.759638353601909, + "tokens_seen": 737376256 + }, + { + "epoch": 2.04, + "learning_rate": 0.00039220661985957874, + "loss": 2.6816, + "theoretical_loss": 3.7596047414860507, + "tokens_seen": 737441792 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003921965897693079, + "loss": 2.841, + "theoretical_loss": 3.7595711331934476, + "tokens_seen": 737507328 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003921865596790371, + "loss": 2.8772, + "theoretical_loss": 3.7595375287233255, + "tokens_seen": 737572864 + }, + { + "epoch": 2.04, + "learning_rate": 0.00039217652958876633, + "loss": 2.7096, + "theoretical_loss": 3.75950392807491, + "tokens_seen": 737638400 + }, + { + "epoch": 2.04, + "learning_rate": 0.00039216649949849546, + "loss": 2.7568, + "theoretical_loss": 3.7594703312474262, + "tokens_seen": 737703936 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003921564694082247, + "loss": 2.7315, + "theoretical_loss": 3.7594367382401015, + "tokens_seen": 737769472 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003921464393179539, + "loss": 3.0837, + "theoretical_loss": 3.759403149052161, + "tokens_seen": 737835008 + }, + { + "epoch": 2.04, + "learning_rate": 0.00039213640922768306, + "loss": 2.7529, + "theoretical_loss": 3.7593695636828324, + "tokens_seen": 737900544 + }, + { + "epoch": 2.04, + "learning_rate": 0.00039212637913741224, + "loss": 2.9824, + "theoretical_loss": 3.7593359821313417, + "tokens_seen": 737966080 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003921163490471414, + "loss": 2.8387, + "theoretical_loss": 3.7593024043969163, + "tokens_seen": 738031616 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003921063189568706, + "loss": 2.9527, + "theoretical_loss": 3.759268830478783, + "tokens_seen": 738097152 + }, + { + "epoch": 2.04, + "learning_rate": 0.00039209628886659984, + "loss": 2.7311, + "theoretical_loss": 3.7592352603761703, + "tokens_seen": 738162688 + }, + { + "epoch": 2.04, + "learning_rate": 0.00039208625877632896, + "loss": 2.7515, + "theoretical_loss": 3.7592016940883046, + "tokens_seen": 738228224 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003920762286860582, + "loss": 3.0343, + "theoretical_loss": 3.7591681316144143, + "tokens_seen": 738293760 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003920661985957873, + "loss": 2.7702, + "theoretical_loss": 3.759134572953728, + "tokens_seen": 738359296 + }, + { + "epoch": 2.04, + "learning_rate": 0.00039205616850551656, + "loss": 2.8235, + "theoretical_loss": 3.7591010181054734, + "tokens_seen": 738424832 + }, + { + "epoch": 2.04, + "learning_rate": 0.00039204613841524574, + "loss": 2.6255, + "theoretical_loss": 3.7590674670688795, + "tokens_seen": 738490368 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003920361083249749, + "loss": 2.8931, + "theoretical_loss": 3.759033919843175, + "tokens_seen": 738555904 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003920260782347041, + "loss": 2.9703, + "theoretical_loss": 3.759000376427589, + "tokens_seen": 738621440 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003920160481444333, + "loss": 2.9791, + "theoretical_loss": 3.75896683682135, + "tokens_seen": 738686976 + }, + { + "epoch": 2.04, + "learning_rate": 0.00039200601805416247, + "loss": 2.8601, + "theoretical_loss": 3.758933301023689, + "tokens_seen": 738752512 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003919959879638917, + "loss": 2.8717, + "theoretical_loss": 3.758899769033835, + "tokens_seen": 738818048 + }, + { + "epoch": 2.04, + "learning_rate": 0.00039198595787362083, + "loss": 3.0159, + "theoretical_loss": 3.7588662408510167, + "tokens_seen": 738883584 + }, + { + "epoch": 2.04, + "objective/train/docs_used": 861921, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8210134506225586, + "objective/train/theoretical_loss": 3.758849478187006, + "objective/train/tokens_used": 759376352, + "theoretical_loss": 3.758849478187006, + "tokens_seen": 738916352 + }, + { + "epoch": 2.04, + "learning_rate": 0.00039197592778335007, + "loss": 2.9699, + "theoretical_loss": 3.7588327164744664, + "tokens_seen": 738949120 + }, + { + "epoch": 2.04, + "learning_rate": 0.00039196589769307925, + "loss": 2.7979, + "theoretical_loss": 3.7587991959034133, + "tokens_seen": 739014656 + }, + { + "epoch": 2.04, + "learning_rate": 0.00039195586760280843, + "loss": 2.8731, + "theoretical_loss": 3.758765679137088, + "tokens_seen": 739080192 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003919458375125376, + "loss": 2.8623, + "theoretical_loss": 3.7587321661747213, + "tokens_seen": 739145728 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003919358074222668, + "loss": 2.7755, + "theoretical_loss": 3.758698657015545, + "tokens_seen": 739211264 + }, + { + "epoch": 2.04, + "learning_rate": 0.00039192577733199597, + "loss": 2.9633, + "theoretical_loss": 3.7586651516587892, + "tokens_seen": 739276800 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003919157472417252, + "loss": 2.9176, + "theoretical_loss": 3.758631650103686, + "tokens_seen": 739342336 + }, + { + "epoch": 2.04, + "learning_rate": 0.00039190571715145433, + "loss": 2.8127, + "theoretical_loss": 3.7585981523494674, + "tokens_seen": 739407872 + }, + { + "epoch": 2.04, + "learning_rate": 0.00039189568706118357, + "loss": 2.8019, + "theoretical_loss": 3.7585646583953656, + "tokens_seen": 739473408 + }, + { + "epoch": 2.04, + "learning_rate": 0.00039188565697091275, + "loss": 2.7282, + "theoretical_loss": 3.7585311682406113, + "tokens_seen": 739538944 + }, + { + "epoch": 2.04, + "learning_rate": 0.00039187562688064193, + "loss": 2.8132, + "theoretical_loss": 3.758497681884439, + "tokens_seen": 739604480 + }, + { + "epoch": 2.04, + "learning_rate": 0.00039186559679037117, + "loss": 3.1263, + "theoretical_loss": 3.7584641993260792, + "tokens_seen": 739670016 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003918555667001003, + "loss": 2.7582, + "theoretical_loss": 3.758430720564766, + "tokens_seen": 739735552 + }, + { + "epoch": 2.04, + "learning_rate": 0.00039184553660982953, + "loss": 2.845, + "theoretical_loss": 3.758397245599732, + "tokens_seen": 739801088 + }, + { + "epoch": 2.04, + "learning_rate": 0.00039183550651955866, + "loss": 2.7409, + "theoretical_loss": 3.7583637744302107, + "tokens_seen": 739866624 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003918254764292879, + "loss": 2.9599, + "theoretical_loss": 3.758330307055436, + "tokens_seen": 739932160 + }, + { + "epoch": 2.04, + "learning_rate": 0.00039181544633901707, + "loss": 2.9472, + "theoretical_loss": 3.758296843474641, + "tokens_seen": 739997696 + }, + { + "epoch": 2.04, + "learning_rate": 0.00039180541624874625, + "loss": 2.7435, + "theoretical_loss": 3.7582633836870603, + "tokens_seen": 740063232 + }, + { + "epoch": 2.04, + "learning_rate": 0.00039179538615847543, + "loss": 2.6432, + "theoretical_loss": 3.758229927691927, + "tokens_seen": 740128768 + }, + { + "epoch": 2.04, + "learning_rate": 0.00039178535606820467, + "loss": 3.0198, + "theoretical_loss": 3.758196475488476, + "tokens_seen": 740194304 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003917753259779338, + "loss": 2.8823, + "theoretical_loss": 3.758163027075943, + "tokens_seen": 740259840 + }, + { + "epoch": 2.04, + "learning_rate": 0.00039176529588766303, + "loss": 2.7262, + "theoretical_loss": 3.758129582453561, + "tokens_seen": 740325376 + }, + { + "epoch": 2.04, + "learning_rate": 0.00039175526579739216, + "loss": 2.9131, + "theoretical_loss": 3.758096141620566, + "tokens_seen": 740390912 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003917452357071214, + "loss": 2.8228, + "theoretical_loss": 3.7580627045761936, + "tokens_seen": 740456448 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003917352056168506, + "loss": 2.7725, + "theoretical_loss": 3.7580292713196792, + "tokens_seen": 740521984 + }, + { + "epoch": 2.04, + "objective/train/docs_used": 862761, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8765251636505127, + "objective/train/theoretical_loss": 3.7580125561116295, + "objective/train/tokens_used": 761014752, + "theoretical_loss": 3.7580125561116295, + "tokens_seen": 740554752 + }, + { + "epoch": 2.04, + "learning_rate": 0.00039172517552657976, + "loss": 2.8848, + "theoretical_loss": 3.757995841850258, + "tokens_seen": 740587520 + }, + { + "epoch": 2.04, + "learning_rate": 0.00039171514543630894, + "loss": 2.6698, + "theoretical_loss": 3.7579624161671665, + "tokens_seen": 740653056 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003917051153460381, + "loss": 2.8602, + "theoretical_loss": 3.75792899426964, + "tokens_seen": 740718592 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003916950852557673, + "loss": 2.8744, + "theoretical_loss": 3.757895576156916, + "tokens_seen": 740784128 + }, + { + "epoch": 2.04, + "learning_rate": 0.00039168505516549653, + "loss": 2.9496, + "theoretical_loss": 3.75786216182823, + "tokens_seen": 740849664 + }, + { + "epoch": 2.04, + "learning_rate": 0.00039167502507522566, + "loss": 2.9308, + "theoretical_loss": 3.757828751282821, + "tokens_seen": 740915200 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003916649949849549, + "loss": 3.0178, + "theoretical_loss": 3.757795344519923, + "tokens_seen": 740980736 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003916549648946841, + "loss": 2.8622, + "theoretical_loss": 3.7577619415387753, + "tokens_seen": 741046272 + }, + { + "epoch": 2.04, + "learning_rate": 0.00039164493480441326, + "loss": 2.8771, + "theoretical_loss": 3.757728542338615, + "tokens_seen": 741111808 + }, + { + "epoch": 2.04, + "learning_rate": 0.00039163490471414244, + "loss": 2.652, + "theoretical_loss": 3.75769514691868, + "tokens_seen": 741177344 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003916248746238716, + "loss": 2.9017, + "theoretical_loss": 3.7576617552782077, + "tokens_seen": 741242880 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003916148445336008, + "loss": 2.9632, + "theoretical_loss": 3.757628367416437, + "tokens_seen": 741308416 + }, + { + "epoch": 2.04, + "learning_rate": 0.00039160481444333004, + "loss": 2.6733, + "theoretical_loss": 3.757594983332605, + "tokens_seen": 741373952 + }, + { + "epoch": 2.04, + "learning_rate": 0.00039159478435305916, + "loss": 2.6975, + "theoretical_loss": 3.7575616030259518, + "tokens_seen": 741439488 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003915847542627884, + "loss": 2.9127, + "theoretical_loss": 3.7575282264957153, + "tokens_seen": 741505024 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003915747241725175, + "loss": 2.8389, + "theoretical_loss": 3.7574948537411355, + "tokens_seen": 741570560 + }, + { + "epoch": 2.04, + "learning_rate": 0.00039156469408224676, + "loss": 3.0937, + "theoretical_loss": 3.7574614847614503, + "tokens_seen": 741636096 + }, + { + "epoch": 2.04, + "learning_rate": 0.00039155466399197594, + "loss": 2.979, + "theoretical_loss": 3.7574281195559003, + "tokens_seen": 741701632 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003915446339017051, + "loss": 2.5656, + "theoretical_loss": 3.7573947581237244, + "tokens_seen": 741767168 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003915346038114343, + "loss": 2.6176, + "theoretical_loss": 3.757361400464163, + "tokens_seen": 741832704 + }, + { + "epoch": 2.04, + "learning_rate": 0.0003915245737211635, + "loss": 2.7122, + "theoretical_loss": 3.7573280465764567, + "tokens_seen": 741898240 + }, + { + "epoch": 2.04, + "learning_rate": 0.00039151454363089267, + "loss": 3.0246, + "theoretical_loss": 3.7572946964598453, + "tokens_seen": 741963776 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003915045135406219, + "loss": 2.7568, + "theoretical_loss": 3.757261350113569, + "tokens_seen": 742029312 + }, + { + "epoch": 2.05, + "learning_rate": 0.00039149448345035103, + "loss": 2.8737, + "theoretical_loss": 3.7572280075368694, + "tokens_seen": 742094848 + }, + { + "epoch": 2.05, + "learning_rate": 0.00039148445336008027, + "loss": 2.8704, + "theoretical_loss": 3.757194668728987, + "tokens_seen": 742160384 + }, + { + "epoch": 2.05, + "objective/train/docs_used": 864185, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.501842498779297, + "objective/train/theoretical_loss": 3.757178000738115, + "objective/train/tokens_used": 762653152, + "theoretical_loss": 3.757178000738115, + "tokens_seen": 742193152 + }, + { + "epoch": 2.05, + "learning_rate": 0.00039147442326980945, + "loss": 3.1039, + "theoretical_loss": 3.7571613336891634, + "tokens_seen": 742225920 + }, + { + "epoch": 2.05, + "learning_rate": 0.00039146439317953863, + "loss": 2.7548, + "theoretical_loss": 3.75712800241664, + "tokens_seen": 742291456 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003914543630892678, + "loss": 2.8036, + "theoretical_loss": 3.7570946749106584, + "tokens_seen": 742356992 + }, + { + "epoch": 2.05, + "learning_rate": 0.000391444332998997, + "loss": 2.584, + "theoretical_loss": 3.7570613511704605, + "tokens_seen": 742422528 + }, + { + "epoch": 2.05, + "learning_rate": 0.00039143430290872617, + "loss": 2.9916, + "theoretical_loss": 3.7570280311952886, + "tokens_seen": 742488064 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003914242728184554, + "loss": 2.9727, + "theoretical_loss": 3.7569947149843843, + "tokens_seen": 742553600 + }, + { + "epoch": 2.05, + "learning_rate": 0.00039141424272818453, + "loss": 2.9331, + "theoretical_loss": 3.7569614025369917, + "tokens_seen": 742619136 + }, + { + "epoch": 2.05, + "learning_rate": 0.00039140421263791377, + "loss": 2.5328, + "theoretical_loss": 3.756928093852352, + "tokens_seen": 742684672 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003913941825476429, + "loss": 2.8034, + "theoretical_loss": 3.756894788929709, + "tokens_seen": 742750208 + }, + { + "epoch": 2.05, + "learning_rate": 0.00039138415245737213, + "loss": 2.7063, + "theoretical_loss": 3.7568614877683064, + "tokens_seen": 742815744 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003913741223671013, + "loss": 2.8524, + "theoretical_loss": 3.7568281903673864, + "tokens_seen": 742881280 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003913640922768305, + "loss": 2.6954, + "theoretical_loss": 3.756794896726194, + "tokens_seen": 742946816 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003913540621865597, + "loss": 2.489, + "theoretical_loss": 3.7567616068439715, + "tokens_seen": 743012352 + }, + { + "epoch": 2.05, + "learning_rate": 0.00039134403209628886, + "loss": 2.9102, + "theoretical_loss": 3.756728320719964, + "tokens_seen": 743077888 + }, + { + "epoch": 2.05, + "learning_rate": 0.00039133400200601804, + "loss": 2.6978, + "theoretical_loss": 3.756695038353416, + "tokens_seen": 743143424 + }, + { + "epoch": 2.05, + "learning_rate": 0.00039132397191574727, + "loss": 2.5701, + "theoretical_loss": 3.7566617597435723, + "tokens_seen": 743208960 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003913139418254764, + "loss": 2.5741, + "theoretical_loss": 3.7566284848896765, + "tokens_seen": 743274496 + }, + { + "epoch": 2.05, + "learning_rate": 0.00039130391173520563, + "loss": 2.8461, + "theoretical_loss": 3.756595213790974, + "tokens_seen": 743340032 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003912938816449348, + "loss": 2.6907, + "theoretical_loss": 3.756561946446711, + "tokens_seen": 743405568 + }, + { + "epoch": 2.05, + "learning_rate": 0.000391283851554664, + "loss": 2.9518, + "theoretical_loss": 3.756528682856131, + "tokens_seen": 743471104 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003912738214643932, + "loss": 2.6059, + "theoretical_loss": 3.7564954230184817, + "tokens_seen": 743536640 + }, + { + "epoch": 2.05, + "learning_rate": 0.00039126379137412236, + "loss": 2.8419, + "theoretical_loss": 3.756462166933008, + "tokens_seen": 743602176 + }, + { + "epoch": 2.05, + "learning_rate": 0.00039125376128385154, + "loss": 2.7538, + "theoretical_loss": 3.7564289145989553, + "tokens_seen": 743667712 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003912437311935808, + "loss": 2.7312, + "theoretical_loss": 3.756395666015571, + "tokens_seen": 743733248 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003912337011033099, + "loss": 3.0143, + "theoretical_loss": 3.756362421182101, + "tokens_seen": 743798784 + }, + { + "epoch": 2.05, + "objective/train/docs_used": 864916, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.2605478763580322, + "objective/train/theoretical_loss": 3.7563458001713483, + "objective/train/tokens_used": 764291552, + "theoretical_loss": 3.7563458001713483, + "tokens_seen": 743831552 + }, + { + "epoch": 2.05, + "learning_rate": 0.00039122367101303914, + "loss": 2.9468, + "theoretical_loss": 3.756329180097792, + "tokens_seen": 743864320 + }, + { + "epoch": 2.05, + "learning_rate": 0.00039121364092276826, + "loss": 2.9104, + "theoretical_loss": 3.756295942761891, + "tokens_seen": 743929856 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003912036108324975, + "loss": 2.8218, + "theoretical_loss": 3.7562627091736456, + "tokens_seen": 743995392 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003911935807422267, + "loss": 2.7912, + "theoretical_loss": 3.7562294793323026, + "tokens_seen": 744060928 + }, + { + "epoch": 2.05, + "learning_rate": 0.00039118355065195586, + "loss": 2.9189, + "theoretical_loss": 3.75619625323711, + "tokens_seen": 744126464 + }, + { + "epoch": 2.05, + "learning_rate": 0.00039117352056168504, + "loss": 2.9026, + "theoretical_loss": 3.7561630308873153, + "tokens_seen": 744192000 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003911634904714143, + "loss": 2.9202, + "theoretical_loss": 3.756129812282167, + "tokens_seen": 744257536 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003911534603811434, + "loss": 2.7454, + "theoretical_loss": 3.7560965974209126, + "tokens_seen": 744323072 + }, + { + "epoch": 2.05, + "learning_rate": 0.00039114343029087264, + "loss": 3.0462, + "theoretical_loss": 3.756063386302801, + "tokens_seen": 744388608 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003911334002006018, + "loss": 2.9229, + "theoretical_loss": 3.7560301789270807, + "tokens_seen": 744454144 + }, + { + "epoch": 2.05, + "learning_rate": 0.000391123370110331, + "loss": 2.6486, + "theoretical_loss": 3.755996975293001, + "tokens_seen": 744519680 + }, + { + "epoch": 2.05, + "learning_rate": 0.00039111334002006024, + "loss": 2.7592, + "theoretical_loss": 3.7559637753998105, + "tokens_seen": 744585216 + }, + { + "epoch": 2.05, + "learning_rate": 0.00039110330992978936, + "loss": 2.579, + "theoretical_loss": 3.755930579246759, + "tokens_seen": 744650752 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003910932798395186, + "loss": 2.6565, + "theoretical_loss": 3.7558973868330954, + "tokens_seen": 744716288 + }, + { + "epoch": 2.05, + "learning_rate": 0.00039108324974924773, + "loss": 2.6632, + "theoretical_loss": 3.7558641981580703, + "tokens_seen": 744781824 + }, + { + "epoch": 2.05, + "learning_rate": 0.00039107321965897696, + "loss": 2.5718, + "theoretical_loss": 3.7558310132209334, + "tokens_seen": 744847360 + }, + { + "epoch": 2.05, + "learning_rate": 0.00039106318956870614, + "loss": 2.6674, + "theoretical_loss": 3.755797832020934, + "tokens_seen": 744912896 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003910531594784353, + "loss": 2.8298, + "theoretical_loss": 3.755764654557324, + "tokens_seen": 744978432 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003910431293881645, + "loss": 3.0806, + "theoretical_loss": 3.755731480829353, + "tokens_seen": 745043968 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003910330992978937, + "loss": 2.6907, + "theoretical_loss": 3.755698310836272, + "tokens_seen": 745109504 + }, + { + "epoch": 2.05, + "learning_rate": 0.00039102306920762287, + "loss": 2.843, + "theoretical_loss": 3.7556651445773324, + "tokens_seen": 745175040 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003910130391173521, + "loss": 2.8829, + "theoretical_loss": 3.755631982051785, + "tokens_seen": 745240576 + }, + { + "epoch": 2.05, + "learning_rate": 0.00039100300902708123, + "loss": 2.7557, + "theoretical_loss": 3.7555988232588815, + "tokens_seen": 745306112 + }, + { + "epoch": 2.05, + "learning_rate": 0.00039099297893681047, + "loss": 2.8817, + "theoretical_loss": 3.7555656681978737, + "tokens_seen": 745371648 + }, + { + "epoch": 2.05, + "learning_rate": 0.00039098294884653965, + "loss": 2.8561, + "theoretical_loss": 3.755532516868014, + "tokens_seen": 745437184 + }, + { + "epoch": 2.05, + "objective/train/docs_used": 865470, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9077301025390625, + "objective/train/theoretical_loss": 3.75551594260203, + "objective/train/tokens_used": 765929952, + "theoretical_loss": 3.75551594260203, + "tokens_seen": 745469952 + }, + { + "epoch": 2.05, + "learning_rate": 0.00039097291875626883, + "loss": 2.9101, + "theoretical_loss": 3.755499369268553, + "tokens_seen": 745502720 + }, + { + "epoch": 2.05, + "learning_rate": 0.000390962888665998, + "loss": 2.9018, + "theoretical_loss": 3.7554662253987443, + "tokens_seen": 745568256 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003909528585757272, + "loss": 2.7832, + "theoretical_loss": 3.755433085257841, + "tokens_seen": 745633792 + }, + { + "epoch": 2.05, + "learning_rate": 0.00039094282848545637, + "loss": 2.9756, + "theoretical_loss": 3.755399948845094, + "tokens_seen": 745699328 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003909327983951856, + "loss": 2.7584, + "theoretical_loss": 3.7553668161597584, + "tokens_seen": 745764864 + }, + { + "epoch": 2.05, + "learning_rate": 0.00039092276830491473, + "loss": 2.886, + "theoretical_loss": 3.7553336872010856, + "tokens_seen": 745830400 + }, + { + "epoch": 2.05, + "learning_rate": 0.00039091273821464397, + "loss": 3.0352, + "theoretical_loss": 3.7553005619683306, + "tokens_seen": 745895936 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003909027081243731, + "loss": 2.733, + "theoretical_loss": 3.755267440460746, + "tokens_seen": 745961472 + }, + { + "epoch": 2.05, + "learning_rate": 0.00039089267803410233, + "loss": 2.7691, + "theoretical_loss": 3.7552343226775857, + "tokens_seen": 746027008 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003908826479438315, + "loss": 2.886, + "theoretical_loss": 3.7552012086181046, + "tokens_seen": 746092544 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003908726178535607, + "loss": 2.7298, + "theoretical_loss": 3.755168098281556, + "tokens_seen": 746158080 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003908625877632899, + "loss": 2.7824, + "theoretical_loss": 3.755134991667195, + "tokens_seen": 746223616 + }, + { + "epoch": 2.05, + "learning_rate": 0.00039085255767301906, + "loss": 2.8606, + "theoretical_loss": 3.755101888774276, + "tokens_seen": 746289152 + }, + { + "epoch": 2.05, + "learning_rate": 0.00039084252758274824, + "loss": 2.481, + "theoretical_loss": 3.755068789602054, + "tokens_seen": 746354688 + }, + { + "epoch": 2.05, + "learning_rate": 0.00039083249749247747, + "loss": 2.5479, + "theoretical_loss": 3.755035694149784, + "tokens_seen": 746420224 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003908224674022066, + "loss": 2.9372, + "theoretical_loss": 3.7550026024167225, + "tokens_seen": 746485760 + }, + { + "epoch": 2.05, + "learning_rate": 0.00039081243731193583, + "loss": 2.9928, + "theoretical_loss": 3.7549695144021236, + "tokens_seen": 746551296 + }, + { + "epoch": 2.05, + "learning_rate": 0.000390802407221665, + "loss": 2.8547, + "theoretical_loss": 3.7549364301052437, + "tokens_seen": 746616832 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003907923771313942, + "loss": 2.6554, + "theoretical_loss": 3.7549033495253386, + "tokens_seen": 746682368 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003907823470411234, + "loss": 2.6698, + "theoretical_loss": 3.754870272661665, + "tokens_seen": 746747904 + }, + { + "epoch": 2.05, + "learning_rate": 0.00039077231695085256, + "loss": 2.8375, + "theoretical_loss": 3.754837199513479, + "tokens_seen": 746813440 + }, + { + "epoch": 2.05, + "learning_rate": 0.00039076228686058174, + "loss": 2.6853, + "theoretical_loss": 3.7548041300800366, + "tokens_seen": 746878976 + }, + { + "epoch": 2.05, + "learning_rate": 0.000390752256770311, + "loss": 2.8241, + "theoretical_loss": 3.7547710643605963, + "tokens_seen": 746944512 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003907422266800401, + "loss": 2.5937, + "theoretical_loss": 3.7547380023544132, + "tokens_seen": 747010048 + }, + { + "epoch": 2.05, + "learning_rate": 0.00039073219658976934, + "loss": 2.6594, + "theoretical_loss": 3.7547049440607467, + "tokens_seen": 747075584 + }, + { + "epoch": 2.05, + "objective/train/docs_used": 865470, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7921359539031982, + "objective/train/theoretical_loss": 3.7546884163058745, + "objective/train/tokens_used": 767568352, + "theoretical_loss": 3.7546884163058745, + "tokens_seen": 747108352 + }, + { + "epoch": 2.05, + "learning_rate": 0.00039072216649949846, + "loss": 2.9316, + "theoretical_loss": 3.7546718894788524, + "tokens_seen": 747141120 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003907121364092277, + "loss": 2.9271, + "theoretical_loss": 3.754638838607989, + "tokens_seen": 747206656 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003907021063189569, + "loss": 2.662, + "theoretical_loss": 3.754605791447415, + "tokens_seen": 747272192 + }, + { + "epoch": 2.05, + "learning_rate": 0.00039069207622868606, + "loss": 2.846, + "theoretical_loss": 3.754572747996387, + "tokens_seen": 747337728 + }, + { + "epoch": 2.05, + "learning_rate": 0.00039068204613841524, + "loss": 2.6688, + "theoretical_loss": 3.754539708254164, + "tokens_seen": 747403264 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003906720160481445, + "loss": 3.0716, + "theoretical_loss": 3.754506672220005, + "tokens_seen": 747468800 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003906619859578736, + "loss": 2.5733, + "theoretical_loss": 3.7544736398931686, + "tokens_seen": 747534336 + }, + { + "epoch": 2.05, + "learning_rate": 0.00039065195586760284, + "loss": 2.8922, + "theoretical_loss": 3.754440611272914, + "tokens_seen": 747599872 + }, + { + "epoch": 2.05, + "learning_rate": 0.00039064192577733197, + "loss": 2.7972, + "theoretical_loss": 3.7544075863585, + "tokens_seen": 747665408 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003906318956870612, + "loss": 2.6071, + "theoretical_loss": 3.7543745651491855, + "tokens_seen": 747730944 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003906218655967904, + "loss": 2.8335, + "theoretical_loss": 3.7543415476442314, + "tokens_seen": 747796480 + }, + { + "epoch": 2.05, + "learning_rate": 0.00039061183550651957, + "loss": 3.1237, + "theoretical_loss": 3.754308533842897, + "tokens_seen": 747862016 + }, + { + "epoch": 2.05, + "learning_rate": 0.00039060180541624875, + "loss": 2.6559, + "theoretical_loss": 3.7542755237444423, + "tokens_seen": 747927552 + }, + { + "epoch": 2.05, + "learning_rate": 0.00039059177532597793, + "loss": 2.877, + "theoretical_loss": 3.754242517348127, + "tokens_seen": 747993088 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003905817452357071, + "loss": 2.6894, + "theoretical_loss": 3.754209514653213, + "tokens_seen": 748058624 + }, + { + "epoch": 2.05, + "learning_rate": 0.00039057171514543634, + "loss": 3.0517, + "theoretical_loss": 3.7541765156589593, + "tokens_seen": 748124160 + }, + { + "epoch": 2.05, + "learning_rate": 0.00039056168505516547, + "loss": 2.8098, + "theoretical_loss": 3.754143520364628, + "tokens_seen": 748189696 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003905516549648947, + "loss": 2.7217, + "theoretical_loss": 3.7541105287694796, + "tokens_seen": 748255232 + }, + { + "epoch": 2.05, + "learning_rate": 0.00039054162487462383, + "loss": 2.8167, + "theoretical_loss": 3.7540775408727765, + "tokens_seen": 748320768 + }, + { + "epoch": 2.05, + "learning_rate": 0.00039053159478435307, + "loss": 2.766, + "theoretical_loss": 3.7540445566737786, + "tokens_seen": 748386304 + }, + { + "epoch": 2.05, + "learning_rate": 0.00039052156469408225, + "loss": 2.7154, + "theoretical_loss": 3.754011576171749, + "tokens_seen": 748451840 + }, + { + "epoch": 2.05, + "learning_rate": 0.00039051153460381143, + "loss": 2.7892, + "theoretical_loss": 3.7539785993659494, + "tokens_seen": 748517376 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003905015045135406, + "loss": 2.6884, + "theoretical_loss": 3.7539456262556414, + "tokens_seen": 748582912 + }, + { + "epoch": 2.05, + "learning_rate": 0.00039049147442326985, + "loss": 3.0133, + "theoretical_loss": 3.753912656840088, + "tokens_seen": 748648448 + }, + { + "epoch": 2.05, + "learning_rate": 0.000390481444332999, + "loss": 2.7864, + "theoretical_loss": 3.7538796911185512, + "tokens_seen": 748713984 + }, + { + "epoch": 2.05, + "objective/train/docs_used": 866234, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.680988311767578, + "objective/train/theoretical_loss": 3.7538632096428093, + "objective/train/tokens_used": 769206752, + "theoretical_loss": 3.7538632096428093, + "tokens_seen": 748746752 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003904714142427282, + "loss": 2.649, + "theoretical_loss": 3.753846729090295, + "tokens_seen": 748779520 + }, + { + "epoch": 2.05, + "learning_rate": 0.00039046138415245734, + "loss": 2.9356, + "theoretical_loss": 3.753813770754581, + "tokens_seen": 748845056 + }, + { + "epoch": 2.05, + "learning_rate": 0.00039045135406218657, + "loss": 2.8491, + "theoretical_loss": 3.753780816110673, + "tokens_seen": 748910592 + }, + { + "epoch": 2.05, + "learning_rate": 0.00039044132397191575, + "loss": 2.9241, + "theoretical_loss": 3.7537478651578358, + "tokens_seen": 748976128 + }, + { + "epoch": 2.05, + "learning_rate": 0.00039043129388164493, + "loss": 2.7688, + "theoretical_loss": 3.7537149178953304, + "tokens_seen": 749041664 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003904212637913741, + "loss": 2.8314, + "theoretical_loss": 3.7536819743224226, + "tokens_seen": 749107200 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003904112337011033, + "loss": 2.9138, + "theoretical_loss": 3.753649034438376, + "tokens_seen": 749172736 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003904012036108325, + "loss": 2.713, + "theoretical_loss": 3.753616098242455, + "tokens_seen": 749238272 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003903911735205617, + "loss": 2.8089, + "theoretical_loss": 3.753583165733924, + "tokens_seen": 749303808 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003903811434302909, + "loss": 2.9077, + "theoretical_loss": 3.7535502369120475, + "tokens_seen": 749369344 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003903711133400201, + "loss": 2.924, + "theoretical_loss": 3.7535173117760907, + "tokens_seen": 749434880 + }, + { + "epoch": 2.05, + "learning_rate": 0.00039036108324974926, + "loss": 2.7313, + "theoretical_loss": 3.753484390325319, + "tokens_seen": 749500416 + }, + { + "epoch": 2.05, + "learning_rate": 0.00039035105315947844, + "loss": 2.7019, + "theoretical_loss": 3.753451472558997, + "tokens_seen": 749565952 + }, + { + "epoch": 2.05, + "learning_rate": 0.00039034102306920767, + "loss": 2.9285, + "theoretical_loss": 3.7534185584763904, + "tokens_seen": 749631488 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003903309929789368, + "loss": 2.5482, + "theoretical_loss": 3.7533856480767662, + "tokens_seen": 749697024 + }, + { + "epoch": 2.05, + "learning_rate": 0.00039032096288866603, + "loss": 2.7619, + "theoretical_loss": 3.753352741359389, + "tokens_seen": 749762560 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003903109327983952, + "loss": 2.9135, + "theoretical_loss": 3.7533198383235256, + "tokens_seen": 749828096 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003903009027081244, + "loss": 2.7497, + "theoretical_loss": 3.753286938968442, + "tokens_seen": 749893632 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003902908726178536, + "loss": 2.8409, + "theoretical_loss": 3.7532540432934054, + "tokens_seen": 749959168 + }, + { + "epoch": 2.05, + "learning_rate": 0.00039028084252758276, + "loss": 2.9118, + "theoretical_loss": 3.7532211512976823, + "tokens_seen": 750024704 + }, + { + "epoch": 2.05, + "learning_rate": 0.00039027081243731194, + "loss": 2.8197, + "theoretical_loss": 3.7531882629805393, + "tokens_seen": 750090240 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003902607823470412, + "loss": 2.7701, + "theoretical_loss": 3.753155378341245, + "tokens_seen": 750155776 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003902507522567703, + "loss": 2.9495, + "theoretical_loss": 3.7531224973790653, + "tokens_seen": 750221312 + }, + { + "epoch": 2.05, + "learning_rate": 0.00039024072216649954, + "loss": 2.7622, + "theoretical_loss": 3.7530896200932684, + "tokens_seen": 750286848 + }, + { + "epoch": 2.05, + "learning_rate": 0.00039023069207622866, + "loss": 2.7241, + "theoretical_loss": 3.7530567464831233, + "tokens_seen": 750352384 + }, + { + "epoch": 2.05, + "objective/train/docs_used": 867529, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8023226261138916, + "objective/train/theoretical_loss": 3.75304031105619, + "objective/train/tokens_used": 770845152, + "theoretical_loss": 3.75304031105619, + "tokens_seen": 750385152 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003902206619859579, + "loss": 2.8509, + "theoretical_loss": 3.7530238765478963, + "tokens_seen": 750417920 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003902106318956871, + "loss": 2.8771, + "theoretical_loss": 3.7529910102868564, + "tokens_seen": 750483456 + }, + { + "epoch": 2.05, + "learning_rate": 0.00039020060180541626, + "loss": 2.7444, + "theoretical_loss": 3.752958147699272, + "tokens_seen": 750548992 + }, + { + "epoch": 2.05, + "learning_rate": 0.00039019057171514544, + "loss": 2.7908, + "theoretical_loss": 3.7529252887844127, + "tokens_seen": 750614528 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003901805416248747, + "loss": 2.9501, + "theoretical_loss": 3.7528924335415463, + "tokens_seen": 750680064 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003901705115346038, + "loss": 2.8367, + "theoretical_loss": 3.7528595819699424, + "tokens_seen": 750745600 + }, + { + "epoch": 2.05, + "learning_rate": 0.00039016048144433304, + "loss": 2.6838, + "theoretical_loss": 3.752826734068871, + "tokens_seen": 750811136 + }, + { + "epoch": 2.05, + "learning_rate": 0.00039015045135406217, + "loss": 2.7132, + "theoretical_loss": 3.7527938898376, + "tokens_seen": 750876672 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003901404212637914, + "loss": 2.7584, + "theoretical_loss": 3.752761049275401, + "tokens_seen": 750942208 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003901303911735206, + "loss": 2.8268, + "theoretical_loss": 3.7527282123815424, + "tokens_seen": 751007744 + }, + { + "epoch": 2.05, + "learning_rate": 0.00039012036108324977, + "loss": 3.2001, + "theoretical_loss": 3.7526953791552953, + "tokens_seen": 751073280 + }, + { + "epoch": 2.05, + "learning_rate": 0.00039011033099297895, + "loss": 2.7938, + "theoretical_loss": 3.7526625495959296, + "tokens_seen": 751138816 + }, + { + "epoch": 2.05, + "learning_rate": 0.00039010030090270813, + "loss": 2.8428, + "theoretical_loss": 3.752629723702717, + "tokens_seen": 751204352 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003900902708124373, + "loss": 2.7589, + "theoretical_loss": 3.752596901474927, + "tokens_seen": 751269888 + }, + { + "epoch": 2.05, + "learning_rate": 0.00039008024072216654, + "loss": 2.8637, + "theoretical_loss": 3.7525640829118307, + "tokens_seen": 751335424 + }, + { + "epoch": 2.05, + "learning_rate": 0.00039007021063189567, + "loss": 2.7897, + "theoretical_loss": 3.7525312680127003, + "tokens_seen": 751400960 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003900601805416249, + "loss": 2.9186, + "theoretical_loss": 3.7524984567768063, + "tokens_seen": 751466496 + }, + { + "epoch": 2.05, + "learning_rate": 0.00039005015045135403, + "loss": 2.819, + "theoretical_loss": 3.7524656492034207, + "tokens_seen": 751532032 + }, + { + "epoch": 2.05, + "learning_rate": 0.00039004012036108327, + "loss": 2.9226, + "theoretical_loss": 3.752432845291816, + "tokens_seen": 751597568 + }, + { + "epoch": 2.05, + "learning_rate": 0.00039003009027081245, + "loss": 2.8002, + "theoretical_loss": 3.752400045041263, + "tokens_seen": 751663104 + }, + { + "epoch": 2.05, + "learning_rate": 0.00039002006018054163, + "loss": 2.6275, + "theoretical_loss": 3.7523672484510344, + "tokens_seen": 751728640 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003900100300902708, + "loss": 3.0589, + "theoretical_loss": 3.752334455520403, + "tokens_seen": 751794176 + }, + { + "epoch": 2.05, + "learning_rate": 0.00039000000000000005, + "loss": 3.0157, + "theoretical_loss": 3.7523016662486413, + "tokens_seen": 751859712 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003899899699097292, + "loss": 2.6684, + "theoretical_loss": 3.752268880635022, + "tokens_seen": 751925248 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003899799398194584, + "loss": 3.0901, + "theoretical_loss": 3.752236098678819, + "tokens_seen": 751990784 + }, + { + "epoch": 2.05, + "objective/train/docs_used": 868214, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.602017402648926, + "objective/train/theoretical_loss": 3.752219709072021, + "objective/train/tokens_used": 772483552, + "theoretical_loss": 3.752219709072021, + "tokens_seen": 752023552 + }, + { + "epoch": 2.05, + "learning_rate": 0.00038996990972918754, + "loss": 2.9366, + "theoretical_loss": 3.7522033203793046, + "tokens_seen": 752056320 + }, + { + "epoch": 2.05, + "learning_rate": 0.00038995987963891677, + "loss": 2.9654, + "theoretical_loss": 3.752170545735753, + "tokens_seen": 752121856 + }, + { + "epoch": 2.05, + "learning_rate": 0.00038994984954864595, + "loss": 2.8639, + "theoretical_loss": 3.752137774747437, + "tokens_seen": 752187392 + }, + { + "epoch": 2.05, + "learning_rate": 0.00038993981945837513, + "loss": 2.8474, + "theoretical_loss": 3.7521050074136317, + "tokens_seen": 752252928 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003899297893681043, + "loss": 2.9317, + "theoretical_loss": 3.7520722437336107, + "tokens_seen": 752318464 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003899197592778335, + "loss": 2.9675, + "theoretical_loss": 3.752039483706648, + "tokens_seen": 752384000 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003899097291875627, + "loss": 3.0981, + "theoretical_loss": 3.752006727332019, + "tokens_seen": 752449536 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003898996990972919, + "loss": 2.8965, + "theoretical_loss": 3.7519739746089975, + "tokens_seen": 752515072 + }, + { + "epoch": 2.05, + "learning_rate": 0.00038988966900702104, + "loss": 2.7717, + "theoretical_loss": 3.751941225536859, + "tokens_seen": 752580608 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003898796389167503, + "loss": 2.8899, + "theoretical_loss": 3.7519084801148788, + "tokens_seen": 752646144 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003898696088264794, + "loss": 2.9638, + "theoretical_loss": 3.751875738342332, + "tokens_seen": 752711680 + }, + { + "epoch": 2.05, + "learning_rate": 0.00038985957873620864, + "loss": 2.8868, + "theoretical_loss": 3.7518430002184937, + "tokens_seen": 752777216 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003898495486459378, + "loss": 2.9566, + "theoretical_loss": 3.751810265742641, + "tokens_seen": 752842752 + }, + { + "epoch": 2.05, + "learning_rate": 0.000389839518555667, + "loss": 2.8835, + "theoretical_loss": 3.7517775349140488, + "tokens_seen": 752908288 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003898294884653962, + "loss": 2.7896, + "theoretical_loss": 3.7517448077319937, + "tokens_seen": 752973824 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003898194583751254, + "loss": 2.6797, + "theoretical_loss": 3.751712084195752, + "tokens_seen": 753039360 + }, + { + "epoch": 2.05, + "learning_rate": 0.00038980942828485454, + "loss": 2.8957, + "theoretical_loss": 3.7516793643046005, + "tokens_seen": 753104896 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003897993981945838, + "loss": 2.8789, + "theoretical_loss": 3.7516466480578154, + "tokens_seen": 753170432 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003897893681043129, + "loss": 2.9751, + "theoretical_loss": 3.751613935454675, + "tokens_seen": 753235968 + }, + { + "epoch": 2.05, + "learning_rate": 0.00038977933801404214, + "loss": 3.1071, + "theoretical_loss": 3.7515812264944555, + "tokens_seen": 753301504 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003897693079237713, + "loss": 2.8043, + "theoretical_loss": 3.751548521176434, + "tokens_seen": 753367040 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003897592778335005, + "loss": 2.8944, + "theoretical_loss": 3.7515158194998897, + "tokens_seen": 753432576 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003897492477432297, + "loss": 2.8687, + "theoretical_loss": 3.7514831214640987, + "tokens_seen": 753498112 + }, + { + "epoch": 2.05, + "learning_rate": 0.00038973921765295886, + "loss": 2.8928, + "theoretical_loss": 3.751450427068341, + "tokens_seen": 753563648 + }, + { + "epoch": 2.05, + "learning_rate": 0.00038972918756268805, + "loss": 3.0386, + "theoretical_loss": 3.7514177363118923, + "tokens_seen": 753629184 + }, + { + "epoch": 2.05, + "objective/train/docs_used": 869556, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.754945755004883, + "objective/train/theoretical_loss": 3.7514013922981846, + "objective/train/tokens_used": 774121952, + "theoretical_loss": 3.7514013922981846, + "tokens_seen": 753661952 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003897191574724173, + "loss": 2.9253, + "theoretical_loss": 3.7513850491940337, + "tokens_seen": 753694720 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003897091273821464, + "loss": 2.8255, + "theoretical_loss": 3.751352365714042, + "tokens_seen": 753760256 + }, + { + "epoch": 2.05, + "learning_rate": 0.00038969909729187564, + "loss": 2.8426, + "theoretical_loss": 3.7513196858711972, + "tokens_seen": 753825792 + }, + { + "epoch": 2.05, + "learning_rate": 0.00038968906720160477, + "loss": 2.9487, + "theoretical_loss": 3.751287009664778, + "tokens_seen": 753891328 + }, + { + "epoch": 2.05, + "learning_rate": 0.000389679037111334, + "loss": 2.7612, + "theoretical_loss": 3.751254337094063, + "tokens_seen": 753956864 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003896690070210632, + "loss": 3.0445, + "theoretical_loss": 3.7512216681583332, + "tokens_seen": 754022400 + }, + { + "epoch": 2.05, + "learning_rate": 0.00038965897693079237, + "loss": 2.9938, + "theoretical_loss": 3.751189002856867, + "tokens_seen": 754087936 + }, + { + "epoch": 2.05, + "learning_rate": 0.00038964894684052155, + "loss": 2.8911, + "theoretical_loss": 3.751156341188945, + "tokens_seen": 754153472 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003896389167502508, + "loss": 2.8629, + "theoretical_loss": 3.751123683153847, + "tokens_seen": 754219008 + }, + { + "epoch": 2.05, + "learning_rate": 0.00038962888665997997, + "loss": 2.8401, + "theoretical_loss": 3.7510910287508534, + "tokens_seen": 754284544 + }, + { + "epoch": 2.05, + "learning_rate": 0.00038961885656970915, + "loss": 2.8568, + "theoretical_loss": 3.751058377979245, + "tokens_seen": 754350080 + }, + { + "epoch": 2.05, + "learning_rate": 0.00038960882647943833, + "loss": 2.8119, + "theoretical_loss": 3.751025730838302, + "tokens_seen": 754415616 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003895987963891675, + "loss": 2.9211, + "theoretical_loss": 3.7509930873273056, + "tokens_seen": 754481152 + }, + { + "epoch": 2.05, + "learning_rate": 0.00038958876629889674, + "loss": 3.0088, + "theoretical_loss": 3.7509604474455376, + "tokens_seen": 754546688 + }, + { + "epoch": 2.05, + "learning_rate": 0.00038957873620862587, + "loss": 2.8507, + "theoretical_loss": 3.750927811192278, + "tokens_seen": 754612224 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003895687061183551, + "loss": 2.8428, + "theoretical_loss": 3.750895178566809, + "tokens_seen": 754677760 + }, + { + "epoch": 2.05, + "learning_rate": 0.00038955867602808423, + "loss": 2.857, + "theoretical_loss": 3.7508625495684136, + "tokens_seen": 754743296 + }, + { + "epoch": 2.05, + "learning_rate": 0.00038954864593781347, + "loss": 2.9037, + "theoretical_loss": 3.7508299241963714, + "tokens_seen": 754808832 + }, + { + "epoch": 2.05, + "learning_rate": 0.00038953861584754265, + "loss": 2.688, + "theoretical_loss": 3.7507973024499663, + "tokens_seen": 754874368 + }, + { + "epoch": 2.05, + "learning_rate": 0.00038952858575727183, + "loss": 2.856, + "theoretical_loss": 3.7507646843284803, + "tokens_seen": 754939904 + }, + { + "epoch": 2.05, + "learning_rate": 0.000389518555667001, + "loss": 2.7452, + "theoretical_loss": 3.7507320698311952, + "tokens_seen": 755005440 + }, + { + "epoch": 2.05, + "learning_rate": 0.00038950852557673025, + "loss": 2.6562, + "theoretical_loss": 3.7506994589573948, + "tokens_seen": 755070976 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003894984954864594, + "loss": 2.8558, + "theoretical_loss": 3.7506668517063613, + "tokens_seen": 755136512 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003894884653961886, + "loss": 3.0562, + "theoretical_loss": 3.750634248077379, + "tokens_seen": 755202048 + }, + { + "epoch": 2.05, + "learning_rate": 0.00038947843530591774, + "loss": 2.8171, + "theoretical_loss": 3.75060164806973, + "tokens_seen": 755267584 + }, + { + "epoch": 2.05, + "objective/train/docs_used": 870017, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.080669403076172, + "objective/train/theoretical_loss": 3.750585349423682, + "objective/train/tokens_used": 775760352, + "theoretical_loss": 3.750585349423682, + "tokens_seen": 755300352 + }, + { + "epoch": 2.05, + "learning_rate": 0.00038946840521564697, + "loss": 2.8293, + "theoretical_loss": 3.7505690516826986, + "tokens_seen": 755333120 + }, + { + "epoch": 2.05, + "learning_rate": 0.00038945837512537615, + "loss": 2.8135, + "theoretical_loss": 3.7505364589155685, + "tokens_seen": 755398656 + }, + { + "epoch": 2.05, + "learning_rate": 0.00038944834503510533, + "loss": 2.8743, + "theoretical_loss": 3.7505038697676234, + "tokens_seen": 755464192 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003894383149448345, + "loss": 2.8677, + "theoretical_loss": 3.750471284238148, + "tokens_seen": 755529728 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003894282848545637, + "loss": 2.7691, + "theoretical_loss": 3.7504387023264263, + "tokens_seen": 755595264 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003894182547642929, + "loss": 2.9833, + "theoretical_loss": 3.750406124031743, + "tokens_seen": 755660800 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003894082246740221, + "loss": 2.9658, + "theoretical_loss": 3.750373549353383, + "tokens_seen": 755726336 + }, + { + "epoch": 2.05, + "learning_rate": 0.00038939819458375124, + "loss": 2.7576, + "theoretical_loss": 3.750340978290632, + "tokens_seen": 755791872 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003893881644934805, + "loss": 2.7332, + "theoretical_loss": 3.7503084108427736, + "tokens_seen": 755857408 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003893781344032096, + "loss": 2.7703, + "theoretical_loss": 3.750275847009095, + "tokens_seen": 755922944 + }, + { + "epoch": 2.05, + "learning_rate": 0.00038936810431293884, + "loss": 2.7189, + "theoretical_loss": 3.7502432867888804, + "tokens_seen": 755988480 + }, + { + "epoch": 2.05, + "learning_rate": 0.000389358074222668, + "loss": 2.7728, + "theoretical_loss": 3.7502107301814167, + "tokens_seen": 756054016 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003893480441323972, + "loss": 2.6577, + "theoretical_loss": 3.750178177185989, + "tokens_seen": 756119552 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003893380140421264, + "loss": 2.7496, + "theoretical_loss": 3.750145627801884, + "tokens_seen": 756185088 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003893279839518556, + "loss": 2.9146, + "theoretical_loss": 3.7501130820283883, + "tokens_seen": 756250624 + }, + { + "epoch": 2.05, + "learning_rate": 0.00038931795386158474, + "loss": 2.7928, + "theoretical_loss": 3.750080539864789, + "tokens_seen": 756316160 + }, + { + "epoch": 2.05, + "learning_rate": 0.000389307923771314, + "loss": 2.8623, + "theoretical_loss": 3.750048001310372, + "tokens_seen": 756381696 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003892978936810431, + "loss": 2.5961, + "theoretical_loss": 3.750015466364424, + "tokens_seen": 756447232 + }, + { + "epoch": 2.05, + "learning_rate": 0.00038928786359077234, + "loss": 2.6931, + "theoretical_loss": 3.7499829350262335, + "tokens_seen": 756512768 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003892778335005015, + "loss": 2.8814, + "theoretical_loss": 3.7499504072950876, + "tokens_seen": 756578304 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003892678034102307, + "loss": 2.7536, + "theoretical_loss": 3.7499178831702737, + "tokens_seen": 756643840 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003892577733199599, + "loss": 2.8347, + "theoretical_loss": 3.7498853626510797, + "tokens_seen": 756709376 + }, + { + "epoch": 2.05, + "learning_rate": 0.00038924774322968906, + "loss": 2.7921, + "theoretical_loss": 3.7498528457367932, + "tokens_seen": 756774912 + }, + { + "epoch": 2.05, + "learning_rate": 0.00038923771313941825, + "loss": 2.9477, + "theoretical_loss": 3.7498203324267037, + "tokens_seen": 756840448 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003892276830491475, + "loss": 3.0231, + "theoretical_loss": 3.7497878227200983, + "tokens_seen": 756905984 + }, + { + "epoch": 2.05, + "objective/train/docs_used": 871516, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.127725601196289, + "objective/train/theoretical_loss": 3.74977156921788, + "objective/train/tokens_used": 777398752, + "theoretical_loss": 3.74977156921788, + "tokens_seen": 756938752 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003892176529588766, + "loss": 2.9308, + "theoretical_loss": 3.749755316616267, + "tokens_seen": 756971520 + }, + { + "epoch": 2.05, + "learning_rate": 0.00038920762286860584, + "loss": 2.8961, + "theoretical_loss": 3.749722814114497, + "tokens_seen": 757037056 + }, + { + "epoch": 2.05, + "learning_rate": 0.00038919759277833497, + "loss": 2.7532, + "theoretical_loss": 3.749690315214079, + "tokens_seen": 757102592 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003891875626880642, + "loss": 3.0619, + "theoretical_loss": 3.7496578199143014, + "tokens_seen": 757168128 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003891775325977934, + "loss": 2.6638, + "theoretical_loss": 3.749625328214454, + "tokens_seen": 757233664 + }, + { + "epoch": 2.05, + "learning_rate": 0.00038916750250752257, + "loss": 2.9055, + "theoretical_loss": 3.7495928401138263, + "tokens_seen": 757299200 + }, + { + "epoch": 2.05, + "learning_rate": 0.00038915747241725175, + "loss": 2.7871, + "theoretical_loss": 3.749560355611708, + "tokens_seen": 757364736 + }, + { + "epoch": 2.05, + "learning_rate": 0.000389147442326981, + "loss": 2.674, + "theoretical_loss": 3.74952787470739, + "tokens_seen": 757430272 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003891374122367101, + "loss": 2.8836, + "theoretical_loss": 3.7494953974001617, + "tokens_seen": 757495808 + }, + { + "epoch": 2.05, + "learning_rate": 0.00038912738214643935, + "loss": 2.6771, + "theoretical_loss": 3.7494629236893138, + "tokens_seen": 757561344 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003891173520561685, + "loss": 2.981, + "theoretical_loss": 3.7494304535741376, + "tokens_seen": 757626880 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003891073219658977, + "loss": 3.0572, + "theoretical_loss": 3.7493979870539227, + "tokens_seen": 757692416 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003890972918756269, + "loss": 3.0317, + "theoretical_loss": 3.7493655241279615, + "tokens_seen": 757757952 + }, + { + "epoch": 2.05, + "learning_rate": 0.00038908726178535607, + "loss": 2.9948, + "theoretical_loss": 3.7493330647955445, + "tokens_seen": 757823488 + }, + { + "epoch": 2.05, + "learning_rate": 0.00038907723169508525, + "loss": 2.7901, + "theoretical_loss": 3.7493006090559637, + "tokens_seen": 757889024 + }, + { + "epoch": 2.05, + "learning_rate": 0.00038906720160481443, + "loss": 2.6746, + "theoretical_loss": 3.74926815690851, + "tokens_seen": 757954560 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003890571715145436, + "loss": 3.0293, + "theoretical_loss": 3.7492357083524768, + "tokens_seen": 758020096 + }, + { + "epoch": 2.05, + "learning_rate": 0.00038904714142427285, + "loss": 2.7772, + "theoretical_loss": 3.7492032633871544, + "tokens_seen": 758085632 + }, + { + "epoch": 2.05, + "learning_rate": 0.000389037111334002, + "loss": 2.8311, + "theoretical_loss": 3.749170822011836, + "tokens_seen": 758151168 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003890270812437312, + "loss": 2.8405, + "theoretical_loss": 3.749138384225814, + "tokens_seen": 758216704 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003890170511534604, + "loss": 2.9285, + "theoretical_loss": 3.7491059500283814, + "tokens_seen": 758282240 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003890070210631896, + "loss": 2.8327, + "theoretical_loss": 3.7490735194188307, + "tokens_seen": 758347776 + }, + { + "epoch": 2.05, + "learning_rate": 0.00038899699097291876, + "loss": 2.7235, + "theoretical_loss": 3.749041092396455, + "tokens_seen": 758413312 + }, + { + "epoch": 2.05, + "learning_rate": 0.00038898696088264794, + "loss": 2.8355, + "theoretical_loss": 3.7490086689605473, + "tokens_seen": 758478848 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003889769307923771, + "loss": 2.8129, + "theoretical_loss": 3.748976249110402, + "tokens_seen": 758544384 + }, + { + "epoch": 2.05, + "objective/train/docs_used": 872211, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.623302936553955, + "objective/train/theoretical_loss": 3.748960040529769, + "objective/train/tokens_used": 779037152, + "theoretical_loss": 3.748960040529769, + "tokens_seen": 758577152 + }, + { + "epoch": 2.05, + "learning_rate": 0.00038896690070210635, + "loss": 2.7462, + "theoretical_loss": 3.7489438328453124, + "tokens_seen": 758609920 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003889568706118355, + "loss": 2.9954, + "theoretical_loss": 3.748911420164572, + "tokens_seen": 758675456 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003889468405215647, + "loss": 2.8864, + "theoretical_loss": 3.7488790110674755, + "tokens_seen": 758740992 + }, + { + "epoch": 2.05, + "learning_rate": 0.00038893681043129384, + "loss": 3.003, + "theoretical_loss": 3.7488466055533163, + "tokens_seen": 758806528 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003889267803410231, + "loss": 2.9352, + "theoretical_loss": 3.74881420362139, + "tokens_seen": 758872064 + }, + { + "epoch": 2.05, + "learning_rate": 0.00038891675025075226, + "loss": 2.7863, + "theoretical_loss": 3.7487818052709905, + "tokens_seen": 758937600 + }, + { + "epoch": 2.05, + "learning_rate": 0.00038890672016048144, + "loss": 2.9018, + "theoretical_loss": 3.7487494105014125, + "tokens_seen": 759003136 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003888966900702106, + "loss": 2.5538, + "theoretical_loss": 3.748717019311952, + "tokens_seen": 759068672 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003888866599799398, + "loss": 2.7822, + "theoretical_loss": 3.748684631701904, + "tokens_seen": 759134208 + }, + { + "epoch": 2.05, + "learning_rate": 0.00038887662988966904, + "loss": 2.7246, + "theoretical_loss": 3.748652247670564, + "tokens_seen": 759199744 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003888665997993982, + "loss": 2.9017, + "theoretical_loss": 3.748619867217227, + "tokens_seen": 759265280 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003888565697091274, + "loss": 2.9178, + "theoretical_loss": 3.7485874903411895, + "tokens_seen": 759330816 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003888465396188566, + "loss": 2.8642, + "theoretical_loss": 3.7485551170417484, + "tokens_seen": 759396352 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003888365095285858, + "loss": 2.8042, + "theoretical_loss": 3.748522747318198, + "tokens_seen": 759461888 + }, + { + "epoch": 2.05, + "learning_rate": 0.00038882647943831494, + "loss": 2.9502, + "theoretical_loss": 3.7484903811698365, + "tokens_seen": 759527424 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003888164493480442, + "loss": 2.9731, + "theoretical_loss": 3.7484580185959597, + "tokens_seen": 759592960 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003888064192577733, + "loss": 2.9822, + "theoretical_loss": 3.748425659595865, + "tokens_seen": 759658496 + }, + { + "epoch": 2.05, + "learning_rate": 0.00038879638916750254, + "loss": 3.0593, + "theoretical_loss": 3.7483933041688493, + "tokens_seen": 759724032 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003887863590772317, + "loss": 2.6031, + "theoretical_loss": 3.74836095231421, + "tokens_seen": 759789568 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003887763289869609, + "loss": 2.9151, + "theoretical_loss": 3.7483286040312445, + "tokens_seen": 759855104 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003887662988966901, + "loss": 2.9807, + "theoretical_loss": 3.7482962593192504, + "tokens_seen": 759920640 + }, + { + "epoch": 2.05, + "learning_rate": 0.00038875626880641927, + "loss": 2.795, + "theoretical_loss": 3.7482639181775252, + "tokens_seen": 759986176 + }, + { + "epoch": 2.05, + "learning_rate": 0.00038874623871614845, + "loss": 2.9018, + "theoretical_loss": 3.748231580605368, + "tokens_seen": 760051712 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003887362086258777, + "loss": 2.9098, + "theoretical_loss": 3.748199246602076, + "tokens_seen": 760117248 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003887261785356068, + "loss": 2.8907, + "theoretical_loss": 3.7481669161669484, + "tokens_seen": 760182784 + }, + { + "epoch": 2.05, + "objective/train/docs_used": 873451, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.31072998046875, + "objective/train/theoretical_loss": 3.748150752287227, + "objective/train/tokens_used": 780675552, + "theoretical_loss": 3.748150752287227, + "tokens_seen": 760215552 + }, + { + "epoch": 2.05, + "learning_rate": 0.00038871614844533604, + "loss": 2.6467, + "theoretical_loss": 3.7481345892992834, + "tokens_seen": 760248320 + }, + { + "epoch": 2.05, + "learning_rate": 0.00038870611835506517, + "loss": 2.7994, + "theoretical_loss": 3.7481022659983805, + "tokens_seen": 760313856 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003886960882647944, + "loss": 2.735, + "theoretical_loss": 3.7480699462635383, + "tokens_seen": 760379392 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003886860581745236, + "loss": 2.9806, + "theoretical_loss": 3.748037630094056, + "tokens_seen": 760444928 + }, + { + "epoch": 2.05, + "learning_rate": 0.00038867602808425277, + "loss": 2.746, + "theoretical_loss": 3.748005317489233, + "tokens_seen": 760510464 + }, + { + "epoch": 2.05, + "learning_rate": 0.00038866599799398195, + "loss": 2.6938, + "theoretical_loss": 3.74797300844837, + "tokens_seen": 760576000 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003886559679037112, + "loss": 2.9767, + "theoretical_loss": 3.747940702970766, + "tokens_seen": 760641536 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003886459378134403, + "loss": 2.9013, + "theoretical_loss": 3.7479084010557204, + "tokens_seen": 760707072 + }, + { + "epoch": 2.05, + "learning_rate": 0.00038863590772316955, + "loss": 2.8877, + "theoretical_loss": 3.7478761027025347, + "tokens_seen": 760772608 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003886258776328987, + "loss": 2.7881, + "theoretical_loss": 3.7478438079105088, + "tokens_seen": 760838144 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003886158475426279, + "loss": 2.8087, + "theoretical_loss": 3.747811516678943, + "tokens_seen": 760903680 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003886058174523571, + "loss": 3.0713, + "theoretical_loss": 3.747779229007139, + "tokens_seen": 760969216 + }, + { + "epoch": 2.05, + "learning_rate": 0.00038859578736208627, + "loss": 2.8158, + "theoretical_loss": 3.7477469448943976, + "tokens_seen": 761034752 + }, + { + "epoch": 2.05, + "learning_rate": 0.00038858575727181545, + "loss": 2.8475, + "theoretical_loss": 3.7477146643400197, + "tokens_seen": 761100288 + }, + { + "epoch": 2.05, + "learning_rate": 0.00038857572718154463, + "loss": 2.8359, + "theoretical_loss": 3.747682387343307, + "tokens_seen": 761165824 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003885656970912738, + "loss": 2.5743, + "theoretical_loss": 3.7476501139035605, + "tokens_seen": 761231360 + }, + { + "epoch": 2.05, + "learning_rate": 0.00038855566700100305, + "loss": 2.9382, + "theoretical_loss": 3.747617844020083, + "tokens_seen": 761296896 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003885456369107322, + "loss": 2.8919, + "theoretical_loss": 3.7475855776921763, + "tokens_seen": 761362432 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003885356068204614, + "loss": 2.9157, + "theoretical_loss": 3.7475533149191422, + "tokens_seen": 761427968 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003885255767301906, + "loss": 2.7488, + "theoretical_loss": 3.7475210557002834, + "tokens_seen": 761493504 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003885155466399198, + "loss": 2.7937, + "theoretical_loss": 3.747488800034903, + "tokens_seen": 761559040 + }, + { + "epoch": 2.05, + "learning_rate": 0.00038850551654964896, + "loss": 2.6836, + "theoretical_loss": 3.747456547922303, + "tokens_seen": 761624576 + }, + { + "epoch": 2.05, + "learning_rate": 0.00038849548645937814, + "loss": 2.8512, + "theoretical_loss": 3.747424299361787, + "tokens_seen": 761690112 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003884854563691073, + "loss": 2.7041, + "theoretical_loss": 3.747392054352658, + "tokens_seen": 761755648 + }, + { + "epoch": 2.05, + "learning_rate": 0.00038847542627883655, + "loss": 2.8625, + "theoretical_loss": 3.7473598128942194, + "tokens_seen": 761821184 + }, + { + "epoch": 2.05, + "objective/train/docs_used": 874119, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.929797649383545, + "objective/train/theoretical_loss": 3.7473436934962914, + "objective/train/tokens_used": 782313952, + "theoretical_loss": 3.7473436934962914, + "tokens_seen": 761853952 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003884653961885657, + "loss": 2.8039, + "theoretical_loss": 3.747327574985775, + "tokens_seen": 761886720 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003884553660982949, + "loss": 2.8516, + "theoretical_loss": 3.7472953406266276, + "tokens_seen": 761952256 + }, + { + "epoch": 2.05, + "learning_rate": 0.00038844533600802404, + "loss": 2.7222, + "theoretical_loss": 3.747263109816083, + "tokens_seen": 762017792 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003884353059177533, + "loss": 2.9182, + "theoretical_loss": 3.747230882553444, + "tokens_seen": 762083328 + }, + { + "epoch": 2.05, + "learning_rate": 0.00038842527582748246, + "loss": 2.6432, + "theoretical_loss": 3.747198658838016, + "tokens_seen": 762148864 + }, + { + "epoch": 2.05, + "learning_rate": 0.00038841524573721164, + "loss": 2.661, + "theoretical_loss": 3.747166438669103, + "tokens_seen": 762214400 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003884052156469408, + "loss": 2.8589, + "theoretical_loss": 3.747134222046009, + "tokens_seen": 762279936 + }, + { + "epoch": 2.05, + "learning_rate": 0.00038839518555667, + "loss": 2.9799, + "theoretical_loss": 3.7471020089680405, + "tokens_seen": 762345472 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003883851554663992, + "loss": 2.8731, + "theoretical_loss": 3.7470697994345024, + "tokens_seen": 762411008 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003883751253761284, + "loss": 2.8761, + "theoretical_loss": 3.7470375934446984, + "tokens_seen": 762476544 + }, + { + "epoch": 2.05, + "learning_rate": 0.00038836509528585755, + "loss": 3.1062, + "theoretical_loss": 3.7470053909979364, + "tokens_seen": 762542080 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003883550651955868, + "loss": 2.8829, + "theoretical_loss": 3.746973192093521, + "tokens_seen": 762607616 + }, + { + "epoch": 2.05, + "learning_rate": 0.00038834503510531596, + "loss": 2.6719, + "theoretical_loss": 3.746940996730758, + "tokens_seen": 762673152 + }, + { + "epoch": 2.05, + "learning_rate": 0.00038833500501504514, + "loss": 2.8273, + "theoretical_loss": 3.7469088049089545, + "tokens_seen": 762738688 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003883249749247743, + "loss": 2.8338, + "theoretical_loss": 3.7468766166274157, + "tokens_seen": 762804224 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003883149448345035, + "loss": 2.8078, + "theoretical_loss": 3.7468444318854486, + "tokens_seen": 762869760 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003883049147442327, + "loss": 2.7179, + "theoretical_loss": 3.7468122506823605, + "tokens_seen": 762935296 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003882948846539619, + "loss": 2.871, + "theoretical_loss": 3.746780073017457, + "tokens_seen": 763000832 + }, + { + "epoch": 2.05, + "learning_rate": 0.00038828485456369105, + "loss": 2.8244, + "theoretical_loss": 3.746747898890047, + "tokens_seen": 763066368 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003882748244734203, + "loss": 2.6443, + "theoretical_loss": 3.7467157282994368, + "tokens_seen": 763131904 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003882647943831494, + "loss": 2.8023, + "theoretical_loss": 3.7466835612449336, + "tokens_seen": 763197440 + }, + { + "epoch": 2.05, + "learning_rate": 0.00038825476429287865, + "loss": 2.7588, + "theoretical_loss": 3.746651397725846, + "tokens_seen": 763262976 + }, + { + "epoch": 2.05, + "learning_rate": 0.00038824473420260783, + "loss": 2.9649, + "theoretical_loss": 3.7466192377414815, + "tokens_seen": 763328512 + }, + { + "epoch": 2.05, + "learning_rate": 0.000388234704112337, + "loss": 3.0581, + "theoretical_loss": 3.746587081291148, + "tokens_seen": 763394048 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003882246740220662, + "loss": 2.9048, + "theoretical_loss": 3.746554928374154, + "tokens_seen": 763459584 + }, + { + "epoch": 2.05, + "objective/train/docs_used": 875620, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.904193878173828, + "objective/train/theoretical_loss": 3.7465388532404433, + "objective/train/tokens_used": 783952352, + "theoretical_loss": 3.7465388532404433, + "tokens_seen": 763492352 + }, + { + "epoch": 2.05, + "learning_rate": 0.00038821464393179537, + "loss": 2.9749, + "theoretical_loss": 3.7465227789898083, + "tokens_seen": 763525120 + }, + { + "epoch": 2.05, + "learning_rate": 0.00038820461384152455, + "loss": 2.8214, + "theoretical_loss": 3.7464906331374195, + "tokens_seen": 763590656 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003881945837512538, + "loss": 2.7125, + "theoretical_loss": 3.7464584908162957, + "tokens_seen": 763656192 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003881845536609829, + "loss": 2.9804, + "theoretical_loss": 3.746426352025747, + "tokens_seen": 763721728 + }, + { + "epoch": 2.05, + "learning_rate": 0.00038817452357071215, + "loss": 2.7881, + "theoretical_loss": 3.7463942167650823, + "tokens_seen": 763787264 + }, + { + "epoch": 2.05, + "learning_rate": 0.00038816449348044133, + "loss": 2.871, + "theoretical_loss": 3.746362085033611, + "tokens_seen": 763852800 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003881544633901705, + "loss": 3.2335, + "theoretical_loss": 3.746329956830643, + "tokens_seen": 763918336 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003881444332998997, + "loss": 2.8939, + "theoretical_loss": 3.746297832155488, + "tokens_seen": 763983872 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003881344032096289, + "loss": 2.8657, + "theoretical_loss": 3.7462657110074558, + "tokens_seen": 764049408 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003881243731193581, + "loss": 2.9157, + "theoretical_loss": 3.746233593385857, + "tokens_seen": 764114944 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003881143430290873, + "loss": 2.9297, + "theoretical_loss": 3.7462014792900016, + "tokens_seen": 764180480 + }, + { + "epoch": 2.05, + "learning_rate": 0.00038810431293881647, + "loss": 2.9193, + "theoretical_loss": 3.746169368719201, + "tokens_seen": 764246016 + }, + { + "epoch": 2.05, + "learning_rate": 0.00038809428284854565, + "loss": 2.6569, + "theoretical_loss": 3.7461372616727657, + "tokens_seen": 764311552 + }, + { + "epoch": 2.05, + "learning_rate": 0.00038808425275827483, + "loss": 2.8671, + "theoretical_loss": 3.746105158150007, + "tokens_seen": 764377088 + }, + { + "epoch": 2.05, + "learning_rate": 0.000388074222668004, + "loss": 2.8244, + "theoretical_loss": 3.746073058150235, + "tokens_seen": 764442624 + }, + { + "epoch": 2.05, + "learning_rate": 0.00038806419257773325, + "loss": 2.7417, + "theoretical_loss": 3.7460409616727626, + "tokens_seen": 764508160 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003880541624874624, + "loss": 2.8929, + "theoretical_loss": 3.7460088687169, + "tokens_seen": 764573696 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003880441323971916, + "loss": 2.9315, + "theoretical_loss": 3.7459767792819605, + "tokens_seen": 764639232 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003880341023069208, + "loss": 2.8129, + "theoretical_loss": 3.745944693367255, + "tokens_seen": 764704768 + }, + { + "epoch": 2.05, + "learning_rate": 0.00038802407221665, + "loss": 2.9542, + "theoretical_loss": 3.7459126109720957, + "tokens_seen": 764770304 + }, + { + "epoch": 2.05, + "learning_rate": 0.00038801404212637916, + "loss": 2.9054, + "theoretical_loss": 3.7458805320957955, + "tokens_seen": 764835840 + }, + { + "epoch": 2.05, + "learning_rate": 0.00038800401203610834, + "loss": 2.7932, + "theoretical_loss": 3.745848456737667, + "tokens_seen": 764901376 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003879939819458375, + "loss": 2.8687, + "theoretical_loss": 3.7458163848970223, + "tokens_seen": 764966912 + }, + { + "epoch": 2.05, + "learning_rate": 0.00038798395185556675, + "loss": 2.8049, + "theoretical_loss": 3.7457843165731752, + "tokens_seen": 765032448 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003879739217652959, + "loss": 2.9075, + "theoretical_loss": 3.7457522517654382, + "tokens_seen": 765097984 + }, + { + "epoch": 2.05, + "objective/train/docs_used": 876301, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8112714290618896, + "objective/train/theoretical_loss": 3.7457362206798965, + "objective/train/tokens_used": 785590752, + "theoretical_loss": 3.7457362206798965, + "tokens_seen": 765130752 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003879638916750251, + "loss": 3.0486, + "theoretical_loss": 3.7457201904731248, + "tokens_seen": 765163520 + }, + { + "epoch": 2.05, + "learning_rate": 0.00038795386158475424, + "loss": 2.8881, + "theoretical_loss": 3.745688132695549, + "tokens_seen": 765229056 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003879438314944835, + "loss": 3.0385, + "theoretical_loss": 3.745656078432024, + "tokens_seen": 765294592 + }, + { + "epoch": 2.05, + "learning_rate": 0.00038793380140421266, + "loss": 2.8354, + "theoretical_loss": 3.7456240276818633, + "tokens_seen": 765360128 + }, + { + "epoch": 2.05, + "learning_rate": 0.00038792377131394184, + "loss": 2.731, + "theoretical_loss": 3.7455919804443827, + "tokens_seen": 765425664 + }, + { + "epoch": 2.05, + "learning_rate": 0.000387913741223671, + "loss": 2.9255, + "theoretical_loss": 3.7455599367188945, + "tokens_seen": 765491200 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003879037111334002, + "loss": 2.842, + "theoretical_loss": 3.745527896504715, + "tokens_seen": 765556736 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003878936810431294, + "loss": 3.034, + "theoretical_loss": 3.7454958598011574, + "tokens_seen": 765622272 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003878836509528586, + "loss": 2.7549, + "theoretical_loss": 3.745463826607537, + "tokens_seen": 765687808 + }, + { + "epoch": 2.05, + "learning_rate": 0.00038787362086258775, + "loss": 2.9639, + "theoretical_loss": 3.7454317969231696, + "tokens_seen": 765753344 + }, + { + "epoch": 2.05, + "learning_rate": 0.000387863590772317, + "loss": 2.8361, + "theoretical_loss": 3.74539977074737, + "tokens_seen": 765818880 + }, + { + "epoch": 2.05, + "learning_rate": 0.00038785356068204616, + "loss": 2.8789, + "theoretical_loss": 3.7453677480794534, + "tokens_seen": 765884416 + }, + { + "epoch": 2.05, + "learning_rate": 0.00038784353059177534, + "loss": 2.9602, + "theoretical_loss": 3.745335728918736, + "tokens_seen": 765949952 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003878335005015045, + "loss": 2.8919, + "theoretical_loss": 3.745303713264533, + "tokens_seen": 766015488 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003878234704112337, + "loss": 2.8526, + "theoretical_loss": 3.745271701116161, + "tokens_seen": 766081024 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003878134403209629, + "loss": 3.0369, + "theoretical_loss": 3.7452396924729356, + "tokens_seen": 766146560 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003878034102306921, + "loss": 2.8223, + "theoretical_loss": 3.745207687334174, + "tokens_seen": 766212096 + }, + { + "epoch": 2.05, + "learning_rate": 0.00038779338014042125, + "loss": 2.8681, + "theoretical_loss": 3.745175685699193, + "tokens_seen": 766277632 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003877833500501505, + "loss": 2.7303, + "theoretical_loss": 3.745143687567308, + "tokens_seen": 766343168 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003877733199598796, + "loss": 2.9516, + "theoretical_loss": 3.7451116929378374, + "tokens_seen": 766408704 + }, + { + "epoch": 2.05, + "learning_rate": 0.00038776328986960885, + "loss": 2.7378, + "theoretical_loss": 3.7450797018100976, + "tokens_seen": 766474240 + }, + { + "epoch": 2.05, + "learning_rate": 0.00038775325977933803, + "loss": 2.6424, + "theoretical_loss": 3.745047714183406, + "tokens_seen": 766539776 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003877432296890672, + "loss": 2.5332, + "theoretical_loss": 3.7450157300570814, + "tokens_seen": 766605312 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003877331995987964, + "loss": 2.8007, + "theoretical_loss": 3.74498374943044, + "tokens_seen": 766670848 + }, + { + "epoch": 2.05, + "learning_rate": 0.00038772316950852557, + "loss": 2.8851, + "theoretical_loss": 3.7449517723028007, + "tokens_seen": 766736384 + }, + { + "epoch": 2.05, + "objective/train/docs_used": 877352, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.128386974334717, + "objective/train/theoretical_loss": 3.7449357850508935, + "objective/train/tokens_used": 787229152, + "theoretical_loss": 3.7449357850508935, + "tokens_seen": 766769152 + }, + { + "epoch": 2.05, + "learning_rate": 0.00038771313941825475, + "loss": 2.7447, + "theoretical_loss": 3.7449197986734806, + "tokens_seen": 766801920 + }, + { + "epoch": 2.05, + "learning_rate": 0.000387703109327984, + "loss": 2.6875, + "theoretical_loss": 3.744887828541799, + "tokens_seen": 766867456 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003876930792377131, + "loss": 2.916, + "theoretical_loss": 3.744855861907075, + "tokens_seen": 766932992 + }, + { + "epoch": 2.05, + "learning_rate": 0.00038768304914744235, + "loss": 2.8973, + "theoretical_loss": 3.744823898768626, + "tokens_seen": 766998528 + }, + { + "epoch": 2.05, + "learning_rate": 0.00038767301905717153, + "loss": 2.7641, + "theoretical_loss": 3.744791939125771, + "tokens_seen": 767064064 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003876629889669007, + "loss": 2.8003, + "theoretical_loss": 3.74475998297783, + "tokens_seen": 767129600 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003876529588766299, + "loss": 2.837, + "theoretical_loss": 3.7447280303241213, + "tokens_seen": 767195136 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003876429287863591, + "loss": 2.6501, + "theoretical_loss": 3.7446960811639656, + "tokens_seen": 767260672 + }, + { + "epoch": 2.05, + "learning_rate": 0.00038763289869608826, + "loss": 3.0486, + "theoretical_loss": 3.7446641354966816, + "tokens_seen": 767326208 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003876228686058175, + "loss": 2.7624, + "theoretical_loss": 3.744632193321589, + "tokens_seen": 767391744 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003876128385155466, + "loss": 2.5624, + "theoretical_loss": 3.7446002546380086, + "tokens_seen": 767457280 + }, + { + "epoch": 2.05, + "learning_rate": 0.00038760280842527585, + "loss": 2.7834, + "theoretical_loss": 3.7445683194452606, + "tokens_seen": 767522816 + }, + { + "epoch": 2.05, + "learning_rate": 0.000387592778335005, + "loss": 2.562, + "theoretical_loss": 3.744536387742665, + "tokens_seen": 767588352 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003875827482447342, + "loss": 2.966, + "theoretical_loss": 3.7445044595295425, + "tokens_seen": 767653888 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003875727181544634, + "loss": 2.8074, + "theoretical_loss": 3.7444725348052144, + "tokens_seen": 767719424 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003875626880641926, + "loss": 2.9211, + "theoretical_loss": 3.744440613569001, + "tokens_seen": 767784960 + }, + { + "epoch": 2.05, + "learning_rate": 0.00038755265797392176, + "loss": 2.9349, + "theoretical_loss": 3.7444086958202245, + "tokens_seen": 767850496 + }, + { + "epoch": 2.05, + "learning_rate": 0.000387542627883651, + "loss": 2.8785, + "theoretical_loss": 3.7443767815582047, + "tokens_seen": 767916032 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003875325977933801, + "loss": 2.9355, + "theoretical_loss": 3.744344870782265, + "tokens_seen": 767981568 + }, + { + "epoch": 2.05, + "learning_rate": 0.00038752256770310936, + "loss": 3.0958, + "theoretical_loss": 3.7443129634917254, + "tokens_seen": 768047104 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003875125376128385, + "loss": 3.0222, + "theoretical_loss": 3.7442810596859095, + "tokens_seen": 768112640 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003875025075225677, + "loss": 2.7181, + "theoretical_loss": 3.7442491593641387, + "tokens_seen": 768178176 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003874924774322969, + "loss": 2.8857, + "theoretical_loss": 3.744217262525735, + "tokens_seen": 768243712 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003874824473420261, + "loss": 2.7694, + "theoretical_loss": 3.744185369170021, + "tokens_seen": 768309248 + }, + { + "epoch": 2.05, + "learning_rate": 0.00038747241725175526, + "loss": 2.9329, + "theoretical_loss": 3.7441534792963203, + "tokens_seen": 768374784 + }, + { + "epoch": 2.05, + "objective/train/docs_used": 878097, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6164374351501465, + "objective/train/theoretical_loss": 3.744137535665013, + "objective/train/tokens_used": 788867552, + "theoretical_loss": 3.744137535665013, + "tokens_seen": 768407552 + }, + { + "epoch": 2.05, + "learning_rate": 0.00038746238716148444, + "loss": 2.7017, + "theoretical_loss": 3.744121592903955, + "tokens_seen": 768440320 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003874523570712136, + "loss": 2.9756, + "theoretical_loss": 3.744089709992248, + "tokens_seen": 768505856 + }, + { + "epoch": 2.05, + "learning_rate": 0.00038744232698094286, + "loss": 2.8532, + "theoretical_loss": 3.744057830560523, + "tokens_seen": 768571392 + }, + { + "epoch": 2.05, + "learning_rate": 0.000387432296890672, + "loss": 2.8827, + "theoretical_loss": 3.7440259546081043, + "tokens_seen": 768636928 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003874222668004012, + "loss": 2.9375, + "theoretical_loss": 3.7439940821343134, + "tokens_seen": 768702464 + }, + { + "epoch": 2.05, + "learning_rate": 0.00038741223671013035, + "loss": 2.7816, + "theoretical_loss": 3.7439622131384764, + "tokens_seen": 768768000 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003874022066198596, + "loss": 2.5517, + "theoretical_loss": 3.7439303476199166, + "tokens_seen": 768833536 + }, + { + "epoch": 2.05, + "learning_rate": 0.00038739217652958876, + "loss": 2.8064, + "theoretical_loss": 3.743898485577957, + "tokens_seen": 768899072 + }, + { + "epoch": 2.05, + "learning_rate": 0.00038738214643931795, + "loss": 2.8739, + "theoretical_loss": 3.743866627011924, + "tokens_seen": 768964608 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003873721163490472, + "loss": 3.0709, + "theoretical_loss": 3.743834771921141, + "tokens_seen": 769030144 + }, + { + "epoch": 2.05, + "learning_rate": 0.00038736208625877636, + "loss": 2.7886, + "theoretical_loss": 3.7438029203049332, + "tokens_seen": 769095680 + }, + { + "epoch": 2.05, + "learning_rate": 0.00038735205616850554, + "loss": 2.9046, + "theoretical_loss": 3.743771072162625, + "tokens_seen": 769161216 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003873420260782347, + "loss": 2.8764, + "theoretical_loss": 3.7437392274935424, + "tokens_seen": 769226752 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003873319959879639, + "loss": 3.125, + "theoretical_loss": 3.74370738629701, + "tokens_seen": 769292288 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003873219658976931, + "loss": 2.9003, + "theoretical_loss": 3.7436755485723543, + "tokens_seen": 769357824 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003873119358074223, + "loss": 2.7719, + "theoretical_loss": 3.7436437143189005, + "tokens_seen": 769423360 + }, + { + "epoch": 2.05, + "learning_rate": 0.00038730190571715145, + "loss": 2.894, + "theoretical_loss": 3.7436118835359737, + "tokens_seen": 769488896 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003872918756268807, + "loss": 2.7787, + "theoretical_loss": 3.7435800562229016, + "tokens_seen": 769554432 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003872818455366098, + "loss": 2.7862, + "theoretical_loss": 3.7435482323790095, + "tokens_seen": 769619968 + }, + { + "epoch": 2.05, + "learning_rate": 0.00038727181544633905, + "loss": 2.8708, + "theoretical_loss": 3.743516412003624, + "tokens_seen": 769685504 + }, + { + "epoch": 2.05, + "learning_rate": 0.00038726178535606823, + "loss": 2.8517, + "theoretical_loss": 3.7434845950960725, + "tokens_seen": 769751040 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003872517552657974, + "loss": 2.9194, + "theoretical_loss": 3.74345278165568, + "tokens_seen": 769816576 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003872417251755266, + "loss": 2.7662, + "theoretical_loss": 3.743420971681776, + "tokens_seen": 769882112 + }, + { + "epoch": 2.05, + "learning_rate": 0.00038723169508525577, + "loss": 2.7826, + "theoretical_loss": 3.743389165173686, + "tokens_seen": 769947648 + }, + { + "epoch": 2.05, + "learning_rate": 0.00038722166499498495, + "loss": 3.1014, + "theoretical_loss": 3.7433573621307383, + "tokens_seen": 770013184 + }, + { + "epoch": 2.05, + "objective/train/docs_used": 879358, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.580174684524536, + "objective/train/theoretical_loss": 3.7433414619084826, + "objective/train/tokens_used": 790505952, + "theoretical_loss": 3.7433414619084826, + "tokens_seen": 770045952 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003872116349047142, + "loss": 2.6705, + "theoretical_loss": 3.7433255625522603, + "tokens_seen": 770078720 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003872016048144433, + "loss": 2.8358, + "theoretical_loss": 3.7432937664375796, + "tokens_seen": 770144256 + }, + { + "epoch": 2.05, + "learning_rate": 0.00038719157472417255, + "loss": 2.8843, + "theoretical_loss": 3.7432619737860247, + "tokens_seen": 770209792 + }, + { + "epoch": 2.05, + "learning_rate": 0.00038718154463390173, + "loss": 2.9577, + "theoretical_loss": 3.7432301845969227, + "tokens_seen": 770275328 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003871715145436309, + "loss": 2.9062, + "theoretical_loss": 3.743198398869603, + "tokens_seen": 770340864 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003871614844533601, + "loss": 2.9596, + "theoretical_loss": 3.743166616603394, + "tokens_seen": 770406400 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003871514543630893, + "loss": 2.9192, + "theoretical_loss": 3.7431348377976246, + "tokens_seen": 770471936 + }, + { + "epoch": 2.05, + "learning_rate": 0.00038714142427281846, + "loss": 3.0561, + "theoretical_loss": 3.743103062451623, + "tokens_seen": 770537472 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003871313941825477, + "loss": 2.8688, + "theoretical_loss": 3.7430712905647185, + "tokens_seen": 770603008 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003871213640922768, + "loss": 2.7494, + "theoretical_loss": 3.7430395221362414, + "tokens_seen": 770668544 + }, + { + "epoch": 2.05, + "learning_rate": 0.00038711133400200605, + "loss": 2.8565, + "theoretical_loss": 3.74300775716552, + "tokens_seen": 770734080 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003871013039117352, + "loss": 2.9329, + "theoretical_loss": 3.7429759956518844, + "tokens_seen": 770799616 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003870912738214644, + "loss": 2.809, + "theoretical_loss": 3.7429442375946644, + "tokens_seen": 770865152 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003870812437311936, + "loss": 2.8026, + "theoretical_loss": 3.742912482993191, + "tokens_seen": 770930688 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003870712136409228, + "loss": 2.8708, + "theoretical_loss": 3.7428807318467925, + "tokens_seen": 770996224 + }, + { + "epoch": 2.05, + "learning_rate": 0.00038706118355065196, + "loss": 2.9398, + "theoretical_loss": 3.742848984154801, + "tokens_seen": 771061760 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003870511534603812, + "loss": 2.8979, + "theoretical_loss": 3.7428172399165467, + "tokens_seen": 771127296 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003870411233701103, + "loss": 2.8455, + "theoretical_loss": 3.74278549913136, + "tokens_seen": 771192832 + }, + { + "epoch": 2.05, + "learning_rate": 0.00038703109327983956, + "loss": 2.998, + "theoretical_loss": 3.742753761798572, + "tokens_seen": 771258368 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003870210631895687, + "loss": 2.836, + "theoretical_loss": 3.742722027917514, + "tokens_seen": 771323904 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003870110330992979, + "loss": 2.9174, + "theoretical_loss": 3.7426902974875182, + "tokens_seen": 771389440 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003870010030090271, + "loss": 2.7942, + "theoretical_loss": 3.7426585705079147, + "tokens_seen": 771454976 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003869909729187563, + "loss": 3.0534, + "theoretical_loss": 3.7426268469780357, + "tokens_seen": 771520512 + }, + { + "epoch": 2.05, + "learning_rate": 0.00038698094282848546, + "loss": 2.9767, + "theoretical_loss": 3.7425951268972133, + "tokens_seen": 771586048 + }, + { + "epoch": 2.05, + "learning_rate": 0.00038697091273821464, + "loss": 2.7632, + "theoretical_loss": 3.74256341026478, + "tokens_seen": 771651584 + }, + { + "epoch": 2.05, + "objective/train/docs_used": 880093, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.807681083679199, + "objective/train/theoretical_loss": 3.7425475532415007, + "objective/train/tokens_used": 792144352, + "theoretical_loss": 3.7425475532415007, + "tokens_seen": 771684352 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003869608826479438, + "loss": 2.8143, + "theoretical_loss": 3.742531697080068, + "tokens_seen": 771717120 + }, + { + "epoch": 2.05, + "learning_rate": 0.00038695085255767306, + "loss": 2.8962, + "theoretical_loss": 3.7424999873424087, + "tokens_seen": 771782656 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003869408224674022, + "loss": 2.7382, + "theoretical_loss": 3.742468281051136, + "tokens_seen": 771848192 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003869307923771314, + "loss": 3.0543, + "theoretical_loss": 3.7424365782055817, + "tokens_seen": 771913728 + }, + { + "epoch": 2.05, + "learning_rate": 0.00038692076228686055, + "loss": 2.814, + "theoretical_loss": 3.74240487880508, + "tokens_seen": 771979264 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003869107321965898, + "loss": 2.8605, + "theoretical_loss": 3.7423731828489633, + "tokens_seen": 772044800 + }, + { + "epoch": 2.05, + "learning_rate": 0.00038690070210631896, + "loss": 2.9045, + "theoretical_loss": 3.7423414903365653, + "tokens_seen": 772110336 + }, + { + "epoch": 2.05, + "learning_rate": 0.00038689067201604815, + "loss": 2.7878, + "theoretical_loss": 3.7423098012672193, + "tokens_seen": 772175872 + }, + { + "epoch": 2.05, + "learning_rate": 0.00038688064192577733, + "loss": 2.7309, + "theoretical_loss": 3.74227811564026, + "tokens_seen": 772241408 + }, + { + "epoch": 2.05, + "learning_rate": 0.00038687061183550656, + "loss": 2.947, + "theoretical_loss": 3.7422464334550196, + "tokens_seen": 772306944 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003868605817452357, + "loss": 2.7292, + "theoretical_loss": 3.742214754710834, + "tokens_seen": 772372480 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003868505516549649, + "loss": 2.8713, + "theoretical_loss": 3.742183079407037, + "tokens_seen": 772438016 + }, + { + "epoch": 2.05, + "learning_rate": 0.00038684052156469405, + "loss": 2.9634, + "theoretical_loss": 3.7421514075429627, + "tokens_seen": 772503552 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003868304914744233, + "loss": 2.7913, + "theoretical_loss": 3.742119739117946, + "tokens_seen": 772569088 + }, + { + "epoch": 2.05, + "learning_rate": 0.00038682046138415247, + "loss": 3.1463, + "theoretical_loss": 3.7420880741313223, + "tokens_seen": 772634624 + }, + { + "epoch": 2.05, + "learning_rate": 0.00038681043129388165, + "loss": 2.6388, + "theoretical_loss": 3.742056412582426, + "tokens_seen": 772700160 + }, + { + "epoch": 2.05, + "learning_rate": 0.00038680040120361083, + "loss": 2.694, + "theoretical_loss": 3.7420247544705925, + "tokens_seen": 772765696 + }, + { + "epoch": 2.05, + "learning_rate": 0.00038679037111334, + "loss": 2.935, + "theoretical_loss": 3.741993099795158, + "tokens_seen": 772831232 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003867803410230692, + "loss": 3.0614, + "theoretical_loss": 3.7419614485554566, + "tokens_seen": 772896768 + }, + { + "epoch": 2.05, + "learning_rate": 0.00038677031093279843, + "loss": 2.7906, + "theoretical_loss": 3.7419298007508255, + "tokens_seen": 772962304 + }, + { + "epoch": 2.05, + "learning_rate": 0.00038676028084252755, + "loss": 3.0528, + "theoretical_loss": 3.7418981563806, + "tokens_seen": 773027840 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003867502507522568, + "loss": 2.9, + "theoretical_loss": 3.7418665154441166, + "tokens_seen": 773093376 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003867402206619859, + "loss": 2.9703, + "theoretical_loss": 3.741834877940711, + "tokens_seen": 773158912 + }, + { + "epoch": 2.05, + "learning_rate": 0.00038673019057171515, + "loss": 3.0419, + "theoretical_loss": 3.7418032438697213, + "tokens_seen": 773224448 + }, + { + "epoch": 2.05, + "learning_rate": 0.00038672016048144433, + "loss": 2.969, + "theoretical_loss": 3.7417716132304824, + "tokens_seen": 773289984 + }, + { + "epoch": 2.05, + "objective/train/docs_used": 881313, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6865322589874268, + "objective/train/theoretical_loss": 3.741755799197563, + "objective/train/tokens_used": 793782752, + "theoretical_loss": 3.741755799197563, + "tokens_seen": 773322752 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003867101303911735, + "loss": 2.8573, + "theoretical_loss": 3.741739986022333, + "tokens_seen": 773355520 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003867001003009027, + "loss": 2.991, + "theoretical_loss": 3.7417083622446086, + "tokens_seen": 773421056 + }, + { + "epoch": 2.05, + "learning_rate": 0.00038669007021063193, + "loss": 2.9242, + "theoretical_loss": 3.7416767418966472, + "tokens_seen": 773486592 + }, + { + "epoch": 2.05, + "learning_rate": 0.00038668004012036106, + "loss": 2.8793, + "theoretical_loss": 3.7416451249777865, + "tokens_seen": 773552128 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003866700100300903, + "loss": 3.0631, + "theoretical_loss": 3.7416135114873637, + "tokens_seen": 773617664 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003866599799398194, + "loss": 2.7995, + "theoretical_loss": 3.741581901424717, + "tokens_seen": 773683200 + }, + { + "epoch": 2.05, + "learning_rate": 0.00038664994984954866, + "loss": 2.7504, + "theoretical_loss": 3.7415502947891843, + "tokens_seen": 773748736 + }, + { + "epoch": 2.05, + "learning_rate": 0.00038663991975927784, + "loss": 2.9901, + "theoretical_loss": 3.7415186915801035, + "tokens_seen": 773814272 + }, + { + "epoch": 2.05, + "learning_rate": 0.000386629889669007, + "loss": 2.9467, + "theoretical_loss": 3.7414870917968135, + "tokens_seen": 773879808 + }, + { + "epoch": 2.05, + "learning_rate": 0.00038661985957873625, + "loss": 2.7866, + "theoretical_loss": 3.7414554954386525, + "tokens_seen": 773945344 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003866098294884654, + "loss": 2.6642, + "theoretical_loss": 3.7414239025049594, + "tokens_seen": 774010880 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003865997993981946, + "loss": 2.7879, + "theoretical_loss": 3.7413923129950737, + "tokens_seen": 774076416 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003865897693079238, + "loss": 3.0073, + "theoretical_loss": 3.741360726908333, + "tokens_seen": 774141952 + }, + { + "epoch": 2.05, + "learning_rate": 0.000386579739217653, + "loss": 2.7793, + "theoretical_loss": 3.7413291442440784, + "tokens_seen": 774207488 + }, + { + "epoch": 2.05, + "learning_rate": 0.00038656970912738216, + "loss": 3.0301, + "theoretical_loss": 3.7412975650016485, + "tokens_seen": 774273024 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003865596790371114, + "loss": 2.8888, + "theoretical_loss": 3.741265989180383, + "tokens_seen": 774338560 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003865496489468405, + "loss": 2.9951, + "theoretical_loss": 3.7412344167796214, + "tokens_seen": 774404096 + }, + { + "epoch": 2.05, + "learning_rate": 0.00038653961885656976, + "loss": 2.8143, + "theoretical_loss": 3.7412028477987045, + "tokens_seen": 774469632 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003865295887662989, + "loss": 2.9018, + "theoretical_loss": 3.7411712822369725, + "tokens_seen": 774535168 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003865195586760281, + "loss": 3.0448, + "theoretical_loss": 3.741139720093765, + "tokens_seen": 774600704 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003865095285857573, + "loss": 2.9451, + "theoretical_loss": 3.741108161368423, + "tokens_seen": 774666240 + }, + { + "epoch": 2.05, + "learning_rate": 0.0003864994984954865, + "loss": 2.8807, + "theoretical_loss": 3.7410766060602887, + "tokens_seen": 774731776 + }, + { + "epoch": 2.05, + "learning_rate": 0.00038648946840521566, + "loss": 3.0032, + "theoretical_loss": 3.7410450541687004, + "tokens_seen": 774797312 + }, + { + "epoch": 2.05, + "learning_rate": 0.00038647943831494484, + "loss": 2.6414, + "theoretical_loss": 3.741013505693001, + "tokens_seen": 774862848 + }, + { + "epoch": 2.05, + "learning_rate": 0.000386469408224674, + "loss": 2.8847, + "theoretical_loss": 3.740981960632532, + "tokens_seen": 774928384 + }, + { + "epoch": 2.05, + "objective/train/docs_used": 882028, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.3022451400756836, + "objective/train/theoretical_loss": 3.740966189382802, + "objective/train/tokens_used": 795421152, + "theoretical_loss": 3.740966189382802, + "tokens_seen": 774961152 + }, + { + "epoch": 2.05, + "learning_rate": 0.00038645937813440326, + "loss": 2.8122, + "theoretical_loss": 3.7409504189866336, + "tokens_seen": 774993920 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003864493480441324, + "loss": 2.8759, + "theoretical_loss": 3.7409188807546485, + "tokens_seen": 775059456 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003864393179538616, + "loss": 2.8984, + "theoretical_loss": 3.740887345935918, + "tokens_seen": 775124992 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038642928786359075, + "loss": 2.902, + "theoretical_loss": 3.7408558145297848, + "tokens_seen": 775190528 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038641925777332, + "loss": 2.8342, + "theoretical_loss": 3.7408242865355907, + "tokens_seen": 775256064 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038640922768304917, + "loss": 2.9901, + "theoretical_loss": 3.7407927619526786, + "tokens_seen": 775321600 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038639919759277835, + "loss": 2.8286, + "theoretical_loss": 3.7407612407803903, + "tokens_seen": 775387136 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038638916750250753, + "loss": 2.8755, + "theoretical_loss": 3.7407297230180694, + "tokens_seen": 775452672 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038637913741223676, + "loss": 2.7858, + "theoretical_loss": 3.7406982086650578, + "tokens_seen": 775518208 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003863691073219659, + "loss": 2.7104, + "theoretical_loss": 3.7406666977207, + "tokens_seen": 775583744 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003863590772316951, + "loss": 2.8863, + "theoretical_loss": 3.740635190184338, + "tokens_seen": 775649280 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038634904714142425, + "loss": 2.9168, + "theoretical_loss": 3.7406036860553167, + "tokens_seen": 775714816 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003863390170511535, + "loss": 2.9828, + "theoretical_loss": 3.7405721853329785, + "tokens_seen": 775780352 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038632898696088267, + "loss": 2.6336, + "theoretical_loss": 3.7405406880166687, + "tokens_seen": 775845888 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038631895687061185, + "loss": 2.7923, + "theoretical_loss": 3.74050919410573, + "tokens_seen": 775911424 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038630892678034103, + "loss": 2.8397, + "theoretical_loss": 3.740477703599507, + "tokens_seen": 775976960 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003862988966900702, + "loss": 2.8475, + "theoretical_loss": 3.740446216497344, + "tokens_seen": 776042496 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003862888665997994, + "loss": 2.7932, + "theoretical_loss": 3.7404147327985866, + "tokens_seen": 776108032 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038627883650952863, + "loss": 2.8546, + "theoretical_loss": 3.7403832525025784, + "tokens_seen": 776173568 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038626880641925775, + "loss": 2.9357, + "theoretical_loss": 3.740351775608665, + "tokens_seen": 776239104 + }, + { + "epoch": 2.06, + "learning_rate": 0.000386258776328987, + "loss": 2.8414, + "theoretical_loss": 3.7403203021161913, + "tokens_seen": 776304640 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003862487462387161, + "loss": 2.916, + "theoretical_loss": 3.740288832024503, + "tokens_seen": 776370176 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038623871614844535, + "loss": 2.9987, + "theoretical_loss": 3.740257365332945, + "tokens_seen": 776435712 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038622868605817453, + "loss": 2.8645, + "theoretical_loss": 3.740225902040864, + "tokens_seen": 776501248 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003862186559679037, + "loss": 2.8856, + "theoretical_loss": 3.740194442147604, + "tokens_seen": 776566784 + }, + { + "epoch": 2.06, + "objective/train/docs_used": 883113, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.753779411315918, + "objective/train/theoretical_loss": 3.740178713475329, + "objective/train/tokens_used": 797059552, + "theoretical_loss": 3.740178713475329, + "tokens_seen": 776599552 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003862086258776329, + "loss": 2.8486, + "theoretical_loss": 3.740162985652513, + "tokens_seen": 776632320 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038619859578736213, + "loss": 2.783, + "theoretical_loss": 3.740131532554937, + "tokens_seen": 776697856 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038618856569709126, + "loss": 2.7231, + "theoretical_loss": 3.7401000828542212, + "tokens_seen": 776763392 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003861785356068205, + "loss": 2.8215, + "theoretical_loss": 3.740068636549713, + "tokens_seen": 776828928 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003861685055165496, + "loss": 2.8173, + "theoretical_loss": 3.740037193640759, + "tokens_seen": 776894464 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038615847542627886, + "loss": 2.7441, + "theoretical_loss": 3.7400057541267064, + "tokens_seen": 776960000 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038614844533600804, + "loss": 2.7466, + "theoretical_loss": 3.739974318006902, + "tokens_seen": 777025536 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003861384152457372, + "loss": 2.9235, + "theoretical_loss": 3.7399428852806937, + "tokens_seen": 777091072 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003861283851554664, + "loss": 2.9365, + "theoretical_loss": 3.7399114559474285, + "tokens_seen": 777156608 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003861183550651956, + "loss": 2.8048, + "theoretical_loss": 3.739880030006454, + "tokens_seen": 777222144 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038610832497492476, + "loss": 2.7476, + "theoretical_loss": 3.7398486074571182, + "tokens_seen": 777287680 + }, + { + "epoch": 2.06, + "learning_rate": 0.000386098294884654, + "loss": 2.7782, + "theoretical_loss": 3.73981718829877, + "tokens_seen": 777353216 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003860882647943831, + "loss": 2.8754, + "theoretical_loss": 3.7397857725307553, + "tokens_seen": 777418752 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038607823470411236, + "loss": 2.8833, + "theoretical_loss": 3.7397543601524252, + "tokens_seen": 777484288 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003860682046138415, + "loss": 2.569, + "theoretical_loss": 3.739722951163127, + "tokens_seen": 777549824 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003860581745235707, + "loss": 2.8597, + "theoretical_loss": 3.7396915455622093, + "tokens_seen": 777615360 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003860481444332999, + "loss": 2.9052, + "theoretical_loss": 3.7396601433490213, + "tokens_seen": 777680896 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003860381143430291, + "loss": 2.8995, + "theoretical_loss": 3.7396287445229124, + "tokens_seen": 777746432 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038602808425275826, + "loss": 2.6981, + "theoretical_loss": 3.7395973490832315, + "tokens_seen": 777811968 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003860180541624875, + "loss": 2.683, + "theoretical_loss": 3.739565957029328, + "tokens_seen": 777877504 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003860080240722166, + "loss": 2.7552, + "theoretical_loss": 3.7395345683605523, + "tokens_seen": 777943040 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038599799398194586, + "loss": 2.8937, + "theoretical_loss": 3.739503183076253, + "tokens_seen": 778008576 + }, + { + "epoch": 2.06, + "learning_rate": 0.000385987963891675, + "loss": 2.646, + "theoretical_loss": 3.7394718011757817, + "tokens_seen": 778074112 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003859779338014042, + "loss": 2.8996, + "theoretical_loss": 3.739440422658487, + "tokens_seen": 778139648 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003859679037111334, + "loss": 2.947, + "theoretical_loss": 3.7394090475237203, + "tokens_seen": 778205184 + }, + { + "epoch": 2.06, + "objective/train/docs_used": 883877, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5676019191741943, + "objective/train/theoretical_loss": 3.7393933612245824, + "objective/train/tokens_used": 798697952, + "theoretical_loss": 3.7393933612245824, + "tokens_seen": 778237952 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003859578736208626, + "loss": 2.6764, + "theoretical_loss": 3.7393776757708324, + "tokens_seen": 778270720 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038594784353059177, + "loss": 2.8117, + "theoretical_loss": 3.7393463073991726, + "tokens_seen": 778336256 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038593781344032095, + "loss": 2.9232, + "theoretical_loss": 3.7393149424080936, + "tokens_seen": 778401792 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038592778335005013, + "loss": 2.7402, + "theoretical_loss": 3.739283580796945, + "tokens_seen": 778467328 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038591775325977937, + "loss": 2.8319, + "theoretical_loss": 3.7392522225650793, + "tokens_seen": 778532864 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003859077231695085, + "loss": 2.9142, + "theoretical_loss": 3.739220867711847, + "tokens_seen": 778598400 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038589769307923773, + "loss": 2.896, + "theoretical_loss": 3.7391895162366002, + "tokens_seen": 778663936 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038588766298896685, + "loss": 2.5929, + "theoretical_loss": 3.7391581681386907, + "tokens_seen": 778729472 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003858776328986961, + "loss": 2.9375, + "theoretical_loss": 3.7391268234174704, + "tokens_seen": 778795008 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003858676028084253, + "loss": 2.8272, + "theoretical_loss": 3.7390954820722917, + "tokens_seen": 778860544 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038585757271815445, + "loss": 2.8612, + "theoretical_loss": 3.7390641441025068, + "tokens_seen": 778926080 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003858475426278837, + "loss": 3.0562, + "theoretical_loss": 3.7390328095074676, + "tokens_seen": 778991616 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038583751253761287, + "loss": 2.9644, + "theoretical_loss": 3.7390014782865277, + "tokens_seen": 779057152 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038582748244734205, + "loss": 2.7878, + "theoretical_loss": 3.73897015043904, + "tokens_seen": 779122688 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038581745235707123, + "loss": 2.7606, + "theoretical_loss": 3.7389388259643566, + "tokens_seen": 779188224 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003858074222668004, + "loss": 2.8087, + "theoretical_loss": 3.7389075048618317, + "tokens_seen": 779253760 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003857973921765296, + "loss": 2.8811, + "theoretical_loss": 3.7388761871308187, + "tokens_seen": 779319296 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038578736208625883, + "loss": 3.0463, + "theoretical_loss": 3.7388448727706707, + "tokens_seen": 779384832 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038577733199598796, + "loss": 2.8511, + "theoretical_loss": 3.738813561780742, + "tokens_seen": 779450368 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003857673019057172, + "loss": 2.763, + "theoretical_loss": 3.7387822541603857, + "tokens_seen": 779515904 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003857572718154463, + "loss": 2.9689, + "theoretical_loss": 3.738750949908957, + "tokens_seen": 779581440 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038574724172517555, + "loss": 2.7182, + "theoretical_loss": 3.738719649025809, + "tokens_seen": 779646976 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038573721163490473, + "loss": 2.8999, + "theoretical_loss": 3.7386883515102975, + "tokens_seen": 779712512 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003857271815446339, + "loss": 2.9385, + "theoretical_loss": 3.7386570573617766, + "tokens_seen": 779778048 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003857171514543631, + "loss": 2.7616, + "theoretical_loss": 3.738625766579601, + "tokens_seen": 779843584 + }, + { + "epoch": 2.06, + "objective/train/docs_used": 885041, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8538613319396973, + "objective/train/theoretical_loss": 3.738610122450691, + "objective/train/tokens_used": 800336352, + "theoretical_loss": 3.738610122450691, + "tokens_seen": 779876352 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038570712136409233, + "loss": 2.8281, + "theoretical_loss": 3.7385944791631256, + "tokens_seen": 779909120 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038569709127382146, + "loss": 2.8344, + "theoretical_loss": 3.738563195111706, + "tokens_seen": 779974656 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003856870611835507, + "loss": 3.0381, + "theoretical_loss": 3.7385319144246973, + "tokens_seen": 780040192 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003856770310932798, + "loss": 2.6559, + "theoretical_loss": 3.7385006371014553, + "tokens_seen": 780105728 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038566700100300906, + "loss": 3.129, + "theoretical_loss": 3.738469363141336, + "tokens_seen": 780171264 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038565697091273824, + "loss": 2.6653, + "theoretical_loss": 3.738438092543694, + "tokens_seen": 780236800 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003856469408224674, + "loss": 2.7597, + "theoretical_loss": 3.7384068253078873, + "tokens_seen": 780302336 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003856369107321966, + "loss": 2.9662, + "theoretical_loss": 3.7383755614332705, + "tokens_seen": 780367872 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003856268806419258, + "loss": 3.0753, + "theoretical_loss": 3.7383443009192012, + "tokens_seen": 780433408 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038561685055165496, + "loss": 2.7267, + "theoretical_loss": 3.738313043765035, + "tokens_seen": 780498944 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003856068204613842, + "loss": 3.0858, + "theoretical_loss": 3.73828178997013, + "tokens_seen": 780564480 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003855967903711133, + "loss": 2.8874, + "theoretical_loss": 3.738250539533842, + "tokens_seen": 780630016 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038558676028084256, + "loss": 2.8781, + "theoretical_loss": 3.738219292455528, + "tokens_seen": 780695552 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003855767301905717, + "loss": 2.8912, + "theoretical_loss": 3.738188048734547, + "tokens_seen": 780761088 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003855667001003009, + "loss": 2.9454, + "theoretical_loss": 3.738156808370255, + "tokens_seen": 780826624 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003855566700100301, + "loss": 2.9832, + "theoretical_loss": 3.7381255713620103, + "tokens_seen": 780892160 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003855466399197593, + "loss": 2.8575, + "theoretical_loss": 3.73809433770917, + "tokens_seen": 780957696 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038553660982948846, + "loss": 2.751, + "theoretical_loss": 3.7380631074110933, + "tokens_seen": 781023232 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003855265797392177, + "loss": 2.7593, + "theoretical_loss": 3.7380318804671377, + "tokens_seen": 781088768 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003855165496489468, + "loss": 2.6554, + "theoretical_loss": 3.7380006568766615, + "tokens_seen": 781154304 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038550651955867606, + "loss": 2.8923, + "theoretical_loss": 3.7379694366390237, + "tokens_seen": 781219840 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003854964894684052, + "loss": 2.8653, + "theoretical_loss": 3.737938219753583, + "tokens_seen": 781285376 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003854864593781344, + "loss": 2.7881, + "theoretical_loss": 3.7379070062196975, + "tokens_seen": 781350912 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003854764292878636, + "loss": 2.9967, + "theoretical_loss": 3.7378757960367275, + "tokens_seen": 781416448 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003854663991975928, + "loss": 2.7699, + "theoretical_loss": 3.7378445892040317, + "tokens_seen": 781481984 + }, + { + "epoch": 2.06, + "objective/train/docs_used": 890004, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.93015456199646, + "objective/train/theoretical_loss": 3.7378289870438364, + "objective/train/tokens_used": 801974752, + "theoretical_loss": 3.7378289870438364, + "tokens_seen": 781514752 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038545636910732197, + "loss": 2.823, + "theoretical_loss": 3.737813385720969, + "tokens_seen": 781547520 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038544633901705115, + "loss": 2.6779, + "theoretical_loss": 3.7377821855869007, + "tokens_seen": 781613056 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038543630892678033, + "loss": 2.9013, + "theoretical_loss": 3.7377509888011846, + "tokens_seen": 781678592 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038542627883650957, + "loss": 2.8486, + "theoretical_loss": 3.7377197953631818, + "tokens_seen": 781744128 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003854162487462387, + "loss": 2.8487, + "theoretical_loss": 3.737688605272252, + "tokens_seen": 781809664 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038540621865596793, + "loss": 2.7669, + "theoretical_loss": 3.7376574185277565, + "tokens_seen": 781875200 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038539618856569705, + "loss": 2.6736, + "theoretical_loss": 3.7376262351290546, + "tokens_seen": 781940736 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003853861584754263, + "loss": 3.0875, + "theoretical_loss": 3.7375950550755075, + "tokens_seen": 782006272 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038537612838515547, + "loss": 2.9593, + "theoretical_loss": 3.737563878366476, + "tokens_seen": 782071808 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038536609829488465, + "loss": 2.6679, + "theoretical_loss": 3.737532705001321, + "tokens_seen": 782137344 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038535606820461383, + "loss": 3.0975, + "theoretical_loss": 3.7375015349794047, + "tokens_seen": 782202880 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038534603811434307, + "loss": 2.8719, + "theoretical_loss": 3.7374703683000865, + "tokens_seen": 782268416 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003853360080240722, + "loss": 3.0239, + "theoretical_loss": 3.7374392049627296, + "tokens_seen": 782333952 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038532597793380143, + "loss": 2.6991, + "theoretical_loss": 3.7374080449666955, + "tokens_seen": 782399488 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038531594784353056, + "loss": 3.0373, + "theoretical_loss": 3.7373768883113456, + "tokens_seen": 782465024 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003853059177532598, + "loss": 2.7967, + "theoretical_loss": 3.737345734996042, + "tokens_seen": 782530560 + }, + { + "epoch": 2.06, + "learning_rate": 0.000385295887662989, + "loss": 2.7177, + "theoretical_loss": 3.7373145850201475, + "tokens_seen": 782596096 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038528585757271816, + "loss": 2.8801, + "theoretical_loss": 3.7372834383830247, + "tokens_seen": 782661632 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038527582748244734, + "loss": 2.6214, + "theoretical_loss": 3.7372522950840352, + "tokens_seen": 782727168 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003852657973921765, + "loss": 2.8109, + "theoretical_loss": 3.7372211551225423, + "tokens_seen": 782792704 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003852557673019057, + "loss": 3.0358, + "theoretical_loss": 3.737190018497909, + "tokens_seen": 782858240 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038524573721163493, + "loss": 2.8844, + "theoretical_loss": 3.7371588852094986, + "tokens_seen": 782923776 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038523570712136406, + "loss": 2.9968, + "theoretical_loss": 3.7371277552566737, + "tokens_seen": 782989312 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003852256770310933, + "loss": 2.758, + "theoretical_loss": 3.737096628638799, + "tokens_seen": 783054848 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003852156469408225, + "loss": 2.8207, + "theoretical_loss": 3.737065505355237, + "tokens_seen": 783120384 + }, + { + "epoch": 2.06, + "objective/train/docs_used": 897763, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7294697761535645, + "objective/train/theoretical_loss": 3.7370499449636245, + "objective/train/tokens_used": 803613152, + "theoretical_loss": 3.7370499449636245, + "tokens_seen": 783153152 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038520561685055166, + "loss": 2.6323, + "theoretical_loss": 3.7370343854053525, + "tokens_seen": 783185920 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038519558676028084, + "loss": 2.9722, + "theoretical_loss": 3.7370032687885084, + "tokens_seen": 783251456 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038518555667001, + "loss": 2.8461, + "theoretical_loss": 3.7369721555040702, + "tokens_seen": 783316992 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003851755265797392, + "loss": 2.7199, + "theoretical_loss": 3.736941045551401, + "tokens_seen": 783382528 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038516549648946844, + "loss": 2.8761, + "theoretical_loss": 3.736909938929866, + "tokens_seen": 783448064 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038515546639919756, + "loss": 3.1976, + "theoretical_loss": 3.7368788356388305, + "tokens_seen": 783513600 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003851454363089268, + "loss": 2.904, + "theoretical_loss": 3.7368477356776584, + "tokens_seen": 783579136 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003851354062186559, + "loss": 2.8346, + "theoretical_loss": 3.7368166390457147, + "tokens_seen": 783644672 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038512537612838516, + "loss": 2.7427, + "theoretical_loss": 3.7367855457423653, + "tokens_seen": 783710208 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003851153460381144, + "loss": 2.7978, + "theoretical_loss": 3.736754455766975, + "tokens_seen": 783775744 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003851053159478435, + "loss": 2.9312, + "theoretical_loss": 3.73672336911891, + "tokens_seen": 783841280 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038509528585757276, + "loss": 2.7718, + "theoretical_loss": 3.7366922857975355, + "tokens_seen": 783906816 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003850852557673019, + "loss": 2.7926, + "theoretical_loss": 3.7366612058022177, + "tokens_seen": 783972352 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003850752256770311, + "loss": 2.9423, + "theoretical_loss": 3.736630129132323, + "tokens_seen": 784037888 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003850651955867603, + "loss": 2.71, + "theoretical_loss": 3.7365990557872166, + "tokens_seen": 784103424 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003850551654964895, + "loss": 2.9139, + "theoretical_loss": 3.7365679857662664, + "tokens_seen": 784168960 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038504513540621866, + "loss": 2.8589, + "theoretical_loss": 3.736536919068838, + "tokens_seen": 784234496 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003850351053159479, + "loss": 2.8182, + "theoretical_loss": 3.7365058556942987, + "tokens_seen": 784300032 + }, + { + "epoch": 2.06, + "learning_rate": 0.000385025075225677, + "loss": 2.9608, + "theoretical_loss": 3.736474795642015, + "tokens_seen": 784365568 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038501504513540626, + "loss": 2.8076, + "theoretical_loss": 3.736443738911354, + "tokens_seen": 784431104 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003850050150451354, + "loss": 2.8234, + "theoretical_loss": 3.7364126855016835, + "tokens_seen": 784496640 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003849949849548646, + "loss": 2.6401, + "theoretical_loss": 3.7363816354123713, + "tokens_seen": 784562176 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003849849548645938, + "loss": 2.8178, + "theoretical_loss": 3.736350588642784, + "tokens_seen": 784627712 + }, + { + "epoch": 2.06, + "learning_rate": 0.000384974924774323, + "loss": 2.9572, + "theoretical_loss": 3.7363195451922904, + "tokens_seen": 784693248 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038496489468405217, + "loss": 2.9677, + "theoretical_loss": 3.7362885050602577, + "tokens_seen": 784758784 + }, + { + "epoch": 2.06, + "objective/train/docs_used": 900073, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.1294591426849365, + "objective/train/theoretical_loss": 3.7362729862384674, + "objective/train/tokens_used": 805251552, + "theoretical_loss": 3.7362729862384674, + "tokens_seen": 784791552 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038495486459378135, + "loss": 2.8987, + "theoretical_loss": 3.736257468246055, + "tokens_seen": 784824320 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038494483450351053, + "loss": 2.8781, + "theoretical_loss": 3.7362264347490504, + "tokens_seen": 784889856 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038493480441323977, + "loss": 2.9147, + "theoretical_loss": 3.7361954045686114, + "tokens_seen": 784955392 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003849247743229689, + "loss": 2.7068, + "theoretical_loss": 3.7361643777041076, + "tokens_seen": 785020928 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038491474423269813, + "loss": 2.716, + "theoretical_loss": 3.7361333541549078, + "tokens_seen": 785086464 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038490471414242725, + "loss": 2.923, + "theoretical_loss": 3.7361023339203814, + "tokens_seen": 785152000 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003848946840521565, + "loss": 2.7817, + "theoretical_loss": 3.736071316999897, + "tokens_seen": 785217536 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038488465396188567, + "loss": 2.8024, + "theoretical_loss": 3.7360403033928242, + "tokens_seen": 785283072 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038487462387161485, + "loss": 2.9971, + "theoretical_loss": 3.7360092930985322, + "tokens_seen": 785348608 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038486459378134403, + "loss": 3.0687, + "theoretical_loss": 3.7359782861163917, + "tokens_seen": 785414144 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038485456369107327, + "loss": 2.8731, + "theoretical_loss": 3.735947282445772, + "tokens_seen": 785479680 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003848445336008024, + "loss": 2.8119, + "theoretical_loss": 3.735916282086043, + "tokens_seen": 785545216 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038483450351053163, + "loss": 2.7403, + "theoretical_loss": 3.735885285036575, + "tokens_seen": 785610752 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038482447342026076, + "loss": 2.7575, + "theoretical_loss": 3.7358542912967385, + "tokens_seen": 785676288 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038481444332999, + "loss": 2.912, + "theoretical_loss": 3.7358233008659045, + "tokens_seen": 785741824 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003848044132397192, + "loss": 2.9174, + "theoretical_loss": 3.7357923137434437, + "tokens_seen": 785807360 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038479438314944836, + "loss": 2.7221, + "theoretical_loss": 3.7357613299287262, + "tokens_seen": 785872896 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038478435305917754, + "loss": 2.7952, + "theoretical_loss": 3.7357303494211243, + "tokens_seen": 785938432 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003847743229689067, + "loss": 2.7357, + "theoretical_loss": 3.735699372220008, + "tokens_seen": 786003968 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003847642928786359, + "loss": 2.6275, + "theoretical_loss": 3.7356683983247496, + "tokens_seen": 786069504 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038475426278836513, + "loss": 2.6422, + "theoretical_loss": 3.735637427734721, + "tokens_seen": 786135040 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038474423269809426, + "loss": 2.8061, + "theoretical_loss": 3.7356064604492936, + "tokens_seen": 786200576 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003847342026078235, + "loss": 2.8188, + "theoretical_loss": 3.735575496467839, + "tokens_seen": 786266112 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003847241725175527, + "loss": 3.1006, + "theoretical_loss": 3.73554453578973, + "tokens_seen": 786331648 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038471414242728186, + "loss": 2.8199, + "theoretical_loss": 3.7355135784143387, + "tokens_seen": 786397184 + }, + { + "epoch": 2.06, + "objective/train/docs_used": 905150, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.815542459487915, + "objective/train/theoretical_loss": 3.735498100964966, + "objective/train/tokens_used": 806889952, + "theoretical_loss": 3.735498100964966, + "tokens_seen": 786429952 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038470411233701104, + "loss": 2.7891, + "theoretical_loss": 3.735482624341037, + "tokens_seen": 786462720 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003846940822467402, + "loss": 2.8774, + "theoretical_loss": 3.735451673569199, + "tokens_seen": 786528256 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003846840521564694, + "loss": 2.9415, + "theoretical_loss": 3.7354207260981953, + "tokens_seen": 786593792 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038467402206619864, + "loss": 2.9055, + "theoretical_loss": 3.7353897819274016, + "tokens_seen": 786659328 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038466399197592776, + "loss": 2.7626, + "theoretical_loss": 3.735358841056189, + "tokens_seen": 786724864 + }, + { + "epoch": 2.06, + "learning_rate": 0.000384653961885657, + "loss": 2.9134, + "theoretical_loss": 3.735327903483931, + "tokens_seen": 786790400 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003846439317953861, + "loss": 2.9937, + "theoretical_loss": 3.7352969692100024, + "tokens_seen": 786855936 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038463390170511536, + "loss": 2.9249, + "theoretical_loss": 3.7352660382337763, + "tokens_seen": 786921472 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038462387161484454, + "loss": 2.8463, + "theoretical_loss": 3.735235110554626, + "tokens_seen": 786987008 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003846138415245737, + "loss": 2.9076, + "theoretical_loss": 3.735204186171926, + "tokens_seen": 787052544 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003846038114343029, + "loss": 2.7049, + "theoretical_loss": 3.7351732650850504, + "tokens_seen": 787118080 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003845937813440321, + "loss": 2.7692, + "theoretical_loss": 3.7351423472933734, + "tokens_seen": 787183616 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038458375125376127, + "loss": 2.8375, + "theoretical_loss": 3.7351114327962702, + "tokens_seen": 787249152 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003845737211634905, + "loss": 2.9005, + "theoretical_loss": 3.735080521593115, + "tokens_seen": 787314688 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038456369107321963, + "loss": 2.7006, + "theoretical_loss": 3.7350496136832825, + "tokens_seen": 787380224 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038455366098294886, + "loss": 2.8368, + "theoretical_loss": 3.735018709066148, + "tokens_seen": 787445760 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038454363089267805, + "loss": 2.8671, + "theoretical_loss": 3.7349878077410867, + "tokens_seen": 787511296 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038453360080240723, + "loss": 2.8951, + "theoretical_loss": 3.734956909707474, + "tokens_seen": 787576832 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003845235707121364, + "loss": 2.8302, + "theoretical_loss": 3.734926014964686, + "tokens_seen": 787642368 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003845135406218656, + "loss": 2.7686, + "theoretical_loss": 3.7348951235120977, + "tokens_seen": 787707904 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038450351053159477, + "loss": 2.7914, + "theoretical_loss": 3.734864235349085, + "tokens_seen": 787773440 + }, + { + "epoch": 2.06, + "learning_rate": 0.000384493480441324, + "loss": 2.7126, + "theoretical_loss": 3.7348333504750246, + "tokens_seen": 787838976 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038448345035105313, + "loss": 3.0019, + "theoretical_loss": 3.734802468889292, + "tokens_seen": 787904512 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038447342026078237, + "loss": 3.1167, + "theoretical_loss": 3.734771590591264, + "tokens_seen": 787970048 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003844633901705115, + "loss": 2.921, + "theoretical_loss": 3.7347407155803176, + "tokens_seen": 788035584 + }, + { + "epoch": 2.06, + "objective/train/docs_used": 910242, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.993212938308716, + "objective/train/theoretical_loss": 3.734725279307305, + "objective/train/tokens_used": 808528352, + "theoretical_loss": 3.734725279307305, + "tokens_seen": 788068352 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038445336008024073, + "loss": 3.0711, + "theoretical_loss": 3.734709843855829, + "tokens_seen": 788101120 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003844433299899699, + "loss": 2.9976, + "theoretical_loss": 3.734678975417175, + "tokens_seen": 788166656 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003844332998996991, + "loss": 2.8315, + "theoretical_loss": 3.7346481102637332, + "tokens_seen": 788232192 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003844232698094283, + "loss": 2.8954, + "theoretical_loss": 3.7346172483948803, + "tokens_seen": 788297728 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038441323971915745, + "loss": 2.7735, + "theoretical_loss": 3.7345863898099942, + "tokens_seen": 788363264 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038440320962888664, + "loss": 2.8864, + "theoretical_loss": 3.7345555345084525, + "tokens_seen": 788428800 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038439317953861587, + "loss": 3.0191, + "theoretical_loss": 3.7345246824896328, + "tokens_seen": 788494336 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038438314944834505, + "loss": 2.9265, + "theoretical_loss": 3.7344938337529134, + "tokens_seen": 788559872 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038437311935807423, + "loss": 2.7427, + "theoretical_loss": 3.7344629882976714, + "tokens_seen": 788625408 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038436308926780347, + "loss": 2.8258, + "theoretical_loss": 3.7344321461232863, + "tokens_seen": 788690944 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003843530591775326, + "loss": 3.0298, + "theoretical_loss": 3.734401307229136, + "tokens_seen": 788756480 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038434302908726183, + "loss": 2.9562, + "theoretical_loss": 3.734370471614599, + "tokens_seen": 788822016 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038433299899699096, + "loss": 2.8713, + "theoretical_loss": 3.734339639279055, + "tokens_seen": 788887552 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003843229689067202, + "loss": 2.6562, + "theoretical_loss": 3.734308810221881, + "tokens_seen": 788953088 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003843129388164494, + "loss": 3.0193, + "theoretical_loss": 3.7342779844424583, + "tokens_seen": 789018624 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038430290872617856, + "loss": 2.7786, + "theoretical_loss": 3.734247161940165, + "tokens_seen": 789084160 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038429287863590774, + "loss": 2.7898, + "theoretical_loss": 3.734216342714381, + "tokens_seen": 789149696 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003842828485456369, + "loss": 2.8075, + "theoretical_loss": 3.7341855267644855, + "tokens_seen": 789215232 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003842728184553661, + "loss": 2.9642, + "theoretical_loss": 3.7341547140898586, + "tokens_seen": 789280768 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038426278836509533, + "loss": 2.824, + "theoretical_loss": 3.7341239046898806, + "tokens_seen": 789346304 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038425275827482446, + "loss": 2.7216, + "theoretical_loss": 3.734093098563931, + "tokens_seen": 789411840 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003842427281845537, + "loss": 2.6687, + "theoretical_loss": 3.7340622957113907, + "tokens_seen": 789477376 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003842326980942829, + "loss": 2.8243, + "theoretical_loss": 3.7340314961316396, + "tokens_seen": 789542912 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038422266800401206, + "loss": 2.8471, + "theoretical_loss": 3.734000699824059, + "tokens_seen": 789608448 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038421263791374124, + "loss": 2.8609, + "theoretical_loss": 3.7339699067880296, + "tokens_seen": 789673984 + }, + { + "epoch": 2.06, + "objective/train/docs_used": 911523, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8604464530944824, + "objective/train/theoretical_loss": 3.733954511496653, + "objective/train/tokens_used": 810166752, + "theoretical_loss": 3.733954511496653, + "tokens_seen": 789706752 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003842026078234704, + "loss": 2.9005, + "theoretical_loss": 3.7339391170229317, + "tokens_seen": 789739520 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003841925777331996, + "loss": 2.9414, + "theoretical_loss": 3.733908330528147, + "tokens_seen": 789805056 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038418254764292884, + "loss": 2.8559, + "theoretical_loss": 3.7338775473030568, + "tokens_seen": 789870592 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038417251755265796, + "loss": 3.1541, + "theoretical_loss": 3.733846767347043, + "tokens_seen": 789936128 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003841624874623872, + "loss": 2.7023, + "theoretical_loss": 3.7338159906594863, + "tokens_seen": 790001664 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003841524573721163, + "loss": 2.8034, + "theoretical_loss": 3.7337852172397694, + "tokens_seen": 790067200 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038414242728184556, + "loss": 2.7727, + "theoretical_loss": 3.7337544470872737, + "tokens_seen": 790132736 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038413239719157474, + "loss": 2.7565, + "theoretical_loss": 3.733723680201382, + "tokens_seen": 790198272 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003841223671013039, + "loss": 2.8344, + "theoretical_loss": 3.733692916581476, + "tokens_seen": 790263808 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003841123370110331, + "loss": 2.8147, + "theoretical_loss": 3.7336621562269383, + "tokens_seen": 790329344 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003841023069207623, + "loss": 2.8566, + "theoretical_loss": 3.7336313991371517, + "tokens_seen": 790394880 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038409227683049147, + "loss": 2.9304, + "theoretical_loss": 3.7336006453114994, + "tokens_seen": 790460416 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003840822467402207, + "loss": 2.9737, + "theoretical_loss": 3.7335698947493645, + "tokens_seen": 790525952 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038407221664994983, + "loss": 2.7628, + "theoretical_loss": 3.733539147450129, + "tokens_seen": 790591488 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038406218655967907, + "loss": 2.6498, + "theoretical_loss": 3.733508403413177, + "tokens_seen": 790657024 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038405215646940825, + "loss": 2.9293, + "theoretical_loss": 3.7334776626378927, + "tokens_seen": 790722560 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038404212637913743, + "loss": 2.8448, + "theoretical_loss": 3.7334469251236584, + "tokens_seen": 790788096 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003840320962888666, + "loss": 2.8674, + "theoretical_loss": 3.733416190869859, + "tokens_seen": 790853632 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003840220661985958, + "loss": 2.9442, + "theoretical_loss": 3.733385459875878, + "tokens_seen": 790919168 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038401203610832497, + "loss": 2.7676, + "theoretical_loss": 3.7333547321411, + "tokens_seen": 790984704 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003840020060180542, + "loss": 2.5382, + "theoretical_loss": 3.7333240076649092, + "tokens_seen": 791050240 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038399197592778333, + "loss": 2.9655, + "theoretical_loss": 3.7332932864466897, + "tokens_seen": 791115776 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038398194583751257, + "loss": 2.9938, + "theoretical_loss": 3.7332625684858263, + "tokens_seen": 791181312 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003839719157472417, + "loss": 2.8769, + "theoretical_loss": 3.733231853781705, + "tokens_seen": 791246848 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038396188565697093, + "loss": 2.8183, + "theoretical_loss": 3.7332011423337086, + "tokens_seen": 791312384 + }, + { + "epoch": 2.06, + "objective/train/docs_used": 912176, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.054990291595459, + "objective/train/theoretical_loss": 3.733185787830566, + "objective/train/tokens_used": 811805152, + "theoretical_loss": 3.733185787830566, + "tokens_seen": 791345152 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003839518555667001, + "loss": 2.8538, + "theoretical_loss": 3.733170434141224, + "tokens_seen": 791377920 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003839418254764293, + "loss": 2.9927, + "theoretical_loss": 3.7331397292036366, + "tokens_seen": 791443456 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003839317953861585, + "loss": 2.9339, + "theoretical_loss": 3.7331090275203307, + "tokens_seen": 791508992 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038392176529588765, + "loss": 2.7836, + "theoretical_loss": 3.733078329090693, + "tokens_seen": 791574528 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038391173520561684, + "loss": 2.5344, + "theoretical_loss": 3.733047633914109, + "tokens_seen": 791640064 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038390170511534607, + "loss": 3.0145, + "theoretical_loss": 3.7330169419899653, + "tokens_seen": 791705600 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003838916750250752, + "loss": 2.9672, + "theoretical_loss": 3.7329862533176463, + "tokens_seen": 791771136 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038388164493480443, + "loss": 2.8305, + "theoretical_loss": 3.732955567896541, + "tokens_seen": 791836672 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003838716148445336, + "loss": 2.9355, + "theoretical_loss": 3.7329248857260335, + "tokens_seen": 791902208 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003838615847542628, + "loss": 2.7786, + "theoretical_loss": 3.7328942068055118, + "tokens_seen": 791967744 + }, + { + "epoch": 2.06, + "learning_rate": 0.000383851554663992, + "loss": 2.8358, + "theoretical_loss": 3.7328635311343623, + "tokens_seen": 792033280 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038384152457372116, + "loss": 2.6601, + "theoretical_loss": 3.732832858711973, + "tokens_seen": 792098816 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038383149448345034, + "loss": 2.618, + "theoretical_loss": 3.7328021895377295, + "tokens_seen": 792164352 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003838214643931796, + "loss": 2.9023, + "theoretical_loss": 3.73277152361102, + "tokens_seen": 792229888 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003838114343029087, + "loss": 2.7241, + "theoretical_loss": 3.732740860931232, + "tokens_seen": 792295424 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038380140421263794, + "loss": 2.7667, + "theoretical_loss": 3.7327102014977527, + "tokens_seen": 792360960 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038379137412236706, + "loss": 2.6392, + "theoretical_loss": 3.732679545309971, + "tokens_seen": 792426496 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003837813440320963, + "loss": 2.877, + "theoretical_loss": 3.732648892367274, + "tokens_seen": 792492032 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003837713139418255, + "loss": 2.8724, + "theoretical_loss": 3.7326182426690506, + "tokens_seen": 792557568 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038376128385155466, + "loss": 2.7809, + "theoretical_loss": 3.732587596214688, + "tokens_seen": 792623104 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038375125376128384, + "loss": 2.858, + "theoretical_loss": 3.732556953003576, + "tokens_seen": 792688640 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003837412236710131, + "loss": 2.7547, + "theoretical_loss": 3.732526313035102, + "tokens_seen": 792754176 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003837311935807422, + "loss": 2.8265, + "theoretical_loss": 3.7324956763086563, + "tokens_seen": 792819712 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038372116349047144, + "loss": 3.1426, + "theoretical_loss": 3.732465042823627, + "tokens_seen": 792885248 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038371113340020057, + "loss": 2.7527, + "theoretical_loss": 3.732434412579403, + "tokens_seen": 792950784 + }, + { + "epoch": 2.06, + "objective/train/docs_used": 913766, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.490590810775757, + "objective/train/theoretical_loss": 3.732419098672402, + "objective/train/tokens_used": 813443552, + "theoretical_loss": 3.732419098672402, + "tokens_seen": 792983552 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003837011033099298, + "loss": 2.7067, + "theoretical_loss": 3.7324037855753742, + "tokens_seen": 793016320 + }, + { + "epoch": 2.06, + "learning_rate": 0.000383691073219659, + "loss": 2.7344, + "theoretical_loss": 3.73237316181093, + "tokens_seen": 793081856 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038368104312938816, + "loss": 2.8914, + "theoretical_loss": 3.73234254128546, + "tokens_seen": 793147392 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038367101303911735, + "loss": 2.8415, + "theoretical_loss": 3.7323119239983544, + "tokens_seen": 793212928 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003836609829488465, + "loss": 2.8043, + "theoretical_loss": 3.732281309949002, + "tokens_seen": 793278464 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003836509528585757, + "loss": 2.803, + "theoretical_loss": 3.732250699136795, + "tokens_seen": 793344000 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038364092276830494, + "loss": 2.7812, + "theoretical_loss": 3.732220091561122, + "tokens_seen": 793409536 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003836308926780341, + "loss": 2.6567, + "theoretical_loss": 3.7321894872213743, + "tokens_seen": 793475072 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003836208625877633, + "loss": 2.6744, + "theoretical_loss": 3.7321588861169417, + "tokens_seen": 793540608 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003836108324974925, + "loss": 2.8148, + "theoretical_loss": 3.7321282882472167, + "tokens_seen": 793606144 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038360080240722167, + "loss": 2.8026, + "theoretical_loss": 3.7320976936115886, + "tokens_seen": 793671680 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003835907723169509, + "loss": 2.8677, + "theoretical_loss": 3.7320671022094496, + "tokens_seen": 793737216 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038358074222668003, + "loss": 2.9286, + "theoretical_loss": 3.7320365140401903, + "tokens_seen": 793802752 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038357071213640927, + "loss": 2.9299, + "theoretical_loss": 3.7320059291032024, + "tokens_seen": 793868288 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038356068204613845, + "loss": 2.8521, + "theoretical_loss": 3.7319753473978783, + "tokens_seen": 793933824 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038355065195586763, + "loss": 2.7029, + "theoretical_loss": 3.7319447689236087, + "tokens_seen": 793999360 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003835406218655968, + "loss": 2.8717, + "theoretical_loss": 3.7319141936797866, + "tokens_seen": 794064896 + }, + { + "epoch": 2.06, + "learning_rate": 0.000383530591775326, + "loss": 2.5418, + "theoretical_loss": 3.731883621665803, + "tokens_seen": 794130432 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038352056168505517, + "loss": 2.831, + "theoretical_loss": 3.731853052881051, + "tokens_seen": 794195968 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003835105315947844, + "loss": 2.8505, + "theoretical_loss": 3.7318224873249237, + "tokens_seen": 794261504 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038350050150451353, + "loss": 2.7467, + "theoretical_loss": 3.731791924996812, + "tokens_seen": 794327040 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038349047141424277, + "loss": 2.6475, + "theoretical_loss": 3.7317613658961104, + "tokens_seen": 794392576 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003834804413239719, + "loss": 2.829, + "theoretical_loss": 3.731730810022211, + "tokens_seen": 794458112 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038347041123370113, + "loss": 2.832, + "theoretical_loss": 3.7317002573745066, + "tokens_seen": 794523648 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003834603811434303, + "loss": 2.8858, + "theoretical_loss": 3.731669707952391, + "tokens_seen": 794589184 + }, + { + "epoch": 2.06, + "objective/train/docs_used": 914513, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7244105339050293, + "objective/train/theoretical_loss": 3.7316544344507396, + "objective/train/tokens_used": 815081952, + "theoretical_loss": 3.7316544344507396, + "tokens_seen": 794621952 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003834503510531595, + "loss": 2.8052, + "theoretical_loss": 3.731639161755258, + "tokens_seen": 794654720 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003834403209628887, + "loss": 2.9226, + "theoretical_loss": 3.731608618782501, + "tokens_seen": 794720256 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038343029087261786, + "loss": 2.8461, + "theoretical_loss": 3.731578079033513, + "tokens_seen": 794785792 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038342026078234704, + "loss": 2.7635, + "theoretical_loss": 3.7315475425076894, + "tokens_seen": 794851328 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038341023069207627, + "loss": 2.7834, + "theoretical_loss": 3.731517009204423, + "tokens_seen": 794916864 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003834002006018054, + "loss": 2.6976, + "theoretical_loss": 3.7314864791231086, + "tokens_seen": 794982400 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038339017051153463, + "loss": 2.7814, + "theoretical_loss": 3.7314559522631408, + "tokens_seen": 795047936 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003833801404212638, + "loss": 2.6001, + "theoretical_loss": 3.7314254286239144, + "tokens_seen": 795113472 + }, + { + "epoch": 2.06, + "learning_rate": 0.000383370110330993, + "loss": 2.9477, + "theoretical_loss": 3.7313949082048232, + "tokens_seen": 795179008 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003833600802407222, + "loss": 2.9413, + "theoretical_loss": 3.731364391005263, + "tokens_seen": 795244544 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038335005015045136, + "loss": 2.7242, + "theoretical_loss": 3.731333877024629, + "tokens_seen": 795310080 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038334002006018054, + "loss": 2.7499, + "theoretical_loss": 3.7313033662623156, + "tokens_seen": 795375616 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003833299899699098, + "loss": 2.7898, + "theoretical_loss": 3.7312728587177193, + "tokens_seen": 795441152 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003833199598796389, + "loss": 2.7396, + "theoretical_loss": 3.7312423543902344, + "tokens_seen": 795506688 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038330992978936814, + "loss": 2.7025, + "theoretical_loss": 3.731211853279258, + "tokens_seen": 795572224 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038329989969909726, + "loss": 2.7248, + "theoretical_loss": 3.7311813553841855, + "tokens_seen": 795637760 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003832898696088265, + "loss": 2.7345, + "theoretical_loss": 3.731150860704413, + "tokens_seen": 795703296 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003832798395185557, + "loss": 2.8407, + "theoretical_loss": 3.7311203692393358, + "tokens_seen": 795768832 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038326980942828486, + "loss": 2.9334, + "theoretical_loss": 3.731089880988352, + "tokens_seen": 795834368 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038325977933801404, + "loss": 2.7804, + "theoretical_loss": 3.731059395950857, + "tokens_seen": 795899904 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003832497492477433, + "loss": 2.7048, + "theoretical_loss": 3.731028914126248, + "tokens_seen": 795965440 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003832397191574724, + "loss": 2.8948, + "theoretical_loss": 3.7309984355139214, + "tokens_seen": 796030976 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038322968906720164, + "loss": 2.953, + "theoretical_loss": 3.730967960113275, + "tokens_seen": 796096512 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038321965897693077, + "loss": 2.8011, + "theoretical_loss": 3.730937487923705, + "tokens_seen": 796162048 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038320962888666, + "loss": 2.8264, + "theoretical_loss": 3.7309070189446096, + "tokens_seen": 796227584 + }, + { + "epoch": 2.06, + "objective/train/docs_used": 915896, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.109905481338501, + "objective/train/theoretical_loss": 3.7308917856588018, + "objective/train/tokens_used": 816720352, + "theoretical_loss": 3.7308917856588018, + "tokens_seen": 796260352 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003831995987963892, + "loss": 2.8979, + "theoretical_loss": 3.7308765531753867, + "tokens_seen": 796293120 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038318956870611836, + "loss": 2.908, + "theoretical_loss": 3.730846090615433, + "tokens_seen": 796358656 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038317953861584755, + "loss": 2.7226, + "theoretical_loss": 3.730815631264147, + "tokens_seen": 796424192 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003831695085255767, + "loss": 2.5663, + "theoretical_loss": 3.7307851751209267, + "tokens_seen": 796489728 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003831594784353059, + "loss": 2.7183, + "theoretical_loss": 3.7307547221851696, + "tokens_seen": 796555264 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038314944834503514, + "loss": 2.8574, + "theoretical_loss": 3.730724272456275, + "tokens_seen": 796620800 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038313941825476427, + "loss": 2.8418, + "theoretical_loss": 3.730693825933641, + "tokens_seen": 796686336 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003831293881644935, + "loss": 2.5915, + "theoretical_loss": 3.7306633826166666, + "tokens_seen": 796751872 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038311935807422263, + "loss": 2.9564, + "theoretical_loss": 3.73063294250475, + "tokens_seen": 796817408 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038310932798395187, + "loss": 2.8469, + "theoretical_loss": 3.730602505597291, + "tokens_seen": 796882944 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038309929789368105, + "loss": 2.887, + "theoretical_loss": 3.730572071893688, + "tokens_seen": 796948480 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038308926780341023, + "loss": 2.8746, + "theoretical_loss": 3.7305416413933408, + "tokens_seen": 797014016 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003830792377131394, + "loss": 2.9179, + "theoretical_loss": 3.7305112140956487, + "tokens_seen": 797079552 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038306920762286865, + "loss": 2.9636, + "theoretical_loss": 3.730480790000011, + "tokens_seen": 797145088 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003830591775325978, + "loss": 2.7864, + "theoretical_loss": 3.730450369105829, + "tokens_seen": 797210624 + }, + { + "epoch": 2.06, + "learning_rate": 0.000383049147442327, + "loss": 3.0204, + "theoretical_loss": 3.7304199514125007, + "tokens_seen": 797276160 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038303911735205614, + "loss": 2.8526, + "theoretical_loss": 3.7303895369194273, + "tokens_seen": 797341696 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038302908726178537, + "loss": 2.7846, + "theoretical_loss": 3.730359125626009, + "tokens_seen": 797407232 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038301905717151455, + "loss": 2.941, + "theoretical_loss": 3.7303287175316466, + "tokens_seen": 797472768 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038300902708124373, + "loss": 2.7052, + "theoretical_loss": 3.73029831263574, + "tokens_seen": 797538304 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003829989969909729, + "loss": 2.7305, + "theoretical_loss": 3.73026791093769, + "tokens_seen": 797603840 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003829889669007021, + "loss": 2.7913, + "theoretical_loss": 3.730237512436899, + "tokens_seen": 797669376 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003829789368104313, + "loss": 2.7654, + "theoretical_loss": 3.730207117132766, + "tokens_seen": 797734912 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003829689067201605, + "loss": 2.7138, + "theoretical_loss": 3.7301767250246938, + "tokens_seen": 797800448 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038295887662988964, + "loss": 2.9547, + "theoretical_loss": 3.7301463361120835, + "tokens_seen": 797865984 + }, + { + "epoch": 2.06, + "objective/train/docs_used": 916418, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8996529579162598, + "objective/train/theoretical_loss": 3.730131142853889, + "objective/train/tokens_used": 818358752, + "theoretical_loss": 3.730131142853889, + "tokens_seen": 797898752 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003829488465396189, + "loss": 3.0152, + "theoretical_loss": 3.7301159503943357, + "tokens_seen": 797931520 + }, + { + "epoch": 2.06, + "learning_rate": 0.000382938816449348, + "loss": 3.0106, + "theoretical_loss": 3.7300855678708533, + "tokens_seen": 797997056 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038292878635907724, + "loss": 2.8793, + "theoretical_loss": 3.7300551885410385, + "tokens_seen": 798062592 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003829187562688064, + "loss": 2.7218, + "theoretical_loss": 3.730024812404292, + "tokens_seen": 798128128 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003829087261785356, + "loss": 2.8625, + "theoretical_loss": 3.7299944394600173, + "tokens_seen": 798193664 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003828986960882648, + "loss": 2.5771, + "theoretical_loss": 3.729964069707616, + "tokens_seen": 798259200 + }, + { + "epoch": 2.06, + "learning_rate": 0.000382888665997994, + "loss": 2.8149, + "theoretical_loss": 3.729933703146491, + "tokens_seen": 798324736 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003828786359077232, + "loss": 2.8857, + "theoretical_loss": 3.729903339776045, + "tokens_seen": 798390272 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003828686058174524, + "loss": 2.9546, + "theoretical_loss": 3.7298729795956804, + "tokens_seen": 798455808 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038285857572718156, + "loss": 2.9196, + "theoretical_loss": 3.729842622604801, + "tokens_seen": 798521344 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038284854563691074, + "loss": 2.921, + "theoretical_loss": 3.72981226880281, + "tokens_seen": 798586880 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038283851554664, + "loss": 3.0057, + "theoretical_loss": 3.7297819181891105, + "tokens_seen": 798652416 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003828284854563691, + "loss": 3.0165, + "theoretical_loss": 3.7297515707631055, + "tokens_seen": 798717952 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038281845536609834, + "loss": 3.0633, + "theoretical_loss": 3.7297212265242, + "tokens_seen": 798783488 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038280842527582746, + "loss": 2.8644, + "theoretical_loss": 3.729690885471796, + "tokens_seen": 798849024 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003827983951855567, + "loss": 2.935, + "theoretical_loss": 3.7296605476052993, + "tokens_seen": 798914560 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003827883650952859, + "loss": 2.9549, + "theoretical_loss": 3.7296302129241132, + "tokens_seen": 798980096 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038277833500501506, + "loss": 2.798, + "theoretical_loss": 3.729599881427642, + "tokens_seen": 799045632 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038276830491474424, + "loss": 2.9321, + "theoretical_loss": 3.72956955311529, + "tokens_seen": 799111168 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003827582748244735, + "loss": 2.8404, + "theoretical_loss": 3.7295392279864625, + "tokens_seen": 799176704 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003827482447342026, + "loss": 2.5635, + "theoretical_loss": 3.729508906040564, + "tokens_seen": 799242240 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038273821464393184, + "loss": 2.7403, + "theoretical_loss": 3.7294785872769993, + "tokens_seen": 799307776 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038272818455366097, + "loss": 2.9023, + "theoretical_loss": 3.7294482716951736, + "tokens_seen": 799373312 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003827181544633902, + "loss": 2.5704, + "theoretical_loss": 3.7294179592944925, + "tokens_seen": 799438848 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003827081243731194, + "loss": 2.8978, + "theoretical_loss": 3.729387650074361, + "tokens_seen": 799504384 + }, + { + "epoch": 2.06, + "objective/train/docs_used": 917678, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9319193363189697, + "objective/train/theoretical_loss": 3.729372496656816, + "objective/train/tokens_used": 819997152, + "theoretical_loss": 3.729372496656816, + "tokens_seen": 799537152 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038269809428284856, + "loss": 2.9678, + "theoretical_loss": 3.729357344034185, + "tokens_seen": 799569920 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038268806419257775, + "loss": 2.7522, + "theoretical_loss": 3.72932704117337, + "tokens_seen": 799635456 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003826780341023069, + "loss": 2.9152, + "theoretical_loss": 3.7292967414913223, + "tokens_seen": 799700992 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003826680040120361, + "loss": 2.5198, + "theoretical_loss": 3.7292664449874477, + "tokens_seen": 799766528 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038265797392176534, + "loss": 2.8087, + "theoretical_loss": 3.729236151661153, + "tokens_seen": 799832064 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038264794383149447, + "loss": 2.8923, + "theoretical_loss": 3.7292058615118435, + "tokens_seen": 799897600 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003826379137412237, + "loss": 2.7683, + "theoretical_loss": 3.729175574538927, + "tokens_seen": 799963136 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038262788365095283, + "loss": 2.7146, + "theoretical_loss": 3.7291452907418092, + "tokens_seen": 800028672 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038261785356068207, + "loss": 2.9047, + "theoretical_loss": 3.729115010119897, + "tokens_seen": 800094208 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038260782347041125, + "loss": 2.712, + "theoretical_loss": 3.7290847326725984, + "tokens_seen": 800159744 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038259779338014043, + "loss": 2.7399, + "theoretical_loss": 3.72905445839932, + "tokens_seen": 800225280 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003825877632898696, + "loss": 2.6716, + "theoretical_loss": 3.729024187299469, + "tokens_seen": 800290816 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038257773319959885, + "loss": 2.878, + "theoretical_loss": 3.7289939193724537, + "tokens_seen": 800356352 + }, + { + "epoch": 2.06, + "learning_rate": 0.000382567703109328, + "loss": 2.7631, + "theoretical_loss": 3.728963654617681, + "tokens_seen": 800421888 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003825576730190572, + "loss": 2.8419, + "theoretical_loss": 3.7289333930345587, + "tokens_seen": 800487424 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038254764292878634, + "loss": 2.5932, + "theoretical_loss": 3.7289031346224952, + "tokens_seen": 800552960 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038253761283851557, + "loss": 2.8904, + "theoretical_loss": 3.7288728793808987, + "tokens_seen": 800618496 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038252758274824475, + "loss": 3.065, + "theoretical_loss": 3.7288426273091773, + "tokens_seen": 800684032 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038251755265797393, + "loss": 2.7377, + "theoretical_loss": 3.7288123784067393, + "tokens_seen": 800749568 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003825075225677031, + "loss": 2.8729, + "theoretical_loss": 3.7287821326729933, + "tokens_seen": 800815104 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003824974924774323, + "loss": 2.7359, + "theoretical_loss": 3.728751890107349, + "tokens_seen": 800880640 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003824874623871615, + "loss": 2.8713, + "theoretical_loss": 3.7287216507092142, + "tokens_seen": 800946176 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003824774322968907, + "loss": 2.5688, + "theoretical_loss": 3.728691414477999, + "tokens_seen": 801011712 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038246740220661984, + "loss": 2.7509, + "theoretical_loss": 3.728661181413112, + "tokens_seen": 801077248 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003824573721163491, + "loss": 2.6961, + "theoretical_loss": 3.7286309515139626, + "tokens_seen": 801142784 + }, + { + "epoch": 2.06, + "objective/train/docs_used": 918390, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.938758134841919, + "objective/train/theoretical_loss": 3.728615837751355, + "objective/train/tokens_used": 821635552, + "theoretical_loss": 3.728615837751355, + "tokens_seen": 801175552 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003824473420260782, + "loss": 3.0026, + "theoretical_loss": 3.7286007247799606, + "tokens_seen": 801208320 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038243731193580744, + "loss": 2.6613, + "theoretical_loss": 3.7285705012105157, + "tokens_seen": 801273856 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003824272818455366, + "loss": 2.668, + "theoretical_loss": 3.7285402808050376, + "tokens_seen": 801339392 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003824172517552658, + "loss": 2.5081, + "theoretical_loss": 3.728510063562937, + "tokens_seen": 801404928 + }, + { + "epoch": 2.06, + "learning_rate": 0.000382407221664995, + "loss": 2.5455, + "theoretical_loss": 3.7284798494836235, + "tokens_seen": 801470464 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003823971915747242, + "loss": 2.7998, + "theoretical_loss": 3.7284496385665076, + "tokens_seen": 801536000 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038238716148445334, + "loss": 2.8151, + "theoretical_loss": 3.728419430811, + "tokens_seen": 801601536 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003823771313941826, + "loss": 2.7684, + "theoretical_loss": 3.7283892262165117, + "tokens_seen": 801667072 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003823671013039117, + "loss": 3.0378, + "theoretical_loss": 3.7283590247824527, + "tokens_seen": 801732608 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038235707121364094, + "loss": 2.9177, + "theoretical_loss": 3.7283288265082346, + "tokens_seen": 801798144 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003823470411233701, + "loss": 3.1085, + "theoretical_loss": 3.7282986313932684, + "tokens_seen": 801863680 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003823370110330993, + "loss": 2.8762, + "theoretical_loss": 3.728268439436966, + "tokens_seen": 801929216 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003823269809428285, + "loss": 2.7137, + "theoretical_loss": 3.7282382506387375, + "tokens_seen": 801994752 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038231695085255766, + "loss": 2.524, + "theoretical_loss": 3.728208064997996, + "tokens_seen": 802060288 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038230692076228685, + "loss": 2.8362, + "theoretical_loss": 3.728177882514152, + "tokens_seen": 802125824 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003822968906720161, + "loss": 2.9109, + "theoretical_loss": 3.728147703186619, + "tokens_seen": 802191360 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003822868605817452, + "loss": 2.7985, + "theoretical_loss": 3.728117527014808, + "tokens_seen": 802256896 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038227683049147444, + "loss": 2.8394, + "theoretical_loss": 3.7280873539981307, + "tokens_seen": 802322432 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038226680040120357, + "loss": 2.7245, + "theoretical_loss": 3.7280571841360013, + "tokens_seen": 802387968 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003822567703109328, + "loss": 2.8553, + "theoretical_loss": 3.7280270174278316, + "tokens_seen": 802453504 + }, + { + "epoch": 2.06, + "learning_rate": 0.000382246740220662, + "loss": 2.6918, + "theoretical_loss": 3.7279968538730333, + "tokens_seen": 802519040 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038223671013039117, + "loss": 2.6755, + "theoretical_loss": 3.7279666934710205, + "tokens_seen": 802584576 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038222668004012035, + "loss": 2.7522, + "theoretical_loss": 3.727936536221206, + "tokens_seen": 802650112 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003822166499498496, + "loss": 2.6767, + "theoretical_loss": 3.7279063821230034, + "tokens_seen": 802715648 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003822066198595787, + "loss": 2.8416, + "theoretical_loss": 3.727876231175825, + "tokens_seen": 802781184 + }, + { + "epoch": 2.06, + "objective/train/docs_used": 919804, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.2665350437164307, + "objective/train/theoretical_loss": 3.727861156883687, + "objective/train/tokens_used": 823273952, + "theoretical_loss": 3.727861156883687, + "tokens_seen": 802813952 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038219658976930795, + "loss": 2.9272, + "theoretical_loss": 3.7278460833790854, + "tokens_seen": 802846720 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003821865596790371, + "loss": 2.8213, + "theoretical_loss": 3.7278159387321974, + "tokens_seen": 802912256 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003821765295887663, + "loss": 2.7012, + "theoretical_loss": 3.727785797234575, + "tokens_seen": 802977792 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003821664994984955, + "loss": 2.6509, + "theoretical_loss": 3.7277556588856333, + "tokens_seen": 803043328 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038215646940822467, + "loss": 2.8218, + "theoretical_loss": 3.7277255236847853, + "tokens_seen": 803108864 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038214643931795385, + "loss": 2.9506, + "theoretical_loss": 3.7276953916314453, + "tokens_seen": 803174400 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038213640922768303, + "loss": 2.8434, + "theoretical_loss": 3.727665262725029, + "tokens_seen": 803239936 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038212637913741227, + "loss": 2.9449, + "theoretical_loss": 3.7276351369649494, + "tokens_seen": 803305472 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038211634904714145, + "loss": 2.6784, + "theoretical_loss": 3.7276050143506225, + "tokens_seen": 803371008 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038210631895687063, + "loss": 2.9453, + "theoretical_loss": 3.727574894881462, + "tokens_seen": 803436544 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003820962888665998, + "loss": 2.7106, + "theoretical_loss": 3.7275447785568847, + "tokens_seen": 803502080 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038208625877632905, + "loss": 2.8271, + "theoretical_loss": 3.7275146653763045, + "tokens_seen": 803567616 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003820762286860582, + "loss": 2.9294, + "theoretical_loss": 3.727484555339137, + "tokens_seen": 803633152 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003820661985957874, + "loss": 2.5488, + "theoretical_loss": 3.7274544484447985, + "tokens_seen": 803698688 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038205616850551654, + "loss": 2.9373, + "theoretical_loss": 3.727424344692704, + "tokens_seen": 803764224 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038204613841524577, + "loss": 2.9778, + "theoretical_loss": 3.7273942440822694, + "tokens_seen": 803829760 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038203610832497495, + "loss": 2.9941, + "theoretical_loss": 3.727364146612911, + "tokens_seen": 803895296 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038202607823470413, + "loss": 2.6439, + "theoretical_loss": 3.727334052284045, + "tokens_seen": 803960832 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003820160481444333, + "loss": 2.7207, + "theoretical_loss": 3.7273039610950875, + "tokens_seen": 804026368 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003820060180541625, + "loss": 2.9206, + "theoretical_loss": 3.7272738730454558, + "tokens_seen": 804091904 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003819959879638917, + "loss": 2.8644, + "theoretical_loss": 3.7272437881345652, + "tokens_seen": 804157440 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003819859578736209, + "loss": 2.964, + "theoretical_loss": 3.7272137063618334, + "tokens_seen": 804222976 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038197592778335004, + "loss": 2.8123, + "theoretical_loss": 3.727183627726677, + "tokens_seen": 804288512 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003819658976930793, + "loss": 2.7005, + "theoretical_loss": 3.727153552228514, + "tokens_seen": 804354048 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003819558676028084, + "loss": 3.1517, + "theoretical_loss": 3.7271234798667603, + "tokens_seen": 804419584 + }, + { + "epoch": 2.06, + "objective/train/docs_used": 920531, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.036098003387451, + "objective/train/theoretical_loss": 3.727108444861855, + "objective/train/tokens_used": 824912352, + "theoretical_loss": 3.727108444861855, + "tokens_seen": 804452352 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038194583751253764, + "loss": 2.6692, + "theoretical_loss": 3.727093410640835, + "tokens_seen": 804485120 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003819358074222668, + "loss": 2.885, + "theoretical_loss": 3.727063344550154, + "tokens_seen": 804550656 + }, + { + "epoch": 2.06, + "learning_rate": 0.000381925777331996, + "loss": 3.1506, + "theoretical_loss": 3.727033281594136, + "tokens_seen": 804616192 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003819157472417252, + "loss": 2.9126, + "theoretical_loss": 3.7270032217721987, + "tokens_seen": 804681728 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003819057171514544, + "loss": 2.7855, + "theoretical_loss": 3.72697316508376, + "tokens_seen": 804747264 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038189568706118354, + "loss": 2.5874, + "theoretical_loss": 3.7269431115282385, + "tokens_seen": 804812800 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003818856569709128, + "loss": 2.7009, + "theoretical_loss": 3.7269130611050523, + "tokens_seen": 804878336 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003818756268806419, + "loss": 2.4843, + "theoretical_loss": 3.72688301381362, + "tokens_seen": 804943872 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038186559679037114, + "loss": 2.7415, + "theoretical_loss": 3.72685296965336, + "tokens_seen": 805009408 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003818555667001003, + "loss": 2.5227, + "theoretical_loss": 3.7268229286236916, + "tokens_seen": 805074944 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003818455366098295, + "loss": 2.7426, + "theoretical_loss": 3.7267928907240337, + "tokens_seen": 805140480 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003818355065195587, + "loss": 2.7953, + "theoretical_loss": 3.7267628559538055, + "tokens_seen": 805206016 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038182547642928786, + "loss": 2.7316, + "theoretical_loss": 3.7267328243124256, + "tokens_seen": 805271552 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038181544633901705, + "loss": 2.7126, + "theoretical_loss": 3.726702795799314, + "tokens_seen": 805337088 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003818054162487463, + "loss": 2.8651, + "theoretical_loss": 3.7266727704138907, + "tokens_seen": 805402624 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003817953861584754, + "loss": 2.7344, + "theoretical_loss": 3.726642748155575, + "tokens_seen": 805468160 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038178535606820464, + "loss": 2.8418, + "theoretical_loss": 3.7266127290237865, + "tokens_seen": 805533696 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038177532597793377, + "loss": 2.887, + "theoretical_loss": 3.7265827130179465, + "tokens_seen": 805599232 + }, + { + "epoch": 2.06, + "learning_rate": 0.000381765295887663, + "loss": 2.8043, + "theoretical_loss": 3.7265527001374736, + "tokens_seen": 805664768 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003817552657973922, + "loss": 2.8358, + "theoretical_loss": 3.7265226903817896, + "tokens_seen": 805730304 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038174523570712137, + "loss": 2.6254, + "theoretical_loss": 3.726492683750314, + "tokens_seen": 805795840 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038173520561685055, + "loss": 2.6947, + "theoretical_loss": 3.7264626802424683, + "tokens_seen": 805861376 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003817251755265798, + "loss": 2.6898, + "theoretical_loss": 3.7264326798576723, + "tokens_seen": 805926912 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003817151454363089, + "loss": 2.7434, + "theoretical_loss": 3.7264026825953485, + "tokens_seen": 805992448 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038170511534603815, + "loss": 2.557, + "theoretical_loss": 3.7263726884549166, + "tokens_seen": 806057984 + }, + { + "epoch": 2.06, + "objective/train/docs_used": 921184, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.061922788619995, + "objective/train/theoretical_loss": 3.7263576925552298, + "objective/train/tokens_used": 826550752, + "theoretical_loss": 3.7263576925552298, + "tokens_seen": 806090752 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003816950852557673, + "loss": 2.9332, + "theoretical_loss": 3.7263426974357987, + "tokens_seen": 806123520 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003816850551654965, + "loss": 2.8546, + "theoretical_loss": 3.7263127095374164, + "tokens_seen": 806189056 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003816750250752257, + "loss": 2.9827, + "theoretical_loss": 3.7262827247591908, + "tokens_seen": 806254592 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038166499498495487, + "loss": 2.7669, + "theoretical_loss": 3.7262527431005443, + "tokens_seen": 806320128 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038165496489468405, + "loss": 2.9307, + "theoretical_loss": 3.726222764560898, + "tokens_seen": 806385664 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038164493480441323, + "loss": 2.855, + "theoretical_loss": 3.7261927891396747, + "tokens_seen": 806451200 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003816349047141424, + "loss": 2.6853, + "theoretical_loss": 3.726162816836296, + "tokens_seen": 806516736 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038162487462387165, + "loss": 2.6587, + "theoretical_loss": 3.7261328476501845, + "tokens_seen": 806582272 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003816148445336008, + "loss": 2.7978, + "theoretical_loss": 3.7261028815807635, + "tokens_seen": 806647808 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038160481444333, + "loss": 2.8737, + "theoretical_loss": 3.726072918627455, + "tokens_seen": 806713344 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038159478435305914, + "loss": 2.6942, + "theoretical_loss": 3.7260429587896815, + "tokens_seen": 806778880 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003815847542627884, + "loss": 2.8407, + "theoretical_loss": 3.7260130020668667, + "tokens_seen": 806844416 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038157472417251756, + "loss": 2.8455, + "theoretical_loss": 3.725983048458434, + "tokens_seen": 806909952 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038156469408224674, + "loss": 2.7394, + "theoretical_loss": 3.7259530979638056, + "tokens_seen": 806975488 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003815546639919759, + "loss": 2.8217, + "theoretical_loss": 3.725923150582406, + "tokens_seen": 807041024 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038154463390170515, + "loss": 2.7258, + "theoretical_loss": 3.7258932063136583, + "tokens_seen": 807106560 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003815346038114343, + "loss": 2.4661, + "theoretical_loss": 3.7258632651569865, + "tokens_seen": 807172096 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003815245737211635, + "loss": 2.8683, + "theoretical_loss": 3.725833327111814, + "tokens_seen": 807237632 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038151454363089264, + "loss": 2.3263, + "theoretical_loss": 3.725803392177566, + "tokens_seen": 807303168 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003815045135406219, + "loss": 2.5954, + "theoretical_loss": 3.7257734603536656, + "tokens_seen": 807368704 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038149448345035106, + "loss": 2.9207, + "theoretical_loss": 3.7257435316395373, + "tokens_seen": 807434240 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038148445336008024, + "loss": 3.1429, + "theoretical_loss": 3.7257136060346063, + "tokens_seen": 807499776 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003814744232698094, + "loss": 2.795, + "theoretical_loss": 3.725683683538297, + "tokens_seen": 807565312 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003814643931795386, + "loss": 2.5561, + "theoretical_loss": 3.7256537641500347, + "tokens_seen": 807630848 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003814543630892678, + "loss": 2.9045, + "theoretical_loss": 3.725623847869244, + "tokens_seen": 807696384 + }, + { + "epoch": 2.06, + "objective/train/docs_used": 922293, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.627511501312256, + "objective/train/theoretical_loss": 3.72560889089397, + "objective/train/tokens_used": 828189152, + "theoretical_loss": 3.72560889089397, + "tokens_seen": 807729152 + }, + { + "epoch": 2.06, + "learning_rate": 0.000381444332998997, + "loss": 2.818, + "theoretical_loss": 3.725593934695349, + "tokens_seen": 807761920 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038143430290872614, + "loss": 2.9807, + "theoretical_loss": 3.7255640246277766, + "tokens_seen": 807827456 + }, + { + "epoch": 2.06, + "learning_rate": 0.0003814242728184554, + "loss": 2.9548, + "theoretical_loss": 3.7255341176659513, + "tokens_seen": 807892992 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038141424272818456, + "loss": 2.9676, + "theoretical_loss": 3.725504213809299, + "tokens_seen": 807958528 + }, + { + "epoch": 2.06, + "learning_rate": 0.00038140421263791374, + "loss": 2.7428, + "theoretical_loss": 3.725474313057246, + "tokens_seen": 808024064 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003813941825476429, + "loss": 2.8616, + "theoretical_loss": 3.7254444154092177, + "tokens_seen": 808089600 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003813841524573721, + "loss": 2.9295, + "theoretical_loss": 3.72541452086464, + "tokens_seen": 808155136 + }, + { + "epoch": 2.07, + "learning_rate": 0.00038137412236710134, + "loss": 2.6852, + "theoretical_loss": 3.7253846294229396, + "tokens_seen": 808220672 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003813640922768305, + "loss": 2.6936, + "theoretical_loss": 3.7253547410835424, + "tokens_seen": 808286208 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003813540621865597, + "loss": 2.5662, + "theoretical_loss": 3.725324855845875, + "tokens_seen": 808351744 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003813440320962889, + "loss": 2.8743, + "theoretical_loss": 3.7252949737093646, + "tokens_seen": 808417280 + }, + { + "epoch": 2.07, + "learning_rate": 0.00038133400200601806, + "loss": 2.6674, + "theoretical_loss": 3.7252650946734374, + "tokens_seen": 808482816 + }, + { + "epoch": 2.07, + "learning_rate": 0.00038132397191574725, + "loss": 2.8592, + "theoretical_loss": 3.7252352187375206, + "tokens_seen": 808548352 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003813139418254765, + "loss": 2.9754, + "theoretical_loss": 3.725205345901042, + "tokens_seen": 808613888 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003813039117352056, + "loss": 2.6084, + "theoretical_loss": 3.725175476163427, + "tokens_seen": 808679424 + }, + { + "epoch": 2.07, + "learning_rate": 0.00038129388164493484, + "loss": 2.7187, + "theoretical_loss": 3.7251456095241053, + "tokens_seen": 808744960 + }, + { + "epoch": 2.07, + "learning_rate": 0.00038128385155466397, + "loss": 2.8086, + "theoretical_loss": 3.725115745982503, + "tokens_seen": 808810496 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003812738214643932, + "loss": 2.8193, + "theoretical_loss": 3.725085885538049, + "tokens_seen": 808876032 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003812637913741224, + "loss": 2.8032, + "theoretical_loss": 3.7250560281901706, + "tokens_seen": 808941568 + }, + { + "epoch": 2.07, + "learning_rate": 0.00038125376128385157, + "loss": 2.766, + "theoretical_loss": 3.7250261739382955, + "tokens_seen": 809007104 + }, + { + "epoch": 2.07, + "learning_rate": 0.00038124373119358075, + "loss": 2.82, + "theoretical_loss": 3.724996322781852, + "tokens_seen": 809072640 + }, + { + "epoch": 2.07, + "learning_rate": 0.00038123370110331, + "loss": 2.5854, + "theoretical_loss": 3.7249664747202686, + "tokens_seen": 809138176 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003812236710130391, + "loss": 2.4659, + "theoretical_loss": 3.7249366297529742, + "tokens_seen": 809203712 + }, + { + "epoch": 2.07, + "learning_rate": 0.00038121364092276835, + "loss": 2.7821, + "theoretical_loss": 3.724906787879397, + "tokens_seen": 809269248 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003812036108324975, + "loss": 2.5499, + "theoretical_loss": 3.7248769490989666, + "tokens_seen": 809334784 + }, + { + "epoch": 2.07, + "objective/train/docs_used": 923007, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.85902738571167, + "objective/train/theoretical_loss": 3.7248620308685023, + "objective/train/tokens_used": 829827552, + "theoretical_loss": 3.7248620308685023, + "tokens_seen": 809367552 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003811935807422267, + "loss": 2.8503, + "theoretical_loss": 3.7248471134111103, + "tokens_seen": 809400320 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003811835506519559, + "loss": 2.7954, + "theoretical_loss": 3.724817280815259, + "tokens_seen": 809465856 + }, + { + "epoch": 2.07, + "learning_rate": 0.00038117352056168507, + "loss": 2.7549, + "theoretical_loss": 3.7247874513108403, + "tokens_seen": 809531392 + }, + { + "epoch": 2.07, + "learning_rate": 0.00038116349047141425, + "loss": 2.5741, + "theoretical_loss": 3.7247576248972853, + "tokens_seen": 809596928 + }, + { + "epoch": 2.07, + "learning_rate": 0.00038115346038114343, + "loss": 2.7751, + "theoretical_loss": 3.724727801574023, + "tokens_seen": 809662464 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003811434302908726, + "loss": 3.0212, + "theoretical_loss": 3.724697981340482, + "tokens_seen": 809728000 + }, + { + "epoch": 2.07, + "learning_rate": 0.00038113340020060185, + "loss": 2.8603, + "theoretical_loss": 3.7246681641960935, + "tokens_seen": 809793536 + }, + { + "epoch": 2.07, + "learning_rate": 0.000381123370110331, + "loss": 2.8533, + "theoretical_loss": 3.724638350140287, + "tokens_seen": 809859072 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003811133400200602, + "loss": 2.8063, + "theoretical_loss": 3.7246085391724932, + "tokens_seen": 809924608 + }, + { + "epoch": 2.07, + "learning_rate": 0.00038110330992978934, + "loss": 2.7438, + "theoretical_loss": 3.7245787312921417, + "tokens_seen": 809990144 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003810932798395186, + "loss": 2.7521, + "theoretical_loss": 3.724548926498663, + "tokens_seen": 810055680 + }, + { + "epoch": 2.07, + "learning_rate": 0.00038108324974924776, + "loss": 2.7021, + "theoretical_loss": 3.7245191247914886, + "tokens_seen": 810121216 + }, + { + "epoch": 2.07, + "learning_rate": 0.00038107321965897694, + "loss": 2.6705, + "theoretical_loss": 3.7244893261700485, + "tokens_seen": 810186752 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003810631895687061, + "loss": 2.8608, + "theoretical_loss": 3.7244595306337733, + "tokens_seen": 810252288 + }, + { + "epoch": 2.07, + "learning_rate": 0.00038105315947843535, + "loss": 2.8353, + "theoretical_loss": 3.7244297381820948, + "tokens_seen": 810317824 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003810431293881645, + "loss": 2.9204, + "theoretical_loss": 3.7243999488144444, + "tokens_seen": 810383360 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003810330992978937, + "loss": 3.0504, + "theoretical_loss": 3.7243701625302528, + "tokens_seen": 810448896 + }, + { + "epoch": 2.07, + "learning_rate": 0.00038102306920762284, + "loss": 2.8244, + "theoretical_loss": 3.724340379328952, + "tokens_seen": 810514432 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003810130391173521, + "loss": 2.8698, + "theoretical_loss": 3.724310599209973, + "tokens_seen": 810579968 + }, + { + "epoch": 2.07, + "learning_rate": 0.00038100300902708126, + "loss": 2.9134, + "theoretical_loss": 3.724280822172749, + "tokens_seen": 810645504 + }, + { + "epoch": 2.07, + "learning_rate": 0.00038099297893681044, + "loss": 2.5862, + "theoretical_loss": 3.724251048216711, + "tokens_seen": 810711040 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003809829488465396, + "loss": 2.6886, + "theoretical_loss": 3.7242212773412913, + "tokens_seen": 810776576 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003809729187562688, + "loss": 2.815, + "theoretical_loss": 3.7241915095459213, + "tokens_seen": 810842112 + }, + { + "epoch": 2.07, + "learning_rate": 0.000380962888665998, + "loss": 2.7922, + "theoretical_loss": 3.724161744830035, + "tokens_seen": 810907648 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003809528585757272, + "loss": 2.9728, + "theoretical_loss": 3.7241319831930646, + "tokens_seen": 810973184 + }, + { + "epoch": 2.07, + "objective/train/docs_used": 924257, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.479335069656372, + "objective/train/theoretical_loss": 3.724117103528995, + "objective/train/tokens_used": 831465952, + "theoretical_loss": 3.724117103528995, + "tokens_seen": 811005952 + }, + { + "epoch": 2.07, + "learning_rate": 0.00038094282848545635, + "loss": 2.5841, + "theoretical_loss": 3.724102224634442, + "tokens_seen": 811038720 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003809327983951856, + "loss": 2.7625, + "theoretical_loss": 3.7240724691536005, + "tokens_seen": 811104256 + }, + { + "epoch": 2.07, + "learning_rate": 0.00038092276830491476, + "loss": 2.6284, + "theoretical_loss": 3.7240427167499734, + "tokens_seen": 811169792 + }, + { + "epoch": 2.07, + "learning_rate": 0.00038091273821464394, + "loss": 2.8429, + "theoretical_loss": 3.724012967422994, + "tokens_seen": 811235328 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003809027081243731, + "loss": 2.6949, + "theoretical_loss": 3.7239832211720953, + "tokens_seen": 811300864 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003808926780341023, + "loss": 2.8882, + "theoretical_loss": 3.7239534779967105, + "tokens_seen": 811366400 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003808826479438315, + "loss": 2.8984, + "theoretical_loss": 3.723923737896274, + "tokens_seen": 811431936 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003808726178535607, + "loss": 2.851, + "theoretical_loss": 3.723894000870219, + "tokens_seen": 811497472 + }, + { + "epoch": 2.07, + "learning_rate": 0.00038086258776328985, + "loss": 2.8951, + "theoretical_loss": 3.72386426691798, + "tokens_seen": 811563008 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003808525576730191, + "loss": 2.6595, + "theoretical_loss": 3.72383453603899, + "tokens_seen": 811628544 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003808425275827482, + "loss": 2.7869, + "theoretical_loss": 3.7238048082326847, + "tokens_seen": 811694080 + }, + { + "epoch": 2.07, + "learning_rate": 0.00038083249749247745, + "loss": 2.7446, + "theoretical_loss": 3.7237750834984977, + "tokens_seen": 811759616 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003808224674022066, + "loss": 2.9705, + "theoretical_loss": 3.7237453618358636, + "tokens_seen": 811825152 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003808124373119358, + "loss": 2.7296, + "theoretical_loss": 3.723715643244217, + "tokens_seen": 811890688 + }, + { + "epoch": 2.07, + "learning_rate": 0.000380802407221665, + "loss": 2.7897, + "theoretical_loss": 3.7236859277229923, + "tokens_seen": 811956224 + }, + { + "epoch": 2.07, + "learning_rate": 0.00038079237713139417, + "loss": 2.8115, + "theoretical_loss": 3.7236562152716255, + "tokens_seen": 812021760 + }, + { + "epoch": 2.07, + "learning_rate": 0.00038078234704112335, + "loss": 2.9943, + "theoretical_loss": 3.723626505889551, + "tokens_seen": 812087296 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003807723169508526, + "loss": 2.7147, + "theoretical_loss": 3.723596799576205, + "tokens_seen": 812152832 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003807622868605817, + "loss": 2.8725, + "theoretical_loss": 3.7235670963310215, + "tokens_seen": 812218368 + }, + { + "epoch": 2.07, + "learning_rate": 0.00038075225677031095, + "loss": 2.9766, + "theoretical_loss": 3.7235373961534375, + "tokens_seen": 812283904 + }, + { + "epoch": 2.07, + "learning_rate": 0.00038074222668004013, + "loss": 2.7214, + "theoretical_loss": 3.7235076990428877, + "tokens_seen": 812349440 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003807321965897693, + "loss": 2.7914, + "theoretical_loss": 3.7234780049988085, + "tokens_seen": 812414976 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003807221664994985, + "loss": 2.7851, + "theoretical_loss": 3.7234483140206356, + "tokens_seen": 812480512 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003807121364092277, + "loss": 2.92, + "theoretical_loss": 3.7234186261078057, + "tokens_seen": 812546048 + }, + { + "epoch": 2.07, + "learning_rate": 0.00038070210631895685, + "loss": 2.7906, + "theoretical_loss": 3.7233889412597545, + "tokens_seen": 812611584 + }, + { + "epoch": 2.07, + "objective/train/docs_used": 924620, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.1071670055389404, + "objective/train/theoretical_loss": 3.723374099984845, + "objective/train/tokens_used": 833104352, + "theoretical_loss": 3.723374099984845, + "tokens_seen": 812644352 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003806920762286861, + "loss": 2.8908, + "theoretical_loss": 3.7233592594759193, + "tokens_seen": 812677120 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003806820461384152, + "loss": 2.8797, + "theoretical_loss": 3.7233295807557356, + "tokens_seen": 812742656 + }, + { + "epoch": 2.07, + "learning_rate": 0.00038067201604814445, + "loss": 2.65, + "theoretical_loss": 3.7232999050986413, + "tokens_seen": 812808192 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003806619859578736, + "loss": 2.9167, + "theoretical_loss": 3.7232702325040727, + "tokens_seen": 812873728 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003806519558676028, + "loss": 2.9298, + "theoretical_loss": 3.7232405629714673, + "tokens_seen": 812939264 + }, + { + "epoch": 2.07, + "learning_rate": 0.000380641925777332, + "loss": 2.8462, + "theoretical_loss": 3.7232108965002615, + "tokens_seen": 813004800 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003806318956870612, + "loss": 2.7202, + "theoretical_loss": 3.7231812330898935, + "tokens_seen": 813070336 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003806218655967904, + "loss": 2.8769, + "theoretical_loss": 3.7231515727398006, + "tokens_seen": 813135872 + }, + { + "epoch": 2.07, + "learning_rate": 0.00038061183550651954, + "loss": 2.5064, + "theoretical_loss": 3.72312191544942, + "tokens_seen": 813201408 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003806018054162488, + "loss": 2.8703, + "theoretical_loss": 3.7230922612181905, + "tokens_seen": 813266944 + }, + { + "epoch": 2.07, + "learning_rate": 0.00038059177532597796, + "loss": 2.68, + "theoretical_loss": 3.7230626100455497, + "tokens_seen": 813332480 + }, + { + "epoch": 2.07, + "learning_rate": 0.00038058174523570714, + "loss": 2.6174, + "theoretical_loss": 3.723032961930935, + "tokens_seen": 813398016 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003805717151454363, + "loss": 2.8521, + "theoretical_loss": 3.7230033168737853, + "tokens_seen": 813463552 + }, + { + "epoch": 2.07, + "learning_rate": 0.00038056168505516555, + "loss": 2.9735, + "theoretical_loss": 3.722973674873539, + "tokens_seen": 813529088 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003805516549648947, + "loss": 2.9433, + "theoretical_loss": 3.7229440359296344, + "tokens_seen": 813594624 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003805416248746239, + "loss": 2.9525, + "theoretical_loss": 3.7229144000415104, + "tokens_seen": 813660160 + }, + { + "epoch": 2.07, + "learning_rate": 0.00038053159478435304, + "loss": 3.0126, + "theoretical_loss": 3.722884767208606, + "tokens_seen": 813725696 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003805215646940823, + "loss": 2.7167, + "theoretical_loss": 3.7228551374303596, + "tokens_seen": 813791232 + }, + { + "epoch": 2.07, + "learning_rate": 0.00038051153460381146, + "loss": 2.8659, + "theoretical_loss": 3.7228255107062114, + "tokens_seen": 813856768 + }, + { + "epoch": 2.07, + "learning_rate": 0.00038050150451354064, + "loss": 2.7003, + "theoretical_loss": 3.7227958870356, + "tokens_seen": 813922304 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003804914744232698, + "loss": 2.6967, + "theoretical_loss": 3.722766266417965, + "tokens_seen": 813987840 + }, + { + "epoch": 2.07, + "learning_rate": 0.000380481444332999, + "loss": 2.6027, + "theoretical_loss": 3.7227366488527456, + "tokens_seen": 814053376 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003804714142427282, + "loss": 2.7122, + "theoretical_loss": 3.722707034339382, + "tokens_seen": 814118912 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003804613841524574, + "loss": 2.9424, + "theoretical_loss": 3.7226774228773145, + "tokens_seen": 814184448 + }, + { + "epoch": 2.07, + "learning_rate": 0.00038045135406218655, + "loss": 2.5289, + "theoretical_loss": 3.722647814465982, + "tokens_seen": 814249984 + }, + { + "epoch": 2.07, + "objective/train/docs_used": 925886, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.111829996109009, + "objective/train/theoretical_loss": 3.7226330114041666, + "objective/train/tokens_used": 834742752, + "theoretical_loss": 3.7226330114041666, + "tokens_seen": 814282752 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003804413239719158, + "loss": 2.7899, + "theoretical_loss": 3.7226182091048257, + "tokens_seen": 814315520 + }, + { + "epoch": 2.07, + "learning_rate": 0.00038043129388164496, + "loss": 2.6081, + "theoretical_loss": 3.7225886067932854, + "tokens_seen": 814381056 + }, + { + "epoch": 2.07, + "learning_rate": 0.00038042126379137414, + "loss": 2.8281, + "theoretical_loss": 3.722559007530802, + "tokens_seen": 814446592 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003804112337011033, + "loss": 2.8378, + "theoretical_loss": 3.722529411316816, + "tokens_seen": 814512128 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003804012036108325, + "loss": 2.812, + "theoretical_loss": 3.7224998181507676, + "tokens_seen": 814577664 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003803911735205617, + "loss": 3.027, + "theoretical_loss": 3.7224702280320985, + "tokens_seen": 814643200 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003803811434302909, + "loss": 2.8126, + "theoretical_loss": 3.722440640960249, + "tokens_seen": 814708736 + }, + { + "epoch": 2.07, + "learning_rate": 0.00038037111334002005, + "loss": 2.8249, + "theoretical_loss": 3.7224110569346616, + "tokens_seen": 814774272 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003803610832497493, + "loss": 2.6414, + "theoretical_loss": 3.7223814759547764, + "tokens_seen": 814839808 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003803510531594784, + "loss": 2.712, + "theoretical_loss": 3.722351898020036, + "tokens_seen": 814905344 + }, + { + "epoch": 2.07, + "learning_rate": 0.00038034102306920765, + "loss": 2.7911, + "theoretical_loss": 3.7223223231298808, + "tokens_seen": 814970880 + }, + { + "epoch": 2.07, + "learning_rate": 0.00038033099297893683, + "loss": 2.5521, + "theoretical_loss": 3.722292751283754, + "tokens_seen": 815036416 + }, + { + "epoch": 2.07, + "learning_rate": 0.000380320962888666, + "loss": 2.783, + "theoretical_loss": 3.722263182481096, + "tokens_seen": 815101952 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003803109327983952, + "loss": 2.7707, + "theoretical_loss": 3.72223361672135, + "tokens_seen": 815167488 + }, + { + "epoch": 2.07, + "learning_rate": 0.00038030090270812437, + "loss": 2.722, + "theoretical_loss": 3.7222040540039583, + "tokens_seen": 815233024 + }, + { + "epoch": 2.07, + "learning_rate": 0.00038029087261785355, + "loss": 2.8075, + "theoretical_loss": 3.722174494328363, + "tokens_seen": 815298560 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003802808425275828, + "loss": 2.805, + "theoretical_loss": 3.7221449376940066, + "tokens_seen": 815364096 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003802708124373119, + "loss": 2.7188, + "theoretical_loss": 3.7221153841003316, + "tokens_seen": 815429632 + }, + { + "epoch": 2.07, + "learning_rate": 0.00038026078234704115, + "loss": 2.6442, + "theoretical_loss": 3.7220858335467817, + "tokens_seen": 815495168 + }, + { + "epoch": 2.07, + "learning_rate": 0.00038025075225677033, + "loss": 2.8672, + "theoretical_loss": 3.722056286032799, + "tokens_seen": 815560704 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003802407221664995, + "loss": 2.8103, + "theoretical_loss": 3.7220267415578268, + "tokens_seen": 815626240 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003802306920762287, + "loss": 2.626, + "theoretical_loss": 3.721997200121309, + "tokens_seen": 815691776 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003802206619859579, + "loss": 2.7419, + "theoretical_loss": 3.7219676617226876, + "tokens_seen": 815757312 + }, + { + "epoch": 2.07, + "learning_rate": 0.00038021063189568705, + "loss": 2.554, + "theoretical_loss": 3.721938126361408, + "tokens_seen": 815822848 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003802006018054163, + "loss": 2.4705, + "theoretical_loss": 3.721908594036913, + "tokens_seen": 815888384 + }, + { + "epoch": 2.07, + "objective/train/docs_used": 927338, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.0634372234344482, + "objective/train/theoretical_loss": 3.7218938290132857, + "objective/train/tokens_used": 836381152, + "theoretical_loss": 3.7218938290132857, + "tokens_seen": 815921152 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003801905717151454, + "loss": 2.8934, + "theoretical_loss": 3.721879064748646, + "tokens_seen": 815953920 + }, + { + "epoch": 2.07, + "learning_rate": 0.00038018054162487465, + "loss": 2.9032, + "theoretical_loss": 3.721849538496052, + "tokens_seen": 816019456 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003801705115346038, + "loss": 2.8216, + "theoretical_loss": 3.721820015278575, + "tokens_seen": 816084992 + }, + { + "epoch": 2.07, + "learning_rate": 0.000380160481444333, + "loss": 2.7976, + "theoretical_loss": 3.721790495095658, + "tokens_seen": 816150528 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003801504513540622, + "loss": 2.4449, + "theoretical_loss": 3.7217609779467473, + "tokens_seen": 816216064 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003801404212637914, + "loss": 2.8176, + "theoretical_loss": 3.7217314638312864, + "tokens_seen": 816281600 + }, + { + "epoch": 2.07, + "learning_rate": 0.00038013039117352056, + "loss": 2.4909, + "theoretical_loss": 3.7217019527487203, + "tokens_seen": 816347136 + }, + { + "epoch": 2.07, + "learning_rate": 0.00038012036108324974, + "loss": 2.7071, + "theoretical_loss": 3.7216724446984943, + "tokens_seen": 816412672 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003801103309929789, + "loss": 2.6279, + "theoretical_loss": 3.7216429396800526, + "tokens_seen": 816478208 + }, + { + "epoch": 2.07, + "learning_rate": 0.00038010030090270816, + "loss": 2.7475, + "theoretical_loss": 3.721613437692841, + "tokens_seen": 816543744 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003800902708124373, + "loss": 2.8536, + "theoretical_loss": 3.7215839387363046, + "tokens_seen": 816609280 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003800802407221665, + "loss": 2.5333, + "theoretical_loss": 3.7215544428098895, + "tokens_seen": 816674816 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003800702106318957, + "loss": 2.5012, + "theoretical_loss": 3.7215249499130403, + "tokens_seen": 816740352 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003800601805416249, + "loss": 2.8211, + "theoretical_loss": 3.721495460045203, + "tokens_seen": 816805888 + }, + { + "epoch": 2.07, + "learning_rate": 0.00038005015045135406, + "loss": 2.6342, + "theoretical_loss": 3.7214659732058246, + "tokens_seen": 816871424 + }, + { + "epoch": 2.07, + "learning_rate": 0.00038004012036108324, + "loss": 2.7294, + "theoretical_loss": 3.7214364893943497, + "tokens_seen": 816936960 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003800300902708124, + "loss": 2.7032, + "theoretical_loss": 3.721407008610225, + "tokens_seen": 817002496 + }, + { + "epoch": 2.07, + "learning_rate": 0.00038002006018054166, + "loss": 2.947, + "theoretical_loss": 3.7213775308528976, + "tokens_seen": 817068032 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003800100300902708, + "loss": 2.8428, + "theoretical_loss": 3.721348056121813, + "tokens_seen": 817133568 + }, + { + "epoch": 2.07, + "learning_rate": 0.00038, + "loss": 2.6575, + "theoretical_loss": 3.7213185844164185, + "tokens_seen": 817199104 + }, + { + "epoch": 2.07, + "learning_rate": 0.00037998996990972915, + "loss": 2.7767, + "theoretical_loss": 3.7212891157361607, + "tokens_seen": 817264640 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003799799398194584, + "loss": 2.8218, + "theoretical_loss": 3.721259650080486, + "tokens_seen": 817330176 + }, + { + "epoch": 2.07, + "learning_rate": 0.00037996990972918756, + "loss": 2.652, + "theoretical_loss": 3.7212301874488425, + "tokens_seen": 817395712 + }, + { + "epoch": 2.07, + "learning_rate": 0.00037995987963891675, + "loss": 2.7455, + "theoretical_loss": 3.7212007278406767, + "tokens_seen": 817461248 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003799498495486459, + "loss": 2.5604, + "theoretical_loss": 3.721171271255436, + "tokens_seen": 817526784 + }, + { + "epoch": 2.07, + "objective/train/docs_used": 928048, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.899970769882202, + "objective/train/theoretical_loss": 3.7211565440962397, + "objective/train/tokens_used": 838019552, + "theoretical_loss": 3.7211565440962397, + "tokens_seen": 817559552 + }, + { + "epoch": 2.07, + "learning_rate": 0.00037993981945837516, + "loss": 2.7355, + "theoretical_loss": 3.721141817692568, + "tokens_seen": 817592320 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003799297893681043, + "loss": 2.7331, + "theoretical_loss": 3.7211123671515214, + "tokens_seen": 817657856 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003799197592778335, + "loss": 2.8934, + "theoretical_loss": 3.7210829196317423, + "tokens_seen": 817723392 + }, + { + "epoch": 2.07, + "learning_rate": 0.00037990972918756265, + "loss": 2.9739, + "theoretical_loss": 3.721053475132679, + "tokens_seen": 817788928 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003798996990972919, + "loss": 2.5838, + "theoretical_loss": 3.7210240336537805, + "tokens_seen": 817854464 + }, + { + "epoch": 2.07, + "learning_rate": 0.00037988966900702107, + "loss": 2.8182, + "theoretical_loss": 3.7209945951944943, + "tokens_seen": 817920000 + }, + { + "epoch": 2.07, + "learning_rate": 0.00037987963891675025, + "loss": 2.7168, + "theoretical_loss": 3.7209651597542694, + "tokens_seen": 817985536 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003798696088264795, + "loss": 2.8628, + "theoretical_loss": 3.7209357273325536, + "tokens_seen": 818051072 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003798595787362086, + "loss": 2.9665, + "theoretical_loss": 3.7209062979287966, + "tokens_seen": 818116608 + }, + { + "epoch": 2.07, + "learning_rate": 0.00037984954864593785, + "loss": 2.6236, + "theoretical_loss": 3.720876871542446, + "tokens_seen": 818182144 + }, + { + "epoch": 2.07, + "learning_rate": 0.00037983951855566703, + "loss": 2.7174, + "theoretical_loss": 3.720847448172951, + "tokens_seen": 818247680 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003798294884653962, + "loss": 2.6124, + "theoretical_loss": 3.720818027819762, + "tokens_seen": 818313216 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003798194583751254, + "loss": 3.0636, + "theoretical_loss": 3.7207886104823267, + "tokens_seen": 818378752 + }, + { + "epoch": 2.07, + "learning_rate": 0.00037980942828485457, + "loss": 2.8202, + "theoretical_loss": 3.7207591961600954, + "tokens_seen": 818444288 + }, + { + "epoch": 2.07, + "learning_rate": 0.00037979939819458375, + "loss": 2.7901, + "theoretical_loss": 3.720729784852517, + "tokens_seen": 818509824 + }, + { + "epoch": 2.07, + "learning_rate": 0.000379789368104313, + "loss": 2.6593, + "theoretical_loss": 3.720700376559042, + "tokens_seen": 818575360 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003797793380140421, + "loss": 2.7329, + "theoretical_loss": 3.7206709712791195, + "tokens_seen": 818640896 + }, + { + "epoch": 2.07, + "learning_rate": 0.00037976930792377135, + "loss": 2.5465, + "theoretical_loss": 3.7206415690121997, + "tokens_seen": 818706432 + }, + { + "epoch": 2.07, + "learning_rate": 0.00037975927783350053, + "loss": 2.8261, + "theoretical_loss": 3.7206121697577332, + "tokens_seen": 818771968 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003797492477432297, + "loss": 2.6284, + "theoretical_loss": 3.7205827735151695, + "tokens_seen": 818837504 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003797392176529589, + "loss": 2.6674, + "theoretical_loss": 3.7205533802839597, + "tokens_seen": 818903040 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003797291875626881, + "loss": 2.696, + "theoretical_loss": 3.720523990063554, + "tokens_seen": 818968576 + }, + { + "epoch": 2.07, + "learning_rate": 0.00037971915747241725, + "loss": 2.8248, + "theoretical_loss": 3.720494602853403, + "tokens_seen": 819034112 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003797091273821465, + "loss": 2.8485, + "theoretical_loss": 3.7204652186529574, + "tokens_seen": 819099648 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003796990972918756, + "loss": 2.8094, + "theoretical_loss": 3.720435837461669, + "tokens_seen": 819165184 + }, + { + "debugging/Self-BLEU-5": 0.5761146013011956, + "debugging/distinct-1-grams": 0.7462239175050402, + "debugging/distinct-2-grams": 0.9529290102306198, + "debugging/entropy-1-grams": 6.250000994807308, + "debugging/entropy-2-grams": 7.4283795811873965, + "debugging/length": 523.9565217391304, + "debugging/num_segments": 23, + "debugging/score": 0.003311065605696948, + "debugging/score_std": 0.0050287850561047285, + "epoch": 2.07, + "objective/train/docs_used": 929355, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8574650287628174, + "objective/train/theoretical_loss": 3.720421147994287, + "objective/train/tokens_used": 839657952, + "theoretical_loss": 3.720421147994287, + "tokens_seen": 819197952 + }, + { + "epoch": 2.07, + "learning_rate": 0.00037968906720160485, + "loss": 2.6181, + "theoretical_loss": 3.720406459278988, + "tokens_seen": 819230720 + }, + { + "epoch": 2.07, + "learning_rate": 0.000379679037111334, + "loss": 2.6622, + "theoretical_loss": 3.720377084104366, + "tokens_seen": 819296256 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003796690070210632, + "loss": 2.7908, + "theoretical_loss": 3.720347711937255, + "tokens_seen": 819361792 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003796589769307924, + "loss": 2.7659, + "theoretical_loss": 3.720318342777106, + "tokens_seen": 819427328 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003796489468405216, + "loss": 2.5301, + "theoretical_loss": 3.7202889766233707, + "tokens_seen": 819492864 + }, + { + "epoch": 2.07, + "learning_rate": 0.00037963891675025076, + "loss": 2.7743, + "theoretical_loss": 3.720259613475501, + "tokens_seen": 819558400 + }, + { + "epoch": 2.07, + "learning_rate": 0.00037962888665997994, + "loss": 2.6859, + "theoretical_loss": 3.720230253332949, + "tokens_seen": 819623936 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003796188565697091, + "loss": 2.9187, + "theoretical_loss": 3.7202008961951667, + "tokens_seen": 819689472 + }, + { + "epoch": 2.07, + "learning_rate": 0.00037960882647943836, + "loss": 2.6287, + "theoretical_loss": 3.720171542061607, + "tokens_seen": 819755008 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003795987963891675, + "loss": 2.8665, + "theoretical_loss": 3.720142190931721, + "tokens_seen": 819820544 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003795887662988967, + "loss": 2.4396, + "theoretical_loss": 3.720112842804962, + "tokens_seen": 819886080 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003795787362086259, + "loss": 2.5603, + "theoretical_loss": 3.7200834976807835, + "tokens_seen": 819951616 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003795687061183551, + "loss": 2.7133, + "theoretical_loss": 3.7200541555586373, + "tokens_seen": 820017152 + }, + { + "epoch": 2.07, + "learning_rate": 0.00037955867602808426, + "loss": 2.7159, + "theoretical_loss": 3.720024816437977, + "tokens_seen": 820082688 + }, + { + "epoch": 2.07, + "learning_rate": 0.00037954864593781344, + "loss": 2.5554, + "theoretical_loss": 3.7199954803182553, + "tokens_seen": 820148224 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003795386158475426, + "loss": 2.7215, + "theoretical_loss": 3.7199661471989254, + "tokens_seen": 820213760 + }, + { + "epoch": 2.07, + "learning_rate": 0.00037952858575727186, + "loss": 2.7678, + "theoretical_loss": 3.7199368170794416, + "tokens_seen": 820279296 + }, + { + "epoch": 2.07, + "learning_rate": 0.000379518555667001, + "loss": 2.7882, + "theoretical_loss": 3.7199074899592564, + "tokens_seen": 820344832 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003795085255767302, + "loss": 2.8856, + "theoretical_loss": 3.7198781658378244, + "tokens_seen": 820410368 + }, + { + "epoch": 2.07, + "learning_rate": 0.00037949849548645935, + "loss": 2.8051, + "theoretical_loss": 3.719848844714599, + "tokens_seen": 820475904 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003794884653961886, + "loss": 2.5591, + "theoretical_loss": 3.719819526589034, + "tokens_seen": 820541440 + }, + { + "epoch": 2.07, + "learning_rate": 0.00037947843530591776, + "loss": 2.6928, + "theoretical_loss": 3.719790211460584, + "tokens_seen": 820606976 + }, + { + "epoch": 2.07, + "learning_rate": 0.00037946840521564695, + "loss": 2.7898, + "theoretical_loss": 3.719760899328703, + "tokens_seen": 820672512 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003794583751253761, + "loss": 2.6418, + "theoretical_loss": 3.719731590192846, + "tokens_seen": 820738048 + }, + { + "epoch": 2.07, + "learning_rate": 0.00037944834503510536, + "loss": 2.6323, + "theoretical_loss": 3.7197022840524667, + "tokens_seen": 820803584 + }, + { + "epoch": 2.07, + "objective/train/docs_used": 929749, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.016310930252075, + "objective/train/theoretical_loss": 3.719687632105411, + "objective/train/tokens_used": 841296352, + "theoretical_loss": 3.719687632105411, + "tokens_seen": 820836352 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003794383149448345, + "loss": 2.5313, + "theoretical_loss": 3.7196729809070206, + "tokens_seen": 820869120 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003794282848545637, + "loss": 2.6753, + "theoretical_loss": 3.7196436807559623, + "tokens_seen": 820934656 + }, + { + "epoch": 2.07, + "learning_rate": 0.00037941825476429285, + "loss": 2.6919, + "theoretical_loss": 3.719614383598746, + "tokens_seen": 821000192 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003794082246740221, + "loss": 3.0039, + "theoretical_loss": 3.719585089434828, + "tokens_seen": 821065728 + }, + { + "epoch": 2.07, + "learning_rate": 0.00037939819458375127, + "loss": 2.8825, + "theoretical_loss": 3.719555798263663, + "tokens_seen": 821131264 + }, + { + "epoch": 2.07, + "learning_rate": 0.00037938816449348045, + "loss": 2.9654, + "theoretical_loss": 3.719526510084707, + "tokens_seen": 821196800 + }, + { + "epoch": 2.07, + "learning_rate": 0.00037937813440320963, + "loss": 2.8574, + "theoretical_loss": 3.719497224897415, + "tokens_seen": 821262336 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003793681043129388, + "loss": 2.7252, + "theoretical_loss": 3.719467942701243, + "tokens_seen": 821327872 + }, + { + "epoch": 2.07, + "learning_rate": 0.000379358074222668, + "loss": 2.876, + "theoretical_loss": 3.7194386634956462, + "tokens_seen": 821393408 + }, + { + "epoch": 2.07, + "learning_rate": 0.00037934804413239723, + "loss": 2.9689, + "theoretical_loss": 3.7194093872800815, + "tokens_seen": 821458944 + }, + { + "epoch": 2.07, + "learning_rate": 0.00037933801404212635, + "loss": 2.6865, + "theoretical_loss": 3.719380114054005, + "tokens_seen": 821524480 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003793279839518556, + "loss": 2.6635, + "theoretical_loss": 3.7193508438168728, + "tokens_seen": 821590016 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003793179538615847, + "loss": 2.9359, + "theoretical_loss": 3.7193215765681407, + "tokens_seen": 821655552 + }, + { + "epoch": 2.07, + "learning_rate": 0.00037930792377131395, + "loss": 2.6683, + "theoretical_loss": 3.7192923123072656, + "tokens_seen": 821721088 + }, + { + "epoch": 2.07, + "learning_rate": 0.00037929789368104313, + "loss": 2.7919, + "theoretical_loss": 3.719263051033705, + "tokens_seen": 821786624 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003792878635907723, + "loss": 2.549, + "theoretical_loss": 3.7192337927469143, + "tokens_seen": 821852160 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003792778335005015, + "loss": 2.543, + "theoretical_loss": 3.7192045374463527, + "tokens_seen": 821917696 + }, + { + "epoch": 2.07, + "learning_rate": 0.00037926780341023073, + "loss": 2.7604, + "theoretical_loss": 3.7191752851314748, + "tokens_seen": 821983232 + }, + { + "epoch": 2.07, + "learning_rate": 0.00037925777331995986, + "loss": 2.9516, + "theoretical_loss": 3.7191460358017396, + "tokens_seen": 822048768 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003792477432296891, + "loss": 2.7581, + "theoretical_loss": 3.719116789456604, + "tokens_seen": 822114304 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003792377131394182, + "loss": 2.814, + "theoretical_loss": 3.7190875460955257, + "tokens_seen": 822179840 + }, + { + "epoch": 2.07, + "learning_rate": 0.00037922768304914746, + "loss": 2.5764, + "theoretical_loss": 3.719058305717962, + "tokens_seen": 822245376 + }, + { + "epoch": 2.07, + "learning_rate": 0.00037921765295887664, + "loss": 2.7264, + "theoretical_loss": 3.7190290683233713, + "tokens_seen": 822310912 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003792076228686058, + "loss": 2.7514, + "theoretical_loss": 3.7189998339112114, + "tokens_seen": 822376448 + }, + { + "epoch": 2.07, + "learning_rate": 0.000379197592778335, + "loss": 2.6486, + "theoretical_loss": 3.7189706024809404, + "tokens_seen": 822441984 + }, + { + "epoch": 2.07, + "objective/train/docs_used": 930378, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.722386360168457, + "objective/train/theoretical_loss": 3.7189559878838434, + "objective/train/tokens_used": 842934752, + "theoretical_loss": 3.7189559878838434, + "tokens_seen": 822474752 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003791875626880642, + "loss": 2.8354, + "theoretical_loss": 3.718941374032016, + "tokens_seen": 822507520 + }, + { + "epoch": 2.07, + "learning_rate": 0.00037917753259779336, + "loss": 3.0609, + "theoretical_loss": 3.7189121485638976, + "tokens_seen": 822573056 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003791675025075226, + "loss": 2.7041, + "theoretical_loss": 3.718882926076043, + "tokens_seen": 822638592 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003791574724172517, + "loss": 2.865, + "theoretical_loss": 3.7188537065679115, + "tokens_seen": 822704128 + }, + { + "epoch": 2.07, + "learning_rate": 0.00037914744232698096, + "loss": 3.0404, + "theoretical_loss": 3.7188244900389615, + "tokens_seen": 822769664 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003791374122367101, + "loss": 2.7809, + "theoretical_loss": 3.718795276488652, + "tokens_seen": 822835200 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003791273821464393, + "loss": 2.6324, + "theoretical_loss": 3.7187660659164425, + "tokens_seen": 822900736 + }, + { + "epoch": 2.07, + "learning_rate": 0.00037911735205616856, + "loss": 3.104, + "theoretical_loss": 3.718736858321792, + "tokens_seen": 822966272 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003791073219658977, + "loss": 2.9184, + "theoretical_loss": 3.7187076537041595, + "tokens_seen": 823031808 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003790972918756269, + "loss": 2.9075, + "theoretical_loss": 3.7186784520630054, + "tokens_seen": 823097344 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003790872617853561, + "loss": 2.9344, + "theoretical_loss": 3.718649253397789, + "tokens_seen": 823162880 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003790772316950853, + "loss": 2.6255, + "theoretical_loss": 3.7186200577079704, + "tokens_seen": 823228416 + }, + { + "epoch": 2.07, + "learning_rate": 0.00037906720160481446, + "loss": 2.7619, + "theoretical_loss": 3.7185908649930086, + "tokens_seen": 823293952 + }, + { + "epoch": 2.07, + "learning_rate": 0.00037905717151454364, + "loss": 2.7724, + "theoretical_loss": 3.7185616752523645, + "tokens_seen": 823359488 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003790471414242728, + "loss": 2.7481, + "theoretical_loss": 3.7185324884854984, + "tokens_seen": 823425024 + }, + { + "epoch": 2.07, + "learning_rate": 0.00037903711133400206, + "loss": 2.8423, + "theoretical_loss": 3.718503304691871, + "tokens_seen": 823490560 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003790270812437312, + "loss": 2.6441, + "theoretical_loss": 3.7184741238709416, + "tokens_seen": 823556096 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003790170511534604, + "loss": 2.6508, + "theoretical_loss": 3.718444946022172, + "tokens_seen": 823621632 + }, + { + "epoch": 2.07, + "learning_rate": 0.00037900702106318955, + "loss": 2.8063, + "theoretical_loss": 3.7184157711450228, + "tokens_seen": 823687168 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003789969909729188, + "loss": 2.9624, + "theoretical_loss": 3.718386599238954, + "tokens_seen": 823752704 + }, + { + "epoch": 2.07, + "learning_rate": 0.00037898696088264796, + "loss": 2.7201, + "theoretical_loss": 3.7183574303034286, + "tokens_seen": 823818240 + }, + { + "epoch": 2.07, + "learning_rate": 0.00037897693079237715, + "loss": 2.7259, + "theoretical_loss": 3.718328264337906, + "tokens_seen": 823883776 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003789669007021063, + "loss": 2.824, + "theoretical_loss": 3.7182991013418483, + "tokens_seen": 823949312 + }, + { + "epoch": 2.07, + "learning_rate": 0.00037895687061183556, + "loss": 2.9522, + "theoretical_loss": 3.718269941314717, + "tokens_seen": 824014848 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003789468405215647, + "loss": 2.7737, + "theoretical_loss": 3.7182407842559746, + "tokens_seen": 824080384 + }, + { + "epoch": 2.07, + "objective/train/docs_used": 931616, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7472262382507324, + "objective/train/theoretical_loss": 3.71822620683958, + "objective/train/tokens_used": 844573152, + "theoretical_loss": 3.71822620683958, + "tokens_seen": 824113152 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003789368104312939, + "loss": 2.5734, + "theoretical_loss": 3.718211630165081, + "tokens_seen": 824145920 + }, + { + "epoch": 2.07, + "learning_rate": 0.00037892678034102305, + "loss": 2.7369, + "theoretical_loss": 3.7181824790415003, + "tokens_seen": 824211456 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003789167502507523, + "loss": 2.887, + "theoretical_loss": 3.718153330884693, + "tokens_seen": 824276992 + }, + { + "epoch": 2.07, + "learning_rate": 0.00037890672016048147, + "loss": 2.9061, + "theoretical_loss": 3.718124185694122, + "tokens_seen": 824342528 + }, + { + "epoch": 2.07, + "learning_rate": 0.00037889669007021065, + "loss": 3.0517, + "theoretical_loss": 3.7180950434692495, + "tokens_seen": 824408064 + }, + { + "epoch": 2.07, + "learning_rate": 0.00037888665997993983, + "loss": 2.7274, + "theoretical_loss": 3.7180659042095376, + "tokens_seen": 824473600 + }, + { + "epoch": 2.07, + "learning_rate": 0.000378876629889669, + "loss": 2.8292, + "theoretical_loss": 3.7180367679144495, + "tokens_seen": 824539136 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003788665997993982, + "loss": 2.8566, + "theoretical_loss": 3.718007634583448, + "tokens_seen": 824604672 + }, + { + "epoch": 2.07, + "learning_rate": 0.00037885656970912743, + "loss": 2.6371, + "theoretical_loss": 3.7179785042159956, + "tokens_seen": 824670208 + }, + { + "epoch": 2.07, + "learning_rate": 0.00037884653961885655, + "loss": 2.8622, + "theoretical_loss": 3.717949376811556, + "tokens_seen": 824735744 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003788365095285858, + "loss": 2.69, + "theoretical_loss": 3.7179202523695913, + "tokens_seen": 824801280 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003788264794383149, + "loss": 2.7501, + "theoretical_loss": 3.7178911308895666, + "tokens_seen": 824866816 + }, + { + "epoch": 2.07, + "learning_rate": 0.00037881644934804415, + "loss": 2.8124, + "theoretical_loss": 3.7178620123709436, + "tokens_seen": 824932352 + }, + { + "epoch": 2.07, + "learning_rate": 0.00037880641925777333, + "loss": 2.9955, + "theoretical_loss": 3.717832896813187, + "tokens_seen": 824997888 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003787963891675025, + "loss": 2.8538, + "theoretical_loss": 3.71780378421576, + "tokens_seen": 825063424 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003787863590772317, + "loss": 2.6721, + "theoretical_loss": 3.7177746745781266, + "tokens_seen": 825128960 + }, + { + "epoch": 2.07, + "learning_rate": 0.00037877632898696093, + "loss": 3.0316, + "theoretical_loss": 3.717745567899751, + "tokens_seen": 825194496 + }, + { + "epoch": 2.07, + "learning_rate": 0.00037876629889669006, + "loss": 2.7912, + "theoretical_loss": 3.7177164641800973, + "tokens_seen": 825260032 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003787562688064193, + "loss": 2.8639, + "theoretical_loss": 3.71768736341863, + "tokens_seen": 825325568 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003787462387161484, + "loss": 2.6579, + "theoretical_loss": 3.717658265614814, + "tokens_seen": 825391104 + }, + { + "epoch": 2.07, + "learning_rate": 0.00037873620862587766, + "loss": 2.9474, + "theoretical_loss": 3.717629170768112, + "tokens_seen": 825456640 + }, + { + "epoch": 2.07, + "learning_rate": 0.00037872617853560684, + "loss": 2.6777, + "theoretical_loss": 3.7176000788779913, + "tokens_seen": 825522176 + }, + { + "epoch": 2.07, + "learning_rate": 0.000378716148445336, + "loss": 2.766, + "theoretical_loss": 3.7175709899439147, + "tokens_seen": 825587712 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003787061183550652, + "loss": 2.6364, + "theoretical_loss": 3.7175419039653486, + "tokens_seen": 825653248 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003786960882647944, + "loss": 3.0823, + "theoretical_loss": 3.7175128209417574, + "tokens_seen": 825718784 + }, + { + "epoch": 2.07, + "objective/train/docs_used": 932123, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9297609329223633, + "objective/train/theoretical_loss": 3.71749828053791, + "objective/train/tokens_used": 846211552, + "theoretical_loss": 3.71749828053791, + "tokens_seen": 825751552 + }, + { + "epoch": 2.07, + "learning_rate": 0.00037868605817452356, + "loss": 3.0315, + "theoretical_loss": 3.7174837408726065, + "tokens_seen": 825784320 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003786760280842528, + "loss": 2.9329, + "theoretical_loss": 3.717454663757361, + "tokens_seen": 825849856 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003786659979939819, + "loss": 2.8325, + "theoretical_loss": 3.7174255895954875, + "tokens_seen": 825915392 + }, + { + "epoch": 2.07, + "learning_rate": 0.00037865596790371116, + "loss": 2.9202, + "theoretical_loss": 3.717396518386451, + "tokens_seen": 825980928 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003786459378134403, + "loss": 2.709, + "theoretical_loss": 3.717367450129718, + "tokens_seen": 826046464 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003786359077231695, + "loss": 2.7433, + "theoretical_loss": 3.7173383848247528, + "tokens_seen": 826112000 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003786258776328987, + "loss": 2.7836, + "theoretical_loss": 3.7173093224710234, + "tokens_seen": 826177536 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003786158475426279, + "loss": 3.0468, + "theoretical_loss": 3.717280263067995, + "tokens_seen": 826243072 + }, + { + "epoch": 2.07, + "learning_rate": 0.00037860581745235706, + "loss": 3.0001, + "theoretical_loss": 3.7172512066151344, + "tokens_seen": 826308608 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003785957873620863, + "loss": 2.9357, + "theoretical_loss": 3.7172221531119085, + "tokens_seen": 826374144 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003785857572718154, + "loss": 2.5982, + "theoretical_loss": 3.7171931025577836, + "tokens_seen": 826439680 + }, + { + "epoch": 2.07, + "learning_rate": 0.00037857572718154466, + "loss": 2.8929, + "theoretical_loss": 3.7171640549522262, + "tokens_seen": 826505216 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003785656970912738, + "loss": 2.8073, + "theoretical_loss": 3.717135010294703, + "tokens_seen": 826570752 + }, + { + "epoch": 2.07, + "learning_rate": 0.000378555667001003, + "loss": 2.7072, + "theoretical_loss": 3.717105968584683, + "tokens_seen": 826636288 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003785456369107322, + "loss": 2.9043, + "theoretical_loss": 3.717076929821631, + "tokens_seen": 826701824 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003785356068204614, + "loss": 3.0974, + "theoretical_loss": 3.717047894005016, + "tokens_seen": 826767360 + }, + { + "epoch": 2.07, + "learning_rate": 0.00037852557673019057, + "loss": 2.5399, + "theoretical_loss": 3.7170188611343047, + "tokens_seen": 826832896 + }, + { + "epoch": 2.07, + "learning_rate": 0.00037851554663991975, + "loss": 3.0706, + "theoretical_loss": 3.7169898312089655, + "tokens_seen": 826898432 + }, + { + "epoch": 2.07, + "learning_rate": 0.00037850551654964893, + "loss": 3.1045, + "theoretical_loss": 3.7169608042284654, + "tokens_seen": 826963968 + }, + { + "epoch": 2.07, + "learning_rate": 0.00037849548645937816, + "loss": 2.8803, + "theoretical_loss": 3.716931780192273, + "tokens_seen": 827029504 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003784854563691073, + "loss": 2.7085, + "theoretical_loss": 3.7169027590998556, + "tokens_seen": 827095040 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003784754262788365, + "loss": 2.8001, + "theoretical_loss": 3.7168737409506822, + "tokens_seen": 827160576 + }, + { + "epoch": 2.07, + "learning_rate": 0.00037846539618856565, + "loss": 3.1734, + "theoretical_loss": 3.7168447257442208, + "tokens_seen": 827226112 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003784553660982949, + "loss": 2.9276, + "theoretical_loss": 3.7168157134799396, + "tokens_seen": 827291648 + }, + { + "epoch": 2.07, + "learning_rate": 0.00037844533600802407, + "loss": 2.7264, + "theoretical_loss": 3.716786704157308, + "tokens_seen": 827357184 + }, + { + "epoch": 2.07, + "objective/train/docs_used": 933422, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9304966926574707, + "objective/train/theoretical_loss": 3.7167722005989443, + "objective/train/tokens_used": 847849952, + "theoretical_loss": 3.7167722005989443, + "tokens_seen": 827389952 + }, + { + "epoch": 2.07, + "learning_rate": 0.00037843530591775325, + "loss": 2.806, + "theoretical_loss": 3.716757697775794, + "tokens_seen": 827422720 + }, + { + "epoch": 2.07, + "learning_rate": 0.00037842527582748243, + "loss": 2.8655, + "theoretical_loss": 3.7167286943348667, + "tokens_seen": 827488256 + }, + { + "epoch": 2.07, + "learning_rate": 0.00037841524573721167, + "loss": 2.9854, + "theoretical_loss": 3.7166996938339953, + "tokens_seen": 827553792 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003784052156469408, + "loss": 2.8557, + "theoretical_loss": 3.716670696272649, + "tokens_seen": 827619328 + }, + { + "epoch": 2.07, + "learning_rate": 0.00037839518555667003, + "loss": 2.8762, + "theoretical_loss": 3.7166417016502966, + "tokens_seen": 827684864 + }, + { + "epoch": 2.07, + "learning_rate": 0.00037838515546639916, + "loss": 2.9352, + "theoretical_loss": 3.7166127099664084, + "tokens_seen": 827750400 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003783751253761284, + "loss": 2.6815, + "theoretical_loss": 3.716583721220453, + "tokens_seen": 827815936 + }, + { + "epoch": 2.07, + "learning_rate": 0.00037836509528585763, + "loss": 2.6254, + "theoretical_loss": 3.7165547354119015, + "tokens_seen": 827881472 + }, + { + "epoch": 2.07, + "learning_rate": 0.00037835506519558675, + "loss": 2.8917, + "theoretical_loss": 3.716525752540222, + "tokens_seen": 827947008 + }, + { + "epoch": 2.07, + "learning_rate": 0.000378345035105316, + "loss": 3.027, + "theoretical_loss": 3.716496772604886, + "tokens_seen": 828012544 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003783350050150451, + "loss": 2.8621, + "theoretical_loss": 3.7164677956053627, + "tokens_seen": 828078080 + }, + { + "epoch": 2.07, + "learning_rate": 0.00037832497492477435, + "loss": 3.0463, + "theoretical_loss": 3.7164388215411233, + "tokens_seen": 828143616 + }, + { + "epoch": 2.07, + "learning_rate": 0.00037831494483450353, + "loss": 2.7848, + "theoretical_loss": 3.7164098504116367, + "tokens_seen": 828209152 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003783049147442327, + "loss": 2.8613, + "theoretical_loss": 3.7163808822163755, + "tokens_seen": 828274688 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003782948846539619, + "loss": 2.9271, + "theoretical_loss": 3.7163519169548085, + "tokens_seen": 828340224 + }, + { + "epoch": 2.07, + "learning_rate": 0.00037828485456369113, + "loss": 2.6843, + "theoretical_loss": 3.7163229546264076, + "tokens_seen": 828405760 + }, + { + "epoch": 2.07, + "learning_rate": 0.00037827482447342026, + "loss": 2.8605, + "theoretical_loss": 3.7162939952306435, + "tokens_seen": 828471296 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003782647943831495, + "loss": 2.9175, + "theoretical_loss": 3.7162650387669873, + "tokens_seen": 828536832 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003782547642928786, + "loss": 2.7707, + "theoretical_loss": 3.71623608523491, + "tokens_seen": 828602368 + }, + { + "epoch": 2.07, + "learning_rate": 0.00037824473420260786, + "loss": 2.676, + "theoretical_loss": 3.7162071346338834, + "tokens_seen": 828667904 + }, + { + "epoch": 2.07, + "learning_rate": 0.00037823470411233704, + "loss": 2.845, + "theoretical_loss": 3.7161781869633788, + "tokens_seen": 828733440 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003782246740220662, + "loss": 2.9766, + "theoretical_loss": 3.7161492422228677, + "tokens_seen": 828798976 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003782146439317954, + "loss": 2.9741, + "theoretical_loss": 3.7161203004118217, + "tokens_seen": 828864512 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003782046138415246, + "loss": 2.9232, + "theoretical_loss": 3.716091361529714, + "tokens_seen": 828930048 + }, + { + "epoch": 2.07, + "learning_rate": 0.00037819458375125376, + "loss": 2.7427, + "theoretical_loss": 3.716062425576015, + "tokens_seen": 828995584 + }, + { + "epoch": 2.07, + "objective/train/docs_used": 934816, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.1100308895111084, + "objective/train/theoretical_loss": 3.716047958697154, + "objective/train/tokens_used": 849488352, + "theoretical_loss": 3.716047958697154, + "tokens_seen": 829028352 + }, + { + "epoch": 2.07, + "learning_rate": 0.000378184553660983, + "loss": 3.0321, + "theoretical_loss": 3.7160334925501974, + "tokens_seen": 829061120 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003781745235707121, + "loss": 2.8202, + "theoretical_loss": 3.7160045624517344, + "tokens_seen": 829126656 + }, + { + "epoch": 2.07, + "learning_rate": 0.00037816449348044136, + "loss": 2.9412, + "theoretical_loss": 3.7159756352800977, + "tokens_seen": 829192192 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003781544633901705, + "loss": 2.7031, + "theoretical_loss": 3.7159467110347597, + "tokens_seen": 829257728 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003781444332998997, + "loss": 3.0871, + "theoretical_loss": 3.715917789715194, + "tokens_seen": 829323264 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003781344032096289, + "loss": 2.8528, + "theoretical_loss": 3.715888871320872, + "tokens_seen": 829388800 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003781243731193581, + "loss": 2.8641, + "theoretical_loss": 3.7158599558512684, + "tokens_seen": 829454336 + }, + { + "epoch": 2.07, + "learning_rate": 0.00037811434302908726, + "loss": 3.02, + "theoretical_loss": 3.7158310433058555, + "tokens_seen": 829519872 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003781043129388165, + "loss": 2.925, + "theoretical_loss": 3.7158021336841065, + "tokens_seen": 829585408 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003780942828485456, + "loss": 2.9142, + "theoretical_loss": 3.715773226985495, + "tokens_seen": 829650944 + }, + { + "epoch": 2.07, + "learning_rate": 0.00037808425275827486, + "loss": 2.7949, + "theoretical_loss": 3.7157443232094947, + "tokens_seen": 829716480 + }, + { + "epoch": 2.07, + "learning_rate": 0.000378074222668004, + "loss": 2.8045, + "theoretical_loss": 3.7157154223555793, + "tokens_seen": 829782016 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003780641925777332, + "loss": 3.2325, + "theoretical_loss": 3.7156865244232224, + "tokens_seen": 829847552 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003780541624874624, + "loss": 2.7547, + "theoretical_loss": 3.7156576294118984, + "tokens_seen": 829913088 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003780441323971916, + "loss": 2.9053, + "theoretical_loss": 3.715628737321081, + "tokens_seen": 829978624 + }, + { + "epoch": 2.07, + "learning_rate": 0.00037803410230692077, + "loss": 2.928, + "theoretical_loss": 3.7155998481502444, + "tokens_seen": 830044160 + }, + { + "epoch": 2.07, + "learning_rate": 0.00037802407221664995, + "loss": 2.8699, + "theoretical_loss": 3.715570961898863, + "tokens_seen": 830109696 + }, + { + "epoch": 2.07, + "learning_rate": 0.00037801404212637913, + "loss": 2.8143, + "theoretical_loss": 3.7155420785664113, + "tokens_seen": 830175232 + }, + { + "epoch": 2.07, + "learning_rate": 0.00037800401203610836, + "loss": 2.7866, + "theoretical_loss": 3.7155131981523644, + "tokens_seen": 830240768 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003779939819458375, + "loss": 2.6816, + "theoretical_loss": 3.7154843206561967, + "tokens_seen": 830306304 + }, + { + "epoch": 2.07, + "learning_rate": 0.00037798395185556673, + "loss": 2.9595, + "theoretical_loss": 3.7154554460773834, + "tokens_seen": 830371840 + }, + { + "epoch": 2.07, + "learning_rate": 0.00037797392176529585, + "loss": 2.9089, + "theoretical_loss": 3.7154265744153996, + "tokens_seen": 830437376 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003779638916750251, + "loss": 2.8497, + "theoretical_loss": 3.71539770566972, + "tokens_seen": 830502912 + }, + { + "epoch": 2.07, + "learning_rate": 0.00037795386158475427, + "loss": 2.9463, + "theoretical_loss": 3.7153688398398197, + "tokens_seen": 830568448 + }, + { + "epoch": 2.07, + "learning_rate": 0.00037794383149448345, + "loss": 2.688, + "theoretical_loss": 3.715339976925175, + "tokens_seen": 830633984 + }, + { + "epoch": 2.07, + "objective/train/docs_used": 935526, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.734133005142212, + "objective/train/theoretical_loss": 3.71532554656091, + "objective/train/tokens_used": 851126752, + "theoretical_loss": 3.71532554656091, + "tokens_seen": 830666752 + }, + { + "epoch": 2.07, + "learning_rate": 0.00037793380140421263, + "loss": 2.9231, + "theoretical_loss": 3.7153111169252613, + "tokens_seen": 830699520 + }, + { + "epoch": 2.07, + "learning_rate": 0.00037792377131394187, + "loss": 2.7234, + "theoretical_loss": 3.7152822598395545, + "tokens_seen": 830765056 + }, + { + "epoch": 2.07, + "learning_rate": 0.000377913741223671, + "loss": 2.9191, + "theoretical_loss": 3.7152534056675295, + "tokens_seen": 830830592 + }, + { + "epoch": 2.07, + "learning_rate": 0.00037790371113340023, + "loss": 2.7125, + "theoretical_loss": 3.715224554408664, + "tokens_seen": 830896128 + }, + { + "epoch": 2.07, + "learning_rate": 0.00037789368104312936, + "loss": 2.7919, + "theoretical_loss": 3.715195706062432, + "tokens_seen": 830961664 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003778836509528586, + "loss": 2.8418, + "theoretical_loss": 3.7151668606283117, + "tokens_seen": 831027200 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003778736208625878, + "loss": 2.7902, + "theoretical_loss": 3.7151380181057783, + "tokens_seen": 831092736 + }, + { + "epoch": 2.07, + "learning_rate": 0.00037786359077231695, + "loss": 2.7545, + "theoretical_loss": 3.715109178494309, + "tokens_seen": 831158272 + }, + { + "epoch": 2.07, + "learning_rate": 0.00037785356068204614, + "loss": 2.7125, + "theoretical_loss": 3.7150803417933798, + "tokens_seen": 831223808 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003778435305917753, + "loss": 2.9552, + "theoretical_loss": 3.7150515080024684, + "tokens_seen": 831289344 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003778335005015045, + "loss": 2.764, + "theoretical_loss": 3.715022677121052, + "tokens_seen": 831354880 + }, + { + "epoch": 2.07, + "learning_rate": 0.00037782347041123373, + "loss": 3.0012, + "theoretical_loss": 3.7149938491486063, + "tokens_seen": 831420416 + }, + { + "epoch": 2.07, + "learning_rate": 0.00037781344032096286, + "loss": 2.675, + "theoretical_loss": 3.7149650240846093, + "tokens_seen": 831485952 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003778034102306921, + "loss": 2.7746, + "theoretical_loss": 3.7149362019285386, + "tokens_seen": 831551488 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003777933801404213, + "loss": 2.5407, + "theoretical_loss": 3.714907382679871, + "tokens_seen": 831617024 + }, + { + "epoch": 2.07, + "learning_rate": 0.00037778335005015046, + "loss": 2.8533, + "theoretical_loss": 3.7148785663380854, + "tokens_seen": 831682560 + }, + { + "epoch": 2.07, + "learning_rate": 0.00037777331995987964, + "loss": 2.7901, + "theoretical_loss": 3.7148497529026585, + "tokens_seen": 831748096 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003777632898696088, + "loss": 2.8641, + "theoretical_loss": 3.714820942373068, + "tokens_seen": 831813632 + }, + { + "epoch": 2.07, + "learning_rate": 0.000377753259779338, + "loss": 2.8166, + "theoretical_loss": 3.7147921347487927, + "tokens_seen": 831879168 + }, + { + "epoch": 2.07, + "learning_rate": 0.00037774322968906724, + "loss": 2.9293, + "theoretical_loss": 3.71476333002931, + "tokens_seen": 831944704 + }, + { + "epoch": 2.07, + "learning_rate": 0.00037773319959879636, + "loss": 2.9647, + "theoretical_loss": 3.714734528214099, + "tokens_seen": 832010240 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003777231695085256, + "loss": 2.8858, + "theoretical_loss": 3.714705729302638, + "tokens_seen": 832075776 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003777131394182547, + "loss": 2.9028, + "theoretical_loss": 3.714676933294405, + "tokens_seen": 832141312 + }, + { + "epoch": 2.07, + "learning_rate": 0.00037770310932798396, + "loss": 2.7915, + "theoretical_loss": 3.71464814018888, + "tokens_seen": 832206848 + }, + { + "epoch": 2.07, + "learning_rate": 0.00037769307923771314, + "loss": 2.9318, + "theoretical_loss": 3.71461934998554, + "tokens_seen": 832272384 + }, + { + "epoch": 2.07, + "objective/train/docs_used": 936975, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9853835105895996, + "objective/train/theoretical_loss": 3.7146049559720273, + "objective/train/tokens_used": 852765152, + "theoretical_loss": 3.7146049559720273, + "tokens_seen": 832305152 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003776830491474423, + "loss": 2.9336, + "theoretical_loss": 3.7145905626838656, + "tokens_seen": 832337920 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003776730190571715, + "loss": 2.6135, + "theoretical_loss": 3.714561778283335, + "tokens_seen": 832403456 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003776629889669007, + "loss": 2.6917, + "theoretical_loss": 3.7145329967834275, + "tokens_seen": 832468992 + }, + { + "epoch": 2.07, + "learning_rate": 0.00037765295887662987, + "loss": 2.6006, + "theoretical_loss": 3.714504218183623, + "tokens_seen": 832534528 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003776429287863591, + "loss": 2.6902, + "theoretical_loss": 3.7144754424834003, + "tokens_seen": 832600064 + }, + { + "epoch": 2.07, + "learning_rate": 0.00037763289869608823, + "loss": 2.7044, + "theoretical_loss": 3.71444666968224, + "tokens_seen": 832665600 + }, + { + "epoch": 2.07, + "learning_rate": 0.00037762286860581746, + "loss": 2.6894, + "theoretical_loss": 3.714417899779621, + "tokens_seen": 832731136 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003776128385155467, + "loss": 2.7718, + "theoretical_loss": 3.7143891327750236, + "tokens_seen": 832796672 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003776028084252758, + "loss": 2.8134, + "theoretical_loss": 3.7143603686679283, + "tokens_seen": 832862208 + }, + { + "epoch": 2.07, + "learning_rate": 0.00037759277833500506, + "loss": 2.5345, + "theoretical_loss": 3.7143316074578143, + "tokens_seen": 832927744 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003775827482447342, + "loss": 2.7188, + "theoretical_loss": 3.7143028491441625, + "tokens_seen": 832993280 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003775727181544634, + "loss": 2.8647, + "theoretical_loss": 3.7142740937264533, + "tokens_seen": 833058816 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003775626880641926, + "loss": 2.874, + "theoretical_loss": 3.7142453412041676, + "tokens_seen": 833124352 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003775526579739218, + "loss": 2.9832, + "theoretical_loss": 3.7142165915767853, + "tokens_seen": 833189888 + }, + { + "epoch": 2.07, + "learning_rate": 0.00037754262788365097, + "loss": 2.7881, + "theoretical_loss": 3.7141878448437886, + "tokens_seen": 833255424 + }, + { + "epoch": 2.07, + "learning_rate": 0.00037753259779338015, + "loss": 2.7939, + "theoretical_loss": 3.7141591010046575, + "tokens_seen": 833320960 + }, + { + "epoch": 2.07, + "learning_rate": 0.00037752256770310933, + "loss": 2.9099, + "theoretical_loss": 3.714130360058873, + "tokens_seen": 833386496 + }, + { + "epoch": 2.07, + "learning_rate": 0.00037751253761283857, + "loss": 2.9208, + "theoretical_loss": 3.7141016220059164, + "tokens_seen": 833452032 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003775025075225677, + "loss": 2.8116, + "theoretical_loss": 3.71407288684527, + "tokens_seen": 833517568 + }, + { + "epoch": 2.07, + "learning_rate": 0.00037749247743229693, + "loss": 2.8814, + "theoretical_loss": 3.7140441545764142, + "tokens_seen": 833583104 + }, + { + "epoch": 2.07, + "learning_rate": 0.00037748244734202605, + "loss": 2.5721, + "theoretical_loss": 3.7140154251988315, + "tokens_seen": 833648640 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003774724172517553, + "loss": 2.5953, + "theoretical_loss": 3.713986698712003, + "tokens_seen": 833714176 + }, + { + "epoch": 2.07, + "learning_rate": 0.00037746238716148447, + "loss": 2.6858, + "theoretical_loss": 3.7139579751154117, + "tokens_seen": 833779712 + }, + { + "epoch": 2.07, + "learning_rate": 0.00037745235707121365, + "loss": 2.8436, + "theoretical_loss": 3.7139292544085385, + "tokens_seen": 833845248 + }, + { + "epoch": 2.07, + "learning_rate": 0.00037744232698094283, + "loss": 2.9229, + "theoretical_loss": 3.713900536590866, + "tokens_seen": 833910784 + }, + { + "epoch": 2.07, + "objective/train/docs_used": 937728, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8705952167510986, + "objective/train/theoretical_loss": 3.7138861787653186, + "objective/train/tokens_used": 854403552, + "theoretical_loss": 3.7138861787653186, + "tokens_seen": 833943552 + }, + { + "epoch": 2.07, + "learning_rate": 0.00037743229689067207, + "loss": 2.8177, + "theoretical_loss": 3.713871821661877, + "tokens_seen": 833976320 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003774222668004012, + "loss": 2.7473, + "theoretical_loss": 3.7138431096210534, + "tokens_seen": 834041856 + }, + { + "epoch": 2.07, + "learning_rate": 0.00037741223671013043, + "loss": 2.7667, + "theoretical_loss": 3.7138144004678777, + "tokens_seen": 834107392 + }, + { + "epoch": 2.07, + "learning_rate": 0.00037740220661985956, + "loss": 2.8914, + "theoretical_loss": 3.7137856942018335, + "tokens_seen": 834172928 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003773921765295888, + "loss": 2.9923, + "theoretical_loss": 3.7137569908224024, + "tokens_seen": 834238464 + }, + { + "epoch": 2.07, + "learning_rate": 0.000377382146439318, + "loss": 2.697, + "theoretical_loss": 3.713728290329068, + "tokens_seen": 834304000 + }, + { + "epoch": 2.07, + "learning_rate": 0.00037737211634904715, + "loss": 2.9992, + "theoretical_loss": 3.7136995927213143, + "tokens_seen": 834369536 + }, + { + "epoch": 2.07, + "learning_rate": 0.00037736208625877634, + "loss": 2.7865, + "theoretical_loss": 3.7136708979986235, + "tokens_seen": 834435072 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003773520561685055, + "loss": 2.7856, + "theoretical_loss": 3.713642206160479, + "tokens_seen": 834500608 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003773420260782347, + "loss": 2.9712, + "theoretical_loss": 3.7136135172063645, + "tokens_seen": 834566144 + }, + { + "epoch": 2.07, + "learning_rate": 0.00037733199598796393, + "loss": 2.6482, + "theoretical_loss": 3.7135848311357638, + "tokens_seen": 834631680 + }, + { + "epoch": 2.07, + "learning_rate": 0.00037732196589769306, + "loss": 2.7742, + "theoretical_loss": 3.713556147948161, + "tokens_seen": 834697216 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003773119358074223, + "loss": 2.7187, + "theoretical_loss": 3.7135274676430394, + "tokens_seen": 834762752 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003773019057171515, + "loss": 2.7803, + "theoretical_loss": 3.7134987902198833, + "tokens_seen": 834828288 + }, + { + "epoch": 2.07, + "learning_rate": 0.00037729187562688066, + "loss": 2.8031, + "theoretical_loss": 3.7134701156781773, + "tokens_seen": 834893824 + }, + { + "epoch": 2.07, + "learning_rate": 0.00037728184553660984, + "loss": 2.7147, + "theoretical_loss": 3.713441444017405, + "tokens_seen": 834959360 + }, + { + "epoch": 2.07, + "learning_rate": 0.000377271815446339, + "loss": 2.5068, + "theoretical_loss": 3.7134127752370514, + "tokens_seen": 835024896 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003772617853560682, + "loss": 2.7243, + "theoretical_loss": 3.713384109336601, + "tokens_seen": 835090432 + }, + { + "epoch": 2.07, + "learning_rate": 0.00037725175526579744, + "loss": 2.7841, + "theoretical_loss": 3.7133554463155383, + "tokens_seen": 835155968 + }, + { + "epoch": 2.07, + "learning_rate": 0.00037724172517552656, + "loss": 2.8078, + "theoretical_loss": 3.713326786173348, + "tokens_seen": 835221504 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003772316950852558, + "loss": 2.5191, + "theoretical_loss": 3.713298128909516, + "tokens_seen": 835287040 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003772216649949849, + "loss": 2.6828, + "theoretical_loss": 3.7132694745235266, + "tokens_seen": 835352576 + }, + { + "epoch": 2.07, + "learning_rate": 0.00037721163490471416, + "loss": 2.8356, + "theoretical_loss": 3.713240823014866, + "tokens_seen": 835418112 + }, + { + "epoch": 2.07, + "learning_rate": 0.00037720160481444334, + "loss": 2.8376, + "theoretical_loss": 3.7132121743830178, + "tokens_seen": 835483648 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003771915747241725, + "loss": 2.9586, + "theoretical_loss": 3.713183528627469, + "tokens_seen": 835549184 + }, + { + "epoch": 2.07, + "objective/train/docs_used": 939195, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6481375694274902, + "objective/train/theoretical_loss": 3.713169206828146, + "objective/train/tokens_used": 856041952, + "theoretical_loss": 3.713169206828146, + "tokens_seen": 835581952 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003771815446339017, + "loss": 2.8213, + "theoretical_loss": 3.713154885747705, + "tokens_seen": 835614720 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003771715145436309, + "loss": 2.7886, + "theoretical_loss": 3.713126245743211, + "tokens_seen": 835680256 + }, + { + "epoch": 2.07, + "learning_rate": 0.00037716148445336007, + "loss": 2.6125, + "theoretical_loss": 3.7130976086134737, + "tokens_seen": 835745792 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003771514543630893, + "loss": 2.9789, + "theoretical_loss": 3.713068974357979, + "tokens_seen": 835811328 + }, + { + "epoch": 2.07, + "learning_rate": 0.00037714142427281843, + "loss": 2.8882, + "theoretical_loss": 3.7130403429762127, + "tokens_seen": 835876864 + }, + { + "epoch": 2.07, + "learning_rate": 0.00037713139418254766, + "loss": 2.7156, + "theoretical_loss": 3.713011714467661, + "tokens_seen": 835942400 + }, + { + "epoch": 2.07, + "learning_rate": 0.00037712136409227685, + "loss": 2.8133, + "theoretical_loss": 3.7129830888318107, + "tokens_seen": 836007936 + }, + { + "epoch": 2.07, + "learning_rate": 0.000377111334002006, + "loss": 2.8748, + "theoretical_loss": 3.7129544660681493, + "tokens_seen": 836073472 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003771013039117352, + "loss": 2.7447, + "theoretical_loss": 3.7129258461761614, + "tokens_seen": 836139008 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003770912738214644, + "loss": 2.4627, + "theoretical_loss": 3.7128972291553355, + "tokens_seen": 836204544 + }, + { + "epoch": 2.07, + "learning_rate": 0.00037708124373119357, + "loss": 2.8791, + "theoretical_loss": 3.712868615005158, + "tokens_seen": 836270080 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003770712136409228, + "loss": 2.9345, + "theoretical_loss": 3.712840003725116, + "tokens_seen": 836335616 + }, + { + "epoch": 2.07, + "learning_rate": 0.00037706118355065193, + "loss": 2.6005, + "theoretical_loss": 3.7128113953146973, + "tokens_seen": 836401152 + }, + { + "epoch": 2.07, + "learning_rate": 0.00037705115346038117, + "loss": 2.7668, + "theoretical_loss": 3.7127827897733887, + "tokens_seen": 836466688 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003770411233701103, + "loss": 2.6267, + "theoretical_loss": 3.712754187100677, + "tokens_seen": 836532224 + }, + { + "epoch": 2.07, + "learning_rate": 0.00037703109327983953, + "loss": 2.8772, + "theoretical_loss": 3.712725587296051, + "tokens_seen": 836597760 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003770210631895687, + "loss": 2.9275, + "theoretical_loss": 3.7126969903589986, + "tokens_seen": 836663296 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003770110330992979, + "loss": 2.8701, + "theoretical_loss": 3.712668396289007, + "tokens_seen": 836728832 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003770010030090271, + "loss": 2.8012, + "theoretical_loss": 3.712639805085564, + "tokens_seen": 836794368 + }, + { + "epoch": 2.07, + "learning_rate": 0.00037699097291875625, + "loss": 2.7351, + "theoretical_loss": 3.7126112167481584, + "tokens_seen": 836859904 + }, + { + "epoch": 2.07, + "learning_rate": 0.00037698094282848544, + "loss": 2.7632, + "theoretical_loss": 3.712582631276278, + "tokens_seen": 836925440 + }, + { + "epoch": 2.07, + "learning_rate": 0.00037697091273821467, + "loss": 2.9011, + "theoretical_loss": 3.712554048669412, + "tokens_seen": 836990976 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003769608826479438, + "loss": 2.8888, + "theoretical_loss": 3.7125254689270486, + "tokens_seen": 837056512 + }, + { + "epoch": 2.07, + "learning_rate": 0.00037695085255767303, + "loss": 2.9281, + "theoretical_loss": 3.7124968920486756, + "tokens_seen": 837122048 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003769408224674022, + "loss": 2.8598, + "theoretical_loss": 3.7124683180337827, + "tokens_seen": 837187584 + }, + { + "epoch": 2.07, + "objective/train/docs_used": 939764, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9354538917541504, + "objective/train/theoretical_loss": 3.712454032099982, + "objective/train/tokens_used": 857680352, + "theoretical_loss": 3.712454032099982, + "tokens_seen": 837220352 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003769307923771314, + "loss": 2.8442, + "theoretical_loss": 3.7124397468818593, + "tokens_seen": 837253120 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003769207622868606, + "loss": 2.6892, + "theoretical_loss": 3.7124111785923937, + "tokens_seen": 837318656 + }, + { + "epoch": 2.07, + "learning_rate": 0.00037691073219658976, + "loss": 2.6348, + "theoretical_loss": 3.712382613164875, + "tokens_seen": 837384192 + }, + { + "epoch": 2.07, + "learning_rate": 0.00037690070210631894, + "loss": 2.7621, + "theoretical_loss": 3.7123540505987926, + "tokens_seen": 837449728 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003768906720160482, + "loss": 2.8155, + "theoretical_loss": 3.7123254908936367, + "tokens_seen": 837515264 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003768806419257773, + "loss": 2.8675, + "theoretical_loss": 3.7122969340488963, + "tokens_seen": 837580800 + }, + { + "epoch": 2.07, + "learning_rate": 0.00037687061183550654, + "loss": 2.7471, + "theoretical_loss": 3.712268380064061, + "tokens_seen": 837646336 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003768605817452357, + "loss": 2.804, + "theoretical_loss": 3.712239828938621, + "tokens_seen": 837711872 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003768505516549649, + "loss": 2.8195, + "theoretical_loss": 3.712211280672067, + "tokens_seen": 837777408 + }, + { + "epoch": 2.07, + "learning_rate": 0.00037684052156469413, + "loss": 2.9182, + "theoretical_loss": 3.7121827352638874, + "tokens_seen": 837842944 + }, + { + "epoch": 2.07, + "learning_rate": 0.00037683049147442326, + "loss": 2.6297, + "theoretical_loss": 3.712154192713574, + "tokens_seen": 837908480 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003768204613841525, + "loss": 2.5391, + "theoretical_loss": 3.7121256530206157, + "tokens_seen": 837974016 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003768104312938817, + "loss": 2.7109, + "theoretical_loss": 3.7120971161845047, + "tokens_seen": 838039552 + }, + { + "epoch": 2.07, + "learning_rate": 0.00037680040120361086, + "loss": 2.8297, + "theoretical_loss": 3.712068582204731, + "tokens_seen": 838105088 + }, + { + "epoch": 2.07, + "learning_rate": 0.00037679037111334004, + "loss": 3.0213, + "theoretical_loss": 3.712040051080785, + "tokens_seen": 838170624 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003767803410230692, + "loss": 2.8937, + "theoretical_loss": 3.7120115228121575, + "tokens_seen": 838236160 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003767703109327984, + "loss": 2.82, + "theoretical_loss": 3.71198299739834, + "tokens_seen": 838301696 + }, + { + "epoch": 2.07, + "learning_rate": 0.00037676028084252764, + "loss": 2.6802, + "theoretical_loss": 3.711954474838824, + "tokens_seen": 838367232 + }, + { + "epoch": 2.07, + "learning_rate": 0.00037675025075225676, + "loss": 2.8553, + "theoretical_loss": 3.7119259551330996, + "tokens_seen": 838432768 + }, + { + "epoch": 2.07, + "learning_rate": 0.000376740220661986, + "loss": 2.6405, + "theoretical_loss": 3.7118974382806598, + "tokens_seen": 838498304 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003767301905717151, + "loss": 2.9445, + "theoretical_loss": 3.711868924280995, + "tokens_seen": 838563840 + }, + { + "epoch": 2.07, + "learning_rate": 0.00037672016048144436, + "loss": 2.845, + "theoretical_loss": 3.7118404131335976, + "tokens_seen": 838629376 + }, + { + "epoch": 2.07, + "learning_rate": 0.00037671013039117354, + "loss": 2.9059, + "theoretical_loss": 3.7118119048379588, + "tokens_seen": 838694912 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003767001003009027, + "loss": 2.8623, + "theoretical_loss": 3.711783399393571, + "tokens_seen": 838760448 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003766900702106319, + "loss": 3.0017, + "theoretical_loss": 3.711754896799926, + "tokens_seen": 838825984 + }, + { + "epoch": 2.07, + "objective/train/docs_used": 940402, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.893378257751465, + "objective/train/theoretical_loss": 3.7117406465719736, + "objective/train/tokens_used": 859318752, + "theoretical_loss": 3.7117406465719736, + "tokens_seen": 838858752 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003766800401203611, + "loss": 2.7411, + "theoretical_loss": 3.711726397056516, + "tokens_seen": 838891520 + }, + { + "epoch": 2.07, + "learning_rate": 0.00037667001003009027, + "loss": 2.524, + "theoretical_loss": 3.711697900162834, + "tokens_seen": 838957056 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003766599799398195, + "loss": 2.8174, + "theoretical_loss": 3.7116694061183715, + "tokens_seen": 839022592 + }, + { + "epoch": 2.07, + "learning_rate": 0.00037664994984954863, + "loss": 2.7206, + "theoretical_loss": 3.7116409149226213, + "tokens_seen": 839088128 + }, + { + "epoch": 2.07, + "learning_rate": 0.00037663991975927786, + "loss": 2.7177, + "theoretical_loss": 3.711612426575077, + "tokens_seen": 839153664 + }, + { + "epoch": 2.07, + "learning_rate": 0.00037662988966900705, + "loss": 2.7343, + "theoretical_loss": 3.711583941075231, + "tokens_seen": 839219200 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003766198595787362, + "loss": 3.0564, + "theoretical_loss": 3.711555458422576, + "tokens_seen": 839284736 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003766098294884654, + "loss": 2.5675, + "theoretical_loss": 3.711526978616605, + "tokens_seen": 839350272 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003765997993981946, + "loss": 2.6686, + "theoretical_loss": 3.711498501656812, + "tokens_seen": 839415808 + }, + { + "epoch": 2.07, + "learning_rate": 0.00037658976930792377, + "loss": 2.8999, + "theoretical_loss": 3.7114700275426893, + "tokens_seen": 839481344 + }, + { + "epoch": 2.07, + "learning_rate": 0.000376579739217653, + "loss": 2.5858, + "theoretical_loss": 3.7114415562737317, + "tokens_seen": 839546880 + }, + { + "epoch": 2.07, + "learning_rate": 0.00037656970912738213, + "loss": 2.8555, + "theoretical_loss": 3.711413087849432, + "tokens_seen": 839612416 + }, + { + "epoch": 2.07, + "learning_rate": 0.00037655967903711137, + "loss": 2.7198, + "theoretical_loss": 3.711384622269284, + "tokens_seen": 839677952 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003765496489468405, + "loss": 2.9233, + "theoretical_loss": 3.711356159532782, + "tokens_seen": 839743488 + }, + { + "epoch": 2.07, + "learning_rate": 0.00037653961885656973, + "loss": 2.7966, + "theoretical_loss": 3.71132769963942, + "tokens_seen": 839809024 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003765295887662989, + "loss": 2.702, + "theoretical_loss": 3.7112992425886917, + "tokens_seen": 839874560 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003765195586760281, + "loss": 2.7411, + "theoretical_loss": 3.7112707883800917, + "tokens_seen": 839940096 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003765095285857573, + "loss": 2.5153, + "theoretical_loss": 3.7112423370131142, + "tokens_seen": 840005632 + }, + { + "epoch": 2.07, + "learning_rate": 0.00037649949849548645, + "loss": 2.5897, + "theoretical_loss": 3.7112138884872543, + "tokens_seen": 840071168 + }, + { + "epoch": 2.07, + "learning_rate": 0.00037648946840521564, + "loss": 2.5425, + "theoretical_loss": 3.711185442802006, + "tokens_seen": 840136704 + }, + { + "epoch": 2.07, + "learning_rate": 0.00037647943831494487, + "loss": 2.8974, + "theoretical_loss": 3.7111569999568648, + "tokens_seen": 840202240 + }, + { + "epoch": 2.07, + "learning_rate": 0.000376469408224674, + "loss": 2.8294, + "theoretical_loss": 3.711128559951325, + "tokens_seen": 840267776 + }, + { + "epoch": 2.07, + "learning_rate": 0.00037645937813440323, + "loss": 2.9157, + "theoretical_loss": 3.711100122784882, + "tokens_seen": 840333312 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003764493480441324, + "loss": 3.2182, + "theoretical_loss": 3.7110716884570305, + "tokens_seen": 840398848 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003764393179538616, + "loss": 2.7897, + "theoretical_loss": 3.711043256967267, + "tokens_seen": 840464384 + }, + { + "epoch": 2.07, + "objective/train/docs_used": 941411, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.780778169631958, + "objective/train/theoretical_loss": 3.71102904228651, + "objective/train/tokens_used": 860957152, + "theoretical_loss": 3.71102904228651, + "tokens_seen": 840497152 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003764292878635908, + "loss": 2.8305, + "theoretical_loss": 3.7110148283150854, + "tokens_seen": 840529920 + }, + { + "epoch": 2.07, + "learning_rate": 0.00037641925777331996, + "loss": 2.7123, + "theoretical_loss": 3.710986402499983, + "tokens_seen": 840595456 + }, + { + "epoch": 2.07, + "learning_rate": 0.00037640922768304914, + "loss": 3.029, + "theoretical_loss": 3.710957979521454, + "tokens_seen": 840660992 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003763991975927784, + "loss": 2.761, + "theoretical_loss": 3.7109295593789953, + "tokens_seen": 840726528 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003763891675025075, + "loss": 2.6939, + "theoretical_loss": 3.7109011420721023, + "tokens_seen": 840792064 + }, + { + "epoch": 2.07, + "learning_rate": 0.00037637913741223674, + "loss": 2.6892, + "theoretical_loss": 3.710872727600271, + "tokens_seen": 840857600 + }, + { + "epoch": 2.07, + "learning_rate": 0.00037636910732196586, + "loss": 2.8136, + "theoretical_loss": 3.710844315962998, + "tokens_seen": 840923136 + }, + { + "epoch": 2.07, + "learning_rate": 0.0003763590772316951, + "loss": 2.6677, + "theoretical_loss": 3.7108159071597795, + "tokens_seen": 840988672 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003763490471414243, + "loss": 2.7426, + "theoretical_loss": 3.710787501190112, + "tokens_seen": 841054208 + }, + { + "epoch": 2.08, + "learning_rate": 0.00037633901705115346, + "loss": 2.7228, + "theoretical_loss": 3.710759098053492, + "tokens_seen": 841119744 + }, + { + "epoch": 2.08, + "learning_rate": 0.00037632898696088264, + "loss": 2.8451, + "theoretical_loss": 3.7107306977494163, + "tokens_seen": 841185280 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003763189568706119, + "loss": 2.7124, + "theoretical_loss": 3.710702300277382, + "tokens_seen": 841250816 + }, + { + "epoch": 2.08, + "learning_rate": 0.000376308926780341, + "loss": 2.8676, + "theoretical_loss": 3.7106739056368863, + "tokens_seen": 841316352 + }, + { + "epoch": 2.08, + "learning_rate": 0.00037629889669007024, + "loss": 2.812, + "theoretical_loss": 3.7106455138274255, + "tokens_seen": 841381888 + }, + { + "epoch": 2.08, + "learning_rate": 0.00037628886659979937, + "loss": 2.8864, + "theoretical_loss": 3.710617124848497, + "tokens_seen": 841447424 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003762788365095286, + "loss": 2.5823, + "theoretical_loss": 3.710588738699599, + "tokens_seen": 841512960 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003762688064192578, + "loss": 2.6779, + "theoretical_loss": 3.7105603553802284, + "tokens_seen": 841578496 + }, + { + "epoch": 2.08, + "learning_rate": 0.00037625877632898696, + "loss": 2.6513, + "theoretical_loss": 3.710531974889883, + "tokens_seen": 841644032 + }, + { + "epoch": 2.08, + "learning_rate": 0.00037624874623871615, + "loss": 2.9036, + "theoretical_loss": 3.710503597228061, + "tokens_seen": 841709568 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003762387161484453, + "loss": 2.9422, + "theoretical_loss": 3.710475222394259, + "tokens_seen": 841775104 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003762286860581745, + "loss": 2.722, + "theoretical_loss": 3.710446850387976, + "tokens_seen": 841840640 + }, + { + "epoch": 2.08, + "learning_rate": 0.00037621865596790374, + "loss": 2.8682, + "theoretical_loss": 3.7104184812087104, + "tokens_seen": 841906176 + }, + { + "epoch": 2.08, + "learning_rate": 0.00037620862587763287, + "loss": 2.5776, + "theoretical_loss": 3.71039011485596, + "tokens_seen": 841971712 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003761985957873621, + "loss": 2.5632, + "theoretical_loss": 3.7103617513292235, + "tokens_seen": 842037248 + }, + { + "epoch": 2.08, + "learning_rate": 0.00037618856569709123, + "loss": 2.5813, + "theoretical_loss": 3.7103333906279987, + "tokens_seen": 842102784 + }, + { + "epoch": 2.08, + "objective/train/docs_used": 942076, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.738717794418335, + "objective/train/theoretical_loss": 3.710319211336797, + "objective/train/tokens_used": 862595552, + "theoretical_loss": 3.710319211336797, + "tokens_seen": 842135552 + }, + { + "epoch": 2.08, + "learning_rate": 0.00037617853560682047, + "loss": 2.8054, + "theoretical_loss": 3.7103050327517852, + "tokens_seen": 842168320 + }, + { + "epoch": 2.08, + "learning_rate": 0.00037616850551654965, + "loss": 2.7416, + "theoretical_loss": 3.7102766777000813, + "tokens_seen": 842233856 + }, + { + "epoch": 2.08, + "learning_rate": 0.00037615847542627883, + "loss": 3.0039, + "theoretical_loss": 3.7102483254723864, + "tokens_seen": 842299392 + }, + { + "epoch": 2.08, + "learning_rate": 0.000376148445336008, + "loss": 2.9361, + "theoretical_loss": 3.710219976068199, + "tokens_seen": 842364928 + }, + { + "epoch": 2.08, + "learning_rate": 0.00037613841524573725, + "loss": 2.7569, + "theoretical_loss": 3.710191629487018, + "tokens_seen": 842430464 + }, + { + "epoch": 2.08, + "learning_rate": 0.00037612838515546637, + "loss": 2.5037, + "theoretical_loss": 3.7101632857283438, + "tokens_seen": 842496000 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003761183550651956, + "loss": 2.8678, + "theoretical_loss": 3.710134944791675, + "tokens_seen": 842561536 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003761083249749248, + "loss": 2.7399, + "theoretical_loss": 3.7101066066765114, + "tokens_seen": 842627072 + }, + { + "epoch": 2.08, + "learning_rate": 0.00037609829488465397, + "loss": 3.0493, + "theoretical_loss": 3.710078271382353, + "tokens_seen": 842692608 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003760882647943832, + "loss": 2.9969, + "theoretical_loss": 3.7100499389086985, + "tokens_seen": 842758144 + }, + { + "epoch": 2.08, + "learning_rate": 0.00037607823470411233, + "loss": 2.8912, + "theoretical_loss": 3.7100216092550493, + "tokens_seen": 842823680 + }, + { + "epoch": 2.08, + "learning_rate": 0.00037606820461384157, + "loss": 2.9114, + "theoretical_loss": 3.7099932824209043, + "tokens_seen": 842889216 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003760581745235707, + "loss": 2.6763, + "theoretical_loss": 3.709964958405765, + "tokens_seen": 842954752 + }, + { + "epoch": 2.08, + "learning_rate": 0.00037604814443329993, + "loss": 2.7973, + "theoretical_loss": 3.7099366372091303, + "tokens_seen": 843020288 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003760381143430291, + "loss": 3.0025, + "theoretical_loss": 3.709908318830501, + "tokens_seen": 843085824 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003760280842527583, + "loss": 3.03, + "theoretical_loss": 3.7098800032693786, + "tokens_seen": 843151360 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003760180541624875, + "loss": 2.791, + "theoretical_loss": 3.709851690525263, + "tokens_seen": 843216896 + }, + { + "epoch": 2.08, + "learning_rate": 0.00037600802407221665, + "loss": 2.9545, + "theoretical_loss": 3.7098233805976557, + "tokens_seen": 843282432 + }, + { + "epoch": 2.08, + "learning_rate": 0.00037599799398194584, + "loss": 2.5329, + "theoretical_loss": 3.7097950734860565, + "tokens_seen": 843347968 + }, + { + "epoch": 2.08, + "learning_rate": 0.00037598796389167507, + "loss": 2.914, + "theoretical_loss": 3.7097667691899674, + "tokens_seen": 843413504 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003759779338014042, + "loss": 2.8874, + "theoretical_loss": 3.709738467708889, + "tokens_seen": 843479040 + }, + { + "epoch": 2.08, + "learning_rate": 0.00037596790371113343, + "loss": 2.6411, + "theoretical_loss": 3.7097101690423235, + "tokens_seen": 843544576 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003759578736208626, + "loss": 2.6696, + "theoretical_loss": 3.7096818731897723, + "tokens_seen": 843610112 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003759478435305918, + "loss": 2.7156, + "theoretical_loss": 3.709653580150736, + "tokens_seen": 843675648 + }, + { + "epoch": 2.08, + "learning_rate": 0.000375937813440321, + "loss": 2.7071, + "theoretical_loss": 3.7096252899247175, + "tokens_seen": 843741184 + }, + { + "epoch": 2.08, + "objective/train/docs_used": 943510, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8786866664886475, + "objective/train/theoretical_loss": 3.709611145866434, + "objective/train/tokens_used": 864233952, + "theoretical_loss": 3.709611145866434, + "tokens_seen": 843773952 + }, + { + "epoch": 2.08, + "learning_rate": 0.00037592778335005016, + "loss": 2.8376, + "theoretical_loss": 3.7095970025112175, + "tokens_seen": 843806720 + }, + { + "epoch": 2.08, + "learning_rate": 0.00037591775325977934, + "loss": 2.8534, + "theoretical_loss": 3.7095687179097387, + "tokens_seen": 843872256 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003759077231695086, + "loss": 2.9706, + "theoretical_loss": 3.7095404361197835, + "tokens_seen": 843937792 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003758976930792377, + "loss": 2.6948, + "theoretical_loss": 3.7095121571408534, + "tokens_seen": 844003328 + }, + { + "epoch": 2.08, + "learning_rate": 0.00037588766298896694, + "loss": 2.7853, + "theoretical_loss": 3.709483880972451, + "tokens_seen": 844068864 + }, + { + "epoch": 2.08, + "learning_rate": 0.00037587763289869606, + "loss": 2.8065, + "theoretical_loss": 3.709455607614079, + "tokens_seen": 844134400 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003758676028084253, + "loss": 2.7958, + "theoretical_loss": 3.7094273370652404, + "tokens_seen": 844199936 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003758575727181545, + "loss": 2.8232, + "theoretical_loss": 3.709399069325437, + "tokens_seen": 844265472 + }, + { + "epoch": 2.08, + "learning_rate": 0.00037584754262788366, + "loss": 3.139, + "theoretical_loss": 3.7093708043941716, + "tokens_seen": 844331008 + }, + { + "epoch": 2.08, + "learning_rate": 0.00037583751253761284, + "loss": 2.8477, + "theoretical_loss": 3.7093425422709485, + "tokens_seen": 844396544 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003758274824473421, + "loss": 2.8708, + "theoretical_loss": 3.70931428295527, + "tokens_seen": 844462080 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003758174523570712, + "loss": 2.5674, + "theoretical_loss": 3.7092860264466387, + "tokens_seen": 844527616 + }, + { + "epoch": 2.08, + "learning_rate": 0.00037580742226680044, + "loss": 2.7413, + "theoretical_loss": 3.709257772744559, + "tokens_seen": 844593152 + }, + { + "epoch": 2.08, + "learning_rate": 0.00037579739217652957, + "loss": 2.7255, + "theoretical_loss": 3.709229521848534, + "tokens_seen": 844658688 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003757873620862588, + "loss": 2.78, + "theoretical_loss": 3.7092012737580675, + "tokens_seen": 844724224 + }, + { + "epoch": 2.08, + "learning_rate": 0.000375777331995988, + "loss": 2.5741, + "theoretical_loss": 3.709173028472663, + "tokens_seen": 844789760 + }, + { + "epoch": 2.08, + "learning_rate": 0.00037576730190571716, + "loss": 2.8994, + "theoretical_loss": 3.7091447859918247, + "tokens_seen": 844855296 + }, + { + "epoch": 2.08, + "learning_rate": 0.00037575727181544635, + "loss": 2.7549, + "theoretical_loss": 3.709116546315056, + "tokens_seen": 844920832 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003757472417251755, + "loss": 2.8349, + "theoretical_loss": 3.709088309441862, + "tokens_seen": 844986368 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003757372116349047, + "loss": 2.9799, + "theoretical_loss": 3.7090600753717458, + "tokens_seen": 845051904 + }, + { + "epoch": 2.08, + "learning_rate": 0.00037572718154463394, + "loss": 2.9412, + "theoretical_loss": 3.7090318441042127, + "tokens_seen": 845117440 + }, + { + "epoch": 2.08, + "learning_rate": 0.00037571715145436307, + "loss": 2.6342, + "theoretical_loss": 3.7090036156387667, + "tokens_seen": 845182976 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003757071213640923, + "loss": 3.0338, + "theoretical_loss": 3.7089753899749134, + "tokens_seen": 845248512 + }, + { + "epoch": 2.08, + "learning_rate": 0.00037569709127382143, + "loss": 2.7319, + "theoretical_loss": 3.708947167112156, + "tokens_seen": 845314048 + }, + { + "epoch": 2.08, + "learning_rate": 0.00037568706118355067, + "loss": 2.6745, + "theoretical_loss": 3.70891894705, + "tokens_seen": 845379584 + }, + { + "epoch": 2.08, + "objective/train/docs_used": 944052, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.632538318634033, + "objective/train/theoretical_loss": 3.7089048380689933, + "objective/train/tokens_used": 865872352, + "theoretical_loss": 3.7089048380689933, + "tokens_seen": 845412352 + }, + { + "epoch": 2.08, + "learning_rate": 0.00037567703109327985, + "loss": 2.8133, + "theoretical_loss": 3.708890729787951, + "tokens_seen": 845445120 + }, + { + "epoch": 2.08, + "learning_rate": 0.00037566700100300903, + "loss": 2.7545, + "theoretical_loss": 3.7088625153255137, + "tokens_seen": 845510656 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003756569709127382, + "loss": 2.8038, + "theoretical_loss": 3.7088343036621936, + "tokens_seen": 845576192 + }, + { + "epoch": 2.08, + "learning_rate": 0.00037564694082246745, + "loss": 2.8285, + "theoretical_loss": 3.7088060947974957, + "tokens_seen": 845641728 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003756369107321966, + "loss": 2.6281, + "theoretical_loss": 3.7087778887309257, + "tokens_seen": 845707264 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003756268806419258, + "loss": 2.9724, + "theoretical_loss": 3.7087496854619895, + "tokens_seen": 845772800 + }, + { + "epoch": 2.08, + "learning_rate": 0.00037561685055165494, + "loss": 2.9071, + "theoretical_loss": 3.708721484990192, + "tokens_seen": 845838336 + }, + { + "epoch": 2.08, + "learning_rate": 0.00037560682046138417, + "loss": 2.8209, + "theoretical_loss": 3.7086932873150404, + "tokens_seen": 845903872 + }, + { + "epoch": 2.08, + "learning_rate": 0.00037559679037111335, + "loss": 2.725, + "theoretical_loss": 3.7086650924360396, + "tokens_seen": 845969408 + }, + { + "epoch": 2.08, + "learning_rate": 0.00037558676028084253, + "loss": 2.7387, + "theoretical_loss": 3.7086369003526967, + "tokens_seen": 846034944 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003755767301905717, + "loss": 2.6394, + "theoretical_loss": 3.7086087110645165, + "tokens_seen": 846100480 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003755667001003009, + "loss": 2.9535, + "theoretical_loss": 3.708580524571007, + "tokens_seen": 846166016 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003755566700100301, + "loss": 2.7447, + "theoretical_loss": 3.708552340871674, + "tokens_seen": 846231552 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003755466399197593, + "loss": 2.691, + "theoretical_loss": 3.708524159966024, + "tokens_seen": 846297088 + }, + { + "epoch": 2.08, + "learning_rate": 0.00037553660982948844, + "loss": 2.8282, + "theoretical_loss": 3.7084959818535648, + "tokens_seen": 846362624 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003755265797392177, + "loss": 2.7942, + "theoretical_loss": 3.7084678065338013, + "tokens_seen": 846428160 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003755165496489468, + "loss": 2.687, + "theoretical_loss": 3.7084396340062424, + "tokens_seen": 846493696 + }, + { + "epoch": 2.08, + "learning_rate": 0.00037550651955867604, + "loss": 2.9069, + "theoretical_loss": 3.7084114642703945, + "tokens_seen": 846559232 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003754964894684052, + "loss": 2.7304, + "theoretical_loss": 3.7083832973257653, + "tokens_seen": 846624768 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003754864593781344, + "loss": 2.9416, + "theoretical_loss": 3.708355133171861, + "tokens_seen": 846690304 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003754764292878636, + "loss": 2.7376, + "theoretical_loss": 3.708326971808191, + "tokens_seen": 846755840 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003754663991975928, + "loss": 2.8387, + "theoretical_loss": 3.7082988132342605, + "tokens_seen": 846821376 + }, + { + "epoch": 2.08, + "learning_rate": 0.00037545636910732194, + "loss": 2.7832, + "theoretical_loss": 3.70827065744958, + "tokens_seen": 846886912 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003754463390170512, + "loss": 2.7474, + "theoretical_loss": 3.708242504453655, + "tokens_seen": 846952448 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003754363089267803, + "loss": 2.9122, + "theoretical_loss": 3.708214354245995, + "tokens_seen": 847017984 + }, + { + "epoch": 2.08, + "objective/train/docs_used": 945481, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.310624122619629, + "objective/train/theoretical_loss": 3.7082002801876106, + "objective/train/tokens_used": 867510752, + "theoretical_loss": 3.7082002801876106, + "tokens_seen": 847050752 + }, + { + "epoch": 2.08, + "learning_rate": 0.00037542627883650954, + "loss": 2.6795, + "theoretical_loss": 3.708186206826108, + "tokens_seen": 847083520 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003754162487462387, + "loss": 2.6511, + "theoretical_loss": 3.708158062193502, + "tokens_seen": 847149056 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003754062186559679, + "loss": 2.9887, + "theoretical_loss": 3.7081299203476847, + "tokens_seen": 847214592 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003753961885656971, + "loss": 2.9141, + "theoretical_loss": 3.708101781288166, + "tokens_seen": 847280128 + }, + { + "epoch": 2.08, + "learning_rate": 0.00037538615847542626, + "loss": 3.0222, + "theoretical_loss": 3.708073645014454, + "tokens_seen": 847345664 + }, + { + "epoch": 2.08, + "learning_rate": 0.00037537612838515544, + "loss": 2.6518, + "theoretical_loss": 3.708045511526057, + "tokens_seen": 847411200 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003753660982948847, + "loss": 2.5696, + "theoretical_loss": 3.7080173808224846, + "tokens_seen": 847476736 + }, + { + "epoch": 2.08, + "learning_rate": 0.00037535606820461386, + "loss": 2.8286, + "theoretical_loss": 3.707989252903245, + "tokens_seen": 847542272 + }, + { + "epoch": 2.08, + "learning_rate": 0.00037534603811434304, + "loss": 2.7683, + "theoretical_loss": 3.707961127767848, + "tokens_seen": 847607808 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003753360080240723, + "loss": 2.7759, + "theoretical_loss": 3.7079330054158026, + "tokens_seen": 847673344 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003753259779338014, + "loss": 3.1806, + "theoretical_loss": 3.7079048858466184, + "tokens_seen": 847738880 + }, + { + "epoch": 2.08, + "learning_rate": 0.00037531594784353064, + "loss": 2.7589, + "theoretical_loss": 3.707876769059805, + "tokens_seen": 847804416 + }, + { + "epoch": 2.08, + "learning_rate": 0.00037530591775325977, + "loss": 2.6025, + "theoretical_loss": 3.7078486550548715, + "tokens_seen": 847869952 + }, + { + "epoch": 2.08, + "learning_rate": 0.000375295887662989, + "loss": 2.7559, + "theoretical_loss": 3.707820543831328, + "tokens_seen": 847935488 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003752858575727182, + "loss": 2.5894, + "theoretical_loss": 3.7077924353886846, + "tokens_seen": 848001024 + }, + { + "epoch": 2.08, + "learning_rate": 0.00037527582748244736, + "loss": 2.745, + "theoretical_loss": 3.707764329726451, + "tokens_seen": 848066560 + }, + { + "epoch": 2.08, + "learning_rate": 0.00037526579739217655, + "loss": 2.6036, + "theoretical_loss": 3.7077362268441374, + "tokens_seen": 848132096 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003752557673019057, + "loss": 2.531, + "theoretical_loss": 3.707708126741254, + "tokens_seen": 848197632 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003752457372116349, + "loss": 2.9765, + "theoretical_loss": 3.7076800294173116, + "tokens_seen": 848263168 + }, + { + "epoch": 2.08, + "learning_rate": 0.00037523570712136414, + "loss": 2.7127, + "theoretical_loss": 3.70765193487182, + "tokens_seen": 848328704 + }, + { + "epoch": 2.08, + "learning_rate": 0.00037522567703109327, + "loss": 2.587, + "theoretical_loss": 3.707623843104291, + "tokens_seen": 848394240 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003752156469408225, + "loss": 2.6971, + "theoretical_loss": 3.707595754114234, + "tokens_seen": 848459776 + }, + { + "epoch": 2.08, + "learning_rate": 0.00037520561685055163, + "loss": 2.6927, + "theoretical_loss": 3.707567667901161, + "tokens_seen": 848525312 + }, + { + "epoch": 2.08, + "learning_rate": 0.00037519558676028087, + "loss": 2.6176, + "theoretical_loss": 3.707539584464582, + "tokens_seen": 848590848 + }, + { + "epoch": 2.08, + "learning_rate": 0.00037518555667001005, + "loss": 2.6565, + "theoretical_loss": 3.7075115038040085, + "tokens_seen": 848656384 + }, + { + "epoch": 2.08, + "objective/train/docs_used": 946246, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.916497230529785, + "objective/train/theoretical_loss": 3.7074974645145717, + "objective/train/tokens_used": 869149152, + "theoretical_loss": 3.7074974645145717, + "tokens_seen": 848689152 + }, + { + "epoch": 2.08, + "learning_rate": 0.00037517552657973923, + "loss": 2.8534, + "theoretical_loss": 3.7074834259189524, + "tokens_seen": 848721920 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003751654964894684, + "loss": 2.5884, + "theoretical_loss": 3.7074553508089245, + "tokens_seen": 848787456 + }, + { + "epoch": 2.08, + "learning_rate": 0.00037515546639919765, + "loss": 2.5969, + "theoretical_loss": 3.7074272784734363, + "tokens_seen": 848852992 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003751454363089268, + "loss": 2.7696, + "theoretical_loss": 3.7073992089119994, + "tokens_seen": 848918528 + }, + { + "epoch": 2.08, + "learning_rate": 0.000375135406218656, + "loss": 2.6844, + "theoretical_loss": 3.7073711421241256, + "tokens_seen": 848984064 + }, + { + "epoch": 2.08, + "learning_rate": 0.00037512537612838514, + "loss": 2.6811, + "theoretical_loss": 3.707343078109327, + "tokens_seen": 849049600 + }, + { + "epoch": 2.08, + "learning_rate": 0.00037511534603811437, + "loss": 2.9713, + "theoretical_loss": 3.707315016867115, + "tokens_seen": 849115136 + }, + { + "epoch": 2.08, + "learning_rate": 0.00037510531594784355, + "loss": 2.8274, + "theoretical_loss": 3.7072869583970025, + "tokens_seen": 849180672 + }, + { + "epoch": 2.08, + "learning_rate": 0.00037509528585757273, + "loss": 2.7507, + "theoretical_loss": 3.7072589026985012, + "tokens_seen": 849246208 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003750852557673019, + "loss": 3.0336, + "theoretical_loss": 3.707230849771124, + "tokens_seen": 849311744 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003750752256770311, + "loss": 2.9991, + "theoretical_loss": 3.707202799614383, + "tokens_seen": 849377280 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003750651955867603, + "loss": 2.6436, + "theoretical_loss": 3.7071747522277905, + "tokens_seen": 849442816 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003750551654964895, + "loss": 2.7344, + "theoretical_loss": 3.70714670761086, + "tokens_seen": 849508352 + }, + { + "epoch": 2.08, + "learning_rate": 0.00037504513540621864, + "loss": 2.7969, + "theoretical_loss": 3.7071186657631037, + "tokens_seen": 849573888 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003750351053159479, + "loss": 2.8591, + "theoretical_loss": 3.7070906266840353, + "tokens_seen": 849639424 + }, + { + "epoch": 2.08, + "learning_rate": 0.000375025075225677, + "loss": 3.018, + "theoretical_loss": 3.707062590373167, + "tokens_seen": 849704960 + }, + { + "epoch": 2.08, + "learning_rate": 0.00037501504513540624, + "loss": 2.6358, + "theoretical_loss": 3.7070345568300125, + "tokens_seen": 849770496 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003750050150451354, + "loss": 2.9031, + "theoretical_loss": 3.707006526054085, + "tokens_seen": 849836032 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003749949849548646, + "loss": 2.6426, + "theoretical_loss": 3.706978498044898, + "tokens_seen": 849901568 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003749849548645938, + "loss": 2.6169, + "theoretical_loss": 3.7069504728019655, + "tokens_seen": 849967104 + }, + { + "epoch": 2.08, + "learning_rate": 0.000374974924774323, + "loss": 2.8302, + "theoretical_loss": 3.706922450324801, + "tokens_seen": 850032640 + }, + { + "epoch": 2.08, + "learning_rate": 0.00037496489468405214, + "loss": 2.7208, + "theoretical_loss": 3.706894430612919, + "tokens_seen": 850098176 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003749548645937814, + "loss": 2.7541, + "theoretical_loss": 3.7068664136658316, + "tokens_seen": 850163712 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003749448345035105, + "loss": 2.7727, + "theoretical_loss": 3.7068383994830545, + "tokens_seen": 850229248 + }, + { + "epoch": 2.08, + "learning_rate": 0.00037493480441323974, + "loss": 2.6672, + "theoretical_loss": 3.7068103880641017, + "tokens_seen": 850294784 + }, + { + "epoch": 2.08, + "objective/train/docs_used": 947576, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4943060874938965, + "objective/train/theoretical_loss": 3.7067963833909072, + "objective/train/tokens_used": 870787552, + "theoretical_loss": 3.7067963833909072, + "tokens_seen": 850327552 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003749247743229689, + "loss": 2.6507, + "theoretical_loss": 3.706782379408487, + "tokens_seen": 850360320 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003749147442326981, + "loss": 2.79, + "theoretical_loss": 3.706754373515725, + "tokens_seen": 850425856 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003749047141424273, + "loss": 2.9062, + "theoretical_loss": 3.7067263703853306, + "tokens_seen": 850491392 + }, + { + "epoch": 2.08, + "learning_rate": 0.00037489468405215646, + "loss": 2.8495, + "theoretical_loss": 3.7066983700168183, + "tokens_seen": 850556928 + }, + { + "epoch": 2.08, + "learning_rate": 0.00037488465396188564, + "loss": 2.6916, + "theoretical_loss": 3.706670372409703, + "tokens_seen": 850622464 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003748746238716149, + "loss": 2.7941, + "theoretical_loss": 3.7066423775635, + "tokens_seen": 850688000 + }, + { + "epoch": 2.08, + "learning_rate": 0.000374864593781344, + "loss": 2.8913, + "theoretical_loss": 3.7066143854777236, + "tokens_seen": 850753536 + }, + { + "epoch": 2.08, + "learning_rate": 0.00037485456369107324, + "loss": 3.0094, + "theoretical_loss": 3.7065863961518897, + "tokens_seen": 850819072 + }, + { + "epoch": 2.08, + "learning_rate": 0.00037484453360080237, + "loss": 2.9763, + "theoretical_loss": 3.7065584095855133, + "tokens_seen": 850884608 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003748345035105316, + "loss": 2.9927, + "theoretical_loss": 3.7065304257781095, + "tokens_seen": 850950144 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003748244734202608, + "loss": 2.7568, + "theoretical_loss": 3.7065024447291943, + "tokens_seen": 851015680 + }, + { + "epoch": 2.08, + "learning_rate": 0.00037481444332998997, + "loss": 2.721, + "theoretical_loss": 3.7064744664382836, + "tokens_seen": 851081216 + }, + { + "epoch": 2.08, + "learning_rate": 0.00037480441323971915, + "loss": 2.5999, + "theoretical_loss": 3.706446490904893, + "tokens_seen": 851146752 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003747943831494484, + "loss": 2.7467, + "theoretical_loss": 3.7064185181285376, + "tokens_seen": 851212288 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003747843530591775, + "loss": 2.6778, + "theoretical_loss": 3.706390548108735, + "tokens_seen": 851277824 + }, + { + "epoch": 2.08, + "learning_rate": 0.00037477432296890675, + "loss": 2.6205, + "theoretical_loss": 3.706362580845, + "tokens_seen": 851343360 + }, + { + "epoch": 2.08, + "learning_rate": 0.00037476429287863587, + "loss": 2.888, + "theoretical_loss": 3.70633461633685, + "tokens_seen": 851408896 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003747542627883651, + "loss": 2.4779, + "theoretical_loss": 3.706306654583801, + "tokens_seen": 851474432 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003747442326980943, + "loss": 2.6989, + "theoretical_loss": 3.706278695585369, + "tokens_seen": 851539968 + }, + { + "epoch": 2.08, + "learning_rate": 0.00037473420260782347, + "loss": 2.9415, + "theoretical_loss": 3.706250739341071, + "tokens_seen": 851605504 + }, + { + "epoch": 2.08, + "learning_rate": 0.00037472417251755265, + "loss": 2.7205, + "theoretical_loss": 3.706222785850424, + "tokens_seen": 851671040 + }, + { + "epoch": 2.08, + "learning_rate": 0.00037471414242728183, + "loss": 2.6934, + "theoretical_loss": 3.706194835112945, + "tokens_seen": 851736576 + }, + { + "epoch": 2.08, + "learning_rate": 0.000374704112337011, + "loss": 2.6926, + "theoretical_loss": 3.706166887128151, + "tokens_seen": 851802112 + }, + { + "epoch": 2.08, + "learning_rate": 0.00037469408224674025, + "loss": 2.8434, + "theoretical_loss": 3.7061389418955586, + "tokens_seen": 851867648 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003746840521564694, + "loss": 2.5986, + "theoretical_loss": 3.706110999414686, + "tokens_seen": 851933184 + }, + { + "epoch": 2.08, + "objective/train/docs_used": 948304, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5480265617370605, + "objective/train/theoretical_loss": 3.706097029205993, + "objective/train/tokens_used": 872425952, + "theoretical_loss": 3.706097029205993, + "tokens_seen": 851965952 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003746740220661986, + "loss": 2.8042, + "theoretical_loss": 3.7060830596850494, + "tokens_seen": 851998720 + }, + { + "epoch": 2.08, + "learning_rate": 0.00037466399197592774, + "loss": 2.5988, + "theoretical_loss": 3.706055122706167, + "tokens_seen": 852064256 + }, + { + "epoch": 2.08, + "learning_rate": 0.000374653961885657, + "loss": 2.5353, + "theoretical_loss": 3.706027188477557, + "tokens_seen": 852129792 + }, + { + "epoch": 2.08, + "learning_rate": 0.00037464393179538615, + "loss": 2.8022, + "theoretical_loss": 3.7059992569987354, + "tokens_seen": 852195328 + }, + { + "epoch": 2.08, + "learning_rate": 0.00037463390170511534, + "loss": 2.8748, + "theoretical_loss": 3.7059713282692224, + "tokens_seen": 852260864 + }, + { + "epoch": 2.08, + "learning_rate": 0.00037462387161484457, + "loss": 2.779, + "theoretical_loss": 3.7059434022885345, + "tokens_seen": 852326400 + }, + { + "epoch": 2.08, + "learning_rate": 0.00037461384152457375, + "loss": 2.7233, + "theoretical_loss": 3.7059154790561895, + "tokens_seen": 852391936 + }, + { + "epoch": 2.08, + "learning_rate": 0.00037460381143430293, + "loss": 2.7193, + "theoretical_loss": 3.705887558571707, + "tokens_seen": 852457472 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003745937813440321, + "loss": 2.6575, + "theoretical_loss": 3.7058596408346043, + "tokens_seen": 852523008 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003745837512537613, + "loss": 2.738, + "theoretical_loss": 3.705831725844401, + "tokens_seen": 852588544 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003745737211634905, + "loss": 2.6115, + "theoretical_loss": 3.7058038136006144, + "tokens_seen": 852654080 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003745636910732197, + "loss": 2.5825, + "theoretical_loss": 3.705775904102764, + "tokens_seen": 852719616 + }, + { + "epoch": 2.08, + "learning_rate": 0.00037455366098294884, + "loss": 2.6179, + "theoretical_loss": 3.705747997350368, + "tokens_seen": 852785152 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003745436308926781, + "loss": 2.7327, + "theoretical_loss": 3.7057200933429466, + "tokens_seen": 852850688 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003745336008024072, + "loss": 2.8464, + "theoretical_loss": 3.7056921920800177, + "tokens_seen": 852916224 + }, + { + "epoch": 2.08, + "learning_rate": 0.00037452357071213644, + "loss": 2.8869, + "theoretical_loss": 3.7056642935611013, + "tokens_seen": 852981760 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003745135406218656, + "loss": 2.6987, + "theoretical_loss": 3.7056363977857156, + "tokens_seen": 853047296 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003745035105315948, + "loss": 2.8166, + "theoretical_loss": 3.7056085047533815, + "tokens_seen": 853112832 + }, + { + "epoch": 2.08, + "learning_rate": 0.000374493480441324, + "loss": 2.4584, + "theoretical_loss": 3.705580614463618, + "tokens_seen": 853178368 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003744834503510532, + "loss": 2.803, + "theoretical_loss": 3.7055527269159443, + "tokens_seen": 853243904 + }, + { + "epoch": 2.08, + "learning_rate": 0.00037447342026078234, + "loss": 2.7901, + "theoretical_loss": 3.705524842109881, + "tokens_seen": 853309440 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003744633901705116, + "loss": 2.9767, + "theoretical_loss": 3.7054969600449477, + "tokens_seen": 853374976 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003744533600802407, + "loss": 2.7746, + "theoretical_loss": 3.705469080720664, + "tokens_seen": 853440512 + }, + { + "epoch": 2.08, + "learning_rate": 0.00037444332998996994, + "loss": 2.8606, + "theoretical_loss": 3.7054412041365508, + "tokens_seen": 853506048 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003744332998996991, + "loss": 2.6336, + "theoretical_loss": 3.705413330292128, + "tokens_seen": 853571584 + }, + { + "epoch": 2.08, + "objective/train/docs_used": 949423, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8431296348571777, + "objective/train/theoretical_loss": 3.7053993943971513, + "objective/train/tokens_used": 874064352, + "theoretical_loss": 3.7053993943971513, + "tokens_seen": 853604352 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003744232698094283, + "loss": 2.6529, + "theoretical_loss": 3.7053854591869166, + "tokens_seen": 853637120 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003744132397191575, + "loss": 3.0092, + "theoretical_loss": 3.7053575908204364, + "tokens_seen": 853702656 + }, + { + "epoch": 2.08, + "learning_rate": 0.00037440320962888666, + "loss": 2.6759, + "theoretical_loss": 3.705329725192209, + "tokens_seen": 853768192 + }, + { + "epoch": 2.08, + "learning_rate": 0.00037439317953861585, + "loss": 2.5965, + "theoretical_loss": 3.705301862301754, + "tokens_seen": 853833728 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003743831494483451, + "loss": 2.7665, + "theoretical_loss": 3.7052740021485926, + "tokens_seen": 853899264 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003743731193580742, + "loss": 3.0544, + "theoretical_loss": 3.7052461447322464, + "tokens_seen": 853964800 + }, + { + "epoch": 2.08, + "learning_rate": 0.00037436308926780344, + "loss": 2.8876, + "theoretical_loss": 3.705218290052237, + "tokens_seen": 854030336 + }, + { + "epoch": 2.08, + "learning_rate": 0.00037435305917753257, + "loss": 2.7202, + "theoretical_loss": 3.705190438108084, + "tokens_seen": 854095872 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003743430290872618, + "loss": 2.7702, + "theoretical_loss": 3.7051625888993103, + "tokens_seen": 854161408 + }, + { + "epoch": 2.08, + "learning_rate": 0.000374332998996991, + "loss": 2.811, + "theoretical_loss": 3.7051347424254364, + "tokens_seen": 854226944 + }, + { + "epoch": 2.08, + "learning_rate": 0.00037432296890672017, + "loss": 3.0093, + "theoretical_loss": 3.705106898685985, + "tokens_seen": 854292480 + }, + { + "epoch": 2.08, + "learning_rate": 0.00037431293881644935, + "loss": 2.5793, + "theoretical_loss": 3.7050790576804764, + "tokens_seen": 854358016 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003743029087261786, + "loss": 2.5169, + "theoretical_loss": 3.7050512194084337, + "tokens_seen": 854423552 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003742928786359077, + "loss": 2.7798, + "theoretical_loss": 3.7050233838693787, + "tokens_seen": 854489088 + }, + { + "epoch": 2.08, + "learning_rate": 0.00037428284854563695, + "loss": 2.5137, + "theoretical_loss": 3.704995551062833, + "tokens_seen": 854554624 + }, + { + "epoch": 2.08, + "learning_rate": 0.00037427281845536607, + "loss": 2.6821, + "theoretical_loss": 3.7049677209883196, + "tokens_seen": 854620160 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003742627883650953, + "loss": 2.8419, + "theoretical_loss": 3.70493989364536, + "tokens_seen": 854685696 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003742527582748245, + "loss": 2.8814, + "theoretical_loss": 3.704912069033477, + "tokens_seen": 854751232 + }, + { + "epoch": 2.08, + "learning_rate": 0.00037424272818455367, + "loss": 2.6939, + "theoretical_loss": 3.7048842471521937, + "tokens_seen": 854816768 + }, + { + "epoch": 2.08, + "learning_rate": 0.00037423269809428285, + "loss": 2.8317, + "theoretical_loss": 3.7048564280010323, + "tokens_seen": 854882304 + }, + { + "epoch": 2.08, + "learning_rate": 0.00037422266800401203, + "loss": 2.9259, + "theoretical_loss": 3.704828611579515, + "tokens_seen": 854947840 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003742126379137412, + "loss": 2.9123, + "theoretical_loss": 3.704800797887166, + "tokens_seen": 855013376 + }, + { + "epoch": 2.08, + "learning_rate": 0.00037420260782347045, + "loss": 2.8707, + "theoretical_loss": 3.7047729869235084, + "tokens_seen": 855078912 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003741925777331996, + "loss": 2.7544, + "theoretical_loss": 3.7047451786880643, + "tokens_seen": 855144448 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003741825476429288, + "loss": 2.8984, + "theoretical_loss": 3.7047173731803573, + "tokens_seen": 855209984 + }, + { + "epoch": 2.08, + "objective/train/docs_used": 949852, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5512642860412598, + "objective/train/theoretical_loss": 3.704703471449257, + "objective/train/tokens_used": 875702752, + "theoretical_loss": 3.704703471449257, + "tokens_seen": 855242752 + }, + { + "epoch": 2.08, + "learning_rate": 0.00037417251755265794, + "loss": 2.6332, + "theoretical_loss": 3.7046895703999114, + "tokens_seen": 855275520 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003741624874623872, + "loss": 2.7775, + "theoretical_loss": 3.7046617703462497, + "tokens_seen": 855341056 + }, + { + "epoch": 2.08, + "learning_rate": 0.00037415245737211635, + "loss": 2.828, + "theoretical_loss": 3.7046339730188964, + "tokens_seen": 855406592 + }, + { + "epoch": 2.08, + "learning_rate": 0.00037414242728184554, + "loss": 2.6766, + "theoretical_loss": 3.7046061784173743, + "tokens_seen": 855472128 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003741323971915747, + "loss": 2.531, + "theoretical_loss": 3.7045783865412085, + "tokens_seen": 855537664 + }, + { + "epoch": 2.08, + "learning_rate": 0.00037412236710130395, + "loss": 2.6946, + "theoretical_loss": 3.704550597389922, + "tokens_seen": 855603200 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003741123370110331, + "loss": 2.7786, + "theoretical_loss": 3.7045228109630397, + "tokens_seen": 855668736 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003741023069207623, + "loss": 2.8376, + "theoretical_loss": 3.7044950272600854, + "tokens_seen": 855734272 + }, + { + "epoch": 2.08, + "learning_rate": 0.00037409227683049144, + "loss": 2.8049, + "theoretical_loss": 3.704467246280584, + "tokens_seen": 855799808 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003740822467402207, + "loss": 2.8418, + "theoretical_loss": 3.7044394680240593, + "tokens_seen": 855865344 + }, + { + "epoch": 2.08, + "learning_rate": 0.00037407221664994986, + "loss": 2.5376, + "theoretical_loss": 3.7044116924900363, + "tokens_seen": 855930880 + }, + { + "epoch": 2.08, + "learning_rate": 0.00037406218655967904, + "loss": 2.565, + "theoretical_loss": 3.7043839196780404, + "tokens_seen": 855996416 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003740521564694082, + "loss": 2.7114, + "theoretical_loss": 3.7043561495875954, + "tokens_seen": 856061952 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003740421263791374, + "loss": 2.8716, + "theoretical_loss": 3.7043283822182262, + "tokens_seen": 856127488 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003740320962888666, + "loss": 2.7345, + "theoretical_loss": 3.704300617569459, + "tokens_seen": 856193024 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003740220661985958, + "loss": 2.7921, + "theoretical_loss": 3.704272855640818, + "tokens_seen": 856258560 + }, + { + "epoch": 2.08, + "learning_rate": 0.00037401203610832494, + "loss": 2.8099, + "theoretical_loss": 3.70424509643183, + "tokens_seen": 856324096 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003740020060180542, + "loss": 2.6542, + "theoretical_loss": 3.7042173399420184, + "tokens_seen": 856389632 + }, + { + "epoch": 2.08, + "learning_rate": 0.00037399197592778336, + "loss": 2.5126, + "theoretical_loss": 3.7041895861709104, + "tokens_seen": 856455168 + }, + { + "epoch": 2.08, + "learning_rate": 0.00037398194583751254, + "loss": 2.8828, + "theoretical_loss": 3.7041618351180308, + "tokens_seen": 856520704 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003739719157472417, + "loss": 2.5162, + "theoretical_loss": 3.704134086782906, + "tokens_seen": 856586240 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003739618856569709, + "loss": 2.7155, + "theoretical_loss": 3.704106341165062, + "tokens_seen": 856651776 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003739518555667001, + "loss": 2.8485, + "theoretical_loss": 3.7040785982640236, + "tokens_seen": 856717312 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003739418254764293, + "loss": 2.7267, + "theoretical_loss": 3.7040508580793188, + "tokens_seen": 856782848 + }, + { + "epoch": 2.08, + "learning_rate": 0.00037393179538615845, + "loss": 2.8417, + "theoretical_loss": 3.704023120610473, + "tokens_seen": 856848384 + }, + { + "epoch": 2.08, + "objective/train/docs_used": 951087, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8513641357421875, + "objective/train/theoretical_loss": 3.7040092528943487, + "objective/train/tokens_used": 877341152, + "theoretical_loss": 3.7040092528943487, + "tokens_seen": 856881152 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003739217652958877, + "loss": 2.7939, + "theoretical_loss": 3.703995385857012, + "tokens_seen": 856913920 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003739117352056168, + "loss": 2.8448, + "theoretical_loss": 3.703967653818464, + "tokens_seen": 856979456 + }, + { + "epoch": 2.08, + "learning_rate": 0.00037390170511534605, + "loss": 2.7197, + "theoretical_loss": 3.703939924494353, + "tokens_seen": 857044992 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003738916750250752, + "loss": 2.5392, + "theoretical_loss": 3.703912197884209, + "tokens_seen": 857110528 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003738816449348044, + "loss": 2.761, + "theoretical_loss": 3.7038844739875567, + "tokens_seen": 857176064 + }, + { + "epoch": 2.08, + "learning_rate": 0.00037387161484453364, + "loss": 2.6546, + "theoretical_loss": 3.703856752803924, + "tokens_seen": 857241600 + }, + { + "epoch": 2.08, + "learning_rate": 0.00037386158475426277, + "loss": 2.7587, + "theoretical_loss": 3.7038290343328377, + "tokens_seen": 857307136 + }, + { + "epoch": 2.08, + "learning_rate": 0.000373851554663992, + "loss": 2.7639, + "theoretical_loss": 3.7038013185738246, + "tokens_seen": 857372672 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003738415245737212, + "loss": 2.6566, + "theoretical_loss": 3.703773605526413, + "tokens_seen": 857438208 + }, + { + "epoch": 2.08, + "learning_rate": 0.00037383149448345037, + "loss": 2.8388, + "theoretical_loss": 3.7037458951901296, + "tokens_seen": 857503744 + }, + { + "epoch": 2.08, + "learning_rate": 0.00037382146439317955, + "loss": 2.6906, + "theoretical_loss": 3.703718187564503, + "tokens_seen": 857569280 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003738114343029088, + "loss": 2.6582, + "theoretical_loss": 3.7036904826490598, + "tokens_seen": 857634816 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003738014042126379, + "loss": 2.8538, + "theoretical_loss": 3.703662780443328, + "tokens_seen": 857700352 + }, + { + "epoch": 2.08, + "learning_rate": 0.00037379137412236715, + "loss": 2.6117, + "theoretical_loss": 3.7036350809468366, + "tokens_seen": 857765888 + }, + { + "epoch": 2.08, + "learning_rate": 0.00037378134403209627, + "loss": 2.6217, + "theoretical_loss": 3.7036073841591124, + "tokens_seen": 857831424 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003737713139418255, + "loss": 2.6182, + "theoretical_loss": 3.7035796900796845, + "tokens_seen": 857896960 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003737612838515547, + "loss": 2.8175, + "theoretical_loss": 3.7035519987080807, + "tokens_seen": 857962496 + }, + { + "epoch": 2.08, + "learning_rate": 0.00037375125376128387, + "loss": 2.9811, + "theoretical_loss": 3.7035243100438295, + "tokens_seen": 858028032 + }, + { + "epoch": 2.08, + "learning_rate": 0.00037374122367101305, + "loss": 2.9455, + "theoretical_loss": 3.7034966240864593, + "tokens_seen": 858093568 + }, + { + "epoch": 2.08, + "learning_rate": 0.00037373119358074223, + "loss": 2.9907, + "theoretical_loss": 3.7034689408354993, + "tokens_seen": 858159104 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003737211634904714, + "loss": 2.851, + "theoretical_loss": 3.7034412602904783, + "tokens_seen": 858224640 + }, + { + "epoch": 2.08, + "learning_rate": 0.00037371113340020065, + "loss": 2.6784, + "theoretical_loss": 3.7034135824509242, + "tokens_seen": 858290176 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003737011033099298, + "loss": 2.8578, + "theoretical_loss": 3.703385907316367, + "tokens_seen": 858355712 + }, + { + "epoch": 2.08, + "learning_rate": 0.000373691073219659, + "loss": 2.8543, + "theoretical_loss": 3.703358234886336, + "tokens_seen": 858421248 + }, + { + "epoch": 2.08, + "learning_rate": 0.00037368104312938814, + "loss": 2.9281, + "theoretical_loss": 3.703330565160359, + "tokens_seen": 858486784 + }, + { + "epoch": 2.08, + "objective/train/docs_used": 951523, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.799268960952759, + "objective/train/theoretical_loss": 3.7033167313112445, + "objective/train/tokens_used": 878979552, + "theoretical_loss": 3.7033167313112445, + "tokens_seen": 858519552 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003736710130391174, + "loss": 2.8842, + "theoretical_loss": 3.7033028981379674, + "tokens_seen": 858552320 + }, + { + "epoch": 2.08, + "learning_rate": 0.00037366098294884655, + "loss": 2.5017, + "theoretical_loss": 3.703275233818689, + "tokens_seen": 858617856 + }, + { + "epoch": 2.08, + "learning_rate": 0.00037365095285857574, + "loss": 2.7812, + "theoretical_loss": 3.7032475722020544, + "tokens_seen": 858683392 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003736409227683049, + "loss": 3.0125, + "theoretical_loss": 3.703219913287593, + "tokens_seen": 858748928 + }, + { + "epoch": 2.08, + "learning_rate": 0.00037363089267803415, + "loss": 2.855, + "theoretical_loss": 3.7031922570748343, + "tokens_seen": 858814464 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003736208625877633, + "loss": 2.7978, + "theoretical_loss": 3.703164603563309, + "tokens_seen": 858880000 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003736108324974925, + "loss": 2.3875, + "theoretical_loss": 3.7031369527525464, + "tokens_seen": 858945536 + }, + { + "epoch": 2.08, + "learning_rate": 0.00037360080240722164, + "loss": 2.9526, + "theoretical_loss": 3.7031093046420773, + "tokens_seen": 859011072 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003735907723169509, + "loss": 2.5221, + "theoretical_loss": 3.703081659231432, + "tokens_seen": 859076608 + }, + { + "epoch": 2.08, + "learning_rate": 0.00037358074222668006, + "loss": 2.9902, + "theoretical_loss": 3.7030540165201407, + "tokens_seen": 859142144 + }, + { + "epoch": 2.08, + "learning_rate": 0.00037357071213640924, + "loss": 2.5429, + "theoretical_loss": 3.7030263765077334, + "tokens_seen": 859207680 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003735606820461384, + "loss": 2.673, + "theoretical_loss": 3.702998739193742, + "tokens_seen": 859273216 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003735506519558676, + "loss": 3.012, + "theoretical_loss": 3.7029711045776965, + "tokens_seen": 859338752 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003735406218655968, + "loss": 2.6181, + "theoretical_loss": 3.7029434726591277, + "tokens_seen": 859404288 + }, + { + "epoch": 2.08, + "learning_rate": 0.000373530591775326, + "loss": 2.7048, + "theoretical_loss": 3.702915843437567, + "tokens_seen": 859469824 + }, + { + "epoch": 2.08, + "learning_rate": 0.00037352056168505514, + "loss": 2.5463, + "theoretical_loss": 3.7028882169125454, + "tokens_seen": 859535360 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003735105315947844, + "loss": 2.394, + "theoretical_loss": 3.7028605930835945, + "tokens_seen": 859600896 + }, + { + "epoch": 2.08, + "learning_rate": 0.00037350050150451356, + "loss": 2.7804, + "theoretical_loss": 3.702832971950245, + "tokens_seen": 859666432 + }, + { + "epoch": 2.08, + "learning_rate": 0.00037349047141424274, + "loss": 2.6185, + "theoretical_loss": 3.702805353512029, + "tokens_seen": 859731968 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003734804413239719, + "loss": 2.8201, + "theoretical_loss": 3.7027777377684776, + "tokens_seen": 859797504 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003734704112337011, + "loss": 3.0059, + "theoretical_loss": 3.7027501247191226, + "tokens_seen": 859863040 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003734603811434303, + "loss": 2.8458, + "theoretical_loss": 3.7027225143634963, + "tokens_seen": 859928576 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003734503510531595, + "loss": 2.6558, + "theoretical_loss": 3.7026949067011303, + "tokens_seen": 859994112 + }, + { + "epoch": 2.08, + "learning_rate": 0.00037344032096288865, + "loss": 2.7805, + "theoretical_loss": 3.7026673017315566, + "tokens_seen": 860059648 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003734302908726179, + "loss": 2.6283, + "theoretical_loss": 3.7026396994543074, + "tokens_seen": 860125184 + }, + { + "epoch": 2.08, + "objective/train/docs_used": 952944, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6841440200805664, + "objective/train/theoretical_loss": 3.7026258993251586, + "objective/train/tokens_used": 880617952, + "theoretical_loss": 3.7026258993251586, + "tokens_seen": 860157952 + }, + { + "epoch": 2.08, + "learning_rate": 0.000373420260782347, + "loss": 2.8276, + "theoretical_loss": 3.702612099868915, + "tokens_seen": 860190720 + }, + { + "epoch": 2.08, + "learning_rate": 0.00037341023069207625, + "loss": 2.9939, + "theoretical_loss": 3.7025845029749123, + "tokens_seen": 860256256 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003734002006018054, + "loss": 2.696, + "theoretical_loss": 3.7025569087718315, + "tokens_seen": 860321792 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003733901705115346, + "loss": 2.8568, + "theoretical_loss": 3.7025293172592053, + "tokens_seen": 860387328 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003733801404212638, + "loss": 2.6686, + "theoretical_loss": 3.702501728436566, + "tokens_seen": 860452864 + }, + { + "epoch": 2.08, + "learning_rate": 0.00037337011033099297, + "loss": 2.752, + "theoretical_loss": 3.7024741423034477, + "tokens_seen": 860518400 + }, + { + "epoch": 2.08, + "learning_rate": 0.00037336008024072215, + "loss": 2.6242, + "theoretical_loss": 3.702446558859382, + "tokens_seen": 860583936 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003733500501504514, + "loss": 2.7184, + "theoretical_loss": 3.702418978103903, + "tokens_seen": 860649472 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003733400200601805, + "loss": 2.8764, + "theoretical_loss": 3.702391400036543, + "tokens_seen": 860715008 + }, + { + "epoch": 2.08, + "learning_rate": 0.00037332998996990975, + "loss": 2.5713, + "theoretical_loss": 3.7023638246568367, + "tokens_seen": 860780544 + }, + { + "epoch": 2.08, + "learning_rate": 0.00037331995987963893, + "loss": 2.7039, + "theoretical_loss": 3.702336251964317, + "tokens_seen": 860846080 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003733099297893681, + "loss": 2.7103, + "theoretical_loss": 3.7023086819585167, + "tokens_seen": 860911616 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003732998996990973, + "loss": 2.808, + "theoretical_loss": 3.7022811146389696, + "tokens_seen": 860977152 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003732898696088265, + "loss": 2.9242, + "theoretical_loss": 3.702253550005211, + "tokens_seen": 861042688 + }, + { + "epoch": 2.08, + "learning_rate": 0.00037327983951855565, + "loss": 2.7875, + "theoretical_loss": 3.7022259880567736, + "tokens_seen": 861108224 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003732698094282849, + "loss": 2.8359, + "theoretical_loss": 3.702198428793192, + "tokens_seen": 861173760 + }, + { + "epoch": 2.08, + "learning_rate": 0.000373259779338014, + "loss": 2.7244, + "theoretical_loss": 3.702170872214, + "tokens_seen": 861239296 + }, + { + "epoch": 2.08, + "learning_rate": 0.00037324974924774325, + "loss": 2.8139, + "theoretical_loss": 3.702143318318732, + "tokens_seen": 861304832 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003732397191574724, + "loss": 2.6474, + "theoretical_loss": 3.702115767106922, + "tokens_seen": 861370368 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003732296890672016, + "loss": 2.7847, + "theoretical_loss": 3.7020882185781057, + "tokens_seen": 861435904 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003732196589769308, + "loss": 2.8831, + "theoretical_loss": 3.7020606727318164, + "tokens_seen": 861501440 + }, + { + "epoch": 2.08, + "learning_rate": 0.00037320962888666, + "loss": 2.6749, + "theoretical_loss": 3.7020331295675897, + "tokens_seen": 861566976 + }, + { + "epoch": 2.08, + "learning_rate": 0.00037319959879638916, + "loss": 2.5583, + "theoretical_loss": 3.70200558908496, + "tokens_seen": 861632512 + }, + { + "epoch": 2.08, + "learning_rate": 0.00037318956870611834, + "loss": 2.8254, + "theoretical_loss": 3.7019780512834632, + "tokens_seen": 861698048 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003731795386158475, + "loss": 2.7722, + "theoretical_loss": 3.7019505161626327, + "tokens_seen": 861763584 + }, + { + "epoch": 2.08, + "objective/train/docs_used": 953723, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7122349739074707, + "objective/train/theoretical_loss": 3.7019367496073228, + "objective/train/tokens_used": 882256352, + "theoretical_loss": 3.7019367496073228, + "tokens_seen": 861796352 + }, + { + "epoch": 2.08, + "learning_rate": 0.00037316950852557675, + "loss": 2.8407, + "theoretical_loss": 3.701922983722006, + "tokens_seen": 861829120 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003731594784353059, + "loss": 2.7291, + "theoretical_loss": 3.701895453961116, + "tokens_seen": 861894656 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003731494483450351, + "loss": 2.7609, + "theoretical_loss": 3.7018679268794994, + "tokens_seen": 861960192 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003731394182547643, + "loss": 2.7356, + "theoretical_loss": 3.701840402476692, + "tokens_seen": 862025728 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003731293881644935, + "loss": 2.8384, + "theoretical_loss": 3.7018128807522297, + "tokens_seen": 862091264 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003731193580742227, + "loss": 2.7187, + "theoretical_loss": 3.701785361705647, + "tokens_seen": 862156800 + }, + { + "epoch": 2.08, + "learning_rate": 0.00037310932798395184, + "loss": 2.7171, + "theoretical_loss": 3.7017578453364806, + "tokens_seen": 862222336 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003730992978936811, + "loss": 2.8718, + "theoretical_loss": 3.701730331644267, + "tokens_seen": 862287872 + }, + { + "epoch": 2.08, + "learning_rate": 0.00037308926780341026, + "loss": 2.8957, + "theoretical_loss": 3.701702820628541, + "tokens_seen": 862353408 + }, + { + "epoch": 2.08, + "learning_rate": 0.00037307923771313944, + "loss": 2.9734, + "theoretical_loss": 3.701675312288841, + "tokens_seen": 862418944 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003730692076228686, + "loss": 2.8563, + "theoretical_loss": 3.701647806624701, + "tokens_seen": 862484480 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003730591775325978, + "loss": 2.724, + "theoretical_loss": 3.701620303635659, + "tokens_seen": 862550016 + }, + { + "epoch": 2.08, + "learning_rate": 0.000373049147442327, + "loss": 2.676, + "theoretical_loss": 3.701592803321251, + "tokens_seen": 862615552 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003730391173520562, + "loss": 2.7422, + "theoretical_loss": 3.7015653056810143, + "tokens_seen": 862681088 + }, + { + "epoch": 2.08, + "learning_rate": 0.00037302908726178534, + "loss": 2.8457, + "theoretical_loss": 3.701537810714485, + "tokens_seen": 862746624 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003730190571715146, + "loss": 2.6289, + "theoretical_loss": 3.7015103184212004, + "tokens_seen": 862812160 + }, + { + "epoch": 2.08, + "learning_rate": 0.00037300902708124376, + "loss": 2.7261, + "theoretical_loss": 3.7014828288006973, + "tokens_seen": 862877696 + }, + { + "epoch": 2.08, + "learning_rate": 0.00037299899699097294, + "loss": 2.7542, + "theoretical_loss": 3.7014553418525136, + "tokens_seen": 862943232 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003729889669007021, + "loss": 2.5174, + "theoretical_loss": 3.701427857576186, + "tokens_seen": 863008768 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003729789368104313, + "loss": 2.8556, + "theoretical_loss": 3.701400375971252, + "tokens_seen": 863074304 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003729689067201605, + "loss": 2.8079, + "theoretical_loss": 3.701372897037249, + "tokens_seen": 863139840 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003729588766298897, + "loss": 2.7774, + "theoretical_loss": 3.7013454207737153, + "tokens_seen": 863205376 + }, + { + "epoch": 2.08, + "learning_rate": 0.00037294884653961885, + "loss": 2.8166, + "theoretical_loss": 3.701317947180188, + "tokens_seen": 863270912 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003729388164493481, + "loss": 2.518, + "theoretical_loss": 3.7012904762562053, + "tokens_seen": 863336448 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003729287863590772, + "loss": 2.8338, + "theoretical_loss": 3.701263008001305, + "tokens_seen": 863401984 + }, + { + "epoch": 2.08, + "objective/train/docs_used": 954800, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.510504961013794, + "objective/train/theoretical_loss": 3.701249274874616, + "objective/train/tokens_used": 883894752, + "theoretical_loss": 3.701249274874616, + "tokens_seen": 863434752 + }, + { + "epoch": 2.08, + "learning_rate": 0.00037291875626880645, + "loss": 2.7716, + "theoretical_loss": 3.7012355424150254, + "tokens_seen": 863467520 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003729087261785356, + "loss": 2.7719, + "theoretical_loss": 3.7012080794969044, + "tokens_seen": 863533056 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003728986960882648, + "loss": 2.7007, + "theoretical_loss": 3.7011806192464807, + "tokens_seen": 863598592 + }, + { + "epoch": 2.08, + "learning_rate": 0.000372888665997994, + "loss": 2.5711, + "theoretical_loss": 3.7011531616632922, + "tokens_seen": 863664128 + }, + { + "epoch": 2.08, + "learning_rate": 0.00037287863590772317, + "loss": 2.7918, + "theoretical_loss": 3.701125706746878, + "tokens_seen": 863729664 + }, + { + "epoch": 2.08, + "learning_rate": 0.00037286860581745235, + "loss": 2.8889, + "theoretical_loss": 3.701098254496777, + "tokens_seen": 863795200 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003728585757271816, + "loss": 2.9669, + "theoretical_loss": 3.7010708049125274, + "tokens_seen": 863860736 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003728485456369107, + "loss": 2.6103, + "theoretical_loss": 3.701043357993668, + "tokens_seen": 863926272 + }, + { + "epoch": 2.08, + "learning_rate": 0.00037283851554663995, + "loss": 2.7198, + "theoretical_loss": 3.701015913739739, + "tokens_seen": 863991808 + }, + { + "epoch": 2.08, + "learning_rate": 0.00037282848545636913, + "loss": 2.7146, + "theoretical_loss": 3.7009884721502777, + "tokens_seen": 864057344 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003728184553660983, + "loss": 2.8361, + "theoretical_loss": 3.700961033224825, + "tokens_seen": 864122880 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003728084252758275, + "loss": 2.7008, + "theoretical_loss": 3.7009335969629196, + "tokens_seen": 864188416 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003727983951855567, + "loss": 2.7614, + "theoretical_loss": 3.7009061633641007, + "tokens_seen": 864253952 + }, + { + "epoch": 2.08, + "learning_rate": 0.00037278836509528585, + "loss": 2.7302, + "theoretical_loss": 3.700878732427908, + "tokens_seen": 864319488 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003727783350050151, + "loss": 2.6105, + "theoretical_loss": 3.7008513041538817, + "tokens_seen": 864385024 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003727683049147442, + "loss": 2.9334, + "theoretical_loss": 3.700823878541562, + "tokens_seen": 864450560 + }, + { + "epoch": 2.08, + "learning_rate": 0.00037275827482447345, + "loss": 2.7052, + "theoretical_loss": 3.7007964555904875, + "tokens_seen": 864516096 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003727482447342026, + "loss": 2.8405, + "theoretical_loss": 3.7007690353001985, + "tokens_seen": 864581632 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003727382146439318, + "loss": 2.6395, + "theoretical_loss": 3.700741617670236, + "tokens_seen": 864647168 + }, + { + "epoch": 2.08, + "learning_rate": 0.000372728184553661, + "loss": 2.6473, + "theoretical_loss": 3.70071420270014, + "tokens_seen": 864712704 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003727181544633902, + "loss": 2.8752, + "theoretical_loss": 3.700686790389451, + "tokens_seen": 864778240 + }, + { + "epoch": 2.08, + "learning_rate": 0.00037270812437311936, + "loss": 2.852, + "theoretical_loss": 3.700659380737709, + "tokens_seen": 864843776 + }, + { + "epoch": 2.08, + "learning_rate": 0.00037269809428284854, + "loss": 2.9933, + "theoretical_loss": 3.7006319737444553, + "tokens_seen": 864909312 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003726880641925777, + "loss": 2.5759, + "theoretical_loss": 3.7006045694092298, + "tokens_seen": 864974848 + }, + { + "epoch": 2.08, + "learning_rate": 0.00037267803410230696, + "loss": 2.9994, + "theoretical_loss": 3.700577167731574, + "tokens_seen": 865040384 + }, + { + "epoch": 2.08, + "objective/train/docs_used": 955501, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.222694158554077, + "objective/train/theoretical_loss": 3.700563467889191, + "objective/train/tokens_used": 885533152, + "theoretical_loss": 3.700563467889191, + "tokens_seen": 865073152 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003726680040120361, + "loss": 2.7528, + "theoretical_loss": 3.7005497687110287, + "tokens_seen": 865105920 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003726579739217653, + "loss": 2.7434, + "theoretical_loss": 3.7005223723471348, + "tokens_seen": 865171456 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003726479438314945, + "loss": 2.5054, + "theoretical_loss": 3.700494978639434, + "tokens_seen": 865236992 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003726379137412237, + "loss": 2.7522, + "theoretical_loss": 3.700467587587467, + "tokens_seen": 865302528 + }, + { + "epoch": 2.08, + "learning_rate": 0.00037262788365095286, + "loss": 2.593, + "theoretical_loss": 3.7004401991907754, + "tokens_seen": 865368064 + }, + { + "epoch": 2.08, + "learning_rate": 0.00037261785356068204, + "loss": 2.4979, + "theoretical_loss": 3.700412813448901, + "tokens_seen": 865433600 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003726078234704112, + "loss": 2.7374, + "theoretical_loss": 3.7003854303613855, + "tokens_seen": 865499136 + }, + { + "epoch": 2.08, + "learning_rate": 0.00037259779338014046, + "loss": 2.7745, + "theoretical_loss": 3.70035804992777, + "tokens_seen": 865564672 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003725877632898696, + "loss": 2.8962, + "theoretical_loss": 3.7003306721475973, + "tokens_seen": 865630208 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003725777331995988, + "loss": 2.8755, + "theoretical_loss": 3.700303297020409, + "tokens_seen": 865695744 + }, + { + "epoch": 2.08, + "learning_rate": 0.00037256770310932795, + "loss": 2.744, + "theoretical_loss": 3.700275924545747, + "tokens_seen": 865761280 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003725576730190572, + "loss": 2.8406, + "theoretical_loss": 3.7002485547231534, + "tokens_seen": 865826816 + }, + { + "epoch": 2.08, + "learning_rate": 0.00037254764292878636, + "loss": 2.3279, + "theoretical_loss": 3.700221187552171, + "tokens_seen": 865892352 + }, + { + "epoch": 2.08, + "learning_rate": 0.00037253761283851554, + "loss": 2.7378, + "theoretical_loss": 3.7001938230323423, + "tokens_seen": 865957888 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003725275827482447, + "loss": 2.1945, + "theoretical_loss": 3.7001664611632092, + "tokens_seen": 866023424 + }, + { + "epoch": 2.08, + "learning_rate": 0.00037251755265797396, + "loss": 2.6143, + "theoretical_loss": 3.700139101944315, + "tokens_seen": 866088960 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003725075225677031, + "loss": 2.816, + "theoretical_loss": 3.700111745375202, + "tokens_seen": 866154496 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003724974924774323, + "loss": 2.4319, + "theoretical_loss": 3.700084391455414, + "tokens_seen": 866220032 + }, + { + "epoch": 2.08, + "learning_rate": 0.00037248746238716145, + "loss": 2.5547, + "theoretical_loss": 3.7000570401844928, + "tokens_seen": 866285568 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003724774322968907, + "loss": 2.7006, + "theoretical_loss": 3.7000296915619826, + "tokens_seen": 866351104 + }, + { + "epoch": 2.08, + "learning_rate": 0.00037246740220661987, + "loss": 2.6894, + "theoretical_loss": 3.700002345587426, + "tokens_seen": 866416640 + }, + { + "epoch": 2.08, + "learning_rate": 0.00037245737211634905, + "loss": 2.7271, + "theoretical_loss": 3.6999750022603664, + "tokens_seen": 866482176 + }, + { + "epoch": 2.08, + "learning_rate": 0.00037244734202607823, + "loss": 2.7118, + "theoretical_loss": 3.6999476615803473, + "tokens_seen": 866547712 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003724373119358074, + "loss": 2.7494, + "theoretical_loss": 3.6999203235469125, + "tokens_seen": 866613248 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003724272818455366, + "loss": 2.8457, + "theoretical_loss": 3.699892988159606, + "tokens_seen": 866678784 + }, + { + "epoch": 2.08, + "objective/train/docs_used": 956034, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.010556221008301, + "objective/train/theoretical_loss": 3.6998793214581074, + "objective/train/tokens_used": 887171552, + "theoretical_loss": 3.6998793214581074, + "tokens_seen": 866711552 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003724172517552658, + "loss": 2.8745, + "theoretical_loss": 3.6998656554179705, + "tokens_seen": 866744320 + }, + { + "epoch": 2.08, + "learning_rate": 0.00037240722166499495, + "loss": 2.7951, + "theoretical_loss": 3.6998383253215508, + "tokens_seen": 866809856 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003723971915747242, + "loss": 2.5935, + "theoretical_loss": 3.699810997869891, + "tokens_seen": 866875392 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003723871614844533, + "loss": 2.6498, + "theoretical_loss": 3.6997836730625346, + "tokens_seen": 866940928 + }, + { + "epoch": 2.08, + "learning_rate": 0.00037237713139418255, + "loss": 2.6788, + "theoretical_loss": 3.6997563508990265, + "tokens_seen": 867006464 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003723671013039118, + "loss": 2.5869, + "theoretical_loss": 3.6997290313789106, + "tokens_seen": 867072000 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003723570712136409, + "loss": 2.6665, + "theoretical_loss": 3.6997017145017326, + "tokens_seen": 867137536 + }, + { + "epoch": 2.08, + "learning_rate": 0.00037234704112337015, + "loss": 2.621, + "theoretical_loss": 3.699674400267035, + "tokens_seen": 867203072 + }, + { + "epoch": 2.08, + "learning_rate": 0.00037233701103309933, + "loss": 2.9424, + "theoretical_loss": 3.699647088674364, + "tokens_seen": 867268608 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003723269809428285, + "loss": 2.6496, + "theoretical_loss": 3.699619779723264, + "tokens_seen": 867334144 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003723169508525577, + "loss": 2.519, + "theoretical_loss": 3.69959247341328, + "tokens_seen": 867399680 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003723069207622869, + "loss": 2.5122, + "theoretical_loss": 3.6995651697439573, + "tokens_seen": 867465216 + }, + { + "epoch": 2.08, + "learning_rate": 0.00037229689067201605, + "loss": 2.5631, + "theoretical_loss": 3.699537868714841, + "tokens_seen": 867530752 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003722868605817453, + "loss": 2.5265, + "theoretical_loss": 3.699510570325475, + "tokens_seen": 867596288 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003722768304914744, + "loss": 2.7926, + "theoretical_loss": 3.699483274575407, + "tokens_seen": 867661824 + }, + { + "epoch": 2.08, + "learning_rate": 0.00037226680040120365, + "loss": 2.9379, + "theoretical_loss": 3.6994559814641814, + "tokens_seen": 867727360 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003722567703109328, + "loss": 2.8544, + "theoretical_loss": 3.6994286909913434, + "tokens_seen": 867792896 + }, + { + "epoch": 2.08, + "learning_rate": 0.000372246740220662, + "loss": 2.4664, + "theoretical_loss": 3.699401403156439, + "tokens_seen": 867858432 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003722367101303912, + "loss": 2.628, + "theoretical_loss": 3.6993741179590143, + "tokens_seen": 867923968 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003722266800401204, + "loss": 2.5972, + "theoretical_loss": 3.6993468353986154, + "tokens_seen": 867989504 + }, + { + "epoch": 2.08, + "learning_rate": 0.00037221664994984956, + "loss": 2.7708, + "theoretical_loss": 3.699319555474788, + "tokens_seen": 868055040 + }, + { + "epoch": 2.08, + "learning_rate": 0.00037220661985957874, + "loss": 2.5467, + "theoretical_loss": 3.6992922781870776, + "tokens_seen": 868120576 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003721965897693079, + "loss": 2.7708, + "theoretical_loss": 3.6992650035350323, + "tokens_seen": 868186112 + }, + { + "epoch": 2.08, + "learning_rate": 0.00037218655967903716, + "loss": 2.4194, + "theoretical_loss": 3.699237731518197, + "tokens_seen": 868251648 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003721765295887663, + "loss": 2.4483, + "theoretical_loss": 3.6992104621361186, + "tokens_seen": 868317184 + }, + { + "epoch": 2.08, + "objective/train/docs_used": 959586, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.304150342941284, + "objective/train/theoretical_loss": 3.6991968284329717, + "objective/train/tokens_used": 888809952, + "theoretical_loss": 3.6991968284329717, + "tokens_seen": 868349952 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003721664994984955, + "loss": 2.646, + "theoretical_loss": 3.699183195388344, + "tokens_seen": 868382720 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003721564694082247, + "loss": 2.6881, + "theoretical_loss": 3.6991559312744196, + "tokens_seen": 868448256 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003721464393179539, + "loss": 2.7205, + "theoretical_loss": 3.6991286697938923, + "tokens_seen": 868513792 + }, + { + "epoch": 2.08, + "learning_rate": 0.00037213640922768306, + "loss": 2.9575, + "theoretical_loss": 3.6991014109463096, + "tokens_seen": 868579328 + }, + { + "epoch": 2.08, + "learning_rate": 0.00037212637913741224, + "loss": 2.7497, + "theoretical_loss": 3.6990741547312176, + "tokens_seen": 868644864 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003721163490471414, + "loss": 2.6964, + "theoretical_loss": 3.699046901148164, + "tokens_seen": 868710400 + }, + { + "epoch": 2.08, + "learning_rate": 0.00037210631895687066, + "loss": 2.8692, + "theoretical_loss": 3.6990196501966963, + "tokens_seen": 868775936 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003720962888665998, + "loss": 2.6819, + "theoretical_loss": 3.6989924018763616, + "tokens_seen": 868841472 + }, + { + "epoch": 2.08, + "learning_rate": 0.000372086258776329, + "loss": 2.6184, + "theoretical_loss": 3.6989651561867074, + "tokens_seen": 868907008 + }, + { + "epoch": 2.08, + "learning_rate": 0.00037207622868605815, + "loss": 2.6148, + "theoretical_loss": 3.698937913127282, + "tokens_seen": 868972544 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003720661985957874, + "loss": 2.5731, + "theoretical_loss": 3.6989106726976324, + "tokens_seen": 869038080 + }, + { + "epoch": 2.08, + "learning_rate": 0.00037205616850551656, + "loss": 2.8067, + "theoretical_loss": 3.6988834348973065, + "tokens_seen": 869103616 + }, + { + "epoch": 2.08, + "learning_rate": 0.00037204613841524575, + "loss": 2.7593, + "theoretical_loss": 3.698856199725853, + "tokens_seen": 869169152 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003720361083249749, + "loss": 2.609, + "theoretical_loss": 3.698828967182819, + "tokens_seen": 869234688 + }, + { + "epoch": 2.08, + "learning_rate": 0.00037202607823470416, + "loss": 2.7118, + "theoretical_loss": 3.698801737267753, + "tokens_seen": 869300224 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003720160481444333, + "loss": 2.7569, + "theoretical_loss": 3.698774509980203, + "tokens_seen": 869365760 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003720060180541625, + "loss": 2.7569, + "theoretical_loss": 3.698747285319719, + "tokens_seen": 869431296 + }, + { + "epoch": 2.08, + "learning_rate": 0.00037199598796389165, + "loss": 2.7691, + "theoretical_loss": 3.6987200632858475, + "tokens_seen": 869496832 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003719859578736209, + "loss": 2.5376, + "theoretical_loss": 3.698692843878139, + "tokens_seen": 869562368 + }, + { + "epoch": 2.08, + "learning_rate": 0.00037197592778335007, + "loss": 2.8628, + "theoretical_loss": 3.6986656270961404, + "tokens_seen": 869627904 + }, + { + "epoch": 2.08, + "learning_rate": 0.00037196589769307925, + "loss": 2.5259, + "theoretical_loss": 3.6986384129394017, + "tokens_seen": 869693440 + }, + { + "epoch": 2.08, + "learning_rate": 0.00037195586760280843, + "loss": 2.6141, + "theoretical_loss": 3.6986112014074717, + "tokens_seen": 869758976 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003719458375125376, + "loss": 2.789, + "theoretical_loss": 3.6985839924998993, + "tokens_seen": 869824512 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003719358074222668, + "loss": 2.7556, + "theoretical_loss": 3.698556786216234, + "tokens_seen": 869890048 + }, + { + "epoch": 2.08, + "learning_rate": 0.000371925777331996, + "loss": 2.6361, + "theoretical_loss": 3.6985295825560245, + "tokens_seen": 869955584 + }, + { + "epoch": 2.08, + "objective/train/docs_used": 961818, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.836449146270752, + "objective/train/theoretical_loss": 3.6985159817095754, + "objective/train/tokens_used": 890267104, + "theoretical_loss": 3.6985159817095754, + "tokens_seen": 869988352 + }, + { + "epoch": 2.08, + "learning_rate": 0.00037191574724172515, + "loss": 2.8748, + "theoretical_loss": 3.698502381518821, + "tokens_seen": 870021120 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003719057171514544, + "loss": 2.8179, + "theoretical_loss": 3.6984751831041724, + "tokens_seen": 870086656 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003718956870611835, + "loss": 2.6425, + "theoretical_loss": 3.6984479873116287, + "tokens_seen": 870152192 + }, + { + "epoch": 2.08, + "learning_rate": 0.00037188565697091275, + "loss": 2.6881, + "theoretical_loss": 3.6984207941407394, + "tokens_seen": 870217728 + }, + { + "epoch": 2.08, + "learning_rate": 0.00037187562688064193, + "loss": 2.9222, + "theoretical_loss": 3.698393603591055, + "tokens_seen": 870283264 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003718655967903711, + "loss": 2.7699, + "theoretical_loss": 3.698366415662125, + "tokens_seen": 870348800 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003718555667001003, + "loss": 2.8366, + "theoretical_loss": 3.698339230353499, + "tokens_seen": 870414336 + }, + { + "epoch": 2.08, + "learning_rate": 0.00037184553660982953, + "loss": 2.8265, + "theoretical_loss": 3.698312047664728, + "tokens_seen": 870479872 + }, + { + "epoch": 2.08, + "learning_rate": 0.00037183550651955866, + "loss": 2.7096, + "theoretical_loss": 3.6982848675953623, + "tokens_seen": 870545408 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003718254764292879, + "loss": 2.8957, + "theoretical_loss": 3.6982576901449518, + "tokens_seen": 870610944 + }, + { + "epoch": 2.08, + "learning_rate": 0.000371815446339017, + "loss": 2.8406, + "theoretical_loss": 3.698230515313047, + "tokens_seen": 870676480 + }, + { + "epoch": 2.08, + "learning_rate": 0.00037180541624874625, + "loss": 2.5983, + "theoretical_loss": 3.698203343099199, + "tokens_seen": 870742016 + }, + { + "epoch": 2.08, + "learning_rate": 0.00037179538615847544, + "loss": 2.5483, + "theoretical_loss": 3.698176173502959, + "tokens_seen": 870807552 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003717853560682046, + "loss": 2.755, + "theoretical_loss": 3.698149006523876, + "tokens_seen": 870873088 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003717753259779338, + "loss": 2.6726, + "theoretical_loss": 3.6981218421615027, + "tokens_seen": 870938624 + }, + { + "epoch": 2.08, + "learning_rate": 0.000371765295887663, + "loss": 2.6759, + "theoretical_loss": 3.69809468041539, + "tokens_seen": 871004160 + }, + { + "epoch": 2.08, + "learning_rate": 0.00037175526579739216, + "loss": 2.8533, + "theoretical_loss": 3.698067521285089, + "tokens_seen": 871069696 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003717452357071214, + "loss": 2.6255, + "theoretical_loss": 3.6980403647701503, + "tokens_seen": 871135232 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003717352056168505, + "loss": 2.589, + "theoretical_loss": 3.698013210870126, + "tokens_seen": 871200768 + }, + { + "epoch": 2.08, + "learning_rate": 0.00037172517552657976, + "loss": 2.7553, + "theoretical_loss": 3.6979860595845673, + "tokens_seen": 871266304 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003717151454363089, + "loss": 2.3292, + "theoretical_loss": 3.6979589109130258, + "tokens_seen": 871331840 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003717051153460381, + "loss": 2.6952, + "theoretical_loss": 3.697931764855054, + "tokens_seen": 871397376 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003716950852557673, + "loss": 3.0049, + "theoretical_loss": 3.697904621410203, + "tokens_seen": 871462912 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003716850551654965, + "loss": 2.7278, + "theoretical_loss": 3.697877480578025, + "tokens_seen": 871528448 + }, + { + "epoch": 2.08, + "learning_rate": 0.00037167502507522566, + "loss": 2.7773, + "theoretical_loss": 3.6978503423580715, + "tokens_seen": 871593984 + }, + { + "epoch": 2.08, + "objective/train/docs_used": 961818, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4541244506835938, + "objective/train/theoretical_loss": 3.69783677422754, + "objective/train/tokens_used": 890267104, + "theoretical_loss": 3.69783677422754, + "tokens_seen": 871626752 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003716649949849549, + "loss": 2.7079, + "theoretical_loss": 3.697823206749896, + "tokens_seen": 871659520 + }, + { + "epoch": 2.08, + "learning_rate": 0.000371654964894684, + "loss": 2.6983, + "theoretical_loss": 3.69779607375305, + "tokens_seen": 871725056 + }, + { + "epoch": 2.08, + "learning_rate": 0.00037164493480441326, + "loss": 2.6245, + "theoretical_loss": 3.6977689433670857, + "tokens_seen": 871790592 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003716349047141424, + "loss": 2.4994, + "theoretical_loss": 3.6977418155915562, + "tokens_seen": 871856128 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003716248746238716, + "loss": 2.4224, + "theoretical_loss": 3.6977146904260136, + "tokens_seen": 871921664 + }, + { + "epoch": 2.08, + "learning_rate": 0.00037161484453360086, + "loss": 2.647, + "theoretical_loss": 3.6976875678700107, + "tokens_seen": 871987200 + }, + { + "epoch": 2.08, + "learning_rate": 0.00037160481444333, + "loss": 2.5049, + "theoretical_loss": 3.697660447923101, + "tokens_seen": 872052736 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003715947843530592, + "loss": 3.0161, + "theoretical_loss": 3.6976333305848366, + "tokens_seen": 872118272 + }, + { + "epoch": 2.08, + "learning_rate": 0.00037158475426278835, + "loss": 2.8686, + "theoretical_loss": 3.6976062158547713, + "tokens_seen": 872183808 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003715747241725176, + "loss": 2.3929, + "theoretical_loss": 3.697579103732458, + "tokens_seen": 872249344 + }, + { + "epoch": 2.08, + "learning_rate": 0.00037156469408224676, + "loss": 2.6487, + "theoretical_loss": 3.6975519942174495, + "tokens_seen": 872314880 + }, + { + "epoch": 2.08, + "learning_rate": 0.00037155466399197595, + "loss": 2.5502, + "theoretical_loss": 3.6975248873093003, + "tokens_seen": 872380416 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003715446339017051, + "loss": 2.8222, + "theoretical_loss": 3.6974977830075626, + "tokens_seen": 872445952 + }, + { + "epoch": 2.08, + "learning_rate": 0.00037153460381143436, + "loss": 2.4791, + "theoretical_loss": 3.697470681311791, + "tokens_seen": 872511488 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003715245737211635, + "loss": 2.5693, + "theoretical_loss": 3.6974435822215392, + "tokens_seen": 872577024 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003715145436308927, + "loss": 2.6591, + "theoretical_loss": 3.6974164857363605, + "tokens_seen": 872642560 + }, + { + "epoch": 2.08, + "learning_rate": 0.00037150451354062185, + "loss": 2.5865, + "theoretical_loss": 3.6973893918558094, + "tokens_seen": 872708096 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003714944834503511, + "loss": 2.414, + "theoretical_loss": 3.6973623005794396, + "tokens_seen": 872773632 + }, + { + "epoch": 2.08, + "learning_rate": 0.00037148445336008027, + "loss": 2.6693, + "theoretical_loss": 3.6973352119068053, + "tokens_seen": 872839168 + }, + { + "epoch": 2.08, + "learning_rate": 0.00037147442326980945, + "loss": 2.7309, + "theoretical_loss": 3.697308125837461, + "tokens_seen": 872904704 + }, + { + "epoch": 2.08, + "learning_rate": 0.00037146439317953863, + "loss": 2.6199, + "theoretical_loss": 3.6972810423709603, + "tokens_seen": 872970240 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003714543630892678, + "loss": 2.5813, + "theoretical_loss": 3.6972539615068594, + "tokens_seen": 873035776 + }, + { + "epoch": 2.08, + "learning_rate": 0.000371444332998997, + "loss": 2.5225, + "theoretical_loss": 3.6972268832447117, + "tokens_seen": 873101312 + }, + { + "epoch": 2.08, + "learning_rate": 0.00037143430290872623, + "loss": 2.7103, + "theoretical_loss": 3.6971998075840715, + "tokens_seen": 873166848 + }, + { + "epoch": 2.08, + "learning_rate": 0.00037142427281845535, + "loss": 2.4854, + "theoretical_loss": 3.6971727345244947, + "tokens_seen": 873232384 + }, + { + "epoch": 2.08, + "objective/train/docs_used": 961818, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5809988975524902, + "objective/train/theoretical_loss": 3.697159198969966, + "objective/train/tokens_used": 890267104, + "theoretical_loss": 3.697159198969966, + "tokens_seen": 873265152 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003714142427281846, + "loss": 2.58, + "theoretical_loss": 3.697145664065536, + "tokens_seen": 873297920 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003714042126379137, + "loss": 2.6467, + "theoretical_loss": 3.69711859620675, + "tokens_seen": 873363456 + }, + { + "epoch": 2.08, + "learning_rate": 0.00037139418254764295, + "loss": 2.4065, + "theoretical_loss": 3.6970915309476915, + "tokens_seen": 873428992 + }, + { + "epoch": 2.08, + "learning_rate": 0.00037138415245737213, + "loss": 2.497, + "theoretical_loss": 3.6970644682879175, + "tokens_seen": 873494528 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003713741223671013, + "loss": 2.7433, + "theoretical_loss": 3.6970374082269815, + "tokens_seen": 873560064 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003713640922768305, + "loss": 2.782, + "theoretical_loss": 3.6970103507644403, + "tokens_seen": 873625600 + }, + { + "epoch": 2.08, + "learning_rate": 0.00037135406218655973, + "loss": 2.5894, + "theoretical_loss": 3.6969832958998485, + "tokens_seen": 873691136 + }, + { + "epoch": 2.08, + "learning_rate": 0.00037134403209628886, + "loss": 2.7581, + "theoretical_loss": 3.6969562436327625, + "tokens_seen": 873756672 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003713340020060181, + "loss": 2.5189, + "theoretical_loss": 3.696929193962738, + "tokens_seen": 873822208 + }, + { + "epoch": 2.08, + "learning_rate": 0.0003713239719157472, + "loss": 2.8104, + "theoretical_loss": 3.6969021468893306, + "tokens_seen": 873887744 + }, + { + "epoch": 2.08, + "learning_rate": 0.00037131394182547645, + "loss": 2.5091, + "theoretical_loss": 3.6968751024120965, + "tokens_seen": 873953280 + }, + { + "epoch": 2.08, + "learning_rate": 0.00037130391173520564, + "loss": 2.8142, + "theoretical_loss": 3.696848060530592, + "tokens_seen": 874018816 + }, + { + "epoch": 2.09, + "learning_rate": 0.0003712938816449348, + "loss": 2.9372, + "theoretical_loss": 3.6968210212443733, + "tokens_seen": 874084352 + }, + { + "epoch": 2.09, + "learning_rate": 0.000371283851554664, + "loss": 2.5234, + "theoretical_loss": 3.6967939845529965, + "tokens_seen": 874149888 + }, + { + "epoch": 2.09, + "learning_rate": 0.0003712738214643932, + "loss": 2.489, + "theoretical_loss": 3.696766950456018, + "tokens_seen": 874215424 + }, + { + "epoch": 2.09, + "learning_rate": 0.00037126379137412236, + "loss": 2.6471, + "theoretical_loss": 3.6967399189529955, + "tokens_seen": 874280960 + }, + { + "epoch": 2.09, + "learning_rate": 0.0003712537612838516, + "loss": 2.6633, + "theoretical_loss": 3.696712890043485, + "tokens_seen": 874346496 + }, + { + "epoch": 2.09, + "learning_rate": 0.0003712437311935807, + "loss": 2.7678, + "theoretical_loss": 3.6966858637270423, + "tokens_seen": 874412032 + }, + { + "epoch": 2.09, + "learning_rate": 0.00037123370110330996, + "loss": 2.727, + "theoretical_loss": 3.6966588400032254, + "tokens_seen": 874477568 + }, + { + "epoch": 2.09, + "learning_rate": 0.0003712236710130391, + "loss": 2.9024, + "theoretical_loss": 3.6966318188715914, + "tokens_seen": 874543104 + }, + { + "epoch": 2.09, + "learning_rate": 0.0003712136409227683, + "loss": 2.6938, + "theoretical_loss": 3.696604800331697, + "tokens_seen": 874608640 + }, + { + "epoch": 2.09, + "learning_rate": 0.0003712036108324975, + "loss": 2.7446, + "theoretical_loss": 3.6965777843830994, + "tokens_seen": 874674176 + }, + { + "epoch": 2.09, + "learning_rate": 0.0003711935807422267, + "loss": 2.8905, + "theoretical_loss": 3.6965507710253562, + "tokens_seen": 874739712 + }, + { + "epoch": 2.09, + "learning_rate": 0.00037118355065195586, + "loss": 2.575, + "theoretical_loss": 3.696523760258025, + "tokens_seen": 874805248 + }, + { + "epoch": 2.09, + "learning_rate": 0.0003711735205616851, + "loss": 2.6711, + "theoretical_loss": 3.696496752080663, + "tokens_seen": 874870784 + }, + { + "epoch": 2.09, + "objective/train/docs_used": 961818, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8190174102783203, + "objective/train/theoretical_loss": 3.696483248963082, + "objective/train/tokens_used": 890267104, + "theoretical_loss": 3.696483248963082, + "tokens_seen": 874903552 + }, + { + "epoch": 2.09, + "learning_rate": 0.0003711634904714142, + "loss": 2.9269, + "theoretical_loss": 3.696469746492828, + "tokens_seen": 874936320 + }, + { + "epoch": 2.09, + "learning_rate": 0.00037115346038114346, + "loss": 2.7968, + "theoretical_loss": 3.6964427434940776, + "tokens_seen": 875001856 + }, + { + "epoch": 2.09, + "learning_rate": 0.0003711434302908726, + "loss": 2.6184, + "theoretical_loss": 3.6964157430839704, + "tokens_seen": 875067392 + }, + { + "epoch": 2.09, + "learning_rate": 0.0003711334002006018, + "loss": 2.5877, + "theoretical_loss": 3.6963887452620634, + "tokens_seen": 875132928 + }, + { + "epoch": 2.09, + "learning_rate": 0.000371123370110331, + "loss": 2.4639, + "theoretical_loss": 3.696361750027915, + "tokens_seen": 875198464 + }, + { + "epoch": 2.09, + "learning_rate": 0.0003711133400200602, + "loss": 2.9735, + "theoretical_loss": 3.696334757381084, + "tokens_seen": 875264000 + }, + { + "epoch": 2.09, + "learning_rate": 0.00037110330992978937, + "loss": 2.6998, + "theoretical_loss": 3.696307767321128, + "tokens_seen": 875329536 + }, + { + "epoch": 2.09, + "learning_rate": 0.00037109327983951855, + "loss": 2.6655, + "theoretical_loss": 3.696280779847606, + "tokens_seen": 875395072 + }, + { + "epoch": 2.09, + "learning_rate": 0.00037108324974924773, + "loss": 2.713, + "theoretical_loss": 3.696253794960076, + "tokens_seen": 875460608 + }, + { + "epoch": 2.09, + "learning_rate": 0.00037107321965897696, + "loss": 2.5791, + "theoretical_loss": 3.696226812658097, + "tokens_seen": 875526144 + }, + { + "epoch": 2.09, + "learning_rate": 0.0003710631895687061, + "loss": 2.7312, + "theoretical_loss": 3.696199832941228, + "tokens_seen": 875591680 + }, + { + "epoch": 2.09, + "learning_rate": 0.0003710531594784353, + "loss": 2.9171, + "theoretical_loss": 3.696172855809027, + "tokens_seen": 875657216 + }, + { + "epoch": 2.09, + "learning_rate": 0.00037104312938816445, + "loss": 2.7087, + "theoretical_loss": 3.696145881261054, + "tokens_seen": 875722752 + }, + { + "epoch": 2.09, + "learning_rate": 0.0003710330992978937, + "loss": 2.726, + "theoretical_loss": 3.6961189092968674, + "tokens_seen": 875788288 + }, + { + "epoch": 2.09, + "learning_rate": 0.00037102306920762287, + "loss": 2.5598, + "theoretical_loss": 3.696091939916027, + "tokens_seen": 875853824 + }, + { + "epoch": 2.09, + "learning_rate": 0.00037101303911735205, + "loss": 2.6229, + "theoretical_loss": 3.696064973118091, + "tokens_seen": 875919360 + }, + { + "epoch": 2.09, + "learning_rate": 0.00037100300902708123, + "loss": 2.7326, + "theoretical_loss": 3.6960380089026197, + "tokens_seen": 875984896 + }, + { + "epoch": 2.09, + "learning_rate": 0.00037099297893681047, + "loss": 2.5333, + "theoretical_loss": 3.696011047269173, + "tokens_seen": 876050432 + }, + { + "epoch": 2.09, + "learning_rate": 0.0003709829488465396, + "loss": 2.4307, + "theoretical_loss": 3.695984088217309, + "tokens_seen": 876115968 + }, + { + "epoch": 2.09, + "learning_rate": 0.00037097291875626883, + "loss": 2.5423, + "theoretical_loss": 3.695957131746589, + "tokens_seen": 876181504 + }, + { + "epoch": 2.09, + "learning_rate": 0.00037096288866599796, + "loss": 2.8121, + "theoretical_loss": 3.695930177856572, + "tokens_seen": 876247040 + }, + { + "epoch": 2.09, + "learning_rate": 0.0003709528585757272, + "loss": 2.4569, + "theoretical_loss": 3.695903226546818, + "tokens_seen": 876312576 + }, + { + "epoch": 2.09, + "learning_rate": 0.0003709428284854564, + "loss": 2.7681, + "theoretical_loss": 3.695876277816888, + "tokens_seen": 876378112 + }, + { + "epoch": 2.09, + "learning_rate": 0.00037093279839518555, + "loss": 2.6246, + "theoretical_loss": 3.6958493316663406, + "tokens_seen": 876443648 + }, + { + "epoch": 2.09, + "learning_rate": 0.00037092276830491474, + "loss": 2.5519, + "theoretical_loss": 3.6958223880947374, + "tokens_seen": 876509184 + }, + { + "epoch": 2.09, + "objective/train/docs_used": 961818, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.962693691253662, + "objective/train/theoretical_loss": 3.695808917275902, + "objective/train/tokens_used": 890267104, + "theoretical_loss": 3.695808917275902, + "tokens_seen": 876541952 + }, + { + "epoch": 2.09, + "learning_rate": 0.0003709127382146439, + "loss": 2.7102, + "theoretical_loss": 3.695795447101638, + "tokens_seen": 876574720 + }, + { + "epoch": 2.09, + "learning_rate": 0.0003709027081243731, + "loss": 2.8035, + "theoretical_loss": 3.695768508686603, + "tokens_seen": 876640256 + }, + { + "epoch": 2.09, + "learning_rate": 0.00037089267803410233, + "loss": 2.7445, + "theoretical_loss": 3.695741572849193, + "tokens_seen": 876705792 + }, + { + "epoch": 2.09, + "learning_rate": 0.00037088264794383146, + "loss": 2.8925, + "theoretical_loss": 3.695714639588969, + "tokens_seen": 876771328 + }, + { + "epoch": 2.09, + "learning_rate": 0.0003708726178535607, + "loss": 2.7924, + "theoretical_loss": 3.695687708905491, + "tokens_seen": 876836864 + }, + { + "epoch": 2.09, + "learning_rate": 0.00037086258776328993, + "loss": 2.5923, + "theoretical_loss": 3.6956607807983213, + "tokens_seen": 876902400 + }, + { + "epoch": 2.09, + "learning_rate": 0.00037085255767301906, + "loss": 2.6051, + "theoretical_loss": 3.69563385526702, + "tokens_seen": 876967936 + }, + { + "epoch": 2.09, + "learning_rate": 0.0003708425275827483, + "loss": 2.7864, + "theoretical_loss": 3.695606932311148, + "tokens_seen": 877033472 + }, + { + "epoch": 2.09, + "learning_rate": 0.0003708324974924774, + "loss": 2.9097, + "theoretical_loss": 3.695580011930267, + "tokens_seen": 877099008 + }, + { + "epoch": 2.09, + "learning_rate": 0.00037082246740220665, + "loss": 2.8712, + "theoretical_loss": 3.6955530941239387, + "tokens_seen": 877164544 + }, + { + "epoch": 2.09, + "learning_rate": 0.00037081243731193584, + "loss": 2.7235, + "theoretical_loss": 3.6955261788917237, + "tokens_seen": 877230080 + }, + { + "epoch": 2.09, + "learning_rate": 0.000370802407221665, + "loss": 2.7495, + "theoretical_loss": 3.6954992662331847, + "tokens_seen": 877295616 + }, + { + "epoch": 2.09, + "learning_rate": 0.0003707923771313942, + "loss": 2.7919, + "theoretical_loss": 3.695472356147882, + "tokens_seen": 877361152 + }, + { + "epoch": 2.09, + "learning_rate": 0.0003707823470411234, + "loss": 2.5835, + "theoretical_loss": 3.695445448635378, + "tokens_seen": 877426688 + }, + { + "epoch": 2.09, + "learning_rate": 0.00037077231695085256, + "loss": 2.6822, + "theoretical_loss": 3.6954185436952347, + "tokens_seen": 877492224 + }, + { + "epoch": 2.09, + "learning_rate": 0.0003707622868605818, + "loss": 2.658, + "theoretical_loss": 3.6953916413270145, + "tokens_seen": 877557760 + }, + { + "epoch": 2.09, + "learning_rate": 0.0003707522567703109, + "loss": 2.4346, + "theoretical_loss": 3.6953647415302786, + "tokens_seen": 877623296 + }, + { + "epoch": 2.09, + "learning_rate": 0.00037074222668004016, + "loss": 2.8774, + "theoretical_loss": 3.6953378443045892, + "tokens_seen": 877688832 + }, + { + "epoch": 2.09, + "learning_rate": 0.0003707321965897693, + "loss": 2.6141, + "theoretical_loss": 3.6953109496495093, + "tokens_seen": 877754368 + }, + { + "epoch": 2.09, + "learning_rate": 0.0003707221664994985, + "loss": 2.683, + "theoretical_loss": 3.695284057564601, + "tokens_seen": 877819904 + }, + { + "epoch": 2.09, + "learning_rate": 0.0003707121364092277, + "loss": 2.7027, + "theoretical_loss": 3.6952571680494275, + "tokens_seen": 877885440 + }, + { + "epoch": 2.09, + "learning_rate": 0.0003707021063189569, + "loss": 2.6527, + "theoretical_loss": 3.6952302811035502, + "tokens_seen": 877950976 + }, + { + "epoch": 2.09, + "learning_rate": 0.00037069207622868606, + "loss": 2.593, + "theoretical_loss": 3.6952033967265328, + "tokens_seen": 878016512 + }, + { + "epoch": 2.09, + "learning_rate": 0.0003706820461384153, + "loss": 2.7754, + "theoretical_loss": 3.6951765149179376, + "tokens_seen": 878082048 + }, + { + "epoch": 2.09, + "learning_rate": 0.0003706720160481444, + "loss": 2.7959, + "theoretical_loss": 3.6951496356773275, + "tokens_seen": 878147584 + }, + { + "epoch": 2.09, + "objective/train/docs_used": 961818, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8952300548553467, + "objective/train/theoretical_loss": 3.6951361970198806, + "objective/train/tokens_used": 890267104, + "theoretical_loss": 3.6951361970198806, + "tokens_seen": 878180352 + }, + { + "epoch": 2.09, + "learning_rate": 0.00037066198595787366, + "loss": 2.6569, + "theoretical_loss": 3.695122759004266, + "tokens_seen": 878213120 + }, + { + "epoch": 2.09, + "learning_rate": 0.0003706519558676028, + "loss": 2.885, + "theoretical_loss": 3.695095884898316, + "tokens_seen": 878278656 + }, + { + "epoch": 2.09, + "learning_rate": 0.000370641925777332, + "loss": 2.7, + "theoretical_loss": 3.695069013359041, + "tokens_seen": 878344192 + }, + { + "epoch": 2.09, + "learning_rate": 0.0003706318956870612, + "loss": 2.8532, + "theoretical_loss": 3.6950421443860044, + "tokens_seen": 878409728 + }, + { + "epoch": 2.09, + "learning_rate": 0.0003706218655967904, + "loss": 2.7147, + "theoretical_loss": 3.6950152779787695, + "tokens_seen": 878475264 + }, + { + "epoch": 2.09, + "learning_rate": 0.00037061183550651957, + "loss": 2.5623, + "theoretical_loss": 3.6949884141369003, + "tokens_seen": 878540800 + }, + { + "epoch": 2.09, + "learning_rate": 0.00037060180541624875, + "loss": 2.584, + "theoretical_loss": 3.6949615528599593, + "tokens_seen": 878606336 + }, + { + "epoch": 2.09, + "learning_rate": 0.00037059177532597793, + "loss": 2.778, + "theoretical_loss": 3.694934694147512, + "tokens_seen": 878671872 + }, + { + "epoch": 2.09, + "learning_rate": 0.00037058174523570716, + "loss": 2.8953, + "theoretical_loss": 3.6949078379991205, + "tokens_seen": 878737408 + }, + { + "epoch": 2.09, + "learning_rate": 0.0003705717151454363, + "loss": 2.7651, + "theoretical_loss": 3.6948809844143504, + "tokens_seen": 878802944 + }, + { + "epoch": 2.09, + "learning_rate": 0.0003705616850551655, + "loss": 2.6819, + "theoretical_loss": 3.694854133392765, + "tokens_seen": 878868480 + }, + { + "epoch": 2.09, + "learning_rate": 0.00037055165496489465, + "loss": 2.6763, + "theoretical_loss": 3.694827284933929, + "tokens_seen": 878934016 + }, + { + "epoch": 2.09, + "learning_rate": 0.0003705416248746239, + "loss": 2.4986, + "theoretical_loss": 3.6948004390374063, + "tokens_seen": 878999552 + }, + { + "epoch": 2.09, + "learning_rate": 0.00037053159478435307, + "loss": 2.6861, + "theoretical_loss": 3.6947735957027614, + "tokens_seen": 879065088 + }, + { + "epoch": 2.09, + "learning_rate": 0.00037052156469408225, + "loss": 2.6115, + "theoretical_loss": 3.694746754929559, + "tokens_seen": 879130624 + }, + { + "epoch": 2.09, + "learning_rate": 0.00037051153460381143, + "loss": 2.5333, + "theoretical_loss": 3.6947199167173643, + "tokens_seen": 879196160 + }, + { + "epoch": 2.09, + "learning_rate": 0.00037050150451354067, + "loss": 2.8628, + "theoretical_loss": 3.6946930810657412, + "tokens_seen": 879261696 + }, + { + "epoch": 2.09, + "learning_rate": 0.0003704914744232698, + "loss": 2.8541, + "theoretical_loss": 3.694666247974255, + "tokens_seen": 879327232 + }, + { + "epoch": 2.09, + "learning_rate": 0.00037048144433299903, + "loss": 2.6677, + "theoretical_loss": 3.6946394174424704, + "tokens_seen": 879392768 + }, + { + "epoch": 2.09, + "learning_rate": 0.00037047141424272816, + "loss": 2.5643, + "theoretical_loss": 3.694612589469953, + "tokens_seen": 879458304 + }, + { + "epoch": 2.09, + "learning_rate": 0.0003704613841524574, + "loss": 2.6471, + "theoretical_loss": 3.6945857640562676, + "tokens_seen": 879523840 + }, + { + "epoch": 2.09, + "learning_rate": 0.0003704513540621866, + "loss": 2.5586, + "theoretical_loss": 3.6945589412009796, + "tokens_seen": 879589376 + }, + { + "epoch": 2.09, + "learning_rate": 0.00037044132397191575, + "loss": 2.6428, + "theoretical_loss": 3.694532120903654, + "tokens_seen": 879654912 + }, + { + "epoch": 2.09, + "learning_rate": 0.00037043129388164494, + "loss": 2.6342, + "theoretical_loss": 3.694505303163857, + "tokens_seen": 879720448 + }, + { + "epoch": 2.09, + "learning_rate": 0.0003704212637913741, + "loss": 2.6444, + "theoretical_loss": 3.6944784879811543, + "tokens_seen": 879785984 + }, + { + "epoch": 2.09, + "objective/train/docs_used": 961818, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.845834732055664, + "objective/train/theoretical_loss": 3.6944650813485773, + "objective/train/tokens_used": 890267104, + "theoretical_loss": 3.6944650813485773, + "tokens_seen": 879818752 + }, + { + "epoch": 2.09, + "learning_rate": 0.0003704112337011033, + "loss": 2.6323, + "theoretical_loss": 3.694451675355111, + "tokens_seen": 879851520 + }, + { + "epoch": 2.09, + "learning_rate": 0.00037040120361083253, + "loss": 2.6846, + "theoretical_loss": 3.694424865285293, + "tokens_seen": 879917056 + }, + { + "epoch": 2.09, + "learning_rate": 0.00037039117352056166, + "loss": 2.2876, + "theoretical_loss": 3.6943980577712665, + "tokens_seen": 879982592 + }, + { + "epoch": 2.09, + "learning_rate": 0.0003703811434302909, + "loss": 2.8253, + "theoretical_loss": 3.694371252812597, + "tokens_seen": 880048128 + }, + { + "epoch": 2.09, + "learning_rate": 0.0003703711133400201, + "loss": 2.8466, + "theoretical_loss": 3.694344450408852, + "tokens_seen": 880113664 + }, + { + "epoch": 2.09, + "learning_rate": 0.00037036108324974926, + "loss": 2.9858, + "theoretical_loss": 3.6943176505595963, + "tokens_seen": 880179200 + }, + { + "epoch": 2.09, + "learning_rate": 0.00037035105315947844, + "loss": 2.9263, + "theoretical_loss": 3.6942908532643974, + "tokens_seen": 880244736 + }, + { + "epoch": 2.09, + "learning_rate": 0.0003703410230692076, + "loss": 2.6523, + "theoretical_loss": 3.6942640585228204, + "tokens_seen": 880310272 + }, + { + "epoch": 2.09, + "learning_rate": 0.0003703309929789368, + "loss": 2.6214, + "theoretical_loss": 3.694237266334433, + "tokens_seen": 880375808 + }, + { + "epoch": 2.09, + "learning_rate": 0.00037032096288866604, + "loss": 2.4765, + "theoretical_loss": 3.694210476698802, + "tokens_seen": 880441344 + }, + { + "epoch": 2.09, + "learning_rate": 0.00037031093279839516, + "loss": 2.6534, + "theoretical_loss": 3.6941836896154934, + "tokens_seen": 880506880 + }, + { + "epoch": 2.09, + "learning_rate": 0.0003703009027081244, + "loss": 2.638, + "theoretical_loss": 3.6941569050840743, + "tokens_seen": 880572416 + }, + { + "epoch": 2.09, + "learning_rate": 0.0003702908726178535, + "loss": 2.7322, + "theoretical_loss": 3.6941301231041117, + "tokens_seen": 880637952 + }, + { + "epoch": 2.09, + "learning_rate": 0.00037028084252758276, + "loss": 2.7069, + "theoretical_loss": 3.6941033436751733, + "tokens_seen": 880703488 + }, + { + "epoch": 2.09, + "learning_rate": 0.00037027081243731194, + "loss": 2.6372, + "theoretical_loss": 3.6940765667968254, + "tokens_seen": 880769024 + }, + { + "epoch": 2.09, + "learning_rate": 0.0003702607823470411, + "loss": 2.6195, + "theoretical_loss": 3.694049792468636, + "tokens_seen": 880834560 + }, + { + "epoch": 2.09, + "learning_rate": 0.0003702507522567703, + "loss": 2.7658, + "theoretical_loss": 3.6940230206901727, + "tokens_seen": 880900096 + }, + { + "epoch": 2.09, + "learning_rate": 0.0003702407221664995, + "loss": 2.5632, + "theoretical_loss": 3.693996251461002, + "tokens_seen": 880965632 + }, + { + "epoch": 2.09, + "learning_rate": 0.00037023069207622867, + "loss": 2.4424, + "theoretical_loss": 3.693969484780692, + "tokens_seen": 881031168 + }, + { + "epoch": 2.09, + "learning_rate": 0.0003702206619859579, + "loss": 2.6033, + "theoretical_loss": 3.693942720648811, + "tokens_seen": 881096704 + }, + { + "epoch": 2.09, + "learning_rate": 0.00037021063189568703, + "loss": 2.6652, + "theoretical_loss": 3.6939159590649266, + "tokens_seen": 881162240 + }, + { + "epoch": 2.09, + "learning_rate": 0.00037020060180541626, + "loss": 2.773, + "theoretical_loss": 3.693889200028606, + "tokens_seen": 881227776 + }, + { + "epoch": 2.09, + "learning_rate": 0.00037019057171514544, + "loss": 2.6614, + "theoretical_loss": 3.6938624435394183, + "tokens_seen": 881293312 + }, + { + "epoch": 2.09, + "learning_rate": 0.0003701805416248746, + "loss": 2.4945, + "theoretical_loss": 3.6938356895969306, + "tokens_seen": 881358848 + }, + { + "epoch": 2.09, + "learning_rate": 0.0003701705115346038, + "loss": 2.6585, + "theoretical_loss": 3.6938089382007124, + "tokens_seen": 881424384 + }, + { + "epoch": 2.09, + "objective/train/docs_used": 961818, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6805853843688965, + "objective/train/theoretical_loss": 3.6937955634573187, + "objective/train/tokens_used": 890267104, + "theoretical_loss": 3.6937955634573187, + "tokens_seen": 881457152 + }, + { + "epoch": 2.09, + "learning_rate": 0.000370160481444333, + "loss": 2.7097, + "theoretical_loss": 3.6937821893503306, + "tokens_seen": 881489920 + }, + { + "epoch": 2.09, + "learning_rate": 0.00037015045135406217, + "loss": 2.8098, + "theoretical_loss": 3.6937554430453554, + "tokens_seen": 881555456 + }, + { + "epoch": 2.09, + "learning_rate": 0.0003701404212637914, + "loss": 2.4764, + "theoretical_loss": 3.6937286992853537, + "tokens_seen": 881620992 + }, + { + "epoch": 2.09, + "learning_rate": 0.00037013039117352053, + "loss": 2.767, + "theoretical_loss": 3.6937019580698953, + "tokens_seen": 881686528 + }, + { + "epoch": 2.09, + "learning_rate": 0.00037012036108324977, + "loss": 2.6104, + "theoretical_loss": 3.6936752193985485, + "tokens_seen": 881752064 + }, + { + "epoch": 2.09, + "learning_rate": 0.00037011033099297895, + "loss": 2.6776, + "theoretical_loss": 3.6936484832708825, + "tokens_seen": 881817600 + }, + { + "epoch": 2.09, + "learning_rate": 0.00037010030090270813, + "loss": 2.8724, + "theoretical_loss": 3.693621749686466, + "tokens_seen": 881883136 + }, + { + "epoch": 2.09, + "learning_rate": 0.00037009027081243736, + "loss": 2.5104, + "theoretical_loss": 3.693595018644868, + "tokens_seen": 881948672 + }, + { + "epoch": 2.09, + "learning_rate": 0.0003700802407221665, + "loss": 2.6103, + "theoretical_loss": 3.6935682901456586, + "tokens_seen": 882014208 + }, + { + "epoch": 2.09, + "learning_rate": 0.0003700702106318957, + "loss": 2.8333, + "theoretical_loss": 3.6935415641884055, + "tokens_seen": 882079744 + }, + { + "epoch": 2.09, + "learning_rate": 0.00037006018054162485, + "loss": 2.4745, + "theoretical_loss": 3.6935148407726794, + "tokens_seen": 882145280 + }, + { + "epoch": 2.09, + "learning_rate": 0.0003700501504513541, + "loss": 2.6831, + "theoretical_loss": 3.69348811989805, + "tokens_seen": 882210816 + }, + { + "epoch": 2.09, + "learning_rate": 0.00037004012036108327, + "loss": 2.5069, + "theoretical_loss": 3.693461401564086, + "tokens_seen": 882276352 + }, + { + "epoch": 2.09, + "learning_rate": 0.00037003009027081245, + "loss": 2.5407, + "theoretical_loss": 3.6934346857703577, + "tokens_seen": 882341888 + }, + { + "epoch": 2.09, + "learning_rate": 0.00037002006018054163, + "loss": 2.9309, + "theoretical_loss": 3.693407972516434, + "tokens_seen": 882407424 + }, + { + "epoch": 2.09, + "learning_rate": 0.00037001003009027087, + "loss": 2.5706, + "theoretical_loss": 3.6933812618018864, + "tokens_seen": 882472960 + }, + { + "epoch": 2.09, + "learning_rate": 0.00037, + "loss": 2.6596, + "theoretical_loss": 3.6933545536262846, + "tokens_seen": 882538496 + }, + { + "epoch": 2.09, + "learning_rate": 0.00036998996990972923, + "loss": 2.8532, + "theoretical_loss": 3.693327847989198, + "tokens_seen": 882604032 + }, + { + "epoch": 2.09, + "learning_rate": 0.00036997993981945836, + "loss": 2.6359, + "theoretical_loss": 3.6933011448901967, + "tokens_seen": 882669568 + }, + { + "epoch": 2.09, + "learning_rate": 0.0003699699097291876, + "loss": 2.7255, + "theoretical_loss": 3.693274444328852, + "tokens_seen": 882735104 + }, + { + "epoch": 2.09, + "learning_rate": 0.0003699598796389168, + "loss": 2.6571, + "theoretical_loss": 3.693247746304734, + "tokens_seen": 882800640 + }, + { + "epoch": 2.09, + "learning_rate": 0.00036994984954864595, + "loss": 2.5259, + "theoretical_loss": 3.693221050817413, + "tokens_seen": 882866176 + }, + { + "epoch": 2.09, + "learning_rate": 0.00036993981945837514, + "loss": 2.8501, + "theoretical_loss": 3.69319435786646, + "tokens_seen": 882931712 + }, + { + "epoch": 2.09, + "learning_rate": 0.0003699297893681043, + "loss": 2.466, + "theoretical_loss": 3.693167667451445, + "tokens_seen": 882997248 + }, + { + "epoch": 2.09, + "learning_rate": 0.0003699197592778335, + "loss": 2.8891, + "theoretical_loss": 3.6931409795719405, + "tokens_seen": 883062784 + }, + { + "epoch": 2.09, + "objective/train/docs_used": 961818, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.3581273555755615, + "objective/train/theoretical_loss": 3.6931276365828696, + "objective/train/tokens_used": 890267104, + "theoretical_loss": 3.6931276365828696, + "tokens_seen": 883095552 + }, + { + "epoch": 2.09, + "learning_rate": 0.00036990972918756273, + "loss": 2.6237, + "theoretical_loss": 3.6931142942275157, + "tokens_seen": 883128320 + }, + { + "epoch": 2.09, + "learning_rate": 0.00036989969909729186, + "loss": 2.4756, + "theoretical_loss": 3.6930876114177433, + "tokens_seen": 883193856 + }, + { + "epoch": 2.09, + "learning_rate": 0.0003698896690070211, + "loss": 2.5828, + "theoretical_loss": 3.6930609311421936, + "tokens_seen": 883259392 + }, + { + "epoch": 2.09, + "learning_rate": 0.0003698796389167503, + "loss": 2.7051, + "theoretical_loss": 3.6930342534004374, + "tokens_seen": 883324928 + }, + { + "epoch": 2.09, + "learning_rate": 0.00036986960882647946, + "loss": 2.4825, + "theoretical_loss": 3.693007578192047, + "tokens_seen": 883390464 + }, + { + "epoch": 2.09, + "learning_rate": 0.00036985957873620864, + "loss": 2.6811, + "theoretical_loss": 3.6929809055165936, + "tokens_seen": 883456000 + }, + { + "epoch": 2.09, + "learning_rate": 0.0003698495486459378, + "loss": 2.8016, + "theoretical_loss": 3.6929542353736493, + "tokens_seen": 883521536 + }, + { + "epoch": 2.09, + "learning_rate": 0.000369839518555667, + "loss": 2.6199, + "theoretical_loss": 3.6929275677627853, + "tokens_seen": 883587072 + }, + { + "epoch": 2.09, + "learning_rate": 0.00036982948846539624, + "loss": 2.6573, + "theoretical_loss": 3.692900902683573, + "tokens_seen": 883652608 + }, + { + "epoch": 2.09, + "learning_rate": 0.00036981945837512536, + "loss": 2.7305, + "theoretical_loss": 3.692874240135585, + "tokens_seen": 883718144 + }, + { + "epoch": 2.09, + "learning_rate": 0.0003698094282848546, + "loss": 2.7184, + "theoretical_loss": 3.6928475801183933, + "tokens_seen": 883783680 + }, + { + "epoch": 2.09, + "learning_rate": 0.0003697993981945837, + "loss": 2.5299, + "theoretical_loss": 3.6928209226315696, + "tokens_seen": 883849216 + }, + { + "epoch": 2.09, + "learning_rate": 0.00036978936810431296, + "loss": 2.5705, + "theoretical_loss": 3.692794267674687, + "tokens_seen": 883914752 + }, + { + "epoch": 2.09, + "learning_rate": 0.00036977933801404214, + "loss": 2.6542, + "theoretical_loss": 3.692767615247317, + "tokens_seen": 883980288 + }, + { + "epoch": 2.09, + "learning_rate": 0.0003697693079237713, + "loss": 2.7557, + "theoretical_loss": 3.6927409653490324, + "tokens_seen": 884045824 + }, + { + "epoch": 2.09, + "learning_rate": 0.0003697592778335005, + "loss": 2.5676, + "theoretical_loss": 3.6927143179794055, + "tokens_seen": 884111360 + }, + { + "epoch": 2.09, + "learning_rate": 0.0003697492477432297, + "loss": 2.7962, + "theoretical_loss": 3.692687673138009, + "tokens_seen": 884176896 + }, + { + "epoch": 2.09, + "learning_rate": 0.00036973921765295887, + "loss": 2.475, + "theoretical_loss": 3.692661030824416, + "tokens_seen": 884242432 + }, + { + "epoch": 2.09, + "learning_rate": 0.0003697291875626881, + "loss": 2.6934, + "theoretical_loss": 3.6926343910381996, + "tokens_seen": 884307968 + }, + { + "epoch": 2.09, + "learning_rate": 0.00036971915747241723, + "loss": 2.5649, + "theoretical_loss": 3.692607753778932, + "tokens_seen": 884373504 + }, + { + "epoch": 2.09, + "learning_rate": 0.00036970912738214646, + "loss": 2.825, + "theoretical_loss": 3.692581119046186, + "tokens_seen": 884439040 + }, + { + "epoch": 2.09, + "learning_rate": 0.00036969909729187565, + "loss": 2.7717, + "theoretical_loss": 3.692554486839536, + "tokens_seen": 884504576 + }, + { + "epoch": 2.09, + "learning_rate": 0.0003696890672016048, + "loss": 2.6135, + "theoretical_loss": 3.692527857158555, + "tokens_seen": 884570112 + }, + { + "epoch": 2.09, + "learning_rate": 0.000369679037111334, + "loss": 2.6465, + "theoretical_loss": 3.692501230002815, + "tokens_seen": 884635648 + }, + { + "epoch": 2.09, + "learning_rate": 0.0003696690070210632, + "loss": 2.7335, + "theoretical_loss": 3.6924746053718915, + "tokens_seen": 884701184 + }, + { + "epoch": 2.09, + "objective/train/docs_used": 961818, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.489288568496704, + "objective/train/theoretical_loss": 3.692461294003102, + "objective/train/tokens_used": 890267104, + "theoretical_loss": 3.692461294003102, + "tokens_seen": 884733952 + }, + { + "epoch": 2.09, + "learning_rate": 0.00036965897693079237, + "loss": 2.7261, + "theoretical_loss": 3.6924479832653567, + "tokens_seen": 884766720 + }, + { + "epoch": 2.09, + "learning_rate": 0.0003696489468405216, + "loss": 2.7215, + "theoretical_loss": 3.6924213636827847, + "tokens_seen": 884832256 + }, + { + "epoch": 2.09, + "learning_rate": 0.00036963891675025073, + "loss": 2.4819, + "theoretical_loss": 3.6923947466237497, + "tokens_seen": 884897792 + }, + { + "epoch": 2.09, + "learning_rate": 0.00036962888665997997, + "loss": 2.7193, + "theoretical_loss": 3.6923681320878243, + "tokens_seen": 884963328 + }, + { + "epoch": 2.09, + "learning_rate": 0.0003696188565697091, + "loss": 2.7264, + "theoretical_loss": 3.692341520074584, + "tokens_seen": 885028864 + }, + { + "epoch": 2.09, + "learning_rate": 0.00036960882647943833, + "loss": 2.6642, + "theoretical_loss": 3.692314910583603, + "tokens_seen": 885094400 + }, + { + "epoch": 2.09, + "learning_rate": 0.0003695987963891675, + "loss": 2.7689, + "theoretical_loss": 3.692288303614454, + "tokens_seen": 885159936 + }, + { + "epoch": 2.09, + "learning_rate": 0.0003695887662988967, + "loss": 2.7526, + "theoretical_loss": 3.692261699166712, + "tokens_seen": 885225472 + }, + { + "epoch": 2.09, + "learning_rate": 0.00036957873620862587, + "loss": 2.6084, + "theoretical_loss": 3.692235097239952, + "tokens_seen": 885291008 + }, + { + "epoch": 2.09, + "learning_rate": 0.00036956870611835505, + "loss": 2.6363, + "theoretical_loss": 3.6922084978337475, + "tokens_seen": 885356544 + }, + { + "epoch": 2.09, + "learning_rate": 0.00036955867602808424, + "loss": 2.6097, + "theoretical_loss": 3.6921819009476744, + "tokens_seen": 885422080 + }, + { + "epoch": 2.09, + "learning_rate": 0.00036954864593781347, + "loss": 2.4996, + "theoretical_loss": 3.6921553065813066, + "tokens_seen": 885487616 + }, + { + "epoch": 2.09, + "learning_rate": 0.0003695386158475426, + "loss": 2.6923, + "theoretical_loss": 3.6921287147342188, + "tokens_seen": 885553152 + }, + { + "epoch": 2.09, + "learning_rate": 0.00036952858575727183, + "loss": 2.6882, + "theoretical_loss": 3.6921021254059863, + "tokens_seen": 885618688 + }, + { + "epoch": 2.09, + "learning_rate": 0.000369518555667001, + "loss": 2.5709, + "theoretical_loss": 3.6920755385961845, + "tokens_seen": 885684224 + }, + { + "epoch": 2.09, + "learning_rate": 0.0003695085255767302, + "loss": 2.6799, + "theoretical_loss": 3.6920489543043873, + "tokens_seen": 885749760 + }, + { + "epoch": 2.09, + "learning_rate": 0.0003694984954864594, + "loss": 2.7231, + "theoretical_loss": 3.692022372530171, + "tokens_seen": 885815296 + }, + { + "epoch": 2.09, + "learning_rate": 0.00036948846539618856, + "loss": 2.5862, + "theoretical_loss": 3.691995793273111, + "tokens_seen": 885880832 + }, + { + "epoch": 2.09, + "learning_rate": 0.00036947843530591774, + "loss": 2.7793, + "theoretical_loss": 3.691969216532782, + "tokens_seen": 885946368 + }, + { + "epoch": 2.09, + "learning_rate": 0.000369468405215647, + "loss": 2.566, + "theoretical_loss": 3.6919426423087605, + "tokens_seen": 886011904 + }, + { + "epoch": 2.09, + "learning_rate": 0.0003694583751253761, + "loss": 2.577, + "theoretical_loss": 3.6919160706006213, + "tokens_seen": 886077440 + }, + { + "epoch": 2.09, + "learning_rate": 0.00036944834503510534, + "loss": 2.7, + "theoretical_loss": 3.6918895014079407, + "tokens_seen": 886142976 + }, + { + "epoch": 2.09, + "learning_rate": 0.00036943831494483446, + "loss": 2.5987, + "theoretical_loss": 3.691862934730294, + "tokens_seen": 886208512 + }, + { + "epoch": 2.09, + "learning_rate": 0.0003694282848545637, + "loss": 2.4723, + "theoretical_loss": 3.691836370567257, + "tokens_seen": 886274048 + }, + { + "epoch": 2.09, + "learning_rate": 0.0003694182547642929, + "loss": 2.4132, + "theoretical_loss": 3.6918098089184075, + "tokens_seen": 886339584 + }, + { + "epoch": 2.09, + "objective/train/docs_used": 961818, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.893305540084839, + "objective/train/theoretical_loss": 3.6917965290366697, + "objective/train/tokens_used": 890267104, + "theoretical_loss": 3.6917965290366697, + "tokens_seen": 886372352 + }, + { + "epoch": 2.09, + "learning_rate": 0.00036940822467402206, + "loss": 2.7739, + "theoretical_loss": 3.6917832497833203, + "tokens_seen": 886405120 + }, + { + "epoch": 2.09, + "learning_rate": 0.00036939819458375124, + "loss": 2.7017, + "theoretical_loss": 3.691756693161571, + "tokens_seen": 886470656 + }, + { + "epoch": 2.09, + "learning_rate": 0.0003693881644934805, + "loss": 2.8098, + "theoretical_loss": 3.6917301390527375, + "tokens_seen": 886536192 + }, + { + "epoch": 2.09, + "learning_rate": 0.0003693781344032096, + "loss": 2.6014, + "theoretical_loss": 3.691703587456395, + "tokens_seen": 886601728 + }, + { + "epoch": 2.09, + "learning_rate": 0.00036936810431293884, + "loss": 2.7644, + "theoretical_loss": 3.691677038372121, + "tokens_seen": 886667264 + }, + { + "epoch": 2.09, + "learning_rate": 0.000369358074222668, + "loss": 2.6346, + "theoretical_loss": 3.691650491799492, + "tokens_seen": 886732800 + }, + { + "epoch": 2.09, + "learning_rate": 0.0003693480441323972, + "loss": 2.534, + "theoretical_loss": 3.6916239477380843, + "tokens_seen": 886798336 + }, + { + "epoch": 2.09, + "learning_rate": 0.00036933801404212644, + "loss": 2.6581, + "theoretical_loss": 3.6915974061874754, + "tokens_seen": 886863872 + }, + { + "epoch": 2.09, + "learning_rate": 0.00036932798395185556, + "loss": 2.7481, + "theoretical_loss": 3.6915708671472416, + "tokens_seen": 886929408 + }, + { + "epoch": 2.09, + "learning_rate": 0.0003693179538615848, + "loss": 2.6258, + "theoretical_loss": 3.691544330616961, + "tokens_seen": 886994944 + }, + { + "epoch": 2.09, + "learning_rate": 0.0003693079237713139, + "loss": 2.6422, + "theoretical_loss": 3.6915177965962096, + "tokens_seen": 887060480 + }, + { + "epoch": 2.09, + "learning_rate": 0.00036929789368104316, + "loss": 2.7657, + "theoretical_loss": 3.691491265084566, + "tokens_seen": 887126016 + }, + { + "epoch": 2.09, + "learning_rate": 0.00036928786359077234, + "loss": 2.5757, + "theoretical_loss": 3.691464736081606, + "tokens_seen": 887191552 + }, + { + "epoch": 2.09, + "learning_rate": 0.0003692778335005015, + "loss": 2.7411, + "theoretical_loss": 3.6914382095869085, + "tokens_seen": 887257088 + }, + { + "epoch": 2.09, + "learning_rate": 0.0003692678034102307, + "loss": 2.9059, + "theoretical_loss": 3.691411685600051, + "tokens_seen": 887322624 + }, + { + "epoch": 2.09, + "learning_rate": 0.0003692577733199599, + "loss": 2.767, + "theoretical_loss": 3.691385164120611, + "tokens_seen": 887388160 + }, + { + "epoch": 2.09, + "learning_rate": 0.00036924774322968907, + "loss": 2.6977, + "theoretical_loss": 3.6913586451481653, + "tokens_seen": 887453696 + }, + { + "epoch": 2.09, + "learning_rate": 0.0003692377131394183, + "loss": 2.6306, + "theoretical_loss": 3.6913321286822924, + "tokens_seen": 887519232 + }, + { + "epoch": 2.09, + "learning_rate": 0.00036922768304914743, + "loss": 2.7088, + "theoretical_loss": 3.6913056147225713, + "tokens_seen": 887584768 + }, + { + "epoch": 2.09, + "learning_rate": 0.00036921765295887666, + "loss": 2.4745, + "theoretical_loss": 3.691279103268579, + "tokens_seen": 887650304 + }, + { + "epoch": 2.09, + "learning_rate": 0.00036920762286860585, + "loss": 2.7917, + "theoretical_loss": 3.691252594319894, + "tokens_seen": 887715840 + }, + { + "epoch": 2.09, + "learning_rate": 0.000369197592778335, + "loss": 2.7978, + "theoretical_loss": 3.691226087876095, + "tokens_seen": 887781376 + }, + { + "epoch": 2.09, + "learning_rate": 0.0003691875626880642, + "loss": 2.7152, + "theoretical_loss": 3.6911995839367604, + "tokens_seen": 887846912 + }, + { + "epoch": 2.09, + "learning_rate": 0.0003691775325977934, + "loss": 2.6977, + "theoretical_loss": 3.691173082501468, + "tokens_seen": 887912448 + }, + { + "epoch": 2.09, + "learning_rate": 0.00036916750250752257, + "loss": 2.769, + "theoretical_loss": 3.6911465835697976, + "tokens_seen": 887977984 + }, + { + "epoch": 2.09, + "objective/train/docs_used": 961818, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6037163734436035, + "objective/train/theoretical_loss": 3.691133335042688, + "objective/train/tokens_used": 890267104, + "theoretical_loss": 3.691133335042688, + "tokens_seen": 888010752 + }, + { + "epoch": 2.09, + "learning_rate": 0.0003691574724172518, + "loss": 2.5662, + "theoretical_loss": 3.6911200871413263, + "tokens_seen": 888043520 + }, + { + "epoch": 2.09, + "learning_rate": 0.00036914744232698093, + "loss": 2.7313, + "theoretical_loss": 3.6910935932156343, + "tokens_seen": 888109056 + }, + { + "epoch": 2.09, + "learning_rate": 0.00036913741223671017, + "loss": 2.8887, + "theoretical_loss": 3.6910671017923002, + "tokens_seen": 888174592 + }, + { + "epoch": 2.09, + "learning_rate": 0.0003691273821464393, + "loss": 2.7674, + "theoretical_loss": 3.6910406128709026, + "tokens_seen": 888240128 + }, + { + "epoch": 2.09, + "learning_rate": 0.00036911735205616853, + "loss": 2.5497, + "theoretical_loss": 3.6910141264510212, + "tokens_seen": 888305664 + }, + { + "epoch": 2.09, + "learning_rate": 0.0003691073219658977, + "loss": 2.6463, + "theoretical_loss": 3.690987642532235, + "tokens_seen": 888371200 + }, + { + "epoch": 2.09, + "learning_rate": 0.0003690972918756269, + "loss": 2.8052, + "theoretical_loss": 3.690961161114123, + "tokens_seen": 888436736 + }, + { + "epoch": 2.09, + "learning_rate": 0.0003690872617853561, + "loss": 2.906, + "theoretical_loss": 3.690934682196265, + "tokens_seen": 888502272 + }, + { + "epoch": 2.09, + "learning_rate": 0.00036907723169508525, + "loss": 2.7999, + "theoretical_loss": 3.690908205778241, + "tokens_seen": 888567808 + }, + { + "epoch": 2.09, + "learning_rate": 0.00036906720160481444, + "loss": 2.7024, + "theoretical_loss": 3.6908817318596303, + "tokens_seen": 888633344 + }, + { + "epoch": 2.09, + "learning_rate": 0.00036905717151454367, + "loss": 2.6924, + "theoretical_loss": 3.6908552604400118, + "tokens_seen": 888698880 + }, + { + "epoch": 2.09, + "learning_rate": 0.0003690471414242728, + "loss": 2.6747, + "theoretical_loss": 3.6908287915189666, + "tokens_seen": 888764416 + }, + { + "epoch": 2.09, + "learning_rate": 0.00036903711133400203, + "loss": 2.6567, + "theoretical_loss": 3.690802325096074, + "tokens_seen": 888829952 + }, + { + "epoch": 2.09, + "learning_rate": 0.0003690270812437312, + "loss": 2.4936, + "theoretical_loss": 3.6907758611709145, + "tokens_seen": 888895488 + }, + { + "epoch": 2.09, + "learning_rate": 0.0003690170511534604, + "loss": 2.4739, + "theoretical_loss": 3.6907493997430674, + "tokens_seen": 888961024 + }, + { + "epoch": 2.09, + "learning_rate": 0.0003690070210631896, + "loss": 2.5213, + "theoretical_loss": 3.6907229408121136, + "tokens_seen": 889026560 + }, + { + "epoch": 2.09, + "learning_rate": 0.00036899699097291876, + "loss": 2.549, + "theoretical_loss": 3.690696484377634, + "tokens_seen": 889092096 + }, + { + "epoch": 2.09, + "learning_rate": 0.00036898696088264794, + "loss": 2.7987, + "theoretical_loss": 3.6906700304392075, + "tokens_seen": 889157632 + }, + { + "epoch": 2.09, + "learning_rate": 0.0003689769307923772, + "loss": 2.4143, + "theoretical_loss": 3.690643578996416, + "tokens_seen": 889223168 + }, + { + "epoch": 2.09, + "learning_rate": 0.0003689669007021063, + "loss": 2.5284, + "theoretical_loss": 3.6906171300488397, + "tokens_seen": 889288704 + }, + { + "epoch": 2.09, + "learning_rate": 0.00036895687061183554, + "loss": 2.5884, + "theoretical_loss": 3.690590683596059, + "tokens_seen": 889354240 + }, + { + "epoch": 2.09, + "learning_rate": 0.00036894684052156466, + "loss": 2.5301, + "theoretical_loss": 3.6905642396376557, + "tokens_seen": 889419776 + }, + { + "epoch": 2.09, + "learning_rate": 0.0003689368104312939, + "loss": 2.7106, + "theoretical_loss": 3.69053779817321, + "tokens_seen": 889485312 + }, + { + "epoch": 2.09, + "learning_rate": 0.0003689267803410231, + "loss": 2.6738, + "theoretical_loss": 3.6905113592023033, + "tokens_seen": 889550848 + }, + { + "epoch": 2.09, + "learning_rate": 0.00036891675025075226, + "loss": 2.9077, + "theoretical_loss": 3.690484922724517, + "tokens_seen": 889616384 + }, + { + "epoch": 2.09, + "objective/train/docs_used": 961818, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.595710515975952, + "objective/train/theoretical_loss": 3.6904717054204124, + "objective/train/tokens_used": 890267104, + "theoretical_loss": 3.6904717054204124, + "tokens_seen": 889649152 + }, + { + "epoch": 2.09, + "learning_rate": 0.00036890672016048144, + "loss": 2.6372, + "theoretical_loss": 3.690458488739431, + "tokens_seen": 889681920 + }, + { + "epoch": 2.09, + "learning_rate": 0.0003688966900702107, + "loss": 2.6164, + "theoretical_loss": 3.6904320572466283, + "tokens_seen": 889747456 + }, + { + "epoch": 2.09, + "learning_rate": 0.0003688866599799398, + "loss": 2.6341, + "theoretical_loss": 3.6904056282456903, + "tokens_seen": 889812992 + }, + { + "epoch": 2.09, + "learning_rate": 0.00036887662988966904, + "loss": 2.7133, + "theoretical_loss": 3.690379201736197, + "tokens_seen": 889878528 + }, + { + "epoch": 2.09, + "learning_rate": 0.00036886659979939817, + "loss": 2.7007, + "theoretical_loss": 3.690352777717732, + "tokens_seen": 889944064 + }, + { + "epoch": 2.09, + "learning_rate": 0.0003688565697091274, + "loss": 2.6807, + "theoretical_loss": 3.6903263561898756, + "tokens_seen": 890009600 + }, + { + "epoch": 2.09, + "learning_rate": 0.0003688465396188566, + "loss": 2.5019, + "theoretical_loss": 3.6902999371522105, + "tokens_seen": 890075136 + }, + { + "epoch": 2.09, + "learning_rate": 0.00036883650952858576, + "loss": 2.8941, + "theoretical_loss": 3.6902735206043182, + "tokens_seen": 890140672 + }, + { + "epoch": 2.09, + "learning_rate": 0.00036882647943831494, + "loss": 2.4319, + "theoretical_loss": 3.6902471065457814, + "tokens_seen": 890206208 + }, + { + "epoch": 2.09, + "learning_rate": 0.0003688164493480441, + "loss": 2.775, + "theoretical_loss": 3.6902206949761815, + "tokens_seen": 890271744 + }, + { + "epoch": 3.0, + "learning_rate": 0.0003688064192577733, + "loss": 3.5933, + "theoretical_loss": 3.690188096626651, + "tokens_seen": 890352640 + }, + { + "epoch": 3.0, + "learning_rate": 0.00036879638916750254, + "loss": 2.7673, + "theoretical_loss": 3.690161690616762, + "tokens_seen": 890418176 + }, + { + "epoch": 3.0, + "learning_rate": 0.00036878635907723167, + "loss": 2.9802, + "theoretical_loss": 3.690135287094459, + "tokens_seen": 890483712 + }, + { + "epoch": 3.0, + "learning_rate": 0.0003687763289869609, + "loss": 2.9272, + "theoretical_loss": 3.690108886059325, + "tokens_seen": 890549248 + }, + { + "epoch": 3.0, + "learning_rate": 0.00036876629889669003, + "loss": 2.8218, + "theoretical_loss": 3.690082487510943, + "tokens_seen": 890614784 + }, + { + "epoch": 3.0, + "learning_rate": 0.00036875626880641927, + "loss": 2.9439, + "theoretical_loss": 3.690056091448896, + "tokens_seen": 890680320 + }, + { + "epoch": 3.0, + "learning_rate": 0.00036874623871614845, + "loss": 2.8595, + "theoretical_loss": 3.690029697872766, + "tokens_seen": 890745856 + }, + { + "epoch": 3.0, + "learning_rate": 0.00036873620862587763, + "loss": 2.8154, + "theoretical_loss": 3.6900033067821374, + "tokens_seen": 890811392 + }, + { + "epoch": 3.0, + "learning_rate": 0.0003687261785356068, + "loss": 2.8048, + "theoretical_loss": 3.6899769181765922, + "tokens_seen": 890876928 + }, + { + "epoch": 3.0, + "learning_rate": 0.00036871614844533605, + "loss": 2.8396, + "theoretical_loss": 3.6899505320557138, + "tokens_seen": 890942464 + }, + { + "epoch": 3.0, + "learning_rate": 0.00036870611835506517, + "loss": 2.7181, + "theoretical_loss": 3.689924148419086, + "tokens_seen": 891008000 + }, + { + "epoch": 3.0, + "learning_rate": 0.0003686960882647944, + "loss": 2.9879, + "theoretical_loss": 3.6898977672662916, + "tokens_seen": 891073536 + }, + { + "epoch": 3.0, + "learning_rate": 0.00036868605817452353, + "loss": 2.7716, + "theoretical_loss": 3.6898713885969148, + "tokens_seen": 891139072 + }, + { + "epoch": 3.0, + "learning_rate": 0.00036867602808425277, + "loss": 2.7015, + "theoretical_loss": 3.6898450124105384, + "tokens_seen": 891204608 + }, + { + "epoch": 3.0, + "learning_rate": 0.00036866599799398195, + "loss": 2.7423, + "theoretical_loss": 3.689818638706747, + "tokens_seen": 891270144 + }, + { + "epoch": 3.0, + "objective/train/docs_used": 1026627, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.057016134262085, + "objective/train/theoretical_loss": 3.6898120456686536, + "objective/train/tokens_used": 911746528, + "theoretical_loss": 3.6898120456686536, + "tokens_seen": 891286528 + }, + { + "epoch": 3.0, + "learning_rate": 0.00036865596790371113, + "loss": 2.8914, + "theoretical_loss": 3.689792267485123, + "tokens_seen": 891335680 + }, + { + "epoch": 3.0, + "learning_rate": 0.0003686459378134403, + "loss": 2.7807, + "theoretical_loss": 3.689765898745252, + "tokens_seen": 891401216 + }, + { + "epoch": 3.0, + "learning_rate": 0.0003686359077231695, + "loss": 2.8683, + "theoretical_loss": 3.689739532486717, + "tokens_seen": 891466752 + }, + { + "epoch": 3.0, + "learning_rate": 0.0003686258776328987, + "loss": 2.6409, + "theoretical_loss": 3.689713168709102, + "tokens_seen": 891532288 + }, + { + "epoch": 3.0, + "learning_rate": 0.0003686158475426279, + "loss": 2.8472, + "theoretical_loss": 3.6896868074119924, + "tokens_seen": 891597824 + }, + { + "epoch": 3.0, + "learning_rate": 0.0003686058174523571, + "loss": 2.619, + "theoretical_loss": 3.689660448594971, + "tokens_seen": 891663360 + }, + { + "epoch": 3.0, + "learning_rate": 0.0003685957873620863, + "loss": 2.788, + "theoretical_loss": 3.689634092257623, + "tokens_seen": 891728896 + }, + { + "epoch": 3.0, + "learning_rate": 0.00036858575727181545, + "loss": 2.9168, + "theoretical_loss": 3.689607738399533, + "tokens_seen": 891794432 + }, + { + "epoch": 3.0, + "learning_rate": 0.00036857572718154464, + "loss": 2.8188, + "theoretical_loss": 3.6895813870202856, + "tokens_seen": 891859968 + }, + { + "epoch": 3.0, + "learning_rate": 0.00036856569709127387, + "loss": 2.744, + "theoretical_loss": 3.6895550381194653, + "tokens_seen": 891925504 + }, + { + "epoch": 3.0, + "learning_rate": 0.000368555667001003, + "loss": 2.8304, + "theoretical_loss": 3.6895286916966565, + "tokens_seen": 891991040 + }, + { + "epoch": 3.0, + "learning_rate": 0.00036854563691073223, + "loss": 2.7051, + "theoretical_loss": 3.689502347751445, + "tokens_seen": 892056576 + }, + { + "epoch": 3.0, + "learning_rate": 0.0003685356068204614, + "loss": 2.781, + "theoretical_loss": 3.6894760062834155, + "tokens_seen": 892122112 + }, + { + "epoch": 3.0, + "learning_rate": 0.0003685255767301906, + "loss": 2.7491, + "theoretical_loss": 3.689449667292153, + "tokens_seen": 892187648 + }, + { + "epoch": 3.0, + "learning_rate": 0.0003685155466399198, + "loss": 2.8092, + "theoretical_loss": 3.689423330777242, + "tokens_seen": 892253184 + }, + { + "epoch": 3.0, + "learning_rate": 0.00036850551654964896, + "loss": 2.8625, + "theoretical_loss": 3.689396996738269, + "tokens_seen": 892318720 + }, + { + "epoch": 3.0, + "learning_rate": 0.00036849548645937814, + "loss": 2.672, + "theoretical_loss": 3.6893706651748195, + "tokens_seen": 892384256 + }, + { + "epoch": 3.0, + "learning_rate": 0.0003684854563691074, + "loss": 2.8162, + "theoretical_loss": 3.6893443360864775, + "tokens_seen": 892449792 + }, + { + "epoch": 3.0, + "learning_rate": 0.0003684754262788365, + "loss": 2.8069, + "theoretical_loss": 3.68931800947283, + "tokens_seen": 892515328 + }, + { + "epoch": 3.0, + "learning_rate": 0.00036846539618856574, + "loss": 2.78, + "theoretical_loss": 3.6892916853334627, + "tokens_seen": 892580864 + }, + { + "epoch": 3.0, + "learning_rate": 0.00036845536609829486, + "loss": 2.7692, + "theoretical_loss": 3.6892653636679604, + "tokens_seen": 892646400 + }, + { + "epoch": 3.0, + "learning_rate": 0.0003684453360080241, + "loss": 2.6777, + "theoretical_loss": 3.68923904447591, + "tokens_seen": 892711936 + }, + { + "epoch": 3.0, + "learning_rate": 0.0003684353059177533, + "loss": 2.7668, + "theoretical_loss": 3.689212727756897, + "tokens_seen": 892777472 + }, + { + "epoch": 3.0, + "learning_rate": 0.00036842527582748246, + "loss": 2.8518, + "theoretical_loss": 3.689186413510508, + "tokens_seen": 892843008 + }, + { + "epoch": 3.0, + "learning_rate": 0.00036841524573721164, + "loss": 2.8783, + "theoretical_loss": 3.689160101736328, + "tokens_seen": 892908544 + }, + { + "epoch": 3.0, + "objective/train/docs_used": 1031633, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.0238943099975586, + "objective/train/theoretical_loss": 3.6891535241790177, + "objective/train/tokens_used": 913384928, + "theoretical_loss": 3.6891535241790177, + "tokens_seen": 892924928 + }, + { + "epoch": 3.0, + "learning_rate": 0.0003684052156469409, + "loss": 2.858, + "theoretical_loss": 3.689133792433945, + "tokens_seen": 892974080 + }, + { + "epoch": 3.0, + "learning_rate": 0.00036839518555667, + "loss": 2.9869, + "theoretical_loss": 3.689107485602944, + "tokens_seen": 893039616 + }, + { + "epoch": 3.0, + "learning_rate": 0.00036838515546639924, + "loss": 2.8007, + "theoretical_loss": 3.689081181242912, + "tokens_seen": 893105152 + }, + { + "epoch": 3.0, + "learning_rate": 0.00036837512537612837, + "loss": 2.9061, + "theoretical_loss": 3.689054879353437, + "tokens_seen": 893170688 + }, + { + "epoch": 3.0, + "learning_rate": 0.0003683650952858576, + "loss": 2.8327, + "theoretical_loss": 3.689028579934103, + "tokens_seen": 893236224 + }, + { + "epoch": 3.0, + "learning_rate": 0.0003683550651955868, + "loss": 2.8884, + "theoretical_loss": 3.6890022829844984, + "tokens_seen": 893301760 + }, + { + "epoch": 3.0, + "learning_rate": 0.00036834503510531596, + "loss": 2.538, + "theoretical_loss": 3.6889759885042106, + "tokens_seen": 893367296 + }, + { + "epoch": 3.0, + "learning_rate": 0.00036833500501504514, + "loss": 2.8365, + "theoretical_loss": 3.6889496964928252, + "tokens_seen": 893432832 + }, + { + "epoch": 3.0, + "learning_rate": 0.0003683249749247743, + "loss": 2.7163, + "theoretical_loss": 3.6889234069499306, + "tokens_seen": 893498368 + }, + { + "epoch": 3.0, + "learning_rate": 0.0003683149448345035, + "loss": 2.7978, + "theoretical_loss": 3.6888971198751133, + "tokens_seen": 893563904 + }, + { + "epoch": 3.0, + "learning_rate": 0.00036830491474423274, + "loss": 2.7502, + "theoretical_loss": 3.688870835267961, + "tokens_seen": 893629440 + }, + { + "epoch": 3.0, + "learning_rate": 0.00036829488465396187, + "loss": 2.7726, + "theoretical_loss": 3.6888445531280603, + "tokens_seen": 893694976 + }, + { + "epoch": 3.0, + "learning_rate": 0.0003682848545636911, + "loss": 2.9963, + "theoretical_loss": 3.688818273455, + "tokens_seen": 893760512 + }, + { + "epoch": 3.0, + "learning_rate": 0.00036827482447342023, + "loss": 2.7254, + "theoretical_loss": 3.688791996248366, + "tokens_seen": 893826048 + }, + { + "epoch": 3.0, + "learning_rate": 0.00036826479438314947, + "loss": 2.8003, + "theoretical_loss": 3.688765721507748, + "tokens_seen": 893891584 + }, + { + "epoch": 3.0, + "learning_rate": 0.00036825476429287865, + "loss": 2.8246, + "theoretical_loss": 3.6887394492327323, + "tokens_seen": 893957120 + }, + { + "epoch": 3.0, + "learning_rate": 0.00036824473420260783, + "loss": 2.5958, + "theoretical_loss": 3.6887131794229076, + "tokens_seen": 894022656 + }, + { + "epoch": 3.0, + "learning_rate": 0.000368234704112337, + "loss": 2.6973, + "theoretical_loss": 3.688686912077861, + "tokens_seen": 894088192 + }, + { + "epoch": 3.0, + "learning_rate": 0.00036822467402206625, + "loss": 2.7454, + "theoretical_loss": 3.6886606471971817, + "tokens_seen": 894153728 + }, + { + "epoch": 3.0, + "learning_rate": 0.00036821464393179537, + "loss": 2.6587, + "theoretical_loss": 3.688634384780457, + "tokens_seen": 894219264 + }, + { + "epoch": 3.0, + "learning_rate": 0.0003682046138415246, + "loss": 2.6925, + "theoretical_loss": 3.688608124827276, + "tokens_seen": 894284800 + }, + { + "epoch": 3.0, + "learning_rate": 0.00036819458375125373, + "loss": 2.8973, + "theoretical_loss": 3.688581867337226, + "tokens_seen": 894350336 + }, + { + "epoch": 3.0, + "learning_rate": 0.00036818455366098297, + "loss": 2.6081, + "theoretical_loss": 3.688555612309897, + "tokens_seen": 894415872 + }, + { + "epoch": 3.0, + "learning_rate": 0.00036817452357071215, + "loss": 2.8145, + "theoretical_loss": 3.6885293597448765, + "tokens_seen": 894481408 + }, + { + "epoch": 3.0, + "learning_rate": 0.00036816449348044133, + "loss": 2.7993, + "theoretical_loss": 3.6885031096417533, + "tokens_seen": 894546944 + }, + { + "epoch": 3.0, + "objective/train/docs_used": 1036652, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8129405975341797, + "objective/train/theoretical_loss": 3.6884965475005957, + "objective/train/tokens_used": 915023328, + "theoretical_loss": 3.6884965475005957, + "tokens_seen": 894563328 + }, + { + "epoch": 3.0, + "learning_rate": 0.0003681544633901705, + "loss": 2.7861, + "theoretical_loss": 3.6884768620001163, + "tokens_seen": 894612480 + }, + { + "epoch": 3.0, + "learning_rate": 0.0003681444332998997, + "loss": 2.7175, + "theoretical_loss": 3.688450616819555, + "tokens_seen": 894678016 + }, + { + "epoch": 3.0, + "learning_rate": 0.0003681344032096289, + "loss": 2.9492, + "theoretical_loss": 3.6884243740996574, + "tokens_seen": 894743552 + }, + { + "epoch": 3.0, + "learning_rate": 0.0003681243731193581, + "loss": 2.6274, + "theoretical_loss": 3.688398133840013, + "tokens_seen": 894809088 + }, + { + "epoch": 3.0, + "learning_rate": 0.00036811434302908724, + "loss": 2.7315, + "theoretical_loss": 3.688371896040211, + "tokens_seen": 894874624 + }, + { + "epoch": 3.0, + "learning_rate": 0.0003681043129388165, + "loss": 2.9269, + "theoretical_loss": 3.6883456606998406, + "tokens_seen": 894940160 + }, + { + "epoch": 3.0, + "learning_rate": 0.0003680942828485456, + "loss": 2.8358, + "theoretical_loss": 3.6883194278184916, + "tokens_seen": 895005696 + }, + { + "epoch": 3.0, + "learning_rate": 0.00036808425275827484, + "loss": 2.8829, + "theoretical_loss": 3.688293197395753, + "tokens_seen": 895071232 + }, + { + "epoch": 3.0, + "learning_rate": 0.000368074222668004, + "loss": 2.7785, + "theoretical_loss": 3.6882669694312145, + "tokens_seen": 895136768 + }, + { + "epoch": 3.0, + "learning_rate": 0.0003680641925777332, + "loss": 2.9305, + "theoretical_loss": 3.688240743924466, + "tokens_seen": 895202304 + }, + { + "epoch": 3.0, + "learning_rate": 0.0003680541624874624, + "loss": 2.7584, + "theoretical_loss": 3.688214520875097, + "tokens_seen": 895267840 + }, + { + "epoch": 3.0, + "learning_rate": 0.0003680441323971916, + "loss": 2.859, + "theoretical_loss": 3.6881883002826976, + "tokens_seen": 895333376 + }, + { + "epoch": 3.0, + "learning_rate": 0.0003680441323971916, + "loss": 2.8224, + "theoretical_loss": 3.6881620821468575, + "tokens_seen": 895398912 + }, + { + "epoch": 3.0, + "learning_rate": 0.00036803410230692074, + "loss": 2.8127, + "theoretical_loss": 3.6881358664671673, + "tokens_seen": 895464448 + }, + { + "epoch": 3.0, + "learning_rate": 0.00036802407221665, + "loss": 2.8278, + "theoretical_loss": 3.6881096532432163, + "tokens_seen": 895529984 + }, + { + "epoch": 3.0, + "learning_rate": 0.0003680140421263791, + "loss": 2.8223, + "theoretical_loss": 3.6880834424745954, + "tokens_seen": 895595520 + }, + { + "epoch": 3.0, + "learning_rate": 0.00036800401203610834, + "loss": 2.8507, + "theoretical_loss": 3.688057234160895, + "tokens_seen": 895661056 + }, + { + "epoch": 3.0, + "learning_rate": 0.0003679939819458375, + "loss": 2.9332, + "theoretical_loss": 3.6880310283017055, + "tokens_seen": 895726592 + }, + { + "epoch": 3.0, + "learning_rate": 0.0003679839518555667, + "loss": 2.8083, + "theoretical_loss": 3.688004824896617, + "tokens_seen": 895792128 + }, + { + "epoch": 3.0, + "learning_rate": 0.0003679739217652959, + "loss": 2.887, + "theoretical_loss": 3.687978623945221, + "tokens_seen": 895857664 + }, + { + "epoch": 3.0, + "learning_rate": 0.00036796389167502506, + "loss": 2.8633, + "theoretical_loss": 3.687952425447108, + "tokens_seen": 895923200 + }, + { + "epoch": 3.0, + "learning_rate": 0.00036795386158475424, + "loss": 2.7929, + "theoretical_loss": 3.687926229401868, + "tokens_seen": 895988736 + }, + { + "epoch": 3.0, + "learning_rate": 0.0003679438314944835, + "loss": 2.8538, + "theoretical_loss": 3.687900035809093, + "tokens_seen": 896054272 + }, + { + "epoch": 3.0, + "learning_rate": 0.0003679338014042126, + "loss": 2.8214, + "theoretical_loss": 3.687873844668373, + "tokens_seen": 896119808 + }, + { + "epoch": 3.0, + "learning_rate": 0.00036792377131394184, + "loss": 2.7678, + "theoretical_loss": 3.6878476559793008, + "tokens_seen": 896185344 + }, + { + "epoch": 3.0, + "objective/train/docs_used": 1041586, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.545072555541992, + "objective/train/theoretical_loss": 3.687841109190055, + "objective/train/tokens_used": 916661728, + "theoretical_loss": 3.687841109190055, + "tokens_seen": 896201728 + }, + { + "epoch": 3.0, + "learning_rate": 0.00036791374122367097, + "loss": 2.575, + "theoretical_loss": 3.6878214697414666, + "tokens_seen": 896250880 + }, + { + "epoch": 3.0, + "learning_rate": 0.0003679037111334002, + "loss": 2.7571, + "theoretical_loss": 3.6877952859544614, + "tokens_seen": 896316416 + }, + { + "epoch": 3.0, + "learning_rate": 0.0003678936810431294, + "loss": 2.8336, + "theoretical_loss": 3.6877691046178773, + "tokens_seen": 896381952 + }, + { + "epoch": 3.0, + "learning_rate": 0.00036788365095285857, + "loss": 2.8234, + "theoretical_loss": 3.687742925731306, + "tokens_seen": 896447488 + }, + { + "epoch": 3.0, + "learning_rate": 0.00036787362086258775, + "loss": 2.7132, + "theoretical_loss": 3.687716749294338, + "tokens_seen": 896513024 + }, + { + "epoch": 3.0, + "learning_rate": 0.000367863590772317, + "loss": 2.751, + "theoretical_loss": 3.6876905753065667, + "tokens_seen": 896578560 + }, + { + "epoch": 3.0, + "learning_rate": 0.00036785356068204616, + "loss": 2.8567, + "theoretical_loss": 3.6876644037675823, + "tokens_seen": 896644096 + }, + { + "epoch": 3.0, + "learning_rate": 0.00036784353059177535, + "loss": 2.8155, + "theoretical_loss": 3.687638234676978, + "tokens_seen": 896709632 + }, + { + "epoch": 3.0, + "learning_rate": 0.0003678335005015045, + "loss": 2.9743, + "theoretical_loss": 3.687612068034345, + "tokens_seen": 896775168 + }, + { + "epoch": 3.0, + "learning_rate": 0.0003678234704112337, + "loss": 2.7782, + "theoretical_loss": 3.6875859038392758, + "tokens_seen": 896840704 + }, + { + "epoch": 3.0, + "learning_rate": 0.00036781344032096294, + "loss": 2.618, + "theoretical_loss": 3.687559742091363, + "tokens_seen": 896906240 + }, + { + "epoch": 3.0, + "learning_rate": 0.00036780341023069207, + "loss": 2.8275, + "theoretical_loss": 3.6875335827901985, + "tokens_seen": 896971776 + }, + { + "epoch": 3.0, + "learning_rate": 0.0003677933801404213, + "loss": 2.7824, + "theoretical_loss": 3.687507425935374, + "tokens_seen": 897037312 + }, + { + "epoch": 3.0, + "learning_rate": 0.0003677933801404213, + "loss": 2.8379, + "theoretical_loss": 3.687481271526484, + "tokens_seen": 897102848 + }, + { + "epoch": 3.0, + "learning_rate": 0.00036778335005015043, + "loss": 2.8418, + "theoretical_loss": 3.6874551195631193, + "tokens_seen": 897168384 + }, + { + "epoch": 3.0, + "learning_rate": 0.00036777331995987967, + "loss": 2.8881, + "theoretical_loss": 3.687428970044873, + "tokens_seen": 897233920 + }, + { + "epoch": 3.0, + "learning_rate": 0.00036776328986960885, + "loss": 2.8368, + "theoretical_loss": 3.6874028229713387, + "tokens_seen": 897299456 + }, + { + "epoch": 3.0, + "learning_rate": 0.00036775325977933803, + "loss": 2.7317, + "theoretical_loss": 3.6873766783421082, + "tokens_seen": 897364992 + }, + { + "epoch": 3.0, + "learning_rate": 0.0003677432296890672, + "loss": 2.9576, + "theoretical_loss": 3.6873505361567753, + "tokens_seen": 897430528 + }, + { + "epoch": 3.0, + "learning_rate": 0.00036773319959879645, + "loss": 2.8593, + "theoretical_loss": 3.687324396414933, + "tokens_seen": 897496064 + }, + { + "epoch": 3.0, + "learning_rate": 0.00036772316950852557, + "loss": 2.8501, + "theoretical_loss": 3.6872982591161745, + "tokens_seen": 897561600 + }, + { + "epoch": 3.0, + "learning_rate": 0.0003677131394182548, + "loss": 2.9521, + "theoretical_loss": 3.6872721242600925, + "tokens_seen": 897627136 + }, + { + "epoch": 3.0, + "learning_rate": 0.00036770310932798393, + "loss": 2.6791, + "theoretical_loss": 3.6872459918462814, + "tokens_seen": 897692672 + }, + { + "epoch": 3.0, + "learning_rate": 0.00036769307923771317, + "loss": 2.6738, + "theoretical_loss": 3.6872198618743344, + "tokens_seen": 897758208 + }, + { + "epoch": 3.0, + "learning_rate": 0.00036768304914744235, + "loss": 3.0484, + "theoretical_loss": 3.687193734343844, + "tokens_seen": 897823744 + }, + { + "epoch": 3.0, + "objective/train/docs_used": 1046594, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.713426113128662, + "objective/train/theoretical_loss": 3.687187202842652, + "objective/train/tokens_used": 918300128, + "theoretical_loss": 3.687187202842652, + "tokens_seen": 897840128 + }, + { + "epoch": 3.0, + "learning_rate": 0.00036767301905717153, + "loss": 2.8236, + "theoretical_loss": 3.6871676092544057, + "tokens_seen": 897889280 + }, + { + "epoch": 3.0, + "learning_rate": 0.0003676629889669007, + "loss": 2.856, + "theoretical_loss": 3.6871414866056123, + "tokens_seen": 897954816 + }, + { + "epoch": 3.0, + "learning_rate": 0.0003676529588766299, + "loss": 2.9411, + "theoretical_loss": 3.687115366397057, + "tokens_seen": 898020352 + }, + { + "epoch": 3.0, + "learning_rate": 0.0003676429287863591, + "loss": 2.622, + "theoretical_loss": 3.6870892486283355, + "tokens_seen": 898085888 + }, + { + "epoch": 3.0, + "learning_rate": 0.0003676328986960883, + "loss": 2.8903, + "theoretical_loss": 3.6870631332990405, + "tokens_seen": 898151424 + }, + { + "epoch": 3.0, + "learning_rate": 0.00036762286860581744, + "loss": 2.7094, + "theoretical_loss": 3.6870370204087664, + "tokens_seen": 898216960 + }, + { + "epoch": 3.0, + "learning_rate": 0.0003676128385155467, + "loss": 2.5918, + "theoretical_loss": 3.6870109099571082, + "tokens_seen": 898282496 + }, + { + "epoch": 3.0, + "learning_rate": 0.0003676028084252758, + "loss": 3.0356, + "theoretical_loss": 3.6869848019436597, + "tokens_seen": 898348032 + }, + { + "epoch": 3.0, + "learning_rate": 0.00036759277833500504, + "loss": 2.8615, + "theoretical_loss": 3.6869586963680154, + "tokens_seen": 898413568 + }, + { + "epoch": 3.0, + "learning_rate": 0.0003675827482447342, + "loss": 2.7986, + "theoretical_loss": 3.68693259322977, + "tokens_seen": 898479104 + }, + { + "epoch": 3.0, + "learning_rate": 0.0003675727181544634, + "loss": 2.7854, + "theoretical_loss": 3.686906492528518, + "tokens_seen": 898544640 + }, + { + "epoch": 3.0, + "learning_rate": 0.0003675626880641926, + "loss": 2.7457, + "theoretical_loss": 3.686880394263854, + "tokens_seen": 898610176 + }, + { + "epoch": 3.0, + "learning_rate": 0.0003675526579739218, + "loss": 2.731, + "theoretical_loss": 3.6868542984353736, + "tokens_seen": 898675712 + }, + { + "epoch": 3.0, + "learning_rate": 0.00036754262788365094, + "loss": 2.8772, + "theoretical_loss": 3.686828205042671, + "tokens_seen": 898741248 + }, + { + "epoch": 3.0, + "learning_rate": 0.0003675325977933802, + "loss": 2.8269, + "theoretical_loss": 3.6868021140853413, + "tokens_seen": 898806784 + }, + { + "epoch": 3.0, + "learning_rate": 0.0003675225677031093, + "loss": 2.8921, + "theoretical_loss": 3.6867760255629802, + "tokens_seen": 898872320 + }, + { + "epoch": 3.0, + "learning_rate": 0.00036751253761283854, + "loss": 2.7749, + "theoretical_loss": 3.686749939475183, + "tokens_seen": 898937856 + }, + { + "epoch": 3.0, + "learning_rate": 0.0003675025075225677, + "loss": 2.6101, + "theoretical_loss": 3.686723855821544, + "tokens_seen": 899003392 + }, + { + "epoch": 3.0, + "learning_rate": 0.0003674924774322969, + "loss": 2.8056, + "theoretical_loss": 3.68669777460166, + "tokens_seen": 899068928 + }, + { + "epoch": 3.0, + "learning_rate": 0.0003674824473420261, + "loss": 2.8025, + "theoretical_loss": 3.686671695815125, + "tokens_seen": 899134464 + }, + { + "epoch": 3.0, + "learning_rate": 0.00036747241725175526, + "loss": 2.7852, + "theoretical_loss": 3.686645619461536, + "tokens_seen": 899200000 + }, + { + "epoch": 3.0, + "learning_rate": 0.00036746238716148444, + "loss": 2.5889, + "theoretical_loss": 3.6866195455404878, + "tokens_seen": 899265536 + }, + { + "epoch": 3.0, + "learning_rate": 0.0003674523570712137, + "loss": 2.9105, + "theoretical_loss": 3.686593474051577, + "tokens_seen": 899331072 + }, + { + "epoch": 3.0, + "learning_rate": 0.0003674423269809428, + "loss": 2.5764, + "theoretical_loss": 3.686567404994399, + "tokens_seen": 899396608 + }, + { + "epoch": 3.0, + "learning_rate": 0.00036743229689067204, + "loss": 2.6419, + "theoretical_loss": 3.68654133836855, + "tokens_seen": 899462144 + }, + { + "epoch": 3.0, + "objective/train/docs_used": 1051693, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.662870407104492, + "objective/train/theoretical_loss": 3.686534822091936, + "objective/train/tokens_used": 919938528, + "theoretical_loss": 3.686534822091936, + "tokens_seen": 899478528 + }, + { + "epoch": 3.0, + "learning_rate": 0.00036742226680040117, + "loss": 2.8781, + "theoretical_loss": 3.686515274173626, + "tokens_seen": 899527680 + }, + { + "epoch": 3.0, + "learning_rate": 0.0003674122367101304, + "loss": 2.8185, + "theoretical_loss": 3.6864892124092234, + "tokens_seen": 899593216 + }, + { + "epoch": 3.0, + "learning_rate": 0.0003674022066198596, + "loss": 2.8196, + "theoretical_loss": 3.6864631530749388, + "tokens_seen": 899658752 + }, + { + "epoch": 3.0, + "learning_rate": 0.00036739217652958877, + "loss": 2.69, + "theoretical_loss": 3.686437096170368, + "tokens_seen": 899724288 + }, + { + "epoch": 3.0, + "learning_rate": 0.00036738214643931795, + "loss": 2.7704, + "theoretical_loss": 3.6864110416951075, + "tokens_seen": 899789824 + }, + { + "epoch": 3.0, + "learning_rate": 0.0003673721163490472, + "loss": 2.8191, + "theoretical_loss": 3.6863849896487544, + "tokens_seen": 899855360 + }, + { + "epoch": 3.0, + "learning_rate": 0.0003673620862587763, + "loss": 2.8529, + "theoretical_loss": 3.686358940030905, + "tokens_seen": 899920896 + }, + { + "epoch": 3.0, + "learning_rate": 0.00036735205616850555, + "loss": 2.7385, + "theoretical_loss": 3.686332892841156, + "tokens_seen": 899986432 + }, + { + "epoch": 3.0, + "learning_rate": 0.00036734202607823467, + "loss": 2.9449, + "theoretical_loss": 3.686306848079105, + "tokens_seen": 900051968 + }, + { + "epoch": 3.0, + "learning_rate": 0.0003673319959879639, + "loss": 2.9013, + "theoretical_loss": 3.686280805744348, + "tokens_seen": 900117504 + }, + { + "epoch": 3.0, + "learning_rate": 0.0003673219658976931, + "loss": 2.87, + "theoretical_loss": 3.686254765836483, + "tokens_seen": 900183040 + }, + { + "epoch": 3.0, + "learning_rate": 0.00036731193580742227, + "loss": 2.7614, + "theoretical_loss": 3.6862287283551067, + "tokens_seen": 900248576 + }, + { + "epoch": 3.0, + "learning_rate": 0.00036730190571715145, + "loss": 2.7338, + "theoretical_loss": 3.686202693299816, + "tokens_seen": 900314112 + }, + { + "epoch": 3.0, + "learning_rate": 0.00036729187562688063, + "loss": 2.5981, + "theoretical_loss": 3.686176660670209, + "tokens_seen": 900379648 + }, + { + "epoch": 3.0, + "learning_rate": 0.0003672818455366098, + "loss": 2.7967, + "theoretical_loss": 3.6861506304658826, + "tokens_seen": 900445184 + }, + { + "epoch": 3.0, + "learning_rate": 0.00036727181544633905, + "loss": 2.9024, + "theoretical_loss": 3.6861246026864354, + "tokens_seen": 900510720 + }, + { + "epoch": 3.0, + "learning_rate": 0.0003672617853560682, + "loss": 2.8047, + "theoretical_loss": 3.6860985773314634, + "tokens_seen": 900576256 + }, + { + "epoch": 3.0, + "learning_rate": 0.0003672517552657974, + "loss": 2.6793, + "theoretical_loss": 3.6860725544005657, + "tokens_seen": 900641792 + }, + { + "epoch": 3.0, + "learning_rate": 0.00036724172517552654, + "loss": 2.9473, + "theoretical_loss": 3.6860465338933395, + "tokens_seen": 900707328 + }, + { + "epoch": 3.0, + "learning_rate": 0.00036723169508525577, + "loss": 2.6197, + "theoretical_loss": 3.686020515809383, + "tokens_seen": 900772864 + }, + { + "epoch": 3.0, + "learning_rate": 0.00036722166499498495, + "loss": 2.8145, + "theoretical_loss": 3.6859945001482943, + "tokens_seen": 900838400 + }, + { + "epoch": 3.0, + "learning_rate": 0.00036721163490471414, + "loss": 2.7004, + "theoretical_loss": 3.685968486909671, + "tokens_seen": 900903936 + }, + { + "epoch": 3.0, + "learning_rate": 0.0003672016048144433, + "loss": 2.9374, + "theoretical_loss": 3.6859424760931123, + "tokens_seen": 900969472 + }, + { + "epoch": 3.0, + "learning_rate": 0.00036719157472417255, + "loss": 2.7168, + "theoretical_loss": 3.6859164676982155, + "tokens_seen": 901035008 + }, + { + "epoch": 3.0, + "learning_rate": 0.0003671815446339017, + "loss": 2.7888, + "theoretical_loss": 3.68589046172458, + "tokens_seen": 901100544 + }, + { + "epoch": 3.0, + "objective/train/docs_used": 1056725, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.974947214126587, + "objective/train/theoretical_loss": 3.685883960609446, + "objective/train/tokens_used": 921576928, + "theoretical_loss": 3.685883960609446, + "tokens_seen": 901116928 + }, + { + "epoch": 3.0, + "learning_rate": 0.0003671715145436309, + "loss": 2.7201, + "theoretical_loss": 3.685864458171803, + "tokens_seen": 901166080 + }, + { + "epoch": 3.0, + "learning_rate": 0.00036716148445336004, + "loss": 2.7771, + "theoretical_loss": 3.685838457039485, + "tokens_seen": 901231616 + }, + { + "epoch": 3.0, + "learning_rate": 0.0003671514543630893, + "loss": 2.7884, + "theoretical_loss": 3.6858124583272227, + "tokens_seen": 901297152 + }, + { + "epoch": 3.0, + "learning_rate": 0.00036714142427281846, + "loss": 2.9321, + "theoretical_loss": 3.685786462034616, + "tokens_seen": 901362688 + }, + { + "epoch": 3.0, + "learning_rate": 0.00036713139418254764, + "loss": 2.7786, + "theoretical_loss": 3.6857604681612646, + "tokens_seen": 901428224 + }, + { + "epoch": 3.0, + "learning_rate": 0.0003671213640922768, + "loss": 2.7034, + "theoretical_loss": 3.6857344767067657, + "tokens_seen": 901493760 + }, + { + "epoch": 3.0, + "learning_rate": 0.000367111334002006, + "loss": 2.8064, + "theoretical_loss": 3.6857084876707193, + "tokens_seen": 901559296 + }, + { + "epoch": 3.0, + "learning_rate": 0.00036710130391173524, + "loss": 2.7424, + "theoretical_loss": 3.6856825010527245, + "tokens_seen": 901624832 + }, + { + "epoch": 3.0, + "learning_rate": 0.0003670912738214644, + "loss": 2.7533, + "theoretical_loss": 3.685656516852381, + "tokens_seen": 901690368 + }, + { + "epoch": 3.0, + "learning_rate": 0.0003670812437311936, + "loss": 2.9162, + "theoretical_loss": 3.6856305350692873, + "tokens_seen": 901755904 + }, + { + "epoch": 3.0, + "learning_rate": 0.0003670712136409228, + "loss": 2.7772, + "theoretical_loss": 3.685604555703044, + "tokens_seen": 901821440 + }, + { + "epoch": 3.0, + "learning_rate": 0.000367061183550652, + "loss": 2.8262, + "theoretical_loss": 3.68557857875325, + "tokens_seen": 901886976 + }, + { + "epoch": 3.0, + "learning_rate": 0.00036705115346038114, + "loss": 2.5888, + "theoretical_loss": 3.6855526042195046, + "tokens_seen": 901952512 + }, + { + "epoch": 3.0, + "learning_rate": 0.0003670411233701104, + "loss": 2.4701, + "theoretical_loss": 3.685526632101408, + "tokens_seen": 902018048 + }, + { + "epoch": 3.0, + "learning_rate": 0.0003670310932798395, + "loss": 3.0106, + "theoretical_loss": 3.6855006623985602, + "tokens_seen": 902083584 + }, + { + "epoch": 3.0, + "learning_rate": 0.00036702106318956874, + "loss": 2.5845, + "theoretical_loss": 3.6854746951105612, + "tokens_seen": 902149120 + }, + { + "epoch": 3.0, + "learning_rate": 0.0003670110330992979, + "loss": 2.738, + "theoretical_loss": 3.6854487302370105, + "tokens_seen": 902214656 + }, + { + "epoch": 3.0, + "learning_rate": 0.0003670010030090271, + "loss": 2.7965, + "theoretical_loss": 3.685422767777509, + "tokens_seen": 902280192 + }, + { + "epoch": 3.0, + "learning_rate": 0.0003669909729187563, + "loss": 2.5952, + "theoretical_loss": 3.6853968077316566, + "tokens_seen": 902345728 + }, + { + "epoch": 3.0, + "learning_rate": 0.00036698094282848546, + "loss": 2.7099, + "theoretical_loss": 3.685370850099053, + "tokens_seen": 902411264 + }, + { + "epoch": 3.0, + "learning_rate": 0.00036697091273821464, + "loss": 2.9008, + "theoretical_loss": 3.6853448948792993, + "tokens_seen": 902476800 + }, + { + "epoch": 3.0, + "learning_rate": 0.0003669608826479439, + "loss": 2.5409, + "theoretical_loss": 3.685318942071996, + "tokens_seen": 902542336 + }, + { + "epoch": 3.0, + "learning_rate": 0.000366950852557673, + "loss": 2.6951, + "theoretical_loss": 3.685292991676744, + "tokens_seen": 902607872 + }, + { + "epoch": 3.0, + "learning_rate": 0.00036694082246740224, + "loss": 2.9099, + "theoretical_loss": 3.685267043693144, + "tokens_seen": 902673408 + }, + { + "epoch": 3.0, + "learning_rate": 0.00036693079237713137, + "loss": 2.5979, + "theoretical_loss": 3.6852410981207955, + "tokens_seen": 902738944 + }, + { + "epoch": 3.0, + "objective/train/docs_used": 1061572, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.726848840713501, + "objective/train/theoretical_loss": 3.68523461210442, + "objective/train/tokens_used": 923215328, + "theoretical_loss": 3.68523461210442, + "tokens_seen": 902755328 + }, + { + "epoch": 3.0, + "learning_rate": 0.0003669207622868606, + "loss": 2.8534, + "theoretical_loss": 3.685215154959301, + "tokens_seen": 902804480 + }, + { + "epoch": 3.0, + "learning_rate": 0.0003669107321965898, + "loss": 2.8183, + "theoretical_loss": 3.6851892142082607, + "tokens_seen": 902870016 + }, + { + "epoch": 3.0, + "learning_rate": 0.00036690070210631897, + "loss": 2.7522, + "theoretical_loss": 3.685163275867276, + "tokens_seen": 902935552 + }, + { + "epoch": 3.0, + "learning_rate": 0.00036689067201604815, + "loss": 2.7613, + "theoretical_loss": 3.685137339935948, + "tokens_seen": 903001088 + }, + { + "epoch": 3.0, + "learning_rate": 0.0003668806419257774, + "loss": 2.7894, + "theoretical_loss": 3.6851114064138777, + "tokens_seen": 903066624 + }, + { + "epoch": 3.0, + "learning_rate": 0.0003668706118355065, + "loss": 2.8125, + "theoretical_loss": 3.685085475300667, + "tokens_seen": 903132160 + }, + { + "epoch": 3.0, + "learning_rate": 0.00036686058174523575, + "loss": 2.7282, + "theoretical_loss": 3.685059546595917, + "tokens_seen": 903197696 + }, + { + "epoch": 3.0, + "learning_rate": 0.00036685055165496487, + "loss": 2.8206, + "theoretical_loss": 3.6850336202992295, + "tokens_seen": 903263232 + }, + { + "epoch": 3.0, + "learning_rate": 0.0003668405215646941, + "loss": 2.7565, + "theoretical_loss": 3.6850076964102056, + "tokens_seen": 903328768 + }, + { + "epoch": 3.0, + "learning_rate": 0.0003668304914744233, + "loss": 2.7826, + "theoretical_loss": 3.684981774928448, + "tokens_seen": 903394304 + }, + { + "epoch": 3.0, + "learning_rate": 0.00036682046138415247, + "loss": 2.8357, + "theoretical_loss": 3.684955855853558, + "tokens_seen": 903459840 + }, + { + "epoch": 3.0, + "learning_rate": 0.00036681043129388165, + "loss": 2.8705, + "theoretical_loss": 3.684929939185137, + "tokens_seen": 903525376 + }, + { + "epoch": 3.0, + "learning_rate": 0.00036680040120361083, + "loss": 2.6921, + "theoretical_loss": 3.6849040249227887, + "tokens_seen": 903590912 + }, + { + "epoch": 3.0, + "learning_rate": 0.00036679037111334, + "loss": 2.8528, + "theoretical_loss": 3.6848781130661137, + "tokens_seen": 903656448 + }, + { + "epoch": 3.0, + "learning_rate": 0.00036678034102306925, + "loss": 2.7383, + "theoretical_loss": 3.6848522036147147, + "tokens_seen": 903721984 + }, + { + "epoch": 3.0, + "learning_rate": 0.0003667703109327984, + "loss": 2.7307, + "theoretical_loss": 3.684826296568194, + "tokens_seen": 903787520 + }, + { + "epoch": 3.0, + "learning_rate": 0.0003667602808425276, + "loss": 2.6868, + "theoretical_loss": 3.6848003919261547, + "tokens_seen": 903853056 + }, + { + "epoch": 3.0, + "learning_rate": 0.00036675025075225674, + "loss": 2.8841, + "theoretical_loss": 3.6847744896881975, + "tokens_seen": 903918592 + }, + { + "epoch": 3.0, + "learning_rate": 0.000366740220661986, + "loss": 2.8175, + "theoretical_loss": 3.6847485898539274, + "tokens_seen": 903984128 + }, + { + "epoch": 3.0, + "learning_rate": 0.00036673019057171515, + "loss": 2.577, + "theoretical_loss": 3.6847226924229455, + "tokens_seen": 904049664 + }, + { + "epoch": 3.0, + "learning_rate": 0.00036672016048144434, + "loss": 2.6837, + "theoretical_loss": 3.684696797394855, + "tokens_seen": 904115200 + }, + { + "epoch": 3.0, + "learning_rate": 0.0003667101303911735, + "loss": 2.7804, + "theoretical_loss": 3.6846709047692587, + "tokens_seen": 904180736 + }, + { + "epoch": 3.0, + "learning_rate": 0.00036670010030090275, + "loss": 2.847, + "theoretical_loss": 3.68464501454576, + "tokens_seen": 904246272 + }, + { + "epoch": 3.0, + "learning_rate": 0.0003666900702106319, + "loss": 2.7734, + "theoretical_loss": 3.6846191267239616, + "tokens_seen": 904311808 + }, + { + "epoch": 3.0, + "learning_rate": 0.0003666800401203611, + "loss": 2.7455, + "theoretical_loss": 3.6845932413034665, + "tokens_seen": 904377344 + }, + { + "epoch": 3.0, + "objective/train/docs_used": 1066630, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.873715877532959, + "objective/train/theoretical_loss": 3.6845867703234996, + "objective/train/tokens_used": 924853728, + "theoretical_loss": 3.6845867703234996, + "tokens_seen": 904393728 + }, + { + "epoch": 3.0, + "learning_rate": 0.00036667001003009024, + "loss": 2.8156, + "theoretical_loss": 3.6845673582838785, + "tokens_seen": 904442880 + }, + { + "epoch": 3.0, + "learning_rate": 0.0003666599799398195, + "loss": 2.7413, + "theoretical_loss": 3.6845414776648004, + "tokens_seen": 904508416 + }, + { + "epoch": 3.0, + "learning_rate": 0.00036664994984954866, + "loss": 2.7902, + "theoretical_loss": 3.6845155994458363, + "tokens_seen": 904573952 + }, + { + "epoch": 3.0, + "learning_rate": 0.00036663991975927784, + "loss": 2.6701, + "theoretical_loss": 3.684489723626589, + "tokens_seen": 904639488 + }, + { + "epoch": 3.0, + "learning_rate": 0.000366629889669007, + "loss": 2.6021, + "theoretical_loss": 3.6844638502066633, + "tokens_seen": 904705024 + }, + { + "epoch": 3.0, + "learning_rate": 0.0003666198595787362, + "loss": 2.6243, + "theoretical_loss": 3.6844379791856614, + "tokens_seen": 904770560 + }, + { + "epoch": 3.0, + "learning_rate": 0.0003666098294884654, + "loss": 2.8246, + "theoretical_loss": 3.684412110563189, + "tokens_seen": 904836096 + }, + { + "epoch": 3.0, + "learning_rate": 0.0003665997993981946, + "loss": 2.5495, + "theoretical_loss": 3.6843862443388478, + "tokens_seen": 904901632 + }, + { + "epoch": 3.0, + "learning_rate": 0.00036658976930792374, + "loss": 2.7142, + "theoretical_loss": 3.684360380512244, + "tokens_seen": 904967168 + }, + { + "epoch": 3.0, + "learning_rate": 0.000366579739217653, + "loss": 2.7124, + "theoretical_loss": 3.68433451908298, + "tokens_seen": 905032704 + }, + { + "epoch": 3.0, + "learning_rate": 0.00036656970912738216, + "loss": 2.8297, + "theoretical_loss": 3.684308660050661, + "tokens_seen": 905098240 + }, + { + "epoch": 3.0, + "learning_rate": 0.00036655967903711134, + "loss": 2.625, + "theoretical_loss": 3.6842828034148916, + "tokens_seen": 905163776 + }, + { + "epoch": 3.0, + "learning_rate": 0.0003665496489468405, + "loss": 2.7145, + "theoretical_loss": 3.684256949175275, + "tokens_seen": 905229312 + }, + { + "epoch": 3.0, + "learning_rate": 0.0003665396188565697, + "loss": 2.7347, + "theoretical_loss": 3.6842310973314163, + "tokens_seen": 905294848 + }, + { + "epoch": 3.0, + "learning_rate": 0.0003665295887662989, + "loss": 2.7359, + "theoretical_loss": 3.684205247882921, + "tokens_seen": 905360384 + }, + { + "epoch": 3.0, + "learning_rate": 0.0003665195586760281, + "loss": 2.8178, + "theoretical_loss": 3.684179400829392, + "tokens_seen": 905425920 + }, + { + "epoch": 3.0, + "learning_rate": 0.00036650952858575725, + "loss": 2.8469, + "theoretical_loss": 3.6841535561704353, + "tokens_seen": 905491456 + }, + { + "epoch": 3.0, + "learning_rate": 0.0003664994984954865, + "loss": 2.7027, + "theoretical_loss": 3.6841277139056556, + "tokens_seen": 905556992 + }, + { + "epoch": 3.0, + "learning_rate": 0.0003664894684052156, + "loss": 2.7502, + "theoretical_loss": 3.6841018740346576, + "tokens_seen": 905622528 + }, + { + "epoch": 3.0, + "learning_rate": 0.00036647943831494484, + "loss": 2.7623, + "theoretical_loss": 3.6840760365570464, + "tokens_seen": 905688064 + }, + { + "epoch": 3.0, + "learning_rate": 0.000366469408224674, + "loss": 2.6832, + "theoretical_loss": 3.684050201472428, + "tokens_seen": 905753600 + }, + { + "epoch": 3.0, + "learning_rate": 0.0003664593781344032, + "loss": 2.9027, + "theoretical_loss": 3.684024368780406, + "tokens_seen": 905819136 + }, + { + "epoch": 3.0, + "learning_rate": 0.0003664493480441324, + "loss": 2.8143, + "theoretical_loss": 3.683998538480587, + "tokens_seen": 905884672 + }, + { + "epoch": 3.0, + "learning_rate": 0.00036643931795386157, + "loss": 2.7001, + "theoretical_loss": 3.6839727105725766, + "tokens_seen": 905950208 + }, + { + "epoch": 3.0, + "learning_rate": 0.00036642928786359075, + "loss": 2.7919, + "theoretical_loss": 3.683946885055979, + "tokens_seen": 906015744 + }, + { + "epoch": 3.0, + "objective/train/docs_used": 1071650, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7729568481445312, + "objective/train/theoretical_loss": 3.683940429050442, + "objective/train/tokens_used": 926492128, + "theoretical_loss": 3.683940429050442, + "tokens_seen": 906032128 + }, + { + "epoch": 3.0, + "learning_rate": 0.00036641925777332, + "loss": 2.7335, + "theoretical_loss": 3.683921061930401, + "tokens_seen": 906081280 + }, + { + "epoch": 3.0, + "learning_rate": 0.0003664092276830491, + "loss": 2.7948, + "theoretical_loss": 3.6838952411954473, + "tokens_seen": 906146816 + }, + { + "epoch": 3.0, + "learning_rate": 0.00036639919759277835, + "loss": 2.8415, + "theoretical_loss": 3.6838694228507256, + "tokens_seen": 906212352 + }, + { + "epoch": 3.0, + "learning_rate": 0.00036638916750250753, + "loss": 2.8463, + "theoretical_loss": 3.68384360689584, + "tokens_seen": 906277888 + }, + { + "epoch": 3.0, + "learning_rate": 0.0003663791374122367, + "loss": 2.9069, + "theoretical_loss": 3.6838177933303964, + "tokens_seen": 906343424 + }, + { + "epoch": 3.0, + "learning_rate": 0.0003663691073219659, + "loss": 2.8556, + "theoretical_loss": 3.683791982154002, + "tokens_seen": 906408960 + }, + { + "epoch": 3.0, + "learning_rate": 0.00036635907723169507, + "loss": 2.725, + "theoretical_loss": 3.683766173366263, + "tokens_seen": 906474496 + }, + { + "epoch": 3.0, + "learning_rate": 0.0003663490471414243, + "loss": 2.5566, + "theoretical_loss": 3.6837403669667843, + "tokens_seen": 906540032 + }, + { + "epoch": 3.0, + "learning_rate": 0.0003663390170511535, + "loss": 2.7766, + "theoretical_loss": 3.683714562955174, + "tokens_seen": 906605568 + }, + { + "epoch": 3.0, + "learning_rate": 0.00036632898696088267, + "loss": 2.9063, + "theoretical_loss": 3.6836887613310374, + "tokens_seen": 906671104 + }, + { + "epoch": 3.0, + "learning_rate": 0.00036631895687061185, + "loss": 2.8221, + "theoretical_loss": 3.6836629620939814, + "tokens_seen": 906736640 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036630892678034103, + "loss": 2.9086, + "theoretical_loss": 3.6836371652436126, + "tokens_seen": 906802176 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003662988966900702, + "loss": 2.8452, + "theoretical_loss": 3.6836113707795377, + "tokens_seen": 906867712 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036628886659979945, + "loss": 2.7909, + "theoretical_loss": 3.6835855787013636, + "tokens_seen": 906933248 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003662788365095286, + "loss": 2.8312, + "theoretical_loss": 3.6835597890086973, + "tokens_seen": 906998784 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003662688064192578, + "loss": 2.5495, + "theoretical_loss": 3.6835340017011458, + "tokens_seen": 907064320 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036625877632898694, + "loss": 2.7974, + "theoretical_loss": 3.683508216778316, + "tokens_seen": 907129856 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003662487462387162, + "loss": 2.7805, + "theoretical_loss": 3.6834824342398154, + "tokens_seen": 907195392 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036623871614844535, + "loss": 2.8686, + "theoretical_loss": 3.6834566540852514, + "tokens_seen": 907260928 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036622868605817454, + "loss": 2.9814, + "theoretical_loss": 3.683430876314231, + "tokens_seen": 907326464 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003662186559679037, + "loss": 2.874, + "theoretical_loss": 3.683405100926362, + "tokens_seen": 907392000 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036620862587763295, + "loss": 2.522, + "theoretical_loss": 3.6833793279212514, + "tokens_seen": 907457536 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003661985957873621, + "loss": 2.6894, + "theoretical_loss": 3.683353557298507, + "tokens_seen": 907523072 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003661885656970913, + "loss": 2.8295, + "theoretical_loss": 3.683327789057738, + "tokens_seen": 907588608 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036617853560682044, + "loss": 2.799, + "theoretical_loss": 3.683302023198551, + "tokens_seen": 907654144 + }, + { + "epoch": 3.01, + "objective/train/docs_used": 1076755, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.107651472091675, + "objective/train/theoretical_loss": 3.68329558210583, + "objective/train/tokens_used": 928130528, + "theoretical_loss": 3.68329558210583, + "tokens_seen": 907670528 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003661685055165497, + "loss": 2.9, + "theoretical_loss": 3.6832762597205533, + "tokens_seen": 907719680 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036615847542627886, + "loss": 2.9029, + "theoretical_loss": 3.6832504986233543, + "tokens_seen": 907785216 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036614844533600804, + "loss": 2.8958, + "theoretical_loss": 3.683224739906561, + "tokens_seen": 907850752 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003661384152457372, + "loss": 2.709, + "theoretical_loss": 3.683198983569783, + "tokens_seen": 907916288 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003661283851554664, + "loss": 2.6734, + "theoretical_loss": 3.6831732296126276, + "tokens_seen": 907981824 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003661183550651956, + "loss": 2.746, + "theoretical_loss": 3.683147478034703, + "tokens_seen": 908047360 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003661083249749248, + "loss": 2.7898, + "theoretical_loss": 3.6831217288356184, + "tokens_seen": 908112896 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036609829488465394, + "loss": 2.7533, + "theoretical_loss": 3.683095982014982, + "tokens_seen": 908178432 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003660882647943832, + "loss": 2.5579, + "theoretical_loss": 3.6830702375724025, + "tokens_seen": 908243968 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036607823470411236, + "loss": 2.7278, + "theoretical_loss": 3.6830444955074895, + "tokens_seen": 908309504 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036606820461384154, + "loss": 2.7775, + "theoretical_loss": 3.683018755819851, + "tokens_seen": 908375040 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003660581745235707, + "loss": 2.8285, + "theoretical_loss": 3.6829930185090958, + "tokens_seen": 908440576 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003660481444332999, + "loss": 2.7422, + "theoretical_loss": 3.6829672835748335, + "tokens_seen": 908506112 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003660381143430291, + "loss": 2.8331, + "theoretical_loss": 3.682941551016673, + "tokens_seen": 908571648 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003660280842527583, + "loss": 2.7105, + "theoretical_loss": 3.682915820834224, + "tokens_seen": 908637184 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036601805416248745, + "loss": 2.7307, + "theoretical_loss": 3.682890093027095, + "tokens_seen": 908702720 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003660080240722167, + "loss": 2.8438, + "theoretical_loss": 3.682864367594896, + "tokens_seen": 908768256 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003659979939819458, + "loss": 2.9085, + "theoretical_loss": 3.682838644537237, + "tokens_seen": 908833792 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036598796389167504, + "loss": 2.607, + "theoretical_loss": 3.6828129238537266, + "tokens_seen": 908899328 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003659779338014042, + "loss": 2.7233, + "theoretical_loss": 3.682787205543975, + "tokens_seen": 908964864 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003659679037111334, + "loss": 2.5585, + "theoretical_loss": 3.682761489607592, + "tokens_seen": 909030400 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003659578736208626, + "loss": 2.6669, + "theoretical_loss": 3.6827357760441877, + "tokens_seen": 909095936 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036594784353059177, + "loss": 2.7123, + "theoretical_loss": 3.6827100648533717, + "tokens_seen": 909161472 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036593781344032095, + "loss": 2.8216, + "theoretical_loss": 3.682684356034754, + "tokens_seen": 909227008 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003659277833500502, + "loss": 2.5725, + "theoretical_loss": 3.682658649587945, + "tokens_seen": 909292544 + }, + { + "epoch": 3.01, + "objective/train/docs_used": 1081876, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.961531162261963, + "objective/train/theoretical_loss": 3.6826522233467927, + "objective/train/tokens_used": 929768928, + "theoretical_loss": 3.6826522233467927, + "tokens_seen": 909308928 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003659177532597793, + "loss": 2.6922, + "theoretical_loss": 3.6826329455125553, + "tokens_seen": 909358080 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036590772316950855, + "loss": 2.8059, + "theoretical_loss": 3.682607243808195, + "tokens_seen": 909423616 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036589769307923773, + "loss": 2.6864, + "theoretical_loss": 3.6825815444744743, + "tokens_seen": 909489152 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003658876629889669, + "loss": 2.6581, + "theoretical_loss": 3.682555847511004, + "tokens_seen": 909554688 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003658776328986961, + "loss": 2.6658, + "theoretical_loss": 3.6825301529173946, + "tokens_seen": 909620224 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036586760280842527, + "loss": 2.9523, + "theoretical_loss": 3.6825044606932567, + "tokens_seen": 909685760 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036585757271815445, + "loss": 2.8693, + "theoretical_loss": 3.682478770838202, + "tokens_seen": 909751296 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003658475426278837, + "loss": 2.8812, + "theoretical_loss": 3.6824530833518394, + "tokens_seen": 909816832 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003658375125376128, + "loss": 2.7843, + "theoretical_loss": 3.682427398233782, + "tokens_seen": 909882368 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036582748244734205, + "loss": 2.7056, + "theoretical_loss": 3.68240171548364, + "tokens_seen": 909947904 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003658174523570712, + "loss": 2.8099, + "theoretical_loss": 3.682376035101025, + "tokens_seen": 910013440 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003658074222668004, + "loss": 2.5475, + "theoretical_loss": 3.6823503570855474, + "tokens_seen": 910078976 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003657973921765296, + "loss": 2.8597, + "theoretical_loss": 3.6823246814368193, + "tokens_seen": 910144512 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003657873620862588, + "loss": 2.8058, + "theoretical_loss": 3.6822990081544518, + "tokens_seen": 910210048 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036577733199598796, + "loss": 2.8276, + "theoretical_loss": 3.6822733372380574, + "tokens_seen": 910275584 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036576730190571714, + "loss": 2.5292, + "theoretical_loss": 3.6822476686872463, + "tokens_seen": 910341120 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003657572718154463, + "loss": 2.7811, + "theoretical_loss": 3.682222002501631, + "tokens_seen": 910406656 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036574724172517555, + "loss": 2.9896, + "theoretical_loss": 3.682196338680823, + "tokens_seen": 910472192 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003657372116349047, + "loss": 2.6515, + "theoretical_loss": 3.682170677224435, + "tokens_seen": 910537728 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003657271815446339, + "loss": 2.842, + "theoretical_loss": 3.682145018132078, + "tokens_seen": 910603264 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003657171514543631, + "loss": 2.5961, + "theoretical_loss": 3.6821193614033643, + "tokens_seen": 910668800 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003657071213640923, + "loss": 2.8136, + "theoretical_loss": 3.682093707037907, + "tokens_seen": 910734336 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036569709127382146, + "loss": 2.8072, + "theoretical_loss": 3.6820680550353178, + "tokens_seen": 910799872 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036568706118355064, + "loss": 2.8705, + "theoretical_loss": 3.682042405395208, + "tokens_seen": 910865408 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003656770310932798, + "loss": 2.6743, + "theoretical_loss": 3.682016758117192, + "tokens_seen": 910930944 + }, + { + "epoch": 3.01, + "objective/train/docs_used": 1086888, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.870638132095337, + "objective/train/theoretical_loss": 3.6820103466667193, + "objective/train/tokens_used": 931407328, + "theoretical_loss": 3.6820103466667193, + "tokens_seen": 910947328 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036566700100300906, + "loss": 2.7901, + "theoretical_loss": 3.681991113200881, + "tokens_seen": 910996480 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003656569709127382, + "loss": 2.7127, + "theoretical_loss": 3.6819654706458884, + "tokens_seen": 911062016 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003656469408224674, + "loss": 2.7062, + "theoretical_loss": 3.6819398304518267, + "tokens_seen": 911127552 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036563691073219655, + "loss": 2.7506, + "theoretical_loss": 3.6819141926183083, + "tokens_seen": 911193088 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003656268806419258, + "loss": 2.6625, + "theoretical_loss": 3.681888557144947, + "tokens_seen": 911258624 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036561685055165496, + "loss": 2.6967, + "theoretical_loss": 3.6818629240313543, + "tokens_seen": 911324160 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036560682046138414, + "loss": 2.8149, + "theoretical_loss": 3.6818372932771455, + "tokens_seen": 911389696 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003655967903711134, + "loss": 2.6254, + "theoretical_loss": 3.681811664881932, + "tokens_seen": 911455232 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036558676028084256, + "loss": 2.7563, + "theoretical_loss": 3.6817860388453276, + "tokens_seen": 911520768 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036557673019057174, + "loss": 2.7822, + "theoretical_loss": 3.681760415166946, + "tokens_seen": 911586304 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003655667001003009, + "loss": 2.8973, + "theoretical_loss": 3.6817347938464002, + "tokens_seen": 911651840 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003655566700100301, + "loss": 2.607, + "theoretical_loss": 3.681709174883304, + "tokens_seen": 911717376 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003655466399197593, + "loss": 2.6586, + "theoretical_loss": 3.681683558277271, + "tokens_seen": 911782912 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003655366098294885, + "loss": 2.852, + "theoretical_loss": 3.6816579440279154, + "tokens_seen": 911848448 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036552657973921765, + "loss": 2.7226, + "theoretical_loss": 3.6816323321348507, + "tokens_seen": 911913984 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003655165496489469, + "loss": 2.6382, + "theoretical_loss": 3.68160672259769, + "tokens_seen": 911979520 + }, + { + "epoch": 3.01, + "learning_rate": 0.000365506519558676, + "loss": 2.8484, + "theoretical_loss": 3.6815811154160487, + "tokens_seen": 912045056 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036549648946840525, + "loss": 2.6745, + "theoretical_loss": 3.68155551058954, + "tokens_seen": 912110592 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003654864593781344, + "loss": 2.8237, + "theoretical_loss": 3.6815299081177786, + "tokens_seen": 912176128 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003654764292878636, + "loss": 3.1232, + "theoretical_loss": 3.6815043080003784, + "tokens_seen": 912241664 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003654663991975928, + "loss": 2.6531, + "theoretical_loss": 3.6814787102369544, + "tokens_seen": 912307200 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036545636910732197, + "loss": 2.7191, + "theoretical_loss": 3.6814531148271206, + "tokens_seen": 912372736 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036544633901705115, + "loss": 2.5767, + "theoretical_loss": 3.681427521770491, + "tokens_seen": 912438272 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003654363089267804, + "loss": 2.8236, + "theoretical_loss": 3.6814019310666812, + "tokens_seen": 912503808 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003654262788365095, + "loss": 2.7541, + "theoretical_loss": 3.6813763427153052, + "tokens_seen": 912569344 + }, + { + "epoch": 3.01, + "objective/train/docs_used": 1091883, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6327216625213623, + "objective/train/theoretical_loss": 3.6813699459949842, + "objective/train/tokens_used": 933045728, + "theoretical_loss": 3.6813699459949842, + "tokens_seen": 912585728 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036541624874623875, + "loss": 2.6855, + "theoretical_loss": 3.681350756715979, + "tokens_seen": 912634880 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036540621865596793, + "loss": 2.6951, + "theoretical_loss": 3.6813251730683163, + "tokens_seen": 912700416 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003653961885656971, + "loss": 2.8262, + "theoretical_loss": 3.681299591771933, + "tokens_seen": 912765952 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003653861584754263, + "loss": 2.7102, + "theoretical_loss": 3.681274012826443, + "tokens_seen": 912831488 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036537612838515547, + "loss": 2.6939, + "theoretical_loss": 3.6812484362314626, + "tokens_seen": 912897024 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036536609829488465, + "loss": 2.8567, + "theoretical_loss": 3.681222861986607, + "tokens_seen": 912962560 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003653560682046139, + "loss": 2.7321, + "theoretical_loss": 3.6811972900914913, + "tokens_seen": 913028096 + }, + { + "epoch": 3.01, + "learning_rate": 0.000365346038114343, + "loss": 2.6525, + "theoretical_loss": 3.6811717205457306, + "tokens_seen": 913093632 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036533600802407225, + "loss": 2.6161, + "theoretical_loss": 3.6811461533489416, + "tokens_seen": 913159168 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003653259779338014, + "loss": 2.7166, + "theoretical_loss": 3.6811205885007388, + "tokens_seen": 913224704 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003653159478435306, + "loss": 2.5017, + "theoretical_loss": 3.6810950260007385, + "tokens_seen": 913290240 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003653059177532598, + "loss": 2.8044, + "theoretical_loss": 3.681069465848556, + "tokens_seen": 913355776 + }, + { + "epoch": 3.01, + "learning_rate": 0.000365295887662989, + "loss": 2.7958, + "theoretical_loss": 3.681043908043808, + "tokens_seen": 913421312 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036528585757271816, + "loss": 2.7558, + "theoretical_loss": 3.6810183525861095, + "tokens_seen": 913486848 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036527582748244734, + "loss": 2.6713, + "theoretical_loss": 3.680992799475078, + "tokens_seen": 913552384 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003652657973921765, + "loss": 2.7104, + "theoretical_loss": 3.6809672487103287, + "tokens_seen": 913617920 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036525576730190575, + "loss": 2.8065, + "theoretical_loss": 3.680941700291478, + "tokens_seen": 913683456 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003652457372116349, + "loss": 2.6175, + "theoretical_loss": 3.680916154218143, + "tokens_seen": 913748992 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003652357071213641, + "loss": 2.7811, + "theoretical_loss": 3.680890610489939, + "tokens_seen": 913814528 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003652256770310933, + "loss": 2.7338, + "theoretical_loss": 3.680865069106483, + "tokens_seen": 913880064 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003652156469408225, + "loss": 2.7676, + "theoretical_loss": 3.680839530067392, + "tokens_seen": 913945600 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036520561685055166, + "loss": 2.6906, + "theoretical_loss": 3.680813993372282, + "tokens_seen": 914011136 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036519558676028084, + "loss": 2.9039, + "theoretical_loss": 3.680788459020771, + "tokens_seen": 914076672 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036518555667001, + "loss": 2.7388, + "theoretical_loss": 3.680762927012475, + "tokens_seen": 914142208 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036517552657973926, + "loss": 2.8223, + "theoretical_loss": 3.680737397347011, + "tokens_seen": 914207744 + }, + { + "epoch": 3.01, + "objective/train/docs_used": 1093340, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.954491138458252, + "objective/train/theoretical_loss": 3.6807310152966677, + "objective/train/tokens_used": 934684128, + "theoretical_loss": 3.6807310152966677, + "tokens_seen": 914224128 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003651654964894684, + "loss": 3.0444, + "theoretical_loss": 3.6807118700239965, + "tokens_seen": 914273280 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003651554663991976, + "loss": 2.813, + "theoretical_loss": 3.6806863450430485, + "tokens_seen": 914338816 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036514543630892675, + "loss": 2.7931, + "theoretical_loss": 3.680660822403784, + "tokens_seen": 914404352 + }, + { + "epoch": 3.01, + "learning_rate": 0.000365135406218656, + "loss": 2.7916, + "theoretical_loss": 3.680635302105821, + "tokens_seen": 914469888 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036512537612838516, + "loss": 2.5788, + "theoretical_loss": 3.6806097841487766, + "tokens_seen": 914535424 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036511534603811434, + "loss": 2.7142, + "theoretical_loss": 3.680584268532268, + "tokens_seen": 914600960 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003651053159478435, + "loss": 2.8717, + "theoretical_loss": 3.680558755255914, + "tokens_seen": 914666496 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036509528585757276, + "loss": 2.8554, + "theoretical_loss": 3.6805332443193306, + "tokens_seen": 914732032 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003650852557673019, + "loss": 2.6752, + "theoretical_loss": 3.680507735722137, + "tokens_seen": 914797568 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003650752256770311, + "loss": 2.6707, + "theoretical_loss": 3.6804822294639505, + "tokens_seen": 914863104 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036506519558676025, + "loss": 2.7038, + "theoretical_loss": 3.6804567255443894, + "tokens_seen": 914928640 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003650551654964895, + "loss": 2.8116, + "theoretical_loss": 3.680431223963071, + "tokens_seen": 914994176 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036504513540621867, + "loss": 2.8997, + "theoretical_loss": 3.6804057247196145, + "tokens_seen": 915059712 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036503510531594785, + "loss": 2.6417, + "theoretical_loss": 3.6803802278136377, + "tokens_seen": 915125248 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036502507522567703, + "loss": 2.8395, + "theoretical_loss": 3.6803547332447595, + "tokens_seen": 915190784 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003650150451354062, + "loss": 2.854, + "theoretical_loss": 3.680329241012597, + "tokens_seen": 915256320 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003650050150451354, + "loss": 2.9282, + "theoretical_loss": 3.68030375111677, + "tokens_seen": 915321856 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003649949849548646, + "loss": 2.7012, + "theoretical_loss": 3.6802782635568967, + "tokens_seen": 915387392 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036498495486459375, + "loss": 2.7663, + "theoretical_loss": 3.6802527783325956, + "tokens_seen": 915452928 + }, + { + "epoch": 3.01, + "learning_rate": 0.000364974924774323, + "loss": 2.8494, + "theoretical_loss": 3.680227295443486, + "tokens_seen": 915518464 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003649648946840521, + "loss": 2.595, + "theoretical_loss": 3.680201814889186, + "tokens_seen": 915584000 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036495486459378135, + "loss": 2.8819, + "theoretical_loss": 3.6801763366693154, + "tokens_seen": 915649536 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036494483450351053, + "loss": 2.5959, + "theoretical_loss": 3.6801508607834927, + "tokens_seen": 915715072 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003649348044132397, + "loss": 2.773, + "theoretical_loss": 3.680125387231338, + "tokens_seen": 915780608 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003649247743229689, + "loss": 2.7749, + "theoretical_loss": 3.680099916012469, + "tokens_seen": 915846144 + }, + { + "epoch": 3.01, + "objective/train/docs_used": 1093676, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.2436115741729736, + "objective/train/theoretical_loss": 3.6800935485722834, + "objective/train/tokens_used": 936322528, + "theoretical_loss": 3.6800935485722834, + "tokens_seen": 915862528 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036491474423269813, + "loss": 2.8122, + "theoretical_loss": 3.6800744471265063, + "tokens_seen": 915911680 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036490471414242726, + "loss": 2.7161, + "theoretical_loss": 3.680048980573069, + "tokens_seen": 915977216 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003648946840521565, + "loss": 2.7632, + "theoretical_loss": 3.6800235163517767, + "tokens_seen": 916042752 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003648846539618856, + "loss": 2.7306, + "theoretical_loss": 3.6799980544622484, + "tokens_seen": 916108288 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036487462387161485, + "loss": 2.8455, + "theoretical_loss": 3.679972594904104, + "tokens_seen": 916173824 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003648645937813441, + "loss": 2.7776, + "theoretical_loss": 3.679947137676964, + "tokens_seen": 916239360 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003648545636910732, + "loss": 2.7945, + "theoretical_loss": 3.6799216827804475, + "tokens_seen": 916304896 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036484453360080245, + "loss": 2.7072, + "theoretical_loss": 3.679896230214175, + "tokens_seen": 916370432 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003648345035105316, + "loss": 2.6346, + "theoretical_loss": 3.679870779977766, + "tokens_seen": 916435968 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003648244734202608, + "loss": 2.9591, + "theoretical_loss": 3.6798453320708413, + "tokens_seen": 916501504 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036481444332999, + "loss": 2.687, + "theoretical_loss": 3.6798198864930205, + "tokens_seen": 916567040 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003648044132397192, + "loss": 2.5467, + "theoretical_loss": 3.679794443243924, + "tokens_seen": 916632576 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036479438314944836, + "loss": 2.6591, + "theoretical_loss": 3.6797690023231726, + "tokens_seen": 916698112 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036478435305917754, + "loss": 2.8308, + "theoretical_loss": 3.6797435637303866, + "tokens_seen": 916763648 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003647743229689067, + "loss": 2.8794, + "theoretical_loss": 3.6797181274651867, + "tokens_seen": 916829184 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036476429287863595, + "loss": 2.9907, + "theoretical_loss": 3.679692693527193, + "tokens_seen": 916894720 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003647542627883651, + "loss": 2.7011, + "theoretical_loss": 3.679667261916027, + "tokens_seen": 916960256 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003647442326980943, + "loss": 2.816, + "theoretical_loss": 3.6796418326313094, + "tokens_seen": 917025792 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003647342026078235, + "loss": 2.8047, + "theoretical_loss": 3.679616405672661, + "tokens_seen": 917091328 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003647241725175527, + "loss": 2.5946, + "theoretical_loss": 3.679590981039703, + "tokens_seen": 917156864 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036471414242728186, + "loss": 2.7901, + "theoretical_loss": 3.6795655587320555, + "tokens_seen": 917222400 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036470411233701104, + "loss": 2.6762, + "theoretical_loss": 3.679540138749341, + "tokens_seen": 917287936 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003646940822467402, + "loss": 2.7547, + "theoretical_loss": 3.6795147210911807, + "tokens_seen": 917353472 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036468405215646946, + "loss": 2.8656, + "theoretical_loss": 3.679489305757196, + "tokens_seen": 917419008 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003646740220661986, + "loss": 2.8932, + "theoretical_loss": 3.679463892747007, + "tokens_seen": 917484544 + }, + { + "epoch": 3.01, + "objective/train/docs_used": 1095045, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7306671142578125, + "objective/train/theoretical_loss": 3.6794575398575087, + "objective/train/tokens_used": 937960928, + "theoretical_loss": 3.6794575398575087, + "tokens_seen": 917500928 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003646639919759278, + "loss": 2.831, + "theoretical_loss": 3.6794384820602364, + "tokens_seen": 917550080 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036465396188565695, + "loss": 2.8848, + "theoretical_loss": 3.679413073696506, + "tokens_seen": 917615616 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003646439317953862, + "loss": 2.7306, + "theoretical_loss": 3.679387667655438, + "tokens_seen": 917681152 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036463390170511536, + "loss": 2.708, + "theoretical_loss": 3.6793622639366523, + "tokens_seen": 917746688 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036462387161484454, + "loss": 2.6047, + "theoretical_loss": 3.679336862539772, + "tokens_seen": 917812224 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003646138415245737, + "loss": 2.7643, + "theoretical_loss": 3.67931146346442, + "tokens_seen": 917877760 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036460381143430296, + "loss": 2.715, + "theoretical_loss": 3.679286066710217, + "tokens_seen": 917943296 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003645937813440321, + "loss": 2.6738, + "theoretical_loss": 3.679260672276786, + "tokens_seen": 918008832 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003645837512537613, + "loss": 2.8271, + "theoretical_loss": 3.679235280163749, + "tokens_seen": 918074368 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036457372116349045, + "loss": 2.6745, + "theoretical_loss": 3.679209890370728, + "tokens_seen": 918139904 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003645636910732197, + "loss": 2.9066, + "theoretical_loss": 3.6791845028973453, + "tokens_seen": 918205440 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036455366098294887, + "loss": 2.7727, + "theoretical_loss": 3.6791591177432252, + "tokens_seen": 918270976 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036454363089267805, + "loss": 2.7415, + "theoretical_loss": 3.679133734907988, + "tokens_seen": 918336512 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036453360080240723, + "loss": 2.9084, + "theoretical_loss": 3.679108354391258, + "tokens_seen": 918402048 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003645235707121364, + "loss": 2.7322, + "theoretical_loss": 3.6790829761926567, + "tokens_seen": 918467584 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003645135406218656, + "loss": 2.6849, + "theoretical_loss": 3.6790576003118085, + "tokens_seen": 918533120 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003645035105315948, + "loss": 2.623, + "theoretical_loss": 3.679032226748335, + "tokens_seen": 918598656 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036449348044132395, + "loss": 2.8732, + "theoretical_loss": 3.6790068555018607, + "tokens_seen": 918664192 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003644834503510532, + "loss": 2.6817, + "theoretical_loss": 3.678981486572007, + "tokens_seen": 918729728 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003644734202607823, + "loss": 2.8337, + "theoretical_loss": 3.6789561199583987, + "tokens_seen": 918795264 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036446339017051155, + "loss": 2.8237, + "theoretical_loss": 3.6789307556606583, + "tokens_seen": 918860800 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036445336008024073, + "loss": 2.7562, + "theoretical_loss": 3.6789053936784093, + "tokens_seen": 918926336 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003644433299899699, + "loss": 2.6789, + "theoretical_loss": 3.6788800340112755, + "tokens_seen": 918991872 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003644332998996991, + "loss": 2.8857, + "theoretical_loss": 3.6788546766588803, + "tokens_seen": 919057408 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036442326980942833, + "loss": 2.9261, + "theoretical_loss": 3.6788293216208476, + "tokens_seen": 919122944 + }, + { + "epoch": 3.01, + "objective/train/docs_used": 1095626, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8860342502593994, + "objective/train/theoretical_loss": 3.678822983222914, + "objective/train/tokens_used": 939599328, + "theoretical_loss": 3.678822983222914, + "tokens_seen": 919139328 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036441323971915746, + "loss": 2.7617, + "theoretical_loss": 3.6788039688968004, + "tokens_seen": 919188480 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003644032096288867, + "loss": 2.6231, + "theoretical_loss": 3.678778618486364, + "tokens_seen": 919254016 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003643931795386158, + "loss": 2.918, + "theoretical_loss": 3.678753270389161, + "tokens_seen": 919319552 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036438314944834505, + "loss": 2.6964, + "theoretical_loss": 3.6787279246048157, + "tokens_seen": 919385088 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036437311935807424, + "loss": 2.5647, + "theoretical_loss": 3.6787025811329523, + "tokens_seen": 919450624 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003643630892678034, + "loss": 2.8001, + "theoretical_loss": 3.6786772399731955, + "tokens_seen": 919516160 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003643530591775326, + "loss": 2.8182, + "theoretical_loss": 3.6786519011251695, + "tokens_seen": 919581696 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003643430290872618, + "loss": 2.8766, + "theoretical_loss": 3.678626564588498, + "tokens_seen": 919647232 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036433299899699096, + "loss": 2.6051, + "theoretical_loss": 3.678601230362806, + "tokens_seen": 919712768 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003643229689067202, + "loss": 2.6735, + "theoretical_loss": 3.678575898447718, + "tokens_seen": 919778304 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003643129388164493, + "loss": 2.7497, + "theoretical_loss": 3.6785505688428586, + "tokens_seen": 919843840 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036430290872617856, + "loss": 2.7237, + "theoretical_loss": 3.6785252415478524, + "tokens_seen": 919909376 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003642928786359077, + "loss": 2.676, + "theoretical_loss": 3.678499916562325, + "tokens_seen": 919974912 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003642828485456369, + "loss": 2.7326, + "theoretical_loss": 3.6784745938859, + "tokens_seen": 920040448 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003642728184553661, + "loss": 2.7578, + "theoretical_loss": 3.6784492735182033, + "tokens_seen": 920105984 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003642627883650953, + "loss": 2.8639, + "theoretical_loss": 3.67842395545886, + "tokens_seen": 920171520 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036425275827482446, + "loss": 2.8007, + "theoretical_loss": 3.678398639707495, + "tokens_seen": 920237056 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003642427281845537, + "loss": 2.8858, + "theoretical_loss": 3.6783733262637335, + "tokens_seen": 920302592 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003642326980942828, + "loss": 2.7203, + "theoretical_loss": 3.678348015127201, + "tokens_seen": 920368128 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036422266800401206, + "loss": 2.795, + "theoretical_loss": 3.678322706297523, + "tokens_seen": 920433664 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003642126379137412, + "loss": 2.7759, + "theoretical_loss": 3.678297399774325, + "tokens_seen": 920499200 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003642026078234704, + "loss": 2.5123, + "theoretical_loss": 3.678272095557232, + "tokens_seen": 920564736 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003641925777331996, + "loss": 2.8862, + "theoretical_loss": 3.6782467936458705, + "tokens_seen": 920630272 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003641825476429288, + "loss": 2.7438, + "theoretical_loss": 3.6782214940398665, + "tokens_seen": 920695808 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036417251755265797, + "loss": 2.6299, + "theoretical_loss": 3.6781961967388446, + "tokens_seen": 920761344 + }, + { + "epoch": 3.01, + "objective/train/docs_used": 1097042, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9992458820343018, + "objective/train/theoretical_loss": 3.6781898727736992, + "objective/train/tokens_used": 941237728, + "theoretical_loss": 3.6781898727736992, + "tokens_seen": 920777728 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036416248746238715, + "loss": 2.9299, + "theoretical_loss": 3.6781709017424324, + "tokens_seen": 920826880 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036415245737211633, + "loss": 2.8001, + "theoretical_loss": 3.6781456090502544, + "tokens_seen": 920892416 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036414242728184556, + "loss": 2.7548, + "theoretical_loss": 3.678120318661938, + "tokens_seen": 920957952 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003641323971915747, + "loss": 2.8482, + "theoretical_loss": 3.6780950305771087, + "tokens_seen": 921023488 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003641223671013039, + "loss": 2.9371, + "theoretical_loss": 3.6780697447953927, + "tokens_seen": 921089024 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036411233701103316, + "loss": 2.6601, + "theoretical_loss": 3.678044461316417, + "tokens_seen": 921154560 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003641023069207623, + "loss": 2.464, + "theoretical_loss": 3.678019180139808, + "tokens_seen": 921220096 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003640922768304915, + "loss": 2.8123, + "theoretical_loss": 3.677993901265191, + "tokens_seen": 921285632 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036408224674022065, + "loss": 2.6211, + "theoretical_loss": 3.6779686246921948, + "tokens_seen": 921351168 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003640722166499499, + "loss": 2.7441, + "theoretical_loss": 3.6779433504204446, + "tokens_seen": 921416704 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036406218655967907, + "loss": 2.6296, + "theoretical_loss": 3.6779180784495678, + "tokens_seen": 921482240 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036405215646940825, + "loss": 2.7529, + "theoretical_loss": 3.677892808779191, + "tokens_seen": 921547776 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036404212637913743, + "loss": 2.6744, + "theoretical_loss": 3.6778675414089417, + "tokens_seen": 921613312 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003640320962888666, + "loss": 2.568, + "theoretical_loss": 3.6778422763384464, + "tokens_seen": 921678848 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003640220661985958, + "loss": 2.6847, + "theoretical_loss": 3.677817013567333, + "tokens_seen": 921744384 + }, + { + "epoch": 3.01, + "learning_rate": 0.000364012036108325, + "loss": 2.672, + "theoretical_loss": 3.677791753095228, + "tokens_seen": 921809920 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036400200601805415, + "loss": 2.6619, + "theoretical_loss": 3.677766494921759, + "tokens_seen": 921875456 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003639919759277834, + "loss": 2.6684, + "theoretical_loss": 3.6777412390465543, + "tokens_seen": 921940992 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003639819458375125, + "loss": 2.7847, + "theoretical_loss": 3.6777159854692396, + "tokens_seen": 922006528 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036397191574724175, + "loss": 2.6895, + "theoretical_loss": 3.6776907341894445, + "tokens_seen": 922072064 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036396188565697093, + "loss": 2.7031, + "theoretical_loss": 3.677665485206796, + "tokens_seen": 922137600 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003639518555667001, + "loss": 2.5351, + "theoretical_loss": 3.6776402385209206, + "tokens_seen": 922203136 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003639418254764293, + "loss": 2.9291, + "theoretical_loss": 3.6776149941314484, + "tokens_seen": 922268672 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036393179538615853, + "loss": 2.5709, + "theoretical_loss": 3.6775897520380054, + "tokens_seen": 922334208 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036392176529588766, + "loss": 2.4427, + "theoretical_loss": 3.6775645122402207, + "tokens_seen": 922399744 + }, + { + "epoch": 3.01, + "objective/train/docs_used": 1097866, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.2819089889526367, + "objective/train/theoretical_loss": 3.6775582026494273, + "objective/train/tokens_used": 942876128, + "theoretical_loss": 3.6775582026494273, + "tokens_seen": 922416128 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003639117352056169, + "loss": 2.811, + "theoretical_loss": 3.6775392747377227, + "tokens_seen": 922465280 + }, + { + "epoch": 3.01, + "learning_rate": 0.000363901705115346, + "loss": 2.6732, + "theoretical_loss": 3.6775140395301387, + "tokens_seen": 922530816 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036389167502507525, + "loss": 2.7929, + "theoretical_loss": 3.677488806617097, + "tokens_seen": 922596352 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036388164493480444, + "loss": 2.6178, + "theoretical_loss": 3.6774635759982273, + "tokens_seen": 922661888 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003638716148445336, + "loss": 2.6879, + "theoretical_loss": 3.677438347673157, + "tokens_seen": 922727424 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003638615847542628, + "loss": 2.5878, + "theoretical_loss": 3.677413121641515, + "tokens_seen": 922792960 + }, + { + "epoch": 3.01, + "learning_rate": 0.000363851554663992, + "loss": 2.6521, + "theoretical_loss": 3.6773878979029293, + "tokens_seen": 922858496 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036384152457372116, + "loss": 2.6997, + "theoretical_loss": 3.67736267645703, + "tokens_seen": 922924032 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003638314944834504, + "loss": 2.7684, + "theoretical_loss": 3.6773374573034445, + "tokens_seen": 922989568 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003638214643931795, + "loss": 2.5416, + "theoretical_loss": 3.6773122404418026, + "tokens_seen": 923055104 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036381143430290876, + "loss": 2.6587, + "theoretical_loss": 3.677287025871733, + "tokens_seen": 923120640 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003638014042126379, + "loss": 2.8315, + "theoretical_loss": 3.6772618135928647, + "tokens_seen": 923186176 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003637913741223671, + "loss": 2.598, + "theoretical_loss": 3.6772366036048276, + "tokens_seen": 923251712 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003637813440320963, + "loss": 2.8329, + "theoretical_loss": 3.67721139590725, + "tokens_seen": 923317248 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003637713139418255, + "loss": 2.7133, + "theoretical_loss": 3.6771861904997616, + "tokens_seen": 923382784 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036376128385155466, + "loss": 2.5512, + "theoretical_loss": 3.677160987381992, + "tokens_seen": 923448320 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003637512537612839, + "loss": 2.7946, + "theoretical_loss": 3.6771357865535705, + "tokens_seen": 923513856 + }, + { + "epoch": 3.01, + "learning_rate": 0.000363741223671013, + "loss": 2.6753, + "theoretical_loss": 3.6771105880141266, + "tokens_seen": 923579392 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036373119358074226, + "loss": 2.7061, + "theoretical_loss": 3.67708539176329, + "tokens_seen": 923644928 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003637211634904714, + "loss": 2.7782, + "theoretical_loss": 3.6770601978006914, + "tokens_seen": 923710464 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003637111334002006, + "loss": 2.6796, + "theoretical_loss": 3.6770350061259593, + "tokens_seen": 923776000 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003637011033099298, + "loss": 2.9432, + "theoretical_loss": 3.6770098167387246, + "tokens_seen": 923841536 + }, + { + "epoch": 3.01, + "learning_rate": 0.000363691073219659, + "loss": 2.8094, + "theoretical_loss": 3.676984629638617, + "tokens_seen": 923907072 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036368104312938817, + "loss": 2.6513, + "theoretical_loss": 3.6769594448252665, + "tokens_seen": 923972608 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036367101303911735, + "loss": 2.7836, + "theoretical_loss": 3.6769342622983032, + "tokens_seen": 924038144 + }, + { + "epoch": 3.01, + "objective/train/docs_used": 1099373, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.109006881713867, + "objective/train/theoretical_loss": 3.676927967023768, + "objective/train/tokens_used": 944514528, + "theoretical_loss": 3.676927967023768, + "tokens_seen": 924054528 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036366098294884653, + "loss": 2.7884, + "theoretical_loss": 3.6769090820573584, + "tokens_seen": 924103680 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036365095285857576, + "loss": 2.8461, + "theoretical_loss": 3.6768839041020613, + "tokens_seen": 924169216 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003636409227683049, + "loss": 2.6621, + "theoretical_loss": 3.676858728432043, + "tokens_seen": 924234752 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003636308926780341, + "loss": 2.8803, + "theoretical_loss": 3.676833555046933, + "tokens_seen": 924300288 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036362086258776325, + "loss": 2.8337, + "theoretical_loss": 3.6768083839463634, + "tokens_seen": 924365824 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003636108324974925, + "loss": 2.8779, + "theoretical_loss": 3.6767832151299644, + "tokens_seen": 924431360 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036360080240722167, + "loss": 2.5029, + "theoretical_loss": 3.6767580485973665, + "tokens_seen": 924496896 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036359077231695085, + "loss": 2.586, + "theoretical_loss": 3.6767328843482012, + "tokens_seen": 924562432 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036358074222668003, + "loss": 2.6846, + "theoretical_loss": 3.6767077223820985, + "tokens_seen": 924627968 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036357071213640927, + "loss": 2.8209, + "theoretical_loss": 3.6766825626986908, + "tokens_seen": 924693504 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003635606820461384, + "loss": 2.9991, + "theoretical_loss": 3.6766574052976075, + "tokens_seen": 924759040 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036355065195586763, + "loss": 2.7583, + "theoretical_loss": 3.676632250178482, + "tokens_seen": 924824576 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036354062186559676, + "loss": 2.6988, + "theoretical_loss": 3.676607097340944, + "tokens_seen": 924890112 + }, + { + "epoch": 3.01, + "learning_rate": 0.000363530591775326, + "loss": 2.7356, + "theoretical_loss": 3.676581946784625, + "tokens_seen": 924955648 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036352056168505517, + "loss": 2.8004, + "theoretical_loss": 3.676556798509157, + "tokens_seen": 925021184 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036351053159478435, + "loss": 2.6135, + "theoretical_loss": 3.676531652514172, + "tokens_seen": 925086720 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036350050150451353, + "loss": 2.7654, + "theoretical_loss": 3.6765065087993007, + "tokens_seen": 925152256 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003634904714142427, + "loss": 2.7685, + "theoretical_loss": 3.6764813673641754, + "tokens_seen": 925217792 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003634804413239719, + "loss": 2.6842, + "theoretical_loss": 3.676456228208428, + "tokens_seen": 925283328 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036347041123370113, + "loss": 2.8569, + "theoretical_loss": 3.67643109133169, + "tokens_seen": 925348864 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036346038114343026, + "loss": 2.7765, + "theoretical_loss": 3.6764059567335936, + "tokens_seen": 925414400 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003634503510531595, + "loss": 2.703, + "theoretical_loss": 3.6763808244137715, + "tokens_seen": 925479936 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003634403209628886, + "loss": 2.7041, + "theoretical_loss": 3.676355694371855, + "tokens_seen": 925545472 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036343029087261786, + "loss": 2.7526, + "theoretical_loss": 3.6763305666074766, + "tokens_seen": 925611008 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036342026078234704, + "loss": 2.7518, + "theoretical_loss": 3.676305441120269, + "tokens_seen": 925676544 + }, + { + "epoch": 3.01, + "objective/train/docs_used": 1100200, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.052302360534668, + "objective/train/theoretical_loss": 3.6762991601042323, + "objective/train/tokens_used": 946152928, + "theoretical_loss": 3.6762991601042323, + "tokens_seen": 925692928 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003634102306920762, + "loss": 2.7231, + "theoretical_loss": 3.6762803179098644, + "tokens_seen": 925742080 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003634002006018054, + "loss": 2.6954, + "theoretical_loss": 3.6762551969758954, + "tokens_seen": 925807616 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036339017051153464, + "loss": 2.6857, + "theoretical_loss": 3.676230078317994, + "tokens_seen": 925873152 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036338014042126376, + "loss": 2.7367, + "theoretical_loss": 3.676204961935794, + "tokens_seen": 925938688 + }, + { + "epoch": 3.01, + "learning_rate": 0.000363370110330993, + "loss": 2.6934, + "theoretical_loss": 3.6761798478289274, + "tokens_seen": 926004224 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003633600802407222, + "loss": 2.6842, + "theoretical_loss": 3.6761547359970272, + "tokens_seen": 926069760 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036335005015045136, + "loss": 2.7722, + "theoretical_loss": 3.6761296264397263, + "tokens_seen": 926135296 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003633400200601806, + "loss": 2.6782, + "theoretical_loss": 3.6761045191566586, + "tokens_seen": 926200832 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003633299899699097, + "loss": 2.9231, + "theoretical_loss": 3.6760794141474555, + "tokens_seen": 926266368 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036331995987963896, + "loss": 2.7677, + "theoretical_loss": 3.6760543114117517, + "tokens_seen": 926331904 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003633099297893681, + "loss": 2.7595, + "theoretical_loss": 3.6760292109491797, + "tokens_seen": 926397440 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003632998996990973, + "loss": 2.8369, + "theoretical_loss": 3.676004112759373, + "tokens_seen": 926462976 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003632898696088265, + "loss": 2.7427, + "theoretical_loss": 3.675979016841966, + "tokens_seen": 926528512 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003632798395185557, + "loss": 2.7186, + "theoretical_loss": 3.675953923196591, + "tokens_seen": 926594048 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036326980942828486, + "loss": 2.7138, + "theoretical_loss": 3.6759288318228815, + "tokens_seen": 926659584 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003632597793380141, + "loss": 2.7673, + "theoretical_loss": 3.6759037427204726, + "tokens_seen": 926725120 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003632497492477432, + "loss": 2.6312, + "theoretical_loss": 3.675878655888997, + "tokens_seen": 926790656 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036323971915747246, + "loss": 2.6802, + "theoretical_loss": 3.675853571328089, + "tokens_seen": 926856192 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003632296890672016, + "loss": 2.9694, + "theoretical_loss": 3.6758284890373822, + "tokens_seen": 926921728 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003632196589769308, + "loss": 2.7553, + "theoretical_loss": 3.675803409016511, + "tokens_seen": 926987264 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036320962888666, + "loss": 2.7427, + "theoretical_loss": 3.6757783312651093, + "tokens_seen": 927052800 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003631995987963892, + "loss": 2.7938, + "theoretical_loss": 3.6757532557828116, + "tokens_seen": 927118336 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036318956870611837, + "loss": 2.8564, + "theoretical_loss": 3.6757281825692525, + "tokens_seen": 927183872 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036317953861584755, + "loss": 2.817, + "theoretical_loss": 3.675703111624065, + "tokens_seen": 927249408 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036316950852557673, + "loss": 2.6925, + "theoretical_loss": 3.6756780429468856, + "tokens_seen": 927314944 + }, + { + "epoch": 3.01, + "objective/train/docs_used": 1101371, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7425971031188965, + "objective/train/theoretical_loss": 3.6756717761319235, + "objective/train/tokens_used": 947791328, + "theoretical_loss": 3.6756717761319235, + "tokens_seen": 927331328 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036315947843530596, + "loss": 2.738, + "theoretical_loss": 3.6756529765373473, + "tokens_seen": 927380480 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003631494483450351, + "loss": 2.9168, + "theoretical_loss": 3.6756279123950852, + "tokens_seen": 927446016 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003631394182547643, + "loss": 2.8124, + "theoretical_loss": 3.675602850519734, + "tokens_seen": 927511552 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036312938816449345, + "loss": 2.9051, + "theoretical_loss": 3.6755777909109284, + "tokens_seen": 927577088 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003631193580742227, + "loss": 2.7232, + "theoretical_loss": 3.675552733568304, + "tokens_seen": 927642624 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036310932798395187, + "loss": 2.5916, + "theoretical_loss": 3.675527678491495, + "tokens_seen": 927708160 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036309929789368105, + "loss": 2.794, + "theoretical_loss": 3.675502625680137, + "tokens_seen": 927773696 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036308926780341023, + "loss": 2.7731, + "theoretical_loss": 3.675477575133865, + "tokens_seen": 927839232 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036307923771313947, + "loss": 2.7715, + "theoretical_loss": 3.6754525268523137, + "tokens_seen": 927904768 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003630692076228686, + "loss": 2.779, + "theoretical_loss": 3.6754274808351193, + "tokens_seen": 927970304 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036305917753259783, + "loss": 2.7027, + "theoretical_loss": 3.6754024370819165, + "tokens_seen": 928035840 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036304914744232696, + "loss": 2.8426, + "theoretical_loss": 3.6753773955923412, + "tokens_seen": 928101376 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003630391173520562, + "loss": 2.7696, + "theoretical_loss": 3.6753523563660293, + "tokens_seen": 928166912 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036302908726178537, + "loss": 2.7511, + "theoretical_loss": 3.6753273194026157, + "tokens_seen": 928232448 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036301905717151455, + "loss": 2.5918, + "theoretical_loss": 3.6753022847017363, + "tokens_seen": 928297984 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036300902708124374, + "loss": 2.5532, + "theoretical_loss": 3.675277252263027, + "tokens_seen": 928363520 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003629989969909729, + "loss": 2.7777, + "theoretical_loss": 3.675252222086124, + "tokens_seen": 928429056 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003629889669007021, + "loss": 2.6777, + "theoretical_loss": 3.675227194170663, + "tokens_seen": 928494592 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036297893681043133, + "loss": 2.8747, + "theoretical_loss": 3.6752021685162806, + "tokens_seen": 928560128 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036296890672016046, + "loss": 2.7622, + "theoretical_loss": 3.6751771451226123, + "tokens_seen": 928625664 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003629588766298897, + "loss": 2.5471, + "theoretical_loss": 3.6751521239892946, + "tokens_seen": 928691200 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003629488465396188, + "loss": 2.5757, + "theoretical_loss": 3.6751271051159637, + "tokens_seen": 928756736 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036293881644934806, + "loss": 2.7508, + "theoretical_loss": 3.675102088502257, + "tokens_seen": 928822272 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036292878635907724, + "loss": 2.8222, + "theoretical_loss": 3.675077074147809, + "tokens_seen": 928887808 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003629187562688064, + "loss": 2.7448, + "theoretical_loss": 3.6750520620522584, + "tokens_seen": 928953344 + }, + { + "epoch": 3.01, + "objective/train/docs_used": 1102157, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.89856219291687, + "objective/train/theoretical_loss": 3.67504580938128, + "objective/train/tokens_used": 949429728, + "theoretical_loss": 3.67504580938128, + "tokens_seen": 928969728 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003629087261785356, + "loss": 2.7577, + "theoretical_loss": 3.67502705221524, + "tokens_seen": 929018880 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036289869608826484, + "loss": 2.764, + "theoretical_loss": 3.6750020446363925, + "tokens_seen": 929084416 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036288866599799396, + "loss": 2.5949, + "theoretical_loss": 3.674977039315351, + "tokens_seen": 929149952 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003628786359077232, + "loss": 2.7679, + "theoretical_loss": 3.674952036251754, + "tokens_seen": 929215488 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003628686058174523, + "loss": 2.8058, + "theoretical_loss": 3.6749270354452372, + "tokens_seen": 929281024 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036285857572718156, + "loss": 2.5555, + "theoretical_loss": 3.674902036895438, + "tokens_seen": 929346560 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036284854563691074, + "loss": 2.6937, + "theoretical_loss": 3.6748770406019937, + "tokens_seen": 929412096 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003628385155466399, + "loss": 2.813, + "theoretical_loss": 3.674852046564542, + "tokens_seen": 929477632 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003628284854563691, + "loss": 2.573, + "theoretical_loss": 3.6748270547827198, + "tokens_seen": 929543168 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003628184553660983, + "loss": 2.8186, + "theoretical_loss": 3.6748020652561646, + "tokens_seen": 929608704 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036280842527582747, + "loss": 2.6764, + "theoretical_loss": 3.6747770779845137, + "tokens_seen": 929674240 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003627983951855567, + "loss": 2.6099, + "theoretical_loss": 3.674752092967405, + "tokens_seen": 929739776 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036278836509528583, + "loss": 2.979, + "theoretical_loss": 3.6747271102044756, + "tokens_seen": 929805312 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036277833500501506, + "loss": 2.8215, + "theoretical_loss": 3.6747021296953646, + "tokens_seen": 929870848 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036276830491474424, + "loss": 2.8194, + "theoretical_loss": 3.6746771514397083, + "tokens_seen": 929936384 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003627582748244734, + "loss": 2.6955, + "theoretical_loss": 3.6746521754371457, + "tokens_seen": 930001920 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003627482447342026, + "loss": 2.886, + "theoretical_loss": 3.674627201687314, + "tokens_seen": 930067456 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003627382146439318, + "loss": 2.6595, + "theoretical_loss": 3.674602230189852, + "tokens_seen": 930132992 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036272818455366097, + "loss": 2.8569, + "theoretical_loss": 3.674577260944398, + "tokens_seen": 930198528 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003627181544633902, + "loss": 2.7004, + "theoretical_loss": 3.6745522939505895, + "tokens_seen": 930264064 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036270812437311933, + "loss": 2.537, + "theoretical_loss": 3.674527329208065, + "tokens_seen": 930329600 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036269809428284857, + "loss": 2.6057, + "theoretical_loss": 3.674502366716464, + "tokens_seen": 930395136 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003626880641925777, + "loss": 2.6501, + "theoretical_loss": 3.6744774064754235, + "tokens_seen": 930460672 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036267803410230693, + "loss": 2.7078, + "theoretical_loss": 3.6744524484845824, + "tokens_seen": 930526208 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003626680040120361, + "loss": 2.6438, + "theoretical_loss": 3.6744274927435807, + "tokens_seen": 930591744 + }, + { + "epoch": 3.01, + "objective/train/docs_used": 1103642, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.0793814659118652, + "objective/train/theoretical_loss": 3.6744212541598245, + "objective/train/tokens_used": 951068128, + "theoretical_loss": 3.6744212541598245, + "tokens_seen": 930608128 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003626579739217653, + "loss": 2.5507, + "theoretical_loss": 3.6744025392520556, + "tokens_seen": 930657280 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036264794383149447, + "loss": 2.7309, + "theoretical_loss": 3.674377588009647, + "tokens_seen": 930722816 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036263791374122365, + "loss": 2.6374, + "theoretical_loss": 3.674352639015993, + "tokens_seen": 930788352 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036262788365095283, + "loss": 2.8798, + "theoretical_loss": 3.674327692270733, + "tokens_seen": 930853888 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036261785356068207, + "loss": 2.7256, + "theoretical_loss": 3.674302747773506, + "tokens_seen": 930919424 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036260782347041125, + "loss": 2.8085, + "theoretical_loss": 3.674277805523952, + "tokens_seen": 930984960 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036259779338014043, + "loss": 2.7442, + "theoretical_loss": 3.6742528655217095, + "tokens_seen": 931050496 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036258776328986967, + "loss": 2.6867, + "theoretical_loss": 3.6742279277664176, + "tokens_seen": 931116032 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003625777331995988, + "loss": 2.8555, + "theoretical_loss": 3.6742029922577157, + "tokens_seen": 931181568 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036256770310932803, + "loss": 2.6521, + "theoretical_loss": 3.674178058995244, + "tokens_seen": 931247104 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036255767301905716, + "loss": 2.7883, + "theoretical_loss": 3.6741531279786424, + "tokens_seen": 931312640 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003625476429287864, + "loss": 2.8195, + "theoretical_loss": 3.674128199207549, + "tokens_seen": 931378176 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003625376128385156, + "loss": 2.6329, + "theoretical_loss": 3.674103272681605, + "tokens_seen": 931443712 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036252758274824475, + "loss": 2.8075, + "theoretical_loss": 3.67407834840045, + "tokens_seen": 931509248 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036251755265797394, + "loss": 2.5847, + "theoretical_loss": 3.6740534263637237, + "tokens_seen": 931574784 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003625075225677031, + "loss": 2.7421, + "theoretical_loss": 3.674028506571066, + "tokens_seen": 931640320 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003624974924774323, + "loss": 2.9755, + "theoretical_loss": 3.6740035890221168, + "tokens_seen": 931705856 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036248746238716153, + "loss": 2.8285, + "theoretical_loss": 3.673978673716517, + "tokens_seen": 931771392 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036247743229689066, + "loss": 2.9064, + "theoretical_loss": 3.673953760653906, + "tokens_seen": 931836928 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003624674022066199, + "loss": 2.7024, + "theoretical_loss": 3.673928849833925, + "tokens_seen": 931902464 + }, + { + "epoch": 3.01, + "learning_rate": 0.000362457372116349, + "loss": 2.83, + "theoretical_loss": 3.6739039412562144, + "tokens_seen": 931968000 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036244734202607826, + "loss": 2.6702, + "theoretical_loss": 3.6738790349204136, + "tokens_seen": 932033536 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036243731193580744, + "loss": 2.7042, + "theoretical_loss": 3.6738541308261645, + "tokens_seen": 932099072 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003624272818455366, + "loss": 2.8335, + "theoretical_loss": 3.673829228973107, + "tokens_seen": 932164608 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003624172517552658, + "loss": 2.7473, + "theoretical_loss": 3.673804329360882, + "tokens_seen": 932230144 + }, + { + "epoch": 3.01, + "objective/train/docs_used": 1104463, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.2996299266815186, + "objective/train/theoretical_loss": 3.673798104807914, + "objective/train/tokens_used": 952706528, + "theoretical_loss": 3.673798104807914, + "tokens_seen": 932246528 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036240722166499504, + "loss": 2.9647, + "theoretical_loss": 3.6737794319891313, + "tokens_seen": 932295680 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036239719157472416, + "loss": 2.7751, + "theoretical_loss": 3.673754536857494, + "tokens_seen": 932361216 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003623871614844534, + "loss": 2.8086, + "theoretical_loss": 3.673729643965612, + "tokens_seen": 932426752 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003623771313941825, + "loss": 2.5253, + "theoretical_loss": 3.6737047533131273, + "tokens_seen": 932492288 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036236710130391176, + "loss": 2.8928, + "theoretical_loss": 3.67367986489968, + "tokens_seen": 932557824 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036235707121364094, + "loss": 2.7164, + "theoretical_loss": 3.6736549787249118, + "tokens_seen": 932623360 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003623470411233701, + "loss": 2.8051, + "theoretical_loss": 3.6736300947884635, + "tokens_seen": 932688896 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003623370110330993, + "loss": 2.6297, + "theoretical_loss": 3.673605213089977, + "tokens_seen": 932754432 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003623269809428285, + "loss": 2.6382, + "theoretical_loss": 3.6735803336290935, + "tokens_seen": 932819968 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036231695085255767, + "loss": 2.8193, + "theoretical_loss": 3.673555456405455, + "tokens_seen": 932885504 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003623069207622869, + "loss": 2.6228, + "theoretical_loss": 3.673530581418703, + "tokens_seen": 932951040 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036229689067201603, + "loss": 2.6079, + "theoretical_loss": 3.673505708668479, + "tokens_seen": 933016576 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036228686058174526, + "loss": 2.7311, + "theoretical_loss": 3.6734808381544255, + "tokens_seen": 933082112 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036227683049147444, + "loss": 2.6853, + "theoretical_loss": 3.6734559698761835, + "tokens_seen": 933147648 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003622668004012036, + "loss": 2.6714, + "theoretical_loss": 3.673431103833396, + "tokens_seen": 933213184 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003622567703109328, + "loss": 2.8825, + "theoretical_loss": 3.673406240025704, + "tokens_seen": 933278720 + }, + { + "epoch": 3.01, + "learning_rate": 0.000362246740220662, + "loss": 2.5792, + "theoretical_loss": 3.6733813784527505, + "tokens_seen": 933344256 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036223671013039117, + "loss": 2.8156, + "theoretical_loss": 3.6733565191141775, + "tokens_seen": 933409792 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003622266800401204, + "loss": 2.9049, + "theoretical_loss": 3.673331662009627, + "tokens_seen": 933475328 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036221664994984953, + "loss": 2.9021, + "theoretical_loss": 3.673306807138742, + "tokens_seen": 933540864 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036220661985957877, + "loss": 2.8239, + "theoretical_loss": 3.6732819545011646, + "tokens_seen": 933606400 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003621965897693079, + "loss": 2.6474, + "theoretical_loss": 3.6732571040965376, + "tokens_seen": 933671936 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036218655967903713, + "loss": 2.8546, + "theoretical_loss": 3.6732322559245034, + "tokens_seen": 933737472 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003621765295887663, + "loss": 2.6471, + "theoretical_loss": 3.6732074099847054, + "tokens_seen": 933803008 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003621664994984955, + "loss": 2.714, + "theoretical_loss": 3.673182566276785, + "tokens_seen": 933868544 + }, + { + "epoch": 3.01, + "objective/train/docs_used": 1105263, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4603049755096436, + "objective/train/theoretical_loss": 3.6731763556984944, + "objective/train/tokens_used": 954344928, + "theoretical_loss": 3.6731763556984944, + "tokens_seen": 933884928 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036215646940822467, + "loss": 2.6152, + "theoretical_loss": 3.6731577248003866, + "tokens_seen": 933934080 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036214643931795385, + "loss": 2.7085, + "theoretical_loss": 3.6731328855551526, + "tokens_seen": 933999616 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036213640922768303, + "loss": 2.6834, + "theoretical_loss": 3.6731080485407266, + "tokens_seen": 934065152 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036212637913741227, + "loss": 2.7269, + "theoretical_loss": 3.6730832137567506, + "tokens_seen": 934130688 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003621163490471414, + "loss": 2.749, + "theoretical_loss": 3.6730583812028685, + "tokens_seen": 934196224 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036210631895687063, + "loss": 2.7608, + "theoretical_loss": 3.673033550878724, + "tokens_seen": 934261760 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003620962888665998, + "loss": 2.9078, + "theoretical_loss": 3.67300872278396, + "tokens_seen": 934327296 + }, + { + "epoch": 3.01, + "learning_rate": 0.000362086258776329, + "loss": 2.8538, + "theoretical_loss": 3.67298389691822, + "tokens_seen": 934392832 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003620762286860582, + "loss": 2.6955, + "theoretical_loss": 3.6729590732811475, + "tokens_seen": 934458368 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036206619859578736, + "loss": 2.7588, + "theoretical_loss": 3.6729342518723866, + "tokens_seen": 934523904 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036205616850551654, + "loss": 2.8656, + "theoretical_loss": 3.67290943269158, + "tokens_seen": 934589440 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003620461384152458, + "loss": 2.8229, + "theoretical_loss": 3.672884615738373, + "tokens_seen": 934654976 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003620361083249749, + "loss": 2.8307, + "theoretical_loss": 3.6728598010124083, + "tokens_seen": 934720512 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036202607823470414, + "loss": 2.7492, + "theoretical_loss": 3.6728349885133307, + "tokens_seen": 934786048 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036201604814443326, + "loss": 2.6536, + "theoretical_loss": 3.672810178240784, + "tokens_seen": 934851584 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003620060180541625, + "loss": 2.8426, + "theoretical_loss": 3.6727853701944118, + "tokens_seen": 934917120 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003619959879638917, + "loss": 2.745, + "theoretical_loss": 3.672760564373859, + "tokens_seen": 934982656 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036198595787362086, + "loss": 2.6526, + "theoretical_loss": 3.672735760778769, + "tokens_seen": 935048192 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036197592778335004, + "loss": 2.4051, + "theoretical_loss": 3.6727109594087874, + "tokens_seen": 935113728 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003619658976930792, + "loss": 2.4586, + "theoretical_loss": 3.672686160263558, + "tokens_seen": 935179264 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003619558676028084, + "loss": 2.8034, + "theoretical_loss": 3.6726613633427254, + "tokens_seen": 935244800 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036194583751253764, + "loss": 2.8574, + "theoretical_loss": 3.6726365686459346, + "tokens_seen": 935310336 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036193580742226677, + "loss": 3.0104, + "theoretical_loss": 3.6726117761728294, + "tokens_seen": 935375872 + }, + { + "epoch": 3.01, + "learning_rate": 0.000361925777331996, + "loss": 2.7326, + "theoretical_loss": 3.672586985923055, + "tokens_seen": 935441408 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003619157472417252, + "loss": 2.5144, + "theoretical_loss": 3.672562197896257, + "tokens_seen": 935506944 + }, + { + "epoch": 3.01, + "objective/train/docs_used": 1106660, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.0614864826202393, + "objective/train/theoretical_loss": 3.6725560012368557, + "objective/train/tokens_used": 955983328, + "theoretical_loss": 3.6725560012368557, + "tokens_seen": 935523328 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036190571715145436, + "loss": 2.7475, + "theoretical_loss": 3.6725374120920797, + "tokens_seen": 935572480 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036189568706118354, + "loss": 2.5965, + "theoretical_loss": 3.672512628510168, + "tokens_seen": 935638016 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003618856569709127, + "loss": 2.6269, + "theoretical_loss": 3.672487847150167, + "tokens_seen": 935703552 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003618756268806419, + "loss": 2.6792, + "theoretical_loss": 3.672463068011723, + "tokens_seen": 935769088 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036186559679037114, + "loss": 2.7316, + "theoretical_loss": 3.6724382910944797, + "tokens_seen": 935834624 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003618555667001003, + "loss": 2.7486, + "theoretical_loss": 3.672413516398083, + "tokens_seen": 935900160 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003618455366098295, + "loss": 2.6199, + "theoretical_loss": 3.672388743922179, + "tokens_seen": 935965696 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003618355065195587, + "loss": 2.6766, + "theoretical_loss": 3.672363973666413, + "tokens_seen": 936031232 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036182547642928787, + "loss": 2.7611, + "theoretical_loss": 3.67233920563043, + "tokens_seen": 936096768 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003618154463390171, + "loss": 2.4793, + "theoretical_loss": 3.672314439813876, + "tokens_seen": 936162304 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036180541624874623, + "loss": 2.5873, + "theoretical_loss": 3.672289676216397, + "tokens_seen": 936227840 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036179538615847546, + "loss": 2.6239, + "theoretical_loss": 3.6722649148376383, + "tokens_seen": 936293376 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036178535606820464, + "loss": 2.7087, + "theoretical_loss": 3.6722401556772466, + "tokens_seen": 936358912 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003617753259779338, + "loss": 2.7876, + "theoretical_loss": 3.672215398734867, + "tokens_seen": 936424448 + }, + { + "epoch": 3.01, + "learning_rate": 0.000361765295887663, + "loss": 2.6702, + "theoretical_loss": 3.672190644010147, + "tokens_seen": 936489984 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003617552657973922, + "loss": 2.788, + "theoretical_loss": 3.672165891502731, + "tokens_seen": 936555520 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036174523570712137, + "loss": 2.7138, + "theoretical_loss": 3.6721411412122666, + "tokens_seen": 936621056 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003617352056168506, + "loss": 2.7349, + "theoretical_loss": 3.6721163931383995, + "tokens_seen": 936686592 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036172517552657973, + "loss": 2.7485, + "theoretical_loss": 3.6720916472807765, + "tokens_seen": 936752128 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036171514543630897, + "loss": 2.8504, + "theoretical_loss": 3.6720669036390436, + "tokens_seen": 936817664 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003617051153460381, + "loss": 2.7481, + "theoretical_loss": 3.672042162212848, + "tokens_seen": 936883200 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036169508525576733, + "loss": 2.9067, + "theoretical_loss": 3.672017423001836, + "tokens_seen": 936948736 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003616850551654965, + "loss": 2.4963, + "theoretical_loss": 3.6719926860056544, + "tokens_seen": 937014272 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003616750250752257, + "loss": 2.6033, + "theoretical_loss": 3.67196795122395, + "tokens_seen": 937079808 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036166499498495487, + "loss": 2.7483, + "theoretical_loss": 3.6719432186563696, + "tokens_seen": 937145344 + }, + { + "epoch": 3.01, + "objective/train/docs_used": 1107254, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6422669887542725, + "objective/train/theoretical_loss": 3.67193703586039, + "objective/train/tokens_used": 957621728, + "theoretical_loss": 3.67193703586039, + "tokens_seen": 937161728 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036165496489468405, + "loss": 2.6973, + "theoretical_loss": 3.67191848830256, + "tokens_seen": 937210880 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036164493480441323, + "loss": 2.6796, + "theoretical_loss": 3.6718937601621695, + "tokens_seen": 937276416 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036163490471414247, + "loss": 2.6766, + "theoretical_loss": 3.6718690342348435, + "tokens_seen": 937341952 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003616248746238716, + "loss": 2.8178, + "theoretical_loss": 3.6718443105202305, + "tokens_seen": 937407488 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036161484453360083, + "loss": 2.6341, + "theoretical_loss": 3.6718195890179772, + "tokens_seen": 937473024 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036160481444333, + "loss": 2.8366, + "theoretical_loss": 3.671794869727731, + "tokens_seen": 937538560 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003615947843530592, + "loss": 2.6977, + "theoretical_loss": 3.67177015264914, + "tokens_seen": 937604096 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003615847542627884, + "loss": 2.6461, + "theoretical_loss": 3.6717454377818513, + "tokens_seen": 937669632 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036157472417251756, + "loss": 2.6455, + "theoretical_loss": 3.671720725125512, + "tokens_seen": 937735168 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036156469408224674, + "loss": 2.7254, + "theoretical_loss": 3.671696014679771, + "tokens_seen": 937800704 + }, + { + "epoch": 3.01, + "learning_rate": 0.000361554663991976, + "loss": 2.6941, + "theoretical_loss": 3.6716713064442743, + "tokens_seen": 937866240 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003615446339017051, + "loss": 2.7385, + "theoretical_loss": 3.671646600418672, + "tokens_seen": 937931776 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036153460381143434, + "loss": 2.5716, + "theoretical_loss": 3.671621896602611, + "tokens_seen": 937997312 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036152457372116346, + "loss": 2.5538, + "theoretical_loss": 3.671597194995739, + "tokens_seen": 938062848 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003615145436308927, + "loss": 2.7646, + "theoretical_loss": 3.671572495597704, + "tokens_seen": 938128384 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003615045135406219, + "loss": 2.5636, + "theoretical_loss": 3.671547798408155, + "tokens_seen": 938193920 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036149448345035106, + "loss": 2.6606, + "theoretical_loss": 3.6715231034267397, + "tokens_seen": 938259456 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036148445336008024, + "loss": 2.7052, + "theoretical_loss": 3.671498410653107, + "tokens_seen": 938324992 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003614744232698094, + "loss": 2.7681, + "theoretical_loss": 3.6714737200869045, + "tokens_seen": 938390528 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003614643931795386, + "loss": 2.3929, + "theoretical_loss": 3.6714490317277817, + "tokens_seen": 938456064 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036145436308926784, + "loss": 2.6983, + "theoretical_loss": 3.6714243455753865, + "tokens_seen": 938521600 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036144433299899697, + "loss": 2.6932, + "theoretical_loss": 3.6713996616293683, + "tokens_seen": 938587136 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003614343029087262, + "loss": 2.5358, + "theoretical_loss": 3.671374979889375, + "tokens_seen": 938652672 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003614242728184554, + "loss": 2.583, + "theoretical_loss": 3.671350300355056, + "tokens_seen": 938718208 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036141424272818456, + "loss": 2.7081, + "theoretical_loss": 3.6713256230260596, + "tokens_seen": 938783744 + }, + { + "epoch": 3.01, + "objective/train/docs_used": 1108649, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.576571464538574, + "objective/train/theoretical_loss": 3.671319454038351, + "objective/train/tokens_used": 959260128, + "theoretical_loss": 3.671319454038351, + "tokens_seen": 938800128 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036140421263791374, + "loss": 2.7355, + "theoretical_loss": 3.6713009479020355, + "tokens_seen": 938849280 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003613941825476429, + "loss": 2.7932, + "theoretical_loss": 3.6712762749826324, + "tokens_seen": 938914816 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003613841524573721, + "loss": 2.659, + "theoretical_loss": 3.6712516042675, + "tokens_seen": 938980352 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036137412236710134, + "loss": 2.4067, + "theoretical_loss": 3.6712269357562866, + "tokens_seen": 939045888 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036136409227683047, + "loss": 2.6325, + "theoretical_loss": 3.671202269448642, + "tokens_seen": 939111424 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003613540621865597, + "loss": 2.7483, + "theoretical_loss": 3.671177605344216, + "tokens_seen": 939176960 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036134403209628883, + "loss": 2.6851, + "theoretical_loss": 3.671152943442658, + "tokens_seen": 939242496 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036133400200601807, + "loss": 2.7459, + "theoretical_loss": 3.6711282837436174, + "tokens_seen": 939308032 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036132397191574725, + "loss": 2.6561, + "theoretical_loss": 3.671103626246743, + "tokens_seen": 939373568 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036131394182547643, + "loss": 2.8424, + "theoretical_loss": 3.671078970951686, + "tokens_seen": 939439104 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003613039117352056, + "loss": 2.7681, + "theoretical_loss": 3.671054317858095, + "tokens_seen": 939504640 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036129388164493484, + "loss": 2.7278, + "theoretical_loss": 3.6710296669656204, + "tokens_seen": 939570176 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036128385155466397, + "loss": 2.9318, + "theoretical_loss": 3.6710050182739127, + "tokens_seen": 939635712 + }, + { + "epoch": 3.01, + "learning_rate": 0.0003612738214643932, + "loss": 2.843, + "theoretical_loss": 3.6709803717826213, + "tokens_seen": 939701248 + }, + { + "epoch": 3.01, + "learning_rate": 0.00036126379137412233, + "loss": 2.7645, + "theoretical_loss": 3.670955727491396, + "tokens_seen": 939766784 + }, + { + "epoch": 3.02, + "learning_rate": 0.00036125376128385157, + "loss": 2.7355, + "theoretical_loss": 3.6709310853998876, + "tokens_seen": 939832320 + }, + { + "epoch": 3.02, + "learning_rate": 0.00036124373119358075, + "loss": 2.8614, + "theoretical_loss": 3.6709064455077467, + "tokens_seen": 939897856 + }, + { + "epoch": 3.02, + "learning_rate": 0.00036123370110330993, + "loss": 2.78, + "theoretical_loss": 3.670881807814623, + "tokens_seen": 939963392 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003612236710130391, + "loss": 2.6633, + "theoretical_loss": 3.6708571723201664, + "tokens_seen": 940028928 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003612136409227683, + "loss": 2.7481, + "theoretical_loss": 3.670832539024029, + "tokens_seen": 940094464 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003612036108324975, + "loss": 2.7512, + "theoretical_loss": 3.6708079079258598, + "tokens_seen": 940160000 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003611935807422267, + "loss": 2.5802, + "theoretical_loss": 3.6707832790253105, + "tokens_seen": 940225536 + }, + { + "epoch": 3.02, + "learning_rate": 0.00036118355065195584, + "loss": 2.8009, + "theoretical_loss": 3.670758652322032, + "tokens_seen": 940291072 + }, + { + "epoch": 3.02, + "learning_rate": 0.00036117352056168507, + "loss": 2.8288, + "theoretical_loss": 3.6707340278156746, + "tokens_seen": 940356608 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003611634904714142, + "loss": 2.9308, + "theoretical_loss": 3.6707094055058898, + "tokens_seen": 940422144 + }, + { + "epoch": 3.02, + "objective/train/docs_used": 1109497, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9072909355163574, + "objective/train/theoretical_loss": 3.670703250271617, + "objective/train/tokens_used": 960898528, + "theoretical_loss": 3.670703250271617, + "tokens_seen": 940438528 + }, + { + "epoch": 3.02, + "learning_rate": 0.00036115346038114343, + "loss": 2.8859, + "theoretical_loss": 3.6706847853923277, + "tokens_seen": 940487680 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003611434302908726, + "loss": 2.8009, + "theoretical_loss": 3.6706601674746397, + "tokens_seen": 940553216 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003611334002006018, + "loss": 2.7237, + "theoretical_loss": 3.6706355517524774, + "tokens_seen": 940618752 + }, + { + "epoch": 3.02, + "learning_rate": 0.000361123370110331, + "loss": 2.5759, + "theoretical_loss": 3.670610938225492, + "tokens_seen": 940684288 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003611133400200602, + "loss": 2.6999, + "theoretical_loss": 3.6705863268933347, + "tokens_seen": 940749824 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003611033099297894, + "loss": 2.6335, + "theoretical_loss": 3.6705617177556564, + "tokens_seen": 940815360 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003610932798395186, + "loss": 2.9326, + "theoretical_loss": 3.670537110812109, + "tokens_seen": 940880896 + }, + { + "epoch": 3.02, + "learning_rate": 0.00036108324974924776, + "loss": 2.7079, + "theoretical_loss": 3.6705125060623445, + "tokens_seen": 940946432 + }, + { + "epoch": 3.02, + "learning_rate": 0.00036107321965897694, + "loss": 2.8174, + "theoretical_loss": 3.670487903506014, + "tokens_seen": 941011968 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003610631895687062, + "loss": 2.6992, + "theoretical_loss": 3.6704633031427694, + "tokens_seen": 941077504 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003610531594784353, + "loss": 2.5846, + "theoretical_loss": 3.6704387049722618, + "tokens_seen": 941143040 + }, + { + "epoch": 3.02, + "learning_rate": 0.00036104312938816454, + "loss": 2.5115, + "theoretical_loss": 3.6704141089941444, + "tokens_seen": 941208576 + }, + { + "epoch": 3.02, + "learning_rate": 0.00036103309929789366, + "loss": 2.8338, + "theoretical_loss": 3.6703895152080683, + "tokens_seen": 941274112 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003610230692076229, + "loss": 2.7403, + "theoretical_loss": 3.670364923613686, + "tokens_seen": 941339648 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003610130391173521, + "loss": 2.7208, + "theoretical_loss": 3.670340334210649, + "tokens_seen": 941405184 + }, + { + "epoch": 3.02, + "learning_rate": 0.00036100300902708126, + "loss": 2.8817, + "theoretical_loss": 3.67031574699861, + "tokens_seen": 941470720 + }, + { + "epoch": 3.02, + "learning_rate": 0.00036099297893681044, + "loss": 2.4179, + "theoretical_loss": 3.6702911619772207, + "tokens_seen": 941536256 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003609829488465396, + "loss": 2.5508, + "theoretical_loss": 3.6702665791461344, + "tokens_seen": 941601792 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003609729187562688, + "loss": 2.8291, + "theoretical_loss": 3.670241998505003, + "tokens_seen": 941667328 + }, + { + "epoch": 3.02, + "learning_rate": 0.00036096288866599804, + "loss": 2.8138, + "theoretical_loss": 3.670217420053479, + "tokens_seen": 941732864 + }, + { + "epoch": 3.02, + "learning_rate": 0.00036095285857572717, + "loss": 2.5917, + "theoretical_loss": 3.670192843791215, + "tokens_seen": 941798400 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003609428284854564, + "loss": 2.6289, + "theoretical_loss": 3.6701682697178635, + "tokens_seen": 941863936 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003609327983951856, + "loss": 2.6654, + "theoretical_loss": 3.670143697833078, + "tokens_seen": 941929472 + }, + { + "epoch": 3.02, + "learning_rate": 0.00036092276830491476, + "loss": 2.7178, + "theoretical_loss": 3.67011912813651, + "tokens_seen": 941995008 + }, + { + "epoch": 3.02, + "learning_rate": 0.00036091273821464394, + "loss": 2.6515, + "theoretical_loss": 3.670094560627814, + "tokens_seen": 942060544 + }, + { + "epoch": 3.02, + "objective/train/docs_used": 1110990, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.0038723945617676, + "objective/train/theoretical_loss": 3.6700884190924543, + "objective/train/tokens_used": 962536928, + "theoretical_loss": 3.6700884190924543, + "tokens_seen": 942076928 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003609027081243731, + "loss": 2.5359, + "theoretical_loss": 3.6700699953066422, + "tokens_seen": 942126080 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003608926780341023, + "loss": 2.7436, + "theoretical_loss": 3.670045432172647, + "tokens_seen": 942191616 + }, + { + "epoch": 3.02, + "learning_rate": 0.00036088264794383154, + "loss": 2.8468, + "theoretical_loss": 3.670020871225483, + "tokens_seen": 942257152 + }, + { + "epoch": 3.02, + "learning_rate": 0.00036087261785356067, + "loss": 2.7068, + "theoretical_loss": 3.669996312464802, + "tokens_seen": 942322688 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003608625877632899, + "loss": 2.7132, + "theoretical_loss": 3.6699717558902583, + "tokens_seen": 942388224 + }, + { + "epoch": 3.02, + "learning_rate": 0.00036085255767301903, + "loss": 2.9056, + "theoretical_loss": 3.669947201501505, + "tokens_seen": 942453760 + }, + { + "epoch": 3.02, + "learning_rate": 0.00036084252758274827, + "loss": 2.6841, + "theoretical_loss": 3.6699226492981953, + "tokens_seen": 942519296 + }, + { + "epoch": 3.02, + "learning_rate": 0.00036083249749247745, + "loss": 2.6888, + "theoretical_loss": 3.6698980992799832, + "tokens_seen": 942584832 + }, + { + "epoch": 3.02, + "learning_rate": 0.00036082246740220663, + "loss": 2.6369, + "theoretical_loss": 3.669873551446522, + "tokens_seen": 942650368 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003608124373119358, + "loss": 2.7913, + "theoretical_loss": 3.6698490057974658, + "tokens_seen": 942715904 + }, + { + "epoch": 3.02, + "learning_rate": 0.00036080240722166505, + "loss": 2.7292, + "theoretical_loss": 3.6698244623324676, + "tokens_seen": 942781440 + }, + { + "epoch": 3.02, + "learning_rate": 0.00036079237713139417, + "loss": 2.8, + "theoretical_loss": 3.6697999210511822, + "tokens_seen": 942846976 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003607823470411234, + "loss": 2.492, + "theoretical_loss": 3.6697753819532624, + "tokens_seen": 942912512 + }, + { + "epoch": 3.02, + "learning_rate": 0.00036077231695085253, + "loss": 2.8689, + "theoretical_loss": 3.6697508450383634, + "tokens_seen": 942978048 + }, + { + "epoch": 3.02, + "learning_rate": 0.00036076228686058177, + "loss": 2.7812, + "theoretical_loss": 3.6697263103061393, + "tokens_seen": 943043584 + }, + { + "epoch": 3.02, + "learning_rate": 0.00036075225677031095, + "loss": 2.5538, + "theoretical_loss": 3.6697017777562433, + "tokens_seen": 943109120 + }, + { + "epoch": 3.02, + "learning_rate": 0.00036074222668004013, + "loss": 2.7327, + "theoretical_loss": 3.6696772473883303, + "tokens_seen": 943174656 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003607321965897693, + "loss": 2.7159, + "theoretical_loss": 3.6696527192020545, + "tokens_seen": 943240192 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003607221664994985, + "loss": 2.5015, + "theoretical_loss": 3.66962819319707, + "tokens_seen": 943305728 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003607121364092277, + "loss": 2.6532, + "theoretical_loss": 3.6696036693730316, + "tokens_seen": 943371264 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003607021063189569, + "loss": 2.7995, + "theoretical_loss": 3.669579147729594, + "tokens_seen": 943436800 + }, + { + "epoch": 3.02, + "learning_rate": 0.00036069207622868604, + "loss": 2.6887, + "theoretical_loss": 3.6695546282664115, + "tokens_seen": 943502336 + }, + { + "epoch": 3.02, + "learning_rate": 0.00036068204613841527, + "loss": 2.9105, + "theoretical_loss": 3.669530110983139, + "tokens_seen": 943567872 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003606720160481444, + "loss": 2.4251, + "theoretical_loss": 3.669505595879432, + "tokens_seen": 943633408 + }, + { + "epoch": 3.02, + "learning_rate": 0.00036066198595787364, + "loss": 2.7481, + "theoretical_loss": 3.6694810829549445, + "tokens_seen": 943698944 + }, + { + "epoch": 3.02, + "objective/train/docs_used": 1111689, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.0604324340820312, + "objective/train/theoretical_loss": 3.669474955064285, + "objective/train/tokens_used": 964175328, + "theoretical_loss": 3.669474955064285, + "tokens_seen": 943715328 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003606519558676028, + "loss": 2.7923, + "theoretical_loss": 3.6694565722093317, + "tokens_seen": 943764480 + }, + { + "epoch": 3.02, + "learning_rate": 0.000360641925777332, + "loss": 2.8104, + "theoretical_loss": 3.6694320636422484, + "tokens_seen": 943830016 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003606318956870612, + "loss": 2.9466, + "theoretical_loss": 3.66940755725335, + "tokens_seen": 943895552 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003606218655967904, + "loss": 2.6342, + "theoretical_loss": 3.669383053042292, + "tokens_seen": 943961088 + }, + { + "epoch": 3.02, + "learning_rate": 0.00036061183550651954, + "loss": 2.73, + "theoretical_loss": 3.6693585510087288, + "tokens_seen": 944026624 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003606018054162488, + "loss": 2.8563, + "theoretical_loss": 3.6693340511523167, + "tokens_seen": 944092160 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003605917753259779, + "loss": 2.6699, + "theoretical_loss": 3.6693095534727105, + "tokens_seen": 944157696 + }, + { + "epoch": 3.02, + "learning_rate": 0.00036058174523570714, + "loss": 2.7349, + "theoretical_loss": 3.6692850579695664, + "tokens_seen": 944223232 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003605717151454363, + "loss": 2.8572, + "theoretical_loss": 3.6692605646425394, + "tokens_seen": 944288768 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003605616850551655, + "loss": 2.6415, + "theoretical_loss": 3.6692360734912848, + "tokens_seen": 944354304 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003605516549648947, + "loss": 2.7184, + "theoretical_loss": 3.6692115845154594, + "tokens_seen": 944419840 + }, + { + "epoch": 3.02, + "learning_rate": 0.00036054162487462386, + "loss": 2.7668, + "theoretical_loss": 3.669187097714718, + "tokens_seen": 944485376 + }, + { + "epoch": 3.02, + "learning_rate": 0.00036053159478435304, + "loss": 2.4856, + "theoretical_loss": 3.6691626130887176, + "tokens_seen": 944550912 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003605215646940823, + "loss": 2.7285, + "theoretical_loss": 3.669138130637114, + "tokens_seen": 944616448 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003605115346038114, + "loss": 2.8301, + "theoretical_loss": 3.669113650359562, + "tokens_seen": 944681984 + }, + { + "epoch": 3.02, + "learning_rate": 0.00036050150451354064, + "loss": 2.9229, + "theoretical_loss": 3.6690891722557186, + "tokens_seen": 944747520 + }, + { + "epoch": 3.02, + "learning_rate": 0.00036049147442326977, + "loss": 2.8787, + "theoretical_loss": 3.6690646963252402, + "tokens_seen": 944813056 + }, + { + "epoch": 3.02, + "learning_rate": 0.000360481444332999, + "loss": 2.8984, + "theoretical_loss": 3.6690402225677827, + "tokens_seen": 944878592 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003604714142427282, + "loss": 2.8173, + "theoretical_loss": 3.6690157509830033, + "tokens_seen": 944944128 + }, + { + "epoch": 3.02, + "learning_rate": 0.00036046138415245737, + "loss": 2.3796, + "theoretical_loss": 3.668991281570557, + "tokens_seen": 945009664 + }, + { + "epoch": 3.02, + "learning_rate": 0.00036045135406218655, + "loss": 2.8952, + "theoretical_loss": 3.668966814330102, + "tokens_seen": 945075200 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003604413239719158, + "loss": 2.8054, + "theoretical_loss": 3.668942349261294, + "tokens_seen": 945140736 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003604312938816449, + "loss": 2.7728, + "theoretical_loss": 3.6689178863637895, + "tokens_seen": 945206272 + }, + { + "epoch": 3.02, + "learning_rate": 0.00036042126379137414, + "loss": 2.8421, + "theoretical_loss": 3.668893425637245, + "tokens_seen": 945271808 + }, + { + "epoch": 3.02, + "learning_rate": 0.00036041123370110327, + "loss": 2.74, + "theoretical_loss": 3.668868967081319, + "tokens_seen": 945337344 + }, + { + "epoch": 3.02, + "objective/train/docs_used": 1112870, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6617534160614014, + "objective/train/theoretical_loss": 3.668862852781456, + "objective/train/tokens_used": 965813728, + "theoretical_loss": 3.668862852781456, + "tokens_seen": 945353728 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003604012036108325, + "loss": 2.8314, + "theoretical_loss": 3.6688445106956666, + "tokens_seen": 945402880 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003603911735205617, + "loss": 2.5809, + "theoretical_loss": 3.668820056479946, + "tokens_seen": 945468416 + }, + { + "epoch": 3.02, + "learning_rate": 0.00036038114343029087, + "loss": 2.5361, + "theoretical_loss": 3.6687956044338135, + "tokens_seen": 945533952 + }, + { + "epoch": 3.02, + "learning_rate": 0.00036037111334002005, + "loss": 2.7613, + "theoretical_loss": 3.6687711545569264, + "tokens_seen": 945599488 + }, + { + "epoch": 3.02, + "learning_rate": 0.00036036108324974923, + "loss": 2.7731, + "theoretical_loss": 3.668746706848942, + "tokens_seen": 945665024 + }, + { + "epoch": 3.02, + "learning_rate": 0.00036035105315947847, + "loss": 2.8019, + "theoretical_loss": 3.668722261309518, + "tokens_seen": 945730560 + }, + { + "epoch": 3.02, + "learning_rate": 0.00036034102306920765, + "loss": 2.8439, + "theoretical_loss": 3.6686978179383116, + "tokens_seen": 945796096 + }, + { + "epoch": 3.02, + "learning_rate": 0.00036033099297893683, + "loss": 2.8839, + "theoretical_loss": 3.6686733767349797, + "tokens_seen": 945861632 + }, + { + "epoch": 3.02, + "learning_rate": 0.000360320962888666, + "loss": 2.5422, + "theoretical_loss": 3.6686489376991807, + "tokens_seen": 945927168 + }, + { + "epoch": 3.02, + "learning_rate": 0.00036031093279839525, + "loss": 2.8942, + "theoretical_loss": 3.668624500830571, + "tokens_seen": 945992704 + }, + { + "epoch": 3.02, + "learning_rate": 0.00036030090270812437, + "loss": 2.5538, + "theoretical_loss": 3.6686000661288105, + "tokens_seen": 946058240 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003602908726178536, + "loss": 2.7451, + "theoretical_loss": 3.6685756335935547, + "tokens_seen": 946123776 + }, + { + "epoch": 3.02, + "learning_rate": 0.00036028084252758273, + "loss": 2.7525, + "theoretical_loss": 3.668551203224463, + "tokens_seen": 946189312 + }, + { + "epoch": 3.02, + "learning_rate": 0.00036027081243731197, + "loss": 2.7951, + "theoretical_loss": 3.668526775021192, + "tokens_seen": 946254848 + }, + { + "epoch": 3.02, + "learning_rate": 0.00036026078234704115, + "loss": 2.6365, + "theoretical_loss": 3.6685023489834006, + "tokens_seen": 946320384 + }, + { + "epoch": 3.02, + "learning_rate": 0.00036025075225677033, + "loss": 2.812, + "theoretical_loss": 3.6684779251107464, + "tokens_seen": 946385920 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003602407221664995, + "loss": 2.5312, + "theoretical_loss": 3.6684535034028887, + "tokens_seen": 946451456 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003602306920762287, + "loss": 2.7091, + "theoretical_loss": 3.668429083859484, + "tokens_seen": 946516992 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003602206619859579, + "loss": 2.9199, + "theoretical_loss": 3.668404666480192, + "tokens_seen": 946582528 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003602106318956871, + "loss": 2.807, + "theoretical_loss": 3.6683802512646704, + "tokens_seen": 946648064 + }, + { + "epoch": 3.02, + "learning_rate": 0.00036020060180541624, + "loss": 2.5709, + "theoretical_loss": 3.6683558382125785, + "tokens_seen": 946713600 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003601905717151455, + "loss": 2.6406, + "theoretical_loss": 3.6683314273235736, + "tokens_seen": 946779136 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003601805416248746, + "loss": 2.895, + "theoretical_loss": 3.6683070185973152, + "tokens_seen": 946844672 + }, + { + "epoch": 3.02, + "learning_rate": 0.00036017051153460384, + "loss": 2.7009, + "theoretical_loss": 3.668282612033462, + "tokens_seen": 946910208 + }, + { + "epoch": 3.02, + "learning_rate": 0.000360160481444333, + "loss": 2.796, + "theoretical_loss": 3.6682582076316725, + "tokens_seen": 946975744 + }, + { + "epoch": 3.02, + "objective/train/docs_used": 1113638, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.09346079826355, + "objective/train/theoretical_loss": 3.6682521068690077, + "objective/train/tokens_used": 967452128, + "theoretical_loss": 3.6682521068690077, + "tokens_seen": 946992128 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003601504513540622, + "loss": 2.9157, + "theoretical_loss": 3.6682338053916057, + "tokens_seen": 947041280 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003601404212637914, + "loss": 2.5278, + "theoretical_loss": 3.6682094053129206, + "tokens_seen": 947106816 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003601303911735206, + "loss": 2.7553, + "theoretical_loss": 3.6681850073952758, + "tokens_seen": 947172352 + }, + { + "epoch": 3.02, + "learning_rate": 0.00036012036108324974, + "loss": 2.7387, + "theoretical_loss": 3.6681606116383314, + "tokens_seen": 947237888 + }, + { + "epoch": 3.02, + "learning_rate": 0.000360110330992979, + "loss": 2.6852, + "theoretical_loss": 3.668136218041745, + "tokens_seen": 947303424 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003601003009027081, + "loss": 2.4123, + "theoretical_loss": 3.6681118266051778, + "tokens_seen": 947368960 + }, + { + "epoch": 3.02, + "learning_rate": 0.00036009027081243734, + "loss": 2.7635, + "theoretical_loss": 3.6680874373282872, + "tokens_seen": 947434496 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003600802407221665, + "loss": 2.7806, + "theoretical_loss": 3.6680630502107334, + "tokens_seen": 947500032 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003600702106318957, + "loss": 2.7379, + "theoretical_loss": 3.668038665252177, + "tokens_seen": 947565568 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003600601805416249, + "loss": 2.7847, + "theoretical_loss": 3.668014282452276, + "tokens_seen": 947631104 + }, + { + "epoch": 3.02, + "learning_rate": 0.00036005015045135406, + "loss": 2.6898, + "theoretical_loss": 3.6679899018106905, + "tokens_seen": 947696640 + }, + { + "epoch": 3.02, + "learning_rate": 0.00036004012036108324, + "loss": 2.7477, + "theoretical_loss": 3.66796552332708, + "tokens_seen": 947762176 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003600300902708125, + "loss": 2.9667, + "theoretical_loss": 3.667941147001105, + "tokens_seen": 947827712 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003600200601805416, + "loss": 2.6944, + "theoretical_loss": 3.6679167728324247, + "tokens_seen": 947893248 + }, + { + "epoch": 3.02, + "learning_rate": 0.00036001003009027084, + "loss": 2.8867, + "theoretical_loss": 3.6678924008206995, + "tokens_seen": 947958784 + }, + { + "epoch": 3.02, + "learning_rate": 0.00035999999999999997, + "loss": 2.6801, + "theoretical_loss": 3.6678680309655896, + "tokens_seen": 948024320 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003599899699097292, + "loss": 2.6097, + "theoretical_loss": 3.667843663266754, + "tokens_seen": 948089856 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003599799398194584, + "loss": 2.7772, + "theoretical_loss": 3.667819297723854, + "tokens_seen": 948155392 + }, + { + "epoch": 3.02, + "learning_rate": 0.00035996990972918757, + "loss": 2.722, + "theoretical_loss": 3.6677949343365492, + "tokens_seen": 948220928 + }, + { + "epoch": 3.02, + "learning_rate": 0.00035995987963891675, + "loss": 2.6455, + "theoretical_loss": 3.6677705731045007, + "tokens_seen": 948286464 + }, + { + "epoch": 3.02, + "learning_rate": 0.000359949849548646, + "loss": 2.7656, + "theoretical_loss": 3.6677462140273684, + "tokens_seen": 948352000 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003599398194583751, + "loss": 2.8047, + "theoretical_loss": 3.6677218571048122, + "tokens_seen": 948417536 + }, + { + "epoch": 3.02, + "learning_rate": 0.00035992978936810434, + "loss": 2.7955, + "theoretical_loss": 3.6676975023364937, + "tokens_seen": 948483072 + }, + { + "epoch": 3.02, + "learning_rate": 0.00035991975927783347, + "loss": 2.789, + "theoretical_loss": 3.667673149722073, + "tokens_seen": 948548608 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003599097291875627, + "loss": 2.9306, + "theoretical_loss": 3.667648799261211, + "tokens_seen": 948614144 + }, + { + "epoch": 3.02, + "objective/train/docs_used": 1115063, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7750394344329834, + "objective/train/theoretical_loss": 3.6676427119824493, + "objective/train/tokens_used": 969090528, + "theoretical_loss": 3.6676427119824493, + "tokens_seen": 948630528 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003598996990972919, + "loss": 2.7399, + "theoretical_loss": 3.6676244509535687, + "tokens_seen": 948679680 + }, + { + "epoch": 3.02, + "learning_rate": 0.00035988966900702107, + "loss": 2.7975, + "theoretical_loss": 3.6676001047988063, + "tokens_seen": 948745216 + }, + { + "epoch": 3.02, + "learning_rate": 0.00035987963891675025, + "loss": 2.7424, + "theoretical_loss": 3.667575760796585, + "tokens_seen": 948810752 + }, + { + "epoch": 3.02, + "learning_rate": 0.00035986960882647943, + "loss": 2.9323, + "theoretical_loss": 3.667551418946566, + "tokens_seen": 948876288 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003598595787362086, + "loss": 2.8068, + "theoretical_loss": 3.6675270792484107, + "tokens_seen": 948941824 + }, + { + "epoch": 3.02, + "learning_rate": 0.00035984954864593785, + "loss": 2.9573, + "theoretical_loss": 3.66750274170178, + "tokens_seen": 949007360 + }, + { + "epoch": 3.02, + "learning_rate": 0.000359839518555667, + "loss": 2.6737, + "theoretical_loss": 3.6674784063063344, + "tokens_seen": 949072896 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003598294884653962, + "loss": 2.7313, + "theoretical_loss": 3.6674540730617364, + "tokens_seen": 949138432 + }, + { + "epoch": 3.02, + "learning_rate": 0.00035981945837512534, + "loss": 2.6642, + "theoretical_loss": 3.6674297419676476, + "tokens_seen": 949203968 + }, + { + "epoch": 3.02, + "learning_rate": 0.00035980942828485457, + "loss": 2.8812, + "theoretical_loss": 3.6674054130237277, + "tokens_seen": 949269504 + }, + { + "epoch": 3.02, + "learning_rate": 0.00035979939819458375, + "loss": 2.6248, + "theoretical_loss": 3.66738108622964, + "tokens_seen": 949335040 + }, + { + "epoch": 3.02, + "learning_rate": 0.00035978936810431293, + "loss": 2.7558, + "theoretical_loss": 3.6673567615850455, + "tokens_seen": 949400576 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003597793380140421, + "loss": 2.6226, + "theoretical_loss": 3.6673324390896056, + "tokens_seen": 949466112 + }, + { + "epoch": 3.02, + "learning_rate": 0.00035976930792377135, + "loss": 2.5855, + "theoretical_loss": 3.667308118742983, + "tokens_seen": 949531648 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003597592778335005, + "loss": 2.7705, + "theoretical_loss": 3.6672838005448387, + "tokens_seen": 949597184 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003597492477432297, + "loss": 2.692, + "theoretical_loss": 3.667259484494835, + "tokens_seen": 949662720 + }, + { + "epoch": 3.02, + "learning_rate": 0.00035973921765295884, + "loss": 2.8313, + "theoretical_loss": 3.667235170592634, + "tokens_seen": 949728256 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003597291875626881, + "loss": 2.8125, + "theoretical_loss": 3.667210858837897, + "tokens_seen": 949793792 + }, + { + "epoch": 3.02, + "learning_rate": 0.00035971915747241726, + "loss": 2.6625, + "theoretical_loss": 3.6671865492302875, + "tokens_seen": 949859328 + }, + { + "epoch": 3.02, + "learning_rate": 0.00035970912738214644, + "loss": 2.6021, + "theoretical_loss": 3.667162241769467, + "tokens_seen": 949924864 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003596990972918756, + "loss": 2.829, + "theoretical_loss": 3.667137936455098, + "tokens_seen": 949990400 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003596890672016048, + "loss": 2.7101, + "theoretical_loss": 3.667113633286842, + "tokens_seen": 950055936 + }, + { + "epoch": 3.02, + "learning_rate": 0.000359679037111334, + "loss": 2.6948, + "theoretical_loss": 3.6670893322643634, + "tokens_seen": 950121472 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003596690070210632, + "loss": 2.716, + "theoretical_loss": 3.667065033387323, + "tokens_seen": 950187008 + }, + { + "epoch": 3.02, + "learning_rate": 0.00035965897693079234, + "loss": 2.7286, + "theoretical_loss": 3.6670407366553848, + "tokens_seen": 950252544 + }, + { + "epoch": 3.02, + "objective/train/docs_used": 1115859, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.0115370750427246, + "objective/train/theoretical_loss": 3.6670346628075325, + "objective/train/tokens_used": 970728928, + "theoretical_loss": 3.6670346628075325, + "tokens_seen": 950268928 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003596489468405216, + "loss": 2.8904, + "theoretical_loss": 3.66701644206821, + "tokens_seen": 950318080 + }, + { + "epoch": 3.02, + "learning_rate": 0.00035963891675025076, + "loss": 2.7355, + "theoretical_loss": 3.6669921496254627, + "tokens_seen": 950383616 + }, + { + "epoch": 3.02, + "learning_rate": 0.00035962888665997994, + "loss": 2.6775, + "theoretical_loss": 3.666967859326805, + "tokens_seen": 950449152 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003596188565697091, + "loss": 2.7278, + "theoretical_loss": 3.6669435711719, + "tokens_seen": 950514688 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003596088264794383, + "loss": 2.8868, + "theoretical_loss": 3.6669192851604113, + "tokens_seen": 950580224 + }, + { + "epoch": 3.02, + "learning_rate": 0.00035959879638916754, + "loss": 2.7949, + "theoretical_loss": 3.666895001292001, + "tokens_seen": 950645760 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003595887662988967, + "loss": 2.8372, + "theoretical_loss": 3.666870719566333, + "tokens_seen": 950711296 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003595787362086259, + "loss": 2.6296, + "theoretical_loss": 3.66684643998307, + "tokens_seen": 950776832 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003595687061183551, + "loss": 2.5486, + "theoretical_loss": 3.6668221625418758, + "tokens_seen": 950842368 + }, + { + "epoch": 3.02, + "learning_rate": 0.00035955867602808426, + "loss": 2.7116, + "theoretical_loss": 3.666797887242414, + "tokens_seen": 950907904 + }, + { + "epoch": 3.02, + "learning_rate": 0.00035954864593781344, + "loss": 2.6305, + "theoretical_loss": 3.6667736140843474, + "tokens_seen": 950973440 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003595386158475427, + "loss": 2.7846, + "theoretical_loss": 3.6667493430673397, + "tokens_seen": 951038976 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003595285857572718, + "loss": 2.7952, + "theoretical_loss": 3.6667250741910546, + "tokens_seen": 951104512 + }, + { + "epoch": 3.02, + "learning_rate": 0.00035951855566700104, + "loss": 2.8643, + "theoretical_loss": 3.666700807455156, + "tokens_seen": 951170048 + }, + { + "epoch": 3.02, + "learning_rate": 0.00035950852557673017, + "loss": 2.6962, + "theoretical_loss": 3.6666765428593076, + "tokens_seen": 951235584 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003594984954864594, + "loss": 2.7865, + "theoretical_loss": 3.6666522804031736, + "tokens_seen": 951301120 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003594884653961886, + "loss": 2.5407, + "theoretical_loss": 3.666628020086417, + "tokens_seen": 951366656 + }, + { + "epoch": 3.02, + "learning_rate": 0.00035947843530591777, + "loss": 2.3828, + "theoretical_loss": 3.6666037619087026, + "tokens_seen": 951432192 + }, + { + "epoch": 3.02, + "learning_rate": 0.00035946840521564695, + "loss": 2.8197, + "theoretical_loss": 3.6665795058696937, + "tokens_seen": 951497728 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003594583751253762, + "loss": 2.7182, + "theoretical_loss": 3.666555251969055, + "tokens_seen": 951563264 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003594483450351053, + "loss": 2.729, + "theoretical_loss": 3.6665310002064513, + "tokens_seen": 951628800 + }, + { + "epoch": 3.02, + "learning_rate": 0.00035943831494483454, + "loss": 2.6999, + "theoretical_loss": 3.6665067505815454, + "tokens_seen": 951694336 + }, + { + "epoch": 3.02, + "learning_rate": 0.00035942828485456367, + "loss": 2.7045, + "theoretical_loss": 3.6664825030940036, + "tokens_seen": 951759872 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003594182547642929, + "loss": 2.5441, + "theoretical_loss": 3.6664582577434888, + "tokens_seen": 951825408 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003594082246740221, + "loss": 2.7172, + "theoretical_loss": 3.666434014529666, + "tokens_seen": 951890944 + }, + { + "epoch": 3.02, + "objective/train/docs_used": 1116983, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5958456993103027, + "objective/train/theoretical_loss": 3.666427954060029, + "objective/train/tokens_used": 972367328, + "theoretical_loss": 3.666427954060029, + "tokens_seen": 951907328 + }, + { + "epoch": 3.02, + "learning_rate": 0.00035939819458375127, + "loss": 2.7405, + "theoretical_loss": 3.6664097734522, + "tokens_seen": 951956480 + }, + { + "epoch": 3.02, + "learning_rate": 0.00035938816449348045, + "loss": 2.8947, + "theoretical_loss": 3.666385534510755, + "tokens_seen": 952022016 + }, + { + "epoch": 3.02, + "learning_rate": 0.00035937813440320963, + "loss": 2.7593, + "theoretical_loss": 3.666361297704997, + "tokens_seen": 952087552 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003593681043129388, + "loss": 2.8236, + "theoretical_loss": 3.666337063034589, + "tokens_seen": 952153088 + }, + { + "epoch": 3.02, + "learning_rate": 0.00035935807422266805, + "loss": 2.4366, + "theoretical_loss": 3.666312830499197, + "tokens_seen": 952218624 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003593480441323972, + "loss": 2.8637, + "theoretical_loss": 3.6662886000984862, + "tokens_seen": 952284160 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003593380140421264, + "loss": 2.8358, + "theoretical_loss": 3.6662643718321215, + "tokens_seen": 952349696 + }, + { + "epoch": 3.02, + "learning_rate": 0.00035932798395185554, + "loss": 2.7572, + "theoretical_loss": 3.6662401456997675, + "tokens_seen": 952415232 + }, + { + "epoch": 3.02, + "learning_rate": 0.00035931795386158477, + "loss": 2.8309, + "theoretical_loss": 3.66621592170109, + "tokens_seen": 952480768 + }, + { + "epoch": 3.02, + "learning_rate": 0.00035930792377131395, + "loss": 3.0672, + "theoretical_loss": 3.6661916998357538, + "tokens_seen": 952546304 + }, + { + "epoch": 3.02, + "learning_rate": 0.00035929789368104313, + "loss": 2.6583, + "theoretical_loss": 3.6661674801034243, + "tokens_seen": 952611840 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003592878635907723, + "loss": 2.6992, + "theoretical_loss": 3.666143262503768, + "tokens_seen": 952677376 + }, + { + "epoch": 3.02, + "learning_rate": 0.00035927783350050155, + "loss": 2.7778, + "theoretical_loss": 3.666119047036449, + "tokens_seen": 952742912 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003592678034102307, + "loss": 2.7916, + "theoretical_loss": 3.6660948337011336, + "tokens_seen": 952808448 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003592577733199599, + "loss": 2.6527, + "theoretical_loss": 3.6660706224974877, + "tokens_seen": 952873984 + }, + { + "epoch": 3.02, + "learning_rate": 0.00035924774322968904, + "loss": 2.7234, + "theoretical_loss": 3.666046413425176, + "tokens_seen": 952939520 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003592377131394183, + "loss": 2.7744, + "theoretical_loss": 3.6660222064838655, + "tokens_seen": 953005056 + }, + { + "epoch": 3.02, + "learning_rate": 0.00035922768304914746, + "loss": 2.608, + "theoretical_loss": 3.665998001673221, + "tokens_seen": 953070592 + }, + { + "epoch": 3.02, + "learning_rate": 0.00035921765295887664, + "loss": 2.6749, + "theoretical_loss": 3.66597379899291, + "tokens_seen": 953136128 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003592076228686058, + "loss": 2.5654, + "theoretical_loss": 3.665949598442597, + "tokens_seen": 953201664 + }, + { + "epoch": 3.02, + "learning_rate": 0.000359197592778335, + "loss": 2.7041, + "theoretical_loss": 3.665925400021949, + "tokens_seen": 953267200 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003591875626880642, + "loss": 2.906, + "theoretical_loss": 3.6659012037306318, + "tokens_seen": 953332736 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003591775325977934, + "loss": 2.7529, + "theoretical_loss": 3.665877009568312, + "tokens_seen": 953398272 + }, + { + "epoch": 3.02, + "learning_rate": 0.00035916750250752254, + "loss": 2.8305, + "theoretical_loss": 3.665852817534655, + "tokens_seen": 953463808 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003591574724172518, + "loss": 2.9454, + "theoretical_loss": 3.6658286276293293, + "tokens_seen": 953529344 + }, + { + "epoch": 3.02, + "objective/train/docs_used": 1117727, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.588460683822632, + "objective/train/theoretical_loss": 3.66582258048551, + "objective/train/tokens_used": 974005728, + "theoretical_loss": 3.66582258048551, + "tokens_seen": 953545728 + }, + { + "epoch": 3.02, + "learning_rate": 0.00035914744232698096, + "loss": 2.6301, + "theoretical_loss": 3.6658044398519993, + "tokens_seen": 953594880 + }, + { + "epoch": 3.02, + "learning_rate": 0.00035913741223671014, + "loss": 2.8647, + "theoretical_loss": 3.6657802542023323, + "tokens_seen": 953660416 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003591273821464393, + "loss": 2.7181, + "theoretical_loss": 3.665756070679995, + "tokens_seen": 953725952 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003591173520561685, + "loss": 2.856, + "theoretical_loss": 3.6657318892846544, + "tokens_seen": 953791488 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003591073219658977, + "loss": 2.7305, + "theoretical_loss": 3.665707710015977, + "tokens_seen": 953857024 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003590972918756269, + "loss": 2.9282, + "theoretical_loss": 3.6656835328736292, + "tokens_seen": 953922560 + }, + { + "epoch": 3.02, + "learning_rate": 0.00035908726178535605, + "loss": 2.6648, + "theoretical_loss": 3.6656593578572787, + "tokens_seen": 953988096 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003590772316950853, + "loss": 2.7987, + "theoretical_loss": 3.6656351849665922, + "tokens_seen": 954053632 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003590672016048144, + "loss": 2.4934, + "theoretical_loss": 3.665611014201237, + "tokens_seen": 954119168 + }, + { + "epoch": 3.02, + "learning_rate": 0.00035905717151454364, + "loss": 2.6069, + "theoretical_loss": 3.66558684556088, + "tokens_seen": 954184704 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003590471414242728, + "loss": 2.8056, + "theoretical_loss": 3.665562679045189, + "tokens_seen": 954250240 + }, + { + "epoch": 3.02, + "learning_rate": 0.000359037111334002, + "loss": 2.9904, + "theoretical_loss": 3.6655385146538304, + "tokens_seen": 954315776 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003590270812437312, + "loss": 2.6449, + "theoretical_loss": 3.665514352386472, + "tokens_seen": 954381312 + }, + { + "epoch": 3.02, + "learning_rate": 0.00035901705115346037, + "loss": 2.7939, + "theoretical_loss": 3.665490192242782, + "tokens_seen": 954446848 + }, + { + "epoch": 3.02, + "learning_rate": 0.00035900702106318955, + "loss": 2.7474, + "theoretical_loss": 3.6654660342224266, + "tokens_seen": 954512384 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003589969909729188, + "loss": 2.6304, + "theoretical_loss": 3.6654418783250744, + "tokens_seen": 954577920 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003589869608826479, + "loss": 2.6741, + "theoretical_loss": 3.6654177245503927, + "tokens_seen": 954643456 + }, + { + "epoch": 3.02, + "learning_rate": 0.00035897693079237715, + "loss": 2.6727, + "theoretical_loss": 3.66539357289805, + "tokens_seen": 954708992 + }, + { + "epoch": 3.02, + "learning_rate": 0.00035896690070210633, + "loss": 2.6506, + "theoretical_loss": 3.6653694233677125, + "tokens_seen": 954774528 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003589568706118355, + "loss": 2.7298, + "theoretical_loss": 3.6653452759590497, + "tokens_seen": 954840064 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003589468405215647, + "loss": 2.7313, + "theoretical_loss": 3.665321130671729, + "tokens_seen": 954905600 + }, + { + "epoch": 3.02, + "learning_rate": 0.00035893681043129387, + "loss": 2.9473, + "theoretical_loss": 3.665296987505419, + "tokens_seen": 954971136 + }, + { + "epoch": 3.02, + "learning_rate": 0.00035892678034102305, + "loss": 2.7933, + "theoretical_loss": 3.6652728464597866, + "tokens_seen": 955036672 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003589167502507523, + "loss": 2.6645, + "theoretical_loss": 3.6652487075345013, + "tokens_seen": 955102208 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003589067201604814, + "loss": 2.605, + "theoretical_loss": 3.6652245707292304, + "tokens_seen": 955167744 + }, + { + "epoch": 3.02, + "objective/train/docs_used": 1119307, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7554943561553955, + "objective/train/theoretical_loss": 3.6652185368591264, + "objective/train/tokens_used": 975644128, + "theoretical_loss": 3.6652185368591264, + "tokens_seen": 955184128 + }, + { + "epoch": 3.02, + "learning_rate": 0.00035889669007021065, + "loss": 2.7885, + "theoretical_loss": 3.6652004360436434, + "tokens_seen": 955233280 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003588866599799398, + "loss": 2.7021, + "theoretical_loss": 3.6651763034774074, + "tokens_seen": 955298816 + }, + { + "epoch": 3.02, + "learning_rate": 0.000358876629889669, + "loss": 2.7756, + "theoretical_loss": 3.665152173030192, + "tokens_seen": 955364352 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003588665997993982, + "loss": 2.6781, + "theoretical_loss": 3.6651280447016656, + "tokens_seen": 955429888 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003588565697091274, + "loss": 2.8244, + "theoretical_loss": 3.6651039184914964, + "tokens_seen": 955495424 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003588465396188566, + "loss": 2.6043, + "theoretical_loss": 3.665079794399354, + "tokens_seen": 955560960 + }, + { + "epoch": 3.02, + "learning_rate": 0.00035883650952858574, + "loss": 2.5664, + "theoretical_loss": 3.6650556724249057, + "tokens_seen": 955626496 + }, + { + "epoch": 3.02, + "learning_rate": 0.00035882647943831497, + "loss": 2.6075, + "theoretical_loss": 3.665031552567822, + "tokens_seen": 955692032 + }, + { + "epoch": 3.02, + "learning_rate": 0.00035881644934804415, + "loss": 2.5505, + "theoretical_loss": 3.665007434827771, + "tokens_seen": 955757568 + }, + { + "epoch": 3.02, + "learning_rate": 0.00035880641925777333, + "loss": 2.5502, + "theoretical_loss": 3.664983319204422, + "tokens_seen": 955823104 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003587963891675025, + "loss": 2.5772, + "theoretical_loss": 3.664959205697444, + "tokens_seen": 955888640 + }, + { + "epoch": 3.02, + "learning_rate": 0.00035878635907723175, + "loss": 2.7139, + "theoretical_loss": 3.664935094306506, + "tokens_seen": 955954176 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003587763289869609, + "loss": 2.8381, + "theoretical_loss": 3.6649109850312778, + "tokens_seen": 956019712 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003587662988966901, + "loss": 2.5263, + "theoretical_loss": 3.664886877871428, + "tokens_seen": 956085248 + }, + { + "epoch": 3.02, + "learning_rate": 0.00035875626880641924, + "loss": 2.7676, + "theoretical_loss": 3.664862772826627, + "tokens_seen": 956150784 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003587462387161485, + "loss": 2.7778, + "theoretical_loss": 3.6648386698965436, + "tokens_seen": 956216320 + }, + { + "epoch": 3.02, + "learning_rate": 0.00035873620862587766, + "loss": 2.637, + "theoretical_loss": 3.664814569080847, + "tokens_seen": 956281856 + }, + { + "epoch": 3.02, + "learning_rate": 0.00035872617853560684, + "loss": 2.7548, + "theoretical_loss": 3.664790470379208, + "tokens_seen": 956347392 + }, + { + "epoch": 3.02, + "learning_rate": 0.000358716148445336, + "loss": 2.8223, + "theoretical_loss": 3.6647663737912954, + "tokens_seen": 956412928 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003587061183550652, + "loss": 2.8319, + "theoretical_loss": 3.664742279316779, + "tokens_seen": 956478464 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003586960882647944, + "loss": 2.6764, + "theoretical_loss": 3.664718186955329, + "tokens_seen": 956544000 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003586860581745236, + "loss": 2.7721, + "theoretical_loss": 3.6646940967066155, + "tokens_seen": 956609536 + }, + { + "epoch": 3.02, + "learning_rate": 0.00035867602808425274, + "loss": 2.5923, + "theoretical_loss": 3.664670008570308, + "tokens_seen": 956675072 + }, + { + "epoch": 3.02, + "learning_rate": 0.000358665997993982, + "loss": 2.7841, + "theoretical_loss": 3.664645922546077, + "tokens_seen": 956740608 + }, + { + "epoch": 3.02, + "learning_rate": 0.00035865596790371116, + "loss": 2.5776, + "theoretical_loss": 3.6646218386335923, + "tokens_seen": 956806144 + }, + { + "epoch": 3.02, + "objective/train/docs_used": 1120060, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.80765438079834, + "objective/train/theoretical_loss": 3.6646158179853927, + "objective/train/tokens_used": 977282528, + "theoretical_loss": 3.6646158179853927, + "tokens_seen": 956822528 + }, + { + "epoch": 3.02, + "learning_rate": 0.00035864593781344034, + "loss": 2.6522, + "theoretical_loss": 3.6645977568325243, + "tokens_seen": 956871680 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003586359077231695, + "loss": 2.8831, + "theoretical_loss": 3.664573677142543, + "tokens_seen": 956937216 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003586258776328987, + "loss": 2.4891, + "theoretical_loss": 3.664549599563319, + "tokens_seen": 957002752 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003586158475426279, + "loss": 2.6942, + "theoretical_loss": 3.6645255240945236, + "tokens_seen": 957068288 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003586058174523571, + "loss": 2.6859, + "theoretical_loss": 3.664501450735826, + "tokens_seen": 957133824 + }, + { + "epoch": 3.02, + "learning_rate": 0.00035859578736208625, + "loss": 2.7707, + "theoretical_loss": 3.6644773794868972, + "tokens_seen": 957199360 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003585857572718155, + "loss": 2.6198, + "theoretical_loss": 3.664453310347408, + "tokens_seen": 957264896 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003585757271815446, + "loss": 2.8187, + "theoretical_loss": 3.6644292433170293, + "tokens_seen": 957330432 + }, + { + "epoch": 3.02, + "learning_rate": 0.00035856569709127384, + "loss": 2.5686, + "theoretical_loss": 3.664405178395432, + "tokens_seen": 957395968 + }, + { + "epoch": 3.02, + "learning_rate": 0.000358555667001003, + "loss": 2.922, + "theoretical_loss": 3.6643811155822865, + "tokens_seen": 957461504 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003585456369107322, + "loss": 2.66, + "theoretical_loss": 3.6643570548772644, + "tokens_seen": 957527040 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003585356068204614, + "loss": 2.8352, + "theoretical_loss": 3.6643329962800353, + "tokens_seen": 957592576 + }, + { + "epoch": 3.02, + "learning_rate": 0.00035852557673019057, + "loss": 2.6895, + "theoretical_loss": 3.6643089397902724, + "tokens_seen": 957658112 + }, + { + "epoch": 3.02, + "learning_rate": 0.00035851554663991975, + "loss": 2.9951, + "theoretical_loss": 3.6642848854076453, + "tokens_seen": 957723648 + }, + { + "epoch": 3.02, + "learning_rate": 0.000358505516549649, + "loss": 2.7052, + "theoretical_loss": 3.6642608331318263, + "tokens_seen": 957789184 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003584954864593781, + "loss": 2.7671, + "theoretical_loss": 3.6642367829624862, + "tokens_seen": 957854720 + }, + { + "epoch": 3.02, + "learning_rate": 0.00035848545636910735, + "loss": 2.7657, + "theoretical_loss": 3.664212734899296, + "tokens_seen": 957920256 + }, + { + "epoch": 3.02, + "learning_rate": 0.00035847542627883653, + "loss": 2.7118, + "theoretical_loss": 3.6641886889419286, + "tokens_seen": 957985792 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003584653961885657, + "loss": 2.7238, + "theoretical_loss": 3.6641646450900542, + "tokens_seen": 958051328 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003584553660982949, + "loss": 2.8321, + "theoretical_loss": 3.664140603343345, + "tokens_seen": 958116864 + }, + { + "epoch": 3.02, + "learning_rate": 0.00035844533600802407, + "loss": 2.7182, + "theoretical_loss": 3.664116563701472, + "tokens_seen": 958182400 + }, + { + "epoch": 3.02, + "learning_rate": 0.00035843530591775325, + "loss": 2.5963, + "theoretical_loss": 3.664092526164108, + "tokens_seen": 958247936 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003584252758274825, + "loss": 2.9829, + "theoretical_loss": 3.6640684907309247, + "tokens_seen": 958313472 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003584152457372116, + "loss": 2.8543, + "theoretical_loss": 3.6640444574015927, + "tokens_seen": 958379008 + }, + { + "epoch": 3.02, + "learning_rate": 0.00035840521564694085, + "loss": 2.6679, + "theoretical_loss": 3.664020426175786, + "tokens_seen": 958444544 + }, + { + "epoch": 3.02, + "objective/train/docs_used": 1121284, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.857239246368408, + "objective/train/theoretical_loss": 3.6640144186979713, + "objective/train/tokens_used": 978920928, + "theoretical_loss": 3.6640144186979713, + "tokens_seen": 958460928 + }, + { + "epoch": 3.02, + "learning_rate": 0.00035839518555667, + "loss": 2.6887, + "theoretical_loss": 3.6639963970531753, + "tokens_seen": 958510080 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003583851554663992, + "loss": 2.8986, + "theoretical_loss": 3.6639723700334335, + "tokens_seen": 958575616 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003583751253761284, + "loss": 2.7256, + "theoretical_loss": 3.663948345116232, + "tokens_seen": 958641152 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003583650952858576, + "loss": 2.7297, + "theoretical_loss": 3.663924322301244, + "tokens_seen": 958706688 + }, + { + "epoch": 3.02, + "learning_rate": 0.00035835506519558676, + "loss": 2.6622, + "theoretical_loss": 3.663900301588141, + "tokens_seen": 958772224 + }, + { + "epoch": 3.02, + "learning_rate": 0.00035834503510531594, + "loss": 2.5843, + "theoretical_loss": 3.663876282976596, + "tokens_seen": 958837760 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003583350050150451, + "loss": 2.6251, + "theoretical_loss": 3.6638522664662814, + "tokens_seen": 958903296 + }, + { + "epoch": 3.02, + "learning_rate": 0.00035832497492477435, + "loss": 2.8493, + "theoretical_loss": 3.66382825205687, + "tokens_seen": 958968832 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003583149448345035, + "loss": 2.9119, + "theoretical_loss": 3.6638042397480337, + "tokens_seen": 959034368 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003583049147442327, + "loss": 2.6118, + "theoretical_loss": 3.663780229539446, + "tokens_seen": 959099904 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003582948846539619, + "loss": 2.6434, + "theoretical_loss": 3.6637562214307797, + "tokens_seen": 959165440 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003582848545636911, + "loss": 2.779, + "theoretical_loss": 3.6637322154217076, + "tokens_seen": 959230976 + }, + { + "epoch": 3.02, + "learning_rate": 0.00035827482447342026, + "loss": 2.773, + "theoretical_loss": 3.6637082115119024, + "tokens_seen": 959296512 + }, + { + "epoch": 3.02, + "learning_rate": 0.00035826479438314944, + "loss": 2.7536, + "theoretical_loss": 3.663684209701037, + "tokens_seen": 959362048 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003582547642928786, + "loss": 2.7197, + "theoretical_loss": 3.6636602099887847, + "tokens_seen": 959427584 + }, + { + "epoch": 3.02, + "learning_rate": 0.00035824473420260786, + "loss": 2.8215, + "theoretical_loss": 3.6636362123748194, + "tokens_seen": 959493120 + }, + { + "epoch": 3.02, + "learning_rate": 0.000358234704112337, + "loss": 2.921, + "theoretical_loss": 3.663612216858813, + "tokens_seen": 959558656 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003582246740220662, + "loss": 2.8807, + "theoretical_loss": 3.6635882234404393, + "tokens_seen": 959624192 + }, + { + "epoch": 3.02, + "learning_rate": 0.00035821464393179535, + "loss": 2.7939, + "theoretical_loss": 3.6635642321193727, + "tokens_seen": 959689728 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003582046138415246, + "loss": 2.8857, + "theoretical_loss": 3.6635402428952855, + "tokens_seen": 959755264 + }, + { + "epoch": 3.02, + "learning_rate": 0.00035819458375125376, + "loss": 2.5746, + "theoretical_loss": 3.6635162557678513, + "tokens_seen": 959820800 + }, + { + "epoch": 3.02, + "learning_rate": 0.00035818455366098294, + "loss": 2.6412, + "theoretical_loss": 3.6634922707367443, + "tokens_seen": 959886336 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003581745235707121, + "loss": 2.4484, + "theoretical_loss": 3.6634682878016376, + "tokens_seen": 959951872 + }, + { + "epoch": 3.02, + "learning_rate": 0.00035816449348044136, + "loss": 2.7905, + "theoretical_loss": 3.6634443069622056, + "tokens_seen": 960017408 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003581544633901705, + "loss": 2.6442, + "theoretical_loss": 3.6634203282181215, + "tokens_seen": 960082944 + }, + { + "epoch": 3.02, + "objective/train/docs_used": 1121988, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.3468472957611084, + "objective/train/theoretical_loss": 3.6634143338594605, + "objective/train/tokens_used": 980559328, + "theoretical_loss": 3.6634143338594605, + "tokens_seen": 960099328 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003581444332998997, + "loss": 2.4599, + "theoretical_loss": 3.6633963515690597, + "tokens_seen": 960148480 + }, + { + "epoch": 3.02, + "learning_rate": 0.00035813440320962885, + "loss": 2.7676, + "theoretical_loss": 3.663372377014694, + "tokens_seen": 960214016 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003581243731193581, + "loss": 2.9168, + "theoretical_loss": 3.6633484045546982, + "tokens_seen": 960279552 + }, + { + "epoch": 3.02, + "learning_rate": 0.00035811434302908727, + "loss": 2.7659, + "theoretical_loss": 3.6633244341887465, + "tokens_seen": 960345088 + }, + { + "epoch": 3.02, + "learning_rate": 0.00035810431293881645, + "loss": 2.8983, + "theoretical_loss": 3.663300465916514, + "tokens_seen": 960410624 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003580942828485457, + "loss": 2.7638, + "theoretical_loss": 3.663276499737673, + "tokens_seen": 960476160 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003580842527582748, + "loss": 2.673, + "theoretical_loss": 3.6632525356518997, + "tokens_seen": 960541696 + }, + { + "epoch": 3.02, + "learning_rate": 0.00035807422266800404, + "loss": 2.8277, + "theoretical_loss": 3.663228573658868, + "tokens_seen": 960607232 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003580641925777332, + "loss": 2.9438, + "theoretical_loss": 3.6632046137582526, + "tokens_seen": 960672768 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003580541624874624, + "loss": 2.6973, + "theoretical_loss": 3.663180655949727, + "tokens_seen": 960738304 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003580441323971916, + "loss": 2.8067, + "theoretical_loss": 3.663156700232967, + "tokens_seen": 960803840 + }, + { + "epoch": 3.02, + "learning_rate": 0.00035803410230692077, + "loss": 2.8689, + "theoretical_loss": 3.663132746607647, + "tokens_seen": 960869376 + }, + { + "epoch": 3.02, + "learning_rate": 0.00035802407221664995, + "loss": 2.9861, + "theoretical_loss": 3.663108795073441, + "tokens_seen": 960934912 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003580140421263792, + "loss": 2.9207, + "theoretical_loss": 3.663084845630025, + "tokens_seen": 961000448 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003580040120361083, + "loss": 2.8269, + "theoretical_loss": 3.663060898277073, + "tokens_seen": 961065984 + }, + { + "epoch": 3.02, + "learning_rate": 0.00035799398194583755, + "loss": 2.7846, + "theoretical_loss": 3.663036953014261, + "tokens_seen": 961131520 + }, + { + "epoch": 3.02, + "learning_rate": 0.00035798395185556673, + "loss": 2.8141, + "theoretical_loss": 3.663013009841263, + "tokens_seen": 961197056 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003579739217652959, + "loss": 2.8821, + "theoretical_loss": 3.6629890687577547, + "tokens_seen": 961262592 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003579638916750251, + "loss": 2.8052, + "theoretical_loss": 3.662965129763412, + "tokens_seen": 961328128 + }, + { + "epoch": 3.02, + "learning_rate": 0.00035795386158475427, + "loss": 2.7982, + "theoretical_loss": 3.6629411928579083, + "tokens_seen": 961393664 + }, + { + "epoch": 3.02, + "learning_rate": 0.00035794383149448345, + "loss": 2.7195, + "theoretical_loss": 3.662917258040921, + "tokens_seen": 961459200 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003579338014042127, + "loss": 2.8697, + "theoretical_loss": 3.6628933253121243, + "tokens_seen": 961524736 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003579237713139418, + "loss": 2.8098, + "theoretical_loss": 3.6628693946711937, + "tokens_seen": 961590272 + }, + { + "epoch": 3.02, + "learning_rate": 0.00035791374122367105, + "loss": 2.8547, + "theoretical_loss": 3.6628454661178056, + "tokens_seen": 961655808 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003579037111334002, + "loss": 2.541, + "theoretical_loss": 3.662821539651635, + "tokens_seen": 961721344 + }, + { + "epoch": 3.02, + "objective/train/docs_used": 1123320, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.837873697280884, + "objective/train/theoretical_loss": 3.6628155583611823, + "objective/train/tokens_used": 982197728, + "theoretical_loss": 3.6628155583611823, + "tokens_seen": 961737728 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003578936810431294, + "loss": 2.8232, + "theoretical_loss": 3.662797615272358, + "tokens_seen": 961786880 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003578836509528586, + "loss": 2.8893, + "theoretical_loss": 3.6627736929796497, + "tokens_seen": 961852416 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003578736208625878, + "loss": 2.537, + "theoretical_loss": 3.6627497727731866, + "tokens_seen": 961917952 + }, + { + "epoch": 3.02, + "learning_rate": 0.00035786359077231696, + "loss": 2.8157, + "theoretical_loss": 3.6627258546526447, + "tokens_seen": 961983488 + }, + { + "epoch": 3.02, + "learning_rate": 0.00035785356068204614, + "loss": 3.0576, + "theoretical_loss": 3.6627019386177, + "tokens_seen": 962049024 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003578435305917753, + "loss": 2.7024, + "theoretical_loss": 3.662678024668028, + "tokens_seen": 962114560 + }, + { + "epoch": 3.02, + "learning_rate": 0.00035783350050150455, + "loss": 2.7436, + "theoretical_loss": 3.6626541128033057, + "tokens_seen": 962180096 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003578234704112337, + "loss": 2.741, + "theoretical_loss": 3.662630203023209, + "tokens_seen": 962245632 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003578134403209629, + "loss": 2.8045, + "theoretical_loss": 3.6626062953274134, + "tokens_seen": 962311168 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003578034102306921, + "loss": 2.6184, + "theoretical_loss": 3.6625823897155962, + "tokens_seen": 962376704 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003577933801404213, + "loss": 2.9925, + "theoretical_loss": 3.662558486187434, + "tokens_seen": 962442240 + }, + { + "epoch": 3.02, + "learning_rate": 0.00035778335005015046, + "loss": 3.0443, + "theoretical_loss": 3.6625345847426027, + "tokens_seen": 962507776 + }, + { + "epoch": 3.02, + "learning_rate": 0.00035777331995987964, + "loss": 2.714, + "theoretical_loss": 3.662510685380779, + "tokens_seen": 962573312 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003577632898696088, + "loss": 2.6968, + "theoretical_loss": 3.6624867881016403, + "tokens_seen": 962638848 + }, + { + "epoch": 3.02, + "learning_rate": 0.00035775325977933806, + "loss": 2.931, + "theoretical_loss": 3.662462892904862, + "tokens_seen": 962704384 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003577432296890672, + "loss": 2.6591, + "theoretical_loss": 3.662438999790122, + "tokens_seen": 962769920 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003577331995987964, + "loss": 3.067, + "theoretical_loss": 3.662415108757097, + "tokens_seen": 962835456 + }, + { + "epoch": 3.02, + "learning_rate": 0.00035772316950852555, + "loss": 2.8951, + "theoretical_loss": 3.6623912198054636, + "tokens_seen": 962900992 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003577131394182548, + "loss": 2.7869, + "theoretical_loss": 3.662367332934899, + "tokens_seen": 962966528 + }, + { + "epoch": 3.02, + "learning_rate": 0.00035770310932798396, + "loss": 3.0066, + "theoretical_loss": 3.66234344814508, + "tokens_seen": 963032064 + }, + { + "epoch": 3.02, + "learning_rate": 0.00035769307923771314, + "loss": 2.7544, + "theoretical_loss": 3.6623195654356846, + "tokens_seen": 963097600 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003576830491474423, + "loss": 2.7924, + "theoretical_loss": 3.6622956848063897, + "tokens_seen": 963163136 + }, + { + "epoch": 3.02, + "learning_rate": 0.00035767301905717156, + "loss": 2.6071, + "theoretical_loss": 3.6622718062568715, + "tokens_seen": 963228672 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003576629889669007, + "loss": 2.8556, + "theoretical_loss": 3.662247929786809, + "tokens_seen": 963294208 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003576529588766299, + "loss": 2.9046, + "theoretical_loss": 3.662224055395879, + "tokens_seen": 963359744 + }, + { + "epoch": 3.02, + "objective/train/docs_used": 1123898, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7481706142425537, + "objective/train/theoretical_loss": 3.662218087122973, + "objective/train/tokens_used": 983836128, + "theoretical_loss": 3.662218087122973, + "tokens_seen": 963376128 + }, + { + "epoch": 3.02, + "learning_rate": 0.00035764292878635905, + "loss": 2.9226, + "theoretical_loss": 3.662200183083759, + "tokens_seen": 963425280 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003576328986960883, + "loss": 2.8547, + "theoretical_loss": 3.6621763128501263, + "tokens_seen": 963490816 + }, + { + "epoch": 3.02, + "learning_rate": 0.00035762286860581747, + "loss": 2.7839, + "theoretical_loss": 3.6621524446946587, + "tokens_seen": 963556352 + }, + { + "epoch": 3.02, + "learning_rate": 0.00035761283851554665, + "loss": 2.7891, + "theoretical_loss": 3.662128578617035, + "tokens_seen": 963621888 + }, + { + "epoch": 3.02, + "learning_rate": 0.00035760280842527583, + "loss": 2.7659, + "theoretical_loss": 3.662104714616931, + "tokens_seen": 963687424 + }, + { + "epoch": 3.02, + "learning_rate": 0.000357592778335005, + "loss": 2.864, + "theoretical_loss": 3.662080852694027, + "tokens_seen": 963752960 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003575827482447342, + "loss": 2.6925, + "theoretical_loss": 3.662056992847999, + "tokens_seen": 963818496 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003575727181544634, + "loss": 2.9939, + "theoretical_loss": 3.6620331350785253, + "tokens_seen": 963884032 + }, + { + "epoch": 3.02, + "learning_rate": 0.00035756268806419255, + "loss": 2.5997, + "theoretical_loss": 3.662009279385285, + "tokens_seen": 963949568 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003575526579739218, + "loss": 2.5903, + "theoretical_loss": 3.661985425767956, + "tokens_seen": 964015104 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003575426278836509, + "loss": 2.9253, + "theoretical_loss": 3.6619615742262157, + "tokens_seen": 964080640 + }, + { + "epoch": 3.02, + "learning_rate": 0.00035753259779338015, + "loss": 3.0226, + "theoretical_loss": 3.661937724759743, + "tokens_seen": 964146176 + }, + { + "epoch": 3.02, + "learning_rate": 0.00035752256770310933, + "loss": 2.7521, + "theoretical_loss": 3.6619138773682165, + "tokens_seen": 964211712 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003575125376128385, + "loss": 2.8833, + "theoretical_loss": 3.6618900320513146, + "tokens_seen": 964277248 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003575025075225677, + "loss": 2.9383, + "theoretical_loss": 3.661866188808715, + "tokens_seen": 964342784 + }, + { + "epoch": 3.02, + "learning_rate": 0.00035749247743229693, + "loss": 2.7874, + "theoretical_loss": 3.661842347640098, + "tokens_seen": 964408320 + }, + { + "epoch": 3.02, + "learning_rate": 0.00035748244734202606, + "loss": 2.7917, + "theoretical_loss": 3.6618185085451405, + "tokens_seen": 964473856 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003574724172517553, + "loss": 2.7022, + "theoretical_loss": 3.661794671523522, + "tokens_seen": 964539392 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003574623871614844, + "loss": 2.6121, + "theoretical_loss": 3.661770836574922, + "tokens_seen": 964604928 + }, + { + "epoch": 3.02, + "learning_rate": 0.00035745235707121365, + "loss": 2.8655, + "theoretical_loss": 3.661747003699018, + "tokens_seen": 964670464 + }, + { + "epoch": 3.02, + "learning_rate": 0.00035744232698094283, + "loss": 2.7242, + "theoretical_loss": 3.66172317289549, + "tokens_seen": 964736000 + }, + { + "epoch": 3.02, + "learning_rate": 0.000357432296890672, + "loss": 2.9418, + "theoretical_loss": 3.661699344164017, + "tokens_seen": 964801536 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003574222668004012, + "loss": 2.9072, + "theoretical_loss": 3.661675517504277, + "tokens_seen": 964867072 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003574122367101304, + "loss": 2.6928, + "theoretical_loss": 3.6616516929159504, + "tokens_seen": 964932608 + }, + { + "epoch": 3.02, + "learning_rate": 0.00035740220661985956, + "loss": 2.5438, + "theoretical_loss": 3.661627870398716, + "tokens_seen": 964998144 + }, + { + "epoch": 3.02, + "objective/train/docs_used": 1124893, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8355164527893066, + "objective/train/theoretical_loss": 3.6616219150929785, + "objective/train/tokens_used": 985474528, + "theoretical_loss": 3.6616219150929785, + "tokens_seen": 965014528 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003573921765295888, + "loss": 2.7413, + "theoretical_loss": 3.6616040499522535, + "tokens_seen": 965063680 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003573821464393179, + "loss": 2.8473, + "theoretical_loss": 3.661580231576242, + "tokens_seen": 965129216 + }, + { + "epoch": 3.02, + "learning_rate": 0.00035737211634904716, + "loss": 2.5775, + "theoretical_loss": 3.6615564152703604, + "tokens_seen": 965194752 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003573620862587763, + "loss": 2.9372, + "theoretical_loss": 3.6615326010342892, + "tokens_seen": 965260288 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003573520561685055, + "loss": 2.6619, + "theoretical_loss": 3.6615087888677076, + "tokens_seen": 965325824 + }, + { + "epoch": 3.02, + "learning_rate": 0.00035734202607823475, + "loss": 2.8003, + "theoretical_loss": 3.661484978770295, + "tokens_seen": 965391360 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003573319959879639, + "loss": 2.6679, + "theoretical_loss": 3.661461170741732, + "tokens_seen": 965456896 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003573219658976931, + "loss": 2.7185, + "theoretical_loss": 3.6614373647816976, + "tokens_seen": 965522432 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003573119358074223, + "loss": 2.7959, + "theoretical_loss": 3.661413560889871, + "tokens_seen": 965587968 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003573019057171515, + "loss": 2.6025, + "theoretical_loss": 3.6613897590659343, + "tokens_seen": 965653504 + }, + { + "epoch": 3.02, + "learning_rate": 0.00035729187562688066, + "loss": 2.8268, + "theoretical_loss": 3.661365959309566, + "tokens_seen": 965719040 + }, + { + "epoch": 3.02, + "learning_rate": 0.00035728184553660984, + "loss": 2.8639, + "theoretical_loss": 3.6613421616204462, + "tokens_seen": 965784576 + }, + { + "epoch": 3.02, + "learning_rate": 0.000357271815446339, + "loss": 2.7539, + "theoretical_loss": 3.6613183659982553, + "tokens_seen": 965850112 + }, + { + "epoch": 3.02, + "learning_rate": 0.00035726178535606826, + "loss": 2.7141, + "theoretical_loss": 3.6612945724426744, + "tokens_seen": 965915648 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003572517552657974, + "loss": 2.6672, + "theoretical_loss": 3.661270780953383, + "tokens_seen": 965981184 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003572417251755266, + "loss": 2.8184, + "theoretical_loss": 3.661246991530061, + "tokens_seen": 966046720 + }, + { + "epoch": 3.02, + "learning_rate": 0.00035723169508525575, + "loss": 2.7297, + "theoretical_loss": 3.66122320417239, + "tokens_seen": 966112256 + }, + { + "epoch": 3.02, + "learning_rate": 0.000357221664994985, + "loss": 2.7294, + "theoretical_loss": 3.661199418880049, + "tokens_seen": 966177792 + }, + { + "epoch": 3.02, + "learning_rate": 0.00035721163490471416, + "loss": 2.9766, + "theoretical_loss": 3.6611756356527203, + "tokens_seen": 966243328 + }, + { + "epoch": 3.02, + "learning_rate": 0.00035720160481444334, + "loss": 2.7843, + "theoretical_loss": 3.661151854490084, + "tokens_seen": 966308864 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003571915747241725, + "loss": 2.684, + "theoretical_loss": 3.6611280753918205, + "tokens_seen": 966374400 + }, + { + "epoch": 3.02, + "learning_rate": 0.00035718154463390176, + "loss": 2.5815, + "theoretical_loss": 3.6611042983576105, + "tokens_seen": 966439936 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003571715145436309, + "loss": 2.6649, + "theoretical_loss": 3.6610805233871355, + "tokens_seen": 966505472 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003571614844533601, + "loss": 2.8501, + "theoretical_loss": 3.6610567504800766, + "tokens_seen": 966571008 + }, + { + "epoch": 3.02, + "learning_rate": 0.00035715145436308925, + "loss": 2.8742, + "theoretical_loss": 3.6610329796361136, + "tokens_seen": 966636544 + }, + { + "epoch": 3.02, + "objective/train/docs_used": 1125576, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.660078525543213, + "objective/train/theoretical_loss": 3.6610270372474445, + "objective/train/tokens_used": 987112928, + "theoretical_loss": 3.6610270372474445, + "tokens_seen": 966652928 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003571414242728185, + "loss": 2.6385, + "theoretical_loss": 3.661009210854929, + "tokens_seen": 966702080 + }, + { + "epoch": 3.02, + "learning_rate": 0.00035713139418254767, + "loss": 2.7278, + "theoretical_loss": 3.660985444136203, + "tokens_seen": 966767616 + }, + { + "epoch": 3.02, + "learning_rate": 0.00035712136409227685, + "loss": 2.8441, + "theoretical_loss": 3.660961679479617, + "tokens_seen": 966833152 + }, + { + "epoch": 3.02, + "learning_rate": 0.00035711133400200603, + "loss": 2.8036, + "theoretical_loss": 3.660937916884853, + "tokens_seen": 966898688 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003571013039117352, + "loss": 2.7275, + "theoretical_loss": 3.660914156351592, + "tokens_seen": 966964224 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003570912738214644, + "loss": 2.7555, + "theoretical_loss": 3.6608903978795153, + "tokens_seen": 967029760 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003570812437311936, + "loss": 3.0237, + "theoretical_loss": 3.6608666414683038, + "tokens_seen": 967095296 + }, + { + "epoch": 3.02, + "learning_rate": 0.00035707121364092275, + "loss": 2.7074, + "theoretical_loss": 3.6608428871176404, + "tokens_seen": 967160832 + }, + { + "epoch": 3.02, + "learning_rate": 0.000357061183550652, + "loss": 2.7985, + "theoretical_loss": 3.6608191348272063, + "tokens_seen": 967226368 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003570511534603811, + "loss": 2.5992, + "theoretical_loss": 3.6607953845966827, + "tokens_seen": 967291904 + }, + { + "epoch": 3.02, + "learning_rate": 0.00035704112337011035, + "loss": 2.8545, + "theoretical_loss": 3.6607716364257525, + "tokens_seen": 967357440 + }, + { + "epoch": 3.02, + "learning_rate": 0.00035703109327983953, + "loss": 2.8628, + "theoretical_loss": 3.660747890314096, + "tokens_seen": 967422976 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003570210631895687, + "loss": 2.6079, + "theoretical_loss": 3.6607241462613964, + "tokens_seen": 967488512 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003570110330992979, + "loss": 2.7346, + "theoretical_loss": 3.6607004042673355, + "tokens_seen": 967554048 + }, + { + "epoch": 3.02, + "learning_rate": 0.00035700100300902713, + "loss": 2.6818, + "theoretical_loss": 3.660676664331595, + "tokens_seen": 967619584 + }, + { + "epoch": 3.02, + "learning_rate": 0.00035699097291875626, + "loss": 2.7036, + "theoretical_loss": 3.6606529264538574, + "tokens_seen": 967685120 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003569809428284855, + "loss": 2.7349, + "theoretical_loss": 3.660629190633805, + "tokens_seen": 967750656 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003569709127382146, + "loss": 3.0038, + "theoretical_loss": 3.66060545687112, + "tokens_seen": 967816192 + }, + { + "epoch": 3.02, + "learning_rate": 0.00035696088264794385, + "loss": 2.7256, + "theoretical_loss": 3.6605817251654846, + "tokens_seen": 967881728 + }, + { + "epoch": 3.02, + "learning_rate": 0.00035695085255767303, + "loss": 2.7116, + "theoretical_loss": 3.6605579955165815, + "tokens_seen": 967947264 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003569408224674022, + "loss": 2.833, + "theoretical_loss": 3.660534267924093, + "tokens_seen": 968012800 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003569307923771314, + "loss": 2.8843, + "theoretical_loss": 3.6605105423877013, + "tokens_seen": 968078336 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003569207622868606, + "loss": 2.5918, + "theoretical_loss": 3.6604868189070903, + "tokens_seen": 968143872 + }, + { + "epoch": 3.02, + "learning_rate": 0.00035691073219658976, + "loss": 2.6069, + "theoretical_loss": 3.6604630974819417, + "tokens_seen": 968209408 + }, + { + "epoch": 3.02, + "learning_rate": 0.000356900702106319, + "loss": 3.0027, + "theoretical_loss": 3.6604393781119384, + "tokens_seen": 968274944 + }, + { + "epoch": 3.02, + "objective/train/docs_used": 1126260, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6405274868011475, + "objective/train/theoretical_loss": 3.660433448590517, + "objective/train/tokens_used": 988751328, + "theoretical_loss": 3.660433448590517, + "tokens_seen": 968291328 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003568906720160481, + "loss": 2.7458, + "theoretical_loss": 3.6604156607967635, + "tokens_seen": 968340480 + }, + { + "epoch": 3.02, + "learning_rate": 0.00035688064192577736, + "loss": 2.8098, + "theoretical_loss": 3.6603919455361, + "tokens_seen": 968406016 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003568706118355065, + "loss": 2.9405, + "theoretical_loss": 3.6603682323296307, + "tokens_seen": 968471552 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003568605817452357, + "loss": 2.5768, + "theoretical_loss": 3.6603445211770387, + "tokens_seen": 968537088 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003568505516549649, + "loss": 2.9667, + "theoretical_loss": 3.660320812078007, + "tokens_seen": 968602624 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003568405215646941, + "loss": 2.7004, + "theoretical_loss": 3.6602971050322197, + "tokens_seen": 968668160 + }, + { + "epoch": 3.02, + "learning_rate": 0.00035683049147442326, + "loss": 2.6829, + "theoretical_loss": 3.660273400039359, + "tokens_seen": 968733696 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003568204613841525, + "loss": 2.6353, + "theoretical_loss": 3.6602496970991085, + "tokens_seen": 968799232 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003568104312938816, + "loss": 2.5665, + "theoretical_loss": 3.660225996211152, + "tokens_seen": 968864768 + }, + { + "epoch": 3.02, + "learning_rate": 0.00035680040120361086, + "loss": 2.9588, + "theoretical_loss": 3.6602022973751724, + "tokens_seen": 968930304 + }, + { + "epoch": 3.02, + "learning_rate": 0.00035679037111334, + "loss": 2.7872, + "theoretical_loss": 3.6601786005908545, + "tokens_seen": 968995840 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003567803410230692, + "loss": 2.8798, + "theoretical_loss": 3.6601549058578806, + "tokens_seen": 969061376 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003567703109327984, + "loss": 2.9378, + "theoretical_loss": 3.660131213175935, + "tokens_seen": 969126912 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003567602808425276, + "loss": 2.8404, + "theoretical_loss": 3.6601075225447017, + "tokens_seen": 969192448 + }, + { + "epoch": 3.02, + "learning_rate": 0.00035675025075225677, + "loss": 2.7157, + "theoretical_loss": 3.6600838339638635, + "tokens_seen": 969257984 + }, + { + "epoch": 3.02, + "learning_rate": 0.00035674022066198595, + "loss": 2.8261, + "theoretical_loss": 3.660060147433106, + "tokens_seen": 969323520 + }, + { + "epoch": 3.02, + "learning_rate": 0.00035673019057171513, + "loss": 2.8776, + "theoretical_loss": 3.660036462952112, + "tokens_seen": 969389056 + }, + { + "epoch": 3.02, + "learning_rate": 0.00035672016048144436, + "loss": 2.857, + "theoretical_loss": 3.6600127805205656, + "tokens_seen": 969454592 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003567101303911735, + "loss": 2.8433, + "theoretical_loss": 3.659989100138151, + "tokens_seen": 969520128 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003567001003009027, + "loss": 2.7035, + "theoretical_loss": 3.6599654218045528, + "tokens_seen": 969585664 + }, + { + "epoch": 3.02, + "learning_rate": 0.00035669007021063185, + "loss": 2.7555, + "theoretical_loss": 3.659941745519455, + "tokens_seen": 969651200 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003566800401203611, + "loss": 2.4502, + "theoretical_loss": 3.659918071282542, + "tokens_seen": 969716736 + }, + { + "epoch": 3.02, + "learning_rate": 0.00035667001003009027, + "loss": 2.8731, + "theoretical_loss": 3.6598943990934987, + "tokens_seen": 969782272 + }, + { + "epoch": 3.02, + "learning_rate": 0.00035665997993981945, + "loss": 2.4678, + "theoretical_loss": 3.6598707289520087, + "tokens_seen": 969847808 + }, + { + "epoch": 3.02, + "learning_rate": 0.00035664994984954863, + "loss": 2.9327, + "theoretical_loss": 3.659847060857757, + "tokens_seen": 969913344 + }, + { + "epoch": 3.02, + "objective/train/docs_used": 1127813, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4290738105773926, + "objective/train/theoretical_loss": 3.659841144154038, + "objective/train/tokens_used": 990389728, + "theoretical_loss": 3.659841144154038, + "tokens_seen": 969929728 + }, + { + "epoch": 3.02, + "learning_rate": 0.00035663991975927787, + "loss": 2.5046, + "theoretical_loss": 3.6598233948104277, + "tokens_seen": 969978880 + }, + { + "epoch": 3.02, + "learning_rate": 0.000356629889669007, + "loss": 2.6093, + "theoretical_loss": 3.6597997308097066, + "tokens_seen": 970044416 + }, + { + "epoch": 3.02, + "learning_rate": 0.00035661985957873623, + "loss": 2.8465, + "theoretical_loss": 3.6597760688552774, + "tokens_seen": 970109952 + }, + { + "epoch": 3.02, + "learning_rate": 0.00035660982948846536, + "loss": 2.9324, + "theoretical_loss": 3.6597524089468254, + "tokens_seen": 970175488 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003565997993981946, + "loss": 2.6557, + "theoretical_loss": 3.6597287510840353, + "tokens_seen": 970241024 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003565897693079238, + "loss": 2.8453, + "theoretical_loss": 3.6597050952665926, + "tokens_seen": 970306560 + }, + { + "epoch": 3.02, + "learning_rate": 0.00035657973921765295, + "loss": 2.8741, + "theoretical_loss": 3.6596814414941816, + "tokens_seen": 970372096 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003565697091273822, + "loss": 2.7244, + "theoretical_loss": 3.6596577897664884, + "tokens_seen": 970437632 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003565596790371113, + "loss": 2.7886, + "theoretical_loss": 3.659634140083197, + "tokens_seen": 970503168 + }, + { + "epoch": 3.02, + "learning_rate": 0.00035654964894684055, + "loss": 2.6121, + "theoretical_loss": 3.659610492443993, + "tokens_seen": 970568704 + }, + { + "epoch": 3.02, + "learning_rate": 0.00035653961885656973, + "loss": 2.6921, + "theoretical_loss": 3.659586846848563, + "tokens_seen": 970634240 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003565295887662989, + "loss": 2.7066, + "theoretical_loss": 3.6595632032965897, + "tokens_seen": 970699776 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003565195586760281, + "loss": 2.7141, + "theoretical_loss": 3.6595395617877613, + "tokens_seen": 970765312 + }, + { + "epoch": 3.02, + "learning_rate": 0.00035650952858575733, + "loss": 2.8058, + "theoretical_loss": 3.659515922321762, + "tokens_seen": 970830848 + }, + { + "epoch": 3.02, + "learning_rate": 0.00035649949849548646, + "loss": 2.8712, + "theoretical_loss": 3.6594922848982776, + "tokens_seen": 970896384 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003564894684052157, + "loss": 2.8611, + "theoretical_loss": 3.6594686495169935, + "tokens_seen": 970961920 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003564794383149448, + "loss": 3.0298, + "theoretical_loss": 3.659445016177596, + "tokens_seen": 971027456 + }, + { + "epoch": 3.02, + "learning_rate": 0.00035646940822467405, + "loss": 2.7334, + "theoretical_loss": 3.6594213848797703, + "tokens_seen": 971092992 + }, + { + "epoch": 3.02, + "learning_rate": 0.00035645937813440323, + "loss": 2.6687, + "theoretical_loss": 3.6593977556232025, + "tokens_seen": 971158528 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003564493480441324, + "loss": 2.9812, + "theoretical_loss": 3.6593741284075794, + "tokens_seen": 971224064 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003564393179538616, + "loss": 2.7254, + "theoretical_loss": 3.6593505032325853, + "tokens_seen": 971289600 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003564292878635908, + "loss": 2.6469, + "theoretical_loss": 3.659326880097908, + "tokens_seen": 971355136 + }, + { + "epoch": 3.02, + "learning_rate": 0.00035641925777331996, + "loss": 2.77, + "theoretical_loss": 3.6593032590032326, + "tokens_seen": 971420672 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003564092276830492, + "loss": 2.8566, + "theoretical_loss": 3.6592796399482452, + "tokens_seen": 971486208 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003563991975927783, + "loss": 2.9147, + "theoretical_loss": 3.659256022932633, + "tokens_seen": 971551744 + }, + { + "epoch": 3.02, + "objective/train/docs_used": 1128556, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.751553773880005, + "objective/train/theoretical_loss": 3.6592501189973454, + "objective/train/tokens_used": 992028128, + "theoretical_loss": 3.6592501189973454, + "tokens_seen": 971568128 + }, + { + "epoch": 3.02, + "learning_rate": 0.00035638916750250756, + "loss": 3.0277, + "theoretical_loss": 3.6592324079560816, + "tokens_seen": 971617280 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003563791374122367, + "loss": 2.8446, + "theoretical_loss": 3.6592087950182783, + "tokens_seen": 971682816 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003563691073219659, + "loss": 2.7536, + "theoretical_loss": 3.6591851841189085, + "tokens_seen": 971748352 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003563590772316951, + "loss": 2.8624, + "theoretical_loss": 3.6591615752576594, + "tokens_seen": 971813888 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003563490471414243, + "loss": 2.779, + "theoretical_loss": 3.659137968434217, + "tokens_seen": 971879424 + }, + { + "epoch": 3.02, + "learning_rate": 0.00035633901705115346, + "loss": 2.836, + "theoretical_loss": 3.6591143636482695, + "tokens_seen": 971944960 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003563289869608827, + "loss": 2.9205, + "theoretical_loss": 3.659090760899502, + "tokens_seen": 972010496 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003563189568706118, + "loss": 2.655, + "theoretical_loss": 3.659067160187602, + "tokens_seen": 972076032 + }, + { + "epoch": 3.02, + "learning_rate": 0.00035630892678034106, + "loss": 2.7509, + "theoretical_loss": 3.6590435615122567, + "tokens_seen": 972141568 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003562988966900702, + "loss": 2.7927, + "theoretical_loss": 3.6590199648731527, + "tokens_seen": 972207104 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003562888665997994, + "loss": 2.8891, + "theoretical_loss": 3.6589963702699775, + "tokens_seen": 972272640 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003562788365095286, + "loss": 2.9476, + "theoretical_loss": 3.658972777702418, + "tokens_seen": 972338176 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003562688064192578, + "loss": 2.8627, + "theoretical_loss": 3.658949187170161, + "tokens_seen": 972403712 + }, + { + "epoch": 3.02, + "learning_rate": 0.00035625877632898697, + "loss": 2.7483, + "theoretical_loss": 3.6589255986728944, + "tokens_seen": 972469248 + }, + { + "epoch": 3.02, + "learning_rate": 0.00035624874623871615, + "loss": 2.5822, + "theoretical_loss": 3.658902012210305, + "tokens_seen": 972534784 + }, + { + "epoch": 3.02, + "learning_rate": 0.00035623871614844533, + "loss": 2.962, + "theoretical_loss": 3.6588784277820805, + "tokens_seen": 972600320 + }, + { + "epoch": 3.02, + "learning_rate": 0.00035622868605817456, + "loss": 2.7324, + "theoretical_loss": 3.658854845387908, + "tokens_seen": 972665856 + }, + { + "epoch": 3.02, + "learning_rate": 0.0003562186559679037, + "loss": 2.7687, + "theoretical_loss": 3.6588312650274757, + "tokens_seen": 972731392 + }, + { + "epoch": 3.03, + "learning_rate": 0.0003562086258776329, + "loss": 2.4566, + "theoretical_loss": 3.658807686700471, + "tokens_seen": 972796928 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035619859578736205, + "loss": 2.6661, + "theoretical_loss": 3.6587841104065815, + "tokens_seen": 972862464 + }, + { + "epoch": 3.03, + "learning_rate": 0.0003561885656970913, + "loss": 2.6573, + "theoretical_loss": 3.6587605361454942, + "tokens_seen": 972928000 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035617853560682047, + "loss": 2.6695, + "theoretical_loss": 3.658736963916898, + "tokens_seen": 972993536 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035616850551654965, + "loss": 2.8766, + "theoretical_loss": 3.6587133937204808, + "tokens_seen": 973059072 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035615847542627883, + "loss": 2.6962, + "theoretical_loss": 3.6586898255559293, + "tokens_seen": 973124608 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035614844533600807, + "loss": 2.7565, + "theoretical_loss": 3.6586662594229327, + "tokens_seen": 973190144 + }, + { + "epoch": 3.03, + "objective/train/docs_used": 1129663, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.72713303565979, + "objective/train/theoretical_loss": 3.6586603682070775, + "objective/train/tokens_used": 993666528, + "theoretical_loss": 3.6586603682070775, + "tokens_seen": 973206528 + }, + { + "epoch": 3.03, + "learning_rate": 0.0003561384152457372, + "loss": 2.8701, + "theoretical_loss": 3.658642695321179, + "tokens_seen": 973255680 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035612838515546643, + "loss": 2.7955, + "theoretical_loss": 3.6586191332503555, + "tokens_seen": 973321216 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035611835506519556, + "loss": 2.8356, + "theoretical_loss": 3.6585955732101514, + "tokens_seen": 973386752 + }, + { + "epoch": 3.03, + "learning_rate": 0.0003561083249749248, + "loss": 2.7941, + "theoretical_loss": 3.6585720152002548, + "tokens_seen": 973452288 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035609829488465397, + "loss": 2.9232, + "theoretical_loss": 3.658548459220354, + "tokens_seen": 973517824 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035608826479438315, + "loss": 2.6519, + "theoretical_loss": 3.658524905270137, + "tokens_seen": 973583360 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035607823470411233, + "loss": 2.7978, + "theoretical_loss": 3.6585013533492927, + "tokens_seen": 973648896 + }, + { + "epoch": 3.03, + "learning_rate": 0.0003560682046138415, + "loss": 2.7026, + "theoretical_loss": 3.65847780345751, + "tokens_seen": 973714432 + }, + { + "epoch": 3.03, + "learning_rate": 0.0003560581745235707, + "loss": 2.689, + "theoretical_loss": 3.6584542555944766, + "tokens_seen": 973779968 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035604814443329993, + "loss": 3.099, + "theoretical_loss": 3.658430709759882, + "tokens_seen": 973845504 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035603811434302906, + "loss": 2.7942, + "theoretical_loss": 3.658407165953415, + "tokens_seen": 973911040 + }, + { + "epoch": 3.03, + "learning_rate": 0.0003560280842527583, + "loss": 2.629, + "theoretical_loss": 3.6583836241747636, + "tokens_seen": 973976576 + }, + { + "epoch": 3.03, + "learning_rate": 0.0003560180541624874, + "loss": 2.7733, + "theoretical_loss": 3.6583600844236175, + "tokens_seen": 974042112 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035600802407221666, + "loss": 2.8486, + "theoretical_loss": 3.658336546699666, + "tokens_seen": 974107648 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035599799398194584, + "loss": 2.648, + "theoretical_loss": 3.658313011002597, + "tokens_seen": 974173184 + }, + { + "epoch": 3.03, + "learning_rate": 0.000355987963891675, + "loss": 2.8866, + "theoretical_loss": 3.658289477332101, + "tokens_seen": 974238720 + }, + { + "epoch": 3.03, + "learning_rate": 0.0003559779338014042, + "loss": 2.6934, + "theoretical_loss": 3.658265945687866, + "tokens_seen": 974304256 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035596790371113344, + "loss": 2.653, + "theoretical_loss": 3.6582424160695814, + "tokens_seen": 974369792 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035595787362086256, + "loss": 2.7027, + "theoretical_loss": 3.658218888476937, + "tokens_seen": 974435328 + }, + { + "epoch": 3.03, + "learning_rate": 0.0003559478435305918, + "loss": 2.7906, + "theoretical_loss": 3.6581953629096224, + "tokens_seen": 974500864 + }, + { + "epoch": 3.03, + "learning_rate": 0.0003559378134403209, + "loss": 2.6374, + "theoretical_loss": 3.658171839367327, + "tokens_seen": 974566400 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035592778335005016, + "loss": 2.64, + "theoretical_loss": 3.6581483178497396, + "tokens_seen": 974631936 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035591775325977934, + "loss": 2.6977, + "theoretical_loss": 3.6581247983565506, + "tokens_seen": 974697472 + }, + { + "epoch": 3.03, + "learning_rate": 0.0003559077231695085, + "loss": 2.9227, + "theoretical_loss": 3.6581012808874487, + "tokens_seen": 974763008 + }, + { + "epoch": 3.03, + "learning_rate": 0.0003558976930792377, + "loss": 2.7544, + "theoretical_loss": 3.6580777654421244, + "tokens_seen": 974828544 + }, + { + "epoch": 3.03, + "objective/train/docs_used": 1130250, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.891390562057495, + "objective/train/theoretical_loss": 3.6580718868969724, + "objective/train/tokens_used": 995304928, + "theoretical_loss": 3.6580718868969724, + "tokens_seen": 974844928 + }, + { + "epoch": 3.03, + "learning_rate": 0.0003558876629889669, + "loss": 2.7537, + "theoretical_loss": 3.6580542520202677, + "tokens_seen": 974894080 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035587763289869607, + "loss": 2.8645, + "theoretical_loss": 3.658030740621568, + "tokens_seen": 974959616 + }, + { + "epoch": 3.03, + "learning_rate": 0.0003558676028084253, + "loss": 2.6031, + "theoretical_loss": 3.6580072312457155, + "tokens_seen": 975025152 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035585757271815443, + "loss": 2.9164, + "theoretical_loss": 3.6579837238924, + "tokens_seen": 975090688 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035584754262788366, + "loss": 2.5411, + "theoretical_loss": 3.657960218561312, + "tokens_seen": 975156224 + }, + { + "epoch": 3.03, + "learning_rate": 0.0003558375125376129, + "loss": 2.7479, + "theoretical_loss": 3.6579367152521414, + "tokens_seen": 975221760 + }, + { + "epoch": 3.03, + "learning_rate": 0.000355827482447342, + "loss": 2.8431, + "theoretical_loss": 3.6579132139645774, + "tokens_seen": 975287296 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035581745235707126, + "loss": 2.8175, + "theoretical_loss": 3.657889714698312, + "tokens_seen": 975352832 + }, + { + "epoch": 3.03, + "learning_rate": 0.0003558074222668004, + "loss": 2.8773, + "theoretical_loss": 3.6578662174530345, + "tokens_seen": 975418368 + }, + { + "epoch": 3.03, + "learning_rate": 0.0003557973921765296, + "loss": 2.6768, + "theoretical_loss": 3.657842722228436, + "tokens_seen": 975483904 + }, + { + "epoch": 3.03, + "learning_rate": 0.0003557873620862588, + "loss": 2.6195, + "theoretical_loss": 3.6578192290242066, + "tokens_seen": 975549440 + }, + { + "epoch": 3.03, + "learning_rate": 0.000355777331995988, + "loss": 2.7406, + "theoretical_loss": 3.6577957378400368, + "tokens_seen": 975614976 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035576730190571717, + "loss": 2.7412, + "theoretical_loss": 3.657772248675617, + "tokens_seen": 975680512 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035575727181544635, + "loss": 2.6126, + "theoretical_loss": 3.6577487615306383, + "tokens_seen": 975746048 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035574724172517553, + "loss": 2.6626, + "theoretical_loss": 3.6577252764047916, + "tokens_seen": 975811584 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035573721163490476, + "loss": 3.0388, + "theoretical_loss": 3.6577017932977673, + "tokens_seen": 975877120 + }, + { + "epoch": 3.03, + "learning_rate": 0.0003557271815446339, + "loss": 2.8096, + "theoretical_loss": 3.6576783122092564, + "tokens_seen": 975942656 + }, + { + "epoch": 3.03, + "learning_rate": 0.0003557171514543631, + "loss": 2.8943, + "theoretical_loss": 3.6576548331389507, + "tokens_seen": 976008192 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035570712136409225, + "loss": 2.8692, + "theoretical_loss": 3.6576313560865397, + "tokens_seen": 976073728 + }, + { + "epoch": 3.03, + "learning_rate": 0.0003556970912738215, + "loss": 2.913, + "theoretical_loss": 3.6576078810517156, + "tokens_seen": 976139264 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035568706118355067, + "loss": 2.7489, + "theoretical_loss": 3.657584408034169, + "tokens_seen": 976204800 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035567703109327985, + "loss": 2.5959, + "theoretical_loss": 3.6575609370335913, + "tokens_seen": 976270336 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035566700100300903, + "loss": 2.962, + "theoretical_loss": 3.657537468049674, + "tokens_seen": 976335872 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035565697091273827, + "loss": 2.7383, + "theoretical_loss": 3.657514001082109, + "tokens_seen": 976401408 + }, + { + "epoch": 3.03, + "learning_rate": 0.0003556469408224674, + "loss": 3.0656, + "theoretical_loss": 3.6574905361305867, + "tokens_seen": 976466944 + }, + { + "epoch": 3.03, + "objective/train/docs_used": 1131717, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6520185470581055, + "objective/train/theoretical_loss": 3.657484670207677, + "objective/train/tokens_used": 996943328, + "theoretical_loss": 3.657484670207677, + "tokens_seen": 976483328 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035563691073219663, + "loss": 2.8189, + "theoretical_loss": 3.657467073194799, + "tokens_seen": 976532480 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035562688064192576, + "loss": 2.6232, + "theoretical_loss": 3.657443612274438, + "tokens_seen": 976598016 + }, + { + "epoch": 3.03, + "learning_rate": 0.000355616850551655, + "loss": 2.6579, + "theoretical_loss": 3.657420153369194, + "tokens_seen": 976663552 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035560682046138417, + "loss": 2.7898, + "theoretical_loss": 3.6573966964787603, + "tokens_seen": 976729088 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035559679037111335, + "loss": 2.707, + "theoretical_loss": 3.6573732416028277, + "tokens_seen": 976794624 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035558676028084253, + "loss": 3.012, + "theoretical_loss": 3.6573497887410884, + "tokens_seen": 976860160 + }, + { + "epoch": 3.03, + "learning_rate": 0.0003555767301905717, + "loss": 2.7479, + "theoretical_loss": 3.6573263378932346, + "tokens_seen": 976925696 + }, + { + "epoch": 3.03, + "learning_rate": 0.0003555667001003009, + "loss": 2.8411, + "theoretical_loss": 3.6573028890589576, + "tokens_seen": 976991232 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035555667001003013, + "loss": 2.8103, + "theoretical_loss": 3.6572794422379493, + "tokens_seen": 977056768 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035554663991975926, + "loss": 2.8471, + "theoretical_loss": 3.6572559974299033, + "tokens_seen": 977122304 + }, + { + "epoch": 3.03, + "learning_rate": 0.0003555366098294885, + "loss": 2.6952, + "theoretical_loss": 3.6572325546345104, + "tokens_seen": 977187840 + }, + { + "epoch": 3.03, + "learning_rate": 0.0003555265797392176, + "loss": 2.4342, + "theoretical_loss": 3.657209113851463, + "tokens_seen": 977253376 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035551654964894686, + "loss": 2.7549, + "theoretical_loss": 3.657185675080454, + "tokens_seen": 977318912 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035550651955867604, + "loss": 2.7508, + "theoretical_loss": 3.657162238321175, + "tokens_seen": 977384448 + }, + { + "epoch": 3.03, + "learning_rate": 0.0003554964894684052, + "loss": 2.8389, + "theoretical_loss": 3.6571388035733197, + "tokens_seen": 977449984 + }, + { + "epoch": 3.03, + "learning_rate": 0.0003554864593781344, + "loss": 3.0033, + "theoretical_loss": 3.657115370836579, + "tokens_seen": 977515520 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035547642928786364, + "loss": 2.782, + "theoretical_loss": 3.657091940110647, + "tokens_seen": 977581056 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035546639919759276, + "loss": 2.7965, + "theoretical_loss": 3.6570685113952153, + "tokens_seen": 977646592 + }, + { + "epoch": 3.03, + "learning_rate": 0.000355456369107322, + "loss": 2.6375, + "theoretical_loss": 3.6570450846899774, + "tokens_seen": 977712128 + }, + { + "epoch": 3.03, + "learning_rate": 0.0003554463390170511, + "loss": 2.6154, + "theoretical_loss": 3.6570216599946255, + "tokens_seen": 977777664 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035543630892678036, + "loss": 2.8124, + "theoretical_loss": 3.656998237308853, + "tokens_seen": 977843200 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035542627883650954, + "loss": 2.8307, + "theoretical_loss": 3.6569748166323524, + "tokens_seen": 977908736 + }, + { + "epoch": 3.03, + "learning_rate": 0.0003554162487462387, + "loss": 2.6667, + "theoretical_loss": 3.656951397964817, + "tokens_seen": 977974272 + }, + { + "epoch": 3.03, + "learning_rate": 0.0003554062186559679, + "loss": 2.8426, + "theoretical_loss": 3.6569279813059397, + "tokens_seen": 978039808 + }, + { + "epoch": 3.03, + "learning_rate": 0.0003553961885656971, + "loss": 2.6248, + "theoretical_loss": 3.656904566655413, + "tokens_seen": 978105344 + }, + { + "epoch": 3.03, + "objective/train/docs_used": 1132115, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.2902779579162598, + "objective/train/theoretical_loss": 3.6568987133065507, + "objective/train/tokens_used": 998581728, + "theoretical_loss": 3.6568987133065507, + "tokens_seen": 978121728 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035538615847542627, + "loss": 2.984, + "theoretical_loss": 3.6568811540129316, + "tokens_seen": 978170880 + }, + { + "epoch": 3.03, + "learning_rate": 0.0003553761283851555, + "loss": 2.8201, + "theoretical_loss": 3.656857743378188, + "tokens_seen": 978236416 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035536609829488463, + "loss": 2.8302, + "theoretical_loss": 3.656834334750875, + "tokens_seen": 978301952 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035535606820461386, + "loss": 2.6622, + "theoretical_loss": 3.6568109281306866, + "tokens_seen": 978367488 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035534603811434304, + "loss": 2.8905, + "theoretical_loss": 3.656787523517316, + "tokens_seen": 978433024 + }, + { + "epoch": 3.03, + "learning_rate": 0.0003553360080240722, + "loss": 2.8052, + "theoretical_loss": 3.656764120910457, + "tokens_seen": 978498560 + }, + { + "epoch": 3.03, + "learning_rate": 0.0003553259779338014, + "loss": 2.9525, + "theoretical_loss": 3.6567407203098035, + "tokens_seen": 978564096 + }, + { + "epoch": 3.03, + "learning_rate": 0.0003553159478435306, + "loss": 2.6697, + "theoretical_loss": 3.6567173217150484, + "tokens_seen": 978629632 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035530591775325977, + "loss": 2.8138, + "theoretical_loss": 3.656693925125886, + "tokens_seen": 978695168 + }, + { + "epoch": 3.03, + "learning_rate": 0.000355295887662989, + "loss": 2.7755, + "theoretical_loss": 3.6566705305420095, + "tokens_seen": 978760704 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035528585757271813, + "loss": 3.0818, + "theoretical_loss": 3.656647137963114, + "tokens_seen": 978826240 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035527582748244737, + "loss": 2.766, + "theoretical_loss": 3.656623747388892, + "tokens_seen": 978891776 + }, + { + "epoch": 3.03, + "learning_rate": 0.0003552657973921765, + "loss": 3.0058, + "theoretical_loss": 3.6566003588190386, + "tokens_seen": 978957312 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035525576730190573, + "loss": 2.8212, + "theoretical_loss": 3.656576972253247, + "tokens_seen": 979022848 + }, + { + "epoch": 3.03, + "learning_rate": 0.0003552457372116349, + "loss": 2.9931, + "theoretical_loss": 3.656553587691212, + "tokens_seen": 979088384 + }, + { + "epoch": 3.03, + "learning_rate": 0.0003552357071213641, + "loss": 2.9403, + "theoretical_loss": 3.6565302051326274, + "tokens_seen": 979153920 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035522567703109327, + "loss": 2.7488, + "theoretical_loss": 3.656506824577188, + "tokens_seen": 979219456 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035521564694082245, + "loss": 2.9203, + "theoretical_loss": 3.6564834460245876, + "tokens_seen": 979284992 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035520561685055163, + "loss": 2.9973, + "theoretical_loss": 3.6564600694745213, + "tokens_seen": 979350528 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035519558676028087, + "loss": 2.8523, + "theoretical_loss": 3.6564366949266827, + "tokens_seen": 979416064 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035518555667001, + "loss": 2.8387, + "theoretical_loss": 3.6564133223807667, + "tokens_seen": 979481600 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035517552657973923, + "loss": 2.8515, + "theoretical_loss": 3.656389951836468, + "tokens_seen": 979547136 + }, + { + "epoch": 3.03, + "learning_rate": 0.0003551654964894684, + "loss": 2.7526, + "theoretical_loss": 3.6563665832934813, + "tokens_seen": 979612672 + }, + { + "epoch": 3.03, + "learning_rate": 0.0003551554663991976, + "loss": 2.8189, + "theoretical_loss": 3.6563432167515018, + "tokens_seen": 979678208 + }, + { + "epoch": 3.03, + "learning_rate": 0.0003551454363089268, + "loss": 2.7372, + "theoretical_loss": 3.656319852210223, + "tokens_seen": 979743744 + }, + { + "epoch": 3.03, + "objective/train/docs_used": 1133253, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4291090965270996, + "objective/train/theoretical_loss": 3.6563140113874777, + "objective/train/tokens_used": 1000220128, + "theoretical_loss": 3.6563140113874777, + "tokens_seen": 979760128 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035513540621865596, + "loss": 2.6382, + "theoretical_loss": 3.6562964896693417, + "tokens_seen": 979809280 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035512537612838514, + "loss": 2.7368, + "theoretical_loss": 3.656273129128551, + "tokens_seen": 979874816 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035511534603811437, + "loss": 2.7358, + "theoretical_loss": 3.6562497705875465, + "tokens_seen": 979940352 + }, + { + "epoch": 3.03, + "learning_rate": 0.0003551053159478435, + "loss": 2.9211, + "theoretical_loss": 3.6562264140460243, + "tokens_seen": 980005888 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035509528585757273, + "loss": 2.8374, + "theoretical_loss": 3.656203059503678, + "tokens_seen": 980071424 + }, + { + "epoch": 3.03, + "learning_rate": 0.0003550852557673019, + "loss": 2.5918, + "theoretical_loss": 3.6561797069602036, + "tokens_seen": 980136960 + }, + { + "epoch": 3.03, + "learning_rate": 0.0003550752256770311, + "loss": 2.7636, + "theoretical_loss": 3.6561563564152966, + "tokens_seen": 980202496 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035506519558676033, + "loss": 2.8558, + "theoretical_loss": 3.656133007868652, + "tokens_seen": 980268032 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035505516549648946, + "loss": 2.6737, + "theoretical_loss": 3.6561096613199653, + "tokens_seen": 980333568 + }, + { + "epoch": 3.03, + "learning_rate": 0.0003550451354062187, + "loss": 2.7461, + "theoretical_loss": 3.656086316768932, + "tokens_seen": 980399104 + }, + { + "epoch": 3.03, + "learning_rate": 0.0003550351053159478, + "loss": 2.9711, + "theoretical_loss": 3.6560629742152475, + "tokens_seen": 980464640 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035502507522567706, + "loss": 2.765, + "theoretical_loss": 3.656039633658607, + "tokens_seen": 980530176 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035501504513540624, + "loss": 2.7839, + "theoretical_loss": 3.6560162950987083, + "tokens_seen": 980595712 + }, + { + "epoch": 3.03, + "learning_rate": 0.0003550050150451354, + "loss": 2.7088, + "theoretical_loss": 3.6559929585352444, + "tokens_seen": 980661248 + }, + { + "epoch": 3.03, + "learning_rate": 0.0003549949849548646, + "loss": 2.9667, + "theoretical_loss": 3.6559696239679127, + "tokens_seen": 980726784 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035498495486459384, + "loss": 2.9507, + "theoretical_loss": 3.6559462913964085, + "tokens_seen": 980792320 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035497492477432296, + "loss": 2.906, + "theoretical_loss": 3.655922960820428, + "tokens_seen": 980857856 + }, + { + "epoch": 3.03, + "learning_rate": 0.0003549648946840522, + "loss": 2.8641, + "theoretical_loss": 3.6558996322396666, + "tokens_seen": 980923392 + }, + { + "epoch": 3.03, + "learning_rate": 0.0003549548645937813, + "loss": 2.6324, + "theoretical_loss": 3.655876305653822, + "tokens_seen": 980988928 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035494483450351056, + "loss": 2.8169, + "theoretical_loss": 3.655852981062589, + "tokens_seen": 981054464 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035493480441323974, + "loss": 2.8943, + "theoretical_loss": 3.6558296584656635, + "tokens_seen": 981120000 + }, + { + "epoch": 3.03, + "learning_rate": 0.0003549247743229689, + "loss": 2.7909, + "theoretical_loss": 3.655806337862743, + "tokens_seen": 981185536 + }, + { + "epoch": 3.03, + "learning_rate": 0.0003549147442326981, + "loss": 2.7996, + "theoretical_loss": 3.655783019253523, + "tokens_seen": 981251072 + }, + { + "epoch": 3.03, + "learning_rate": 0.0003549047141424273, + "loss": 2.625, + "theoretical_loss": 3.6557597026377, + "tokens_seen": 981316608 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035489468405215647, + "loss": 2.9213, + "theoretical_loss": 3.655736388014971, + "tokens_seen": 981382144 + }, + { + "epoch": 3.03, + "objective/train/docs_used": 1133558, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7345311641693115, + "objective/train/theoretical_loss": 3.655730559670674, + "objective/train/tokens_used": 1001858528, + "theoretical_loss": 3.655730559670674, + "tokens_seen": 981398528 + }, + { + "epoch": 3.03, + "learning_rate": 0.0003548846539618857, + "loss": 2.7113, + "theoretical_loss": 3.655713075385032, + "tokens_seen": 981447680 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035487462387161483, + "loss": 2.9233, + "theoretical_loss": 3.6556897647475797, + "tokens_seen": 981513216 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035486459378134406, + "loss": 2.635, + "theoretical_loss": 3.6556664561023116, + "tokens_seen": 981578752 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035485456369107324, + "loss": 2.9342, + "theoretical_loss": 3.655643149448923, + "tokens_seen": 981644288 + }, + { + "epoch": 3.03, + "learning_rate": 0.0003548445336008024, + "loss": 2.739, + "theoretical_loss": 3.655619844787112, + "tokens_seen": 981709824 + }, + { + "epoch": 3.03, + "learning_rate": 0.0003548345035105316, + "loss": 2.8912, + "theoretical_loss": 3.6555965421165744, + "tokens_seen": 981775360 + }, + { + "epoch": 3.03, + "learning_rate": 0.0003548244734202608, + "loss": 3.0486, + "theoretical_loss": 3.6555732414370077, + "tokens_seen": 981840896 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035481444332998997, + "loss": 3.1694, + "theoretical_loss": 3.6555499427481095, + "tokens_seen": 981906432 + }, + { + "epoch": 3.03, + "learning_rate": 0.0003548044132397192, + "loss": 3.1176, + "theoretical_loss": 3.655526646049576, + "tokens_seen": 981971968 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035479438314944833, + "loss": 2.8801, + "theoretical_loss": 3.655503351341105, + "tokens_seen": 982037504 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035478435305917757, + "loss": 2.9109, + "theoretical_loss": 3.655480058622393, + "tokens_seen": 982103040 + }, + { + "epoch": 3.03, + "learning_rate": 0.0003547743229689067, + "loss": 2.9983, + "theoretical_loss": 3.6554567678931376, + "tokens_seen": 982168576 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035476429287863593, + "loss": 2.7778, + "theoretical_loss": 3.6554334791530363, + "tokens_seen": 982234112 + }, + { + "epoch": 3.03, + "learning_rate": 0.0003547542627883651, + "loss": 2.7951, + "theoretical_loss": 3.6554101924017863, + "tokens_seen": 982299648 + }, + { + "epoch": 3.03, + "learning_rate": 0.0003547442326980943, + "loss": 2.9124, + "theoretical_loss": 3.6553869076390857, + "tokens_seen": 982365184 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035473420260782347, + "loss": 2.6658, + "theoretical_loss": 3.6553636248646306, + "tokens_seen": 982430720 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035472417251755265, + "loss": 3.0599, + "theoretical_loss": 3.6553403440781205, + "tokens_seen": 982496256 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035471414242728183, + "loss": 2.8763, + "theoretical_loss": 3.6553170652792515, + "tokens_seen": 982561792 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035470411233701107, + "loss": 2.8212, + "theoretical_loss": 3.655293788467722, + "tokens_seen": 982627328 + }, + { + "epoch": 3.03, + "learning_rate": 0.0003546940822467402, + "loss": 2.677, + "theoretical_loss": 3.65527051364323, + "tokens_seen": 982692864 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035468405215646943, + "loss": 2.8286, + "theoretical_loss": 3.655247240805473, + "tokens_seen": 982758400 + }, + { + "epoch": 3.03, + "learning_rate": 0.0003546740220661986, + "loss": 2.8833, + "theoretical_loss": 3.6552239699541493, + "tokens_seen": 982823936 + }, + { + "epoch": 3.03, + "learning_rate": 0.0003546639919759278, + "loss": 2.8848, + "theoretical_loss": 3.6552007010889564, + "tokens_seen": 982889472 + }, + { + "epoch": 3.03, + "learning_rate": 0.000354653961885657, + "loss": 2.9692, + "theoretical_loss": 3.6551774342095933, + "tokens_seen": 982955008 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035464393179538616, + "loss": 2.8621, + "theoretical_loss": 3.6551541693157574, + "tokens_seen": 983020544 + }, + { + "debugging/Self-BLEU-5": 0.39763851797645977, + "debugging/distinct-1-grams": 0.7825305380824156, + "debugging/distinct-2-grams": 0.9587387596654564, + "debugging/entropy-1-grams": 5.676356416295845, + "debugging/entropy-2-grams": 6.320223384568614, + "debugging/length": 540.7142857142857, + "debugging/num_segments": 7, + "debugging/score": 0.006025538542352743, + "debugging/score_std": 0.005169474872870312, + "epoch": 3.03, + "objective/train/docs_used": 1133558, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.444734573364258, + "objective/train/theoretical_loss": 3.6551483534025015, + "objective/train/tokens_used": 1003496928, + "theoretical_loss": 3.6551483534025015, + "tokens_seen": 983036928 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035463390170511534, + "loss": 2.7331, + "theoretical_loss": 3.6551309064071464, + "tokens_seen": 983086080 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035462387161484457, + "loss": 2.7984, + "theoretical_loss": 3.6551076454834597, + "tokens_seen": 983151616 + }, + { + "epoch": 3.03, + "learning_rate": 0.0003546138415245737, + "loss": 2.6689, + "theoretical_loss": 3.655084386544395, + "tokens_seen": 983217152 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035460381143430293, + "loss": 2.9601, + "theoretical_loss": 3.655061129589651, + "tokens_seen": 983282688 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035459378134403206, + "loss": 2.8545, + "theoretical_loss": 3.655037874618926, + "tokens_seen": 983348224 + }, + { + "epoch": 3.03, + "learning_rate": 0.0003545837512537613, + "loss": 2.8236, + "theoretical_loss": 3.655014621631918, + "tokens_seen": 983413760 + }, + { + "epoch": 3.03, + "learning_rate": 0.0003545737211634905, + "loss": 2.8477, + "theoretical_loss": 3.654991370628327, + "tokens_seen": 983479296 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035456369107321966, + "loss": 3.001, + "theoretical_loss": 3.6549681216078502, + "tokens_seen": 983544832 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035455366098294884, + "loss": 2.8405, + "theoretical_loss": 3.654944874570187, + "tokens_seen": 983610368 + }, + { + "epoch": 3.03, + "learning_rate": 0.000354543630892678, + "loss": 2.7946, + "theoretical_loss": 3.6549216295150364, + "tokens_seen": 983675904 + }, + { + "epoch": 3.03, + "learning_rate": 0.0003545336008024072, + "loss": 2.7789, + "theoretical_loss": 3.6548983864420967, + "tokens_seen": 983741440 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035452357071213644, + "loss": 2.9531, + "theoretical_loss": 3.6548751453510677, + "tokens_seen": 983806976 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035451354062186556, + "loss": 2.8242, + "theoretical_loss": 3.654851906241647, + "tokens_seen": 983872512 + }, + { + "epoch": 3.03, + "learning_rate": 0.0003545035105315948, + "loss": 2.8175, + "theoretical_loss": 3.6548286691135354, + "tokens_seen": 983938048 + }, + { + "epoch": 3.03, + "learning_rate": 0.000354493480441324, + "loss": 2.9619, + "theoretical_loss": 3.6548054339664304, + "tokens_seen": 984003584 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035448345035105316, + "loss": 2.8656, + "theoretical_loss": 3.6547822008000326, + "tokens_seen": 984069120 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035447342026078234, + "loss": 2.7168, + "theoretical_loss": 3.654758969614041, + "tokens_seen": 984134656 + }, + { + "epoch": 3.03, + "learning_rate": 0.0003544633901705115, + "loss": 2.741, + "theoretical_loss": 3.654735740408153, + "tokens_seen": 984200192 + }, + { + "epoch": 3.03, + "learning_rate": 0.0003544533600802407, + "loss": 3.0168, + "theoretical_loss": 3.6547125131820706, + "tokens_seen": 984265728 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035444332998996994, + "loss": 3.0056, + "theoretical_loss": 3.654689287935492, + "tokens_seen": 984331264 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035443329989969907, + "loss": 2.8746, + "theoretical_loss": 3.654666064668117, + "tokens_seen": 984396800 + }, + { + "epoch": 3.03, + "learning_rate": 0.0003544232698094283, + "loss": 3.1804, + "theoretical_loss": 3.654642843379645, + "tokens_seen": 984462336 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035441323971915743, + "loss": 3.1955, + "theoretical_loss": 3.6546196240697757, + "tokens_seen": 984527872 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035440320962888667, + "loss": 2.9598, + "theoretical_loss": 3.6545964067382086, + "tokens_seen": 984593408 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035439317953861585, + "loss": 2.938, + "theoretical_loss": 3.654573191384644, + "tokens_seen": 984658944 + }, + { + "epoch": 3.03, + "objective/train/docs_used": 1134314, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.677119493484497, + "objective/train/theoretical_loss": 3.6545673878552805, + "objective/train/tokens_used": 1005135328, + "theoretical_loss": 3.6545673878552805, + "tokens_seen": 984675328 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035438314944834503, + "loss": 3.1099, + "theoretical_loss": 3.654549978008782, + "tokens_seen": 984724480 + }, + { + "epoch": 3.03, + "learning_rate": 0.0003543731193580742, + "loss": 3.0527, + "theoretical_loss": 3.6545267666103207, + "tokens_seen": 984790016 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035436308926780344, + "loss": 3.0231, + "theoretical_loss": 3.6545035571889626, + "tokens_seen": 984855552 + }, + { + "epoch": 3.03, + "learning_rate": 0.0003543530591775326, + "loss": 2.8865, + "theoretical_loss": 3.6544803497444063, + "tokens_seen": 984921088 + }, + { + "epoch": 3.03, + "learning_rate": 0.0003543430290872618, + "loss": 2.7854, + "theoretical_loss": 3.6544571442763516, + "tokens_seen": 984986624 + }, + { + "epoch": 3.03, + "learning_rate": 0.000354332998996991, + "loss": 2.712, + "theoretical_loss": 3.6544339407844997, + "tokens_seen": 985052160 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035432296890672017, + "loss": 2.8541, + "theoretical_loss": 3.6544107392685503, + "tokens_seen": 985117696 + }, + { + "epoch": 3.03, + "learning_rate": 0.0003543129388164494, + "loss": 2.9849, + "theoretical_loss": 3.6543875397282037, + "tokens_seen": 985183232 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035430290872617853, + "loss": 2.9454, + "theoretical_loss": 3.65436434216316, + "tokens_seen": 985248768 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035429287863590777, + "loss": 3.1817, + "theoretical_loss": 3.654341146573121, + "tokens_seen": 985314304 + }, + { + "epoch": 3.03, + "learning_rate": 0.0003542828485456369, + "loss": 2.9778, + "theoretical_loss": 3.6543179529577854, + "tokens_seen": 985379840 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035427281845536613, + "loss": 2.9435, + "theoretical_loss": 3.654294761316855, + "tokens_seen": 985445376 + }, + { + "epoch": 3.03, + "learning_rate": 0.0003542627883650953, + "loss": 2.8006, + "theoretical_loss": 3.6542715716500296, + "tokens_seen": 985510912 + }, + { + "epoch": 3.03, + "learning_rate": 0.0003542527582748245, + "loss": 2.9925, + "theoretical_loss": 3.6542483839570106, + "tokens_seen": 985576448 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035424272818455367, + "loss": 3.0758, + "theoretical_loss": 3.6542251982374983, + "tokens_seen": 985641984 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035423269809428285, + "loss": 2.8809, + "theoretical_loss": 3.6542020144911938, + "tokens_seen": 985707520 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035422266800401203, + "loss": 2.9749, + "theoretical_loss": 3.6541788327177978, + "tokens_seen": 985773056 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035421263791374127, + "loss": 3.0246, + "theoretical_loss": 3.6541556529170114, + "tokens_seen": 985838592 + }, + { + "epoch": 3.03, + "learning_rate": 0.0003542026078234704, + "loss": 3.0333, + "theoretical_loss": 3.6541324750885362, + "tokens_seen": 985904128 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035419257773319963, + "loss": 2.9751, + "theoretical_loss": 3.654109299232072, + "tokens_seen": 985969664 + }, + { + "epoch": 3.03, + "learning_rate": 0.0003541825476429288, + "loss": 3.002, + "theoretical_loss": 3.6540861253473205, + "tokens_seen": 986035200 + }, + { + "epoch": 3.03, + "learning_rate": 0.000354172517552658, + "loss": 3.0969, + "theoretical_loss": 3.654062953433984, + "tokens_seen": 986100736 + }, + { + "epoch": 3.03, + "learning_rate": 0.0003541624874623872, + "loss": 3.0918, + "theoretical_loss": 3.654039783491762, + "tokens_seen": 986166272 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035415245737211636, + "loss": 2.738, + "theoretical_loss": 3.654016615520357, + "tokens_seen": 986231808 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035414242728184554, + "loss": 2.915, + "theoretical_loss": 3.65399344951947, + "tokens_seen": 986297344 + }, + { + "epoch": 3.03, + "objective/train/docs_used": 1135754, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.907013416290283, + "objective/train/theoretical_loss": 3.6539876583271065, + "objective/train/tokens_used": 1006773728, + "theoretical_loss": 3.6539876583271065, + "tokens_seen": 986313728 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035413239719157477, + "loss": 2.919, + "theoretical_loss": 3.6539702854888025, + "tokens_seen": 986362880 + }, + { + "epoch": 3.03, + "learning_rate": 0.0003541223671013039, + "loss": 2.9234, + "theoretical_loss": 3.6539471234280567, + "tokens_seen": 986428416 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035411233701103314, + "loss": 2.8228, + "theoretical_loss": 3.6539239633369327, + "tokens_seen": 986493952 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035410230692076226, + "loss": 2.9489, + "theoretical_loss": 3.653900805215134, + "tokens_seen": 986559488 + }, + { + "epoch": 3.03, + "learning_rate": 0.0003540922768304915, + "loss": 3.0417, + "theoretical_loss": 3.6538776490623612, + "tokens_seen": 986625024 + }, + { + "epoch": 3.03, + "learning_rate": 0.0003540822467402207, + "loss": 2.7938, + "theoretical_loss": 3.6538544948783165, + "tokens_seen": 986690560 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035407221664994986, + "loss": 2.9534, + "theoretical_loss": 3.6538313426627016, + "tokens_seen": 986756096 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035406218655967904, + "loss": 2.8762, + "theoretical_loss": 3.6538081924152186, + "tokens_seen": 986821632 + }, + { + "epoch": 3.03, + "learning_rate": 0.0003540521564694082, + "loss": 2.8072, + "theoretical_loss": 3.653785044135569, + "tokens_seen": 986887168 + }, + { + "epoch": 3.03, + "learning_rate": 0.0003540421263791374, + "loss": 3.145, + "theoretical_loss": 3.653761897823456, + "tokens_seen": 986952704 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035403209628886664, + "loss": 3.0583, + "theoretical_loss": 3.6537387534785806, + "tokens_seen": 987018240 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035402206619859577, + "loss": 3.0636, + "theoretical_loss": 3.6537156111006457, + "tokens_seen": 987083776 + }, + { + "epoch": 3.03, + "learning_rate": 0.000354012036108325, + "loss": 2.9563, + "theoretical_loss": 3.6536924706893528, + "tokens_seen": 987149312 + }, + { + "epoch": 3.03, + "learning_rate": 0.0003540020060180542, + "loss": 3.0312, + "theoretical_loss": 3.6536693322444047, + "tokens_seen": 987214848 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035399197592778336, + "loss": 3.0896, + "theoretical_loss": 3.6536461957655044, + "tokens_seen": 987280384 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035398194583751254, + "loss": 2.9546, + "theoretical_loss": 3.653623061252353, + "tokens_seen": 987345920 + }, + { + "epoch": 3.03, + "learning_rate": 0.0003539719157472417, + "loss": 2.9942, + "theoretical_loss": 3.6535999287046543, + "tokens_seen": 987411456 + }, + { + "epoch": 3.03, + "learning_rate": 0.0003539618856569709, + "loss": 3.0153, + "theoretical_loss": 3.65357679812211, + "tokens_seen": 987476992 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035395185556670014, + "loss": 2.7674, + "theoretical_loss": 3.6535536695044235, + "tokens_seen": 987542528 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035394182547642927, + "loss": 3.0445, + "theoretical_loss": 3.653530542851297, + "tokens_seen": 987608064 + }, + { + "epoch": 3.03, + "learning_rate": 0.0003539317953861585, + "loss": 3.0266, + "theoretical_loss": 3.6535074181624334, + "tokens_seen": 987673600 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035392176529588763, + "loss": 2.6688, + "theoretical_loss": 3.6534842954375355, + "tokens_seen": 987739136 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035391173520561687, + "loss": 2.8888, + "theoretical_loss": 3.6534611746763064, + "tokens_seen": 987804672 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035390170511534605, + "loss": 3.0519, + "theoretical_loss": 3.653438055878449, + "tokens_seen": 987870208 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035389167502507523, + "loss": 2.8346, + "theoretical_loss": 3.6534149390436657, + "tokens_seen": 987935744 + }, + { + "epoch": 3.03, + "objective/train/docs_used": 1136387, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.134091854095459, + "objective/train/theoretical_loss": 3.6534091601416656, + "objective/train/tokens_used": 1008412128, + "theoretical_loss": 3.6534091601416656, + "tokens_seen": 987952128 + }, + { + "epoch": 3.03, + "learning_rate": 0.0003538816449348044, + "loss": 2.9636, + "theoretical_loss": 3.6533918241716608, + "tokens_seen": 988001280 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035387161484453364, + "loss": 2.8623, + "theoretical_loss": 3.6533687112621367, + "tokens_seen": 988066816 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035386158475426277, + "loss": 3.0957, + "theoretical_loss": 3.6533456003147964, + "tokens_seen": 988132352 + }, + { + "epoch": 3.03, + "learning_rate": 0.000353851554663992, + "loss": 2.8578, + "theoretical_loss": 3.653322491329344, + "tokens_seen": 988197888 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035384152457372113, + "loss": 2.8403, + "theoretical_loss": 3.653299384305482, + "tokens_seen": 988263424 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035383149448345037, + "loss": 2.8318, + "theoretical_loss": 3.653276279242915, + "tokens_seen": 988328960 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035382146439317955, + "loss": 2.8225, + "theoretical_loss": 3.6532531761413454, + "tokens_seen": 988394496 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035381143430290873, + "loss": 2.7515, + "theoretical_loss": 3.6532300750004767, + "tokens_seen": 988460032 + }, + { + "epoch": 3.03, + "learning_rate": 0.0003538014042126379, + "loss": 2.8674, + "theoretical_loss": 3.6532069758200136, + "tokens_seen": 988525568 + }, + { + "epoch": 3.03, + "learning_rate": 0.0003537913741223671, + "loss": 2.8581, + "theoretical_loss": 3.653183878599659, + "tokens_seen": 988591104 + }, + { + "epoch": 3.03, + "learning_rate": 0.0003537813440320963, + "loss": 2.8162, + "theoretical_loss": 3.6531607833391164, + "tokens_seen": 988656640 + }, + { + "epoch": 3.03, + "learning_rate": 0.0003537713139418255, + "loss": 2.9214, + "theoretical_loss": 3.6531376900380903, + "tokens_seen": 988722176 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035376128385155464, + "loss": 2.9007, + "theoretical_loss": 3.653114598696284, + "tokens_seen": 988787712 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035375125376128387, + "loss": 3.061, + "theoretical_loss": 3.653091509313402, + "tokens_seen": 988853248 + }, + { + "epoch": 3.03, + "learning_rate": 0.000353741223671013, + "loss": 3.2347, + "theoretical_loss": 3.653068421889148, + "tokens_seen": 988918784 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035373119358074223, + "loss": 2.7098, + "theoretical_loss": 3.653045336423226, + "tokens_seen": 988984320 + }, + { + "epoch": 3.03, + "learning_rate": 0.0003537211634904714, + "loss": 2.6685, + "theoretical_loss": 3.65302225291534, + "tokens_seen": 989049856 + }, + { + "epoch": 3.03, + "learning_rate": 0.0003537111334002006, + "loss": 3.1113, + "theoretical_loss": 3.6529991713651944, + "tokens_seen": 989115392 + }, + { + "epoch": 3.03, + "learning_rate": 0.0003537011033099298, + "loss": 2.8261, + "theoretical_loss": 3.652976091772494, + "tokens_seen": 989180928 + }, + { + "epoch": 3.03, + "learning_rate": 0.000353691073219659, + "loss": 2.706, + "theoretical_loss": 3.6529530141369424, + "tokens_seen": 989246464 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035368104312938814, + "loss": 2.8985, + "theoretical_loss": 3.652929938458244, + "tokens_seen": 989312000 + }, + { + "epoch": 3.03, + "learning_rate": 0.0003536710130391174, + "loss": 2.8457, + "theoretical_loss": 3.652906864736103, + "tokens_seen": 989377536 + }, + { + "epoch": 3.03, + "learning_rate": 0.0003536609829488465, + "loss": 2.9615, + "theoretical_loss": 3.6528837929702256, + "tokens_seen": 989443072 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035365095285857574, + "loss": 2.7433, + "theoretical_loss": 3.6528607231603143, + "tokens_seen": 989508608 + }, + { + "epoch": 3.03, + "learning_rate": 0.0003536409227683049, + "loss": 2.8964, + "theoretical_loss": 3.6528376553060746, + "tokens_seen": 989574144 + }, + { + "epoch": 3.03, + "objective/train/docs_used": 1137615, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.197035074234009, + "objective/train/theoretical_loss": 3.6528318886480537, + "objective/train/tokens_used": 1010050528, + "theoretical_loss": 3.6528318886480537, + "tokens_seen": 989590528 + }, + { + "epoch": 3.03, + "learning_rate": 0.0003536308926780341, + "loss": 3.1049, + "theoretical_loss": 3.6528145894072113, + "tokens_seen": 989639680 + }, + { + "epoch": 3.03, + "learning_rate": 0.0003536208625877633, + "loss": 2.9796, + "theoretical_loss": 3.6527915254634293, + "tokens_seen": 989705216 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035361083249749246, + "loss": 2.7986, + "theoretical_loss": 3.652768463474433, + "tokens_seen": 989770752 + }, + { + "epoch": 3.03, + "learning_rate": 0.0003536008024072217, + "loss": 2.8468, + "theoretical_loss": 3.652745403439928, + "tokens_seen": 989836288 + }, + { + "epoch": 3.03, + "learning_rate": 0.0003535907723169509, + "loss": 2.8229, + "theoretical_loss": 3.652722345359618, + "tokens_seen": 989901824 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035358074222668006, + "loss": 2.7055, + "theoretical_loss": 3.65269928923321, + "tokens_seen": 989967360 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035357071213640924, + "loss": 2.593, + "theoretical_loss": 3.652676235060407, + "tokens_seen": 990032896 + }, + { + "epoch": 3.03, + "learning_rate": 0.0003535606820461384, + "loss": 2.8729, + "theoretical_loss": 3.6526531828409157, + "tokens_seen": 990098432 + }, + { + "epoch": 3.03, + "learning_rate": 0.0003535506519558676, + "loss": 2.9613, + "theoretical_loss": 3.6526301325744406, + "tokens_seen": 990163968 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035354062186559684, + "loss": 3.0164, + "theoretical_loss": 3.652607084260687, + "tokens_seen": 990229504 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035353059177532597, + "loss": 2.9109, + "theoretical_loss": 3.65258403789936, + "tokens_seen": 990295040 + }, + { + "epoch": 3.03, + "learning_rate": 0.0003535205616850552, + "loss": 2.8497, + "theoretical_loss": 3.652560993490166, + "tokens_seen": 990360576 + }, + { + "epoch": 3.03, + "learning_rate": 0.0003535105315947844, + "loss": 2.9282, + "theoretical_loss": 3.6525379510328095, + "tokens_seen": 990426112 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035350050150451356, + "loss": 2.9462, + "theoretical_loss": 3.652514910526997, + "tokens_seen": 990491648 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035349047141424274, + "loss": 2.7424, + "theoretical_loss": 3.652491871972433, + "tokens_seen": 990557184 + }, + { + "epoch": 3.03, + "learning_rate": 0.0003534804413239719, + "loss": 2.738, + "theoretical_loss": 3.6524688353688237, + "tokens_seen": 990622720 + }, + { + "epoch": 3.03, + "learning_rate": 0.0003534704112337011, + "loss": 3.0098, + "theoretical_loss": 3.652445800715875, + "tokens_seen": 990688256 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035346038114343034, + "loss": 2.6713, + "theoretical_loss": 3.6524227680132926, + "tokens_seen": 990753792 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035345035105315947, + "loss": 3.0378, + "theoretical_loss": 3.652399737260782, + "tokens_seen": 990819328 + }, + { + "epoch": 3.03, + "learning_rate": 0.0003534403209628887, + "loss": 3.0039, + "theoretical_loss": 3.65237670845805, + "tokens_seen": 990884864 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035343029087261783, + "loss": 2.8203, + "theoretical_loss": 3.6523536816048017, + "tokens_seen": 990950400 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035342026078234707, + "loss": 2.9151, + "theoretical_loss": 3.6523306567007436, + "tokens_seen": 991015936 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035341023069207625, + "loss": 3.0935, + "theoretical_loss": 3.6523076337455813, + "tokens_seen": 991081472 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035340020060180543, + "loss": 2.7343, + "theoretical_loss": 3.652284612739022, + "tokens_seen": 991147008 + }, + { + "epoch": 3.03, + "learning_rate": 0.0003533901705115346, + "loss": 2.9753, + "theoretical_loss": 3.6522615936807705, + "tokens_seen": 991212544 + }, + { + "epoch": 3.03, + "objective/train/docs_used": 1138406, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.1054980754852295, + "objective/train/theoretical_loss": 3.6522558392205973, + "objective/train/tokens_used": 1011688928, + "theoretical_loss": 3.6522558392205973, + "tokens_seen": 991228928 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035338014042126384, + "loss": 2.9782, + "theoretical_loss": 3.652238576570535, + "tokens_seen": 991278080 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035337011033099297, + "loss": 2.844, + "theoretical_loss": 3.6522155614080205, + "tokens_seen": 991343616 + }, + { + "epoch": 3.03, + "learning_rate": 0.0003533600802407222, + "loss": 2.9592, + "theoretical_loss": 3.652192548192933, + "tokens_seen": 991409152 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035335005015045133, + "loss": 2.976, + "theoretical_loss": 3.6521695369249807, + "tokens_seen": 991474688 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035334002006018057, + "loss": 2.9461, + "theoretical_loss": 3.652146527603869, + "tokens_seen": 991540224 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035332998996990975, + "loss": 2.8723, + "theoretical_loss": 3.6521235202293045, + "tokens_seen": 991605760 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035331995987963893, + "loss": 2.7463, + "theoretical_loss": 3.6521005148009946, + "tokens_seen": 991671296 + }, + { + "epoch": 3.03, + "learning_rate": 0.0003533099297893681, + "loss": 2.8138, + "theoretical_loss": 3.6520775113186454, + "tokens_seen": 991736832 + }, + { + "epoch": 3.03, + "learning_rate": 0.0003532998996990973, + "loss": 2.8062, + "theoretical_loss": 3.652054509781964, + "tokens_seen": 991802368 + }, + { + "epoch": 3.03, + "learning_rate": 0.0003532898696088265, + "loss": 2.9703, + "theoretical_loss": 3.652031510190657, + "tokens_seen": 991867904 + }, + { + "epoch": 3.03, + "learning_rate": 0.0003532798395185557, + "loss": 2.7611, + "theoretical_loss": 3.6520085125444313, + "tokens_seen": 991933440 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035326980942828484, + "loss": 2.7765, + "theoretical_loss": 3.651985516842995, + "tokens_seen": 991998976 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035325977933801407, + "loss": 2.7648, + "theoretical_loss": 3.6519625230860537, + "tokens_seen": 992064512 + }, + { + "epoch": 3.03, + "learning_rate": 0.0003532497492477432, + "loss": 3.091, + "theoretical_loss": 3.6519395312733156, + "tokens_seen": 992130048 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035323971915747243, + "loss": 2.7488, + "theoretical_loss": 3.651916541404487, + "tokens_seen": 992195584 + }, + { + "epoch": 3.03, + "learning_rate": 0.0003532296890672016, + "loss": 2.7365, + "theoretical_loss": 3.651893553479276, + "tokens_seen": 992261120 + }, + { + "epoch": 3.03, + "learning_rate": 0.0003532196589769308, + "loss": 2.9087, + "theoretical_loss": 3.6518705674973893, + "tokens_seen": 992326656 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035320962888666, + "loss": 2.8225, + "theoretical_loss": 3.651847583458535, + "tokens_seen": 992392192 + }, + { + "epoch": 3.03, + "learning_rate": 0.0003531995987963892, + "loss": 2.5887, + "theoretical_loss": 3.65182460136242, + "tokens_seen": 992457728 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035318956870611834, + "loss": 2.8481, + "theoretical_loss": 3.651801621208752, + "tokens_seen": 992523264 + }, + { + "epoch": 3.03, + "learning_rate": 0.0003531795386158476, + "loss": 2.7536, + "theoretical_loss": 3.651778642997238, + "tokens_seen": 992588800 + }, + { + "epoch": 3.03, + "learning_rate": 0.0003531695085255767, + "loss": 2.8793, + "theoretical_loss": 3.6517556667275866, + "tokens_seen": 992654336 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035315947843530594, + "loss": 2.8578, + "theoretical_loss": 3.651732692399505, + "tokens_seen": 992719872 + }, + { + "epoch": 3.03, + "learning_rate": 0.0003531494483450351, + "loss": 3.0135, + "theoretical_loss": 3.6517097200127013, + "tokens_seen": 992785408 + }, + { + "epoch": 3.03, + "learning_rate": 0.0003531394182547643, + "loss": 2.7965, + "theoretical_loss": 3.651686749566883, + "tokens_seen": 992850944 + }, + { + "epoch": 3.03, + "objective/train/docs_used": 1143666, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9665863513946533, + "objective/train/theoretical_loss": 3.6516810072586727, + "objective/train/tokens_used": 1013327328, + "theoretical_loss": 3.6516810072586727, + "tokens_seen": 992867328 + }, + { + "epoch": 3.03, + "learning_rate": 0.0003531293881644935, + "loss": 2.9326, + "theoretical_loss": 3.6516637810617576, + "tokens_seen": 992916480 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035311935807422266, + "loss": 2.9258, + "theoretical_loss": 3.651640814497034, + "tokens_seen": 992982016 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035310932798395184, + "loss": 3.1624, + "theoretical_loss": 3.6516178498724194, + "tokens_seen": 993047552 + }, + { + "epoch": 3.03, + "learning_rate": 0.0003530992978936811, + "loss": 2.8843, + "theoretical_loss": 3.6515948871876227, + "tokens_seen": 993113088 + }, + { + "epoch": 3.03, + "learning_rate": 0.0003530892678034102, + "loss": 2.7025, + "theoretical_loss": 3.6515719264423514, + "tokens_seen": 993178624 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035307923771313944, + "loss": 2.7606, + "theoretical_loss": 3.651548967636314, + "tokens_seen": 993244160 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035306920762286857, + "loss": 2.9293, + "theoretical_loss": 3.651526010769219, + "tokens_seen": 993309696 + }, + { + "epoch": 3.03, + "learning_rate": 0.0003530591775325978, + "loss": 2.6736, + "theoretical_loss": 3.6515030558407737, + "tokens_seen": 993375232 + }, + { + "epoch": 3.03, + "learning_rate": 0.000353049147442327, + "loss": 2.9679, + "theoretical_loss": 3.651480102850688, + "tokens_seen": 993440768 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035303911735205617, + "loss": 2.6353, + "theoretical_loss": 3.651457151798669, + "tokens_seen": 993506304 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035302908726178535, + "loss": 2.9455, + "theoretical_loss": 3.6514342026844258, + "tokens_seen": 993571840 + }, + { + "epoch": 3.03, + "learning_rate": 0.0003530190571715146, + "loss": 2.9826, + "theoretical_loss": 3.6514112555076674, + "tokens_seen": 993637376 + }, + { + "epoch": 3.03, + "learning_rate": 0.0003530090270812437, + "loss": 2.8492, + "theoretical_loss": 3.651388310268102, + "tokens_seen": 993702912 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035299899699097294, + "loss": 2.8351, + "theoretical_loss": 3.6513653669654387, + "tokens_seen": 993768448 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035298896690070207, + "loss": 2.9885, + "theoretical_loss": 3.6513424255993856, + "tokens_seen": 993833984 + }, + { + "epoch": 3.03, + "learning_rate": 0.0003529789368104313, + "loss": 2.7295, + "theoretical_loss": 3.651319486169652, + "tokens_seen": 993899520 + }, + { + "epoch": 3.03, + "learning_rate": 0.0003529689067201605, + "loss": 2.9383, + "theoretical_loss": 3.6512965486759468, + "tokens_seen": 993965056 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035295887662988967, + "loss": 2.6859, + "theoretical_loss": 3.651273613117979, + "tokens_seen": 994030592 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035294884653961885, + "loss": 2.8415, + "theoretical_loss": 3.6512506794954573, + "tokens_seen": 994096128 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035293881644934803, + "loss": 2.697, + "theoretical_loss": 3.651227747808091, + "tokens_seen": 994161664 + }, + { + "epoch": 3.03, + "learning_rate": 0.0003529287863590772, + "loss": 2.8838, + "theoretical_loss": 3.6512048180555894, + "tokens_seen": 994227200 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035291875626880645, + "loss": 2.9552, + "theoretical_loss": 3.6511818902376616, + "tokens_seen": 994292736 + }, + { + "epoch": 3.03, + "learning_rate": 0.0003529087261785356, + "loss": 3.0095, + "theoretical_loss": 3.6511589643540168, + "tokens_seen": 994358272 + }, + { + "epoch": 3.03, + "learning_rate": 0.0003528986960882648, + "loss": 2.6717, + "theoretical_loss": 3.6511360404043645, + "tokens_seen": 994423808 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035288866599799394, + "loss": 3.1065, + "theoretical_loss": 3.6511131183884142, + "tokens_seen": 994489344 + }, + { + "epoch": 3.03, + "objective/train/docs_used": 1148851, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7076807022094727, + "objective/train/theoretical_loss": 3.6511073881865337, + "objective/train/tokens_used": 1014965728, + "theoretical_loss": 3.6511073881865337, + "tokens_seen": 994505728 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035287863590772317, + "loss": 2.8518, + "theoretical_loss": 3.6510901983058757, + "tokens_seen": 994554880 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035286860581745235, + "loss": 3.0575, + "theoretical_loss": 3.6510672801564574, + "tokens_seen": 994620416 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035285857572718153, + "loss": 2.7546, + "theoretical_loss": 3.65104436393987, + "tokens_seen": 994685952 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035284854563691077, + "loss": 2.8476, + "theoretical_loss": 3.651021449655822, + "tokens_seen": 994751488 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035283851554663995, + "loss": 2.7623, + "theoretical_loss": 3.650998537304025, + "tokens_seen": 994817024 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035282848545636913, + "loss": 2.9836, + "theoretical_loss": 3.6509756268841866, + "tokens_seen": 994882560 + }, + { + "epoch": 3.03, + "learning_rate": 0.0003528184553660983, + "loss": 2.7934, + "theoretical_loss": 3.6509527183960184, + "tokens_seen": 994948096 + }, + { + "epoch": 3.03, + "learning_rate": 0.0003528084252758275, + "loss": 2.8664, + "theoretical_loss": 3.6509298118392293, + "tokens_seen": 995013632 + }, + { + "epoch": 3.03, + "learning_rate": 0.0003527983951855567, + "loss": 2.8624, + "theoretical_loss": 3.650906907213529, + "tokens_seen": 995079168 + }, + { + "epoch": 3.03, + "learning_rate": 0.0003527883650952859, + "loss": 2.958, + "theoretical_loss": 3.650884004518629, + "tokens_seen": 995144704 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035277833500501504, + "loss": 2.9275, + "theoretical_loss": 3.650861103754238, + "tokens_seen": 995210240 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035276830491474427, + "loss": 3.1767, + "theoretical_loss": 3.650838204920067, + "tokens_seen": 995275776 + }, + { + "epoch": 3.03, + "learning_rate": 0.0003527582748244734, + "loss": 2.8752, + "theoretical_loss": 3.650815308015826, + "tokens_seen": 995341312 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035274824473420263, + "loss": 2.9867, + "theoretical_loss": 3.650792413041225, + "tokens_seen": 995406848 + }, + { + "epoch": 3.03, + "learning_rate": 0.0003527382146439318, + "loss": 2.9126, + "theoretical_loss": 3.6507695199959747, + "tokens_seen": 995472384 + }, + { + "epoch": 3.03, + "learning_rate": 0.000352728184553661, + "loss": 2.957, + "theoretical_loss": 3.650746628879785, + "tokens_seen": 995537920 + }, + { + "epoch": 3.03, + "learning_rate": 0.0003527181544633902, + "loss": 2.9834, + "theoretical_loss": 3.650723739692367, + "tokens_seen": 995603456 + }, + { + "epoch": 3.03, + "learning_rate": 0.0003527081243731194, + "loss": 2.6613, + "theoretical_loss": 3.650700852433431, + "tokens_seen": 995668992 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035269809428284854, + "loss": 2.4629, + "theoretical_loss": 3.6506779671026877, + "tokens_seen": 995734528 + }, + { + "epoch": 3.03, + "learning_rate": 0.0003526880641925778, + "loss": 2.7763, + "theoretical_loss": 3.6506550836998475, + "tokens_seen": 995800064 + }, + { + "epoch": 3.03, + "learning_rate": 0.0003526780341023069, + "loss": 2.7792, + "theoretical_loss": 3.650632202224621, + "tokens_seen": 995865600 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035266800401203614, + "loss": 3.1098, + "theoretical_loss": 3.6506093226767193, + "tokens_seen": 995931136 + }, + { + "epoch": 3.03, + "learning_rate": 0.0003526579739217653, + "loss": 2.7007, + "theoretical_loss": 3.650586445055853, + "tokens_seen": 995996672 + }, + { + "epoch": 3.03, + "learning_rate": 0.0003526479438314945, + "loss": 2.9314, + "theoretical_loss": 3.650563569361734, + "tokens_seen": 996062208 + }, + { + "epoch": 3.03, + "learning_rate": 0.0003526379137412237, + "loss": 3.065, + "theoretical_loss": 3.6505406955940725, + "tokens_seen": 996127744 + }, + { + "epoch": 3.03, + "objective/train/docs_used": 1153874, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.706052541732788, + "objective/train/theoretical_loss": 3.650534977453132, + "objective/train/tokens_used": 1016604128, + "theoretical_loss": 3.650534977453132, + "tokens_seen": 996144128 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035262788365095286, + "loss": 2.8504, + "theoretical_loss": 3.650517823752579, + "tokens_seen": 996193280 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035261785356068204, + "loss": 2.9067, + "theoretical_loss": 3.650494953836965, + "tokens_seen": 996258816 + }, + { + "epoch": 3.03, + "learning_rate": 0.0003526078234704113, + "loss": 2.7841, + "theoretical_loss": 3.650472085846942, + "tokens_seen": 996324352 + }, + { + "epoch": 3.03, + "learning_rate": 0.0003525977933801404, + "loss": 2.7586, + "theoretical_loss": 3.6504492197822214, + "tokens_seen": 996389888 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035258776328986964, + "loss": 2.6799, + "theoretical_loss": 3.650426355642514, + "tokens_seen": 996455424 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035257773319959877, + "loss": 2.8452, + "theoretical_loss": 3.6504034934275307, + "tokens_seen": 996520960 + }, + { + "epoch": 3.03, + "learning_rate": 0.000352567703109328, + "loss": 2.7273, + "theoretical_loss": 3.650380633136984, + "tokens_seen": 996586496 + }, + { + "epoch": 3.03, + "learning_rate": 0.0003525576730190572, + "loss": 2.9257, + "theoretical_loss": 3.6503577747705847, + "tokens_seen": 996652032 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035254764292878637, + "loss": 2.8902, + "theoretical_loss": 3.6503349183280447, + "tokens_seen": 996717568 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035253761283851555, + "loss": 2.8466, + "theoretical_loss": 3.650312063809075, + "tokens_seen": 996783104 + }, + { + "epoch": 3.03, + "learning_rate": 0.0003525275827482448, + "loss": 2.9663, + "theoretical_loss": 3.6502892112133876, + "tokens_seen": 996848640 + }, + { + "epoch": 3.03, + "learning_rate": 0.0003525175526579739, + "loss": 2.9607, + "theoretical_loss": 3.6502663605406944, + "tokens_seen": 996914176 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035250752256770314, + "loss": 2.795, + "theoretical_loss": 3.6502435117907073, + "tokens_seen": 996979712 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035249749247743227, + "loss": 2.8096, + "theoretical_loss": 3.6502206649631375, + "tokens_seen": 997045248 + }, + { + "epoch": 3.03, + "learning_rate": 0.0003524874623871615, + "loss": 2.9291, + "theoretical_loss": 3.6501978200576977, + "tokens_seen": 997110784 + }, + { + "epoch": 3.03, + "learning_rate": 0.0003524774322968907, + "loss": 2.7279, + "theoretical_loss": 3.6501749770740988, + "tokens_seen": 997176320 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035246740220661987, + "loss": 2.9556, + "theoretical_loss": 3.6501521360120543, + "tokens_seen": 997241856 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035245737211634905, + "loss": 2.9142, + "theoretical_loss": 3.650129296871275, + "tokens_seen": 997307392 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035244734202607823, + "loss": 2.9961, + "theoretical_loss": 3.6501064596514734, + "tokens_seen": 997372928 + }, + { + "epoch": 3.03, + "learning_rate": 0.0003524373119358074, + "loss": 2.7048, + "theoretical_loss": 3.6500836243523627, + "tokens_seen": 997438464 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035242728184553665, + "loss": 2.8819, + "theoretical_loss": 3.6500607909736535, + "tokens_seen": 997504000 + }, + { + "epoch": 3.03, + "learning_rate": 0.0003524172517552658, + "loss": 2.9255, + "theoretical_loss": 3.6500379595150596, + "tokens_seen": 997569536 + }, + { + "epoch": 3.03, + "learning_rate": 0.000352407221664995, + "loss": 2.8446, + "theoretical_loss": 3.6500151299762926, + "tokens_seen": 997635072 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035239719157472414, + "loss": 2.6841, + "theoretical_loss": 3.649992302357065, + "tokens_seen": 997700608 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035238716148445337, + "loss": 2.8274, + "theoretical_loss": 3.64996947665709, + "tokens_seen": 997766144 + }, + { + "epoch": 3.03, + "objective/train/docs_used": 1156592, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5675315856933594, + "objective/train/theoretical_loss": 3.6499637705319454, + "objective/train/tokens_used": 1018242528, + "theoretical_loss": 3.6499637705319454, + "tokens_seen": 997782528 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035237713139418255, + "loss": 2.9074, + "theoretical_loss": 3.6499466528760793, + "tokens_seen": 997831680 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035236710130391173, + "loss": 2.8998, + "theoretical_loss": 3.649923831013746, + "tokens_seen": 997897216 + }, + { + "epoch": 3.03, + "learning_rate": 0.0003523570712136409, + "loss": 2.7522, + "theoretical_loss": 3.6499010110698027, + "tokens_seen": 997962752 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035234704112337015, + "loss": 2.8103, + "theoretical_loss": 3.6498781930439623, + "tokens_seen": 998028288 + }, + { + "epoch": 3.03, + "learning_rate": 0.0003523370110330993, + "loss": 2.8486, + "theoretical_loss": 3.6498553769359376, + "tokens_seen": 998093824 + }, + { + "epoch": 3.03, + "learning_rate": 0.0003523269809428285, + "loss": 2.8552, + "theoretical_loss": 3.6498325627454413, + "tokens_seen": 998159360 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035231695085255764, + "loss": 2.8996, + "theoretical_loss": 3.649809750472187, + "tokens_seen": 998224896 + }, + { + "epoch": 3.03, + "learning_rate": 0.0003523069207622869, + "loss": 2.8666, + "theoretical_loss": 3.649786940115887, + "tokens_seen": 998290432 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035229689067201606, + "loss": 2.8501, + "theoretical_loss": 3.649764131676255, + "tokens_seen": 998355968 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035228686058174524, + "loss": 2.7246, + "theoretical_loss": 3.649741325153003, + "tokens_seen": 998421504 + }, + { + "epoch": 3.03, + "learning_rate": 0.0003522768304914744, + "loss": 2.7134, + "theoretical_loss": 3.6497185205458456, + "tokens_seen": 998487040 + }, + { + "epoch": 3.03, + "learning_rate": 0.0003522668004012036, + "loss": 2.9018, + "theoretical_loss": 3.6496957178544953, + "tokens_seen": 998552576 + }, + { + "epoch": 3.03, + "learning_rate": 0.0003522567703109328, + "loss": 2.7131, + "theoretical_loss": 3.649672917078666, + "tokens_seen": 998618112 + }, + { + "epoch": 3.03, + "learning_rate": 0.000352246740220662, + "loss": 2.9995, + "theoretical_loss": 3.6496501182180703, + "tokens_seen": 998683648 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035223671013039114, + "loss": 2.8589, + "theoretical_loss": 3.649627321272422, + "tokens_seen": 998749184 + }, + { + "epoch": 3.03, + "learning_rate": 0.0003522266800401204, + "loss": 2.7328, + "theoretical_loss": 3.649604526241435, + "tokens_seen": 998814720 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035221664994984956, + "loss": 2.9369, + "theoretical_loss": 3.6495817331248226, + "tokens_seen": 998880256 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035220661985957874, + "loss": 3.0445, + "theoretical_loss": 3.6495589419222982, + "tokens_seen": 998945792 + }, + { + "epoch": 3.03, + "learning_rate": 0.0003521965897693079, + "loss": 2.8777, + "theoretical_loss": 3.6495361526335754, + "tokens_seen": 999011328 + }, + { + "epoch": 3.03, + "learning_rate": 0.0003521865596790371, + "loss": 2.8657, + "theoretical_loss": 3.649513365258369, + "tokens_seen": 999076864 + }, + { + "epoch": 3.03, + "learning_rate": 0.0003521765295887663, + "loss": 2.7128, + "theoretical_loss": 3.6494905797963915, + "tokens_seen": 999142400 + }, + { + "epoch": 3.03, + "learning_rate": 0.0003521664994984955, + "loss": 2.704, + "theoretical_loss": 3.649467796247358, + "tokens_seen": 999207936 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035215646940822465, + "loss": 2.8895, + "theoretical_loss": 3.649445014610981, + "tokens_seen": 999273472 + }, + { + "epoch": 3.03, + "learning_rate": 0.0003521464393179539, + "loss": 2.8723, + "theoretical_loss": 3.649422234886976, + "tokens_seen": 999339008 + }, + { + "epoch": 3.03, + "learning_rate": 0.000352136409227683, + "loss": 3.0843, + "theoretical_loss": 3.6493994570750568, + "tokens_seen": 999404544 + }, + { + "epoch": 3.03, + "objective/train/docs_used": 1159661, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.2150416374206543, + "objective/train/theoretical_loss": 3.6493937629208064, + "objective/train/tokens_used": 1019880928, + "theoretical_loss": 3.6493937629208064, + "tokens_seen": 999420928 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035212637913741224, + "loss": 2.8963, + "theoretical_loss": 3.649376681174936, + "tokens_seen": 999470080 + }, + { + "epoch": 3.03, + "learning_rate": 0.0003521163490471414, + "loss": 3.0057, + "theoretical_loss": 3.64935390718633, + "tokens_seen": 999535616 + }, + { + "epoch": 3.03, + "learning_rate": 0.0003521063189568706, + "loss": 2.9464, + "theoretical_loss": 3.6493311351089517, + "tokens_seen": 999601152 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035209628886659984, + "loss": 3.0535, + "theoretical_loss": 3.6493083649425153, + "tokens_seen": 999666688 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035208625877632897, + "loss": 3.0811, + "theoretical_loss": 3.649285596686736, + "tokens_seen": 999732224 + }, + { + "epoch": 3.03, + "learning_rate": 0.0003520762286860582, + "loss": 2.69, + "theoretical_loss": 3.6492628303413275, + "tokens_seen": 999797760 + }, + { + "epoch": 3.03, + "learning_rate": 0.0003520661985957874, + "loss": 3.0515, + "theoretical_loss": 3.649240065906005, + "tokens_seen": 999863296 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035205616850551657, + "loss": 2.9586, + "theoretical_loss": 3.6492173033804827, + "tokens_seen": 999928832 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035204613841524575, + "loss": 3.0117, + "theoretical_loss": 3.649194542764475, + "tokens_seen": 999994368 + }, + { + "epoch": 3.03, + "learning_rate": 0.000352036108324975, + "loss": 2.9995, + "theoretical_loss": 3.6491717840576974, + "tokens_seen": 1000059904 + }, + { + "epoch": 3.03, + "learning_rate": 0.0003520260782347041, + "loss": 3.0554, + "theoretical_loss": 3.6491490272598637, + "tokens_seen": 1000125440 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035201604814443334, + "loss": 3.0433, + "theoretical_loss": 3.649126272370689, + "tokens_seen": 1000190976 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035200601805416247, + "loss": 2.8908, + "theoretical_loss": 3.6491035193898886, + "tokens_seen": 1000256512 + }, + { + "epoch": 3.03, + "learning_rate": 0.0003519959879638917, + "loss": 2.9182, + "theoretical_loss": 3.6490807683171766, + "tokens_seen": 1000322048 + }, + { + "epoch": 3.03, + "learning_rate": 0.0003519859578736209, + "loss": 2.7976, + "theoretical_loss": 3.649058019152269, + "tokens_seen": 1000387584 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035197592778335007, + "loss": 2.671, + "theoretical_loss": 3.6490352718948795, + "tokens_seen": 1000453120 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035196589769307925, + "loss": 2.8257, + "theoretical_loss": 3.649012526544725, + "tokens_seen": 1000518656 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035195586760280843, + "loss": 2.5004, + "theoretical_loss": 3.6489897831015194, + "tokens_seen": 1000584192 + }, + { + "epoch": 3.03, + "learning_rate": 0.0003519458375125376, + "loss": 2.5683, + "theoretical_loss": 3.648967041564978, + "tokens_seen": 1000649728 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035193580742226685, + "loss": 3.0786, + "theoretical_loss": 3.648944301934817, + "tokens_seen": 1000715264 + }, + { + "epoch": 3.03, + "learning_rate": 0.000351925777331996, + "loss": 2.8131, + "theoretical_loss": 3.6489215642107506, + "tokens_seen": 1000780800 + }, + { + "epoch": 3.03, + "learning_rate": 0.0003519157472417252, + "loss": 2.787, + "theoretical_loss": 3.6488988283924946, + "tokens_seen": 1000846336 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035190571715145434, + "loss": 2.8895, + "theoretical_loss": 3.648876094479765, + "tokens_seen": 1000911872 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035189568706118357, + "loss": 2.5759, + "theoretical_loss": 3.648853362472277, + "tokens_seen": 1000977408 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035188565697091275, + "loss": 2.6727, + "theoretical_loss": 3.648830632369746, + "tokens_seen": 1001042944 + }, + { + "epoch": 3.03, + "objective/train/docs_used": 1160836, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.1073474884033203, + "objective/train/theoretical_loss": 3.648824950141729, + "objective/train/tokens_used": 1021519328, + "theoretical_loss": 3.648824950141729, + "tokens_seen": 1001059328 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035187562688064193, + "loss": 3.1555, + "theoretical_loss": 3.6488079041718877, + "tokens_seen": 1001108480 + }, + { + "epoch": 3.03, + "learning_rate": 0.0003518655967903711, + "loss": 2.9301, + "theoretical_loss": 3.648785177878418, + "tokens_seen": 1001174016 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035185556670010035, + "loss": 2.7881, + "theoretical_loss": 3.648762453489053, + "tokens_seen": 1001239552 + }, + { + "epoch": 3.03, + "learning_rate": 0.0003518455366098295, + "loss": 2.9136, + "theoretical_loss": 3.6487397310035083, + "tokens_seen": 1001305088 + }, + { + "epoch": 3.03, + "learning_rate": 0.0003518355065195587, + "loss": 2.9075, + "theoretical_loss": 3.648717010421499, + "tokens_seen": 1001370624 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035182547642928784, + "loss": 2.561, + "theoretical_loss": 3.6486942917427427, + "tokens_seen": 1001436160 + }, + { + "epoch": 3.03, + "learning_rate": 0.0003518154463390171, + "loss": 3.1295, + "theoretical_loss": 3.648671574966954, + "tokens_seen": 1001501696 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035180541624874626, + "loss": 2.7492, + "theoretical_loss": 3.6486488600938496, + "tokens_seen": 1001567232 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035179538615847544, + "loss": 2.8224, + "theoretical_loss": 3.648626147123146, + "tokens_seen": 1001632768 + }, + { + "epoch": 3.03, + "learning_rate": 0.0003517853560682046, + "loss": 2.9366, + "theoretical_loss": 3.648603436054558, + "tokens_seen": 1001698304 + }, + { + "epoch": 3.03, + "learning_rate": 0.0003517753259779338, + "loss": 2.9208, + "theoretical_loss": 3.648580726887803, + "tokens_seen": 1001763840 + }, + { + "epoch": 3.03, + "learning_rate": 0.000351765295887663, + "loss": 2.8788, + "theoretical_loss": 3.648558019622598, + "tokens_seen": 1001829376 + }, + { + "epoch": 3.03, + "learning_rate": 0.0003517552657973922, + "loss": 2.9987, + "theoretical_loss": 3.648535314258658, + "tokens_seen": 1001894912 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035174523570712134, + "loss": 3.0191, + "theoretical_loss": 3.6485126107957004, + "tokens_seen": 1001960448 + }, + { + "epoch": 3.03, + "learning_rate": 0.0003517352056168506, + "loss": 2.8817, + "theoretical_loss": 3.648489909233441, + "tokens_seen": 1002025984 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035172517552657976, + "loss": 2.9599, + "theoretical_loss": 3.648467209571597, + "tokens_seen": 1002091520 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035171514543630894, + "loss": 2.7182, + "theoretical_loss": 3.6484445118098847, + "tokens_seen": 1002157056 + }, + { + "epoch": 3.03, + "learning_rate": 0.0003517051153460381, + "loss": 2.9031, + "theoretical_loss": 3.648421815948021, + "tokens_seen": 1002222592 + }, + { + "epoch": 3.03, + "learning_rate": 0.0003516950852557673, + "loss": 2.6092, + "theoretical_loss": 3.6483991219857224, + "tokens_seen": 1002288128 + }, + { + "epoch": 3.03, + "learning_rate": 0.0003516850551654965, + "loss": 2.6493, + "theoretical_loss": 3.648376429922706, + "tokens_seen": 1002353664 + }, + { + "epoch": 3.03, + "learning_rate": 0.0003516750250752257, + "loss": 2.7576, + "theoretical_loss": 3.6483537397586887, + "tokens_seen": 1002419200 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035166499498495485, + "loss": 2.9421, + "theoretical_loss": 3.648331051493387, + "tokens_seen": 1002484736 + }, + { + "epoch": 3.03, + "learning_rate": 0.0003516549648946841, + "loss": 2.6704, + "theoretical_loss": 3.6483083651265185, + "tokens_seen": 1002550272 + }, + { + "epoch": 3.03, + "learning_rate": 0.0003516449348044132, + "loss": 2.8942, + "theoretical_loss": 3.6482856806578, + "tokens_seen": 1002615808 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035163490471414244, + "loss": 2.9315, + "theoretical_loss": 3.648262998086948, + "tokens_seen": 1002681344 + }, + { + "epoch": 3.03, + "objective/train/docs_used": 1161424, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.439594030380249, + "objective/train/theoretical_loss": 3.648257327740744, + "objective/train/tokens_used": 1023157728, + "theoretical_loss": 3.648257327740744, + "tokens_seen": 1002697728 + }, + { + "epoch": 3.03, + "learning_rate": 0.0003516248746238716, + "loss": 2.7253, + "theoretical_loss": 3.6482403174136806, + "tokens_seen": 1002746880 + }, + { + "epoch": 3.03, + "learning_rate": 0.0003516148445336008, + "loss": 2.9394, + "theoretical_loss": 3.648217638637715, + "tokens_seen": 1002812416 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035160481444333, + "loss": 2.498, + "theoretical_loss": 3.648194961758768, + "tokens_seen": 1002877952 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035159478435305917, + "loss": 2.7678, + "theoretical_loss": 3.6481722867765574, + "tokens_seen": 1002943488 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035158475426278835, + "loss": 2.7227, + "theoretical_loss": 3.6481496136908, + "tokens_seen": 1003009024 + }, + { + "epoch": 3.03, + "learning_rate": 0.0003515747241725176, + "loss": 2.6544, + "theoretical_loss": 3.6481269425012144, + "tokens_seen": 1003074560 + }, + { + "epoch": 3.03, + "learning_rate": 0.0003515646940822467, + "loss": 2.7992, + "theoretical_loss": 3.648104273207517, + "tokens_seen": 1003140096 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035155466399197595, + "loss": 2.8121, + "theoretical_loss": 3.648081605809426, + "tokens_seen": 1003205632 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035154463390170513, + "loss": 2.8442, + "theoretical_loss": 3.648058940306659, + "tokens_seen": 1003271168 + }, + { + "epoch": 3.03, + "learning_rate": 0.0003515346038114343, + "loss": 2.91, + "theoretical_loss": 3.648036276698934, + "tokens_seen": 1003336704 + }, + { + "epoch": 3.03, + "learning_rate": 0.0003515245737211635, + "loss": 2.9568, + "theoretical_loss": 3.6480136149859677, + "tokens_seen": 1003402240 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035151454363089267, + "loss": 2.7311, + "theoretical_loss": 3.6479909551674794, + "tokens_seen": 1003467776 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035150451354062185, + "loss": 2.8667, + "theoretical_loss": 3.647968297243186, + "tokens_seen": 1003533312 + }, + { + "epoch": 3.03, + "learning_rate": 0.0003514944834503511, + "loss": 2.7325, + "theoretical_loss": 3.6479456412128064, + "tokens_seen": 1003598848 + }, + { + "epoch": 3.03, + "learning_rate": 0.0003514844533600802, + "loss": 2.9816, + "theoretical_loss": 3.6479229870760577, + "tokens_seen": 1003664384 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035147442326980945, + "loss": 2.8746, + "theoretical_loss": 3.6479003348326584, + "tokens_seen": 1003729920 + }, + { + "epoch": 3.03, + "learning_rate": 0.0003514643931795386, + "loss": 2.9706, + "theoretical_loss": 3.647877684482326, + "tokens_seen": 1003795456 + }, + { + "epoch": 3.03, + "learning_rate": 0.0003514543630892678, + "loss": 2.769, + "theoretical_loss": 3.64785503602478, + "tokens_seen": 1003860992 + }, + { + "epoch": 3.03, + "learning_rate": 0.000351444332998997, + "loss": 2.815, + "theoretical_loss": 3.647832389459738, + "tokens_seen": 1003926528 + }, + { + "epoch": 3.03, + "learning_rate": 0.0003514343029087262, + "loss": 2.6748, + "theoretical_loss": 3.647809744786918, + "tokens_seen": 1003992064 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035142427281845536, + "loss": 2.787, + "theoretical_loss": 3.6477871020060393, + "tokens_seen": 1004057600 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035141424272818454, + "loss": 2.9168, + "theoretical_loss": 3.647764461116819, + "tokens_seen": 1004123136 + }, + { + "epoch": 3.03, + "learning_rate": 0.0003514042126379137, + "loss": 2.7364, + "theoretical_loss": 3.6477418221189772, + "tokens_seen": 1004188672 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035139418254764295, + "loss": 2.7187, + "theoretical_loss": 3.647719185012231, + "tokens_seen": 1004254208 + }, + { + "epoch": 3.03, + "learning_rate": 0.0003513841524573721, + "loss": 2.7207, + "theoretical_loss": 3.6476965497963, + "tokens_seen": 1004319744 + }, + { + "epoch": 3.03, + "objective/train/docs_used": 1162777, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7652225494384766, + "objective/train/theoretical_loss": 3.6476908912877244, + "objective/train/tokens_used": 1024796128, + "theoretical_loss": 3.6476908912877244, + "tokens_seen": 1004336128 + }, + { + "epoch": 3.03, + "learning_rate": 0.0003513741223671013, + "loss": 2.9625, + "theoretical_loss": 3.647673916470903, + "tokens_seen": 1004385280 + }, + { + "epoch": 3.03, + "learning_rate": 0.0003513640922768305, + "loss": 2.8803, + "theoretical_loss": 3.647651285035758, + "tokens_seen": 1004450816 + }, + { + "epoch": 3.03, + "learning_rate": 0.0003513540621865597, + "loss": 2.7448, + "theoretical_loss": 3.647628655490584, + "tokens_seen": 1004516352 + }, + { + "epoch": 3.03, + "learning_rate": 0.0003513440320962889, + "loss": 3.026, + "theoretical_loss": 3.6476060278351, + "tokens_seen": 1004581888 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035133400200601804, + "loss": 2.9949, + "theoretical_loss": 3.6475834020690256, + "tokens_seen": 1004647424 + }, + { + "epoch": 3.03, + "learning_rate": 0.0003513239719157473, + "loss": 3.0162, + "theoretical_loss": 3.647560778192079, + "tokens_seen": 1004712960 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035131394182547646, + "loss": 2.7788, + "theoretical_loss": 3.6475381562039795, + "tokens_seen": 1004778496 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035130391173520564, + "loss": 2.9032, + "theoretical_loss": 3.647515536104446, + "tokens_seen": 1004844032 + }, + { + "epoch": 3.03, + "learning_rate": 0.0003512938816449348, + "loss": 2.8642, + "theoretical_loss": 3.6474929178931976, + "tokens_seen": 1004909568 + }, + { + "epoch": 3.03, + "learning_rate": 0.000351283851554664, + "loss": 2.7253, + "theoretical_loss": 3.647470301569954, + "tokens_seen": 1004975104 + }, + { + "epoch": 3.03, + "learning_rate": 0.0003512738214643932, + "loss": 2.9281, + "theoretical_loss": 3.647447687134435, + "tokens_seen": 1005040640 + }, + { + "epoch": 3.03, + "learning_rate": 0.0003512637913741224, + "loss": 2.8002, + "theoretical_loss": 3.6474250745863586, + "tokens_seen": 1005106176 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035125376128385154, + "loss": 2.7841, + "theoretical_loss": 3.647402463925445, + "tokens_seen": 1005171712 + }, + { + "epoch": 3.03, + "learning_rate": 0.0003512437311935808, + "loss": 2.7919, + "theoretical_loss": 3.6473798551514136, + "tokens_seen": 1005237248 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035123370110330996, + "loss": 2.9419, + "theoretical_loss": 3.647357248263984, + "tokens_seen": 1005302784 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035122367101303914, + "loss": 2.7465, + "theoretical_loss": 3.6473346432628753, + "tokens_seen": 1005368320 + }, + { + "epoch": 3.03, + "learning_rate": 0.0003512136409227683, + "loss": 2.811, + "theoretical_loss": 3.647312040147808, + "tokens_seen": 1005433856 + }, + { + "epoch": 3.03, + "learning_rate": 0.0003512036108324975, + "loss": 2.8367, + "theoretical_loss": 3.6472894389185013, + "tokens_seen": 1005499392 + }, + { + "epoch": 3.03, + "learning_rate": 0.0003511935807422267, + "loss": 2.87, + "theoretical_loss": 3.647266839574675, + "tokens_seen": 1005564928 + }, + { + "epoch": 3.03, + "learning_rate": 0.0003511835506519559, + "loss": 2.8118, + "theoretical_loss": 3.6472442421160487, + "tokens_seen": 1005630464 + }, + { + "epoch": 3.03, + "learning_rate": 0.00035117352056168505, + "loss": 2.947, + "theoretical_loss": 3.6472216465423424, + "tokens_seen": 1005696000 + }, + { + "epoch": 3.03, + "learning_rate": 0.0003511634904714143, + "loss": 2.6745, + "theoretical_loss": 3.647199052853277, + "tokens_seen": 1005761536 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003511534603811434, + "loss": 2.8839, + "theoretical_loss": 3.6471764610485713, + "tokens_seen": 1005827072 + }, + { + "epoch": 3.04, + "learning_rate": 0.00035114343029087264, + "loss": 2.8895, + "theoretical_loss": 3.647153871127946, + "tokens_seen": 1005892608 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003511334002006018, + "loss": 2.7014, + "theoretical_loss": 3.647131283091121, + "tokens_seen": 1005958144 + }, + { + "epoch": 3.04, + "objective/train/docs_used": 1163373, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.477107524871826, + "objective/train/theoretical_loss": 3.6471256363762254, + "objective/train/tokens_used": 1026434528, + "theoretical_loss": 3.6471256363762254, + "tokens_seen": 1005974528 + }, + { + "epoch": 3.04, + "learning_rate": 0.000351123370110331, + "loss": 2.8156, + "theoretical_loss": 3.6471086969378166, + "tokens_seen": 1006023680 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003511133400200602, + "loss": 2.5824, + "theoretical_loss": 3.647086112667753, + "tokens_seen": 1006089216 + }, + { + "epoch": 3.04, + "learning_rate": 0.00035110330992978937, + "loss": 2.8562, + "theoretical_loss": 3.6470635302806507, + "tokens_seen": 1006154752 + }, + { + "epoch": 3.04, + "learning_rate": 0.00035109327983951855, + "loss": 2.9694, + "theoretical_loss": 3.6470409497762297, + "tokens_seen": 1006220288 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003510832497492478, + "loss": 2.7441, + "theoretical_loss": 3.647018371154211, + "tokens_seen": 1006285824 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003510732196589769, + "loss": 2.8582, + "theoretical_loss": 3.6469957944143143, + "tokens_seen": 1006351360 + }, + { + "epoch": 3.04, + "learning_rate": 0.00035106318956870615, + "loss": 2.764, + "theoretical_loss": 3.646973219556261, + "tokens_seen": 1006416896 + }, + { + "epoch": 3.04, + "learning_rate": 0.00035105315947843533, + "loss": 2.6505, + "theoretical_loss": 3.6469506465797714, + "tokens_seen": 1006482432 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003510431293881645, + "loss": 2.6349, + "theoretical_loss": 3.6469280754845657, + "tokens_seen": 1006547968 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003510330992978937, + "loss": 2.8268, + "theoretical_loss": 3.6469055062703655, + "tokens_seen": 1006613504 + }, + { + "epoch": 3.04, + "learning_rate": 0.00035102306920762287, + "loss": 2.8678, + "theoretical_loss": 3.6468829389368906, + "tokens_seen": 1006679040 + }, + { + "epoch": 3.04, + "learning_rate": 0.00035101303911735205, + "loss": 3.0746, + "theoretical_loss": 3.6468603734838627, + "tokens_seen": 1006744576 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003510030090270813, + "loss": 2.9625, + "theoretical_loss": 3.6468378099110024, + "tokens_seen": 1006810112 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003509929789368104, + "loss": 2.8395, + "theoretical_loss": 3.6468152482180307, + "tokens_seen": 1006875648 + }, + { + "epoch": 3.04, + "learning_rate": 0.00035098294884653965, + "loss": 2.5887, + "theoretical_loss": 3.646792688404669, + "tokens_seen": 1006941184 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003509729187562688, + "loss": 2.7411, + "theoretical_loss": 3.646770130470637, + "tokens_seen": 1007006720 + }, + { + "epoch": 3.04, + "learning_rate": 0.000350962888665998, + "loss": 3.0379, + "theoretical_loss": 3.6467475744156577, + "tokens_seen": 1007072256 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003509528585757272, + "loss": 2.7484, + "theoretical_loss": 3.646725020239451, + "tokens_seen": 1007137792 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003509428284854564, + "loss": 2.8597, + "theoretical_loss": 3.646702467941739, + "tokens_seen": 1007203328 + }, + { + "epoch": 3.04, + "learning_rate": 0.00035093279839518556, + "loss": 2.8188, + "theoretical_loss": 3.646679917522243, + "tokens_seen": 1007268864 + }, + { + "epoch": 3.04, + "learning_rate": 0.00035092276830491474, + "loss": 2.9834, + "theoretical_loss": 3.646657368980683, + "tokens_seen": 1007334400 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003509127382146439, + "loss": 2.7302, + "theoretical_loss": 3.6466348223167824, + "tokens_seen": 1007399936 + }, + { + "epoch": 3.04, + "learning_rate": 0.00035090270812437315, + "loss": 2.7753, + "theoretical_loss": 3.646612277530261, + "tokens_seen": 1007465472 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003508926780341023, + "loss": 2.6536, + "theoretical_loss": 3.646589734620842, + "tokens_seen": 1007531008 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003508826479438315, + "loss": 2.6738, + "theoretical_loss": 3.646567193588246, + "tokens_seen": 1007596544 + }, + { + "epoch": 3.04, + "objective/train/docs_used": 1164906, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.129481554031372, + "objective/train/theoretical_loss": 3.6465615586233175, + "objective/train/tokens_used": 1028072928, + "theoretical_loss": 3.6465615586233175, + "tokens_seen": 1007612928 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003508726178535607, + "loss": 2.9174, + "theoretical_loss": 3.6465446544321947, + "tokens_seen": 1007662080 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003508625877632899, + "loss": 2.7308, + "theoretical_loss": 3.6465221171524105, + "tokens_seen": 1007727616 + }, + { + "epoch": 3.04, + "learning_rate": 0.00035085255767301906, + "loss": 2.8025, + "theoretical_loss": 3.646499581748614, + "tokens_seen": 1007793152 + }, + { + "epoch": 3.04, + "learning_rate": 0.00035084252758274824, + "loss": 2.8899, + "theoretical_loss": 3.646477048220528, + "tokens_seen": 1007858688 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003508324974924774, + "loss": 2.8594, + "theoretical_loss": 3.646454516567875, + "tokens_seen": 1007924224 + }, + { + "epoch": 3.04, + "learning_rate": 0.00035082246740220666, + "loss": 2.9397, + "theoretical_loss": 3.6464319867903754, + "tokens_seen": 1007989760 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003508124373119358, + "loss": 2.8024, + "theoretical_loss": 3.646409458887752, + "tokens_seen": 1008055296 + }, + { + "epoch": 3.04, + "learning_rate": 0.000350802407221665, + "loss": 2.8186, + "theoretical_loss": 3.646386932859728, + "tokens_seen": 1008120832 + }, + { + "epoch": 3.04, + "learning_rate": 0.00035079237713139415, + "loss": 2.7744, + "theoretical_loss": 3.646364408706024, + "tokens_seen": 1008186368 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003507823470411234, + "loss": 2.6754, + "theoretical_loss": 3.646341886426362, + "tokens_seen": 1008251904 + }, + { + "epoch": 3.04, + "learning_rate": 0.00035077231695085256, + "loss": 2.9253, + "theoretical_loss": 3.646319366020466, + "tokens_seen": 1008317440 + }, + { + "epoch": 3.04, + "learning_rate": 0.00035076228686058174, + "loss": 2.7929, + "theoretical_loss": 3.646296847488057, + "tokens_seen": 1008382976 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003507522567703109, + "loss": 2.9397, + "theoretical_loss": 3.6462743308288577, + "tokens_seen": 1008448512 + }, + { + "epoch": 3.04, + "learning_rate": 0.00035074222668004016, + "loss": 2.8635, + "theoretical_loss": 3.646251816042591, + "tokens_seen": 1008514048 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003507321965897693, + "loss": 2.8216, + "theoretical_loss": 3.6462293031289787, + "tokens_seen": 1008579584 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003507221664994985, + "loss": 2.8308, + "theoretical_loss": 3.6462067920877437, + "tokens_seen": 1008645120 + }, + { + "epoch": 3.04, + "learning_rate": 0.00035071213640922765, + "loss": 2.9977, + "theoretical_loss": 3.646184282918609, + "tokens_seen": 1008710656 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003507021063189569, + "loss": 2.8788, + "theoretical_loss": 3.6461617756212963, + "tokens_seen": 1008776192 + }, + { + "epoch": 3.04, + "learning_rate": 0.00035069207622868607, + "loss": 2.6056, + "theoretical_loss": 3.6461392701955297, + "tokens_seen": 1008841728 + }, + { + "epoch": 3.04, + "learning_rate": 0.00035068204613841525, + "loss": 2.8894, + "theoretical_loss": 3.646116766641031, + "tokens_seen": 1008907264 + }, + { + "epoch": 3.04, + "learning_rate": 0.00035067201604814443, + "loss": 2.8793, + "theoretical_loss": 3.646094264957523, + "tokens_seen": 1008972800 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003506619859578736, + "loss": 2.6769, + "theoretical_loss": 3.64607176514473, + "tokens_seen": 1009038336 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003506519558676028, + "loss": 2.6217, + "theoretical_loss": 3.6460492672023728, + "tokens_seen": 1009103872 + }, + { + "epoch": 3.04, + "learning_rate": 0.000350641925777332, + "loss": 2.7837, + "theoretical_loss": 3.6460267711301766, + "tokens_seen": 1009169408 + }, + { + "epoch": 3.04, + "learning_rate": 0.00035063189568706115, + "loss": 2.8821, + "theoretical_loss": 3.6460042769278633, + "tokens_seen": 1009234944 + }, + { + "epoch": 3.04, + "objective/train/docs_used": 1165274, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.818286657333374, + "objective/train/theoretical_loss": 3.6459986536694218, + "objective/train/tokens_used": 1029711328, + "theoretical_loss": 3.6459986536694218, + "tokens_seen": 1009251328 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003506218655967904, + "loss": 2.8747, + "theoretical_loss": 3.6459817845951563, + "tokens_seen": 1009300480 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003506118355065195, + "loss": 2.6386, + "theoretical_loss": 3.645959294131779, + "tokens_seen": 1009366016 + }, + { + "epoch": 3.04, + "learning_rate": 0.00035060180541624875, + "loss": 2.8541, + "theoretical_loss": 3.6459368055374544, + "tokens_seen": 1009431552 + }, + { + "epoch": 3.04, + "learning_rate": 0.000350591775325978, + "loss": 2.925, + "theoretical_loss": 3.6459143188119056, + "tokens_seen": 1009497088 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003505817452357071, + "loss": 2.7224, + "theoretical_loss": 3.645891833954857, + "tokens_seen": 1009562624 + }, + { + "epoch": 3.04, + "learning_rate": 0.00035057171514543635, + "loss": 2.9708, + "theoretical_loss": 3.645869350966031, + "tokens_seen": 1009628160 + }, + { + "epoch": 3.04, + "learning_rate": 0.00035056168505516553, + "loss": 2.6602, + "theoretical_loss": 3.645846869845152, + "tokens_seen": 1009693696 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003505516549648947, + "loss": 2.8586, + "theoretical_loss": 3.645824390591943, + "tokens_seen": 1009759232 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003505416248746239, + "loss": 2.6716, + "theoretical_loss": 3.6458019132061272, + "tokens_seen": 1009824768 + }, + { + "epoch": 3.04, + "learning_rate": 0.00035053159478435307, + "loss": 3.0043, + "theoretical_loss": 3.6457794376874295, + "tokens_seen": 1009890304 + }, + { + "epoch": 3.04, + "learning_rate": 0.00035052156469408225, + "loss": 2.7406, + "theoretical_loss": 3.645756964035573, + "tokens_seen": 1009955840 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003505115346038115, + "loss": 2.8073, + "theoretical_loss": 3.6457344922502815, + "tokens_seen": 1010021376 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003505015045135406, + "loss": 2.9574, + "theoretical_loss": 3.6457120223312787, + "tokens_seen": 1010086912 + }, + { + "epoch": 3.04, + "learning_rate": 0.00035049147442326985, + "loss": 2.8053, + "theoretical_loss": 3.645689554278289, + "tokens_seen": 1010152448 + }, + { + "epoch": 3.04, + "learning_rate": 0.000350481444332999, + "loss": 2.7435, + "theoretical_loss": 3.6456670880910362, + "tokens_seen": 1010217984 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003504714142427282, + "loss": 2.9574, + "theoretical_loss": 3.645644623769244, + "tokens_seen": 1010283520 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003504613841524574, + "loss": 2.8229, + "theoretical_loss": 3.645622161312637, + "tokens_seen": 1010349056 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003504513540621866, + "loss": 2.9436, + "theoretical_loss": 3.645599700720939, + "tokens_seen": 1010414592 + }, + { + "epoch": 3.04, + "learning_rate": 0.00035044132397191576, + "loss": 2.8731, + "theoretical_loss": 3.6455772419938746, + "tokens_seen": 1010480128 + }, + { + "epoch": 3.04, + "learning_rate": 0.00035043129388164494, + "loss": 2.8602, + "theoretical_loss": 3.645554785131168, + "tokens_seen": 1010545664 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003504212637913741, + "loss": 2.721, + "theoretical_loss": 3.645532330132543, + "tokens_seen": 1010611200 + }, + { + "epoch": 3.04, + "learning_rate": 0.00035041123370110335, + "loss": 2.6924, + "theoretical_loss": 3.6455098769977248, + "tokens_seen": 1010676736 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003504012036108325, + "loss": 2.9447, + "theoretical_loss": 3.645487425726437, + "tokens_seen": 1010742272 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003503911735205617, + "loss": 2.7244, + "theoretical_loss": 3.6454649763184044, + "tokens_seen": 1010807808 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003503811434302909, + "loss": 2.8867, + "theoretical_loss": 3.6454425287733527, + "tokens_seen": 1010873344 + }, + { + "epoch": 3.04, + "objective/train/docs_used": 1166720, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7130308151245117, + "objective/train/theoretical_loss": 3.6454369171781478, + "objective/train/tokens_used": 1031349728, + "theoretical_loss": 3.6454369171781478, + "tokens_seen": 1010889728 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003503711133400201, + "loss": 2.7468, + "theoretical_loss": 3.645420083091005, + "tokens_seen": 1010938880 + }, + { + "epoch": 3.04, + "learning_rate": 0.00035036108324974926, + "loss": 2.7862, + "theoretical_loss": 3.6453976392710867, + "tokens_seen": 1011004416 + }, + { + "epoch": 3.04, + "learning_rate": 0.00035035105315947844, + "loss": 2.9374, + "theoretical_loss": 3.6453751973133226, + "tokens_seen": 1011069952 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003503410230692076, + "loss": 2.6861, + "theoretical_loss": 3.645352757217437, + "tokens_seen": 1011135488 + }, + { + "epoch": 3.04, + "learning_rate": 0.00035033099297893686, + "loss": 2.7727, + "theoretical_loss": 3.6453303189831554, + "tokens_seen": 1011201024 + }, + { + "epoch": 3.04, + "learning_rate": 0.000350320962888666, + "loss": 2.9209, + "theoretical_loss": 3.6453078826102026, + "tokens_seen": 1011266560 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003503109327983952, + "loss": 2.7929, + "theoretical_loss": 3.6452854480983032, + "tokens_seen": 1011332096 + }, + { + "epoch": 3.04, + "learning_rate": 0.00035030090270812435, + "loss": 2.8713, + "theoretical_loss": 3.6452630154471826, + "tokens_seen": 1011397632 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003502908726178536, + "loss": 2.8707, + "theoretical_loss": 3.645240584656566, + "tokens_seen": 1011463168 + }, + { + "epoch": 3.04, + "learning_rate": 0.00035028084252758276, + "loss": 2.7903, + "theoretical_loss": 3.645218155726178, + "tokens_seen": 1011528704 + }, + { + "epoch": 3.04, + "learning_rate": 0.00035027081243731194, + "loss": 2.8254, + "theoretical_loss": 3.6451957286557444, + "tokens_seen": 1011594240 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003502607823470411, + "loss": 2.9904, + "theoretical_loss": 3.64517330344499, + "tokens_seen": 1011659776 + }, + { + "epoch": 3.04, + "learning_rate": 0.00035025075225677036, + "loss": 2.8842, + "theoretical_loss": 3.6451508800936407, + "tokens_seen": 1011725312 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003502407221664995, + "loss": 2.885, + "theoretical_loss": 3.6451284586014214, + "tokens_seen": 1011790848 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003502306920762287, + "loss": 2.5298, + "theoretical_loss": 3.6451060389680574, + "tokens_seen": 1011856384 + }, + { + "epoch": 3.04, + "learning_rate": 0.00035022066198595785, + "loss": 2.8897, + "theoretical_loss": 3.645083621193275, + "tokens_seen": 1011921920 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003502106318956871, + "loss": 2.5561, + "theoretical_loss": 3.6450612052767997, + "tokens_seen": 1011987456 + }, + { + "epoch": 3.04, + "learning_rate": 0.00035020060180541627, + "loss": 2.7428, + "theoretical_loss": 3.645038791218356, + "tokens_seen": 1012052992 + }, + { + "epoch": 3.04, + "learning_rate": 0.00035019057171514545, + "loss": 3.0237, + "theoretical_loss": 3.6450163790176706, + "tokens_seen": 1012118528 + }, + { + "epoch": 3.04, + "learning_rate": 0.00035018054162487463, + "loss": 2.6427, + "theoretical_loss": 3.644993968674469, + "tokens_seen": 1012184064 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003501705115346038, + "loss": 3.029, + "theoretical_loss": 3.644971560188477, + "tokens_seen": 1012249600 + }, + { + "epoch": 3.04, + "learning_rate": 0.000350160481444333, + "loss": 2.8482, + "theoretical_loss": 3.6449491535594207, + "tokens_seen": 1012315136 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003501504513540622, + "loss": 2.7981, + "theoretical_loss": 3.644926748787025, + "tokens_seen": 1012380672 + }, + { + "epoch": 3.04, + "learning_rate": 0.00035014042126379135, + "loss": 2.9763, + "theoretical_loss": 3.6449043458710175, + "tokens_seen": 1012446208 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003501303911735206, + "loss": 2.8806, + "theoretical_loss": 3.6448819448111234, + "tokens_seen": 1012511744 + }, + { + "epoch": 3.04, + "objective/train/docs_used": 1167486, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.346987724304199, + "objective/train/theoretical_loss": 3.644876344836135, + "objective/train/tokens_used": 1032988128, + "theoretical_loss": 3.644876344836135, + "tokens_seen": 1012528128 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003501203610832497, + "loss": 2.9117, + "theoretical_loss": 3.6448595456070683, + "tokens_seen": 1012577280 + }, + { + "epoch": 3.04, + "learning_rate": 0.00035011033099297895, + "loss": 2.748, + "theoretical_loss": 3.644837148258579, + "tokens_seen": 1012642816 + }, + { + "epoch": 3.04, + "learning_rate": 0.00035010030090270813, + "loss": 2.9558, + "theoretical_loss": 3.644814752765382, + "tokens_seen": 1012708352 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003500902708124373, + "loss": 2.8949, + "theoretical_loss": 3.644792359127202, + "tokens_seen": 1012773888 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003500802407221665, + "loss": 2.8262, + "theoretical_loss": 3.644769967343767, + "tokens_seen": 1012839424 + }, + { + "epoch": 3.04, + "learning_rate": 0.00035007021063189573, + "loss": 2.6742, + "theoretical_loss": 3.644747577414803, + "tokens_seen": 1012904960 + }, + { + "epoch": 3.04, + "learning_rate": 0.00035006018054162486, + "loss": 2.7622, + "theoretical_loss": 3.6447251893400363, + "tokens_seen": 1012970496 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003500501504513541, + "loss": 2.7589, + "theoretical_loss": 3.644702803119193, + "tokens_seen": 1013036032 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003500401203610832, + "loss": 2.6765, + "theoretical_loss": 3.6446804187520003, + "tokens_seen": 1013101568 + }, + { + "epoch": 3.04, + "learning_rate": 0.00035003009027081245, + "loss": 2.7089, + "theoretical_loss": 3.644658036238184, + "tokens_seen": 1013167104 + }, + { + "epoch": 3.04, + "learning_rate": 0.00035002006018054163, + "loss": 2.7562, + "theoretical_loss": 3.644635655577472, + "tokens_seen": 1013232640 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003500100300902708, + "loss": 2.9357, + "theoretical_loss": 3.64461327676959, + "tokens_seen": 1013298176 + }, + { + "epoch": 3.04, + "learning_rate": 0.00035, + "loss": 2.6649, + "theoretical_loss": 3.6445908998142653, + "tokens_seen": 1013363712 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003499899699097292, + "loss": 2.4005, + "theoretical_loss": 3.6445685247112247, + "tokens_seen": 1013429248 + }, + { + "epoch": 3.04, + "learning_rate": 0.00034997993981945836, + "loss": 3.0307, + "theoretical_loss": 3.6445461514601947, + "tokens_seen": 1013494784 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003499699097291876, + "loss": 2.6786, + "theoretical_loss": 3.6445237800609025, + "tokens_seen": 1013560320 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003499598796389167, + "loss": 2.9899, + "theoretical_loss": 3.6445014105130755, + "tokens_seen": 1013625856 + }, + { + "epoch": 3.04, + "learning_rate": 0.00034994984954864596, + "loss": 2.7386, + "theoretical_loss": 3.64447904281644, + "tokens_seen": 1013691392 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003499398194583751, + "loss": 2.9648, + "theoretical_loss": 3.6444566769707243, + "tokens_seen": 1013756928 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003499297893681043, + "loss": 2.7837, + "theoretical_loss": 3.6444343129756547, + "tokens_seen": 1013822464 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003499197592778335, + "loss": 2.7852, + "theoretical_loss": 3.6444119508309583, + "tokens_seen": 1013888000 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003499097291875627, + "loss": 2.8927, + "theoretical_loss": 3.644389590536363, + "tokens_seen": 1013953536 + }, + { + "epoch": 3.04, + "learning_rate": 0.00034989969909729186, + "loss": 2.9017, + "theoretical_loss": 3.6443672320915956, + "tokens_seen": 1014019072 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003498896690070211, + "loss": 2.9671, + "theoretical_loss": 3.644344875496384, + "tokens_seen": 1014084608 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003498796389167502, + "loss": 2.7448, + "theoretical_loss": 3.644322520750456, + "tokens_seen": 1014150144 + }, + { + "epoch": 3.04, + "objective/train/docs_used": 1168150, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5092766284942627, + "objective/train/theoretical_loss": 3.6443169323528926, + "objective/train/tokens_used": 1034626528, + "theoretical_loss": 3.6443169323528926, + "tokens_seen": 1014166528 + }, + { + "epoch": 3.04, + "learning_rate": 0.00034986960882647946, + "loss": 2.6011, + "theoretical_loss": 3.644300167853538, + "tokens_seen": 1014215680 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003498595787362086, + "loss": 2.7115, + "theoretical_loss": 3.6442778168053587, + "tokens_seen": 1014281216 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003498495486459378, + "loss": 2.6693, + "theoretical_loss": 3.644255467605645, + "tokens_seen": 1014346752 + }, + { + "epoch": 3.04, + "learning_rate": 0.00034983951855566706, + "loss": 2.6503, + "theoretical_loss": 3.6442331202541256, + "tokens_seen": 1014412288 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003498294884653962, + "loss": 2.8226, + "theoretical_loss": 3.644210774750527, + "tokens_seen": 1014477824 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003498194583751254, + "loss": 2.8758, + "theoretical_loss": 3.6441884310945785, + "tokens_seen": 1014543360 + }, + { + "epoch": 3.04, + "learning_rate": 0.00034980942828485455, + "loss": 2.9343, + "theoretical_loss": 3.6441660892860064, + "tokens_seen": 1014608896 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003497993981945838, + "loss": 2.8126, + "theoretical_loss": 3.6441437493245394, + "tokens_seen": 1014674432 + }, + { + "epoch": 3.04, + "learning_rate": 0.00034978936810431296, + "loss": 2.891, + "theoretical_loss": 3.6441214112099054, + "tokens_seen": 1014739968 + }, + { + "epoch": 3.04, + "learning_rate": 0.00034977933801404214, + "loss": 2.9544, + "theoretical_loss": 3.644099074941833, + "tokens_seen": 1014805504 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003497693079237713, + "loss": 2.8482, + "theoretical_loss": 3.64407674052005, + "tokens_seen": 1014871040 + }, + { + "epoch": 3.04, + "learning_rate": 0.00034975927783350056, + "loss": 2.7608, + "theoretical_loss": 3.6440544079442843, + "tokens_seen": 1014936576 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003497492477432297, + "loss": 2.5706, + "theoretical_loss": 3.644032077214264, + "tokens_seen": 1015002112 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003497392176529589, + "loss": 2.6036, + "theoretical_loss": 3.644009748329718, + "tokens_seen": 1015067648 + }, + { + "epoch": 3.04, + "learning_rate": 0.00034972918756268805, + "loss": 2.8312, + "theoretical_loss": 3.6439874212903742, + "tokens_seen": 1015133184 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003497191574724173, + "loss": 2.8235, + "theoretical_loss": 3.643965096095961, + "tokens_seen": 1015198720 + }, + { + "epoch": 3.04, + "learning_rate": 0.00034970912738214647, + "loss": 2.8537, + "theoretical_loss": 3.6439427727462075, + "tokens_seen": 1015264256 + }, + { + "epoch": 3.04, + "learning_rate": 0.00034969909729187565, + "loss": 2.8265, + "theoretical_loss": 3.643920451240841, + "tokens_seen": 1015329792 + }, + { + "epoch": 3.04, + "learning_rate": 0.00034968906720160483, + "loss": 2.8553, + "theoretical_loss": 3.6438981315795917, + "tokens_seen": 1015395328 + }, + { + "epoch": 3.04, + "learning_rate": 0.000349679037111334, + "loss": 2.6262, + "theoretical_loss": 3.643875813762187, + "tokens_seen": 1015460864 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003496690070210632, + "loss": 2.93, + "theoretical_loss": 3.6438534977883554, + "tokens_seen": 1015526400 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003496589769307924, + "loss": 2.9229, + "theoretical_loss": 3.643831183657827, + "tokens_seen": 1015591936 + }, + { + "epoch": 3.04, + "learning_rate": 0.00034964894684052155, + "loss": 3.0048, + "theoretical_loss": 3.6438088713703296, + "tokens_seen": 1015657472 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003496389167502508, + "loss": 2.8082, + "theoretical_loss": 3.6437865609255917, + "tokens_seen": 1015723008 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003496288866599799, + "loss": 2.6069, + "theoretical_loss": 3.6437642523233436, + "tokens_seen": 1015788544 + }, + { + "epoch": 3.04, + "objective/train/docs_used": 1169618, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.381481885910034, + "objective/train/theoretical_loss": 3.6437586754606386, + "objective/train/tokens_used": 1036264928, + "theoretical_loss": 3.6437586754606386, + "tokens_seen": 1015804928 + }, + { + "epoch": 3.04, + "learning_rate": 0.00034961885656970915, + "loss": 2.815, + "theoretical_loss": 3.6437419455633133, + "tokens_seen": 1015854080 + }, + { + "epoch": 3.04, + "learning_rate": 0.00034960882647943833, + "loss": 2.7737, + "theoretical_loss": 3.64371964064523, + "tokens_seen": 1015919616 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003495987963891675, + "loss": 2.606, + "theoretical_loss": 3.6436973375688226, + "tokens_seen": 1015985152 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003495887662988967, + "loss": 2.8198, + "theoretical_loss": 3.6436750363338204, + "tokens_seen": 1016050688 + }, + { + "epoch": 3.04, + "learning_rate": 0.00034957873620862593, + "loss": 2.8019, + "theoretical_loss": 3.643652736939953, + "tokens_seen": 1016116224 + }, + { + "epoch": 3.04, + "learning_rate": 0.00034956870611835506, + "loss": 2.8078, + "theoretical_loss": 3.643630439386949, + "tokens_seen": 1016181760 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003495586760280843, + "loss": 2.7872, + "theoretical_loss": 3.6436081436745384, + "tokens_seen": 1016247296 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003495486459378134, + "loss": 2.7448, + "theoretical_loss": 3.6435858498024505, + "tokens_seen": 1016312832 + }, + { + "epoch": 3.04, + "learning_rate": 0.00034953861584754265, + "loss": 2.9181, + "theoretical_loss": 3.643563557770414, + "tokens_seen": 1016378368 + }, + { + "epoch": 3.04, + "learning_rate": 0.00034952858575727183, + "loss": 2.8828, + "theoretical_loss": 3.643541267578159, + "tokens_seen": 1016443904 + }, + { + "epoch": 3.04, + "learning_rate": 0.000349518555667001, + "loss": 2.9088, + "theoretical_loss": 3.643518979225415, + "tokens_seen": 1016509440 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003495085255767302, + "loss": 2.7972, + "theoretical_loss": 3.6434966927119117, + "tokens_seen": 1016574976 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003494984954864594, + "loss": 2.7734, + "theoretical_loss": 3.6434744080373784, + "tokens_seen": 1016640512 + }, + { + "epoch": 3.04, + "learning_rate": 0.00034948846539618856, + "loss": 2.6963, + "theoretical_loss": 3.643452125201545, + "tokens_seen": 1016706048 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003494784353059178, + "loss": 2.9125, + "theoretical_loss": 3.6434298442041415, + "tokens_seen": 1016771584 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003494684052156469, + "loss": 2.8687, + "theoretical_loss": 3.643407565044898, + "tokens_seen": 1016837120 + }, + { + "epoch": 3.04, + "learning_rate": 0.00034945837512537616, + "loss": 2.6719, + "theoretical_loss": 3.6433852877235435, + "tokens_seen": 1016902656 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003494483450351053, + "loss": 2.8081, + "theoretical_loss": 3.6433630122398086, + "tokens_seen": 1016968192 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003494383149448345, + "loss": 2.8221, + "theoretical_loss": 3.643340738593423, + "tokens_seen": 1017033728 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003494282848545637, + "loss": 2.6224, + "theoretical_loss": 3.643318466784117, + "tokens_seen": 1017099264 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003494182547642929, + "loss": 2.7745, + "theoretical_loss": 3.6432961968116206, + "tokens_seen": 1017164800 + }, + { + "epoch": 3.04, + "learning_rate": 0.00034940822467402206, + "loss": 2.7596, + "theoretical_loss": 3.643273928675664, + "tokens_seen": 1017230336 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003493981945837513, + "loss": 2.5713, + "theoretical_loss": 3.643251662375978, + "tokens_seen": 1017295872 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003493881644934804, + "loss": 2.7915, + "theoretical_loss": 3.643229397912292, + "tokens_seen": 1017361408 + }, + { + "epoch": 3.04, + "learning_rate": 0.00034937813440320966, + "loss": 2.6503, + "theoretical_loss": 3.6432071352843365, + "tokens_seen": 1017426944 + }, + { + "epoch": 3.04, + "objective/train/docs_used": 1170378, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.776099681854248, + "objective/train/theoretical_loss": 3.643201569914149, + "objective/train/tokens_used": 1037903328, + "theoretical_loss": 3.643201569914149, + "tokens_seen": 1017443328 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003493681043129388, + "loss": 3.0182, + "theoretical_loss": 3.6431848744918423, + "tokens_seen": 1017492480 + }, + { + "epoch": 3.04, + "learning_rate": 0.000349358074222668, + "loss": 2.9773, + "theoretical_loss": 3.6431626155345396, + "tokens_seen": 1017558016 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003493480441323972, + "loss": 2.822, + "theoretical_loss": 3.6431403584121593, + "tokens_seen": 1017623552 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003493380140421264, + "loss": 2.7286, + "theoretical_loss": 3.643118103124431, + "tokens_seen": 1017689088 + }, + { + "epoch": 3.04, + "learning_rate": 0.00034932798395185557, + "loss": 2.7543, + "theoretical_loss": 3.643095849671087, + "tokens_seen": 1017754624 + }, + { + "epoch": 3.04, + "learning_rate": 0.00034931795386158475, + "loss": 2.7698, + "theoretical_loss": 3.6430735980518563, + "tokens_seen": 1017820160 + }, + { + "epoch": 3.04, + "learning_rate": 0.00034930792377131393, + "loss": 2.4205, + "theoretical_loss": 3.6430513482664706, + "tokens_seen": 1017885696 + }, + { + "epoch": 3.04, + "learning_rate": 0.00034929789368104316, + "loss": 2.6192, + "theoretical_loss": 3.643029100314661, + "tokens_seen": 1017951232 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003492878635907723, + "loss": 2.8012, + "theoretical_loss": 3.643006854196157, + "tokens_seen": 1018016768 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003492778335005015, + "loss": 2.6011, + "theoretical_loss": 3.642984609910691, + "tokens_seen": 1018082304 + }, + { + "epoch": 3.04, + "learning_rate": 0.00034926780341023065, + "loss": 2.6748, + "theoretical_loss": 3.6429623674579927, + "tokens_seen": 1018147840 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003492577733199599, + "loss": 2.789, + "theoretical_loss": 3.642940126837795, + "tokens_seen": 1018213376 + }, + { + "epoch": 3.04, + "learning_rate": 0.00034924774322968907, + "loss": 2.691, + "theoretical_loss": 3.6429178880498263, + "tokens_seen": 1018278912 + }, + { + "epoch": 3.04, + "learning_rate": 0.00034923771313941825, + "loss": 2.813, + "theoretical_loss": 3.64289565109382, + "tokens_seen": 1018344448 + }, + { + "epoch": 3.04, + "learning_rate": 0.00034922768304914743, + "loss": 2.7174, + "theoretical_loss": 3.6428734159695066, + "tokens_seen": 1018409984 + }, + { + "epoch": 3.04, + "learning_rate": 0.00034921765295887667, + "loss": 2.7577, + "theoretical_loss": 3.6428511826766172, + "tokens_seen": 1018475520 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003492076228686058, + "loss": 2.6778, + "theoretical_loss": 3.6428289512148835, + "tokens_seen": 1018541056 + }, + { + "epoch": 3.04, + "learning_rate": 0.00034919759277833503, + "loss": 2.6804, + "theoretical_loss": 3.6428067215840363, + "tokens_seen": 1018606592 + }, + { + "epoch": 3.04, + "learning_rate": 0.00034918756268806416, + "loss": 2.679, + "theoretical_loss": 3.6427844937838074, + "tokens_seen": 1018672128 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003491775325977934, + "loss": 2.8358, + "theoretical_loss": 3.6427622678139286, + "tokens_seen": 1018737664 + }, + { + "epoch": 3.04, + "learning_rate": 0.00034916750250752257, + "loss": 2.7548, + "theoretical_loss": 3.642740043674131, + "tokens_seen": 1018803200 + }, + { + "epoch": 3.04, + "learning_rate": 0.00034915747241725175, + "loss": 2.7795, + "theoretical_loss": 3.642717821364146, + "tokens_seen": 1018868736 + }, + { + "epoch": 3.04, + "learning_rate": 0.00034914744232698093, + "loss": 2.7847, + "theoretical_loss": 3.6426956008837053, + "tokens_seen": 1018934272 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003491374122367101, + "loss": 2.8055, + "theoretical_loss": 3.642673382232541, + "tokens_seen": 1018999808 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003491273821464393, + "loss": 2.8508, + "theoretical_loss": 3.642651165410385, + "tokens_seen": 1019065344 + }, + { + "epoch": 3.04, + "objective/train/docs_used": 1171971, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9963715076446533, + "objective/train/theoretical_loss": 3.6426456114905967, + "objective/train/tokens_used": 1039541728, + "theoretical_loss": 3.6426456114905967, + "tokens_seen": 1019081728 + }, + { + "epoch": 3.04, + "learning_rate": 0.00034911735205616853, + "loss": 2.9141, + "theoretical_loss": 3.642628950416969, + "tokens_seen": 1019130880 + }, + { + "epoch": 3.04, + "learning_rate": 0.00034910732196589766, + "loss": 2.7184, + "theoretical_loss": 3.6426067372520246, + "tokens_seen": 1019196416 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003490972918756269, + "loss": 2.6736, + "theoretical_loss": 3.6425845259152836, + "tokens_seen": 1019261952 + }, + { + "epoch": 3.04, + "learning_rate": 0.00034908726178535613, + "loss": 2.6398, + "theoretical_loss": 3.642562316406478, + "tokens_seen": 1019327488 + }, + { + "epoch": 3.04, + "learning_rate": 0.00034907723169508526, + "loss": 2.7368, + "theoretical_loss": 3.642540108725341, + "tokens_seen": 1019393024 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003490672016048145, + "loss": 2.6245, + "theoretical_loss": 3.6425179028716035, + "tokens_seen": 1019458560 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003490571715145436, + "loss": 2.7568, + "theoretical_loss": 3.642495698844998, + "tokens_seen": 1019524096 + }, + { + "epoch": 3.04, + "learning_rate": 0.00034904714142427285, + "loss": 2.9744, + "theoretical_loss": 3.6424734966452568, + "tokens_seen": 1019589632 + }, + { + "epoch": 3.04, + "learning_rate": 0.00034903711133400203, + "loss": 2.8797, + "theoretical_loss": 3.642451296272112, + "tokens_seen": 1019655168 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003490270812437312, + "loss": 2.6659, + "theoretical_loss": 3.642429097725296, + "tokens_seen": 1019720704 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003490170511534604, + "loss": 2.8605, + "theoretical_loss": 3.642406901004542, + "tokens_seen": 1019786240 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003490070210631896, + "loss": 2.6881, + "theoretical_loss": 3.6423847061095813, + "tokens_seen": 1019851776 + }, + { + "epoch": 3.04, + "learning_rate": 0.00034899699097291876, + "loss": 2.6267, + "theoretical_loss": 3.6423625130401467, + "tokens_seen": 1019917312 + }, + { + "epoch": 3.04, + "learning_rate": 0.000348986960882648, + "loss": 2.8009, + "theoretical_loss": 3.6423403217959707, + "tokens_seen": 1019982848 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003489769307923771, + "loss": 2.8462, + "theoretical_loss": 3.6423181323767864, + "tokens_seen": 1020048384 + }, + { + "epoch": 3.04, + "learning_rate": 0.00034896690070210636, + "loss": 2.9541, + "theoretical_loss": 3.642295944782326, + "tokens_seen": 1020113920 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003489568706118355, + "loss": 2.6883, + "theoretical_loss": 3.6422737590123226, + "tokens_seen": 1020179456 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003489468405215647, + "loss": 2.6475, + "theoretical_loss": 3.6422515750665085, + "tokens_seen": 1020244992 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003489368104312939, + "loss": 2.7722, + "theoretical_loss": 3.6422293929446172, + "tokens_seen": 1020310528 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003489267803410231, + "loss": 2.7078, + "theoretical_loss": 3.6422072126463814, + "tokens_seen": 1020376064 + }, + { + "epoch": 3.04, + "learning_rate": 0.00034891675025075226, + "loss": 2.7822, + "theoretical_loss": 3.6421850341715336, + "tokens_seen": 1020441600 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003489067201604815, + "loss": 2.6382, + "theoretical_loss": 3.642162857519807, + "tokens_seen": 1020507136 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003488966900702106, + "loss": 2.6841, + "theoretical_loss": 3.6421406826909353, + "tokens_seen": 1020572672 + }, + { + "epoch": 3.04, + "learning_rate": 0.00034888665997993986, + "loss": 2.8371, + "theoretical_loss": 3.642118509684651, + "tokens_seen": 1020638208 + }, + { + "epoch": 3.04, + "learning_rate": 0.000348876629889669, + "loss": 2.7937, + "theoretical_loss": 3.642096338500687, + "tokens_seen": 1020703744 + }, + { + "epoch": 3.04, + "objective/train/docs_used": 1172849, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6258201599121094, + "objective/train/theoretical_loss": 3.6420907959894024, + "objective/train/tokens_used": 1041180128, + "theoretical_loss": 3.6420907959894024, + "tokens_seen": 1020720128 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003488665997993982, + "loss": 2.9837, + "theoretical_loss": 3.6420741691387772, + "tokens_seen": 1020769280 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003488565697091274, + "loss": 2.67, + "theoretical_loss": 3.642052001598655, + "tokens_seen": 1020834816 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003488465396188566, + "loss": 2.7801, + "theoretical_loss": 3.6420298358800527, + "tokens_seen": 1020900352 + }, + { + "epoch": 3.04, + "learning_rate": 0.00034883650952858577, + "loss": 2.6078, + "theoretical_loss": 3.6420076719827046, + "tokens_seen": 1020965888 + }, + { + "epoch": 3.04, + "learning_rate": 0.00034882647943831495, + "loss": 2.8249, + "theoretical_loss": 3.6419855099063443, + "tokens_seen": 1021031424 + }, + { + "epoch": 3.04, + "learning_rate": 0.00034881644934804413, + "loss": 2.9316, + "theoretical_loss": 3.641963349650705, + "tokens_seen": 1021096960 + }, + { + "epoch": 3.04, + "learning_rate": 0.00034880641925777336, + "loss": 2.6782, + "theoretical_loss": 3.64194119121552, + "tokens_seen": 1021162496 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003487963891675025, + "loss": 2.8126, + "theoretical_loss": 3.641919034600523, + "tokens_seen": 1021228032 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003487863590772317, + "loss": 2.7651, + "theoretical_loss": 3.6418968798054485, + "tokens_seen": 1021293568 + }, + { + "epoch": 3.04, + "learning_rate": 0.00034877632898696085, + "loss": 2.8099, + "theoretical_loss": 3.6418747268300296, + "tokens_seen": 1021359104 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003487662988966901, + "loss": 2.6514, + "theoretical_loss": 3.6418525756739992, + "tokens_seen": 1021424640 + }, + { + "epoch": 3.04, + "learning_rate": 0.00034875626880641927, + "loss": 2.7745, + "theoretical_loss": 3.6418304263370933, + "tokens_seen": 1021490176 + }, + { + "epoch": 3.04, + "learning_rate": 0.00034874623871614845, + "loss": 2.7871, + "theoretical_loss": 3.6418082788190436, + "tokens_seen": 1021555712 + }, + { + "epoch": 3.04, + "learning_rate": 0.00034873620862587763, + "loss": 2.4823, + "theoretical_loss": 3.6417861331195853, + "tokens_seen": 1021621248 + }, + { + "epoch": 3.04, + "learning_rate": 0.00034872617853560687, + "loss": 2.7061, + "theoretical_loss": 3.6417639892384526, + "tokens_seen": 1021686784 + }, + { + "epoch": 3.04, + "learning_rate": 0.000348716148445336, + "loss": 2.7506, + "theoretical_loss": 3.6417418471753793, + "tokens_seen": 1021752320 + }, + { + "epoch": 3.04, + "learning_rate": 0.00034870611835506523, + "loss": 2.7926, + "theoretical_loss": 3.6417197069300986, + "tokens_seen": 1021817856 + }, + { + "epoch": 3.04, + "learning_rate": 0.00034869608826479436, + "loss": 2.745, + "theoretical_loss": 3.6416975685023463, + "tokens_seen": 1021883392 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003486860581745236, + "loss": 2.4741, + "theoretical_loss": 3.6416754318918554, + "tokens_seen": 1021948928 + }, + { + "epoch": 3.04, + "learning_rate": 0.00034867602808425277, + "loss": 2.8679, + "theoretical_loss": 3.641653297098361, + "tokens_seen": 1022014464 + }, + { + "epoch": 3.04, + "learning_rate": 0.00034866599799398195, + "loss": 2.9177, + "theoretical_loss": 3.6416311641215966, + "tokens_seen": 1022080000 + }, + { + "epoch": 3.04, + "learning_rate": 0.00034865596790371113, + "loss": 2.7663, + "theoretical_loss": 3.6416090329612976, + "tokens_seen": 1022145536 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003486459378134403, + "loss": 2.794, + "theoretical_loss": 3.6415869036171973, + "tokens_seen": 1022211072 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003486359077231695, + "loss": 2.6762, + "theoretical_loss": 3.641564776089032, + "tokens_seen": 1022276608 + }, + { + "epoch": 3.04, + "learning_rate": 0.00034862587763289873, + "loss": 2.6958, + "theoretical_loss": 3.6415426503765342, + "tokens_seen": 1022342144 + }, + { + "epoch": 3.04, + "objective/train/docs_used": 1174219, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4945321083068848, + "objective/train/theoretical_loss": 3.641537119232077, + "objective/train/tokens_used": 1042818528, + "theoretical_loss": 3.641537119232077, + "tokens_seen": 1022358528 + }, + { + "epoch": 3.04, + "learning_rate": 0.00034861584754262786, + "loss": 2.6434, + "theoretical_loss": 3.6415205264794404, + "tokens_seen": 1022407680 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003486058174523571, + "loss": 2.8606, + "theoretical_loss": 3.641498404397484, + "tokens_seen": 1022473216 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003485957873620862, + "loss": 2.6391, + "theoretical_loss": 3.6414762841304, + "tokens_seen": 1022538752 + }, + { + "epoch": 3.04, + "learning_rate": 0.00034858575727181546, + "loss": 2.6538, + "theoretical_loss": 3.6414541656779233, + "tokens_seen": 1022604288 + }, + { + "epoch": 3.04, + "learning_rate": 0.00034857572718154464, + "loss": 2.664, + "theoretical_loss": 3.6414320490397896, + "tokens_seen": 1022669824 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003485656970912738, + "loss": 2.7767, + "theoretical_loss": 3.6414099342157327, + "tokens_seen": 1022735360 + }, + { + "epoch": 3.04, + "learning_rate": 0.000348555667001003, + "loss": 2.6541, + "theoretical_loss": 3.6413878212054875, + "tokens_seen": 1022800896 + }, + { + "epoch": 3.04, + "learning_rate": 0.00034854563691073223, + "loss": 2.902, + "theoretical_loss": 3.64136571000879, + "tokens_seen": 1022866432 + }, + { + "epoch": 3.04, + "learning_rate": 0.00034853560682046136, + "loss": 2.764, + "theoretical_loss": 3.641343600625375, + "tokens_seen": 1022931968 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003485255767301906, + "loss": 2.7848, + "theoretical_loss": 3.641321493054977, + "tokens_seen": 1022997504 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003485155466399197, + "loss": 2.6366, + "theoretical_loss": 3.6412993872973316, + "tokens_seen": 1023063040 + }, + { + "epoch": 3.04, + "learning_rate": 0.00034850551654964896, + "loss": 2.6445, + "theoretical_loss": 3.6412772833521743, + "tokens_seen": 1023128576 + }, + { + "epoch": 3.04, + "learning_rate": 0.00034849548645937814, + "loss": 2.8639, + "theoretical_loss": 3.6412551812192397, + "tokens_seen": 1023194112 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003484854563691073, + "loss": 2.8851, + "theoretical_loss": 3.641233080898264, + "tokens_seen": 1023259648 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003484754262788365, + "loss": 2.7964, + "theoretical_loss": 3.6412109823889818, + "tokens_seen": 1023325184 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003484653961885657, + "loss": 2.8088, + "theoretical_loss": 3.64118888569113, + "tokens_seen": 1023390720 + }, + { + "epoch": 3.04, + "learning_rate": 0.00034845536609829486, + "loss": 2.6254, + "theoretical_loss": 3.641166790804442, + "tokens_seen": 1023456256 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003484453360080241, + "loss": 2.6364, + "theoretical_loss": 3.641144697728655, + "tokens_seen": 1023521792 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003484353059177532, + "loss": 2.822, + "theoretical_loss": 3.6411226064635045, + "tokens_seen": 1023587328 + }, + { + "epoch": 3.04, + "learning_rate": 0.00034842527582748246, + "loss": 2.6328, + "theoretical_loss": 3.6411005170087254, + "tokens_seen": 1023652864 + }, + { + "epoch": 3.04, + "learning_rate": 0.00034841524573721164, + "loss": 2.8881, + "theoretical_loss": 3.641078429364054, + "tokens_seen": 1023718400 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003484052156469408, + "loss": 2.8342, + "theoretical_loss": 3.6410563435292262, + "tokens_seen": 1023783936 + }, + { + "epoch": 3.04, + "learning_rate": 0.00034839518555667, + "loss": 2.63, + "theoretical_loss": 3.6410342595039777, + "tokens_seen": 1023849472 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003483851554663992, + "loss": 2.7483, + "theoretical_loss": 3.6410121772880446, + "tokens_seen": 1023915008 + }, + { + "epoch": 3.04, + "learning_rate": 0.00034837512537612837, + "loss": 2.6859, + "theoretical_loss": 3.640990096881162, + "tokens_seen": 1023980544 + }, + { + "epoch": 3.04, + "objective/train/docs_used": 1174839, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9092800617218018, + "objective/train/theoretical_loss": 3.640984577062075, + "objective/train/tokens_used": 1044456928, + "theoretical_loss": 3.640984577062075, + "tokens_seen": 1023996928 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003483650952858576, + "loss": 2.8045, + "theoretical_loss": 3.6409680182830675, + "tokens_seen": 1024046080 + }, + { + "epoch": 3.04, + "learning_rate": 0.00034835506519558673, + "loss": 2.8116, + "theoretical_loss": 3.6409459414934955, + "tokens_seen": 1024111616 + }, + { + "epoch": 3.04, + "learning_rate": 0.00034834503510531597, + "loss": 2.7341, + "theoretical_loss": 3.6409238665121832, + "tokens_seen": 1024177152 + }, + { + "epoch": 3.04, + "learning_rate": 0.00034833500501504515, + "loss": 2.4435, + "theoretical_loss": 3.640901793338867, + "tokens_seen": 1024242688 + }, + { + "epoch": 3.04, + "learning_rate": 0.00034832497492477433, + "loss": 2.6926, + "theoretical_loss": 3.640879721973282, + "tokens_seen": 1024308224 + }, + { + "epoch": 3.04, + "learning_rate": 0.00034831494483450356, + "loss": 2.8559, + "theoretical_loss": 3.6408576524151655, + "tokens_seen": 1024373760 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003483049147442327, + "loss": 2.7453, + "theoretical_loss": 3.6408355846642535, + "tokens_seen": 1024439296 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003482948846539619, + "loss": 2.7137, + "theoretical_loss": 3.6408135187202824, + "tokens_seen": 1024504832 + }, + { + "epoch": 3.04, + "learning_rate": 0.00034828485456369105, + "loss": 2.573, + "theoretical_loss": 3.640791454582989, + "tokens_seen": 1024570368 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003482748244734203, + "loss": 2.6886, + "theoretical_loss": 3.640769392252109, + "tokens_seen": 1024635904 + }, + { + "epoch": 3.04, + "learning_rate": 0.00034826479438314947, + "loss": 2.9169, + "theoretical_loss": 3.64074733172738, + "tokens_seen": 1024701440 + }, + { + "epoch": 3.04, + "learning_rate": 0.00034825476429287865, + "loss": 2.6807, + "theoretical_loss": 3.640725273008538, + "tokens_seen": 1024766976 + }, + { + "epoch": 3.04, + "learning_rate": 0.00034824473420260783, + "loss": 2.8173, + "theoretical_loss": 3.64070321609532, + "tokens_seen": 1024832512 + }, + { + "epoch": 3.04, + "learning_rate": 0.00034823470411233707, + "loss": 2.8759, + "theoretical_loss": 3.6406811609874628, + "tokens_seen": 1024898048 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003482246740220662, + "loss": 2.6937, + "theoretical_loss": 3.640659107684703, + "tokens_seen": 1024963584 + }, + { + "epoch": 3.04, + "learning_rate": 0.00034821464393179543, + "loss": 2.8059, + "theoretical_loss": 3.640637056186777, + "tokens_seen": 1025029120 + }, + { + "epoch": 3.04, + "learning_rate": 0.00034820461384152456, + "loss": 2.7328, + "theoretical_loss": 3.6406150064934226, + "tokens_seen": 1025094656 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003481945837512538, + "loss": 2.8426, + "theoretical_loss": 3.6405929586043766, + "tokens_seen": 1025160192 + }, + { + "epoch": 3.04, + "learning_rate": 0.00034818455366098297, + "loss": 2.5166, + "theoretical_loss": 3.640570912519375, + "tokens_seen": 1025225728 + }, + { + "epoch": 3.04, + "learning_rate": 0.00034817452357071215, + "loss": 2.7644, + "theoretical_loss": 3.640548868238157, + "tokens_seen": 1025291264 + }, + { + "epoch": 3.04, + "learning_rate": 0.00034816449348044133, + "loss": 2.7304, + "theoretical_loss": 3.640526825760457, + "tokens_seen": 1025356800 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003481544633901705, + "loss": 2.6769, + "theoretical_loss": 3.6405047850860144, + "tokens_seen": 1025422336 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003481444332998997, + "loss": 2.8822, + "theoretical_loss": 3.6404827462145652, + "tokens_seen": 1025487872 + }, + { + "epoch": 3.04, + "learning_rate": 0.00034813440320962893, + "loss": 2.8437, + "theoretical_loss": 3.6404607091458474, + "tokens_seen": 1025553408 + }, + { + "epoch": 3.04, + "learning_rate": 0.00034812437311935806, + "loss": 2.6955, + "theoretical_loss": 3.640438673879598, + "tokens_seen": 1025618944 + }, + { + "epoch": 3.04, + "objective/train/docs_used": 1176332, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5118682384490967, + "objective/train/theoretical_loss": 3.640433165344641, + "objective/train/tokens_used": 1046095328, + "theoretical_loss": 3.640433165344641, + "tokens_seen": 1025635328 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003481143430290873, + "loss": 2.6492, + "theoretical_loss": 3.6404166404155545, + "tokens_seen": 1025684480 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003481043129388164, + "loss": 2.468, + "theoretical_loss": 3.6403946087534544, + "tokens_seen": 1025750016 + }, + { + "epoch": 3.04, + "learning_rate": 0.00034809428284854566, + "loss": 2.7733, + "theoretical_loss": 3.6403725788930354, + "tokens_seen": 1025815552 + }, + { + "epoch": 3.04, + "learning_rate": 0.00034808425275827484, + "loss": 2.8966, + "theoretical_loss": 3.6403505508340346, + "tokens_seen": 1025881088 + }, + { + "epoch": 3.04, + "learning_rate": 0.000348074222668004, + "loss": 2.8089, + "theoretical_loss": 3.6403285245761903, + "tokens_seen": 1025946624 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003480641925777332, + "loss": 2.7961, + "theoretical_loss": 3.640306500119239, + "tokens_seen": 1026012160 + }, + { + "epoch": 3.04, + "learning_rate": 0.00034805416248746243, + "loss": 2.691, + "theoretical_loss": 3.64028447746292, + "tokens_seen": 1026077696 + }, + { + "epoch": 3.04, + "learning_rate": 0.00034804413239719156, + "loss": 2.7155, + "theoretical_loss": 3.64026245660697, + "tokens_seen": 1026143232 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003480341023069208, + "loss": 2.7533, + "theoretical_loss": 3.6402404375511273, + "tokens_seen": 1026208768 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003480240722166499, + "loss": 2.8656, + "theoretical_loss": 3.6402184202951293, + "tokens_seen": 1026274304 + }, + { + "epoch": 3.04, + "learning_rate": 0.00034801404212637916, + "loss": 2.9105, + "theoretical_loss": 3.640196404838715, + "tokens_seen": 1026339840 + }, + { + "epoch": 3.04, + "learning_rate": 0.00034800401203610834, + "loss": 2.545, + "theoretical_loss": 3.6401743911816213, + "tokens_seen": 1026405376 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003479939819458375, + "loss": 2.7461, + "theoretical_loss": 3.6401523793235864, + "tokens_seen": 1026470912 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003479839518555667, + "loss": 2.6097, + "theoretical_loss": 3.640130369264349, + "tokens_seen": 1026536448 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003479739217652959, + "loss": 2.6789, + "theoretical_loss": 3.640108361003647, + "tokens_seen": 1026601984 + }, + { + "epoch": 3.04, + "learning_rate": 0.00034796389167502506, + "loss": 2.8538, + "theoretical_loss": 3.6400863545412188, + "tokens_seen": 1026667520 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003479538615847543, + "loss": 2.9017, + "theoretical_loss": 3.640064349876803, + "tokens_seen": 1026733056 + }, + { + "epoch": 3.04, + "learning_rate": 0.00034794383149448343, + "loss": 2.6433, + "theoretical_loss": 3.6400423470101364, + "tokens_seen": 1026798592 + }, + { + "epoch": 3.04, + "learning_rate": 0.00034793380140421266, + "loss": 2.8106, + "theoretical_loss": 3.6400203459409592, + "tokens_seen": 1026864128 + }, + { + "epoch": 3.04, + "learning_rate": 0.00034792377131394184, + "loss": 2.5955, + "theoretical_loss": 3.6399983466690085, + "tokens_seen": 1026929664 + }, + { + "epoch": 3.04, + "learning_rate": 0.000347913741223671, + "loss": 2.7773, + "theoretical_loss": 3.639976349194024, + "tokens_seen": 1026995200 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003479037111334002, + "loss": 2.7691, + "theoretical_loss": 3.6399543535157433, + "tokens_seen": 1027060736 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003478936810431294, + "loss": 2.9683, + "theoretical_loss": 3.639932359633906, + "tokens_seen": 1027126272 + }, + { + "epoch": 3.04, + "learning_rate": 0.00034788365095285857, + "loss": 2.7609, + "theoretical_loss": 3.63991036754825, + "tokens_seen": 1027191808 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003478736208625878, + "loss": 2.8229, + "theoretical_loss": 3.639888377258514, + "tokens_seen": 1027257344 + }, + { + "epoch": 3.04, + "objective/train/docs_used": 1176787, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9666268825531006, + "objective/train/theoretical_loss": 3.639882879966662, + "objective/train/tokens_used": 1047733728, + "theoretical_loss": 3.639882879966662, + "tokens_seen": 1027273728 + }, + { + "epoch": 3.04, + "learning_rate": 0.00034786359077231693, + "loss": 2.7606, + "theoretical_loss": 3.639866388764437, + "tokens_seen": 1027322880 + }, + { + "epoch": 3.04, + "learning_rate": 0.00034785356068204617, + "loss": 2.8858, + "theoretical_loss": 3.6398444020657577, + "tokens_seen": 1027388416 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003478435305917753, + "loss": 2.8103, + "theoretical_loss": 3.6398224171622156, + "tokens_seen": 1027453952 + }, + { + "epoch": 3.04, + "learning_rate": 0.00034783350050150453, + "loss": 2.8517, + "theoretical_loss": 3.639800434053549, + "tokens_seen": 1027519488 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003478234704112337, + "loss": 2.7297, + "theoretical_loss": 3.6397784527394967, + "tokens_seen": 1027585024 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003478134403209629, + "loss": 3.0084, + "theoretical_loss": 3.6397564732197987, + "tokens_seen": 1027650560 + }, + { + "epoch": 3.04, + "learning_rate": 0.00034780341023069207, + "loss": 2.8202, + "theoretical_loss": 3.639734495494193, + "tokens_seen": 1027716096 + }, + { + "epoch": 3.04, + "learning_rate": 0.00034779338014042125, + "loss": 2.7186, + "theoretical_loss": 3.6397125195624196, + "tokens_seen": 1027781632 + }, + { + "epoch": 3.04, + "learning_rate": 0.00034778335005015043, + "loss": 2.8238, + "theoretical_loss": 3.6396905454242177, + "tokens_seen": 1027847168 + }, + { + "epoch": 3.04, + "learning_rate": 0.00034777331995987967, + "loss": 2.8337, + "theoretical_loss": 3.6396685730793257, + "tokens_seen": 1027912704 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003477632898696088, + "loss": 2.7286, + "theoretical_loss": 3.6396466025274834, + "tokens_seen": 1027978240 + }, + { + "epoch": 3.04, + "learning_rate": 0.00034775325977933803, + "loss": 2.846, + "theoretical_loss": 3.6396246337684306, + "tokens_seen": 1028043776 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003477432296890672, + "loss": 2.7812, + "theoretical_loss": 3.639602666801906, + "tokens_seen": 1028109312 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003477331995987964, + "loss": 2.6417, + "theoretical_loss": 3.6395807016276502, + "tokens_seen": 1028174848 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003477231695085256, + "loss": 2.6741, + "theoretical_loss": 3.6395587382454018, + "tokens_seen": 1028240384 + }, + { + "epoch": 3.04, + "learning_rate": 0.00034771313941825476, + "loss": 2.9862, + "theoretical_loss": 3.639536776654901, + "tokens_seen": 1028305920 + }, + { + "epoch": 3.04, + "learning_rate": 0.00034770310932798394, + "loss": 2.9435, + "theoretical_loss": 3.6395148168558866, + "tokens_seen": 1028371456 + }, + { + "epoch": 3.04, + "learning_rate": 0.00034769307923771317, + "loss": 2.5491, + "theoretical_loss": 3.639492858848099, + "tokens_seen": 1028436992 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003476830491474423, + "loss": 2.8525, + "theoretical_loss": 3.6394709026312775, + "tokens_seen": 1028502528 + }, + { + "epoch": 3.04, + "learning_rate": 0.00034767301905717153, + "loss": 2.7869, + "theoretical_loss": 3.6394489482051626, + "tokens_seen": 1028568064 + }, + { + "epoch": 3.04, + "learning_rate": 0.00034766298896690066, + "loss": 2.9154, + "theoretical_loss": 3.639426995569494, + "tokens_seen": 1028633600 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003476529588766299, + "loss": 2.6809, + "theoretical_loss": 3.6394050447240107, + "tokens_seen": 1028699136 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003476429287863591, + "loss": 2.9911, + "theoretical_loss": 3.6393830956684536, + "tokens_seen": 1028764672 + }, + { + "epoch": 3.04, + "learning_rate": 0.00034763289869608826, + "loss": 2.6227, + "theoretical_loss": 3.6393611484025628, + "tokens_seen": 1028830208 + }, + { + "epoch": 3.04, + "learning_rate": 0.00034762286860581744, + "loss": 2.775, + "theoretical_loss": 3.639339202926078, + "tokens_seen": 1028895744 + }, + { + "epoch": 3.04, + "objective/train/docs_used": 1178361, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4620256423950195, + "objective/train/theoretical_loss": 3.6393337168365205, + "objective/train/tokens_used": 1049372128, + "theoretical_loss": 3.6393337168365205, + "tokens_seen": 1028912128 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003476128385155466, + "loss": 2.7881, + "theoretical_loss": 3.639317259238739, + "tokens_seen": 1028961280 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003476028084252758, + "loss": 2.8292, + "theoretical_loss": 3.639295317340287, + "tokens_seen": 1029026816 + }, + { + "epoch": 3.04, + "learning_rate": 0.00034759277833500504, + "loss": 2.6422, + "theoretical_loss": 3.6392733772304613, + "tokens_seen": 1029092352 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003475827482447342, + "loss": 2.971, + "theoretical_loss": 3.639251438909003, + "tokens_seen": 1029157888 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003475727181544634, + "loss": 2.6485, + "theoretical_loss": 3.6392295023756516, + "tokens_seen": 1029223424 + }, + { + "epoch": 3.04, + "learning_rate": 0.00034756268806419263, + "loss": 2.9472, + "theoretical_loss": 3.6392075676301485, + "tokens_seen": 1029288960 + }, + { + "epoch": 3.04, + "learning_rate": 0.00034755265797392176, + "loss": 2.6777, + "theoretical_loss": 3.6391856346722333, + "tokens_seen": 1029354496 + }, + { + "epoch": 3.04, + "learning_rate": 0.000347542627883651, + "loss": 2.6996, + "theoretical_loss": 3.639163703501647, + "tokens_seen": 1029420032 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003475325977933801, + "loss": 2.8, + "theoretical_loss": 3.63914177411813, + "tokens_seen": 1029485568 + }, + { + "epoch": 3.04, + "learning_rate": 0.00034752256770310936, + "loss": 2.7562, + "theoretical_loss": 3.6391198465214227, + "tokens_seen": 1029551104 + }, + { + "epoch": 3.04, + "learning_rate": 0.00034751253761283854, + "loss": 2.6978, + "theoretical_loss": 3.6390979207112664, + "tokens_seen": 1029616640 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003475025075225677, + "loss": 2.6952, + "theoretical_loss": 3.639075996687401, + "tokens_seen": 1029682176 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003474924774322969, + "loss": 2.7032, + "theoretical_loss": 3.6390540744495685, + "tokens_seen": 1029747712 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003474824473420261, + "loss": 2.6098, + "theoretical_loss": 3.639032153997509, + "tokens_seen": 1029813248 + }, + { + "epoch": 3.04, + "learning_rate": 0.00034747241725175527, + "loss": 2.9094, + "theoretical_loss": 3.6390102353309626, + "tokens_seen": 1029878784 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003474623871614845, + "loss": 2.6849, + "theoretical_loss": 3.6389883184496723, + "tokens_seen": 1029944320 + }, + { + "epoch": 3.04, + "learning_rate": 0.00034745235707121363, + "loss": 2.8267, + "theoretical_loss": 3.638966403353377, + "tokens_seen": 1030009856 + }, + { + "epoch": 3.04, + "learning_rate": 0.00034744232698094286, + "loss": 2.8079, + "theoretical_loss": 3.6389444900418186, + "tokens_seen": 1030075392 + }, + { + "epoch": 3.04, + "learning_rate": 0.00034743229689067204, + "loss": 2.6925, + "theoretical_loss": 3.6389225785147388, + "tokens_seen": 1030140928 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003474222668004012, + "loss": 2.8071, + "theoretical_loss": 3.638900668771878, + "tokens_seen": 1030206464 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003474122367101304, + "loss": 2.8804, + "theoretical_loss": 3.638878760812978, + "tokens_seen": 1030272000 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003474022066198596, + "loss": 2.6228, + "theoretical_loss": 3.6388568546377793, + "tokens_seen": 1030337536 + }, + { + "epoch": 3.04, + "learning_rate": 0.00034739217652958877, + "loss": 2.6782, + "theoretical_loss": 3.6388349502460238, + "tokens_seen": 1030403072 + }, + { + "epoch": 3.04, + "learning_rate": 0.000347382146439318, + "loss": 2.8143, + "theoretical_loss": 3.638813047637453, + "tokens_seen": 1030468608 + }, + { + "epoch": 3.04, + "learning_rate": 0.00034737211634904713, + "loss": 2.5234, + "theoretical_loss": 3.6387911468118075, + "tokens_seen": 1030534144 + }, + { + "epoch": 3.04, + "objective/train/docs_used": 1179088, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.1466615200042725, + "objective/train/theoretical_loss": 3.6387856718839484, + "objective/train/tokens_used": 1051010528, + "theoretical_loss": 3.6387856718839484, + "tokens_seen": 1030550528 + }, + { + "epoch": 3.04, + "learning_rate": 0.00034736208625877637, + "loss": 2.859, + "theoretical_loss": 3.6387692477688303, + "tokens_seen": 1030599680 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003473520561685055, + "loss": 2.7063, + "theoretical_loss": 3.6387473505082615, + "tokens_seen": 1030665216 + }, + { + "epoch": 3.04, + "learning_rate": 0.00034734202607823473, + "loss": 2.4725, + "theoretical_loss": 3.6387254550298436, + "tokens_seen": 1030730752 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003473319959879639, + "loss": 2.8637, + "theoretical_loss": 3.6387035613333176, + "tokens_seen": 1030796288 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003473219658976931, + "loss": 2.9199, + "theoretical_loss": 3.6386816694184256, + "tokens_seen": 1030861824 + }, + { + "epoch": 3.04, + "learning_rate": 0.00034731193580742227, + "loss": 2.8224, + "theoretical_loss": 3.6386597792849096, + "tokens_seen": 1030927360 + }, + { + "epoch": 3.04, + "learning_rate": 0.00034730190571715145, + "loss": 2.6093, + "theoretical_loss": 3.638637890932511, + "tokens_seen": 1030992896 + }, + { + "epoch": 3.04, + "learning_rate": 0.00034729187562688063, + "loss": 2.783, + "theoretical_loss": 3.638616004360972, + "tokens_seen": 1031058432 + }, + { + "epoch": 3.04, + "learning_rate": 0.00034728184553660987, + "loss": 2.7837, + "theoretical_loss": 3.6385941195700338, + "tokens_seen": 1031123968 + }, + { + "epoch": 3.04, + "learning_rate": 0.000347271815446339, + "loss": 2.644, + "theoretical_loss": 3.638572236559439, + "tokens_seen": 1031189504 + }, + { + "epoch": 3.04, + "learning_rate": 0.00034726178535606823, + "loss": 2.708, + "theoretical_loss": 3.63855035532893, + "tokens_seen": 1031255040 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003472517552657974, + "loss": 2.8875, + "theoretical_loss": 3.638528475878248, + "tokens_seen": 1031320576 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003472417251755266, + "loss": 2.712, + "theoretical_loss": 3.6385065982071354, + "tokens_seen": 1031386112 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003472316950852558, + "loss": 2.7669, + "theoretical_loss": 3.6384847223153347, + "tokens_seen": 1031451648 + }, + { + "epoch": 3.04, + "learning_rate": 0.00034722166499498496, + "loss": 2.8086, + "theoretical_loss": 3.6384628482025887, + "tokens_seen": 1031517184 + }, + { + "epoch": 3.04, + "learning_rate": 0.00034721163490471414, + "loss": 2.6404, + "theoretical_loss": 3.6384409758686385, + "tokens_seen": 1031582720 + }, + { + "epoch": 3.04, + "learning_rate": 0.00034720160481444337, + "loss": 2.6375, + "theoretical_loss": 3.6384191053132264, + "tokens_seen": 1031648256 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003471915747241725, + "loss": 2.8026, + "theoretical_loss": 3.638397236536096, + "tokens_seen": 1031713792 + }, + { + "epoch": 3.04, + "learning_rate": 0.00034718154463390173, + "loss": 2.5592, + "theoretical_loss": 3.6383753695369894, + "tokens_seen": 1031779328 + }, + { + "epoch": 3.04, + "learning_rate": 0.00034717151454363086, + "loss": 2.6478, + "theoretical_loss": 3.6383535043156483, + "tokens_seen": 1031844864 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003471614844533601, + "loss": 2.977, + "theoretical_loss": 3.638331640871816, + "tokens_seen": 1031910400 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003471514543630893, + "loss": 2.4206, + "theoretical_loss": 3.638309779205235, + "tokens_seen": 1031975936 + }, + { + "epoch": 3.04, + "learning_rate": 0.00034714142427281846, + "loss": 2.4946, + "theoretical_loss": 3.6382879193156485, + "tokens_seen": 1032041472 + }, + { + "epoch": 3.04, + "learning_rate": 0.00034713139418254764, + "loss": 2.6483, + "theoretical_loss": 3.6382660612027977, + "tokens_seen": 1032107008 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003471213640922768, + "loss": 2.6798, + "theoretical_loss": 3.638244204866427, + "tokens_seen": 1032172544 + }, + { + "epoch": 3.04, + "objective/train/docs_used": 1180390, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7738821506500244, + "objective/train/theoretical_loss": 3.6382387410598787, + "objective/train/tokens_used": 1052648928, + "theoretical_loss": 3.6382387410598787, + "tokens_seen": 1032188928 + }, + { + "epoch": 3.04, + "learning_rate": 0.000347111334002006, + "loss": 2.8191, + "theoretical_loss": 3.638222350306278, + "tokens_seen": 1032238080 + }, + { + "epoch": 3.04, + "learning_rate": 0.00034710130391173524, + "loss": 2.733, + "theoretical_loss": 3.638200497522095, + "tokens_seen": 1032303616 + }, + { + "epoch": 3.04, + "learning_rate": 0.00034709127382146436, + "loss": 2.7866, + "theoretical_loss": 3.6381786465136194, + "tokens_seen": 1032369152 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003470812437311936, + "loss": 2.7378, + "theoretical_loss": 3.6381567972805957, + "tokens_seen": 1032434688 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003470712136409228, + "loss": 2.5182, + "theoretical_loss": 3.6381349498227653, + "tokens_seen": 1032500224 + }, + { + "epoch": 3.04, + "learning_rate": 0.00034706118355065196, + "loss": 2.5097, + "theoretical_loss": 3.6381131041398724, + "tokens_seen": 1032565760 + }, + { + "epoch": 3.04, + "learning_rate": 0.00034705115346038114, + "loss": 2.8444, + "theoretical_loss": 3.6380912602316604, + "tokens_seen": 1032631296 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003470411233701103, + "loss": 2.9808, + "theoretical_loss": 3.6380694180978717, + "tokens_seen": 1032696832 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003470310932798395, + "loss": 2.7574, + "theoretical_loss": 3.6380475777382504, + "tokens_seen": 1032762368 + }, + { + "epoch": 3.04, + "learning_rate": 0.00034702106318956874, + "loss": 2.7757, + "theoretical_loss": 3.638025739152539, + "tokens_seen": 1032827904 + }, + { + "epoch": 3.04, + "learning_rate": 0.00034701103309929787, + "loss": 2.676, + "theoretical_loss": 3.638003902340481, + "tokens_seen": 1032893440 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003470010030090271, + "loss": 2.6281, + "theoretical_loss": 3.63798206730182, + "tokens_seen": 1032958976 + }, + { + "epoch": 3.04, + "learning_rate": 0.00034699097291875623, + "loss": 2.5487, + "theoretical_loss": 3.6379602340363, + "tokens_seen": 1033024512 + }, + { + "epoch": 3.04, + "learning_rate": 0.00034698094282848547, + "loss": 2.8011, + "theoretical_loss": 3.637938402543664, + "tokens_seen": 1033090048 + }, + { + "epoch": 3.04, + "learning_rate": 0.00034697091273821465, + "loss": 2.71, + "theoretical_loss": 3.6379165728236553, + "tokens_seen": 1033155584 + }, + { + "epoch": 3.04, + "learning_rate": 0.00034696088264794383, + "loss": 2.7223, + "theoretical_loss": 3.637894744876018, + "tokens_seen": 1033221120 + }, + { + "epoch": 3.04, + "learning_rate": 0.000346950852557673, + "loss": 2.4191, + "theoretical_loss": 3.637872918700496, + "tokens_seen": 1033286656 + }, + { + "epoch": 3.04, + "learning_rate": 0.00034694082246740224, + "loss": 2.9634, + "theoretical_loss": 3.6378510942968325, + "tokens_seen": 1033352192 + }, + { + "epoch": 3.04, + "learning_rate": 0.00034693079237713137, + "loss": 2.8381, + "theoretical_loss": 3.6378292716647715, + "tokens_seen": 1033417728 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003469207622868606, + "loss": 2.7367, + "theoretical_loss": 3.637807450804057, + "tokens_seen": 1033483264 + }, + { + "epoch": 3.04, + "learning_rate": 0.00034691073219658973, + "loss": 2.5099, + "theoretical_loss": 3.637785631714433, + "tokens_seen": 1033548800 + }, + { + "epoch": 3.04, + "learning_rate": 0.00034690070210631897, + "loss": 2.8073, + "theoretical_loss": 3.637763814395643, + "tokens_seen": 1033614336 + }, + { + "epoch": 3.04, + "learning_rate": 0.00034689067201604815, + "loss": 2.9439, + "theoretical_loss": 3.6377419988474315, + "tokens_seen": 1033679872 + }, + { + "epoch": 3.04, + "learning_rate": 0.00034688064192577733, + "loss": 2.8287, + "theoretical_loss": 3.6377201850695418, + "tokens_seen": 1033745408 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003468706118355065, + "loss": 2.6707, + "theoretical_loss": 3.6376983730617187, + "tokens_seen": 1033810944 + }, + { + "epoch": 3.04, + "objective/train/docs_used": 1181093, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.383265256881714, + "objective/train/theoretical_loss": 3.637692920336306, + "objective/train/tokens_used": 1054287328, + "theoretical_loss": 3.637692920336306, + "tokens_seen": 1033827328 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003468605817452357, + "loss": 2.6716, + "theoretical_loss": 3.6376765628237067, + "tokens_seen": 1033876480 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003468505516549649, + "loss": 2.4495, + "theoretical_loss": 3.6376547543552493, + "tokens_seen": 1033942016 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003468405215646941, + "loss": 2.8331, + "theoretical_loss": 3.6376329476560914, + "tokens_seen": 1034007552 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003468304914744233, + "loss": 2.8199, + "theoretical_loss": 3.6376111427259765, + "tokens_seen": 1034073088 + }, + { + "epoch": 3.04, + "learning_rate": 0.00034682046138415247, + "loss": 2.6642, + "theoretical_loss": 3.6375893395646495, + "tokens_seen": 1034138624 + }, + { + "epoch": 3.04, + "learning_rate": 0.00034681043129388165, + "loss": 2.6186, + "theoretical_loss": 3.637567538171855, + "tokens_seen": 1034204160 + }, + { + "epoch": 3.04, + "learning_rate": 0.00034680040120361083, + "loss": 2.8684, + "theoretical_loss": 3.637545738547337, + "tokens_seen": 1034269696 + }, + { + "epoch": 3.04, + "learning_rate": 0.00034679037111334007, + "loss": 2.6562, + "theoretical_loss": 3.6375239406908406, + "tokens_seen": 1034335232 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003467803410230692, + "loss": 2.7554, + "theoretical_loss": 3.6375021446021103, + "tokens_seen": 1034400768 + }, + { + "epoch": 3.04, + "learning_rate": 0.00034677031093279843, + "loss": 3.0065, + "theoretical_loss": 3.63748035028089, + "tokens_seen": 1034466304 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003467602808425276, + "loss": 2.8307, + "theoretical_loss": 3.6374585577269256, + "tokens_seen": 1034531840 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003467502507522568, + "loss": 2.647, + "theoretical_loss": 3.6374367669399605, + "tokens_seen": 1034597376 + }, + { + "epoch": 3.04, + "learning_rate": 0.000346740220661986, + "loss": 2.8376, + "theoretical_loss": 3.637414977919741, + "tokens_seen": 1034662912 + }, + { + "epoch": 3.04, + "learning_rate": 0.00034673019057171516, + "loss": 2.7634, + "theoretical_loss": 3.6373931906660113, + "tokens_seen": 1034728448 + }, + { + "epoch": 3.04, + "learning_rate": 0.00034672016048144434, + "loss": 2.6724, + "theoretical_loss": 3.6373714051785155, + "tokens_seen": 1034793984 + }, + { + "epoch": 3.04, + "learning_rate": 0.00034671013039117357, + "loss": 2.6667, + "theoretical_loss": 3.637349621457, + "tokens_seen": 1034859520 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003467001003009027, + "loss": 3.0254, + "theoretical_loss": 3.6373278395012085, + "tokens_seen": 1034925056 + }, + { + "epoch": 3.04, + "learning_rate": 0.00034669007021063193, + "loss": 2.7656, + "theoretical_loss": 3.6373060593108875, + "tokens_seen": 1034990592 + }, + { + "epoch": 3.04, + "learning_rate": 0.00034668004012036106, + "loss": 2.6329, + "theoretical_loss": 3.6372842808857806, + "tokens_seen": 1035056128 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003466700100300903, + "loss": 2.7772, + "theoretical_loss": 3.6372625042256344, + "tokens_seen": 1035121664 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003466599799398195, + "loss": 2.7761, + "theoretical_loss": 3.637240729330193, + "tokens_seen": 1035187200 + }, + { + "epoch": 3.04, + "learning_rate": 0.00034664994984954866, + "loss": 2.7121, + "theoretical_loss": 3.637218956199202, + "tokens_seen": 1035252736 + }, + { + "epoch": 3.04, + "learning_rate": 0.00034663991975927784, + "loss": 2.8773, + "theoretical_loss": 3.637197184832407, + "tokens_seen": 1035318272 + }, + { + "epoch": 3.04, + "learning_rate": 0.000346629889669007, + "loss": 2.7028, + "theoretical_loss": 3.6371754152295535, + "tokens_seen": 1035383808 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003466198595787362, + "loss": 2.7789, + "theoretical_loss": 3.6371536473903863, + "tokens_seen": 1035449344 + }, + { + "epoch": 3.04, + "objective/train/docs_used": 1182527, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.771360158920288, + "objective/train/theoretical_loss": 3.637148205706141, + "objective/train/tokens_used": 1055925728, + "theoretical_loss": 3.637148205706141, + "tokens_seen": 1035465728 + }, + { + "epoch": 3.04, + "learning_rate": 0.00034660982948846544, + "loss": 2.901, + "theoretical_loss": 3.6371318813146516, + "tokens_seen": 1035514880 + }, + { + "epoch": 3.04, + "learning_rate": 0.00034659979939819456, + "loss": 2.9041, + "theoretical_loss": 3.637110117002095, + "tokens_seen": 1035580416 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003465897693079238, + "loss": 2.9562, + "theoretical_loss": 3.6370883544524615, + "tokens_seen": 1035645952 + }, + { + "epoch": 3.04, + "learning_rate": 0.000346579739217653, + "loss": 2.843, + "theoretical_loss": 3.637066593665497, + "tokens_seen": 1035711488 + }, + { + "epoch": 3.04, + "learning_rate": 0.00034656970912738216, + "loss": 2.7811, + "theoretical_loss": 3.637044834640947, + "tokens_seen": 1035777024 + }, + { + "epoch": 3.04, + "learning_rate": 0.00034655967903711134, + "loss": 2.5779, + "theoretical_loss": 3.637023077378558, + "tokens_seen": 1035842560 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003465496489468405, + "loss": 2.7218, + "theoretical_loss": 3.637001321878075, + "tokens_seen": 1035908096 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003465396188565697, + "loss": 2.7265, + "theoretical_loss": 3.6369795681392443, + "tokens_seen": 1035973632 + }, + { + "epoch": 3.04, + "learning_rate": 0.00034652958876629894, + "loss": 2.8509, + "theoretical_loss": 3.6369578161618117, + "tokens_seen": 1036039168 + }, + { + "epoch": 3.04, + "learning_rate": 0.00034651955867602807, + "loss": 2.6018, + "theoretical_loss": 3.636936065945523, + "tokens_seen": 1036104704 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003465095285857573, + "loss": 2.5405, + "theoretical_loss": 3.6369143174901244, + "tokens_seen": 1036170240 + }, + { + "epoch": 3.04, + "learning_rate": 0.00034649949849548643, + "loss": 2.644, + "theoretical_loss": 3.6368925707953625, + "tokens_seen": 1036235776 + }, + { + "epoch": 3.04, + "learning_rate": 0.00034648946840521567, + "loss": 2.7706, + "theoretical_loss": 3.6368708258609823, + "tokens_seen": 1036301312 + }, + { + "epoch": 3.04, + "learning_rate": 0.00034647943831494485, + "loss": 2.7791, + "theoretical_loss": 3.636849082686731, + "tokens_seen": 1036366848 + }, + { + "epoch": 3.04, + "learning_rate": 0.00034646940822467403, + "loss": 2.528, + "theoretical_loss": 3.6368273412723546, + "tokens_seen": 1036432384 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003464593781344032, + "loss": 2.7708, + "theoretical_loss": 3.6368056016175987, + "tokens_seen": 1036497920 + }, + { + "epoch": 3.04, + "learning_rate": 0.00034644934804413244, + "loss": 2.8057, + "theoretical_loss": 3.6367838637222105, + "tokens_seen": 1036563456 + }, + { + "epoch": 3.04, + "learning_rate": 0.00034643931795386157, + "loss": 2.8408, + "theoretical_loss": 3.6367621275859356, + "tokens_seen": 1036628992 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003464292878635908, + "loss": 2.7432, + "theoretical_loss": 3.6367403932085214, + "tokens_seen": 1036694528 + }, + { + "epoch": 3.04, + "learning_rate": 0.00034641925777331993, + "loss": 2.7451, + "theoretical_loss": 3.636718660589714, + "tokens_seen": 1036760064 + }, + { + "epoch": 3.04, + "learning_rate": 0.00034640922768304917, + "loss": 2.7478, + "theoretical_loss": 3.6366969297292595, + "tokens_seen": 1036825600 + }, + { + "epoch": 3.04, + "learning_rate": 0.00034639919759277835, + "loss": 2.71, + "theoretical_loss": 3.6366752006269056, + "tokens_seen": 1036891136 + }, + { + "epoch": 3.04, + "learning_rate": 0.00034638916750250753, + "loss": 2.8828, + "theoretical_loss": 3.636653473282397, + "tokens_seen": 1036956672 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003463791374122367, + "loss": 2.5511, + "theoretical_loss": 3.636631747695483, + "tokens_seen": 1037022208 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003463691073219659, + "loss": 2.2055, + "theoretical_loss": 3.6366100238659085, + "tokens_seen": 1037087744 + }, + { + "epoch": 3.04, + "objective/train/docs_used": 1183367, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4552834033966064, + "objective/train/theoretical_loss": 3.636604593183069, + "objective/train/tokens_used": 1057564128, + "theoretical_loss": 3.636604593183069, + "tokens_seen": 1037104128 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003463590772316951, + "loss": 2.5444, + "theoretical_loss": 3.63658830179342, + "tokens_seen": 1037153280 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003463490471414243, + "loss": 2.6151, + "theoretical_loss": 3.6365665814777666, + "tokens_seen": 1037218816 + }, + { + "epoch": 3.04, + "learning_rate": 0.00034633901705115344, + "loss": 2.7626, + "theoretical_loss": 3.6365448629186927, + "tokens_seen": 1037284352 + }, + { + "epoch": 3.04, + "learning_rate": 0.00034632898696088267, + "loss": 2.8501, + "theoretical_loss": 3.6365231461159473, + "tokens_seen": 1037349888 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003463189568706118, + "loss": 2.8395, + "theoretical_loss": 3.6365014310692754, + "tokens_seen": 1037415424 + }, + { + "epoch": 3.04, + "learning_rate": 0.00034630892678034103, + "loss": 2.8243, + "theoretical_loss": 3.636479717778426, + "tokens_seen": 1037480960 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003462988966900702, + "loss": 2.8931, + "theoretical_loss": 3.6364580062431453, + "tokens_seen": 1037546496 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003462888665997994, + "loss": 2.4737, + "theoretical_loss": 3.6364362964631805, + "tokens_seen": 1037612032 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003462788365095286, + "loss": 2.6304, + "theoretical_loss": 3.636414588438279, + "tokens_seen": 1037677568 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003462688064192578, + "loss": 2.7085, + "theoretical_loss": 3.636392882168188, + "tokens_seen": 1037743104 + }, + { + "epoch": 3.04, + "learning_rate": 0.00034625877632898694, + "loss": 2.7709, + "theoretical_loss": 3.636371177652655, + "tokens_seen": 1037808640 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003462487462387162, + "loss": 2.7462, + "theoretical_loss": 3.6363494748914267, + "tokens_seen": 1037874176 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003462387161484453, + "loss": 2.7757, + "theoretical_loss": 3.6363277738842514, + "tokens_seen": 1037939712 + }, + { + "epoch": 3.04, + "learning_rate": 0.00034622868605817454, + "loss": 2.7106, + "theoretical_loss": 3.636306074630876, + "tokens_seen": 1038005248 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003462186559679037, + "loss": 2.5829, + "theoretical_loss": 3.636284377131049, + "tokens_seen": 1038070784 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003462086258776329, + "loss": 2.6035, + "theoretical_loss": 3.636262681384517, + "tokens_seen": 1038136320 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003461985957873621, + "loss": 2.6727, + "theoretical_loss": 3.636240987391028, + "tokens_seen": 1038201856 + }, + { + "epoch": 3.04, + "learning_rate": 0.00034618856569709126, + "loss": 2.8162, + "theoretical_loss": 3.6362192951503296, + "tokens_seen": 1038267392 + }, + { + "epoch": 3.04, + "learning_rate": 0.00034617853560682044, + "loss": 2.7681, + "theoretical_loss": 3.6361976046621702, + "tokens_seen": 1038332928 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003461685055165497, + "loss": 2.7701, + "theoretical_loss": 3.6361759159262963, + "tokens_seen": 1038398464 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003461584754262788, + "loss": 2.7035, + "theoretical_loss": 3.636154228942457, + "tokens_seen": 1038464000 + }, + { + "epoch": 3.04, + "learning_rate": 0.00034614844533600804, + "loss": 2.986, + "theoretical_loss": 3.6361325437103993, + "tokens_seen": 1038529536 + }, + { + "epoch": 3.04, + "learning_rate": 0.00034613841524573717, + "loss": 2.9617, + "theoretical_loss": 3.636110860229872, + "tokens_seen": 1038595072 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003461283851554664, + "loss": 2.6108, + "theoretical_loss": 3.636089178500623, + "tokens_seen": 1038660608 + }, + { + "epoch": 3.04, + "learning_rate": 0.0003461183550651956, + "loss": 2.639, + "theoretical_loss": 3.6360674985223995, + "tokens_seen": 1038726144 + }, + { + "epoch": 3.04, + "objective/train/docs_used": 1184791, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.915973663330078, + "objective/train/theoretical_loss": 3.636062078801412, + "objective/train/tokens_used": 1059202528, + "theoretical_loss": 3.636062078801412, + "tokens_seen": 1038742528 + }, + { + "epoch": 3.05, + "learning_rate": 0.00034610832497492476, + "loss": 2.8912, + "theoretical_loss": 3.6360458202949504, + "tokens_seen": 1038791680 + }, + { + "epoch": 3.05, + "learning_rate": 0.00034609829488465395, + "loss": 2.5813, + "theoretical_loss": 3.636024143818024, + "tokens_seen": 1038857216 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003460882647943832, + "loss": 2.761, + "theoretical_loss": 3.636002469091368, + "tokens_seen": 1038922752 + }, + { + "epoch": 3.05, + "learning_rate": 0.00034607823470411236, + "loss": 2.6937, + "theoretical_loss": 3.6359807961147306, + "tokens_seen": 1038988288 + }, + { + "epoch": 3.05, + "learning_rate": 0.00034606820461384154, + "loss": 2.5175, + "theoretical_loss": 3.6359591248878607, + "tokens_seen": 1039053824 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003460581745235707, + "loss": 2.9226, + "theoretical_loss": 3.635937455410507, + "tokens_seen": 1039119360 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003460481444332999, + "loss": 2.6023, + "theoretical_loss": 3.6359157876824164, + "tokens_seen": 1039184896 + }, + { + "epoch": 3.05, + "learning_rate": 0.00034603811434302914, + "loss": 2.7549, + "theoretical_loss": 3.6358941217033385, + "tokens_seen": 1039250432 + }, + { + "epoch": 3.05, + "learning_rate": 0.00034602808425275827, + "loss": 2.6038, + "theoretical_loss": 3.635872457473022, + "tokens_seen": 1039315968 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003460180541624875, + "loss": 2.8857, + "theoretical_loss": 3.635850794991215, + "tokens_seen": 1039381504 + }, + { + "epoch": 3.05, + "learning_rate": 0.00034600802407221663, + "loss": 2.6358, + "theoretical_loss": 3.6358291342576665, + "tokens_seen": 1039447040 + }, + { + "epoch": 3.05, + "learning_rate": 0.00034599799398194587, + "loss": 2.8824, + "theoretical_loss": 3.635807475272125, + "tokens_seen": 1039512576 + }, + { + "epoch": 3.05, + "learning_rate": 0.00034598796389167505, + "loss": 2.8954, + "theoretical_loss": 3.635785818034339, + "tokens_seen": 1039578112 + }, + { + "epoch": 3.05, + "learning_rate": 0.00034597793380140423, + "loss": 2.682, + "theoretical_loss": 3.6357641625440578, + "tokens_seen": 1039643648 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003459679037111334, + "loss": 2.6801, + "theoretical_loss": 3.6357425088010302, + "tokens_seen": 1039709184 + }, + { + "epoch": 3.05, + "learning_rate": 0.00034595787362086264, + "loss": 2.5574, + "theoretical_loss": 3.635720856805005, + "tokens_seen": 1039774720 + }, + { + "epoch": 3.05, + "learning_rate": 0.00034594784353059177, + "loss": 2.7004, + "theoretical_loss": 3.63569920655573, + "tokens_seen": 1039840256 + }, + { + "epoch": 3.05, + "learning_rate": 0.000345937813440321, + "loss": 2.4317, + "theoretical_loss": 3.635677558052956, + "tokens_seen": 1039905792 + }, + { + "epoch": 3.05, + "learning_rate": 0.00034592778335005013, + "loss": 2.69, + "theoretical_loss": 3.635655911296431, + "tokens_seen": 1039971328 + }, + { + "epoch": 3.05, + "learning_rate": 0.00034591775325977937, + "loss": 2.6949, + "theoretical_loss": 3.635634266285905, + "tokens_seen": 1040036864 + }, + { + "epoch": 3.05, + "learning_rate": 0.00034590772316950855, + "loss": 2.9375, + "theoretical_loss": 3.6356126230211263, + "tokens_seen": 1040102400 + }, + { + "epoch": 3.05, + "learning_rate": 0.00034589769307923773, + "loss": 2.5751, + "theoretical_loss": 3.635590981501845, + "tokens_seen": 1040167936 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003458876629889669, + "loss": 2.4966, + "theoretical_loss": 3.635569341727809, + "tokens_seen": 1040233472 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003458776328986961, + "loss": 2.6717, + "theoretical_loss": 3.6355477036987685, + "tokens_seen": 1040299008 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003458676028084253, + "loss": 2.7597, + "theoretical_loss": 3.6355260674144727, + "tokens_seen": 1040364544 + }, + { + "epoch": 3.05, + "objective/train/docs_used": 1185522, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9252572059631348, + "objective/train/theoretical_loss": 3.6355206586159863, + "objective/train/tokens_used": 1060840928, + "theoretical_loss": 3.6355206586159863, + "tokens_seen": 1040380928 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003458575727181545, + "loss": 2.8402, + "theoretical_loss": 3.6355044328746713, + "tokens_seen": 1040430080 + }, + { + "epoch": 3.05, + "learning_rate": 0.00034584754262788364, + "loss": 2.8633, + "theoretical_loss": 3.6354828000791137, + "tokens_seen": 1040495616 + }, + { + "epoch": 3.05, + "learning_rate": 0.00034583751253761287, + "loss": 2.6076, + "theoretical_loss": 3.6354611690275496, + "tokens_seen": 1040561152 + }, + { + "epoch": 3.05, + "learning_rate": 0.000345827482447342, + "loss": 2.67, + "theoretical_loss": 3.6354395397197274, + "tokens_seen": 1040626688 + }, + { + "epoch": 3.05, + "learning_rate": 0.00034581745235707123, + "loss": 2.7122, + "theoretical_loss": 3.6354179121553987, + "tokens_seen": 1040692224 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003458074222668004, + "loss": 2.7741, + "theoretical_loss": 3.635396286334311, + "tokens_seen": 1040757760 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003457973921765296, + "loss": 2.7348, + "theoretical_loss": 3.6353746622562157, + "tokens_seen": 1040823296 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003457873620862588, + "loss": 2.8063, + "theoretical_loss": 3.6353530399208616, + "tokens_seen": 1040888832 + }, + { + "epoch": 3.05, + "learning_rate": 0.000345777331995988, + "loss": 2.4041, + "theoretical_loss": 3.6353314193279993, + "tokens_seen": 1040954368 + }, + { + "epoch": 3.05, + "learning_rate": 0.00034576730190571714, + "loss": 2.9207, + "theoretical_loss": 3.635309800477378, + "tokens_seen": 1041019904 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003457572718154464, + "loss": 2.7454, + "theoretical_loss": 3.6352881833687483, + "tokens_seen": 1041085440 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003457472417251755, + "loss": 2.7788, + "theoretical_loss": 3.6352665680018594, + "tokens_seen": 1041150976 + }, + { + "epoch": 3.05, + "learning_rate": 0.00034573721163490474, + "loss": 2.909, + "theoretical_loss": 3.6352449543764624, + "tokens_seen": 1041216512 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003457271815446339, + "loss": 2.8034, + "theoretical_loss": 3.6352233424923064, + "tokens_seen": 1041282048 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003457171514543631, + "loss": 2.8243, + "theoretical_loss": 3.6352017323491417, + "tokens_seen": 1041347584 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003457071213640923, + "loss": 2.6581, + "theoretical_loss": 3.635180123946719, + "tokens_seen": 1041413120 + }, + { + "epoch": 3.05, + "learning_rate": 0.00034569709127382146, + "loss": 2.7839, + "theoretical_loss": 3.6351585172847884, + "tokens_seen": 1041478656 + }, + { + "epoch": 3.05, + "learning_rate": 0.00034568706118355064, + "loss": 2.6316, + "theoretical_loss": 3.6351369123631, + "tokens_seen": 1041544192 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003456770310932799, + "loss": 2.5668, + "theoretical_loss": 3.635115309181404, + "tokens_seen": 1041609728 + }, + { + "epoch": 3.05, + "learning_rate": 0.000345667001003009, + "loss": 2.7055, + "theoretical_loss": 3.6350937077394514, + "tokens_seen": 1041675264 + }, + { + "epoch": 3.05, + "learning_rate": 0.00034565697091273824, + "loss": 2.7145, + "theoretical_loss": 3.635072108036992, + "tokens_seen": 1041740800 + }, + { + "epoch": 3.05, + "learning_rate": 0.00034564694082246737, + "loss": 2.8442, + "theoretical_loss": 3.6350505100737767, + "tokens_seen": 1041806336 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003456369107321966, + "loss": 2.6584, + "theoretical_loss": 3.6350289138495557, + "tokens_seen": 1041871872 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003456268806419258, + "loss": 2.7529, + "theoretical_loss": 3.63500731936408, + "tokens_seen": 1041937408 + }, + { + "epoch": 3.05, + "learning_rate": 0.00034561685055165496, + "loss": 2.7175, + "theoretical_loss": 3.6349857266171, + "tokens_seen": 1042002944 + }, + { + "epoch": 3.05, + "objective/train/docs_used": 1186074, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.702031135559082, + "objective/train/theoretical_loss": 3.6349803287019657, + "objective/train/tokens_used": 1062479328, + "theoretical_loss": 3.6349803287019657, + "tokens_seen": 1042019328 + }, + { + "epoch": 3.05, + "learning_rate": 0.00034560682046138415, + "loss": 2.7838, + "theoretical_loss": 3.634964135608367, + "tokens_seen": 1042068480 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003455967903711134, + "loss": 2.7338, + "theoretical_loss": 3.634942546337631, + "tokens_seen": 1042134016 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003455867602808425, + "loss": 2.6965, + "theoretical_loss": 3.634920958804643, + "tokens_seen": 1042199552 + }, + { + "epoch": 3.05, + "learning_rate": 0.00034557673019057174, + "loss": 2.7916, + "theoretical_loss": 3.634899373009154, + "tokens_seen": 1042265088 + }, + { + "epoch": 3.05, + "learning_rate": 0.00034556670010030087, + "loss": 2.4773, + "theoretical_loss": 3.6348777889509147, + "tokens_seen": 1042330624 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003455566700100301, + "loss": 2.3448, + "theoretical_loss": 3.634856206629677, + "tokens_seen": 1042396160 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003455466399197593, + "loss": 2.6472, + "theoretical_loss": 3.6348346260451905, + "tokens_seen": 1042461696 + }, + { + "epoch": 3.05, + "learning_rate": 0.00034553660982948847, + "loss": 2.8073, + "theoretical_loss": 3.634813047197207, + "tokens_seen": 1042527232 + }, + { + "epoch": 3.05, + "learning_rate": 0.00034552657973921765, + "loss": 2.7546, + "theoretical_loss": 3.6347914700854775, + "tokens_seen": 1042592768 + }, + { + "epoch": 3.05, + "learning_rate": 0.00034551654964894683, + "loss": 2.8483, + "theoretical_loss": 3.6347698947097538, + "tokens_seen": 1042658304 + }, + { + "epoch": 3.05, + "learning_rate": 0.000345506519558676, + "loss": 2.6587, + "theoretical_loss": 3.634748321069786, + "tokens_seen": 1042723840 + }, + { + "epoch": 3.05, + "learning_rate": 0.00034549648946840525, + "loss": 2.675, + "theoretical_loss": 3.634726749165326, + "tokens_seen": 1042789376 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003454864593781344, + "loss": 2.642, + "theoretical_loss": 3.634705178996125, + "tokens_seen": 1042854912 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003454764292878636, + "loss": 2.4778, + "theoretical_loss": 3.6346836105619347, + "tokens_seen": 1042920448 + }, + { + "epoch": 3.05, + "learning_rate": 0.00034546639919759274, + "loss": 2.4542, + "theoretical_loss": 3.634662043862506, + "tokens_seen": 1042985984 + }, + { + "epoch": 3.05, + "learning_rate": 0.00034545636910732197, + "loss": 2.4854, + "theoretical_loss": 3.634640478897591, + "tokens_seen": 1043051520 + }, + { + "epoch": 3.05, + "learning_rate": 0.00034544633901705115, + "loss": 2.834, + "theoretical_loss": 3.6346189156669406, + "tokens_seen": 1043117056 + }, + { + "epoch": 3.05, + "learning_rate": 0.00034543630892678033, + "loss": 3.0203, + "theoretical_loss": 3.6345973541703067, + "tokens_seen": 1043182592 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003454262788365095, + "loss": 2.9097, + "theoretical_loss": 3.634575794407441, + "tokens_seen": 1043248128 + }, + { + "epoch": 3.05, + "learning_rate": 0.00034541624874623875, + "loss": 2.8772, + "theoretical_loss": 3.6345542363780954, + "tokens_seen": 1043313664 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003454062186559679, + "loss": 2.6594, + "theoretical_loss": 3.634532680082021, + "tokens_seen": 1043379200 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003453961885656971, + "loss": 2.8194, + "theoretical_loss": 3.63451112551897, + "tokens_seen": 1043444736 + }, + { + "epoch": 3.05, + "learning_rate": 0.00034538615847542624, + "loss": 2.8311, + "theoretical_loss": 3.634489572688694, + "tokens_seen": 1043510272 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003453761283851555, + "loss": 2.6407, + "theoretical_loss": 3.634468021590945, + "tokens_seen": 1043575808 + }, + { + "epoch": 3.05, + "learning_rate": 0.00034536609829488466, + "loss": 2.7576, + "theoretical_loss": 3.6344464722254757, + "tokens_seen": 1043641344 + }, + { + "epoch": 3.05, + "objective/train/docs_used": 1186076, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4055583477020264, + "objective/train/theoretical_loss": 3.634441085154748, + "objective/train/tokens_used": 1064117728, + "theoretical_loss": 3.634441085154748, + "tokens_seen": 1043657728 + }, + { + "epoch": 3.05, + "learning_rate": 0.00034535606820461384, + "loss": 2.7181, + "theoretical_loss": 3.634424924592037, + "tokens_seen": 1043706880 + }, + { + "epoch": 3.05, + "learning_rate": 0.000345346038114343, + "loss": 2.7111, + "theoretical_loss": 3.634403378690381, + "tokens_seen": 1043772416 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003453360080240722, + "loss": 2.8271, + "theoretical_loss": 3.6343818345202603, + "tokens_seen": 1043837952 + }, + { + "epoch": 3.05, + "learning_rate": 0.00034532597793380143, + "loss": 2.7099, + "theoretical_loss": 3.634360292081427, + "tokens_seen": 1043903488 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003453159478435306, + "loss": 2.8388, + "theoretical_loss": 3.634338751373633, + "tokens_seen": 1043969024 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003453059177532598, + "loss": 2.695, + "theoretical_loss": 3.634317212396631, + "tokens_seen": 1044034560 + }, + { + "epoch": 3.05, + "learning_rate": 0.000345295887662989, + "loss": 2.5767, + "theoretical_loss": 3.634295675150173, + "tokens_seen": 1044100096 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003452858575727182, + "loss": 2.8101, + "theoretical_loss": 3.634274139634011, + "tokens_seen": 1044165632 + }, + { + "epoch": 3.05, + "learning_rate": 0.00034527582748244734, + "loss": 3.153, + "theoretical_loss": 3.634252605847898, + "tokens_seen": 1044231168 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003452657973921766, + "loss": 2.9676, + "theoretical_loss": 3.634231073791586, + "tokens_seen": 1044296704 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003452557673019057, + "loss": 2.6099, + "theoretical_loss": 3.634209543464828, + "tokens_seen": 1044362240 + }, + { + "epoch": 3.05, + "learning_rate": 0.00034524573721163494, + "loss": 2.8402, + "theoretical_loss": 3.6341880148673766, + "tokens_seen": 1044427776 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003452357071213641, + "loss": 2.9848, + "theoretical_loss": 3.6341664879989835, + "tokens_seen": 1044493312 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003452256770310933, + "loss": 2.7283, + "theoretical_loss": 3.634144962859402, + "tokens_seen": 1044558848 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003452156469408225, + "loss": 2.6106, + "theoretical_loss": 3.634123439448385, + "tokens_seen": 1044624384 + }, + { + "epoch": 3.05, + "learning_rate": 0.00034520561685055166, + "loss": 2.696, + "theoretical_loss": 3.634101917765685, + "tokens_seen": 1044689920 + }, + { + "epoch": 3.05, + "learning_rate": 0.00034519558676028084, + "loss": 2.7361, + "theoretical_loss": 3.6340803978110543, + "tokens_seen": 1044755456 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003451855566700101, + "loss": 2.813, + "theoretical_loss": 3.634058879584247, + "tokens_seen": 1044820992 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003451755265797392, + "loss": 2.8119, + "theoretical_loss": 3.6340373630850147, + "tokens_seen": 1044886528 + }, + { + "epoch": 3.05, + "learning_rate": 0.00034516549648946844, + "loss": 2.7531, + "theoretical_loss": 3.6340158483131115, + "tokens_seen": 1044952064 + }, + { + "epoch": 3.05, + "learning_rate": 0.00034515546639919757, + "loss": 2.8369, + "theoretical_loss": 3.6339943352682895, + "tokens_seen": 1045017600 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003451454363089268, + "loss": 2.7817, + "theoretical_loss": 3.633972823950302, + "tokens_seen": 1045083136 + }, + { + "epoch": 3.05, + "learning_rate": 0.000345135406218656, + "loss": 2.572, + "theoretical_loss": 3.633951314358903, + "tokens_seen": 1045148672 + }, + { + "epoch": 3.05, + "learning_rate": 0.00034512537612838517, + "loss": 3.0036, + "theoretical_loss": 3.6339298064938443, + "tokens_seen": 1045214208 + }, + { + "epoch": 3.05, + "learning_rate": 0.00034511534603811435, + "loss": 2.9289, + "theoretical_loss": 3.6339083003548796, + "tokens_seen": 1045279744 + }, + { + "epoch": 3.05, + "objective/train/docs_used": 1186840, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6896257400512695, + "objective/train/theoretical_loss": 3.6339029240898117, + "objective/train/tokens_used": 1065756128, + "theoretical_loss": 3.6339029240898117, + "tokens_seen": 1045296128 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003451053159478436, + "loss": 2.7279, + "theoretical_loss": 3.6338867959417622, + "tokens_seen": 1045345280 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003450952858575727, + "loss": 2.8756, + "theoretical_loss": 3.633865293254246, + "tokens_seen": 1045410816 + }, + { + "epoch": 3.05, + "learning_rate": 0.00034508525576730194, + "loss": 2.6824, + "theoretical_loss": 3.633843792292083, + "tokens_seen": 1045476352 + }, + { + "epoch": 3.05, + "learning_rate": 0.00034507522567703107, + "loss": 2.7099, + "theoretical_loss": 3.6338222930550286, + "tokens_seen": 1045541888 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003450651955867603, + "loss": 2.6207, + "theoretical_loss": 3.6338007955428346, + "tokens_seen": 1045607424 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003450551654964895, + "loss": 2.7052, + "theoretical_loss": 3.633779299755255, + "tokens_seen": 1045672960 + }, + { + "epoch": 3.05, + "learning_rate": 0.00034504513540621867, + "loss": 2.7676, + "theoretical_loss": 3.633757805692044, + "tokens_seen": 1045738496 + }, + { + "epoch": 3.05, + "learning_rate": 0.00034503510531594785, + "loss": 2.7697, + "theoretical_loss": 3.633736313352954, + "tokens_seen": 1045804032 + }, + { + "epoch": 3.05, + "learning_rate": 0.00034502507522567703, + "loss": 2.6363, + "theoretical_loss": 3.6337148227377396, + "tokens_seen": 1045869568 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003450150451354062, + "loss": 2.8698, + "theoretical_loss": 3.633693333846154, + "tokens_seen": 1045935104 + }, + { + "epoch": 3.05, + "learning_rate": 0.00034500501504513545, + "loss": 2.7141, + "theoretical_loss": 3.6336718466779514, + "tokens_seen": 1046000640 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003449949849548646, + "loss": 2.8788, + "theoretical_loss": 3.6336503612328857, + "tokens_seen": 1046066176 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003449849548645938, + "loss": 2.9623, + "theoretical_loss": 3.63362887751071, + "tokens_seen": 1046131712 + }, + { + "epoch": 3.05, + "learning_rate": 0.00034497492477432294, + "loss": 2.7589, + "theoretical_loss": 3.633607395511179, + "tokens_seen": 1046197248 + }, + { + "epoch": 3.05, + "learning_rate": 0.00034496489468405217, + "loss": 2.716, + "theoretical_loss": 3.6335859152340464, + "tokens_seen": 1046262784 + }, + { + "epoch": 3.05, + "learning_rate": 0.00034495486459378135, + "loss": 2.9333, + "theoretical_loss": 3.633564436679066, + "tokens_seen": 1046328320 + }, + { + "epoch": 3.05, + "learning_rate": 0.00034494483450351053, + "loss": 2.7096, + "theoretical_loss": 3.6335429598459927, + "tokens_seen": 1046393856 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003449348044132397, + "loss": 2.5753, + "theoretical_loss": 3.6335214847345796, + "tokens_seen": 1046459392 + }, + { + "epoch": 3.05, + "learning_rate": 0.00034492477432296895, + "loss": 2.7974, + "theoretical_loss": 3.633500011344582, + "tokens_seen": 1046524928 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003449147442326981, + "loss": 2.733, + "theoretical_loss": 3.6334785396757527, + "tokens_seen": 1046590464 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003449047141424273, + "loss": 2.6909, + "theoretical_loss": 3.633457069727847, + "tokens_seen": 1046656000 + }, + { + "epoch": 3.05, + "learning_rate": 0.00034489468405215644, + "loss": 2.966, + "theoretical_loss": 3.6334356015006186, + "tokens_seen": 1046721536 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003448846539618857, + "loss": 2.6867, + "theoretical_loss": 3.633414134993823, + "tokens_seen": 1046787072 + }, + { + "epoch": 3.05, + "learning_rate": 0.00034487462387161486, + "loss": 2.9336, + "theoretical_loss": 3.6333926702072135, + "tokens_seen": 1046852608 + }, + { + "epoch": 3.05, + "learning_rate": 0.00034486459378134404, + "loss": 2.6975, + "theoretical_loss": 3.6333712071405446, + "tokens_seen": 1046918144 + }, + { + "epoch": 3.05, + "objective/train/docs_used": 1188135, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7929697036743164, + "objective/train/theoretical_loss": 3.63336584164259, + "objective/train/tokens_used": 1067394528, + "theoretical_loss": 3.63336584164259, + "tokens_seen": 1046934528 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003448545636910732, + "loss": 2.5809, + "theoretical_loss": 3.6333497457935717, + "tokens_seen": 1046983680 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003448445336008024, + "loss": 2.8726, + "theoretical_loss": 3.6333282861660487, + "tokens_seen": 1047049216 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003448345035105316, + "loss": 2.6964, + "theoretical_loss": 3.63330682825773, + "tokens_seen": 1047114752 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003448244734202608, + "loss": 2.8283, + "theoretical_loss": 3.6332853720683715, + "tokens_seen": 1047180288 + }, + { + "epoch": 3.05, + "learning_rate": 0.00034481444332998994, + "loss": 3.0839, + "theoretical_loss": 3.633263917597726, + "tokens_seen": 1047245824 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003448044132397192, + "loss": 2.8138, + "theoretical_loss": 3.6332424648455506, + "tokens_seen": 1047311360 + }, + { + "epoch": 3.05, + "learning_rate": 0.00034479438314944836, + "loss": 2.6576, + "theoretical_loss": 3.633221013811598, + "tokens_seen": 1047376896 + }, + { + "epoch": 3.05, + "learning_rate": 0.00034478435305917754, + "loss": 2.7789, + "theoretical_loss": 3.6331995644956243, + "tokens_seen": 1047442432 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003447743229689067, + "loss": 2.8961, + "theoretical_loss": 3.6331781168973842, + "tokens_seen": 1047507968 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003447642928786359, + "loss": 2.6799, + "theoretical_loss": 3.6331566710166325, + "tokens_seen": 1047573504 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003447542627883651, + "loss": 2.9108, + "theoretical_loss": 3.6331352268531245, + "tokens_seen": 1047639040 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003447442326980943, + "loss": 2.7632, + "theoretical_loss": 3.633113784406615, + "tokens_seen": 1047704576 + }, + { + "epoch": 3.05, + "learning_rate": 0.00034473420260782345, + "loss": 2.8166, + "theoretical_loss": 3.6330923436768594, + "tokens_seen": 1047770112 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003447241725175527, + "loss": 2.8229, + "theoretical_loss": 3.6330709046636125, + "tokens_seen": 1047835648 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003447141424272818, + "loss": 3.0599, + "theoretical_loss": 3.63304946736663, + "tokens_seen": 1047901184 + }, + { + "epoch": 3.05, + "learning_rate": 0.00034470411233701104, + "loss": 2.8003, + "theoretical_loss": 3.633028031785667, + "tokens_seen": 1047966720 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003446940822467402, + "loss": 2.8111, + "theoretical_loss": 3.633006597920479, + "tokens_seen": 1048032256 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003446840521564694, + "loss": 2.7593, + "theoretical_loss": 3.6329851657708208, + "tokens_seen": 1048097792 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003446740220661986, + "loss": 2.6875, + "theoretical_loss": 3.632963735336448, + "tokens_seen": 1048163328 + }, + { + "epoch": 3.05, + "learning_rate": 0.00034466399197592777, + "loss": 2.8356, + "theoretical_loss": 3.632942306617117, + "tokens_seen": 1048228864 + }, + { + "epoch": 3.05, + "learning_rate": 0.00034465396188565695, + "loss": 2.591, + "theoretical_loss": 3.6329208796125823, + "tokens_seen": 1048294400 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003446439317953862, + "loss": 2.6914, + "theoretical_loss": 3.6328994543226, + "tokens_seen": 1048359936 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003446339017051153, + "loss": 2.757, + "theoretical_loss": 3.632878030746925, + "tokens_seen": 1048425472 + }, + { + "epoch": 3.05, + "learning_rate": 0.00034462387161484455, + "loss": 2.9459, + "theoretical_loss": 3.632856608885314, + "tokens_seen": 1048491008 + }, + { + "epoch": 3.05, + "learning_rate": 0.00034461384152457373, + "loss": 2.9613, + "theoretical_loss": 3.6328351887375225, + "tokens_seen": 1048556544 + }, + { + "epoch": 3.05, + "objective/train/docs_used": 1188820, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7827885150909424, + "objective/train/theoretical_loss": 3.63282983396833, + "objective/train/tokens_used": 1069032928, + "theoretical_loss": 3.63282983396833, + "tokens_seen": 1048572928 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003446038114343029, + "loss": 2.8882, + "theoretical_loss": 3.6328137703033057, + "tokens_seen": 1048622080 + }, + { + "epoch": 3.05, + "learning_rate": 0.00034459378134403214, + "loss": 2.6796, + "theoretical_loss": 3.63279235358242, + "tokens_seen": 1048687616 + }, + { + "epoch": 3.05, + "learning_rate": 0.00034458375125376127, + "loss": 2.7243, + "theoretical_loss": 3.632770938574621, + "tokens_seen": 1048753152 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003445737211634905, + "loss": 2.835, + "theoretical_loss": 3.6327495252796647, + "tokens_seen": 1048818688 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003445636910732197, + "loss": 2.756, + "theoretical_loss": 3.632728113697307, + "tokens_seen": 1048884224 + }, + { + "epoch": 3.05, + "learning_rate": 0.00034455366098294887, + "loss": 2.7928, + "theoretical_loss": 3.6327067038273047, + "tokens_seen": 1048949760 + }, + { + "epoch": 3.05, + "learning_rate": 0.00034454363089267805, + "loss": 2.5859, + "theoretical_loss": 3.632685295669413, + "tokens_seen": 1049015296 + }, + { + "epoch": 3.05, + "learning_rate": 0.00034453360080240723, + "loss": 2.7753, + "theoretical_loss": 3.6326638892233873, + "tokens_seen": 1049080832 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003445235707121364, + "loss": 2.7935, + "theoretical_loss": 3.6326424844889855, + "tokens_seen": 1049146368 + }, + { + "epoch": 3.05, + "learning_rate": 0.00034451354062186565, + "loss": 2.6532, + "theoretical_loss": 3.6326210814659636, + "tokens_seen": 1049211904 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003445035105315948, + "loss": 2.7596, + "theoretical_loss": 3.632599680154077, + "tokens_seen": 1049277440 + }, + { + "epoch": 3.05, + "learning_rate": 0.000344493480441324, + "loss": 2.7922, + "theoretical_loss": 3.632578280553082, + "tokens_seen": 1049342976 + }, + { + "epoch": 3.05, + "learning_rate": 0.00034448345035105314, + "loss": 2.6624, + "theoretical_loss": 3.632556882662736, + "tokens_seen": 1049408512 + }, + { + "epoch": 3.05, + "learning_rate": 0.00034447342026078237, + "loss": 2.6397, + "theoretical_loss": 3.6325354864827943, + "tokens_seen": 1049474048 + }, + { + "epoch": 3.05, + "learning_rate": 0.00034446339017051155, + "loss": 2.7459, + "theoretical_loss": 3.6325140920130146, + "tokens_seen": 1049539584 + }, + { + "epoch": 3.05, + "learning_rate": 0.00034445336008024073, + "loss": 2.6949, + "theoretical_loss": 3.6324926992531523, + "tokens_seen": 1049605120 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003444433299899699, + "loss": 2.4783, + "theoretical_loss": 3.6324713082029643, + "tokens_seen": 1049670656 + }, + { + "epoch": 3.05, + "learning_rate": 0.00034443329989969915, + "loss": 2.788, + "theoretical_loss": 3.632449918862208, + "tokens_seen": 1049736192 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003444232698094283, + "loss": 2.8112, + "theoretical_loss": 3.632428531230639, + "tokens_seen": 1049801728 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003444132397191575, + "loss": 2.7413, + "theoretical_loss": 3.6324071453080147, + "tokens_seen": 1049867264 + }, + { + "epoch": 3.05, + "learning_rate": 0.00034440320962888664, + "loss": 2.6558, + "theoretical_loss": 3.6323857610940915, + "tokens_seen": 1049932800 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003443931795386159, + "loss": 2.7811, + "theoretical_loss": 3.6323643785886266, + "tokens_seen": 1049998336 + }, + { + "epoch": 3.05, + "learning_rate": 0.00034438314944834506, + "loss": 2.9847, + "theoretical_loss": 3.6323429977913766, + "tokens_seen": 1050063872 + }, + { + "epoch": 3.05, + "learning_rate": 0.00034437311935807424, + "loss": 2.6683, + "theoretical_loss": 3.6323216187020986, + "tokens_seen": 1050129408 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003443630892678034, + "loss": 2.7443, + "theoretical_loss": 3.632300241320549, + "tokens_seen": 1050194944 + }, + { + "epoch": 3.05, + "objective/train/docs_used": 1190162, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4736685752868652, + "objective/train/theoretical_loss": 3.6322948972419664, + "objective/train/tokens_used": 1070671328, + "theoretical_loss": 3.6322948972419664, + "tokens_seen": 1050211328 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003443530591775326, + "loss": 2.7721, + "theoretical_loss": 3.6322788656464864, + "tokens_seen": 1050260480 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003443430290872618, + "loss": 2.5936, + "theoretical_loss": 3.632257491679666, + "tokens_seen": 1050326016 + }, + { + "epoch": 3.05, + "learning_rate": 0.000344332998996991, + "loss": 2.5859, + "theoretical_loss": 3.6322361194198463, + "tokens_seen": 1050391552 + }, + { + "epoch": 3.05, + "learning_rate": 0.00034432296890672014, + "loss": 2.5561, + "theoretical_loss": 3.632214748866784, + "tokens_seen": 1050457088 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003443129388164494, + "loss": 2.7315, + "theoretical_loss": 3.6321933800202357, + "tokens_seen": 1050522624 + }, + { + "epoch": 3.05, + "learning_rate": 0.00034430290872617856, + "loss": 2.6789, + "theoretical_loss": 3.63217201287996, + "tokens_seen": 1050588160 + }, + { + "epoch": 3.05, + "learning_rate": 0.00034429287863590774, + "loss": 2.7474, + "theoretical_loss": 3.632150647445713, + "tokens_seen": 1050653696 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003442828485456369, + "loss": 2.8795, + "theoretical_loss": 3.6321292837172523, + "tokens_seen": 1050719232 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003442728184553661, + "loss": 2.7036, + "theoretical_loss": 3.6321079216943364, + "tokens_seen": 1050784768 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003442627883650953, + "loss": 2.6738, + "theoretical_loss": 3.6320865613767213, + "tokens_seen": 1050850304 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003442527582748245, + "loss": 2.6909, + "theoretical_loss": 3.6320652027641653, + "tokens_seen": 1050915840 + }, + { + "epoch": 3.05, + "learning_rate": 0.00034424272818455365, + "loss": 2.6339, + "theoretical_loss": 3.6320438458564266, + "tokens_seen": 1050981376 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003442326980942829, + "loss": 2.8449, + "theoretical_loss": 3.632022490653261, + "tokens_seen": 1051046912 + }, + { + "epoch": 3.05, + "learning_rate": 0.000344222668004012, + "loss": 2.759, + "theoretical_loss": 3.6320011371544285, + "tokens_seen": 1051112448 + }, + { + "epoch": 3.05, + "learning_rate": 0.00034421263791374124, + "loss": 2.7718, + "theoretical_loss": 3.631979785359685, + "tokens_seen": 1051177984 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003442026078234704, + "loss": 2.5683, + "theoretical_loss": 3.6319584352687895, + "tokens_seen": 1051243520 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003441925777331996, + "loss": 2.87, + "theoretical_loss": 3.6319370868814986, + "tokens_seen": 1051309056 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003441825476429288, + "loss": 2.9015, + "theoretical_loss": 3.631915740197571, + "tokens_seen": 1051374592 + }, + { + "epoch": 3.05, + "learning_rate": 0.00034417251755265797, + "loss": 2.717, + "theoretical_loss": 3.6318943952167646, + "tokens_seen": 1051440128 + }, + { + "epoch": 3.05, + "learning_rate": 0.00034416248746238715, + "loss": 2.6497, + "theoretical_loss": 3.631873051938837, + "tokens_seen": 1051505664 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003441524573721164, + "loss": 2.7191, + "theoretical_loss": 3.631851710363547, + "tokens_seen": 1051571200 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003441424272818455, + "loss": 2.7562, + "theoretical_loss": 3.6318303704906514, + "tokens_seen": 1051636736 + }, + { + "epoch": 3.05, + "learning_rate": 0.00034413239719157475, + "loss": 2.6796, + "theoretical_loss": 3.6318090323199095, + "tokens_seen": 1051702272 + }, + { + "epoch": 3.05, + "learning_rate": 0.00034412236710130393, + "loss": 2.7178, + "theoretical_loss": 3.631787695851079, + "tokens_seen": 1051767808 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003441123370110331, + "loss": 2.9961, + "theoretical_loss": 3.6317663610839177, + "tokens_seen": 1051833344 + }, + { + "epoch": 3.05, + "objective/train/docs_used": 1190623, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.515566825866699, + "objective/train/theoretical_loss": 3.6317610276579853, + "objective/train/tokens_used": 1072309728, + "theoretical_loss": 3.6317610276579853, + "tokens_seen": 1051849728 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003441023069207623, + "loss": 2.5649, + "theoretical_loss": 3.6317450280181847, + "tokens_seen": 1051898880 + }, + { + "epoch": 3.05, + "learning_rate": 0.00034409227683049147, + "loss": 2.9343, + "theoretical_loss": 3.631723696653638, + "tokens_seen": 1051964416 + }, + { + "epoch": 3.05, + "learning_rate": 0.00034408224674022065, + "loss": 2.8018, + "theoretical_loss": 3.631702366990036, + "tokens_seen": 1052029952 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003440722166499499, + "loss": 2.5757, + "theoretical_loss": 3.6316810390271366, + "tokens_seen": 1052095488 + }, + { + "epoch": 3.05, + "learning_rate": 0.000344062186559679, + "loss": 2.7342, + "theoretical_loss": 3.631659712764699, + "tokens_seen": 1052161024 + }, + { + "epoch": 3.05, + "learning_rate": 0.00034405215646940825, + "loss": 2.7447, + "theoretical_loss": 3.6316383882024814, + "tokens_seen": 1052226560 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003440421263791374, + "loss": 2.5221, + "theoretical_loss": 3.631617065340243, + "tokens_seen": 1052292096 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003440320962888666, + "loss": 2.8617, + "theoretical_loss": 3.6315957441777407, + "tokens_seen": 1052357632 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003440220661985958, + "loss": 2.8462, + "theoretical_loss": 3.631574424714735, + "tokens_seen": 1052423168 + }, + { + "epoch": 3.05, + "learning_rate": 0.000344012036108325, + "loss": 2.7823, + "theoretical_loss": 3.6315531069509834, + "tokens_seen": 1052488704 + }, + { + "epoch": 3.05, + "learning_rate": 0.00034400200601805416, + "loss": 2.7497, + "theoretical_loss": 3.6315317908862457, + "tokens_seen": 1052554240 + }, + { + "epoch": 3.05, + "learning_rate": 0.00034399197592778334, + "loss": 2.7485, + "theoretical_loss": 3.6315104765202797, + "tokens_seen": 1052619776 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003439819458375125, + "loss": 2.6464, + "theoretical_loss": 3.6314891638528453, + "tokens_seen": 1052685312 + }, + { + "epoch": 3.05, + "learning_rate": 0.00034397191574724175, + "loss": 2.9019, + "theoretical_loss": 3.6314678528837003, + "tokens_seen": 1052750848 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003439618856569709, + "loss": 2.8454, + "theoretical_loss": 3.6314465436126047, + "tokens_seen": 1052816384 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003439518555667001, + "loss": 2.9119, + "theoretical_loss": 3.6314252360393167, + "tokens_seen": 1052881920 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003439418254764293, + "loss": 2.8862, + "theoretical_loss": 3.6314039301635956, + "tokens_seen": 1052947456 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003439317953861585, + "loss": 2.6123, + "theoretical_loss": 3.631382625985201, + "tokens_seen": 1053012992 + }, + { + "epoch": 3.05, + "learning_rate": 0.00034392176529588766, + "loss": 2.746, + "theoretical_loss": 3.6313613235038913, + "tokens_seen": 1053078528 + }, + { + "epoch": 3.05, + "learning_rate": 0.00034391173520561684, + "loss": 2.8266, + "theoretical_loss": 3.6313400227194266, + "tokens_seen": 1053144064 + }, + { + "epoch": 3.05, + "learning_rate": 0.000343901705115346, + "loss": 2.7095, + "theoretical_loss": 3.631318723631565, + "tokens_seen": 1053209600 + }, + { + "epoch": 3.05, + "learning_rate": 0.00034389167502507526, + "loss": 2.7095, + "theoretical_loss": 3.631297426240067, + "tokens_seen": 1053275136 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003438816449348044, + "loss": 2.7669, + "theoretical_loss": 3.6312761305446912, + "tokens_seen": 1053340672 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003438716148445336, + "loss": 2.7469, + "theoretical_loss": 3.631254836545197, + "tokens_seen": 1053406208 + }, + { + "epoch": 3.05, + "learning_rate": 0.00034386158475426275, + "loss": 2.6697, + "theoretical_loss": 3.6312335442413444, + "tokens_seen": 1053471744 + }, + { + "epoch": 3.05, + "objective/train/docs_used": 1192122, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.786816120147705, + "objective/train/theoretical_loss": 3.6312282214302964, + "objective/train/tokens_used": 1073948128, + "theoretical_loss": 3.6312282214302964, + "tokens_seen": 1053488128 + }, + { + "epoch": 3.05, + "learning_rate": 0.000343851554663992, + "loss": 2.7388, + "theoretical_loss": 3.631212253632892, + "tokens_seen": 1053537280 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003438415245737212, + "loss": 2.8104, + "theoretical_loss": 3.6311909647196003, + "tokens_seen": 1053602816 + }, + { + "epoch": 3.05, + "learning_rate": 0.00034383149448345034, + "loss": 2.8182, + "theoretical_loss": 3.6311696775012283, + "tokens_seen": 1053668352 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003438214643931796, + "loss": 2.8246, + "theoretical_loss": 3.6311483919775362, + "tokens_seen": 1053733888 + }, + { + "epoch": 3.05, + "learning_rate": 0.00034381143430290876, + "loss": 2.7451, + "theoretical_loss": 3.6311271081482834, + "tokens_seen": 1053799424 + }, + { + "epoch": 3.05, + "learning_rate": 0.00034380140421263794, + "loss": 2.7069, + "theoretical_loss": 3.631105826013229, + "tokens_seen": 1053864960 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003437913741223671, + "loss": 2.5447, + "theoretical_loss": 3.631084545572134, + "tokens_seen": 1053930496 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003437813440320963, + "loss": 2.8172, + "theoretical_loss": 3.6310632668247576, + "tokens_seen": 1053996032 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003437713139418255, + "loss": 2.8105, + "theoretical_loss": 3.6310419897708597, + "tokens_seen": 1054061568 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003437612838515547, + "loss": 2.7633, + "theoretical_loss": 3.6310207144102, + "tokens_seen": 1054127104 + }, + { + "epoch": 3.05, + "learning_rate": 0.00034375125376128385, + "loss": 2.7244, + "theoretical_loss": 3.630999440742539, + "tokens_seen": 1054192640 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003437412236710131, + "loss": 2.6294, + "theoretical_loss": 3.6309781687676366, + "tokens_seen": 1054258176 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003437311935807422, + "loss": 2.688, + "theoretical_loss": 3.6309568984852527, + "tokens_seen": 1054323712 + }, + { + "epoch": 3.05, + "learning_rate": 0.00034372116349047144, + "loss": 2.7278, + "theoretical_loss": 3.6309356298951476, + "tokens_seen": 1054389248 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003437111334002006, + "loss": 2.8325, + "theoretical_loss": 3.630914362997082, + "tokens_seen": 1054454784 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003437011033099298, + "loss": 2.5317, + "theoretical_loss": 3.630893097790815, + "tokens_seen": 1054520320 + }, + { + "epoch": 3.05, + "learning_rate": 0.000343691073219659, + "loss": 2.6591, + "theoretical_loss": 3.6308718342761073, + "tokens_seen": 1054585856 + }, + { + "epoch": 3.05, + "learning_rate": 0.00034368104312938817, + "loss": 2.8177, + "theoretical_loss": 3.6308505724527196, + "tokens_seen": 1054651392 + }, + { + "epoch": 3.05, + "learning_rate": 0.00034367101303911735, + "loss": 2.5762, + "theoretical_loss": 3.6308293123204125, + "tokens_seen": 1054716928 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003436609829488466, + "loss": 2.7574, + "theoretical_loss": 3.630808053878946, + "tokens_seen": 1054782464 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003436509528585757, + "loss": 2.8421, + "theoretical_loss": 3.6307867971280796, + "tokens_seen": 1054848000 + }, + { + "epoch": 3.05, + "learning_rate": 0.00034364092276830495, + "loss": 2.7248, + "theoretical_loss": 3.630765542067576, + "tokens_seen": 1054913536 + }, + { + "epoch": 3.05, + "learning_rate": 0.00034363089267803413, + "loss": 2.6222, + "theoretical_loss": 3.630744288697194, + "tokens_seen": 1054979072 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003436208625877633, + "loss": 2.7392, + "theoretical_loss": 3.630723037016695, + "tokens_seen": 1055044608 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003436108324974925, + "loss": 2.685, + "theoretical_loss": 3.63070178702584, + "tokens_seen": 1055110144 + }, + { + "epoch": 3.05, + "objective/train/docs_used": 1192817, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.987161874771118, + "objective/train/theoretical_loss": 3.6306964747921047, + "objective/train/tokens_used": 1075586528, + "theoretical_loss": 3.6306964747921047, + "tokens_seen": 1055126528 + }, + { + "epoch": 3.05, + "learning_rate": 0.00034360080240722167, + "loss": 2.8193, + "theoretical_loss": 3.6306805387243886, + "tokens_seen": 1055175680 + }, + { + "epoch": 3.05, + "learning_rate": 0.00034359077231695085, + "loss": 2.6656, + "theoretical_loss": 3.6306592921121026, + "tokens_seen": 1055241216 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003435807422266801, + "loss": 2.8565, + "theoretical_loss": 3.630638047188742, + "tokens_seen": 1055306752 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003435707121364092, + "loss": 2.8089, + "theoretical_loss": 3.630616803954069, + "tokens_seen": 1055372288 + }, + { + "epoch": 3.05, + "learning_rate": 0.00034356068204613845, + "loss": 2.7897, + "theoretical_loss": 3.6305955624078434, + "tokens_seen": 1055437824 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003435506519558676, + "loss": 2.8819, + "theoretical_loss": 3.630574322549826, + "tokens_seen": 1055503360 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003435406218655968, + "loss": 2.6583, + "theoretical_loss": 3.6305530843797786, + "tokens_seen": 1055568896 + }, + { + "epoch": 3.05, + "learning_rate": 0.000343530591775326, + "loss": 2.8231, + "theoretical_loss": 3.630531847897462, + "tokens_seen": 1055634432 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003435205616850552, + "loss": 2.5255, + "theoretical_loss": 3.6305106131026363, + "tokens_seen": 1055699968 + }, + { + "epoch": 3.05, + "learning_rate": 0.00034351053159478436, + "loss": 2.8292, + "theoretical_loss": 3.6304893799950646, + "tokens_seen": 1055765504 + }, + { + "epoch": 3.05, + "learning_rate": 0.00034350050150451354, + "loss": 2.8406, + "theoretical_loss": 3.630468148574507, + "tokens_seen": 1055831040 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003434904714142427, + "loss": 2.6635, + "theoretical_loss": 3.6304469188407245, + "tokens_seen": 1055896576 + }, + { + "epoch": 3.05, + "learning_rate": 0.00034348044132397195, + "loss": 2.5332, + "theoretical_loss": 3.630425690793479, + "tokens_seen": 1055962112 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003434704112337011, + "loss": 2.7283, + "theoretical_loss": 3.6304044644325315, + "tokens_seen": 1056027648 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003434603811434303, + "loss": 2.8458, + "theoretical_loss": 3.6303832397576437, + "tokens_seen": 1056093184 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003434503510531595, + "loss": 2.8332, + "theoretical_loss": 3.6303620167685766, + "tokens_seen": 1056158720 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003434403209628887, + "loss": 2.8633, + "theoretical_loss": 3.6303407954650924, + "tokens_seen": 1056224256 + }, + { + "epoch": 3.05, + "learning_rate": 0.00034343029087261786, + "loss": 2.8437, + "theoretical_loss": 3.630319575846952, + "tokens_seen": 1056289792 + }, + { + "epoch": 3.05, + "learning_rate": 0.00034342026078234704, + "loss": 2.837, + "theoretical_loss": 3.6302983579139174, + "tokens_seen": 1056355328 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003434102306920762, + "loss": 2.7527, + "theoretical_loss": 3.6302771416657498, + "tokens_seen": 1056420864 + }, + { + "epoch": 3.05, + "learning_rate": 0.00034340020060180546, + "loss": 2.8107, + "theoretical_loss": 3.6302559271022115, + "tokens_seen": 1056486400 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003433901705115346, + "loss": 2.751, + "theoretical_loss": 3.6302347142230644, + "tokens_seen": 1056551936 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003433801404212638, + "loss": 2.834, + "theoretical_loss": 3.6302135030280693, + "tokens_seen": 1056617472 + }, + { + "epoch": 3.05, + "learning_rate": 0.00034337011033099295, + "loss": 2.6986, + "theoretical_loss": 3.6301922935169886, + "tokens_seen": 1056683008 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003433600802407222, + "loss": 2.7787, + "theoretical_loss": 3.630171085689584, + "tokens_seen": 1056748544 + }, + { + "epoch": 3.05, + "objective/train/docs_used": 1193467, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6657423973083496, + "objective/train/theoretical_loss": 3.6301657839957793, + "objective/train/tokens_used": 1077224928, + "theoretical_loss": 3.6301657839957793, + "tokens_seen": 1056764928 + }, + { + "epoch": 3.05, + "learning_rate": 0.00034335005015045136, + "loss": 2.6549, + "theoretical_loss": 3.630149879545618, + "tokens_seen": 1056814080 + }, + { + "epoch": 3.05, + "learning_rate": 0.00034334002006018054, + "loss": 2.6641, + "theoretical_loss": 3.630128675084852, + "tokens_seen": 1056879616 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003433299899699097, + "loss": 2.6756, + "theoretical_loss": 3.6301074723070483, + "tokens_seen": 1056945152 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003433199598796389, + "loss": 2.8726, + "theoretical_loss": 3.6300862712119693, + "tokens_seen": 1057010688 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003433099297893681, + "loss": 2.8562, + "theoretical_loss": 3.6300650717993763, + "tokens_seen": 1057076224 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003432998996990973, + "loss": 2.4702, + "theoretical_loss": 3.630043874069032, + "tokens_seen": 1057141760 + }, + { + "epoch": 3.05, + "learning_rate": 0.00034328986960882645, + "loss": 2.7899, + "theoretical_loss": 3.630022678020699, + "tokens_seen": 1057207296 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003432798395185557, + "loss": 2.6209, + "theoretical_loss": 3.630001483654139, + "tokens_seen": 1057272832 + }, + { + "epoch": 3.05, + "learning_rate": 0.00034326980942828487, + "loss": 2.8341, + "theoretical_loss": 3.6299802909691143, + "tokens_seen": 1057338368 + }, + { + "epoch": 3.05, + "learning_rate": 0.00034325977933801405, + "loss": 2.6814, + "theoretical_loss": 3.6299590999653875, + "tokens_seen": 1057403904 + }, + { + "epoch": 3.05, + "learning_rate": 0.00034324974924774323, + "loss": 2.6038, + "theoretical_loss": 3.629937910642721, + "tokens_seen": 1057469440 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003432397191574724, + "loss": 2.6944, + "theoretical_loss": 3.6299167230008775, + "tokens_seen": 1057534976 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003432296890672016, + "loss": 2.8614, + "theoretical_loss": 3.6298955370396193, + "tokens_seen": 1057600512 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003432196589769308, + "loss": 2.753, + "theoretical_loss": 3.6298743527587085, + "tokens_seen": 1057666048 + }, + { + "epoch": 3.05, + "learning_rate": 0.00034320962888665995, + "loss": 2.697, + "theoretical_loss": 3.629853170157909, + "tokens_seen": 1057731584 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003431995987963892, + "loss": 2.6929, + "theoretical_loss": 3.6298319892369815, + "tokens_seen": 1057797120 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003431895687061183, + "loss": 2.7009, + "theoretical_loss": 3.629810809995691, + "tokens_seen": 1057862656 + }, + { + "epoch": 3.05, + "learning_rate": 0.00034317953861584755, + "loss": 2.9626, + "theoretical_loss": 3.6297896324337984, + "tokens_seen": 1057928192 + }, + { + "epoch": 3.05, + "learning_rate": 0.00034316950852557673, + "loss": 2.6848, + "theoretical_loss": 3.6297684565510675, + "tokens_seen": 1057993728 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003431594784353059, + "loss": 2.7164, + "theoretical_loss": 3.629747282347261, + "tokens_seen": 1058059264 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003431494483450351, + "loss": 2.4662, + "theoretical_loss": 3.6297261098221414, + "tokens_seen": 1058124800 + }, + { + "epoch": 3.05, + "learning_rate": 0.00034313941825476433, + "loss": 2.8341, + "theoretical_loss": 3.629704938975472, + "tokens_seen": 1058190336 + }, + { + "epoch": 3.05, + "learning_rate": 0.00034312938816449345, + "loss": 2.8161, + "theoretical_loss": 3.6296837698070163, + "tokens_seen": 1058255872 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003431193580742227, + "loss": 3.0423, + "theoretical_loss": 3.6296626023165364, + "tokens_seen": 1058321408 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003431093279839518, + "loss": 2.9207, + "theoretical_loss": 3.6296414365037957, + "tokens_seen": 1058386944 + }, + { + "epoch": 3.05, + "objective/train/docs_used": 1194725, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.449052095413208, + "objective/train/theoretical_loss": 3.6296361453127295, + "objective/train/tokens_used": 1078863328, + "theoretical_loss": 3.6296361453127295, + "tokens_seen": 1058403328 + }, + { + "epoch": 3.05, + "learning_rate": 0.00034309929789368105, + "loss": 2.7992, + "theoretical_loss": 3.6296202723685576, + "tokens_seen": 1058452480 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003430892678034103, + "loss": 2.9941, + "theoretical_loss": 3.6295991099105853, + "tokens_seen": 1058518016 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003430792377131394, + "loss": 2.6418, + "theoretical_loss": 3.629577949129642, + "tokens_seen": 1058583552 + }, + { + "epoch": 3.05, + "learning_rate": 0.00034306920762286865, + "loss": 2.7596, + "theoretical_loss": 3.6295567900254904, + "tokens_seen": 1058649088 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003430591775325978, + "loss": 2.7151, + "theoretical_loss": 3.6295356325978947, + "tokens_seen": 1058714624 + }, + { + "epoch": 3.05, + "learning_rate": 0.000343049147442327, + "loss": 2.839, + "theoretical_loss": 3.6295144768466177, + "tokens_seen": 1058780160 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003430391173520562, + "loss": 2.7535, + "theoretical_loss": 3.629493322771423, + "tokens_seen": 1058845696 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003430290872617854, + "loss": 2.7381, + "theoretical_loss": 3.6294721703720745, + "tokens_seen": 1058911232 + }, + { + "epoch": 3.05, + "learning_rate": 0.00034301905717151456, + "loss": 2.8283, + "theoretical_loss": 3.629451019648335, + "tokens_seen": 1058976768 + }, + { + "epoch": 3.05, + "learning_rate": 0.00034300902708124374, + "loss": 2.5179, + "theoretical_loss": 3.6294298705999686, + "tokens_seen": 1059042304 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003429989969909729, + "loss": 2.844, + "theoretical_loss": 3.6294087232267387, + "tokens_seen": 1059107840 + }, + { + "epoch": 3.05, + "learning_rate": 0.00034298896690070215, + "loss": 2.883, + "theoretical_loss": 3.629387577528409, + "tokens_seen": 1059173376 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003429789368104313, + "loss": 2.6924, + "theoretical_loss": 3.6293664335047433, + "tokens_seen": 1059238912 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003429689067201605, + "loss": 2.8918, + "theoretical_loss": 3.6293452911555057, + "tokens_seen": 1059304448 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003429588766298897, + "loss": 2.8112, + "theoretical_loss": 3.6293241504804596, + "tokens_seen": 1059369984 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003429488465396189, + "loss": 2.9203, + "theoretical_loss": 3.6293030114793687, + "tokens_seen": 1059435520 + }, + { + "epoch": 3.05, + "learning_rate": 0.00034293881644934806, + "loss": 2.7102, + "theoretical_loss": 3.629281874151997, + "tokens_seen": 1059501056 + }, + { + "epoch": 3.05, + "learning_rate": 0.00034292878635907724, + "loss": 2.6518, + "theoretical_loss": 3.6292607384981093, + "tokens_seen": 1059566592 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003429187562688064, + "loss": 2.6567, + "theoretical_loss": 3.629239604517468, + "tokens_seen": 1059632128 + }, + { + "epoch": 3.05, + "learning_rate": 0.00034290872617853566, + "loss": 2.9782, + "theoretical_loss": 3.629218472209839, + "tokens_seen": 1059697664 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003428986960882648, + "loss": 2.6975, + "theoretical_loss": 3.6291973415749847, + "tokens_seen": 1059763200 + }, + { + "epoch": 3.05, + "learning_rate": 0.000342888665997994, + "loss": 2.7407, + "theoretical_loss": 3.6291762126126708, + "tokens_seen": 1059828736 + }, + { + "epoch": 3.05, + "learning_rate": 0.00034287863590772315, + "loss": 2.9337, + "theoretical_loss": 3.6291550853226604, + "tokens_seen": 1059894272 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003428686058174524, + "loss": 2.6709, + "theoretical_loss": 3.629133959704718, + "tokens_seen": 1059959808 + }, + { + "epoch": 3.05, + "learning_rate": 0.00034285857572718156, + "loss": 2.6925, + "theoretical_loss": 3.629112835758608, + "tokens_seen": 1060025344 + }, + { + "epoch": 3.05, + "objective/train/docs_used": 1195453, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8471500873565674, + "objective/train/theoretical_loss": 3.629107555033277, + "objective/train/tokens_used": 1080501728, + "theoretical_loss": 3.629107555033277, + "tokens_seen": 1060041728 + }, + { + "epoch": 3.05, + "learning_rate": 0.00034284854563691074, + "loss": 2.9138, + "theoretical_loss": 3.629091713484095, + "tokens_seen": 1060090880 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003428385155466399, + "loss": 2.6919, + "theoretical_loss": 3.629070592880943, + "tokens_seen": 1060156416 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003428284854563691, + "loss": 2.7959, + "theoretical_loss": 3.6290494739489167, + "tokens_seen": 1060221952 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003428184553660983, + "loss": 2.7872, + "theoretical_loss": 3.62902835668778, + "tokens_seen": 1060287488 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003428084252758275, + "loss": 2.6846, + "theoretical_loss": 3.6290072410972987, + "tokens_seen": 1060353024 + }, + { + "epoch": 3.05, + "learning_rate": 0.00034279839518555665, + "loss": 2.8737, + "theoretical_loss": 3.628986127177236, + "tokens_seen": 1060418560 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003427883650952859, + "loss": 2.9303, + "theoretical_loss": 3.6289650149273576, + "tokens_seen": 1060484096 + }, + { + "epoch": 3.05, + "learning_rate": 0.00034277833500501507, + "loss": 2.7662, + "theoretical_loss": 3.628943904347427, + "tokens_seen": 1060549632 + }, + { + "epoch": 3.05, + "learning_rate": 0.00034276830491474425, + "loss": 2.8635, + "theoretical_loss": 3.6289227954372105, + "tokens_seen": 1060615168 + }, + { + "epoch": 3.05, + "learning_rate": 0.00034275827482447343, + "loss": 2.7954, + "theoretical_loss": 3.6289016881964713, + "tokens_seen": 1060680704 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003427482447342026, + "loss": 2.7364, + "theoretical_loss": 3.6288805826249755, + "tokens_seen": 1060746240 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003427382146439318, + "loss": 2.9048, + "theoretical_loss": 3.628859478722487, + "tokens_seen": 1060811776 + }, + { + "epoch": 3.05, + "learning_rate": 0.000342728184553661, + "loss": 2.9484, + "theoretical_loss": 3.6288383764887713, + "tokens_seen": 1060877312 + }, + { + "epoch": 3.05, + "learning_rate": 0.00034271815446339015, + "loss": 2.8514, + "theoretical_loss": 3.628817275923593, + "tokens_seen": 1060942848 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003427081243731194, + "loss": 2.9913, + "theoretical_loss": 3.628796177026718, + "tokens_seen": 1061008384 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003426980942828485, + "loss": 2.7835, + "theoretical_loss": 3.6287750797979097, + "tokens_seen": 1061073920 + }, + { + "epoch": 3.05, + "learning_rate": 0.00034268806419257775, + "loss": 2.787, + "theoretical_loss": 3.628753984236935, + "tokens_seen": 1061139456 + }, + { + "epoch": 3.05, + "learning_rate": 0.00034267803410230693, + "loss": 2.6012, + "theoretical_loss": 3.628732890343558, + "tokens_seen": 1061204992 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003426680040120361, + "loss": 2.8233, + "theoretical_loss": 3.6287117981175436, + "tokens_seen": 1061270528 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003426579739217653, + "loss": 2.9159, + "theoretical_loss": 3.6286907075586585, + "tokens_seen": 1061336064 + }, + { + "epoch": 3.05, + "learning_rate": 0.00034264794383149453, + "loss": 2.754, + "theoretical_loss": 3.6286696186666667, + "tokens_seen": 1061401600 + }, + { + "epoch": 3.05, + "learning_rate": 0.00034263791374122366, + "loss": 2.5168, + "theoretical_loss": 3.6286485314413337, + "tokens_seen": 1061467136 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003426278836509529, + "loss": 2.8263, + "theoretical_loss": 3.6286274458824255, + "tokens_seen": 1061532672 + }, + { + "epoch": 3.05, + "learning_rate": 0.000342617853560682, + "loss": 2.6702, + "theoretical_loss": 3.6286063619897067, + "tokens_seen": 1061598208 + }, + { + "epoch": 3.05, + "learning_rate": 0.00034260782347041125, + "loss": 3.0237, + "theoretical_loss": 3.6285852797629436, + "tokens_seen": 1061663744 + }, + { + "epoch": 3.05, + "objective/train/docs_used": 1196907, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8292043209075928, + "objective/train/theoretical_loss": 3.628580009466531, + "objective/train/tokens_used": 1082140128, + "theoretical_loss": 3.628580009466531, + "tokens_seen": 1061680128 + }, + { + "epoch": 3.05, + "learning_rate": 0.00034259779338014043, + "loss": 2.798, + "theoretical_loss": 3.628564199201901, + "tokens_seen": 1061729280 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003425877632898696, + "loss": 2.5776, + "theoretical_loss": 3.628543120306346, + "tokens_seen": 1061794816 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003425777331995988, + "loss": 2.8096, + "theoretical_loss": 3.628522043076042, + "tokens_seen": 1061860352 + }, + { + "epoch": 3.05, + "learning_rate": 0.000342567703109328, + "loss": 2.8131, + "theoretical_loss": 3.6285009675107567, + "tokens_seen": 1061925888 + }, + { + "epoch": 3.05, + "learning_rate": 0.00034255767301905716, + "loss": 2.8516, + "theoretical_loss": 3.628479893610254, + "tokens_seen": 1061991424 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003425476429287864, + "loss": 2.6119, + "theoretical_loss": 3.628458821374301, + "tokens_seen": 1062056960 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003425376128385155, + "loss": 2.7599, + "theoretical_loss": 3.6284377508026635, + "tokens_seen": 1062122496 + }, + { + "epoch": 3.05, + "learning_rate": 0.00034252758274824476, + "loss": 2.6915, + "theoretical_loss": 3.6284166818951067, + "tokens_seen": 1062188032 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003425175526579739, + "loss": 2.7241, + "theoretical_loss": 3.6283956146513967, + "tokens_seen": 1062253568 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003425075225677031, + "loss": 2.6911, + "theoretical_loss": 3.6283745490712995, + "tokens_seen": 1062319104 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003424974924774323, + "loss": 2.8022, + "theoretical_loss": 3.628353485154581, + "tokens_seen": 1062384640 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003424874623871615, + "loss": 2.5889, + "theoretical_loss": 3.6283324229010083, + "tokens_seen": 1062450176 + }, + { + "epoch": 3.05, + "learning_rate": 0.00034247743229689066, + "loss": 2.8766, + "theoretical_loss": 3.6283113623103462, + "tokens_seen": 1062515712 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003424674022066199, + "loss": 2.7854, + "theoretical_loss": 3.6282903033823612, + "tokens_seen": 1062581248 + }, + { + "epoch": 3.05, + "learning_rate": 0.000342457372116349, + "loss": 2.838, + "theoretical_loss": 3.62826924611682, + "tokens_seen": 1062646784 + }, + { + "epoch": 3.05, + "learning_rate": 0.00034244734202607826, + "loss": 2.7359, + "theoretical_loss": 3.628248190513488, + "tokens_seen": 1062712320 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003424373119358074, + "loss": 2.9226, + "theoretical_loss": 3.628227136572132, + "tokens_seen": 1062777856 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003424272818455366, + "loss": 2.942, + "theoretical_loss": 3.6282060842925183, + "tokens_seen": 1062843392 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003424172517552658, + "loss": 2.8092, + "theoretical_loss": 3.6281850336744137, + "tokens_seen": 1062908928 + }, + { + "epoch": 3.05, + "learning_rate": 0.000342407221664995, + "loss": 2.8666, + "theoretical_loss": 3.6281639847175837, + "tokens_seen": 1062974464 + }, + { + "epoch": 3.05, + "learning_rate": 0.00034239719157472416, + "loss": 2.6878, + "theoretical_loss": 3.6281429374217953, + "tokens_seen": 1063040000 + }, + { + "epoch": 3.05, + "learning_rate": 0.00034238716148445335, + "loss": 2.7101, + "theoretical_loss": 3.628121891786815, + "tokens_seen": 1063105536 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003423771313941825, + "loss": 2.9153, + "theoretical_loss": 3.62810084781241, + "tokens_seen": 1063171072 + }, + { + "epoch": 3.05, + "learning_rate": 0.00034236710130391176, + "loss": 2.8423, + "theoretical_loss": 3.628079805498346, + "tokens_seen": 1063236608 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003423570712136409, + "loss": 2.5257, + "theoretical_loss": 3.62805876484439, + "tokens_seen": 1063302144 + }, + { + "epoch": 3.05, + "objective/train/docs_used": 1197465, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.75803279876709, + "objective/train/theoretical_loss": 3.628053504940265, + "objective/train/tokens_used": 1083778528, + "theoretical_loss": 3.628053504940265, + "tokens_seen": 1063318528 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003423470411233701, + "loss": 2.5156, + "theoretical_loss": 3.6280377258503087, + "tokens_seen": 1063367680 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003423370110330993, + "loss": 2.8494, + "theoretical_loss": 3.628016688515869, + "tokens_seen": 1063433216 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003423269809428285, + "loss": 2.8854, + "theoretical_loss": 3.6279956528408377, + "tokens_seen": 1063498752 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003423169508525577, + "loss": 2.6486, + "theoretical_loss": 3.627974618824981, + "tokens_seen": 1063564288 + }, + { + "epoch": 3.05, + "learning_rate": 0.00034230692076228685, + "loss": 2.765, + "theoretical_loss": 3.6279535864680668, + "tokens_seen": 1063629824 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003422968906720161, + "loss": 2.6579, + "theoretical_loss": 3.6279325557698616, + "tokens_seen": 1063695360 + }, + { + "epoch": 3.05, + "learning_rate": 0.00034228686058174527, + "loss": 2.7489, + "theoretical_loss": 3.6279115267301325, + "tokens_seen": 1063760896 + }, + { + "epoch": 3.05, + "learning_rate": 0.00034227683049147445, + "loss": 2.7437, + "theoretical_loss": 3.6278904993486467, + "tokens_seen": 1063826432 + }, + { + "epoch": 3.05, + "learning_rate": 0.00034226680040120363, + "loss": 2.9753, + "theoretical_loss": 3.6278694736251706, + "tokens_seen": 1063891968 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003422567703109328, + "loss": 2.8276, + "theoretical_loss": 3.627848449559472, + "tokens_seen": 1063957504 + }, + { + "epoch": 3.05, + "learning_rate": 0.000342246740220662, + "loss": 2.6558, + "theoretical_loss": 3.6278274271513182, + "tokens_seen": 1064023040 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003422367101303912, + "loss": 2.8123, + "theoretical_loss": 3.627806406400476, + "tokens_seen": 1064088576 + }, + { + "epoch": 3.05, + "learning_rate": 0.00034222668004012035, + "loss": 2.515, + "theoretical_loss": 3.6277853873067127, + "tokens_seen": 1064154112 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003422166499498496, + "loss": 2.9208, + "theoretical_loss": 3.627764369869796, + "tokens_seen": 1064219648 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003422066198595787, + "loss": 2.8514, + "theoretical_loss": 3.6277433540894926, + "tokens_seen": 1064285184 + }, + { + "epoch": 3.05, + "learning_rate": 0.00034219658976930795, + "loss": 2.6061, + "theoretical_loss": 3.6277223399655707, + "tokens_seen": 1064350720 + }, + { + "epoch": 3.05, + "learning_rate": 0.00034218655967903713, + "loss": 2.8817, + "theoretical_loss": 3.627701327497798, + "tokens_seen": 1064416256 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003421765295887663, + "loss": 2.7702, + "theoretical_loss": 3.627680316685941, + "tokens_seen": 1064481792 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003421664994984955, + "loss": 2.9302, + "theoretical_loss": 3.6276593075297674, + "tokens_seen": 1064547328 + }, + { + "epoch": 3.05, + "learning_rate": 0.00034215646940822473, + "loss": 2.8216, + "theoretical_loss": 3.6276383000290457, + "tokens_seen": 1064612864 + }, + { + "epoch": 3.05, + "learning_rate": 0.00034214643931795386, + "loss": 2.6398, + "theoretical_loss": 3.627617294183543, + "tokens_seen": 1064678400 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003421364092276831, + "loss": 2.873, + "theoretical_loss": 3.6275962899930265, + "tokens_seen": 1064743936 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003421263791374122, + "loss": 2.887, + "theoretical_loss": 3.627575287457265, + "tokens_seen": 1064809472 + }, + { + "epoch": 3.05, + "learning_rate": 0.00034211634904714145, + "loss": 2.7337, + "theoretical_loss": 3.6275542865760255, + "tokens_seen": 1064875008 + }, + { + "epoch": 3.05, + "learning_rate": 0.00034210631895687063, + "loss": 3.067, + "theoretical_loss": 3.6275332873490767, + "tokens_seen": 1064940544 + }, + { + "epoch": 3.05, + "objective/train/docs_used": 1198703, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5579044818878174, + "objective/train/theoretical_loss": 3.627528037800795, + "objective/train/tokens_used": 1085416928, + "theoretical_loss": 3.627528037800795, + "tokens_seen": 1064956928 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003420962888665998, + "loss": 2.7133, + "theoretical_loss": 3.627512289776185, + "tokens_seen": 1065006080 + }, + { + "epoch": 3.05, + "learning_rate": 0.000342086258776329, + "loss": 2.7935, + "theoretical_loss": 3.62749129385712, + "tokens_seen": 1065071616 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003420762286860582, + "loss": 2.6849, + "theoretical_loss": 3.6274702995916486, + "tokens_seen": 1065137152 + }, + { + "epoch": 3.05, + "learning_rate": 0.00034206619859578736, + "loss": 2.8924, + "theoretical_loss": 3.627449306979539, + "tokens_seen": 1065202688 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003420561685055166, + "loss": 2.7858, + "theoretical_loss": 3.62742831602056, + "tokens_seen": 1065268224 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003420461384152457, + "loss": 2.8484, + "theoretical_loss": 3.6274073267144793, + "tokens_seen": 1065333760 + }, + { + "epoch": 3.05, + "learning_rate": 0.00034203610832497496, + "loss": 2.7231, + "theoretical_loss": 3.627386339061064, + "tokens_seen": 1065399296 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003420260782347041, + "loss": 2.8332, + "theoretical_loss": 3.6273653530600845, + "tokens_seen": 1065464832 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003420160481444333, + "loss": 2.7879, + "theoretical_loss": 3.627344368711307, + "tokens_seen": 1065530368 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003420060180541625, + "loss": 2.4803, + "theoretical_loss": 3.6273233860145013, + "tokens_seen": 1065595904 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003419959879638917, + "loss": 2.8658, + "theoretical_loss": 3.627302404969435, + "tokens_seen": 1065661440 + }, + { + "epoch": 3.05, + "learning_rate": 0.00034198595787362086, + "loss": 2.7391, + "theoretical_loss": 3.6272814255758763, + "tokens_seen": 1065726976 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003419759277833501, + "loss": 2.9654, + "theoretical_loss": 3.6272604478335944, + "tokens_seen": 1065792512 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003419658976930792, + "loss": 2.5866, + "theoretical_loss": 3.6272394717423575, + "tokens_seen": 1065858048 + }, + { + "epoch": 3.05, + "learning_rate": 0.00034195586760280846, + "loss": 2.7491, + "theoretical_loss": 3.627218497301934, + "tokens_seen": 1065923584 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003419458375125376, + "loss": 2.94, + "theoretical_loss": 3.627197524512092, + "tokens_seen": 1065989120 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003419358074222668, + "loss": 2.7148, + "theoretical_loss": 3.6271765533726015, + "tokens_seen": 1066054656 + }, + { + "epoch": 3.05, + "learning_rate": 0.000341925777331996, + "loss": 3.0207, + "theoretical_loss": 3.62715558388323, + "tokens_seen": 1066120192 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003419157472417252, + "loss": 2.8715, + "theoretical_loss": 3.6271346160437465, + "tokens_seen": 1066185728 + }, + { + "epoch": 3.05, + "learning_rate": 0.00034190571715145436, + "loss": 2.8406, + "theoretical_loss": 3.6271136498539205, + "tokens_seen": 1066251264 + }, + { + "epoch": 3.05, + "learning_rate": 0.00034189568706118355, + "loss": 2.6654, + "theoretical_loss": 3.6270926853135195, + "tokens_seen": 1066316800 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003418856569709127, + "loss": 2.7278, + "theoretical_loss": 3.627071722422314, + "tokens_seen": 1066382336 + }, + { + "epoch": 3.05, + "learning_rate": 0.00034187562688064196, + "loss": 2.7196, + "theoretical_loss": 3.6270507611800715, + "tokens_seen": 1066447872 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003418655967903711, + "loss": 2.8842, + "theoretical_loss": 3.6270298015865614, + "tokens_seen": 1066513408 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003418555667001003, + "loss": 2.5783, + "theoretical_loss": 3.627008843641553, + "tokens_seen": 1066578944 + }, + { + "epoch": 3.05, + "objective/train/docs_used": 1199964, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9109983444213867, + "objective/train/theoretical_loss": 3.6270036044128524, + "objective/train/tokens_used": 1087055328, + "theoretical_loss": 3.6270036044128524, + "tokens_seen": 1066595328 + }, + { + "epoch": 3.05, + "learning_rate": 0.00034184553660982945, + "loss": 3.0025, + "theoretical_loss": 3.626987887344815, + "tokens_seen": 1066644480 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003418355065195587, + "loss": 2.7713, + "theoretical_loss": 3.626966932696117, + "tokens_seen": 1066710016 + }, + { + "epoch": 3.05, + "learning_rate": 0.00034182547642928787, + "loss": 2.7343, + "theoretical_loss": 3.626945979695228, + "tokens_seen": 1066775552 + }, + { + "epoch": 3.05, + "learning_rate": 0.00034181544633901705, + "loss": 2.942, + "theoretical_loss": 3.626925028341917, + "tokens_seen": 1066841088 + }, + { + "epoch": 3.05, + "learning_rate": 0.00034180541624874623, + "loss": 3.002, + "theoretical_loss": 3.6269040786359534, + "tokens_seen": 1066906624 + }, + { + "epoch": 3.05, + "learning_rate": 0.00034179538615847547, + "loss": 2.9148, + "theoretical_loss": 3.626883130577106, + "tokens_seen": 1066972160 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003417853560682046, + "loss": 2.864, + "theoretical_loss": 3.6268621841651445, + "tokens_seen": 1067037696 + }, + { + "epoch": 3.05, + "learning_rate": 0.00034177532597793383, + "loss": 2.8888, + "theoretical_loss": 3.626841239399839, + "tokens_seen": 1067103232 + }, + { + "epoch": 3.05, + "learning_rate": 0.00034176529588766295, + "loss": 2.9927, + "theoretical_loss": 3.626820296280958, + "tokens_seen": 1067168768 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003417552657973922, + "loss": 2.6687, + "theoretical_loss": 3.626799354808271, + "tokens_seen": 1067234304 + }, + { + "epoch": 3.05, + "learning_rate": 0.00034174523570712137, + "loss": 2.7331, + "theoretical_loss": 3.626778414981548, + "tokens_seen": 1067299840 + }, + { + "epoch": 3.05, + "learning_rate": 0.00034173520561685055, + "loss": 2.6517, + "theoretical_loss": 3.626757476800558, + "tokens_seen": 1067365376 + }, + { + "epoch": 3.05, + "learning_rate": 0.00034172517552657973, + "loss": 3.0107, + "theoretical_loss": 3.6267365402650715, + "tokens_seen": 1067430912 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003417151454363089, + "loss": 3.1109, + "theoretical_loss": 3.6267156053748577, + "tokens_seen": 1067496448 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003417051153460381, + "loss": 2.6219, + "theoretical_loss": 3.6266946721296858, + "tokens_seen": 1067561984 + }, + { + "epoch": 3.05, + "learning_rate": 0.00034169508525576733, + "loss": 2.7849, + "theoretical_loss": 3.626673740529326, + "tokens_seen": 1067627520 + }, + { + "epoch": 3.05, + "learning_rate": 0.00034168505516549646, + "loss": 2.7899, + "theoretical_loss": 3.6266528105735487, + "tokens_seen": 1067693056 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003416750250752257, + "loss": 2.6937, + "theoretical_loss": 3.6266318822621226, + "tokens_seen": 1067758592 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003416649949849548, + "loss": 2.7576, + "theoretical_loss": 3.6266109555948187, + "tokens_seen": 1067824128 + }, + { + "epoch": 3.05, + "learning_rate": 0.00034165496489468406, + "loss": 2.7506, + "theoretical_loss": 3.626590030571406, + "tokens_seen": 1067889664 + }, + { + "epoch": 3.05, + "learning_rate": 0.00034164493480441324, + "loss": 2.7177, + "theoretical_loss": 3.626569107191655, + "tokens_seen": 1067955200 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003416349047141424, + "loss": 2.6327, + "theoretical_loss": 3.6265481854553356, + "tokens_seen": 1068020736 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003416248746238716, + "loss": 2.8429, + "theoretical_loss": 3.6265272653622183, + "tokens_seen": 1068086272 + }, + { + "epoch": 3.05, + "learning_rate": 0.00034161484453360083, + "loss": 2.7674, + "theoretical_loss": 3.6265063469120724, + "tokens_seen": 1068151808 + }, + { + "epoch": 3.05, + "learning_rate": 0.00034160481444332996, + "loss": 2.796, + "theoretical_loss": 3.626485430104669, + "tokens_seen": 1068217344 + }, + { + "epoch": 3.05, + "objective/train/docs_used": 1200699, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.70048451423645, + "objective/train/theoretical_loss": 3.6264802011594695, + "objective/train/tokens_used": 1088693728, + "theoretical_loss": 3.6264802011594695, + "tokens_seen": 1068233728 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003415947843530592, + "loss": 2.6839, + "theoretical_loss": 3.6264645149397774, + "tokens_seen": 1068282880 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003415847542627884, + "loss": 2.7033, + "theoretical_loss": 3.6264436014171686, + "tokens_seen": 1068348416 + }, + { + "epoch": 3.05, + "learning_rate": 0.00034157472417251756, + "loss": 2.6647, + "theoretical_loss": 3.6264226895366125, + "tokens_seen": 1068413952 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003415646940822468, + "loss": 2.8639, + "theoretical_loss": 3.62640177929788, + "tokens_seen": 1068479488 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003415546639919759, + "loss": 2.8761, + "theoretical_loss": 3.626380870700741, + "tokens_seen": 1068545024 + }, + { + "epoch": 3.05, + "learning_rate": 0.00034154463390170516, + "loss": 2.8552, + "theoretical_loss": 3.6263599637449655, + "tokens_seen": 1068610560 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003415346038114343, + "loss": 2.8734, + "theoretical_loss": 3.6263390584303252, + "tokens_seen": 1068676096 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003415245737211635, + "loss": 2.7517, + "theoretical_loss": 3.6263181547565893, + "tokens_seen": 1068741632 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003415145436308927, + "loss": 2.7747, + "theoretical_loss": 3.62629725272353, + "tokens_seen": 1068807168 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003415045135406219, + "loss": 2.8721, + "theoretical_loss": 3.6262763523309163, + "tokens_seen": 1068872704 + }, + { + "epoch": 3.05, + "learning_rate": 0.00034149448345035106, + "loss": 2.7056, + "theoretical_loss": 3.62625545357852, + "tokens_seen": 1068938240 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003414844533600803, + "loss": 2.6699, + "theoretical_loss": 3.6262345564661116, + "tokens_seen": 1069003776 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003414744232698094, + "loss": 2.8384, + "theoretical_loss": 3.6262136609934617, + "tokens_seen": 1069069312 + }, + { + "epoch": 3.05, + "learning_rate": 0.00034146439317953866, + "loss": 2.6113, + "theoretical_loss": 3.626192767160341, + "tokens_seen": 1069134848 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003414543630892678, + "loss": 2.8847, + "theoretical_loss": 3.6261718749665204, + "tokens_seen": 1069200384 + }, + { + "epoch": 3.05, + "learning_rate": 0.000341444332998997, + "loss": 2.9789, + "theoretical_loss": 3.6261509844117708, + "tokens_seen": 1069265920 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003414343029087262, + "loss": 2.69, + "theoretical_loss": 3.626130095495864, + "tokens_seen": 1069331456 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003414242728184554, + "loss": 2.8193, + "theoretical_loss": 3.62610920821857, + "tokens_seen": 1069396992 + }, + { + "epoch": 3.05, + "learning_rate": 0.00034141424272818456, + "loss": 2.7256, + "theoretical_loss": 3.62608832257966, + "tokens_seen": 1069462528 + }, + { + "epoch": 3.05, + "learning_rate": 0.00034140421263791375, + "loss": 2.8196, + "theoretical_loss": 3.626067438578905, + "tokens_seen": 1069528064 + }, + { + "epoch": 3.05, + "learning_rate": 0.00034139418254764293, + "loss": 2.8286, + "theoretical_loss": 3.626046556216077, + "tokens_seen": 1069593600 + }, + { + "epoch": 3.05, + "learning_rate": 0.00034138415245737216, + "loss": 2.8624, + "theoretical_loss": 3.626025675490946, + "tokens_seen": 1069659136 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003413741223671013, + "loss": 2.8349, + "theoretical_loss": 3.6260047964032847, + "tokens_seen": 1069724672 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003413640922768305, + "loss": 2.8984, + "theoretical_loss": 3.625983918952863, + "tokens_seen": 1069790208 + }, + { + "epoch": 3.05, + "learning_rate": 0.00034135406218655965, + "loss": 2.7387, + "theoretical_loss": 3.6259630431394525, + "tokens_seen": 1069855744 + }, + { + "epoch": 3.05, + "objective/train/docs_used": 1201919, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 1.7883902788162231, + "objective/train/theoretical_loss": 3.625957824441856, + "objective/train/tokens_used": 1090332128, + "theoretical_loss": 3.625957824441856, + "tokens_seen": 1069872128 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003413440320962889, + "loss": 2.4208, + "theoretical_loss": 3.6259421689628253, + "tokens_seen": 1069921280 + }, + { + "epoch": 3.05, + "learning_rate": 0.00034133400200601807, + "loss": 2.8423, + "theoretical_loss": 3.625921296422752, + "tokens_seen": 1069986816 + }, + { + "epoch": 3.05, + "learning_rate": 0.00034132397191574725, + "loss": 2.7369, + "theoretical_loss": 3.625900425519004, + "tokens_seen": 1070052352 + }, + { + "epoch": 3.05, + "learning_rate": 0.00034131394182547643, + "loss": 2.9399, + "theoretical_loss": 3.6258795562513537, + "tokens_seen": 1070117888 + }, + { + "epoch": 3.05, + "learning_rate": 0.00034130391173520567, + "loss": 2.7031, + "theoretical_loss": 3.6258586886195725, + "tokens_seen": 1070183424 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003412938816449348, + "loss": 2.8105, + "theoretical_loss": 3.6258378226234314, + "tokens_seen": 1070248960 + }, + { + "epoch": 3.05, + "learning_rate": 0.00034128385155466403, + "loss": 2.6984, + "theoretical_loss": 3.6258169582627024, + "tokens_seen": 1070314496 + }, + { + "epoch": 3.05, + "learning_rate": 0.00034127382146439315, + "loss": 2.7699, + "theoretical_loss": 3.625796095537157, + "tokens_seen": 1070380032 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003412637913741224, + "loss": 2.79, + "theoretical_loss": 3.625775234446567, + "tokens_seen": 1070445568 + }, + { + "epoch": 3.05, + "learning_rate": 0.00034125376128385157, + "loss": 2.5291, + "theoretical_loss": 3.6257543749907044, + "tokens_seen": 1070511104 + }, + { + "epoch": 3.05, + "learning_rate": 0.00034124373119358075, + "loss": 2.7412, + "theoretical_loss": 3.625733517169341, + "tokens_seen": 1070576640 + }, + { + "epoch": 3.05, + "learning_rate": 0.00034123370110330993, + "loss": 2.7322, + "theoretical_loss": 3.6257126609822485, + "tokens_seen": 1070642176 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003412236710130391, + "loss": 2.7944, + "theoretical_loss": 3.6256918064291987, + "tokens_seen": 1070707712 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003412136409227683, + "loss": 2.6609, + "theoretical_loss": 3.625670953509964, + "tokens_seen": 1070773248 + }, + { + "epoch": 3.05, + "learning_rate": 0.00034120361083249753, + "loss": 2.6465, + "theoretical_loss": 3.625650102224316, + "tokens_seen": 1070838784 + }, + { + "epoch": 3.05, + "learning_rate": 0.00034119358074222666, + "loss": 2.5707, + "theoretical_loss": 3.6256292525720273, + "tokens_seen": 1070904320 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003411835506519559, + "loss": 2.4912, + "theoretical_loss": 3.625608404552869, + "tokens_seen": 1070969856 + }, + { + "epoch": 3.05, + "learning_rate": 0.000341173520561685, + "loss": 2.5813, + "theoretical_loss": 3.6255875581666146, + "tokens_seen": 1071035392 + }, + { + "epoch": 3.05, + "learning_rate": 0.00034116349047141426, + "loss": 2.8009, + "theoretical_loss": 3.6255667134130354, + "tokens_seen": 1071100928 + }, + { + "epoch": 3.05, + "learning_rate": 0.00034115346038114344, + "loss": 2.7488, + "theoretical_loss": 3.6255458702919037, + "tokens_seen": 1071166464 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003411434302908726, + "loss": 2.8851, + "theoretical_loss": 3.6255250288029917, + "tokens_seen": 1071232000 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003411334002006018, + "loss": 2.879, + "theoretical_loss": 3.6255041889460724, + "tokens_seen": 1071297536 + }, + { + "epoch": 3.05, + "learning_rate": 0.00034112337011033103, + "loss": 2.8509, + "theoretical_loss": 3.625483350720918, + "tokens_seen": 1071363072 + }, + { + "epoch": 3.05, + "learning_rate": 0.00034111334002006016, + "loss": 2.7807, + "theoretical_loss": 3.6254625141273, + "tokens_seen": 1071428608 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003411033099297894, + "loss": 2.8458, + "theoretical_loss": 3.625441679164992, + "tokens_seen": 1071494144 + }, + { + "epoch": 3.05, + "objective/train/docs_used": 1202634, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.0612995624542236, + "objective/train/theoretical_loss": 3.6254364706792805, + "objective/train/tokens_used": 1091970528, + "theoretical_loss": 3.6254364706792805, + "tokens_seen": 1071510528 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003410932798395185, + "loss": 2.8313, + "theoretical_loss": 3.6254208458337662, + "tokens_seen": 1071559680 + }, + { + "epoch": 3.05, + "learning_rate": 0.00034108324974924776, + "loss": 2.7383, + "theoretical_loss": 3.625400014133395, + "tokens_seen": 1071625216 + }, + { + "epoch": 3.05, + "learning_rate": 0.00034107321965897694, + "loss": 2.9735, + "theoretical_loss": 3.6253791840636507, + "tokens_seen": 1071690752 + }, + { + "epoch": 3.05, + "learning_rate": 0.0003410631895687061, + "loss": 2.9058, + "theoretical_loss": 3.6253583556243063, + "tokens_seen": 1071756288 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003410531594784353, + "loss": 2.8198, + "theoretical_loss": 3.625337528815135, + "tokens_seen": 1071821824 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003410431293881645, + "loss": 2.8492, + "theoretical_loss": 3.6253167036359084, + "tokens_seen": 1071887360 + }, + { + "epoch": 3.06, + "learning_rate": 0.00034103309929789366, + "loss": 2.7865, + "theoretical_loss": 3.6252958800864006, + "tokens_seen": 1071952896 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003410230692076229, + "loss": 2.9489, + "theoretical_loss": 3.625275058166384, + "tokens_seen": 1072018432 + }, + { + "epoch": 3.06, + "learning_rate": 0.000341013039117352, + "loss": 2.99, + "theoretical_loss": 3.6252542378756303, + "tokens_seen": 1072083968 + }, + { + "epoch": 3.06, + "learning_rate": 0.00034100300902708126, + "loss": 2.6486, + "theoretical_loss": 3.6252334192139144, + "tokens_seen": 1072149504 + }, + { + "epoch": 3.06, + "learning_rate": 0.00034099297893681044, + "loss": 2.6888, + "theoretical_loss": 3.6252126021810076, + "tokens_seen": 1072215040 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003409829488465396, + "loss": 2.738, + "theoretical_loss": 3.625191786776684, + "tokens_seen": 1072280576 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003409729187562688, + "loss": 2.8253, + "theoretical_loss": 3.6251709730007162, + "tokens_seen": 1072346112 + }, + { + "epoch": 3.06, + "learning_rate": 0.000340962888665998, + "loss": 2.5057, + "theoretical_loss": 3.6251501608528773, + "tokens_seen": 1072411648 + }, + { + "epoch": 3.06, + "learning_rate": 0.00034095285857572717, + "loss": 2.8812, + "theoretical_loss": 3.6251293503329407, + "tokens_seen": 1072477184 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003409428284854564, + "loss": 2.9102, + "theoretical_loss": 3.625108541440679, + "tokens_seen": 1072542720 + }, + { + "epoch": 3.06, + "learning_rate": 0.00034093279839518553, + "loss": 2.6922, + "theoretical_loss": 3.6250877341758665, + "tokens_seen": 1072608256 + }, + { + "epoch": 3.06, + "learning_rate": 0.00034092276830491477, + "loss": 2.8168, + "theoretical_loss": 3.6250669285382755, + "tokens_seen": 1072673792 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003409127382146439, + "loss": 2.7439, + "theoretical_loss": 3.62504612452768, + "tokens_seen": 1072739328 + }, + { + "epoch": 3.06, + "learning_rate": 0.00034090270812437313, + "loss": 2.9342, + "theoretical_loss": 3.625025322143853, + "tokens_seen": 1072804864 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003408926780341023, + "loss": 2.8336, + "theoretical_loss": 3.625004521386568, + "tokens_seen": 1072870400 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003408826479438315, + "loss": 2.8846, + "theoretical_loss": 3.6249837222555987, + "tokens_seen": 1072935936 + }, + { + "epoch": 3.06, + "learning_rate": 0.00034087261785356067, + "loss": 2.703, + "theoretical_loss": 3.624962924750718, + "tokens_seen": 1073001472 + }, + { + "epoch": 3.06, + "learning_rate": 0.00034086258776328985, + "loss": 2.6356, + "theoretical_loss": 3.6249421288717, + "tokens_seen": 1073067008 + }, + { + "epoch": 3.06, + "learning_rate": 0.00034085255767301903, + "loss": 2.8443, + "theoretical_loss": 3.6249213346183184, + "tokens_seen": 1073132544 + }, + { + "epoch": 3.06, + "objective/train/docs_used": 1203719, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.52689266204834, + "objective/train/theoretical_loss": 3.6249161363089524, + "objective/train/tokens_used": 1093608928, + "theoretical_loss": 3.6249161363089524, + "tokens_seen": 1073148928 + }, + { + "epoch": 3.06, + "learning_rate": 0.00034084252758274827, + "loss": 2.6909, + "theoretical_loss": 3.624900541990346, + "tokens_seen": 1073198080 + }, + { + "epoch": 3.06, + "learning_rate": 0.00034083249749247745, + "loss": 2.9078, + "theoretical_loss": 3.6248797509875583, + "tokens_seen": 1073263616 + }, + { + "epoch": 3.06, + "learning_rate": 0.00034082246740220663, + "loss": 2.8621, + "theoretical_loss": 3.6248589616097275, + "tokens_seen": 1073329152 + }, + { + "epoch": 3.06, + "learning_rate": 0.00034081243731193587, + "loss": 2.7082, + "theoretical_loss": 3.6248381738566278, + "tokens_seen": 1073394688 + }, + { + "epoch": 3.06, + "learning_rate": 0.000340802407221665, + "loss": 2.8163, + "theoretical_loss": 3.624817387728033, + "tokens_seen": 1073460224 + }, + { + "epoch": 3.06, + "learning_rate": 0.00034079237713139423, + "loss": 2.8633, + "theoretical_loss": 3.624796603223717, + "tokens_seen": 1073525760 + }, + { + "epoch": 3.06, + "learning_rate": 0.00034078234704112335, + "loss": 2.6537, + "theoretical_loss": 3.6247758203434537, + "tokens_seen": 1073591296 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003407723169508526, + "loss": 2.843, + "theoretical_loss": 3.624755039087017, + "tokens_seen": 1073656832 + }, + { + "epoch": 3.06, + "learning_rate": 0.00034076228686058177, + "loss": 2.9657, + "theoretical_loss": 3.624734259454182, + "tokens_seen": 1073722368 + }, + { + "epoch": 3.06, + "learning_rate": 0.00034075225677031095, + "loss": 2.7164, + "theoretical_loss": 3.6247134814447204, + "tokens_seen": 1073787904 + }, + { + "epoch": 3.06, + "learning_rate": 0.00034074222668004013, + "loss": 2.6927, + "theoretical_loss": 3.6246927050584086, + "tokens_seen": 1073853440 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003407321965897693, + "loss": 2.7393, + "theoretical_loss": 3.62467193029502, + "tokens_seen": 1073918976 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003407221664994985, + "loss": 2.8091, + "theoretical_loss": 3.624651157154328, + "tokens_seen": 1073984512 + }, + { + "epoch": 3.06, + "learning_rate": 0.00034071213640922773, + "loss": 2.6501, + "theoretical_loss": 3.6246303856361077, + "tokens_seen": 1074050048 + }, + { + "epoch": 3.06, + "learning_rate": 0.00034070210631895686, + "loss": 2.7927, + "theoretical_loss": 3.6246096157401335, + "tokens_seen": 1074115584 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003406920762286861, + "loss": 2.8565, + "theoretical_loss": 3.6245888474661796, + "tokens_seen": 1074181120 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003406820461384152, + "loss": 2.7706, + "theoretical_loss": 3.62456808081402, + "tokens_seen": 1074246656 + }, + { + "epoch": 3.06, + "learning_rate": 0.00034067201604814446, + "loss": 2.7304, + "theoretical_loss": 3.6245473157834294, + "tokens_seen": 1074312192 + }, + { + "epoch": 3.06, + "learning_rate": 0.00034066198595787364, + "loss": 2.8795, + "theoretical_loss": 3.6245265523741823, + "tokens_seen": 1074377728 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003406519558676028, + "loss": 2.8952, + "theoretical_loss": 3.6245057905860527, + "tokens_seen": 1074443264 + }, + { + "epoch": 3.06, + "learning_rate": 0.000340641925777332, + "loss": 2.854, + "theoretical_loss": 3.624485030418816, + "tokens_seen": 1074508800 + }, + { + "epoch": 3.06, + "learning_rate": 0.00034063189568706123, + "loss": 2.7502, + "theoretical_loss": 3.624464271872246, + "tokens_seen": 1074574336 + }, + { + "epoch": 3.06, + "learning_rate": 0.00034062186559679036, + "loss": 2.5958, + "theoretical_loss": 3.6244435149461185, + "tokens_seen": 1074639872 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003406118355065196, + "loss": 2.8345, + "theoretical_loss": 3.624422759640207, + "tokens_seen": 1074705408 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003406018054162487, + "loss": 2.8674, + "theoretical_loss": 3.6244020059542867, + "tokens_seen": 1074770944 + }, + { + "epoch": 3.06, + "objective/train/docs_used": 1204483, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6257479190826416, + "objective/train/theoretical_loss": 3.624396817785904, + "objective/train/tokens_used": 1095247328, + "theoretical_loss": 3.624396817785904, + "tokens_seen": 1074787328 + }, + { + "epoch": 3.06, + "learning_rate": 0.00034059177532597796, + "loss": 2.763, + "theoretical_loss": 3.6243812538881324, + "tokens_seen": 1074836480 + }, + { + "epoch": 3.06, + "learning_rate": 0.00034058174523570714, + "loss": 2.8225, + "theoretical_loss": 3.624360503441519, + "tokens_seen": 1074902016 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003405717151454363, + "loss": 2.7167, + "theoretical_loss": 3.6243397546142218, + "tokens_seen": 1074967552 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003405616850551655, + "loss": 2.7695, + "theoretical_loss": 3.6243190074060143, + "tokens_seen": 1075033088 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003405516549648947, + "loss": 2.664, + "theoretical_loss": 3.624298261816673, + "tokens_seen": 1075098624 + }, + { + "epoch": 3.06, + "learning_rate": 0.00034054162487462386, + "loss": 2.8657, + "theoretical_loss": 3.6242775178459716, + "tokens_seen": 1075164160 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003405315947843531, + "loss": 2.707, + "theoretical_loss": 3.624256775493687, + "tokens_seen": 1075229696 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003405215646940822, + "loss": 2.8292, + "theoretical_loss": 3.6242360347595923, + "tokens_seen": 1075295232 + }, + { + "epoch": 3.06, + "learning_rate": 0.00034051153460381146, + "loss": 2.7094, + "theoretical_loss": 3.6242152956434635, + "tokens_seen": 1075360768 + }, + { + "epoch": 3.06, + "learning_rate": 0.00034050150451354064, + "loss": 2.9164, + "theoretical_loss": 3.624194558145076, + "tokens_seen": 1075426304 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003404914744232698, + "loss": 2.6311, + "theoretical_loss": 3.624173822264205, + "tokens_seen": 1075491840 + }, + { + "epoch": 3.06, + "learning_rate": 0.000340481444332999, + "loss": 2.5965, + "theoretical_loss": 3.624153088000625, + "tokens_seen": 1075557376 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003404714142427282, + "loss": 2.8016, + "theoretical_loss": 3.6241323553541127, + "tokens_seen": 1075622912 + }, + { + "epoch": 3.06, + "learning_rate": 0.00034046138415245737, + "loss": 2.7171, + "theoretical_loss": 3.624111624324442, + "tokens_seen": 1075688448 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003404513540621866, + "loss": 2.8304, + "theoretical_loss": 3.6240908949113897, + "tokens_seen": 1075753984 + }, + { + "epoch": 3.06, + "learning_rate": 0.00034044132397191573, + "loss": 2.9595, + "theoretical_loss": 3.62407016711473, + "tokens_seen": 1075819520 + }, + { + "epoch": 3.06, + "learning_rate": 0.00034043129388164497, + "loss": 2.6081, + "theoretical_loss": 3.6240494409342396, + "tokens_seen": 1075885056 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003404212637913741, + "loss": 3.0219, + "theoretical_loss": 3.6240287163696934, + "tokens_seen": 1075950592 + }, + { + "epoch": 3.06, + "learning_rate": 0.00034041123370110333, + "loss": 2.8945, + "theoretical_loss": 3.624007993420866, + "tokens_seen": 1076016128 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003404012036108325, + "loss": 2.9308, + "theoretical_loss": 3.6239872720875352, + "tokens_seen": 1076081664 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003403911735205617, + "loss": 2.6346, + "theoretical_loss": 3.623966552369475, + "tokens_seen": 1076147200 + }, + { + "epoch": 3.06, + "learning_rate": 0.00034038114343029087, + "loss": 2.754, + "theoretical_loss": 3.623945834266462, + "tokens_seen": 1076212736 + }, + { + "epoch": 3.06, + "learning_rate": 0.00034037111334002005, + "loss": 2.5849, + "theoretical_loss": 3.6239251177782714, + "tokens_seen": 1076278272 + }, + { + "epoch": 3.06, + "learning_rate": 0.00034036108324974923, + "loss": 2.8053, + "theoretical_loss": 3.6239044029046794, + "tokens_seen": 1076343808 + }, + { + "epoch": 3.06, + "learning_rate": 0.00034035105315947847, + "loss": 2.7989, + "theoretical_loss": 3.623883689645462, + "tokens_seen": 1076409344 + }, + { + "epoch": 3.06, + "objective/train/docs_used": 1205647, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.873117446899414, + "objective/train/theoretical_loss": 3.623878511582877, + "objective/train/tokens_used": 1096885728, + "theoretical_loss": 3.623878511582877, + "tokens_seen": 1076425728 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003403410230692076, + "loss": 2.8016, + "theoretical_loss": 3.6238629780003944, + "tokens_seen": 1076474880 + }, + { + "epoch": 3.06, + "learning_rate": 0.00034033099297893683, + "loss": 2.8755, + "theoretical_loss": 3.6238422679692532, + "tokens_seen": 1076540416 + }, + { + "epoch": 3.06, + "learning_rate": 0.000340320962888666, + "loss": 2.6786, + "theoretical_loss": 3.623821559551814, + "tokens_seen": 1076605952 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003403109327983952, + "loss": 2.5573, + "theoretical_loss": 3.623800852747854, + "tokens_seen": 1076671488 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003403009027081244, + "loss": 2.8492, + "theoretical_loss": 3.623780147557147, + "tokens_seen": 1076737024 + }, + { + "epoch": 3.06, + "learning_rate": 0.00034029087261785356, + "loss": 2.7657, + "theoretical_loss": 3.623759443979471, + "tokens_seen": 1076802560 + }, + { + "epoch": 3.06, + "learning_rate": 0.00034028084252758274, + "loss": 2.8066, + "theoretical_loss": 3.623738742014602, + "tokens_seen": 1076868096 + }, + { + "epoch": 3.06, + "learning_rate": 0.00034027081243731197, + "loss": 2.7101, + "theoretical_loss": 3.6237180416623156, + "tokens_seen": 1076933632 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003402607823470411, + "loss": 2.8152, + "theoretical_loss": 3.623697342922388, + "tokens_seen": 1076999168 + }, + { + "epoch": 3.06, + "learning_rate": 0.00034025075225677033, + "loss": 2.6212, + "theoretical_loss": 3.623676645794596, + "tokens_seen": 1077064704 + }, + { + "epoch": 3.06, + "learning_rate": 0.00034024072216649946, + "loss": 2.7761, + "theoretical_loss": 3.6236559502787165, + "tokens_seen": 1077130240 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003402306920762287, + "loss": 2.7202, + "theoretical_loss": 3.6236352563745244, + "tokens_seen": 1077195776 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003402206619859579, + "loss": 2.6571, + "theoretical_loss": 3.623614564081797, + "tokens_seen": 1077261312 + }, + { + "epoch": 3.06, + "learning_rate": 0.00034021063189568706, + "loss": 2.8752, + "theoretical_loss": 3.6235938734003117, + "tokens_seen": 1077326848 + }, + { + "epoch": 3.06, + "learning_rate": 0.00034020060180541624, + "loss": 2.6293, + "theoretical_loss": 3.6235731843298433, + "tokens_seen": 1077392384 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003401905717151454, + "loss": 2.7159, + "theoretical_loss": 3.623552496870169, + "tokens_seen": 1077457920 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003401805416248746, + "loss": 2.7924, + "theoretical_loss": 3.6235318110210657, + "tokens_seen": 1077523456 + }, + { + "epoch": 3.06, + "learning_rate": 0.00034017051153460384, + "loss": 2.7635, + "theoretical_loss": 3.62351112678231, + "tokens_seen": 1077588992 + }, + { + "epoch": 3.06, + "learning_rate": 0.00034016048144433296, + "loss": 2.9351, + "theoretical_loss": 3.6234904441536786, + "tokens_seen": 1077654528 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003401504513540622, + "loss": 2.9664, + "theoretical_loss": 3.623469763134948, + "tokens_seen": 1077720064 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003401404212637914, + "loss": 2.7644, + "theoretical_loss": 3.6234490837258955, + "tokens_seen": 1077785600 + }, + { + "epoch": 3.06, + "learning_rate": 0.00034013039117352056, + "loss": 2.8821, + "theoretical_loss": 3.6234284059262976, + "tokens_seen": 1077851136 + }, + { + "epoch": 3.06, + "learning_rate": 0.00034012036108324974, + "loss": 2.9963, + "theoretical_loss": 3.6234077297359315, + "tokens_seen": 1077916672 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003401103309929789, + "loss": 2.5783, + "theoretical_loss": 3.6233870551545735, + "tokens_seen": 1077982208 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003401003009027081, + "loss": 2.8951, + "theoretical_loss": 3.6233663821820015, + "tokens_seen": 1078047744 + }, + { + "epoch": 3.06, + "objective/train/docs_used": 1210610, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8817059993743896, + "objective/train/theoretical_loss": 3.6233612141902043, + "objective/train/tokens_used": 1098524128, + "theoretical_loss": 3.6233612141902043, + "tokens_seen": 1078064128 + }, + { + "epoch": 3.06, + "learning_rate": 0.00034009027081243734, + "loss": 2.9285, + "theoretical_loss": 3.623345710817991, + "tokens_seen": 1078113280 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003400802407221665, + "loss": 2.929, + "theoretical_loss": 3.623325041062321, + "tokens_seen": 1078178816 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003400702106318957, + "loss": 2.7234, + "theoretical_loss": 3.623304372914767, + "tokens_seen": 1078244352 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003400601805416249, + "loss": 2.6798, + "theoretical_loss": 3.6232837063751075, + "tokens_seen": 1078309888 + }, + { + "epoch": 3.06, + "learning_rate": 0.00034005015045135406, + "loss": 2.9209, + "theoretical_loss": 3.6232630414431184, + "tokens_seen": 1078375424 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003400401203610833, + "loss": 2.576, + "theoretical_loss": 3.6232423781185776, + "tokens_seen": 1078440960 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003400300902708124, + "loss": 2.7724, + "theoretical_loss": 3.623221716401263, + "tokens_seen": 1078506496 + }, + { + "epoch": 3.06, + "learning_rate": 0.00034002006018054166, + "loss": 2.7263, + "theoretical_loss": 3.6232010562909505, + "tokens_seen": 1078572032 + }, + { + "epoch": 3.06, + "learning_rate": 0.00034001003009027084, + "loss": 2.8475, + "theoretical_loss": 3.623180397787419, + "tokens_seen": 1078637568 + }, + { + "epoch": 3.06, + "learning_rate": 0.00034, + "loss": 2.863, + "theoretical_loss": 3.623159740890445, + "tokens_seen": 1078703104 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003399899699097292, + "loss": 2.8764, + "theoretical_loss": 3.6231390855998056, + "tokens_seen": 1078768640 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003399799398194584, + "loss": 2.6731, + "theoretical_loss": 3.6231184319152794, + "tokens_seen": 1078834176 + }, + { + "epoch": 3.06, + "learning_rate": 0.00033996990972918757, + "loss": 2.9074, + "theoretical_loss": 3.6230977798366437, + "tokens_seen": 1078899712 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003399598796389168, + "loss": 2.6713, + "theoretical_loss": 3.6230771293636757, + "tokens_seen": 1078965248 + }, + { + "epoch": 3.06, + "learning_rate": 0.00033994984954864593, + "loss": 2.5203, + "theoretical_loss": 3.623056480496153, + "tokens_seen": 1079030784 + }, + { + "epoch": 3.06, + "learning_rate": 0.00033993981945837517, + "loss": 2.8662, + "theoretical_loss": 3.6230358332338533, + "tokens_seen": 1079096320 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003399297893681043, + "loss": 2.7636, + "theoretical_loss": 3.623015187576555, + "tokens_seen": 1079161856 + }, + { + "epoch": 3.06, + "learning_rate": 0.00033991975927783353, + "loss": 2.7715, + "theoretical_loss": 3.622994543524035, + "tokens_seen": 1079227392 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003399097291875627, + "loss": 2.8445, + "theoretical_loss": 3.6229739010760715, + "tokens_seen": 1079292928 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003398996990972919, + "loss": 2.9655, + "theoretical_loss": 3.6229532602324426, + "tokens_seen": 1079358464 + }, + { + "epoch": 3.06, + "learning_rate": 0.00033988966900702107, + "loss": 2.6079, + "theoretical_loss": 3.622932620992926, + "tokens_seen": 1079424000 + }, + { + "epoch": 3.06, + "learning_rate": 0.00033987963891675025, + "loss": 2.6488, + "theoretical_loss": 3.6229119833572994, + "tokens_seen": 1079489536 + }, + { + "epoch": 3.06, + "learning_rate": 0.00033986960882647943, + "loss": 2.5739, + "theoretical_loss": 3.622891347325341, + "tokens_seen": 1079555072 + }, + { + "epoch": 3.06, + "learning_rate": 0.00033985957873620867, + "loss": 2.8467, + "theoretical_loss": 3.6228707128968294, + "tokens_seen": 1079620608 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003398495486459378, + "loss": 2.8359, + "theoretical_loss": 3.622850080071542, + "tokens_seen": 1079686144 + }, + { + "epoch": 3.06, + "objective/train/docs_used": 1215587, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9586844444274902, + "objective/train/theoretical_loss": 3.6228449221156978, + "objective/train/tokens_used": 1100162528, + "theoretical_loss": 3.6228449221156978, + "tokens_seen": 1079702528 + }, + { + "epoch": 3.06, + "learning_rate": 0.00033983951855566703, + "loss": 2.838, + "theoretical_loss": 3.6228294488492567, + "tokens_seen": 1079751680 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003398294884653962, + "loss": 2.6845, + "theoretical_loss": 3.6228088192297525, + "tokens_seen": 1079817216 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003398194583751254, + "loss": 2.6935, + "theoretical_loss": 3.622788191212807, + "tokens_seen": 1079882752 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003398094282848546, + "loss": 2.8, + "theoretical_loss": 3.622767564798199, + "tokens_seen": 1079948288 + }, + { + "epoch": 3.06, + "learning_rate": 0.00033979939819458376, + "loss": 2.8069, + "theoretical_loss": 3.622746939985706, + "tokens_seen": 1080013824 + }, + { + "epoch": 3.06, + "learning_rate": 0.00033978936810431294, + "loss": 2.7078, + "theoretical_loss": 3.622726316775107, + "tokens_seen": 1080079360 + }, + { + "epoch": 3.06, + "learning_rate": 0.00033977933801404217, + "loss": 2.7325, + "theoretical_loss": 3.6227056951661805, + "tokens_seen": 1080144896 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003397693079237713, + "loss": 2.7843, + "theoretical_loss": 3.6226850751587043, + "tokens_seen": 1080210432 + }, + { + "epoch": 3.06, + "learning_rate": 0.00033975927783350053, + "loss": 2.8463, + "theoretical_loss": 3.6226644567524575, + "tokens_seen": 1080275968 + }, + { + "epoch": 3.06, + "learning_rate": 0.00033974924774322966, + "loss": 2.7421, + "theoretical_loss": 3.622643839947218, + "tokens_seen": 1080341504 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003397392176529589, + "loss": 2.8555, + "theoretical_loss": 3.622623224742765, + "tokens_seen": 1080407040 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003397291875626881, + "loss": 2.8457, + "theoretical_loss": 3.622602611138877, + "tokens_seen": 1080472576 + }, + { + "epoch": 3.06, + "learning_rate": 0.00033971915747241726, + "loss": 2.8681, + "theoretical_loss": 3.6225819991353325, + "tokens_seen": 1080538112 + }, + { + "epoch": 3.06, + "learning_rate": 0.00033970912738214644, + "loss": 2.5826, + "theoretical_loss": 3.6225613887319104, + "tokens_seen": 1080603648 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003396990972918756, + "loss": 2.5601, + "theoretical_loss": 3.622540779928389, + "tokens_seen": 1080669184 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003396890672016048, + "loss": 2.7977, + "theoretical_loss": 3.6225201727245473, + "tokens_seen": 1080734720 + }, + { + "epoch": 3.06, + "learning_rate": 0.00033967903711133404, + "loss": 2.592, + "theoretical_loss": 3.6224995671201645, + "tokens_seen": 1080800256 + }, + { + "epoch": 3.06, + "learning_rate": 0.00033966900702106316, + "loss": 2.6062, + "theoretical_loss": 3.622478963115019, + "tokens_seen": 1080865792 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003396589769307924, + "loss": 2.6532, + "theoretical_loss": 3.6224583607088903, + "tokens_seen": 1080931328 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003396489468405216, + "loss": 2.7851, + "theoretical_loss": 3.6224377599015565, + "tokens_seen": 1080996864 + }, + { + "epoch": 3.06, + "learning_rate": 0.00033963891675025076, + "loss": 2.8381, + "theoretical_loss": 3.6224171606927973, + "tokens_seen": 1081062400 + }, + { + "epoch": 3.06, + "learning_rate": 0.00033962888665997994, + "loss": 2.8092, + "theoretical_loss": 3.6223965630823916, + "tokens_seen": 1081127936 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003396188565697091, + "loss": 2.791, + "theoretical_loss": 3.6223759670701186, + "tokens_seen": 1081193472 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003396088264794383, + "loss": 2.8194, + "theoretical_loss": 3.622355372655757, + "tokens_seen": 1081259008 + }, + { + "epoch": 3.06, + "learning_rate": 0.00033959879638916754, + "loss": 3.0145, + "theoretical_loss": 3.622334779839087, + "tokens_seen": 1081324544 + }, + { + "epoch": 3.06, + "objective/train/docs_used": 1220679, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.306006908416748, + "objective/train/theoretical_loss": 3.6223296318845324, + "objective/train/tokens_used": 1101800928, + "theoretical_loss": 3.6223296318845324, + "tokens_seen": 1081340928 + }, + { + "epoch": 3.06, + "learning_rate": 0.00033958876629889667, + "loss": 2.6374, + "theoretical_loss": 3.622314188619886, + "tokens_seen": 1081390080 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003395787362086259, + "loss": 2.7159, + "theoretical_loss": 3.6222935989979352, + "tokens_seen": 1081455616 + }, + { + "epoch": 3.06, + "learning_rate": 0.00033956870611835503, + "loss": 2.764, + "theoretical_loss": 3.622273010973013, + "tokens_seen": 1081521152 + }, + { + "epoch": 3.06, + "learning_rate": 0.00033955867602808426, + "loss": 2.8377, + "theoretical_loss": 3.622252424544899, + "tokens_seen": 1081586688 + }, + { + "epoch": 3.06, + "learning_rate": 0.00033954864593781345, + "loss": 2.7197, + "theoretical_loss": 3.622231839713372, + "tokens_seen": 1081652224 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003395386158475426, + "loss": 2.7794, + "theoretical_loss": 3.6222112564782125, + "tokens_seen": 1081717760 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003395285857572718, + "loss": 2.7446, + "theoretical_loss": 3.6221906748391994, + "tokens_seen": 1081783296 + }, + { + "epoch": 3.06, + "learning_rate": 0.00033951855566700104, + "loss": 2.8116, + "theoretical_loss": 3.622170094796112, + "tokens_seen": 1081848832 + }, + { + "epoch": 3.06, + "learning_rate": 0.00033950852557673017, + "loss": 2.8156, + "theoretical_loss": 3.6221495163487303, + "tokens_seen": 1081914368 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003394984954864594, + "loss": 2.8413, + "theoretical_loss": 3.622128939496834, + "tokens_seen": 1081979904 + }, + { + "epoch": 3.06, + "learning_rate": 0.00033948846539618853, + "loss": 2.7511, + "theoretical_loss": 3.6221083642402023, + "tokens_seen": 1082045440 + }, + { + "epoch": 3.06, + "learning_rate": 0.00033947843530591777, + "loss": 2.7974, + "theoretical_loss": 3.6220877905786155, + "tokens_seen": 1082110976 + }, + { + "epoch": 3.06, + "learning_rate": 0.00033946840521564695, + "loss": 2.7031, + "theoretical_loss": 3.6220672185118525, + "tokens_seen": 1082176512 + }, + { + "epoch": 3.06, + "learning_rate": 0.00033945837512537613, + "loss": 2.4711, + "theoretical_loss": 3.622046648039694, + "tokens_seen": 1082242048 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003394483450351053, + "loss": 2.6463, + "theoretical_loss": 3.6220260791619197, + "tokens_seen": 1082307584 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003394383149448345, + "loss": 2.7821, + "theoretical_loss": 3.622005511878309, + "tokens_seen": 1082373120 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003394282848545637, + "loss": 2.7633, + "theoretical_loss": 3.6219849461886424, + "tokens_seen": 1082438656 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003394182547642929, + "loss": 2.7907, + "theoretical_loss": 3.621964382092699, + "tokens_seen": 1082504192 + }, + { + "epoch": 3.06, + "learning_rate": 0.00033940822467402204, + "loss": 2.5404, + "theoretical_loss": 3.62194381959026, + "tokens_seen": 1082569728 + }, + { + "epoch": 3.06, + "learning_rate": 0.00033939819458375127, + "loss": 2.6696, + "theoretical_loss": 3.6219232586811048, + "tokens_seen": 1082635264 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003393881644934804, + "loss": 2.72, + "theoretical_loss": 3.6219026993650134, + "tokens_seen": 1082700800 + }, + { + "epoch": 3.06, + "learning_rate": 0.00033937813440320963, + "loss": 2.7538, + "theoretical_loss": 3.621882141641766, + "tokens_seen": 1082766336 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003393681043129388, + "loss": 2.7411, + "theoretical_loss": 3.6218615855111436, + "tokens_seen": 1082831872 + }, + { + "epoch": 3.06, + "learning_rate": 0.000339358074222668, + "loss": 2.7286, + "theoretical_loss": 3.6218410309729254, + "tokens_seen": 1082897408 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003393480441323972, + "loss": 3.0193, + "theoretical_loss": 3.6218204780268914, + "tokens_seen": 1082962944 + }, + { + "epoch": 3.06, + "objective/train/docs_used": 1225756, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9983677864074707, + "objective/train/theoretical_loss": 3.621815340039136, + "objective/train/tokens_used": 1103439328, + "theoretical_loss": 3.621815340039136, + "tokens_seen": 1082979328 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003393380140421264, + "loss": 2.7483, + "theoretical_loss": 3.6217999266728236, + "tokens_seen": 1083028480 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003393279839518556, + "loss": 2.7398, + "theoretical_loss": 3.621779376910501, + "tokens_seen": 1083094016 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003393179538615848, + "loss": 2.7203, + "theoretical_loss": 3.6217588287397042, + "tokens_seen": 1083159552 + }, + { + "epoch": 3.06, + "learning_rate": 0.00033930792377131396, + "loss": 2.6191, + "theoretical_loss": 3.621738282160214, + "tokens_seen": 1083225088 + }, + { + "epoch": 3.06, + "learning_rate": 0.00033929789368104314, + "loss": 2.97, + "theoretical_loss": 3.6217177371718106, + "tokens_seen": 1083290624 + }, + { + "epoch": 3.06, + "learning_rate": 0.00033928786359077237, + "loss": 2.6384, + "theoretical_loss": 3.621697193774275, + "tokens_seen": 1083356160 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003392778335005015, + "loss": 2.5806, + "theoretical_loss": 3.6216766519673866, + "tokens_seen": 1083421696 + }, + { + "epoch": 3.06, + "learning_rate": 0.00033926780341023073, + "loss": 2.8141, + "theoretical_loss": 3.621656111750928, + "tokens_seen": 1083487232 + }, + { + "epoch": 3.06, + "learning_rate": 0.00033925777331995986, + "loss": 2.7713, + "theoretical_loss": 3.621635573124678, + "tokens_seen": 1083552768 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003392477432296891, + "loss": 2.7386, + "theoretical_loss": 3.6216150360884183, + "tokens_seen": 1083618304 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003392377131394183, + "loss": 2.8081, + "theoretical_loss": 3.6215945006419297, + "tokens_seen": 1083683840 + }, + { + "epoch": 3.06, + "learning_rate": 0.00033922768304914746, + "loss": 2.8118, + "theoretical_loss": 3.621573966784992, + "tokens_seen": 1083749376 + }, + { + "epoch": 3.06, + "learning_rate": 0.00033921765295887664, + "loss": 3.0737, + "theoretical_loss": 3.621553434517388, + "tokens_seen": 1083814912 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003392076228686058, + "loss": 2.7613, + "theoretical_loss": 3.6215329038388964, + "tokens_seen": 1083880448 + }, + { + "epoch": 3.06, + "learning_rate": 0.000339197592778335, + "loss": 2.6075, + "theoretical_loss": 3.621512374749299, + "tokens_seen": 1083945984 + }, + { + "epoch": 3.06, + "learning_rate": 0.00033918756268806424, + "loss": 2.7083, + "theoretical_loss": 3.621491847248378, + "tokens_seen": 1084011520 + }, + { + "epoch": 3.06, + "learning_rate": 0.00033917753259779336, + "loss": 2.7446, + "theoretical_loss": 3.621471321335912, + "tokens_seen": 1084077056 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003391675025075226, + "loss": 2.9731, + "theoretical_loss": 3.6214507970116845, + "tokens_seen": 1084142592 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003391574724172518, + "loss": 2.687, + "theoretical_loss": 3.6214302742754754, + "tokens_seen": 1084208128 + }, + { + "epoch": 3.06, + "learning_rate": 0.00033914744232698096, + "loss": 2.7052, + "theoretical_loss": 3.6214097531270655, + "tokens_seen": 1084273664 + }, + { + "epoch": 3.06, + "learning_rate": 0.00033913741223671014, + "loss": 2.7163, + "theoretical_loss": 3.6213892335662363, + "tokens_seen": 1084339200 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003391273821464393, + "loss": 2.7634, + "theoretical_loss": 3.6213687155927694, + "tokens_seen": 1084404736 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003391173520561685, + "loss": 2.651, + "theoretical_loss": 3.6213481992064462, + "tokens_seen": 1084470272 + }, + { + "epoch": 3.06, + "learning_rate": 0.00033910732196589774, + "loss": 2.7872, + "theoretical_loss": 3.6213276844070474, + "tokens_seen": 1084535808 + }, + { + "epoch": 3.06, + "learning_rate": 0.00033909729187562687, + "loss": 2.7807, + "theoretical_loss": 3.6213071711943545, + "tokens_seen": 1084601344 + }, + { + "epoch": 3.06, + "objective/train/docs_used": 1230848, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.071620225906372, + "objective/train/theoretical_loss": 3.621302043139079, + "objective/train/tokens_used": 1105077728, + "theoretical_loss": 3.621302043139079, + "tokens_seen": 1084617728 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003390872617853561, + "loss": 2.2767, + "theoretical_loss": 3.6212866595681494, + "tokens_seen": 1084666880 + }, + { + "epoch": 3.06, + "learning_rate": 0.00033907723169508523, + "loss": 2.7164, + "theoretical_loss": 3.6212661495282132, + "tokens_seen": 1084732416 + }, + { + "epoch": 3.06, + "learning_rate": 0.00033906720160481446, + "loss": 2.5967, + "theoretical_loss": 3.621245641074328, + "tokens_seen": 1084797952 + }, + { + "epoch": 3.06, + "learning_rate": 0.00033905717151454365, + "loss": 2.7439, + "theoretical_loss": 3.6212251342062736, + "tokens_seen": 1084863488 + }, + { + "epoch": 3.06, + "learning_rate": 0.00033904714142427283, + "loss": 2.8463, + "theoretical_loss": 3.6212046289238335, + "tokens_seen": 1084929024 + }, + { + "epoch": 3.06, + "learning_rate": 0.000339037111334002, + "loss": 2.9693, + "theoretical_loss": 3.6211841252267885, + "tokens_seen": 1084994560 + }, + { + "epoch": 3.06, + "learning_rate": 0.00033902708124373124, + "loss": 2.7778, + "theoretical_loss": 3.6211636231149202, + "tokens_seen": 1085060096 + }, + { + "epoch": 3.06, + "learning_rate": 0.00033901705115346037, + "loss": 2.9191, + "theoretical_loss": 3.621143122588011, + "tokens_seen": 1085125632 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003390070210631896, + "loss": 2.738, + "theoretical_loss": 3.6211226236458414, + "tokens_seen": 1085191168 + }, + { + "epoch": 3.06, + "learning_rate": 0.00033899699097291873, + "loss": 2.6476, + "theoretical_loss": 3.6211021262881946, + "tokens_seen": 1085256704 + }, + { + "epoch": 3.06, + "learning_rate": 0.00033898696088264797, + "loss": 2.8807, + "theoretical_loss": 3.621081630514851, + "tokens_seen": 1085322240 + }, + { + "epoch": 3.06, + "learning_rate": 0.00033897693079237715, + "loss": 2.5783, + "theoretical_loss": 3.6210611363255945, + "tokens_seen": 1085387776 + }, + { + "epoch": 3.06, + "learning_rate": 0.00033896690070210633, + "loss": 2.8129, + "theoretical_loss": 3.621040643720205, + "tokens_seen": 1085453312 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003389568706118355, + "loss": 2.9223, + "theoretical_loss": 3.621020152698465, + "tokens_seen": 1085518848 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003389468405215647, + "loss": 2.8439, + "theoretical_loss": 3.6209996632601573, + "tokens_seen": 1085584384 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003389368104312939, + "loss": 2.4224, + "theoretical_loss": 3.620979175405064, + "tokens_seen": 1085649920 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003389267803410231, + "loss": 2.8556, + "theoretical_loss": 3.6209586891329657, + "tokens_seen": 1085715456 + }, + { + "epoch": 3.06, + "learning_rate": 0.00033891675025075224, + "loss": 2.9056, + "theoretical_loss": 3.620938204443646, + "tokens_seen": 1085780992 + }, + { + "epoch": 3.06, + "learning_rate": 0.00033890672016048147, + "loss": 2.9773, + "theoretical_loss": 3.6209177213368866, + "tokens_seen": 1085846528 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003388966900702106, + "loss": 2.5649, + "theoretical_loss": 3.6208972398124697, + "tokens_seen": 1085912064 + }, + { + "epoch": 3.06, + "learning_rate": 0.00033888665997993983, + "loss": 2.7481, + "theoretical_loss": 3.6208767598701774, + "tokens_seen": 1085977600 + }, + { + "epoch": 3.06, + "learning_rate": 0.000338876629889669, + "loss": 2.7252, + "theoretical_loss": 3.6208562815097927, + "tokens_seen": 1086043136 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003388665997993982, + "loss": 2.7418, + "theoretical_loss": 3.6208358047310973, + "tokens_seen": 1086108672 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003388565697091274, + "loss": 2.7919, + "theoretical_loss": 3.6208153295338734, + "tokens_seen": 1086174208 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003388465396188566, + "loss": 2.8255, + "theoretical_loss": 3.6207948559179046, + "tokens_seen": 1086239744 + }, + { + "epoch": 3.06, + "objective/train/docs_used": 1232129, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.1969611644744873, + "objective/train/theoretical_loss": 3.6207897377609575, + "objective/train/tokens_used": 1106716128, + "theoretical_loss": 3.6207897377609575, + "tokens_seen": 1086256128 + }, + { + "epoch": 3.06, + "learning_rate": 0.00033883650952858574, + "loss": 2.8786, + "theoretical_loss": 3.6207743838829725, + "tokens_seen": 1086305280 + }, + { + "epoch": 3.06, + "learning_rate": 0.000338826479438315, + "loss": 2.6704, + "theoretical_loss": 3.6207539134288593, + "tokens_seen": 1086370816 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003388164493480441, + "loss": 2.6908, + "theoretical_loss": 3.6207334445553485, + "tokens_seen": 1086436352 + }, + { + "epoch": 3.06, + "learning_rate": 0.00033880641925777334, + "loss": 2.5391, + "theoretical_loss": 3.6207129772622224, + "tokens_seen": 1086501888 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003387963891675025, + "loss": 2.8369, + "theoretical_loss": 3.6206925115492634, + "tokens_seen": 1086567424 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003387863590772317, + "loss": 2.7543, + "theoretical_loss": 3.620672047416254, + "tokens_seen": 1086632960 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003387763289869609, + "loss": 2.7815, + "theoretical_loss": 3.6206515848629777, + "tokens_seen": 1086698496 + }, + { + "epoch": 3.06, + "learning_rate": 0.00033876629889669006, + "loss": 2.6934, + "theoretical_loss": 3.6206311238892166, + "tokens_seen": 1086764032 + }, + { + "epoch": 3.06, + "learning_rate": 0.00033875626880641924, + "loss": 2.7152, + "theoretical_loss": 3.6206106644947544, + "tokens_seen": 1086829568 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003387462387161485, + "loss": 3.0106, + "theoretical_loss": 3.620590206679373, + "tokens_seen": 1086895104 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003387362086258776, + "loss": 2.9409, + "theoretical_loss": 3.620569750442856, + "tokens_seen": 1086960640 + }, + { + "epoch": 3.06, + "learning_rate": 0.00033872617853560684, + "loss": 2.7743, + "theoretical_loss": 3.6205492957849863, + "tokens_seen": 1087026176 + }, + { + "epoch": 3.06, + "learning_rate": 0.00033871614844533597, + "loss": 2.5937, + "theoretical_loss": 3.620528842705546, + "tokens_seen": 1087091712 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003387061183550652, + "loss": 2.9943, + "theoretical_loss": 3.6205083912043197, + "tokens_seen": 1087157248 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003386960882647944, + "loss": 2.6698, + "theoretical_loss": 3.620487941281089, + "tokens_seen": 1087222784 + }, + { + "epoch": 3.06, + "learning_rate": 0.00033868605817452356, + "loss": 2.5315, + "theoretical_loss": 3.6204674929356377, + "tokens_seen": 1087288320 + }, + { + "epoch": 3.06, + "learning_rate": 0.00033867602808425275, + "loss": 2.8809, + "theoretical_loss": 3.620447046167749, + "tokens_seen": 1087353856 + }, + { + "epoch": 3.06, + "learning_rate": 0.000338665997993982, + "loss": 2.6834, + "theoretical_loss": 3.620426600977207, + "tokens_seen": 1087419392 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003386559679037111, + "loss": 2.7171, + "theoretical_loss": 3.620406157363793, + "tokens_seen": 1087484928 + }, + { + "epoch": 3.06, + "learning_rate": 0.00033864593781344034, + "loss": 2.8595, + "theoretical_loss": 3.6203857153272923, + "tokens_seen": 1087550464 + }, + { + "epoch": 3.06, + "learning_rate": 0.00033863590772316947, + "loss": 2.8387, + "theoretical_loss": 3.6203652748674866, + "tokens_seen": 1087616000 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003386258776328987, + "loss": 2.9063, + "theoretical_loss": 3.6203448359841603, + "tokens_seen": 1087681536 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003386158475426279, + "loss": 2.7254, + "theoretical_loss": 3.6203243986770968, + "tokens_seen": 1087747072 + }, + { + "epoch": 3.06, + "learning_rate": 0.00033860581745235707, + "loss": 2.8621, + "theoretical_loss": 3.620303962946079, + "tokens_seen": 1087812608 + }, + { + "epoch": 3.06, + "learning_rate": 0.00033859578736208625, + "loss": 2.8689, + "theoretical_loss": 3.6202835287908908, + "tokens_seen": 1087878144 + }, + { + "epoch": 3.06, + "objective/train/docs_used": 1232782, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.946256637573242, + "objective/train/theoretical_loss": 3.620278420498292, + "objective/train/tokens_used": 1108354528, + "theoretical_loss": 3.620278420498292, + "tokens_seen": 1087894528 + }, + { + "epoch": 3.06, + "learning_rate": 0.00033858575727181543, + "loss": 2.9374, + "theoretical_loss": 3.620263096211316, + "tokens_seen": 1087943680 + }, + { + "epoch": 3.06, + "learning_rate": 0.00033857572718154467, + "loss": 2.9483, + "theoretical_loss": 3.6202426652071376, + "tokens_seen": 1088009216 + }, + { + "epoch": 3.06, + "learning_rate": 0.00033856569709127385, + "loss": 2.8696, + "theoretical_loss": 3.6202222357781397, + "tokens_seen": 1088074752 + }, + { + "epoch": 3.06, + "learning_rate": 0.00033855566700100303, + "loss": 2.6028, + "theoretical_loss": 3.620201807924106, + "tokens_seen": 1088140288 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003385456369107322, + "loss": 2.7729, + "theoretical_loss": 3.62018138164482, + "tokens_seen": 1088205824 + }, + { + "epoch": 3.06, + "learning_rate": 0.00033853560682046144, + "loss": 3.0473, + "theoretical_loss": 3.6201609569400657, + "tokens_seen": 1088271360 + }, + { + "epoch": 3.06, + "learning_rate": 0.00033852557673019057, + "loss": 2.5592, + "theoretical_loss": 3.620140533809627, + "tokens_seen": 1088336896 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003385155466399198, + "loss": 2.6389, + "theoretical_loss": 3.620120112253287, + "tokens_seen": 1088402432 + }, + { + "epoch": 3.06, + "learning_rate": 0.00033850551654964893, + "loss": 2.6717, + "theoretical_loss": 3.6200996922708306, + "tokens_seen": 1088467968 + }, + { + "epoch": 3.06, + "learning_rate": 0.00033849548645937817, + "loss": 2.7892, + "theoretical_loss": 3.620079273862041, + "tokens_seen": 1088533504 + }, + { + "epoch": 3.06, + "learning_rate": 0.00033848545636910735, + "loss": 2.7012, + "theoretical_loss": 3.620058857026703, + "tokens_seen": 1088599040 + }, + { + "epoch": 3.06, + "learning_rate": 0.00033847542627883653, + "loss": 2.4863, + "theoretical_loss": 3.6200384417646, + "tokens_seen": 1088664576 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003384653961885657, + "loss": 2.754, + "theoretical_loss": 3.6200180280755165, + "tokens_seen": 1088730112 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003384553660982949, + "loss": 2.6993, + "theoretical_loss": 3.619997615959236, + "tokens_seen": 1088795648 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003384453360080241, + "loss": 2.6705, + "theoretical_loss": 3.6199772054155432, + "tokens_seen": 1088861184 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003384353059177533, + "loss": 2.8679, + "theoretical_loss": 3.619956796444222, + "tokens_seen": 1088926720 + }, + { + "epoch": 3.06, + "learning_rate": 0.00033842527582748244, + "loss": 2.7584, + "theoretical_loss": 3.619936389045057, + "tokens_seen": 1088992256 + }, + { + "epoch": 3.06, + "learning_rate": 0.00033841524573721167, + "loss": 2.7861, + "theoretical_loss": 3.6199159832178323, + "tokens_seen": 1089057792 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003384052156469408, + "loss": 2.8264, + "theoretical_loss": 3.619895578962332, + "tokens_seen": 1089123328 + }, + { + "epoch": 3.06, + "learning_rate": 0.00033839518555667003, + "loss": 2.8653, + "theoretical_loss": 3.6198751762783408, + "tokens_seen": 1089188864 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003383851554663992, + "loss": 2.864, + "theoretical_loss": 3.619854775165643, + "tokens_seen": 1089254400 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003383751253761284, + "loss": 2.793, + "theoretical_loss": 3.6198343756240225, + "tokens_seen": 1089319936 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003383650952858576, + "loss": 2.7411, + "theoretical_loss": 3.6198139776532656, + "tokens_seen": 1089385472 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003383550651955868, + "loss": 2.8531, + "theoretical_loss": 3.6197935812531545, + "tokens_seen": 1089451008 + }, + { + "epoch": 3.06, + "learning_rate": 0.00033834503510531594, + "loss": 2.6371, + "theoretical_loss": 3.6197731864234752, + "tokens_seen": 1089516544 + }, + { + "epoch": 3.06, + "objective/train/docs_used": 1234372, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.325249433517456, + "objective/train/theoretical_loss": 3.6197680879614103, + "objective/train/tokens_used": 1109992928, + "theoretical_loss": 3.6197680879614103, + "tokens_seen": 1089532928 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003383350050150452, + "loss": 2.4403, + "theoretical_loss": 3.6197527931640123, + "tokens_seen": 1089582080 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003383249749247743, + "loss": 2.7791, + "theoretical_loss": 3.6197324014745504, + "tokens_seen": 1089647616 + }, + { + "epoch": 3.06, + "learning_rate": 0.00033831494483450354, + "loss": 2.9964, + "theoretical_loss": 3.6197120113548733, + "tokens_seen": 1089713152 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003383049147442327, + "loss": 2.7922, + "theoretical_loss": 3.619691622804767, + "tokens_seen": 1089778688 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003382948846539619, + "loss": 2.7788, + "theoretical_loss": 3.6196712358240157, + "tokens_seen": 1089844224 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003382848545636911, + "loss": 2.8688, + "theoretical_loss": 3.619650850412404, + "tokens_seen": 1089909760 + }, + { + "epoch": 3.06, + "learning_rate": 0.00033827482447342026, + "loss": 2.7936, + "theoretical_loss": 3.6196304665697174, + "tokens_seen": 1089975296 + }, + { + "epoch": 3.06, + "learning_rate": 0.00033826479438314944, + "loss": 2.8542, + "theoretical_loss": 3.619610084295741, + "tokens_seen": 1090040832 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003382547642928787, + "loss": 2.6332, + "theoretical_loss": 3.6195897035902584, + "tokens_seen": 1090106368 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003382447342026078, + "loss": 2.6517, + "theoretical_loss": 3.619569324453056, + "tokens_seen": 1090171904 + }, + { + "epoch": 3.06, + "learning_rate": 0.00033823470411233704, + "loss": 2.6189, + "theoretical_loss": 3.6195489468839184, + "tokens_seen": 1090237440 + }, + { + "epoch": 3.06, + "learning_rate": 0.00033822467402206617, + "loss": 2.8172, + "theoretical_loss": 3.619528570882631, + "tokens_seen": 1090302976 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003382146439317954, + "loss": 2.8496, + "theoretical_loss": 3.6195081964489777, + "tokens_seen": 1090368512 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003382046138415246, + "loss": 2.9001, + "theoretical_loss": 3.6194878235827455, + "tokens_seen": 1090434048 + }, + { + "epoch": 3.06, + "learning_rate": 0.00033819458375125376, + "loss": 3.004, + "theoretical_loss": 3.6194674522837182, + "tokens_seen": 1090499584 + }, + { + "epoch": 3.06, + "learning_rate": 0.00033818455366098295, + "loss": 2.5051, + "theoretical_loss": 3.6194470825516816, + "tokens_seen": 1090565120 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003381745235707122, + "loss": 2.6448, + "theoretical_loss": 3.619426714386421, + "tokens_seen": 1090630656 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003381644934804413, + "loss": 2.7133, + "theoretical_loss": 3.619406347787722, + "tokens_seen": 1090696192 + }, + { + "epoch": 3.06, + "learning_rate": 0.00033815446339017054, + "loss": 2.8944, + "theoretical_loss": 3.61938598275537, + "tokens_seen": 1090761728 + }, + { + "epoch": 3.06, + "learning_rate": 0.00033814443329989967, + "loss": 2.7101, + "theoretical_loss": 3.6193656192891495, + "tokens_seen": 1090827264 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003381344032096289, + "loss": 2.7511, + "theoretical_loss": 3.619345257388847, + "tokens_seen": 1090892800 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003381243731193581, + "loss": 2.7677, + "theoretical_loss": 3.6193248970542475, + "tokens_seen": 1090958336 + }, + { + "epoch": 3.06, + "learning_rate": 0.00033811434302908727, + "loss": 2.7238, + "theoretical_loss": 3.619304538285137, + "tokens_seen": 1091023872 + }, + { + "epoch": 3.06, + "learning_rate": 0.00033810431293881645, + "loss": 2.8521, + "theoretical_loss": 3.6192841810813006, + "tokens_seen": 1091089408 + }, + { + "epoch": 3.06, + "learning_rate": 0.00033809428284854563, + "loss": 2.5637, + "theoretical_loss": 3.6192638254425242, + "tokens_seen": 1091154944 + }, + { + "epoch": 3.06, + "objective/train/docs_used": 1235119, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5149593353271484, + "objective/train/theoretical_loss": 3.619258736777346, + "objective/train/tokens_used": 1111631328, + "theoretical_loss": 3.619258736777346, + "tokens_seen": 1091171328 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003380842527582748, + "loss": 2.4153, + "theoretical_loss": 3.619243471368594, + "tokens_seen": 1091220480 + }, + { + "epoch": 3.06, + "learning_rate": 0.00033807422266800405, + "loss": 2.6798, + "theoretical_loss": 3.6192231188592947, + "tokens_seen": 1091286016 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003380641925777332, + "loss": 2.8856, + "theoretical_loss": 3.6192027679144125, + "tokens_seen": 1091351552 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003380541624874624, + "loss": 2.6799, + "theoretical_loss": 3.619182418533734, + "tokens_seen": 1091417088 + }, + { + "epoch": 3.06, + "learning_rate": 0.00033804413239719154, + "loss": 2.7602, + "theoretical_loss": 3.619162070717044, + "tokens_seen": 1091482624 + }, + { + "epoch": 3.06, + "learning_rate": 0.00033803410230692077, + "loss": 2.7477, + "theoretical_loss": 3.6191417244641286, + "tokens_seen": 1091548160 + }, + { + "epoch": 3.06, + "learning_rate": 0.00033802407221664995, + "loss": 2.757, + "theoretical_loss": 3.619121379774774, + "tokens_seen": 1091613696 + }, + { + "epoch": 3.06, + "learning_rate": 0.00033801404212637913, + "loss": 2.7874, + "theoretical_loss": 3.6191010366487664, + "tokens_seen": 1091679232 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003380040120361083, + "loss": 2.8289, + "theoretical_loss": 3.619080695085891, + "tokens_seen": 1091744768 + }, + { + "epoch": 3.06, + "learning_rate": 0.00033799398194583755, + "loss": 2.8967, + "theoretical_loss": 3.619060355085934, + "tokens_seen": 1091810304 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003379839518555667, + "loss": 2.6525, + "theoretical_loss": 3.6190400166486834, + "tokens_seen": 1091875840 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003379739217652959, + "loss": 2.4763, + "theoretical_loss": 3.6190196797739227, + "tokens_seen": 1091941376 + }, + { + "epoch": 3.06, + "learning_rate": 0.00033796389167502504, + "loss": 2.6815, + "theoretical_loss": 3.6189993444614394, + "tokens_seen": 1092006912 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003379538615847543, + "loss": 2.6668, + "theoretical_loss": 3.61897901071102, + "tokens_seen": 1092072448 + }, + { + "epoch": 3.06, + "learning_rate": 0.00033794383149448346, + "loss": 2.7857, + "theoretical_loss": 3.61895867852245, + "tokens_seen": 1092137984 + }, + { + "epoch": 3.06, + "learning_rate": 0.00033793380140421264, + "loss": 2.7561, + "theoretical_loss": 3.6189383478955164, + "tokens_seen": 1092203520 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003379237713139418, + "loss": 2.8636, + "theoretical_loss": 3.618918018830005, + "tokens_seen": 1092269056 + }, + { + "epoch": 3.06, + "learning_rate": 0.000337913741223671, + "loss": 2.7024, + "theoretical_loss": 3.6188976913257025, + "tokens_seen": 1092334592 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003379037111334002, + "loss": 2.6201, + "theoretical_loss": 3.618877365382396, + "tokens_seen": 1092400128 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003378936810431294, + "loss": 2.7568, + "theoretical_loss": 3.6188570409998704, + "tokens_seen": 1092465664 + }, + { + "epoch": 3.06, + "learning_rate": 0.00033788365095285854, + "loss": 2.8051, + "theoretical_loss": 3.618836718177913, + "tokens_seen": 1092531200 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003378736208625878, + "loss": 2.7044, + "theoretical_loss": 3.618816396916311, + "tokens_seen": 1092596736 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003378635907723169, + "loss": 2.8435, + "theoretical_loss": 3.6187960772148506, + "tokens_seen": 1092662272 + }, + { + "epoch": 3.06, + "learning_rate": 0.00033785356068204614, + "loss": 2.6148, + "theoretical_loss": 3.6187757590733183, + "tokens_seen": 1092727808 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003378435305917753, + "loss": 2.8435, + "theoretical_loss": 3.6187554424915005, + "tokens_seen": 1092793344 + }, + { + "epoch": 3.06, + "objective/train/docs_used": 1236502, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.975677490234375, + "objective/train/theoretical_loss": 3.6187503635897267, + "objective/train/tokens_used": 1113269728, + "theoretical_loss": 3.6187503635897267, + "tokens_seen": 1092809728 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003378335005015045, + "loss": 2.84, + "theoretical_loss": 3.6187351274691846, + "tokens_seen": 1092858880 + }, + { + "epoch": 3.06, + "learning_rate": 0.00033782347041123374, + "loss": 2.7963, + "theoretical_loss": 3.6187148140061574, + "tokens_seen": 1092924416 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003378134403209629, + "loss": 2.7715, + "theoretical_loss": 3.618694502102205, + "tokens_seen": 1092989952 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003378034102306921, + "loss": 2.7411, + "theoretical_loss": 3.618674191757115, + "tokens_seen": 1093055488 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003377933801404213, + "loss": 2.8463, + "theoretical_loss": 3.618653882970674, + "tokens_seen": 1093121024 + }, + { + "epoch": 3.06, + "learning_rate": 0.00033778335005015046, + "loss": 2.7457, + "theoretical_loss": 3.6186335757426686, + "tokens_seen": 1093186560 + }, + { + "epoch": 3.06, + "learning_rate": 0.00033777331995987964, + "loss": 2.7479, + "theoretical_loss": 3.6186132700728866, + "tokens_seen": 1093252096 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003377632898696089, + "loss": 2.8089, + "theoretical_loss": 3.6185929659611142, + "tokens_seen": 1093317632 + }, + { + "epoch": 3.06, + "learning_rate": 0.000337753259779338, + "loss": 2.8787, + "theoretical_loss": 3.6185726634071393, + "tokens_seen": 1093383168 + }, + { + "epoch": 3.06, + "learning_rate": 0.00033774322968906724, + "loss": 2.6973, + "theoretical_loss": 3.6185523624107487, + "tokens_seen": 1093448704 + }, + { + "epoch": 3.06, + "learning_rate": 0.00033773319959879637, + "loss": 2.8744, + "theoretical_loss": 3.6185320629717292, + "tokens_seen": 1093514240 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003377231695085256, + "loss": 2.5628, + "theoretical_loss": 3.6185117650898677, + "tokens_seen": 1093579776 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003377131394182548, + "loss": 2.6778, + "theoretical_loss": 3.6184914687649528, + "tokens_seen": 1093645312 + }, + { + "epoch": 3.06, + "learning_rate": 0.00033770310932798396, + "loss": 2.8106, + "theoretical_loss": 3.6184711739967708, + "tokens_seen": 1093710848 + }, + { + "epoch": 3.06, + "learning_rate": 0.00033769307923771315, + "loss": 2.6888, + "theoretical_loss": 3.618450880785109, + "tokens_seen": 1093776384 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003376830491474424, + "loss": 2.8349, + "theoretical_loss": 3.618430589129755, + "tokens_seen": 1093841920 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003376730190571715, + "loss": 2.8202, + "theoretical_loss": 3.6184102990304963, + "tokens_seen": 1093907456 + }, + { + "epoch": 3.06, + "learning_rate": 0.00033766298896690074, + "loss": 2.6311, + "theoretical_loss": 3.6183900104871203, + "tokens_seen": 1093972992 + }, + { + "epoch": 3.06, + "learning_rate": 0.00033765295887662987, + "loss": 2.8867, + "theoretical_loss": 3.6183697234994145, + "tokens_seen": 1094038528 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003376429287863591, + "loss": 2.5946, + "theoretical_loss": 3.6183494380671664, + "tokens_seen": 1094104064 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003376328986960883, + "loss": 2.5993, + "theoretical_loss": 3.618329154190163, + "tokens_seen": 1094169600 + }, + { + "epoch": 3.06, + "learning_rate": 0.00033762286860581747, + "loss": 2.6903, + "theoretical_loss": 3.618308871868193, + "tokens_seen": 1094235136 + }, + { + "epoch": 3.06, + "learning_rate": 0.00033761283851554665, + "loss": 2.8344, + "theoretical_loss": 3.6182885911010434, + "tokens_seen": 1094300672 + }, + { + "epoch": 3.06, + "learning_rate": 0.00033760280842527583, + "loss": 2.6523, + "theoretical_loss": 3.618268311888502, + "tokens_seen": 1094366208 + }, + { + "epoch": 3.06, + "learning_rate": 0.000337592778335005, + "loss": 2.6933, + "theoretical_loss": 3.6182480342303567, + "tokens_seen": 1094431744 + }, + { + "epoch": 3.06, + "objective/train/docs_used": 1237024, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6148250102996826, + "objective/train/theoretical_loss": 3.6182429650586703, + "objective/train/tokens_used": 1114908128, + "theoretical_loss": 3.6182429650586703, + "tokens_seen": 1094448128 + }, + { + "epoch": 3.06, + "learning_rate": 0.00033758274824473425, + "loss": 2.6435, + "theoretical_loss": 3.6182277581263955, + "tokens_seen": 1094497280 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003375727181544634, + "loss": 2.6805, + "theoretical_loss": 3.6182074835764055, + "tokens_seen": 1094562816 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003375626880641926, + "loss": 2.5208, + "theoretical_loss": 3.6181872105801753, + "tokens_seen": 1094628352 + }, + { + "epoch": 3.06, + "learning_rate": 0.00033755265797392174, + "loss": 2.7542, + "theoretical_loss": 3.618166939137492, + "tokens_seen": 1094693888 + }, + { + "epoch": 3.06, + "learning_rate": 0.00033754262788365097, + "loss": 2.6371, + "theoretical_loss": 3.6181466692481443, + "tokens_seen": 1094759424 + }, + { + "epoch": 3.06, + "learning_rate": 0.00033753259779338015, + "loss": 2.8191, + "theoretical_loss": 3.61812640091192, + "tokens_seen": 1094824960 + }, + { + "epoch": 3.06, + "learning_rate": 0.00033752256770310933, + "loss": 2.7634, + "theoretical_loss": 3.6181061341286074, + "tokens_seen": 1094890496 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003375125376128385, + "loss": 2.7474, + "theoretical_loss": 3.6180858688979938, + "tokens_seen": 1094956032 + }, + { + "epoch": 3.06, + "learning_rate": 0.00033750250752256775, + "loss": 2.5913, + "theoretical_loss": 3.6180656052198676, + "tokens_seen": 1095021568 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003374924774322969, + "loss": 2.8893, + "theoretical_loss": 3.6180453430940176, + "tokens_seen": 1095087104 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003374824473420261, + "loss": 2.675, + "theoretical_loss": 3.618025082520232, + "tokens_seen": 1095152640 + }, + { + "epoch": 3.06, + "learning_rate": 0.00033747241725175524, + "loss": 2.7979, + "theoretical_loss": 3.618004823498297, + "tokens_seen": 1095218176 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003374623871614845, + "loss": 2.8325, + "theoretical_loss": 3.617984566028004, + "tokens_seen": 1095283712 + }, + { + "epoch": 3.06, + "learning_rate": 0.00033745235707121366, + "loss": 2.6937, + "theoretical_loss": 3.617964310109139, + "tokens_seen": 1095349248 + }, + { + "epoch": 3.06, + "learning_rate": 0.00033744232698094284, + "loss": 2.777, + "theoretical_loss": 3.617944055741492, + "tokens_seen": 1095414784 + }, + { + "epoch": 3.06, + "learning_rate": 0.000337432296890672, + "loss": 2.5714, + "theoretical_loss": 3.6179238029248495, + "tokens_seen": 1095480320 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003374222668004012, + "loss": 2.7678, + "theoretical_loss": 3.6179035516590017, + "tokens_seen": 1095545856 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003374122367101304, + "loss": 2.6847, + "theoretical_loss": 3.617883301943736, + "tokens_seen": 1095611392 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003374022066198596, + "loss": 2.733, + "theoretical_loss": 3.6178630537788417, + "tokens_seen": 1095676928 + }, + { + "epoch": 3.06, + "learning_rate": 0.00033739217652958874, + "loss": 2.5637, + "theoretical_loss": 3.6178428071641076, + "tokens_seen": 1095742464 + }, + { + "epoch": 3.06, + "learning_rate": 0.000337382146439318, + "loss": 2.716, + "theoretical_loss": 3.6178225620993207, + "tokens_seen": 1095808000 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003373721163490471, + "loss": 2.7915, + "theoretical_loss": 3.617802318584271, + "tokens_seen": 1095873536 + }, + { + "epoch": 3.06, + "learning_rate": 0.00033736208625877634, + "loss": 2.8944, + "theoretical_loss": 3.6177820766187474, + "tokens_seen": 1095939072 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003373520561685055, + "loss": 2.7548, + "theoretical_loss": 3.617761836202538, + "tokens_seen": 1096004608 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003373420260782347, + "loss": 2.6252, + "theoretical_loss": 3.6177415973354314, + "tokens_seen": 1096070144 + }, + { + "epoch": 3.06, + "objective/train/docs_used": 1237677, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6507112979888916, + "objective/train/theoretical_loss": 3.6177365378606776, + "objective/train/tokens_used": 1116546528, + "theoretical_loss": 3.6177365378606776, + "tokens_seen": 1096086528 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003373319959879639, + "loss": 2.7237, + "theoretical_loss": 3.6177213600172173, + "tokens_seen": 1096135680 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003373219658976931, + "loss": 2.6584, + "theoretical_loss": 3.6177011242476835, + "tokens_seen": 1096201216 + }, + { + "epoch": 3.06, + "learning_rate": 0.00033731193580742225, + "loss": 2.6034, + "theoretical_loss": 3.6176808900266204, + "tokens_seen": 1096266752 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003373019057171515, + "loss": 2.7871, + "theoretical_loss": 3.6176606573538153, + "tokens_seen": 1096332288 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003372918756268806, + "loss": 2.7767, + "theoretical_loss": 3.617640426229058, + "tokens_seen": 1096397824 + }, + { + "epoch": 3.06, + "learning_rate": 0.00033728184553660984, + "loss": 2.5391, + "theoretical_loss": 3.6176201966521373, + "tokens_seen": 1096463360 + }, + { + "epoch": 3.06, + "learning_rate": 0.000337271815446339, + "loss": 2.7906, + "theoretical_loss": 3.6175999686228426, + "tokens_seen": 1096528896 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003372617853560682, + "loss": 2.8622, + "theoretical_loss": 3.6175797421409626, + "tokens_seen": 1096594432 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003372517552657974, + "loss": 2.6529, + "theoretical_loss": 3.6175595172062867, + "tokens_seen": 1096659968 + }, + { + "epoch": 3.06, + "learning_rate": 0.00033724172517552657, + "loss": 2.5864, + "theoretical_loss": 3.617539293818605, + "tokens_seen": 1096725504 + }, + { + "epoch": 3.06, + "learning_rate": 0.00033723169508525575, + "loss": 2.6748, + "theoretical_loss": 3.6175190719777044, + "tokens_seen": 1096791040 + }, + { + "epoch": 3.06, + "learning_rate": 0.000337221664994985, + "loss": 2.7057, + "theoretical_loss": 3.6174988516833766, + "tokens_seen": 1096856576 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003372116349047141, + "loss": 2.7422, + "theoretical_loss": 3.617478632935409, + "tokens_seen": 1096922112 + }, + { + "epoch": 3.06, + "learning_rate": 0.00033720160481444335, + "loss": 2.813, + "theoretical_loss": 3.617458415733593, + "tokens_seen": 1096987648 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003371915747241725, + "loss": 2.785, + "theoretical_loss": 3.617438200077716, + "tokens_seen": 1097053184 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003371815446339017, + "loss": 2.5565, + "theoretical_loss": 3.617417985967569, + "tokens_seen": 1097118720 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003371715145436309, + "loss": 2.7586, + "theoretical_loss": 3.6173977734029403, + "tokens_seen": 1097184256 + }, + { + "epoch": 3.06, + "learning_rate": 0.00033716148445336007, + "loss": 2.7171, + "theoretical_loss": 3.6173775623836204, + "tokens_seen": 1097249792 + }, + { + "epoch": 3.06, + "learning_rate": 0.00033715145436308925, + "loss": 2.8573, + "theoretical_loss": 3.6173573529093983, + "tokens_seen": 1097315328 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003371414242728185, + "loss": 2.4732, + "theoretical_loss": 3.6173371449800635, + "tokens_seen": 1097380864 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003371313941825476, + "loss": 2.6875, + "theoretical_loss": 3.6173169385954065, + "tokens_seen": 1097446400 + }, + { + "epoch": 3.06, + "learning_rate": 0.00033712136409227685, + "loss": 2.6837, + "theoretical_loss": 3.6172967337552153, + "tokens_seen": 1097511936 + }, + { + "epoch": 3.06, + "learning_rate": 0.000337111334002006, + "loss": 2.6919, + "theoretical_loss": 3.6172765304592813, + "tokens_seen": 1097577472 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003371013039117352, + "loss": 2.5633, + "theoretical_loss": 3.617256328707394, + "tokens_seen": 1097643008 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003370912738214644, + "loss": 2.4857, + "theoretical_loss": 3.617236128499343, + "tokens_seen": 1097708544 + }, + { + "epoch": 3.06, + "objective/train/docs_used": 1238996, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.87542986869812, + "objective/train/theoretical_loss": 3.6172310786885298, + "objective/train/tokens_used": 1118184928, + "theoretical_loss": 3.6172310786885298, + "tokens_seen": 1097724928 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003370812437311936, + "loss": 2.9281, + "theoretical_loss": 3.6172159298349174, + "tokens_seen": 1097774080 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003370712136409228, + "loss": 2.849, + "theoretical_loss": 3.617195732713908, + "tokens_seen": 1097839616 + }, + { + "epoch": 3.06, + "learning_rate": 0.00033706118355065194, + "loss": 2.582, + "theoretical_loss": 3.617175537136105, + "tokens_seen": 1097905152 + }, + { + "epoch": 3.06, + "learning_rate": 0.00033705115346038117, + "loss": 2.6484, + "theoretical_loss": 3.617155343101297, + "tokens_seen": 1097970688 + }, + { + "epoch": 3.06, + "learning_rate": 0.00033704112337011035, + "loss": 2.8651, + "theoretical_loss": 3.6171351506092755, + "tokens_seen": 1098036224 + }, + { + "epoch": 3.06, + "learning_rate": 0.00033703109327983953, + "loss": 2.8084, + "theoretical_loss": 3.61711495965983, + "tokens_seen": 1098101760 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003370210631895687, + "loss": 2.72, + "theoretical_loss": 3.6170947702527503, + "tokens_seen": 1098167296 + }, + { + "epoch": 3.06, + "learning_rate": 0.00033701103309929795, + "loss": 2.5411, + "theoretical_loss": 3.6170745823878274, + "tokens_seen": 1098232832 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003370010030090271, + "loss": 2.634, + "theoretical_loss": 3.6170543960648507, + "tokens_seen": 1098298368 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003369909729187563, + "loss": 2.6711, + "theoretical_loss": 3.6170342112836105, + "tokens_seen": 1098363904 + }, + { + "epoch": 3.06, + "learning_rate": 0.00033698094282848544, + "loss": 2.7764, + "theoretical_loss": 3.6170140280438967, + "tokens_seen": 1098429440 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003369709127382147, + "loss": 2.6749, + "theoretical_loss": 3.6169938463455003, + "tokens_seen": 1098494976 + }, + { + "epoch": 3.06, + "learning_rate": 0.00033696088264794386, + "loss": 2.9666, + "theoretical_loss": 3.616973666188212, + "tokens_seen": 1098560512 + }, + { + "epoch": 3.06, + "learning_rate": 0.00033695085255767304, + "loss": 2.7339, + "theoretical_loss": 3.6169534875718217, + "tokens_seen": 1098626048 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003369408224674022, + "loss": 2.6009, + "theoretical_loss": 3.616933310496119, + "tokens_seen": 1098691584 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003369307923771314, + "loss": 2.4912, + "theoretical_loss": 3.616913134960896, + "tokens_seen": 1098757120 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003369207622868606, + "loss": 2.8986, + "theoretical_loss": 3.6168929609659424, + "tokens_seen": 1098822656 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003369107321965898, + "loss": 2.7956, + "theoretical_loss": 3.6168727885110483, + "tokens_seen": 1098888192 + }, + { + "epoch": 3.06, + "learning_rate": 0.00033690070210631894, + "loss": 2.6026, + "theoretical_loss": 3.6168526175960047, + "tokens_seen": 1098953728 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003368906720160482, + "loss": 3.0197, + "theoretical_loss": 3.6168324482206025, + "tokens_seen": 1099019264 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003368806419257773, + "loss": 2.8678, + "theoretical_loss": 3.6168122803846323, + "tokens_seen": 1099084800 + }, + { + "epoch": 3.06, + "learning_rate": 0.00033687061183550654, + "loss": 2.5931, + "theoretical_loss": 3.6167921140878843, + "tokens_seen": 1099150336 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003368605817452357, + "loss": 2.5982, + "theoretical_loss": 3.6167719493301496, + "tokens_seen": 1099215872 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003368505516549649, + "loss": 2.6875, + "theoretical_loss": 3.6167517861112195, + "tokens_seen": 1099281408 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003368405215646941, + "loss": 2.6566, + "theoretical_loss": 3.6167316244308845, + "tokens_seen": 1099346944 + }, + { + "epoch": 3.06, + "objective/train/docs_used": 1239758, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.1782515048980713, + "objective/train/theoretical_loss": 3.6167265842511815, + "objective/train/tokens_used": 1119823328, + "theoretical_loss": 3.6167265842511815, + "tokens_seen": 1099363328 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003368304914744233, + "loss": 2.6187, + "theoretical_loss": 3.616711464288935, + "tokens_seen": 1099412480 + }, + { + "epoch": 3.06, + "learning_rate": 0.00033682046138415245, + "loss": 2.8087, + "theoretical_loss": 3.616691305685162, + "tokens_seen": 1099478016 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003368104312938817, + "loss": 2.7666, + "theoretical_loss": 3.616671148619357, + "tokens_seen": 1099543552 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003368004012036108, + "loss": 2.8175, + "theoretical_loss": 3.6166509930913113, + "tokens_seen": 1099609088 + }, + { + "epoch": 3.06, + "learning_rate": 0.00033679037111334004, + "loss": 2.6504, + "theoretical_loss": 3.6166308391008153, + "tokens_seen": 1099674624 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003367803410230692, + "loss": 2.7364, + "theoretical_loss": 3.6166106866476593, + "tokens_seen": 1099740160 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003367703109327984, + "loss": 2.6164, + "theoretical_loss": 3.6165905357316364, + "tokens_seen": 1099805696 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003367602808425276, + "loss": 2.7391, + "theoretical_loss": 3.616570386352536, + "tokens_seen": 1099871232 + }, + { + "epoch": 3.06, + "learning_rate": 0.00033675025075225677, + "loss": 2.539, + "theoretical_loss": 3.6165502385101505, + "tokens_seen": 1099936768 + }, + { + "epoch": 3.06, + "learning_rate": 0.00033674022066198595, + "loss": 2.6233, + "theoretical_loss": 3.61653009220427, + "tokens_seen": 1100002304 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003367301905717152, + "loss": 2.7204, + "theoretical_loss": 3.616509947434687, + "tokens_seen": 1100067840 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003367201604814443, + "loss": 2.6064, + "theoretical_loss": 3.6164898042011924, + "tokens_seen": 1100133376 + }, + { + "epoch": 3.06, + "learning_rate": 0.00033671013039117355, + "loss": 2.7523, + "theoretical_loss": 3.616469662503577, + "tokens_seen": 1100198912 + }, + { + "epoch": 3.06, + "learning_rate": 0.00033670010030090273, + "loss": 2.7206, + "theoretical_loss": 3.616449522341633, + "tokens_seen": 1100264448 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003366900702106319, + "loss": 2.7963, + "theoretical_loss": 3.6164293837151513, + "tokens_seen": 1100329984 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003366800401203611, + "loss": 2.5174, + "theoretical_loss": 3.616409246623924, + "tokens_seen": 1100395520 + }, + { + "epoch": 3.06, + "learning_rate": 0.00033667001003009027, + "loss": 2.8013, + "theoretical_loss": 3.6163891110677415, + "tokens_seen": 1100461056 + }, + { + "epoch": 3.06, + "learning_rate": 0.00033665997993981945, + "loss": 2.6539, + "theoretical_loss": 3.6163689770463967, + "tokens_seen": 1100526592 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003366499498495487, + "loss": 2.6839, + "theoretical_loss": 3.6163488445596808, + "tokens_seen": 1100592128 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003366399197592778, + "loss": 2.7934, + "theoretical_loss": 3.616328713607385, + "tokens_seen": 1100657664 + }, + { + "epoch": 3.06, + "learning_rate": 0.00033662988966900705, + "loss": 2.7716, + "theoretical_loss": 3.6163085841893015, + "tokens_seen": 1100723200 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003366198595787362, + "loss": 2.7189, + "theoretical_loss": 3.616288456305222, + "tokens_seen": 1100788736 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003366098294884654, + "loss": 2.5594, + "theoretical_loss": 3.616268329954938, + "tokens_seen": 1100854272 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003365997993981946, + "loss": 2.6865, + "theoretical_loss": 3.616248205138241, + "tokens_seen": 1100919808 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003365897693079238, + "loss": 2.812, + "theoretical_loss": 3.616228081854924, + "tokens_seen": 1100985344 + }, + { + "epoch": 3.06, + "objective/train/docs_used": 1241137, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8605268001556396, + "objective/train/theoretical_loss": 3.6162230512736606, + "objective/train/tokens_used": 1121461728, + "theoretical_loss": 3.6162230512736606, + "tokens_seen": 1101001728 + }, + { + "epoch": 3.06, + "learning_rate": 0.00033657973921765295, + "loss": 2.9682, + "theoretical_loss": 3.616207960104778, + "tokens_seen": 1101050880 + }, + { + "epoch": 3.06, + "learning_rate": 0.00033656970912738214, + "loss": 2.8834, + "theoretical_loss": 3.616187839887595, + "tokens_seen": 1101116416 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003365596790371113, + "loss": 2.8656, + "theoretical_loss": 3.6161677212031673, + "tokens_seen": 1101181952 + }, + { + "epoch": 3.06, + "learning_rate": 0.00033654964894684055, + "loss": 2.6308, + "theoretical_loss": 3.616147604051287, + "tokens_seen": 1101247488 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003365396188565697, + "loss": 2.7918, + "theoretical_loss": 3.6161274884317454, + "tokens_seen": 1101313024 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003365295887662989, + "loss": 2.6168, + "theoretical_loss": 3.6161073743443355, + "tokens_seen": 1101378560 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003365195586760281, + "loss": 2.7806, + "theoretical_loss": 3.616087261788849, + "tokens_seen": 1101444096 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003365095285857573, + "loss": 2.6078, + "theoretical_loss": 3.6160671507650783, + "tokens_seen": 1101509632 + }, + { + "epoch": 3.06, + "learning_rate": 0.00033649949849548646, + "loss": 2.5558, + "theoretical_loss": 3.6160470412728154, + "tokens_seen": 1101575168 + }, + { + "epoch": 3.06, + "learning_rate": 0.00033648946840521564, + "loss": 3.0053, + "theoretical_loss": 3.616026933311853, + "tokens_seen": 1101640704 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003364794383149448, + "loss": 2.5638, + "theoretical_loss": 3.6160068268819825, + "tokens_seen": 1101706240 + }, + { + "epoch": 3.06, + "learning_rate": 0.00033646940822467406, + "loss": 2.4866, + "theoretical_loss": 3.6159867219829973, + "tokens_seen": 1101771776 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003364593781344032, + "loss": 2.546, + "theoretical_loss": 3.615966618614689, + "tokens_seen": 1101837312 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003364493480441324, + "loss": 2.6268, + "theoretical_loss": 3.6159465167768507, + "tokens_seen": 1101902848 + }, + { + "epoch": 3.06, + "learning_rate": 0.00033643931795386154, + "loss": 2.7514, + "theoretical_loss": 3.615926416469274, + "tokens_seen": 1101968384 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003364292878635908, + "loss": 2.7597, + "theoretical_loss": 3.6159063176917527, + "tokens_seen": 1102033920 + }, + { + "epoch": 3.06, + "learning_rate": 0.00033641925777331996, + "loss": 2.7634, + "theoretical_loss": 3.615886220444078, + "tokens_seen": 1102099456 + }, + { + "epoch": 3.06, + "learning_rate": 0.00033640922768304914, + "loss": 2.8582, + "theoretical_loss": 3.6158661247260433, + "tokens_seen": 1102164992 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003363991975927783, + "loss": 2.7127, + "theoretical_loss": 3.6158460305374405, + "tokens_seen": 1102230528 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003363891675025075, + "loss": 2.6674, + "theoretical_loss": 3.6158259378780633, + "tokens_seen": 1102296064 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003363791374122367, + "loss": 2.9135, + "theoretical_loss": 3.615805846747704, + "tokens_seen": 1102361600 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003363691073219659, + "loss": 2.7931, + "theoretical_loss": 3.6157857571461554, + "tokens_seen": 1102427136 + }, + { + "epoch": 3.06, + "learning_rate": 0.00033635907723169505, + "loss": 2.7545, + "theoretical_loss": 3.61576566907321, + "tokens_seen": 1102492672 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003363490471414243, + "loss": 2.4555, + "theoretical_loss": 3.6157455825286604, + "tokens_seen": 1102558208 + }, + { + "epoch": 3.06, + "learning_rate": 0.00033633901705115346, + "loss": 2.8385, + "theoretical_loss": 3.6157254975123, + "tokens_seen": 1102623744 + }, + { + "epoch": 3.06, + "objective/train/docs_used": 1241790, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5041520595550537, + "objective/train/theoretical_loss": 3.615720476496965, + "objective/train/tokens_used": 1123100128, + "theoretical_loss": 3.615720476496965, + "tokens_seen": 1102640128 + }, + { + "epoch": 3.06, + "learning_rate": 0.00033632898696088265, + "loss": 2.6422, + "theoretical_loss": 3.6157054140239215, + "tokens_seen": 1102689280 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003363189568706119, + "loss": 2.8461, + "theoretical_loss": 3.615685332063318, + "tokens_seen": 1102754816 + }, + { + "epoch": 3.06, + "learning_rate": 0.000336308926780341, + "loss": 2.71, + "theoretical_loss": 3.6156652516302827, + "tokens_seen": 1102820352 + }, + { + "epoch": 3.06, + "learning_rate": 0.00033629889669007024, + "loss": 2.789, + "theoretical_loss": 3.615645172724608, + "tokens_seen": 1102885888 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003362888665997994, + "loss": 2.747, + "theoretical_loss": 3.6156250953460876, + "tokens_seen": 1102951424 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003362788365095286, + "loss": 2.6397, + "theoretical_loss": 3.615605019494514, + "tokens_seen": 1103016960 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003362688064192578, + "loss": 2.6001, + "theoretical_loss": 3.6155849451696813, + "tokens_seen": 1103082496 + }, + { + "epoch": 3.06, + "learning_rate": 0.00033625877632898697, + "loss": 2.4701, + "theoretical_loss": 3.615564872371382, + "tokens_seen": 1103148032 + }, + { + "epoch": 3.06, + "learning_rate": 0.00033624874623871615, + "loss": 2.6813, + "theoretical_loss": 3.615544801099409, + "tokens_seen": 1103213568 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003362387161484454, + "loss": 2.7879, + "theoretical_loss": 3.615524731353556, + "tokens_seen": 1103279104 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003362286860581745, + "loss": 2.6216, + "theoretical_loss": 3.615504663133617, + "tokens_seen": 1103344640 + }, + { + "epoch": 3.06, + "learning_rate": 0.00033621865596790375, + "loss": 2.383, + "theoretical_loss": 3.615484596439384, + "tokens_seen": 1103410176 + }, + { + "epoch": 3.06, + "learning_rate": 0.00033620862587763293, + "loss": 2.4897, + "theoretical_loss": 3.6154645312706513, + "tokens_seen": 1103475712 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003361985957873621, + "loss": 2.7389, + "theoretical_loss": 3.615444467627212, + "tokens_seen": 1103541248 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003361885656970913, + "loss": 2.7377, + "theoretical_loss": 3.6154244055088602, + "tokens_seen": 1103606784 + }, + { + "epoch": 3.06, + "learning_rate": 0.00033617853560682047, + "loss": 2.8105, + "theoretical_loss": 3.6154043449153885, + "tokens_seen": 1103672320 + }, + { + "epoch": 3.06, + "learning_rate": 0.00033616850551654965, + "loss": 2.6996, + "theoretical_loss": 3.615384285846591, + "tokens_seen": 1103737856 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003361584754262789, + "loss": 2.4126, + "theoretical_loss": 3.6153642283022616, + "tokens_seen": 1103803392 + }, + { + "epoch": 3.06, + "learning_rate": 0.000336148445336008, + "loss": 2.6139, + "theoretical_loss": 3.6153441722821933, + "tokens_seen": 1103868928 + }, + { + "epoch": 3.06, + "learning_rate": 0.00033613841524573725, + "loss": 2.6361, + "theoretical_loss": 3.6153241177861797, + "tokens_seen": 1103934464 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003361283851554664, + "loss": 2.4851, + "theoretical_loss": 3.615304064814015, + "tokens_seen": 1104000000 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003361183550651956, + "loss": 2.6684, + "theoretical_loss": 3.615284013365493, + "tokens_seen": 1104065536 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003361083249749248, + "loss": 2.7077, + "theoretical_loss": 3.615263963440407, + "tokens_seen": 1104131072 + }, + { + "epoch": 3.06, + "learning_rate": 0.000336098294884654, + "loss": 2.7662, + "theoretical_loss": 3.615243915038551, + "tokens_seen": 1104196608 + }, + { + "epoch": 3.06, + "learning_rate": 0.00033608826479438316, + "loss": 2.6451, + "theoretical_loss": 3.61522386815972, + "tokens_seen": 1104262144 + }, + { + "epoch": 3.06, + "objective/train/docs_used": 1242899, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7701282501220703, + "objective/train/theoretical_loss": 3.61521885667796, + "objective/train/tokens_used": 1124738528, + "theoretical_loss": 3.61521885667796, + "tokens_seen": 1104278528 + }, + { + "epoch": 3.06, + "learning_rate": 0.00033607823470411234, + "loss": 2.5712, + "theoretical_loss": 3.6152038228037062, + "tokens_seen": 1104327680 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003360682046138415, + "loss": 2.7366, + "theoretical_loss": 3.6151837789703043, + "tokens_seen": 1104393216 + }, + { + "epoch": 3.06, + "learning_rate": 0.00033605817452357075, + "loss": 2.6724, + "theoretical_loss": 3.6151637366593086, + "tokens_seen": 1104458752 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003360481444332999, + "loss": 2.6507, + "theoretical_loss": 3.615143695870512, + "tokens_seen": 1104524288 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003360381143430291, + "loss": 2.8124, + "theoretical_loss": 3.6151236566037106, + "tokens_seen": 1104589824 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003360280842527583, + "loss": 2.6883, + "theoretical_loss": 3.6151036188586967, + "tokens_seen": 1104655360 + }, + { + "epoch": 3.06, + "learning_rate": 0.0003360180541624875, + "loss": 2.6271, + "theoretical_loss": 3.6150835826352656, + "tokens_seen": 1104720896 + }, + { + "epoch": 3.06, + "learning_rate": 0.00033600802407221666, + "loss": 2.679, + "theoretical_loss": 3.6150635479332105, + "tokens_seen": 1104786432 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033599799398194584, + "loss": 2.6898, + "theoretical_loss": 3.615043514752326, + "tokens_seen": 1104851968 + }, + { + "epoch": 3.07, + "learning_rate": 0.000335987963891675, + "loss": 2.6468, + "theoretical_loss": 3.615023483092407, + "tokens_seen": 1104917504 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033597793380140426, + "loss": 2.7967, + "theoretical_loss": 3.6150034529532475, + "tokens_seen": 1104983040 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003359679037111334, + "loss": 2.8978, + "theoretical_loss": 3.6149834243346417, + "tokens_seen": 1105048576 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003359578736208626, + "loss": 2.6337, + "theoretical_loss": 3.6149633972363837, + "tokens_seen": 1105114112 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033594784353059174, + "loss": 2.6396, + "theoretical_loss": 3.614943371658268, + "tokens_seen": 1105179648 + }, + { + "epoch": 3.07, + "learning_rate": 0.000335937813440321, + "loss": 2.8301, + "theoretical_loss": 3.61492334760009, + "tokens_seen": 1105245184 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033592778335005016, + "loss": 3.037, + "theoretical_loss": 3.6149033250616434, + "tokens_seen": 1105310720 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033591775325977934, + "loss": 2.6608, + "theoretical_loss": 3.6148833040427224, + "tokens_seen": 1105376256 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003359077231695085, + "loss": 2.8969, + "theoretical_loss": 3.614863284543123, + "tokens_seen": 1105441792 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003358976930792377, + "loss": 2.7132, + "theoretical_loss": 3.6148432665626387, + "tokens_seen": 1105507328 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003358876629889669, + "loss": 2.4359, + "theoretical_loss": 3.6148232501010638, + "tokens_seen": 1105572864 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003358776328986961, + "loss": 2.9279, + "theoretical_loss": 3.6148032351581945, + "tokens_seen": 1105638400 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033586760280842525, + "loss": 2.7633, + "theoretical_loss": 3.6147832217338243, + "tokens_seen": 1105703936 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003358575727181545, + "loss": 2.7452, + "theoretical_loss": 3.614763209827749, + "tokens_seen": 1105769472 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033584754262788366, + "loss": 2.6413, + "theoretical_loss": 3.6147431994397623, + "tokens_seen": 1105835008 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033583751253761285, + "loss": 2.5344, + "theoretical_loss": 3.61472319056966, + "tokens_seen": 1105900544 + }, + { + "epoch": 3.07, + "objective/train/docs_used": 1243613, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9853389263153076, + "objective/train/theoretical_loss": 3.6147181885892796, + "objective/train/tokens_used": 1126376928, + "theoretical_loss": 3.6147181885892796, + "tokens_seen": 1105916928 + }, + { + "epoch": 3.07, + "learning_rate": 0.000335827482447342, + "loss": 2.7355, + "theoretical_loss": 3.6147031832172365, + "tokens_seen": 1105966080 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003358174523570712, + "loss": 2.6175, + "theoretical_loss": 3.614683177382287, + "tokens_seen": 1106031616 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003358074222668004, + "loss": 2.9476, + "theoretical_loss": 3.6146631730646064, + "tokens_seen": 1106097152 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003357973921765296, + "loss": 2.7897, + "theoretical_loss": 3.6146431702639896, + "tokens_seen": 1106162688 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033578736208625875, + "loss": 2.6465, + "theoretical_loss": 3.6146231689802324, + "tokens_seen": 1106228224 + }, + { + "epoch": 3.07, + "learning_rate": 0.000335777331995988, + "loss": 2.6784, + "theoretical_loss": 3.6146031692131295, + "tokens_seen": 1106293760 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003357673019057171, + "loss": 2.7595, + "theoretical_loss": 3.614583170962475, + "tokens_seen": 1106359296 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033575727181544635, + "loss": 2.5581, + "theoretical_loss": 3.614563174228066, + "tokens_seen": 1106424832 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033574724172517553, + "loss": 2.7814, + "theoretical_loss": 3.6145431790096962, + "tokens_seen": 1106490368 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003357372116349047, + "loss": 2.6407, + "theoretical_loss": 3.6145231853071618, + "tokens_seen": 1106555904 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003357271815446339, + "loss": 2.7503, + "theoretical_loss": 3.6145031931202576, + "tokens_seen": 1106621440 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033571715145436313, + "loss": 2.794, + "theoretical_loss": 3.6144832024487794, + "tokens_seen": 1106686976 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033570712136409225, + "loss": 2.765, + "theoretical_loss": 3.614463213292522, + "tokens_seen": 1106752512 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003356970912738215, + "loss": 2.7039, + "theoretical_loss": 3.6144432256512817, + "tokens_seen": 1106818048 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003356870611835506, + "loss": 2.5687, + "theoretical_loss": 3.6144232395248532, + "tokens_seen": 1106883584 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033567703109327985, + "loss": 2.7355, + "theoretical_loss": 3.6144032549130323, + "tokens_seen": 1106949120 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033566700100300903, + "loss": 2.7997, + "theoretical_loss": 3.6143832718156146, + "tokens_seen": 1107014656 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003356569709127382, + "loss": 2.6971, + "theoretical_loss": 3.6143632902323954, + "tokens_seen": 1107080192 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003356469408224674, + "loss": 2.8226, + "theoretical_loss": 3.6143433101631706, + "tokens_seen": 1107145728 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003356369107321966, + "loss": 2.458, + "theoretical_loss": 3.614323331607736, + "tokens_seen": 1107211264 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033562688064192576, + "loss": 2.6713, + "theoretical_loss": 3.6143033545658874, + "tokens_seen": 1107276800 + }, + { + "epoch": 3.07, + "learning_rate": 0.000335616850551655, + "loss": 2.5748, + "theoretical_loss": 3.61428337903742, + "tokens_seen": 1107342336 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003356068204613841, + "loss": 2.8478, + "theoretical_loss": 3.61426340502213, + "tokens_seen": 1107407872 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033559679037111336, + "loss": 2.8482, + "theoretical_loss": 3.614243432519813, + "tokens_seen": 1107473408 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003355867602808425, + "loss": 2.7905, + "theoretical_loss": 3.6142234615302655, + "tokens_seen": 1107538944 + }, + { + "epoch": 3.07, + "objective/train/docs_used": 1244863, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.85904860496521, + "objective/train/theoretical_loss": 3.6142184690192245, + "objective/train/tokens_used": 1128015328, + "theoretical_loss": 3.6142184690192245, + "tokens_seen": 1107555328 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003355767301905717, + "loss": 2.8847, + "theoretical_loss": 3.6142034920532824, + "tokens_seen": 1107604480 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033556670010030095, + "loss": 2.7116, + "theoretical_loss": 3.6141835240886606, + "tokens_seen": 1107670016 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003355566700100301, + "loss": 2.6555, + "theoretical_loss": 3.6141635576361955, + "tokens_seen": 1107735552 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003355466399197593, + "loss": 2.5725, + "theoretical_loss": 3.6141435926956835, + "tokens_seen": 1107801088 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003355366098294885, + "loss": 2.6107, + "theoretical_loss": 3.6141236292669205, + "tokens_seen": 1107866624 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003355265797392177, + "loss": 2.4844, + "theoretical_loss": 3.6141036673497027, + "tokens_seen": 1107932160 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033551654964894686, + "loss": 2.7763, + "theoretical_loss": 3.6140837069438256, + "tokens_seen": 1107997696 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033550651955867604, + "loss": 2.7287, + "theoretical_loss": 3.6140637480490865, + "tokens_seen": 1108063232 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003354964894684052, + "loss": 2.6421, + "theoretical_loss": 3.6140437906652805, + "tokens_seen": 1108128768 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033548645937813446, + "loss": 2.595, + "theoretical_loss": 3.614023834792205, + "tokens_seen": 1108194304 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003354764292878636, + "loss": 2.7723, + "theoretical_loss": 3.6140038804296557, + "tokens_seen": 1108259840 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003354663991975928, + "loss": 2.8769, + "theoretical_loss": 3.6139839275774293, + "tokens_seen": 1108325376 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033545636910732195, + "loss": 2.587, + "theoretical_loss": 3.613963976235321, + "tokens_seen": 1108390912 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003354463390170512, + "loss": 2.7087, + "theoretical_loss": 3.613944026403129, + "tokens_seen": 1108456448 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033543630892678036, + "loss": 2.8548, + "theoretical_loss": 3.6139240780806485, + "tokens_seen": 1108521984 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033542627883650954, + "loss": 2.5704, + "theoretical_loss": 3.6139041312676765, + "tokens_seen": 1108587520 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003354162487462387, + "loss": 2.7445, + "theoretical_loss": 3.61388418596401, + "tokens_seen": 1108653056 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003354062186559679, + "loss": 2.8618, + "theoretical_loss": 3.613864242169444, + "tokens_seen": 1108718592 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003353961885656971, + "loss": 2.6855, + "theoretical_loss": 3.6138442998837768, + "tokens_seen": 1108784128 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003353861584754263, + "loss": 2.51, + "theoretical_loss": 3.613824359106805, + "tokens_seen": 1108849664 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033537612838515545, + "loss": 2.6772, + "theoretical_loss": 3.6138044198383237, + "tokens_seen": 1108915200 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003353660982948847, + "loss": 2.6674, + "theoretical_loss": 3.613784482078131, + "tokens_seen": 1108980736 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033535606820461386, + "loss": 2.6406, + "theoretical_loss": 3.6137645458260237, + "tokens_seen": 1109046272 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033534603811434305, + "loss": 2.678, + "theoretical_loss": 3.613744611081798, + "tokens_seen": 1109111808 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003353360080240722, + "loss": 2.7416, + "theoretical_loss": 3.613724677845251, + "tokens_seen": 1109177344 + }, + { + "epoch": 3.07, + "objective/train/docs_used": 1245226, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.714931011199951, + "objective/train/theoretical_loss": 3.6137196947716657, + "objective/train/tokens_used": 1129653728, + "theoretical_loss": 3.6137196947716657, + "tokens_seen": 1109193728 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003353259779338014, + "loss": 2.8707, + "theoretical_loss": 3.6137047461161798, + "tokens_seen": 1109242880 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003353159478435306, + "loss": 2.908, + "theoretical_loss": 3.6136848158943815, + "tokens_seen": 1109308416 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003353059177532598, + "loss": 2.5868, + "theoretical_loss": 3.6136648871796524, + "tokens_seen": 1109373952 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033529588766298895, + "loss": 2.7557, + "theoretical_loss": 3.61364495997179, + "tokens_seen": 1109439488 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003352858575727182, + "loss": 2.4417, + "theoretical_loss": 3.6136250342705907, + "tokens_seen": 1109505024 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003352758274824473, + "loss": 2.693, + "theoretical_loss": 3.613605110075853, + "tokens_seen": 1109570560 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033526579739217655, + "loss": 2.9798, + "theoretical_loss": 3.6135851873873728, + "tokens_seen": 1109636096 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033525576730190573, + "loss": 2.6654, + "theoretical_loss": 3.613565266204948, + "tokens_seen": 1109701632 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003352457372116349, + "loss": 2.5048, + "theoretical_loss": 3.613545346528375, + "tokens_seen": 1109767168 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003352357071213641, + "loss": 2.8293, + "theoretical_loss": 3.613525428357452, + "tokens_seen": 1109832704 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033522567703109333, + "loss": 2.658, + "theoretical_loss": 3.6135055116919754, + "tokens_seen": 1109898240 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033521564694082245, + "loss": 2.6935, + "theoretical_loss": 3.6134855965317434, + "tokens_seen": 1109963776 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003352056168505517, + "loss": 2.8863, + "theoretical_loss": 3.6134656828765523, + "tokens_seen": 1110029312 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003351955867602808, + "loss": 2.6411, + "theoretical_loss": 3.613445770726201, + "tokens_seen": 1110094848 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033518555667001005, + "loss": 2.7521, + "theoretical_loss": 3.613425860080486, + "tokens_seen": 1110160384 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033517552657973923, + "loss": 2.4854, + "theoretical_loss": 3.6134059509392045, + "tokens_seen": 1110225920 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003351654964894684, + "loss": 2.8907, + "theoretical_loss": 3.6133860433021545, + "tokens_seen": 1110291456 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003351554663991976, + "loss": 2.616, + "theoretical_loss": 3.613366137169134, + "tokens_seen": 1110356992 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003351454363089268, + "loss": 2.7054, + "theoretical_loss": 3.6133462325399393, + "tokens_seen": 1110422528 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033513540621865596, + "loss": 2.7419, + "theoretical_loss": 3.6133263294143694, + "tokens_seen": 1110488064 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003351253761283852, + "loss": 2.721, + "theoretical_loss": 3.613306427792221, + "tokens_seen": 1110553600 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003351153460381143, + "loss": 2.791, + "theoretical_loss": 3.6132865276732926, + "tokens_seen": 1110619136 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033510531594784356, + "loss": 2.5685, + "theoretical_loss": 3.6132666290573816, + "tokens_seen": 1110684672 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003350952858575727, + "loss": 2.8731, + "theoretical_loss": 3.613246731944286, + "tokens_seen": 1110750208 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003350852557673019, + "loss": 2.5971, + "theoretical_loss": 3.613226836333803, + "tokens_seen": 1110815744 + }, + { + "epoch": 3.07, + "objective/train/docs_used": 1246492, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.347496747970581, + "objective/train/theoretical_loss": 3.6132218626659425, + "objective/train/tokens_used": 1131292128, + "theoretical_loss": 3.6132218626659425, + "tokens_seen": 1110832128 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003350752256770311, + "loss": 2.4939, + "theoretical_loss": 3.6132069422257316, + "tokens_seen": 1110881280 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003350651955867603, + "loss": 2.3556, + "theoretical_loss": 3.613187049619869, + "tokens_seen": 1110946816 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033505516549648946, + "loss": 2.7816, + "theoretical_loss": 3.6131671585160126, + "tokens_seen": 1111012352 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003350451354062187, + "loss": 2.3472, + "theoretical_loss": 3.6131472689139614, + "tokens_seen": 1111077888 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003350351053159478, + "loss": 2.8442, + "theoretical_loss": 3.6131273808135136, + "tokens_seen": 1111143424 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033502507522567706, + "loss": 2.6176, + "theoretical_loss": 3.613107494214466, + "tokens_seen": 1111208960 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003350150451354062, + "loss": 2.7964, + "theoretical_loss": 3.6130876091166177, + "tokens_seen": 1111274496 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003350050150451354, + "loss": 2.5386, + "theoretical_loss": 3.613067725519767, + "tokens_seen": 1111340032 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003349949849548646, + "loss": 2.7889, + "theoretical_loss": 3.613047843423711, + "tokens_seen": 1111405568 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003349849548645938, + "loss": 2.5481, + "theoretical_loss": 3.61302796282825, + "tokens_seen": 1111471104 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033497492477432296, + "loss": 2.804, + "theoretical_loss": 3.6130080837331797, + "tokens_seen": 1111536640 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033496489468405215, + "loss": 2.6183, + "theoretical_loss": 3.6129882061383, + "tokens_seen": 1111602176 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003349548645937813, + "loss": 2.6518, + "theoretical_loss": 3.612968330043409, + "tokens_seen": 1111667712 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033494483450351056, + "loss": 2.731, + "theoretical_loss": 3.612948455448305, + "tokens_seen": 1111733248 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003349348044132397, + "loss": 2.8229, + "theoretical_loss": 3.6129285823527866, + "tokens_seen": 1111798784 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003349247743229689, + "loss": 2.8859, + "theoretical_loss": 3.6129087107566518, + "tokens_seen": 1111864320 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033491474423269805, + "loss": 2.5248, + "theoretical_loss": 3.6128888406597, + "tokens_seen": 1111929856 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003349047141424273, + "loss": 2.869, + "theoretical_loss": 3.612868972061728, + "tokens_seen": 1111995392 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033489468405215647, + "loss": 2.6972, + "theoretical_loss": 3.6128491049625366, + "tokens_seen": 1112060928 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033488465396188565, + "loss": 2.6138, + "theoretical_loss": 3.612829239361923, + "tokens_seen": 1112126464 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033487462387161483, + "loss": 2.9363, + "theoretical_loss": 3.612809375259686, + "tokens_seen": 1112192000 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033486459378134406, + "loss": 2.9163, + "theoretical_loss": 3.612789512655625, + "tokens_seen": 1112257536 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003348545636910732, + "loss": 2.9406, + "theoretical_loss": 3.612769651549538, + "tokens_seen": 1112323072 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003348445336008024, + "loss": 2.6882, + "theoretical_loss": 3.6127497919412237, + "tokens_seen": 1112388608 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003348345035105316, + "loss": 2.8227, + "theoretical_loss": 3.6127299338304812, + "tokens_seen": 1112454144 + }, + { + "epoch": 3.07, + "objective/train/docs_used": 1247248, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 1.9692270755767822, + "objective/train/theoretical_loss": 3.6127249695367682, + "objective/train/tokens_used": 1132930528, + "theoretical_loss": 3.6127249695367682, + "tokens_seen": 1112470528 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003348244734202608, + "loss": 2.3856, + "theoretical_loss": 3.61271007721711, + "tokens_seen": 1112519680 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033481444332999, + "loss": 2.7777, + "theoretical_loss": 3.612690222100908, + "tokens_seen": 1112585216 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033480441323971915, + "loss": 2.5555, + "theoretical_loss": 3.612670368481675, + "tokens_seen": 1112650752 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003347943831494484, + "loss": 2.7394, + "theoretical_loss": 3.612650516359209, + "tokens_seen": 1112716288 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003347843530591775, + "loss": 2.5448, + "theoretical_loss": 3.6126306657333105, + "tokens_seen": 1112781824 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033477432296890675, + "loss": 2.6602, + "theoretical_loss": 3.6126108166037767, + "tokens_seen": 1112847360 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033476429287863593, + "loss": 2.5948, + "theoretical_loss": 3.612590968970408, + "tokens_seen": 1112912896 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003347542627883651, + "loss": 2.8068, + "theoretical_loss": 3.6125711228330024, + "tokens_seen": 1112978432 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003347442326980943, + "loss": 2.6638, + "theoretical_loss": 3.6125512781913605, + "tokens_seen": 1113043968 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033473420260782353, + "loss": 2.8068, + "theoretical_loss": 3.6125314350452804, + "tokens_seen": 1113109504 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033472417251755265, + "loss": 2.7284, + "theoretical_loss": 3.6125115933945624, + "tokens_seen": 1113175040 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003347141424272819, + "loss": 2.6465, + "theoretical_loss": 3.612491753239004, + "tokens_seen": 1113240576 + }, + { + "epoch": 3.07, + "learning_rate": 0.000334704112337011, + "loss": 2.5866, + "theoretical_loss": 3.6124719145784066, + "tokens_seen": 1113306112 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033469408224674025, + "loss": 2.776, + "theoretical_loss": 3.612452077412568, + "tokens_seen": 1113371648 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033468405215646943, + "loss": 2.5994, + "theoretical_loss": 3.612432241741289, + "tokens_seen": 1113437184 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003346740220661986, + "loss": 2.7386, + "theoretical_loss": 3.612412407564367, + "tokens_seen": 1113502720 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003346639919759278, + "loss": 2.4634, + "theoretical_loss": 3.6123925748816035, + "tokens_seen": 1113568256 + }, + { + "epoch": 3.07, + "learning_rate": 0.000334653961885657, + "loss": 2.6224, + "theoretical_loss": 3.6123727436927977, + "tokens_seen": 1113633792 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033464393179538616, + "loss": 2.8138, + "theoretical_loss": 3.612352913997748, + "tokens_seen": 1113699328 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003346339017051154, + "loss": 2.7124, + "theoretical_loss": 3.612333085796255, + "tokens_seen": 1113764864 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003346238716148445, + "loss": 2.6895, + "theoretical_loss": 3.6123132590881175, + "tokens_seen": 1113830400 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033461384152457376, + "loss": 2.6007, + "theoretical_loss": 3.612293433873136, + "tokens_seen": 1113895936 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003346038114343029, + "loss": 2.5594, + "theoretical_loss": 3.61227361015111, + "tokens_seen": 1113961472 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003345937813440321, + "loss": 2.8955, + "theoretical_loss": 3.612253787921839, + "tokens_seen": 1114027008 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003345837512537613, + "loss": 2.637, + "theoretical_loss": 3.612233967185123, + "tokens_seen": 1114092544 + }, + { + "epoch": 3.07, + "objective/train/docs_used": 1248654, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.329355001449585, + "objective/train/theoretical_loss": 3.612229012234132, + "objective/train/tokens_used": 1134568928, + "theoretical_loss": 3.612229012234132, + "tokens_seen": 1114108928 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003345737211634905, + "loss": 2.763, + "theoretical_loss": 3.6122141479407617, + "tokens_seen": 1114158080 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033456369107321966, + "loss": 2.599, + "theoretical_loss": 3.6121943301885553, + "tokens_seen": 1114223616 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003345536609829489, + "loss": 2.896, + "theoretical_loss": 3.612174513928303, + "tokens_seen": 1114289152 + }, + { + "epoch": 3.07, + "learning_rate": 0.000334543630892678, + "loss": 2.5523, + "theoretical_loss": 3.612154699159806, + "tokens_seen": 1114354688 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033453360080240726, + "loss": 2.5513, + "theoretical_loss": 3.6121348858828624, + "tokens_seen": 1114420224 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003345235707121364, + "loss": 2.6675, + "theoretical_loss": 3.612115074097274, + "tokens_seen": 1114485760 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003345135406218656, + "loss": 2.7643, + "theoretical_loss": 3.6120952638028396, + "tokens_seen": 1114551296 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003345035105315948, + "loss": 2.8072, + "theoretical_loss": 3.61207545499936, + "tokens_seen": 1114616832 + }, + { + "epoch": 3.07, + "learning_rate": 0.000334493480441324, + "loss": 2.7499, + "theoretical_loss": 3.6120556476866352, + "tokens_seen": 1114682368 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033448345035105316, + "loss": 2.7357, + "theoretical_loss": 3.612035841864466, + "tokens_seen": 1114747904 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033447342026078235, + "loss": 2.9083, + "theoretical_loss": 3.612016037532651, + "tokens_seen": 1114813440 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003344633901705115, + "loss": 2.9005, + "theoretical_loss": 3.6119962346909915, + "tokens_seen": 1114878976 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033445336008024076, + "loss": 2.7063, + "theoretical_loss": 3.6119764333392883, + "tokens_seen": 1114944512 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003344433299899699, + "loss": 2.5994, + "theoretical_loss": 3.61195663347734, + "tokens_seen": 1115010048 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003344332998996991, + "loss": 2.6802, + "theoretical_loss": 3.6119368351049492, + "tokens_seen": 1115075584 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033442326980942825, + "loss": 2.5207, + "theoretical_loss": 3.6119170382219146, + "tokens_seen": 1115141120 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003344132397191575, + "loss": 2.4613, + "theoretical_loss": 3.6118972428280376, + "tokens_seen": 1115206656 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033440320962888667, + "loss": 2.5894, + "theoretical_loss": 3.611877448923118, + "tokens_seen": 1115272192 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033439317953861585, + "loss": 2.5237, + "theoretical_loss": 3.611857656506957, + "tokens_seen": 1115337728 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033438314944834503, + "loss": 2.6439, + "theoretical_loss": 3.6118378655793544, + "tokens_seen": 1115403264 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033437311935807427, + "loss": 2.4564, + "theoretical_loss": 3.611818076140111, + "tokens_seen": 1115468800 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003343630892678034, + "loss": 2.7816, + "theoretical_loss": 3.611798288189028, + "tokens_seen": 1115534336 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033435305917753263, + "loss": 2.8158, + "theoretical_loss": 3.6117785017259054, + "tokens_seen": 1115599872 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033434302908726175, + "loss": 2.7997, + "theoretical_loss": 3.6117587167505447, + "tokens_seen": 1115665408 + }, + { + "epoch": 3.07, + "learning_rate": 0.000334332998996991, + "loss": 2.7841, + "theoretical_loss": 3.611738933262746, + "tokens_seen": 1115730944 + }, + { + "epoch": 3.07, + "objective/train/docs_used": 1249313, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.326357126235962, + "objective/train/theoretical_loss": 3.6117339876232046, + "objective/train/tokens_used": 1136207328, + "theoretical_loss": 3.6117339876232046, + "tokens_seen": 1115747328 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033432296890672017, + "loss": 2.5598, + "theoretical_loss": 3.6117191512623097, + "tokens_seen": 1115796480 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033431293881644935, + "loss": 2.6363, + "theoretical_loss": 3.6116993707490375, + "tokens_seen": 1115862016 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033430290872617853, + "loss": 2.5902, + "theoretical_loss": 3.61167959172273, + "tokens_seen": 1115927552 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003342928786359077, + "loss": 2.859, + "theoretical_loss": 3.611659814183188, + "tokens_seen": 1115993088 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003342828485456369, + "loss": 2.7009, + "theoretical_loss": 3.6116400381302123, + "tokens_seen": 1116058624 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033427281845536613, + "loss": 2.8771, + "theoretical_loss": 3.611620263563605, + "tokens_seen": 1116124160 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033426278836509526, + "loss": 2.7368, + "theoretical_loss": 3.611600490483165, + "tokens_seen": 1116189696 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003342527582748245, + "loss": 2.8553, + "theoretical_loss": 3.611580718888695, + "tokens_seen": 1116255232 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003342427281845536, + "loss": 2.6817, + "theoretical_loss": 3.611560948779996, + "tokens_seen": 1116320768 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033423269809428285, + "loss": 2.788, + "theoretical_loss": 3.611541180156868, + "tokens_seen": 1116386304 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033422266800401204, + "loss": 2.6987, + "theoretical_loss": 3.611521413019113, + "tokens_seen": 1116451840 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003342126379137412, + "loss": 2.5525, + "theoretical_loss": 3.6115016473665325, + "tokens_seen": 1116517376 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003342026078234704, + "loss": 2.5741, + "theoretical_loss": 3.6114818831989277, + "tokens_seen": 1116582912 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033419257773319963, + "loss": 2.6247, + "theoretical_loss": 3.611462120516099, + "tokens_seen": 1116648448 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033418254764292876, + "loss": 2.6781, + "theoretical_loss": 3.6114423593178486, + "tokens_seen": 1116713984 + }, + { + "epoch": 3.07, + "learning_rate": 0.000334172517552658, + "loss": 2.8599, + "theoretical_loss": 3.6114225996039773, + "tokens_seen": 1116779520 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003341624874623871, + "loss": 2.5877, + "theoretical_loss": 3.6114028413742867, + "tokens_seen": 1116845056 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033415245737211636, + "loss": 2.6957, + "theoretical_loss": 3.611383084628579, + "tokens_seen": 1116910592 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033414242728184554, + "loss": 2.8918, + "theoretical_loss": 3.6113633293666547, + "tokens_seen": 1116976128 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003341323971915747, + "loss": 2.6015, + "theoretical_loss": 3.6113435755883154, + "tokens_seen": 1117041664 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003341223671013039, + "loss": 2.7051, + "theoretical_loss": 3.611323823293363, + "tokens_seen": 1117107200 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003341123370110331, + "loss": 2.6845, + "theoretical_loss": 3.611304072481598, + "tokens_seen": 1117172736 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033410230692076226, + "loss": 2.5767, + "theoretical_loss": 3.6112843231528244, + "tokens_seen": 1117238272 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003340922768304915, + "loss": 2.4481, + "theoretical_loss": 3.6112645753068415, + "tokens_seen": 1117303808 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003340822467402207, + "loss": 2.6844, + "theoretical_loss": 3.6112448289434527, + "tokens_seen": 1117369344 + }, + { + "epoch": 3.07, + "objective/train/docs_used": 1250355, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9746341705322266, + "objective/train/theoretical_loss": 3.6112398925842375, + "objective/train/tokens_used": 1137845728, + "theoretical_loss": 3.6112398925842375, + "tokens_seen": 1117385728 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033407221664994986, + "loss": 2.6491, + "theoretical_loss": 3.6112250840624585, + "tokens_seen": 1117434880 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003340621865596791, + "loss": 2.6187, + "theoretical_loss": 3.6112053406636617, + "tokens_seen": 1117500416 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003340521564694082, + "loss": 2.5762, + "theoretical_loss": 3.611185598746863, + "tokens_seen": 1117565952 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033404212637913746, + "loss": 2.8125, + "theoretical_loss": 3.611165858311865, + "tokens_seen": 1117631488 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003340320962888666, + "loss": 2.8009, + "theoretical_loss": 3.6111461193584695, + "tokens_seen": 1117697024 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003340220661985958, + "loss": 2.7583, + "theoretical_loss": 3.6111263818864785, + "tokens_seen": 1117762560 + }, + { + "epoch": 3.07, + "learning_rate": 0.000334012036108325, + "loss": 2.6939, + "theoretical_loss": 3.6111066458956937, + "tokens_seen": 1117828096 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003340020060180542, + "loss": 2.9579, + "theoretical_loss": 3.6110869113859176, + "tokens_seen": 1117893632 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033399197592778336, + "loss": 2.9776, + "theoretical_loss": 3.611067178356952, + "tokens_seen": 1117959168 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033398194583751255, + "loss": 2.8083, + "theoretical_loss": 3.6110474468085987, + "tokens_seen": 1118024704 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003339719157472417, + "loss": 2.5772, + "theoretical_loss": 3.61102771674066, + "tokens_seen": 1118090240 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033396188565697096, + "loss": 2.6275, + "theoretical_loss": 3.611007988152938, + "tokens_seen": 1118155776 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003339518555667001, + "loss": 2.4687, + "theoretical_loss": 3.6109882610452355, + "tokens_seen": 1118221312 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003339418254764293, + "loss": 2.7554, + "theoretical_loss": 3.6109685354173537, + "tokens_seen": 1118286848 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033393179538615845, + "loss": 2.6071, + "theoretical_loss": 3.610948811269096, + "tokens_seen": 1118352384 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003339217652958877, + "loss": 2.705, + "theoretical_loss": 3.6109290886002636, + "tokens_seen": 1118417920 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033391173520561687, + "loss": 2.7084, + "theoretical_loss": 3.6109093674106596, + "tokens_seen": 1118483456 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033390170511534605, + "loss": 2.9054, + "theoretical_loss": 3.6108896477000862, + "tokens_seen": 1118548992 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033389167502507523, + "loss": 2.7773, + "theoretical_loss": 3.610869929468346, + "tokens_seen": 1118614528 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033388164493480447, + "loss": 2.8347, + "theoretical_loss": 3.610850212715241, + "tokens_seen": 1118680064 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003338716148445336, + "loss": 2.7266, + "theoretical_loss": 3.610830497440574, + "tokens_seen": 1118745600 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033386158475426283, + "loss": 2.4787, + "theoretical_loss": 3.610810783644148, + "tokens_seen": 1118811136 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033385155466399195, + "loss": 2.643, + "theoretical_loss": 3.6107910713257647, + "tokens_seen": 1118876672 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003338415245737212, + "loss": 2.7258, + "theoretical_loss": 3.610771360485227, + "tokens_seen": 1118942208 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033383149448345037, + "loss": 2.9903, + "theoretical_loss": 3.6107516511223383, + "tokens_seen": 1119007744 + }, + { + "epoch": 3.07, + "objective/train/docs_used": 1250984, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.452301502227783, + "objective/train/theoretical_loss": 3.6107467240124755, + "objective/train/tokens_used": 1139484128, + "theoretical_loss": 3.6107467240124755, + "tokens_seen": 1119024128 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033382146439317955, + "loss": 2.5613, + "theoretical_loss": 3.6107319432369005, + "tokens_seen": 1119073280 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033381143430290873, + "loss": 2.445, + "theoretical_loss": 3.610712236828716, + "tokens_seen": 1119138816 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003338014042126379, + "loss": 2.8158, + "theoretical_loss": 3.610692531897589, + "tokens_seen": 1119204352 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003337913741223671, + "loss": 2.8361, + "theoretical_loss": 3.610672828443321, + "tokens_seen": 1119269888 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033378134403209633, + "loss": 2.8296, + "theoretical_loss": 3.6106531264657153, + "tokens_seen": 1119335424 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033377131394182546, + "loss": 2.545, + "theoretical_loss": 3.6106334259645747, + "tokens_seen": 1119400960 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003337612838515547, + "loss": 2.8547, + "theoretical_loss": 3.610613726939702, + "tokens_seen": 1119466496 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003337512537612838, + "loss": 2.8374, + "theoretical_loss": 3.6105940293909002, + "tokens_seen": 1119532032 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033374122367101306, + "loss": 2.6208, + "theoretical_loss": 3.610574333317973, + "tokens_seen": 1119597568 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033373119358074224, + "loss": 2.6924, + "theoretical_loss": 3.610554638720722, + "tokens_seen": 1119663104 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003337211634904714, + "loss": 2.8094, + "theoretical_loss": 3.610534945598952, + "tokens_seen": 1119728640 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003337111334002006, + "loss": 2.7446, + "theoretical_loss": 3.6105152539524648, + "tokens_seen": 1119794176 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033370110330992983, + "loss": 2.81, + "theoretical_loss": 3.6104955637810643, + "tokens_seen": 1119859712 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033369107321965896, + "loss": 2.6061, + "theoretical_loss": 3.6104758750845534, + "tokens_seen": 1119925248 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003336810431293882, + "loss": 2.8181, + "theoretical_loss": 3.6104561878627353, + "tokens_seen": 1119990784 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003336710130391173, + "loss": 2.9375, + "theoretical_loss": 3.6104365021154132, + "tokens_seen": 1120056320 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033366098294884656, + "loss": 2.6833, + "theoretical_loss": 3.61041681784239, + "tokens_seen": 1120121856 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033365095285857574, + "loss": 2.9931, + "theoretical_loss": 3.61039713504347, + "tokens_seen": 1120187392 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003336409227683049, + "loss": 2.5654, + "theoretical_loss": 3.6103774537184563, + "tokens_seen": 1120252928 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003336308926780341, + "loss": 2.7439, + "theoretical_loss": 3.6103577738671513, + "tokens_seen": 1120318464 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003336208625877633, + "loss": 2.6787, + "theoretical_loss": 3.61033809548936, + "tokens_seen": 1120384000 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033361083249749246, + "loss": 2.7574, + "theoretical_loss": 3.6103184185848853, + "tokens_seen": 1120449536 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003336008024072217, + "loss": 2.5471, + "theoretical_loss": 3.61029874315353, + "tokens_seen": 1120515072 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003335907723169508, + "loss": 2.8783, + "theoretical_loss": 3.6102790691950988, + "tokens_seen": 1120580608 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033358074222668006, + "loss": 2.819, + "theoretical_loss": 3.6102593967093943, + "tokens_seen": 1120646144 + }, + { + "epoch": 3.07, + "objective/train/docs_used": 1252222, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9513473510742188, + "objective/train/theoretical_loss": 3.610254478818059, + "objective/train/tokens_used": 1141122528, + "theoretical_loss": 3.610254478818059, + "tokens_seen": 1120662528 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033357071213640924, + "loss": 2.7699, + "theoretical_loss": 3.6102397256962204, + "tokens_seen": 1120711680 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003335606820461384, + "loss": 2.6794, + "theoretical_loss": 3.6102200561553817, + "tokens_seen": 1120777216 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003335506519558676, + "loss": 2.694, + "theoretical_loss": 3.6102003880866804, + "tokens_seen": 1120842752 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003335406218655968, + "loss": 2.8837, + "theoretical_loss": 3.6101807214899218, + "tokens_seen": 1120908288 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033353059177532597, + "loss": 2.7733, + "theoretical_loss": 3.610161056364908, + "tokens_seen": 1120973824 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003335205616850552, + "loss": 2.7387, + "theoretical_loss": 3.6101413927114443, + "tokens_seen": 1121039360 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033351053159478433, + "loss": 2.7936, + "theoretical_loss": 3.6101217305293343, + "tokens_seen": 1121104896 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033350050150451356, + "loss": 2.9508, + "theoretical_loss": 3.610102069818381, + "tokens_seen": 1121170432 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003334904714142427, + "loss": 2.6335, + "theoretical_loss": 3.6100824105783893, + "tokens_seen": 1121235968 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003334804413239719, + "loss": 2.6319, + "theoretical_loss": 3.610062752809163, + "tokens_seen": 1121301504 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003334704112337011, + "loss": 2.7402, + "theoretical_loss": 3.6100430965105055, + "tokens_seen": 1121367040 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003334603811434303, + "loss": 2.7282, + "theoretical_loss": 3.6100234416822223, + "tokens_seen": 1121432576 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033345035105315947, + "loss": 2.7094, + "theoretical_loss": 3.6100037883241156, + "tokens_seen": 1121498112 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033344032096288865, + "loss": 2.8534, + "theoretical_loss": 3.6099841364359904, + "tokens_seen": 1121563648 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033343029087261783, + "loss": 2.909, + "theoretical_loss": 3.609964486017651, + "tokens_seen": 1121629184 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033342026078234707, + "loss": 2.9188, + "theoretical_loss": 3.609944837068902, + "tokens_seen": 1121694720 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003334102306920762, + "loss": 2.8513, + "theoretical_loss": 3.609925189589547, + "tokens_seen": 1121760256 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033340020060180543, + "loss": 2.5853, + "theoretical_loss": 3.60990554357939, + "tokens_seen": 1121825792 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003333901705115346, + "loss": 2.6128, + "theoretical_loss": 3.609885899038236, + "tokens_seen": 1121891328 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003333801404212638, + "loss": 2.6936, + "theoretical_loss": 3.6098662559658887, + "tokens_seen": 1121956864 + }, + { + "epoch": 3.07, + "learning_rate": 0.000333370110330993, + "loss": 2.5918, + "theoretical_loss": 3.6098466143621533, + "tokens_seen": 1122022400 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033336008024072215, + "loss": 2.6725, + "theoretical_loss": 3.609826974226834, + "tokens_seen": 1122087936 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033335005015045134, + "loss": 2.7519, + "theoretical_loss": 3.6098073355597347, + "tokens_seen": 1122153472 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033334002006018057, + "loss": 2.6089, + "theoretical_loss": 3.6097876983606607, + "tokens_seen": 1122219008 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033332998996990975, + "loss": 2.4769, + "theoretical_loss": 3.6097680626294157, + "tokens_seen": 1122284544 + }, + { + "epoch": 3.07, + "objective/train/docs_used": 1252729, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7386367321014404, + "objective/train/theoretical_loss": 3.60976315392593, + "objective/train/tokens_used": 1142760928, + "theoretical_loss": 3.60976315392593, + "tokens_seen": 1122300928 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033331995987963893, + "loss": 2.7276, + "theoretical_loss": 3.609748428365805, + "tokens_seen": 1122350080 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003333099297893681, + "loss": 2.7054, + "theoretical_loss": 3.609728795569633, + "tokens_seen": 1122415616 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003332998996990973, + "loss": 2.8766, + "theoretical_loss": 3.6097091642407038, + "tokens_seen": 1122481152 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033328986960882653, + "loss": 2.7833, + "theoretical_loss": 3.6096895343788225, + "tokens_seen": 1122546688 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033327983951855566, + "loss": 2.7231, + "theoretical_loss": 3.6096699059837944, + "tokens_seen": 1122612224 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003332698094282849, + "loss": 3.13, + "theoretical_loss": 3.6096502790554235, + "tokens_seen": 1122677760 + }, + { + "epoch": 3.07, + "learning_rate": 0.000333259779338014, + "loss": 2.7866, + "theoretical_loss": 3.6096306535935154, + "tokens_seen": 1122743296 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033324974924774326, + "loss": 2.7094, + "theoretical_loss": 3.6096110295978736, + "tokens_seen": 1122808832 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033323971915747244, + "loss": 2.699, + "theoretical_loss": 3.6095914070683044, + "tokens_seen": 1122874368 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003332296890672016, + "loss": 2.8131, + "theoretical_loss": 3.609571786004612, + "tokens_seen": 1122939904 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003332196589769308, + "loss": 2.669, + "theoretical_loss": 3.6095521664066013, + "tokens_seen": 1123005440 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033320962888666003, + "loss": 2.7178, + "theoretical_loss": 3.6095325482740774, + "tokens_seen": 1123070976 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033319959879638916, + "loss": 2.848, + "theoretical_loss": 3.6095129316068455, + "tokens_seen": 1123136512 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003331895687061184, + "loss": 2.7948, + "theoretical_loss": 3.609493316404711, + "tokens_seen": 1123202048 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003331795386158475, + "loss": 2.5098, + "theoretical_loss": 3.6094737026674775, + "tokens_seen": 1123267584 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033316950852557676, + "loss": 2.8703, + "theoretical_loss": 3.609454090394952, + "tokens_seen": 1123333120 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033315947843530594, + "loss": 2.6075, + "theoretical_loss": 3.6094344795869384, + "tokens_seen": 1123398656 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003331494483450351, + "loss": 2.7532, + "theoretical_loss": 3.609414870243243, + "tokens_seen": 1123464192 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003331394182547643, + "loss": 2.7733, + "theoretical_loss": 3.6093952623636696, + "tokens_seen": 1123529728 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003331293881644935, + "loss": 2.8531, + "theoretical_loss": 3.609375655948025, + "tokens_seen": 1123595264 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033311935807422266, + "loss": 2.5311, + "theoretical_loss": 3.609356050996113, + "tokens_seen": 1123660800 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003331093279839519, + "loss": 2.9696, + "theoretical_loss": 3.609336447507741, + "tokens_seen": 1123726336 + }, + { + "epoch": 3.07, + "learning_rate": 0.000333099297893681, + "loss": 2.9202, + "theoretical_loss": 3.609316845482712, + "tokens_seen": 1123791872 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033308926780341026, + "loss": 2.9049, + "theoretical_loss": 3.6092972449208336, + "tokens_seen": 1123857408 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033307923771313944, + "loss": 2.842, + "theoretical_loss": 3.60927764582191, + "tokens_seen": 1123922944 + }, + { + "epoch": 3.07, + "objective/train/docs_used": 1254028, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9574615955352783, + "objective/train/theoretical_loss": 3.609272746275743, + "objective/train/tokens_used": 1144399328, + "theoretical_loss": 3.609272746275743, + "tokens_seen": 1123939328 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003330692076228686, + "loss": 2.911, + "theoretical_loss": 3.6092580481857466, + "tokens_seen": 1123988480 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003330591775325978, + "loss": 2.7521, + "theoretical_loss": 3.60923845201215, + "tokens_seen": 1124054016 + }, + { + "epoch": 3.07, + "learning_rate": 0.000333049147442327, + "loss": 2.762, + "theoretical_loss": 3.6092188573009247, + "tokens_seen": 1124119552 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033303911735205617, + "loss": 3.0036, + "theoretical_loss": 3.6091992640518766, + "tokens_seen": 1124185088 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003330290872617854, + "loss": 2.87, + "theoretical_loss": 3.6091796722648124, + "tokens_seen": 1124250624 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033301905717151453, + "loss": 2.833, + "theoretical_loss": 3.609160081939536, + "tokens_seen": 1124316160 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033300902708124376, + "loss": 3.1339, + "theoretical_loss": 3.609140493075855, + "tokens_seen": 1124381696 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003329989969909729, + "loss": 2.846, + "theoretical_loss": 3.6091209056735734, + "tokens_seen": 1124447232 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003329889669007021, + "loss": 2.7986, + "theoretical_loss": 3.609101319732498, + "tokens_seen": 1124512768 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003329789368104313, + "loss": 2.8447, + "theoretical_loss": 3.6090817352524347, + "tokens_seen": 1124578304 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003329689067201605, + "loss": 2.6167, + "theoretical_loss": 3.6090621522331894, + "tokens_seen": 1124643840 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033295887662988967, + "loss": 2.6057, + "theoretical_loss": 3.6090425706745677, + "tokens_seen": 1124709376 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033294884653961885, + "loss": 2.8129, + "theoretical_loss": 3.609022990576376, + "tokens_seen": 1124774912 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033293881644934803, + "loss": 2.5224, + "theoretical_loss": 3.609003411938419, + "tokens_seen": 1124840448 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033292878635907727, + "loss": 2.826, + "theoretical_loss": 3.6089838347605045, + "tokens_seen": 1124905984 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003329187562688064, + "loss": 2.4935, + "theoretical_loss": 3.6089642590424376, + "tokens_seen": 1124971520 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033290872617853563, + "loss": 2.5188, + "theoretical_loss": 3.6089446847840243, + "tokens_seen": 1125037056 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003328986960882648, + "loss": 2.6992, + "theoretical_loss": 3.608925111985071, + "tokens_seen": 1125102592 + }, + { + "epoch": 3.07, + "learning_rate": 0.000332888665997994, + "loss": 2.7602, + "theoretical_loss": 3.6089055406453845, + "tokens_seen": 1125168128 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003328786359077232, + "loss": 2.6183, + "theoretical_loss": 3.60888597076477, + "tokens_seen": 1125233664 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033286860581745235, + "loss": 2.7598, + "theoretical_loss": 3.608866402343034, + "tokens_seen": 1125299200 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033285857572718154, + "loss": 2.5389, + "theoretical_loss": 3.608846835379983, + "tokens_seen": 1125364736 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033284854563691077, + "loss": 2.928, + "theoretical_loss": 3.6088272698754236, + "tokens_seen": 1125430272 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003328385155466399, + "loss": 2.8564, + "theoretical_loss": 3.6088077058291614, + "tokens_seen": 1125495808 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033282848545636913, + "loss": 2.6543, + "theoretical_loss": 3.6087881432410036, + "tokens_seen": 1125561344 + }, + { + "epoch": 3.07, + "objective/train/docs_used": 1254775, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.805901527404785, + "objective/train/theoretical_loss": 3.60878325282177, + "objective/train/tokens_used": 1146037728, + "theoretical_loss": 3.60878325282177, + "tokens_seen": 1125577728 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033281845536609826, + "loss": 2.7767, + "theoretical_loss": 3.608768582110756, + "tokens_seen": 1125626880 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003328084252758275, + "loss": 2.6417, + "theoretical_loss": 3.608749022438226, + "tokens_seen": 1125692416 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003327983951855567, + "loss": 2.6314, + "theoretical_loss": 3.6087294642232184, + "tokens_seen": 1125757952 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033278836509528586, + "loss": 2.4157, + "theoretical_loss": 3.6087099074655415, + "tokens_seen": 1125823488 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033277833500501504, + "loss": 2.8866, + "theoretical_loss": 3.6086903521650013, + "tokens_seen": 1125889024 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003327683049147442, + "loss": 2.9665, + "theoretical_loss": 3.608670798321404, + "tokens_seen": 1125954560 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003327582748244734, + "loss": 2.6735, + "theoretical_loss": 3.608651245934557, + "tokens_seen": 1126020096 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033274824473420264, + "loss": 2.656, + "theoretical_loss": 3.6086316950042665, + "tokens_seen": 1126085632 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033273821464393176, + "loss": 2.5484, + "theoretical_loss": 3.608612145530339, + "tokens_seen": 1126151168 + }, + { + "epoch": 3.07, + "learning_rate": 0.000332728184553661, + "loss": 2.5752, + "theoretical_loss": 3.608592597512582, + "tokens_seen": 1126216704 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003327181544633902, + "loss": 2.5263, + "theoretical_loss": 3.6085730509508016, + "tokens_seen": 1126282240 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033270812437311936, + "loss": 2.968, + "theoretical_loss": 3.608553505844805, + "tokens_seen": 1126347776 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033269809428284854, + "loss": 2.5893, + "theoretical_loss": 3.6085339621943993, + "tokens_seen": 1126413312 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003326880641925777, + "loss": 2.4944, + "theoretical_loss": 3.6085144199993904, + "tokens_seen": 1126478848 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003326780341023069, + "loss": 2.5355, + "theoretical_loss": 3.608494879259587, + "tokens_seen": 1126544384 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033266800401203614, + "loss": 2.702, + "theoretical_loss": 3.6084753399747944, + "tokens_seen": 1126609920 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033265797392176527, + "loss": 2.6876, + "theoretical_loss": 3.6084558021448205, + "tokens_seen": 1126675456 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003326479438314945, + "loss": 2.7097, + "theoretical_loss": 3.6084362657694724, + "tokens_seen": 1126740992 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033263791374122363, + "loss": 2.8215, + "theoretical_loss": 3.608416730848557, + "tokens_seen": 1126806528 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033262788365095286, + "loss": 2.7713, + "theoretical_loss": 3.608397197381881, + "tokens_seen": 1126872064 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033261785356068205, + "loss": 2.6377, + "theoretical_loss": 3.6083776653692525, + "tokens_seen": 1126937600 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003326078234704112, + "loss": 2.8388, + "theoretical_loss": 3.6083581348104783, + "tokens_seen": 1127003136 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003325977933801404, + "loss": 2.8913, + "theoretical_loss": 3.608338605705365, + "tokens_seen": 1127068672 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033258776328986964, + "loss": 2.7082, + "theoretical_loss": 3.6083190780537207, + "tokens_seen": 1127134208 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003325777331995988, + "loss": 2.6834, + "theoretical_loss": 3.608299551855353, + "tokens_seen": 1127199744 + }, + { + "epoch": 3.07, + "objective/train/docs_used": 1256132, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.0457611083984375, + "objective/train/theoretical_loss": 3.6082946705328123, + "objective/train/tokens_used": 1147676128, + "theoretical_loss": 3.6082946705328123, + "tokens_seen": 1127216128 + }, + { + "epoch": 3.07, + "learning_rate": 0.000332567703109328, + "loss": 3.1307, + "theoretical_loss": 3.608280027110068, + "tokens_seen": 1127265280 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003325576730190572, + "loss": 2.5617, + "theoretical_loss": 3.6082605038176743, + "tokens_seen": 1127330816 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033254764292878637, + "loss": 2.6656, + "theoretical_loss": 3.6082409819779784, + "tokens_seen": 1127396352 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003325376128385156, + "loss": 2.9161, + "theoretical_loss": 3.608221461590789, + "tokens_seen": 1127461888 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033252758274824473, + "loss": 2.7213, + "theoretical_loss": 3.6082019426559127, + "tokens_seen": 1127527424 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033251755265797396, + "loss": 2.6276, + "theoretical_loss": 3.6081824251731565, + "tokens_seen": 1127592960 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003325075225677031, + "loss": 2.6259, + "theoretical_loss": 3.6081629091423295, + "tokens_seen": 1127658496 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033249749247743233, + "loss": 2.7656, + "theoretical_loss": 3.608143394563238, + "tokens_seen": 1127724032 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003324874623871615, + "loss": 2.7165, + "theoretical_loss": 3.6081238814356906, + "tokens_seen": 1127789568 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003324774322968907, + "loss": 2.8234, + "theoretical_loss": 3.608104369759494, + "tokens_seen": 1127855104 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033246740220661987, + "loss": 2.9237, + "theoretical_loss": 3.608084859534457, + "tokens_seen": 1127920640 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033245737211634905, + "loss": 2.7804, + "theoretical_loss": 3.6080653507603864, + "tokens_seen": 1127986176 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033244734202607823, + "loss": 2.7878, + "theoretical_loss": 3.6080458434370906, + "tokens_seen": 1128051712 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033243731193580747, + "loss": 2.8638, + "theoretical_loss": 3.608026337564377, + "tokens_seen": 1128117248 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003324272818455366, + "loss": 2.5568, + "theoretical_loss": 3.6080068331420545, + "tokens_seen": 1128182784 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033241725175526583, + "loss": 2.7804, + "theoretical_loss": 3.6079873301699292, + "tokens_seen": 1128248320 + }, + { + "epoch": 3.07, + "learning_rate": 0.000332407221664995, + "loss": 2.6939, + "theoretical_loss": 3.607967828647811, + "tokens_seen": 1128313856 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003323971915747242, + "loss": 2.9341, + "theoretical_loss": 3.6079483285755067, + "tokens_seen": 1128379392 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003323871614844534, + "loss": 2.8499, + "theoretical_loss": 3.6079288299528245, + "tokens_seen": 1128444928 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033237713139418255, + "loss": 2.6364, + "theoretical_loss": 3.6079093327795726, + "tokens_seen": 1128510464 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033236710130391174, + "loss": 2.692, + "theoretical_loss": 3.607889837055559, + "tokens_seen": 1128576000 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033235707121364097, + "loss": 2.7181, + "theoretical_loss": 3.607870342780592, + "tokens_seen": 1128641536 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003323470411233701, + "loss": 2.6819, + "theoretical_loss": 3.6078508499544797, + "tokens_seen": 1128707072 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033233701103309933, + "loss": 2.7742, + "theoretical_loss": 3.6078313585770303, + "tokens_seen": 1128772608 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033232698094282846, + "loss": 2.8938, + "theoretical_loss": 3.607811868648052, + "tokens_seen": 1128838144 + }, + { + "epoch": 3.07, + "objective/train/docs_used": 1256724, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9342129230499268, + "objective/train/theoretical_loss": 3.6078069963921084, + "objective/train/tokens_used": 1149314528, + "theoretical_loss": 3.6078069963921084, + "tokens_seen": 1128854528 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003323169508525577, + "loss": 2.6303, + "theoretical_loss": 3.6077923801673526, + "tokens_seen": 1128903680 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003323069207622869, + "loss": 2.9345, + "theoretical_loss": 3.607772893134741, + "tokens_seen": 1128969216 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033229689067201606, + "loss": 2.5573, + "theoretical_loss": 3.6077534075500264, + "tokens_seen": 1129034752 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033228686058174524, + "loss": 2.6129, + "theoretical_loss": 3.607733923413015, + "tokens_seen": 1129100288 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003322768304914744, + "loss": 2.7973, + "theoretical_loss": 3.607714440723517, + "tokens_seen": 1129165824 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003322668004012036, + "loss": 2.7817, + "theoretical_loss": 3.60769495948134, + "tokens_seen": 1129231360 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033225677031093284, + "loss": 2.9254, + "theoretical_loss": 3.6076754796862933, + "tokens_seen": 1129296896 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033224674022066196, + "loss": 2.8688, + "theoretical_loss": 3.6076560013381846, + "tokens_seen": 1129362432 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003322367101303912, + "loss": 2.731, + "theoretical_loss": 3.607636524436823, + "tokens_seen": 1129427968 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003322266800401204, + "loss": 2.3886, + "theoretical_loss": 3.607617048982017, + "tokens_seen": 1129493504 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033221664994984956, + "loss": 2.4788, + "theoretical_loss": 3.6075975749735747, + "tokens_seen": 1129559040 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033220661985957874, + "loss": 2.6113, + "theoretical_loss": 3.6075781024113054, + "tokens_seen": 1129624576 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003321965897693079, + "loss": 2.7369, + "theoretical_loss": 3.607558631295017, + "tokens_seen": 1129690112 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003321865596790371, + "loss": 2.7931, + "theoretical_loss": 3.60753916162452, + "tokens_seen": 1129755648 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033217652958876634, + "loss": 2.6653, + "theoretical_loss": 3.607519693399621, + "tokens_seen": 1129821184 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033216649949849547, + "loss": 2.771, + "theoretical_loss": 3.6075002266201306, + "tokens_seen": 1129886720 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003321564694082247, + "loss": 2.8361, + "theoretical_loss": 3.6074807612858573, + "tokens_seen": 1129952256 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033214643931795383, + "loss": 2.4755, + "theoretical_loss": 3.6074612973966085, + "tokens_seen": 1130017792 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033213640922768306, + "loss": 2.882, + "theoretical_loss": 3.607441834952195, + "tokens_seen": 1130083328 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033212637913741225, + "loss": 2.6837, + "theoretical_loss": 3.607422373952425, + "tokens_seen": 1130148864 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003321163490471414, + "loss": 2.825, + "theoretical_loss": 3.607402914397108, + "tokens_seen": 1130214400 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003321063189568706, + "loss": 2.7678, + "theoretical_loss": 3.607383456286051, + "tokens_seen": 1130279936 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033209628886659984, + "loss": 2.7812, + "theoretical_loss": 3.607363999619066, + "tokens_seen": 1130345472 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033208625877632897, + "loss": 2.7462, + "theoretical_loss": 3.6073445443959606, + "tokens_seen": 1130411008 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003320762286860582, + "loss": 2.9633, + "theoretical_loss": 3.607325090616544, + "tokens_seen": 1130476544 + }, + { + "epoch": 3.07, + "objective/train/docs_used": 1258334, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.567551851272583, + "objective/train/theoretical_loss": 3.607320227397244, + "objective/train/tokens_used": 1150952928, + "theoretical_loss": 3.607320227397244, + "tokens_seen": 1130492928 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033206619859578733, + "loss": 2.753, + "theoretical_loss": 3.607305638280626, + "tokens_seen": 1130542080 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033205616850551657, + "loss": 2.5546, + "theoretical_loss": 3.6072861873880147, + "tokens_seen": 1130607616 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033204613841524575, + "loss": 2.8395, + "theoretical_loss": 3.6072667379385197, + "tokens_seen": 1130673152 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033203610832497493, + "loss": 2.8606, + "theoretical_loss": 3.607247289931951, + "tokens_seen": 1130738688 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003320260782347041, + "loss": 2.7365, + "theoretical_loss": 3.6072278433681175, + "tokens_seen": 1130804224 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003320160481444333, + "loss": 2.7856, + "theoretical_loss": 3.6072083982468284, + "tokens_seen": 1130869760 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003320060180541625, + "loss": 2.5667, + "theoretical_loss": 3.607188954567894, + "tokens_seen": 1130935296 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003319959879638917, + "loss": 2.629, + "theoretical_loss": 3.6071695123311223, + "tokens_seen": 1131000832 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033198595787362084, + "loss": 2.7256, + "theoretical_loss": 3.6071500715363234, + "tokens_seen": 1131066368 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033197592778335007, + "loss": 3.0258, + "theoretical_loss": 3.607130632183307, + "tokens_seen": 1131131904 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003319658976930792, + "loss": 2.8214, + "theoretical_loss": 3.6071111942718836, + "tokens_seen": 1131197440 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033195586760280843, + "loss": 2.6264, + "theoretical_loss": 3.607091757801861, + "tokens_seen": 1131262976 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003319458375125376, + "loss": 2.6839, + "theoretical_loss": 3.607072322773049, + "tokens_seen": 1131328512 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003319358074222668, + "loss": 2.9318, + "theoretical_loss": 3.6070528891852582, + "tokens_seen": 1131394048 + }, + { + "epoch": 3.07, + "learning_rate": 0.000331925777331996, + "loss": 2.7424, + "theoretical_loss": 3.6070334570382983, + "tokens_seen": 1131459584 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003319157472417252, + "loss": 2.7137, + "theoretical_loss": 3.6070140263319783, + "tokens_seen": 1131525120 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033190571715145434, + "loss": 2.8819, + "theoretical_loss": 3.6069945970661084, + "tokens_seen": 1131590656 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003318956870611836, + "loss": 2.8069, + "theoretical_loss": 3.6069751692404983, + "tokens_seen": 1131656192 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003318856569709127, + "loss": 2.7572, + "theoretical_loss": 3.606955742854958, + "tokens_seen": 1131721728 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033187562688064194, + "loss": 2.646, + "theoretical_loss": 3.606936317909297, + "tokens_seen": 1131787264 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003318655967903711, + "loss": 2.8115, + "theoretical_loss": 3.6069168944033256, + "tokens_seen": 1131852800 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003318555667001003, + "loss": 2.629, + "theoretical_loss": 3.606897472336853, + "tokens_seen": 1131918336 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003318455366098295, + "loss": 2.5971, + "theoretical_loss": 3.6068780517096903, + "tokens_seen": 1131983872 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033183550651955866, + "loss": 2.7799, + "theoretical_loss": 3.606858632521647, + "tokens_seen": 1132049408 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003318254764292879, + "loss": 2.7061, + "theoretical_loss": 3.606839214772533, + "tokens_seen": 1132114944 + }, + { + "epoch": 3.07, + "objective/train/docs_used": 1259128, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.0284805297851562, + "objective/train/theoretical_loss": 3.606834360560065, + "objective/train/tokens_used": 1152591328, + "theoretical_loss": 3.606834360560065, + "tokens_seen": 1132131328 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003318154463390171, + "loss": 2.8331, + "theoretical_loss": 3.6068197984621584, + "tokens_seen": 1132180480 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033180541624874626, + "loss": 2.4103, + "theoretical_loss": 3.6068003835903335, + "tokens_seen": 1132246016 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033179538615847544, + "loss": 2.5474, + "theoretical_loss": 3.6067809701568687, + "tokens_seen": 1132311552 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003317853560682046, + "loss": 2.5511, + "theoretical_loss": 3.6067615581615735, + "tokens_seen": 1132377088 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003317753259779338, + "loss": 2.7727, + "theoretical_loss": 3.606742147604259, + "tokens_seen": 1132442624 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033176529588766304, + "loss": 2.8224, + "theoretical_loss": 3.6067227384847347, + "tokens_seen": 1132508160 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033175526579739216, + "loss": 2.6972, + "theoretical_loss": 3.6067033308028114, + "tokens_seen": 1132573696 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003317452357071214, + "loss": 2.626, + "theoretical_loss": 3.6066839245582987, + "tokens_seen": 1132639232 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003317352056168506, + "loss": 2.6497, + "theoretical_loss": 3.6066645197510083, + "tokens_seen": 1132704768 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033172517552657976, + "loss": 2.6526, + "theoretical_loss": 3.6066451163807494, + "tokens_seen": 1132770304 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033171514543630894, + "loss": 2.8228, + "theoretical_loss": 3.606625714447333, + "tokens_seen": 1132835840 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003317051153460381, + "loss": 2.9014, + "theoretical_loss": 3.6066063139505697, + "tokens_seen": 1132901376 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003316950852557673, + "loss": 2.4233, + "theoretical_loss": 3.6065869148902694, + "tokens_seen": 1132966912 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033168505516549654, + "loss": 2.7184, + "theoretical_loss": 3.6065675172662433, + "tokens_seen": 1133032448 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033167502507522567, + "loss": 2.7, + "theoretical_loss": 3.606548121078302, + "tokens_seen": 1133097984 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003316649949849549, + "loss": 2.771, + "theoretical_loss": 3.6065287263262555, + "tokens_seen": 1133163520 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033165496489468403, + "loss": 2.7794, + "theoretical_loss": 3.6065093330099147, + "tokens_seen": 1133229056 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033164493480441326, + "loss": 2.5853, + "theoretical_loss": 3.606489941129091, + "tokens_seen": 1133294592 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033163490471414245, + "loss": 2.5687, + "theoretical_loss": 3.6064705506835946, + "tokens_seen": 1133360128 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003316248746238716, + "loss": 2.7562, + "theoretical_loss": 3.6064511616732355, + "tokens_seen": 1133425664 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003316148445336008, + "loss": 2.6475, + "theoretical_loss": 3.6064317740978264, + "tokens_seen": 1133491200 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033160481444333004, + "loss": 2.8928, + "theoretical_loss": 3.6064123879571763, + "tokens_seen": 1133556736 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033159478435305917, + "loss": 2.8312, + "theoretical_loss": 3.606393003251097, + "tokens_seen": 1133622272 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003315847542627884, + "loss": 2.723, + "theoretical_loss": 3.606373619979399, + "tokens_seen": 1133687808 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033157472417251753, + "loss": 2.8412, + "theoretical_loss": 3.606354238141894, + "tokens_seen": 1133753344 + }, + { + "epoch": 3.07, + "objective/train/docs_used": 1260370, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.1477930545806885, + "objective/train/theoretical_loss": 3.606349392906588, + "objective/train/tokens_used": 1154229728, + "theoretical_loss": 3.606349392906588, + "tokens_seen": 1133769728 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033156469408224677, + "loss": 2.5493, + "theoretical_loss": 3.606334857738392, + "tokens_seen": 1133818880 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033155466399197595, + "loss": 2.5485, + "theoretical_loss": 3.606315478768704, + "tokens_seen": 1133884416 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033154463390170513, + "loss": 2.8706, + "theoretical_loss": 3.6062961012326427, + "tokens_seen": 1133949952 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003315346038114343, + "loss": 2.6468, + "theoretical_loss": 3.6062767251300176, + "tokens_seen": 1134015488 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003315245737211635, + "loss": 2.7919, + "theoretical_loss": 3.6062573504606403, + "tokens_seen": 1134081024 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003315145436308927, + "loss": 2.8601, + "theoretical_loss": 3.6062379772243216, + "tokens_seen": 1134146560 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003315045135406219, + "loss": 2.712, + "theoretical_loss": 3.6062186054208736, + "tokens_seen": 1134212096 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033149448345035104, + "loss": 2.5173, + "theoretical_loss": 3.6061992350501066, + "tokens_seen": 1134277632 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033148445336008027, + "loss": 2.4526, + "theoretical_loss": 3.606179866111832, + "tokens_seen": 1134343168 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003314744232698094, + "loss": 2.8311, + "theoretical_loss": 3.6061604986058615, + "tokens_seen": 1134408704 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033146439317953863, + "loss": 2.8495, + "theoretical_loss": 3.6061411325320067, + "tokens_seen": 1134474240 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003314543630892678, + "loss": 2.9322, + "theoretical_loss": 3.6061217678900785, + "tokens_seen": 1134539776 + }, + { + "epoch": 3.07, + "learning_rate": 0.000331444332998997, + "loss": 2.5131, + "theoretical_loss": 3.6061024046798886, + "tokens_seen": 1134605312 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003314343029087262, + "loss": 2.8674, + "theoretical_loss": 3.606083042901248, + "tokens_seen": 1134670848 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003314242728184554, + "loss": 2.8922, + "theoretical_loss": 3.6060636825539687, + "tokens_seen": 1134736384 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033141424272818454, + "loss": 2.7273, + "theoretical_loss": 3.6060443236378616, + "tokens_seen": 1134801920 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003314042126379138, + "loss": 2.7511, + "theoretical_loss": 3.606024966152739, + "tokens_seen": 1134867456 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003313941825476429, + "loss": 2.6649, + "theoretical_loss": 3.606005610098412, + "tokens_seen": 1134932992 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033138415245737214, + "loss": 2.8792, + "theoretical_loss": 3.6059862554746926, + "tokens_seen": 1134998528 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003313741223671013, + "loss": 2.6218, + "theoretical_loss": 3.605966902281392, + "tokens_seen": 1135064064 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003313640922768305, + "loss": 2.811, + "theoretical_loss": 3.605947550518322, + "tokens_seen": 1135129600 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003313540621865597, + "loss": 2.5705, + "theoretical_loss": 3.6059282001852946, + "tokens_seen": 1135195136 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033134403209628886, + "loss": 2.9007, + "theoretical_loss": 3.6059088512821216, + "tokens_seen": 1135260672 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033133400200601804, + "loss": 2.7223, + "theoretical_loss": 3.6058895038086147, + "tokens_seen": 1135326208 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003313239719157473, + "loss": 2.8814, + "theoretical_loss": 3.6058701577645857, + "tokens_seen": 1135391744 + }, + { + "epoch": 3.07, + "objective/train/docs_used": 1261008, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7761924266815186, + "objective/train/theoretical_loss": 3.6058653214769123, + "objective/train/tokens_used": 1155868128, + "theoretical_loss": 3.6058653214769123, + "tokens_seen": 1135408128 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003313139418254764, + "loss": 2.8254, + "theoretical_loss": 3.605850813149847, + "tokens_seen": 1135457280 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033130391173520564, + "loss": 2.6401, + "theoretical_loss": 3.605831469964209, + "tokens_seen": 1135522816 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033129388164493477, + "loss": 2.9165, + "theoretical_loss": 3.6058121282074853, + "tokens_seen": 1135588352 + }, + { + "epoch": 3.07, + "learning_rate": 0.000331283851554664, + "loss": 2.6665, + "theoretical_loss": 3.605792787879487, + "tokens_seen": 1135653888 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003312738214643932, + "loss": 2.8296, + "theoretical_loss": 3.6057734489800266, + "tokens_seen": 1135719424 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033126379137412236, + "loss": 2.4184, + "theoretical_loss": 3.605754111508916, + "tokens_seen": 1135784960 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033125376128385155, + "loss": 2.8367, + "theoretical_loss": 3.605734775465967, + "tokens_seen": 1135850496 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003312437311935808, + "loss": 2.6468, + "theoretical_loss": 3.605715440850992, + "tokens_seen": 1135916032 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003312337011033099, + "loss": 2.5701, + "theoretical_loss": 3.6056961076638037, + "tokens_seen": 1135981568 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033122367101303914, + "loss": 2.7412, + "theoretical_loss": 3.6056767759042136, + "tokens_seen": 1136047104 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033121364092276827, + "loss": 2.4669, + "theoretical_loss": 3.6056574455720334, + "tokens_seen": 1136112640 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003312036108324975, + "loss": 2.7462, + "theoretical_loss": 3.605638116667077, + "tokens_seen": 1136178176 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003311935807422267, + "loss": 2.8023, + "theoretical_loss": 3.605618789189155, + "tokens_seen": 1136243712 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033118355065195587, + "loss": 2.5532, + "theoretical_loss": 3.6055994631380806, + "tokens_seen": 1136309248 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033117352056168505, + "loss": 2.6124, + "theoretical_loss": 3.6055801385136665, + "tokens_seen": 1136374784 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033116349047141423, + "loss": 2.5696, + "theoretical_loss": 3.605560815315725, + "tokens_seen": 1136440320 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003311534603811434, + "loss": 2.5819, + "theoretical_loss": 3.605541493544067, + "tokens_seen": 1136505856 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033114343029087265, + "loss": 2.8007, + "theoretical_loss": 3.6055221731985068, + "tokens_seen": 1136571392 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033113340020060177, + "loss": 2.6863, + "theoretical_loss": 3.605502854278857, + "tokens_seen": 1136636928 + }, + { + "epoch": 3.07, + "learning_rate": 0.000331123370110331, + "loss": 2.8361, + "theoretical_loss": 3.605483536784929, + "tokens_seen": 1136702464 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033111334002006013, + "loss": 2.8174, + "theoretical_loss": 3.605464220716536, + "tokens_seen": 1136768000 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033110330992978937, + "loss": 2.7956, + "theoretical_loss": 3.6054449060734903, + "tokens_seen": 1136833536 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033109327983951855, + "loss": 2.8733, + "theoretical_loss": 3.605425592855605, + "tokens_seen": 1136899072 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033108324974924773, + "loss": 2.6193, + "theoretical_loss": 3.6054062810626926, + "tokens_seen": 1136964608 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033107321965897697, + "loss": 2.8384, + "theoretical_loss": 3.605386970694566, + "tokens_seen": 1137030144 + }, + { + "epoch": 3.07, + "objective/train/docs_used": 1262017, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.1084911823272705, + "objective/train/theoretical_loss": 3.6053821433251354, + "objective/train/tokens_used": 1157506528, + "theoretical_loss": 3.6053821433251354, + "tokens_seen": 1137046528 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033106318956870615, + "loss": 2.7541, + "theoretical_loss": 3.6053676617510377, + "tokens_seen": 1137095680 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033105315947843533, + "loss": 2.5768, + "theoretical_loss": 3.6053483542319205, + "tokens_seen": 1137161216 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003310431293881645, + "loss": 2.6472, + "theoretical_loss": 3.6053290481370275, + "tokens_seen": 1137226752 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003310330992978937, + "loss": 2.4923, + "theoretical_loss": 3.6053097434661714, + "tokens_seen": 1137292288 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003310230692076229, + "loss": 2.7072, + "theoretical_loss": 3.6052904402191652, + "tokens_seen": 1137357824 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003310130391173521, + "loss": 2.7793, + "theoretical_loss": 3.6052711383958216, + "tokens_seen": 1137423360 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033100300902708124, + "loss": 2.8709, + "theoretical_loss": 3.605251837995954, + "tokens_seen": 1137488896 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033099297893681047, + "loss": 2.58, + "theoretical_loss": 3.605232539019375, + "tokens_seen": 1137554432 + }, + { + "epoch": 3.07, + "learning_rate": 0.0003309829488465396, + "loss": 2.7207, + "theoretical_loss": 3.605213241465898, + "tokens_seen": 1137619968 + }, + { + "epoch": 3.07, + "learning_rate": 0.00033097291875626883, + "loss": 2.5586, + "theoretical_loss": 3.605193945335336, + "tokens_seen": 1137685504 + }, + { + "epoch": 3.07, + "learning_rate": 0.000330962888665998, + "loss": 2.5498, + "theoretical_loss": 3.605174650627502, + "tokens_seen": 1137751040 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003309528585757272, + "loss": 2.7564, + "theoretical_loss": 3.6051553573422086, + "tokens_seen": 1137816576 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003309428284854564, + "loss": 2.8811, + "theoretical_loss": 3.60513606547927, + "tokens_seen": 1137882112 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003309327983951856, + "loss": 2.7868, + "theoretical_loss": 3.6051167750384994, + "tokens_seen": 1137947648 + }, + { + "epoch": 3.08, + "learning_rate": 0.00033092276830491474, + "loss": 2.8146, + "theoretical_loss": 3.6050974860197096, + "tokens_seen": 1138013184 + }, + { + "epoch": 3.08, + "learning_rate": 0.000330912738214644, + "loss": 2.7553, + "theoretical_loss": 3.6050781984227136, + "tokens_seen": 1138078720 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003309027081243731, + "loss": 2.8369, + "theoretical_loss": 3.6050589122473253, + "tokens_seen": 1138144256 + }, + { + "epoch": 3.08, + "learning_rate": 0.00033089267803410234, + "loss": 2.5716, + "theoretical_loss": 3.605039627493358, + "tokens_seen": 1138209792 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003308826479438315, + "loss": 2.9157, + "theoretical_loss": 3.605020344160625, + "tokens_seen": 1138275328 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003308726178535607, + "loss": 2.565, + "theoretical_loss": 3.60500106224894, + "tokens_seen": 1138340864 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003308625877632899, + "loss": 2.4975, + "theoretical_loss": 3.6049817817581156, + "tokens_seen": 1138406400 + }, + { + "epoch": 3.08, + "learning_rate": 0.00033085255767301906, + "loss": 2.6338, + "theoretical_loss": 3.6049625026879664, + "tokens_seen": 1138471936 + }, + { + "epoch": 3.08, + "learning_rate": 0.00033084252758274824, + "loss": 2.6649, + "theoretical_loss": 3.604943225038305, + "tokens_seen": 1138537472 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003308324974924775, + "loss": 2.669, + "theoretical_loss": 3.604923948808946, + "tokens_seen": 1138603008 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003308224674022066, + "loss": 2.7445, + "theoretical_loss": 3.604904673999702, + "tokens_seen": 1138668544 + }, + { + "epoch": 3.08, + "objective/train/docs_used": 1262682, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.618945837020874, + "objective/train/theoretical_loss": 3.604899855519262, + "objective/train/tokens_used": 1159144928, + "theoretical_loss": 3.604899855519262, + "tokens_seen": 1138684928 + }, + { + "epoch": 3.08, + "learning_rate": 0.00033081243731193584, + "loss": 2.759, + "theoretical_loss": 3.604885400610388, + "tokens_seen": 1138734080 + }, + { + "epoch": 3.08, + "learning_rate": 0.00033080240722166497, + "loss": 2.7804, + "theoretical_loss": 3.6048661286408157, + "tokens_seen": 1138799616 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003307923771313942, + "loss": 2.8219, + "theoretical_loss": 3.6048468580908004, + "tokens_seen": 1138865152 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003307823470411234, + "loss": 2.6843, + "theoretical_loss": 3.6048275889601555, + "tokens_seen": 1138930688 + }, + { + "epoch": 3.08, + "learning_rate": 0.00033077231695085256, + "loss": 2.9408, + "theoretical_loss": 3.6048083212486945, + "tokens_seen": 1138996224 + }, + { + "epoch": 3.08, + "learning_rate": 0.00033076228686058175, + "loss": 2.6726, + "theoretical_loss": 3.604789054956232, + "tokens_seen": 1139061760 + }, + { + "epoch": 3.08, + "learning_rate": 0.000330752256770311, + "loss": 2.8145, + "theoretical_loss": 3.604769790082581, + "tokens_seen": 1139127296 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003307422266800401, + "loss": 2.6603, + "theoretical_loss": 3.604750526627555, + "tokens_seen": 1139192832 + }, + { + "epoch": 3.08, + "learning_rate": 0.00033073219658976934, + "loss": 2.6343, + "theoretical_loss": 3.6047312645909697, + "tokens_seen": 1139258368 + }, + { + "epoch": 3.08, + "learning_rate": 0.00033072216649949847, + "loss": 2.5703, + "theoretical_loss": 3.6047120039726375, + "tokens_seen": 1139323904 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003307121364092277, + "loss": 3.0216, + "theoretical_loss": 3.6046927447723727, + "tokens_seen": 1139389440 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003307021063189569, + "loss": 2.7889, + "theoretical_loss": 3.6046734869899892, + "tokens_seen": 1139454976 + }, + { + "epoch": 3.08, + "learning_rate": 0.00033069207622868607, + "loss": 2.7531, + "theoretical_loss": 3.604654230625302, + "tokens_seen": 1139520512 + }, + { + "epoch": 3.08, + "learning_rate": 0.00033068204613841525, + "loss": 2.6034, + "theoretical_loss": 3.6046349756781244, + "tokens_seen": 1139586048 + }, + { + "epoch": 3.08, + "learning_rate": 0.00033067201604814443, + "loss": 2.7754, + "theoretical_loss": 3.604615722148271, + "tokens_seen": 1139651584 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003306619859578736, + "loss": 2.7181, + "theoretical_loss": 3.6045964700355553, + "tokens_seen": 1139717120 + }, + { + "epoch": 3.08, + "learning_rate": 0.00033065195586760285, + "loss": 2.8112, + "theoretical_loss": 3.6045772193397916, + "tokens_seen": 1139782656 + }, + { + "epoch": 3.08, + "learning_rate": 0.00033064192577733197, + "loss": 2.7175, + "theoretical_loss": 3.6045579700607955, + "tokens_seen": 1139848192 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003306318956870612, + "loss": 2.7196, + "theoretical_loss": 3.6045387221983796, + "tokens_seen": 1139913728 + }, + { + "epoch": 3.08, + "learning_rate": 0.00033062186559679034, + "loss": 2.5488, + "theoretical_loss": 3.604519475752359, + "tokens_seen": 1139979264 + }, + { + "epoch": 3.08, + "learning_rate": 0.00033061183550651957, + "loss": 2.7686, + "theoretical_loss": 3.604500230722548, + "tokens_seen": 1140044800 + }, + { + "epoch": 3.08, + "learning_rate": 0.00033060180541624875, + "loss": 2.8398, + "theoretical_loss": 3.6044809871087606, + "tokens_seen": 1140110336 + }, + { + "epoch": 3.08, + "learning_rate": 0.00033059177532597793, + "loss": 2.8059, + "theoretical_loss": 3.604461744910812, + "tokens_seen": 1140175872 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003305817452357071, + "loss": 2.8029, + "theoretical_loss": 3.6044425041285164, + "tokens_seen": 1140241408 + }, + { + "epoch": 3.08, + "learning_rate": 0.00033057171514543635, + "loss": 2.8484, + "theoretical_loss": 3.604423264761688, + "tokens_seen": 1140306944 + }, + { + "epoch": 3.08, + "objective/train/docs_used": 1264116, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.787461757659912, + "objective/train/theoretical_loss": 3.6044184551411256, + "objective/train/tokens_used": 1160783328, + "theoretical_loss": 3.6044184551411256, + "tokens_seen": 1140323328 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003305616850551655, + "loss": 2.7743, + "theoretical_loss": 3.604404026810141, + "tokens_seen": 1140372480 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003305516549648947, + "loss": 2.7962, + "theoretical_loss": 3.6043847902736905, + "tokens_seen": 1140438016 + }, + { + "epoch": 3.08, + "learning_rate": 0.00033054162487462384, + "loss": 2.5863, + "theoretical_loss": 3.6043655551521514, + "tokens_seen": 1140503552 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003305315947843531, + "loss": 2.7442, + "theoretical_loss": 3.6043463214453375, + "tokens_seen": 1140569088 + }, + { + "epoch": 3.08, + "learning_rate": 0.00033052156469408225, + "loss": 2.7425, + "theoretical_loss": 3.604327089153064, + "tokens_seen": 1140634624 + }, + { + "epoch": 3.08, + "learning_rate": 0.00033051153460381144, + "loss": 2.7151, + "theoretical_loss": 3.6043078582751455, + "tokens_seen": 1140700160 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003305015045135406, + "loss": 2.7677, + "theoretical_loss": 3.604288628811397, + "tokens_seen": 1140765696 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003304914744232698, + "loss": 2.8816, + "theoretical_loss": 3.6042694007616327, + "tokens_seen": 1140831232 + }, + { + "epoch": 3.08, + "learning_rate": 0.000330481444332999, + "loss": 2.75, + "theoretical_loss": 3.604250174125668, + "tokens_seen": 1140896768 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003304714142427282, + "loss": 2.8542, + "theoretical_loss": 3.604230948903317, + "tokens_seen": 1140962304 + }, + { + "epoch": 3.08, + "learning_rate": 0.00033046138415245734, + "loss": 2.7008, + "theoretical_loss": 3.6042117250943955, + "tokens_seen": 1141027840 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003304513540621866, + "loss": 2.8737, + "theoretical_loss": 3.6041925026987176, + "tokens_seen": 1141093376 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003304413239719157, + "loss": 2.6164, + "theoretical_loss": 3.6041732817160987, + "tokens_seen": 1141158912 + }, + { + "epoch": 3.08, + "learning_rate": 0.00033043129388164494, + "loss": 2.9281, + "theoretical_loss": 3.604154062146354, + "tokens_seen": 1141224448 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003304212637913741, + "loss": 2.714, + "theoretical_loss": 3.604134843989298, + "tokens_seen": 1141289984 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003304112337011033, + "loss": 2.6748, + "theoretical_loss": 3.604115627244746, + "tokens_seen": 1141355520 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003304012036108325, + "loss": 2.5346, + "theoretical_loss": 3.604096411912513, + "tokens_seen": 1141421056 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003303911735205617, + "loss": 2.7443, + "theoretical_loss": 3.604077197992414, + "tokens_seen": 1141486592 + }, + { + "epoch": 3.08, + "learning_rate": 0.00033038114343029084, + "loss": 2.901, + "theoretical_loss": 3.604057985484265, + "tokens_seen": 1141552128 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003303711133400201, + "loss": 2.9053, + "theoretical_loss": 3.6040387743878797, + "tokens_seen": 1141617664 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003303610832497492, + "loss": 2.9141, + "theoretical_loss": 3.6040195647030746, + "tokens_seen": 1141683200 + }, + { + "epoch": 3.08, + "learning_rate": 0.00033035105315947844, + "loss": 2.6902, + "theoretical_loss": 3.604000356429664, + "tokens_seen": 1141748736 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003303410230692076, + "loss": 2.7633, + "theoretical_loss": 3.6039811495674643, + "tokens_seen": 1141814272 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003303309929789368, + "loss": 2.9135, + "theoretical_loss": 3.60396194411629, + "tokens_seen": 1141879808 + }, + { + "epoch": 3.08, + "learning_rate": 0.00033032096288866604, + "loss": 2.5842, + "theoretical_loss": 3.6039427400759565, + "tokens_seen": 1141945344 + }, + { + "epoch": 3.08, + "objective/train/docs_used": 1264658, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5670065879821777, + "objective/train/theoretical_loss": 3.603937939286295, + "objective/train/tokens_used": 1162421728, + "theoretical_loss": 3.603937939286295, + "tokens_seen": 1141961728 + }, + { + "epoch": 3.08, + "learning_rate": 0.00033031093279839517, + "loss": 2.6734, + "theoretical_loss": 3.6039235374462795, + "tokens_seen": 1142010880 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003303009027081244, + "loss": 2.6518, + "theoretical_loss": 3.6039043362270746, + "tokens_seen": 1142076416 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003302908726178536, + "loss": 2.6918, + "theoretical_loss": 3.6038851364181568, + "tokens_seen": 1142141952 + }, + { + "epoch": 3.08, + "learning_rate": 0.00033028084252758276, + "loss": 2.8456, + "theoretical_loss": 3.603865938019341, + "tokens_seen": 1142207488 + }, + { + "epoch": 3.08, + "learning_rate": 0.00033027081243731195, + "loss": 2.6436, + "theoretical_loss": 3.603846741030445, + "tokens_seen": 1142273024 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003302607823470412, + "loss": 2.6678, + "theoretical_loss": 3.603827545451282, + "tokens_seen": 1142338560 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003302507522567703, + "loss": 2.7732, + "theoretical_loss": 3.6038083512816685, + "tokens_seen": 1142404096 + }, + { + "epoch": 3.08, + "learning_rate": 0.00033024072216649954, + "loss": 2.7161, + "theoretical_loss": 3.6037891585214203, + "tokens_seen": 1142469632 + }, + { + "epoch": 3.08, + "learning_rate": 0.00033023069207622867, + "loss": 2.7033, + "theoretical_loss": 3.6037699671703534, + "tokens_seen": 1142535168 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003302206619859579, + "loss": 2.7151, + "theoretical_loss": 3.6037507772282824, + "tokens_seen": 1142600704 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003302106318956871, + "loss": 2.7908, + "theoretical_loss": 3.603731588695024, + "tokens_seen": 1142666240 + }, + { + "epoch": 3.08, + "learning_rate": 0.00033020060180541627, + "loss": 2.8021, + "theoretical_loss": 3.6037124015703936, + "tokens_seen": 1142731776 + }, + { + "epoch": 3.08, + "learning_rate": 0.00033019057171514545, + "loss": 2.6327, + "theoretical_loss": 3.603693215854207, + "tokens_seen": 1142797312 + }, + { + "epoch": 3.08, + "learning_rate": 0.00033018054162487463, + "loss": 2.7521, + "theoretical_loss": 3.6036740315462805, + "tokens_seen": 1142862848 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003301705115346038, + "loss": 2.6045, + "theoretical_loss": 3.6036548486464293, + "tokens_seen": 1142928384 + }, + { + "epoch": 3.08, + "learning_rate": 0.00033016048144433305, + "loss": 2.6104, + "theoretical_loss": 3.60363566715447, + "tokens_seen": 1142993920 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003301504513540622, + "loss": 2.602, + "theoretical_loss": 3.603616487070218, + "tokens_seen": 1143059456 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003301404212637914, + "loss": 2.6504, + "theoretical_loss": 3.6035973083934896, + "tokens_seen": 1143124992 + }, + { + "epoch": 3.08, + "learning_rate": 0.00033013039117352054, + "loss": 2.6241, + "theoretical_loss": 3.6035781311241006, + "tokens_seen": 1143190528 + }, + { + "epoch": 3.08, + "learning_rate": 0.00033012036108324977, + "loss": 2.4047, + "theoretical_loss": 3.6035589552618674, + "tokens_seen": 1143256064 + }, + { + "epoch": 3.08, + "learning_rate": 0.00033011033099297895, + "loss": 2.6619, + "theoretical_loss": 3.6035397808066056, + "tokens_seen": 1143321600 + }, + { + "epoch": 3.08, + "learning_rate": 0.00033010030090270813, + "loss": 2.7529, + "theoretical_loss": 3.6035206077581314, + "tokens_seen": 1143387136 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003300902708124373, + "loss": 2.5172, + "theoretical_loss": 3.603501436116262, + "tokens_seen": 1143452672 + }, + { + "epoch": 3.08, + "learning_rate": 0.00033008024072216655, + "loss": 2.3102, + "theoretical_loss": 3.603482265880812, + "tokens_seen": 1143518208 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003300702106318957, + "loss": 2.8067, + "theoretical_loss": 3.6034630970515984, + "tokens_seen": 1143583744 + }, + { + "epoch": 3.08, + "objective/train/docs_used": 1266087, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.628338098526001, + "objective/train/theoretical_loss": 3.603458305063998, + "objective/train/tokens_used": 1164060128, + "theoretical_loss": 3.603458305063998, + "tokens_seen": 1143600128 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003300601805416249, + "loss": 2.8557, + "theoretical_loss": 3.6034439296284377, + "tokens_seen": 1143649280 + }, + { + "epoch": 3.08, + "learning_rate": 0.00033005015045135404, + "loss": 2.7171, + "theoretical_loss": 3.6034247636111463, + "tokens_seen": 1143714816 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003300401203610833, + "loss": 2.6378, + "theoretical_loss": 3.6034055989995393, + "tokens_seen": 1143780352 + }, + { + "epoch": 3.08, + "learning_rate": 0.00033003009027081245, + "loss": 2.5898, + "theoretical_loss": 3.6033864357934347, + "tokens_seen": 1143845888 + }, + { + "epoch": 3.08, + "learning_rate": 0.00033002006018054164, + "loss": 2.6563, + "theoretical_loss": 3.603367273992648, + "tokens_seen": 1143911424 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003300100300902708, + "loss": 2.9048, + "theoretical_loss": 3.603348113596996, + "tokens_seen": 1143976960 + }, + { + "epoch": 3.08, + "learning_rate": 0.00033, + "loss": 2.9012, + "theoretical_loss": 3.6033289546062948, + "tokens_seen": 1144042496 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003299899699097292, + "loss": 2.437, + "theoretical_loss": 3.603309797020361, + "tokens_seen": 1144108032 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003299799398194584, + "loss": 2.7824, + "theoretical_loss": 3.6032906408390115, + "tokens_seen": 1144173568 + }, + { + "epoch": 3.08, + "learning_rate": 0.00032996990972918754, + "loss": 2.5932, + "theoretical_loss": 3.603271486062062, + "tokens_seen": 1144239104 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003299598796389168, + "loss": 2.8508, + "theoretical_loss": 3.60325233268933, + "tokens_seen": 1144304640 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003299498495486459, + "loss": 2.912, + "theoretical_loss": 3.6032331807206326, + "tokens_seen": 1144370176 + }, + { + "epoch": 3.08, + "learning_rate": 0.00032993981945837514, + "loss": 2.7717, + "theoretical_loss": 3.603214030155785, + "tokens_seen": 1144435712 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003299297893681043, + "loss": 2.4534, + "theoretical_loss": 3.6031948809946046, + "tokens_seen": 1144501248 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003299197592778335, + "loss": 2.6931, + "theoretical_loss": 3.6031757332369088, + "tokens_seen": 1144566784 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003299097291875627, + "loss": 2.7032, + "theoretical_loss": 3.603156586882513, + "tokens_seen": 1144632320 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003298996990972919, + "loss": 2.4714, + "theoretical_loss": 3.6031374419312354, + "tokens_seen": 1144697856 + }, + { + "epoch": 3.08, + "learning_rate": 0.00032988966900702104, + "loss": 2.7777, + "theoretical_loss": 3.603118298382892, + "tokens_seen": 1144763392 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003298796389167503, + "loss": 2.6707, + "theoretical_loss": 3.6030991562373003, + "tokens_seen": 1144828928 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003298696088264794, + "loss": 2.808, + "theoretical_loss": 3.6030800154942764, + "tokens_seen": 1144894464 + }, + { + "epoch": 3.08, + "learning_rate": 0.00032985957873620864, + "loss": 2.8968, + "theoretical_loss": 3.603060876153638, + "tokens_seen": 1144960000 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003298495486459378, + "loss": 2.7766, + "theoretical_loss": 3.603041738215202, + "tokens_seen": 1145025536 + }, + { + "epoch": 3.08, + "learning_rate": 0.000329839518555667, + "loss": 2.5747, + "theoretical_loss": 3.6030226016787847, + "tokens_seen": 1145091072 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003298294884653962, + "loss": 2.5625, + "theoretical_loss": 3.603003466544204, + "tokens_seen": 1145156608 + }, + { + "epoch": 3.08, + "learning_rate": 0.00032981945837512537, + "loss": 2.6457, + "theoretical_loss": 3.6029843328112765, + "tokens_seen": 1145222144 + }, + { + "epoch": 3.08, + "objective/train/docs_used": 1266852, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9177322387695312, + "objective/train/theoretical_loss": 3.602979549597032, + "objective/train/tokens_used": 1165698528, + "theoretical_loss": 3.602979549597032, + "tokens_seen": 1145238528 + }, + { + "epoch": 3.08, + "learning_rate": 0.00032980942828485455, + "loss": 2.7197, + "theoretical_loss": 3.6029652004798196, + "tokens_seen": 1145287680 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003297993981945838, + "loss": 2.5158, + "theoretical_loss": 3.6029460695496507, + "tokens_seen": 1145353216 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003297893681043129, + "loss": 2.5897, + "theoretical_loss": 3.6029269400205868, + "tokens_seen": 1145418752 + }, + { + "epoch": 3.08, + "learning_rate": 0.00032977933801404215, + "loss": 2.7719, + "theoretical_loss": 3.602907811892444, + "tokens_seen": 1145484288 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003297693079237713, + "loss": 2.9219, + "theoretical_loss": 3.6028886851650412, + "tokens_seen": 1145549824 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003297592778335005, + "loss": 2.6883, + "theoretical_loss": 3.6028695598381955, + "tokens_seen": 1145615360 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003297492477432297, + "loss": 2.7103, + "theoretical_loss": 3.6028504359117233, + "tokens_seen": 1145680896 + }, + { + "epoch": 3.08, + "learning_rate": 0.00032973921765295887, + "loss": 2.7711, + "theoretical_loss": 3.6028313133854426, + "tokens_seen": 1145746432 + }, + { + "epoch": 3.08, + "learning_rate": 0.00032972918756268805, + "loss": 2.6575, + "theoretical_loss": 3.6028121922591705, + "tokens_seen": 1145811968 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003297191574724173, + "loss": 2.783, + "theoretical_loss": 3.602793072532725, + "tokens_seen": 1145877504 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003297091273821464, + "loss": 2.6168, + "theoretical_loss": 3.6027739542059223, + "tokens_seen": 1145943040 + }, + { + "epoch": 3.08, + "learning_rate": 0.00032969909729187565, + "loss": 2.8734, + "theoretical_loss": 3.6027548372785816, + "tokens_seen": 1146008576 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003296890672016048, + "loss": 2.8027, + "theoretical_loss": 3.60273572175052, + "tokens_seen": 1146074112 + }, + { + "epoch": 3.08, + "learning_rate": 0.000329679037111334, + "loss": 2.6679, + "theoretical_loss": 3.6027166076215535, + "tokens_seen": 1146139648 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003296690070210632, + "loss": 2.7966, + "theoretical_loss": 3.6026974948915016, + "tokens_seen": 1146205184 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003296589769307924, + "loss": 2.6869, + "theoretical_loss": 3.602678383560181, + "tokens_seen": 1146270720 + }, + { + "epoch": 3.08, + "learning_rate": 0.00032964894684052155, + "loss": 2.7343, + "theoretical_loss": 3.6026592736274097, + "tokens_seen": 1146336256 + }, + { + "epoch": 3.08, + "learning_rate": 0.00032963891675025074, + "loss": 2.6343, + "theoretical_loss": 3.602640165093005, + "tokens_seen": 1146401792 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003296288866599799, + "loss": 2.9379, + "theoretical_loss": 3.6026210579567852, + "tokens_seen": 1146467328 + }, + { + "epoch": 3.08, + "learning_rate": 0.00032961885656970915, + "loss": 2.5409, + "theoretical_loss": 3.6026019522185684, + "tokens_seen": 1146532864 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003296088264794383, + "loss": 2.711, + "theoretical_loss": 3.602582847878171, + "tokens_seen": 1146598400 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003295987963891675, + "loss": 2.8479, + "theoretical_loss": 3.6025637449354124, + "tokens_seen": 1146663936 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003295887662988967, + "loss": 2.7248, + "theoretical_loss": 3.6025446433901096, + "tokens_seen": 1146729472 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003295787362086259, + "loss": 2.7041, + "theoretical_loss": 3.6025255432420806, + "tokens_seen": 1146795008 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003295687061183551, + "loss": 2.8072, + "theoretical_loss": 3.6025064444911434, + "tokens_seen": 1146860544 + }, + { + "debugging/Self-BLEU-5": 0.4473570585925922, + "debugging/distinct-1-grams": 0.7971195468618069, + "debugging/distinct-2-grams": 0.9608189858650149, + "debugging/entropy-1-grams": 5.677884863520164, + "debugging/entropy-2-grams": 6.546455298638856, + "debugging/length": 517.5, + "debugging/num_segments": 10, + "debugging/score": 0.004519826938251854, + "debugging/score_std": 0.004873429920306367, + "epoch": 3.08, + "objective/train/docs_used": 1267667, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.091930866241455, + "objective/train/theoretical_loss": 3.602501670021683, + "objective/train/tokens_used": 1167336928, + "theoretical_loss": 3.602501670021683, + "tokens_seen": 1146876928 + }, + { + "epoch": 3.08, + "learning_rate": 0.00032955867602808424, + "loss": 2.711, + "theoretical_loss": 3.6024873471371155, + "tokens_seen": 1146926080 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003295486459378135, + "loss": 2.6344, + "theoretical_loss": 3.602468251179816, + "tokens_seen": 1146991616 + }, + { + "epoch": 3.08, + "learning_rate": 0.00032953861584754265, + "loss": 2.6278, + "theoretical_loss": 3.6024491566190626, + "tokens_seen": 1147057152 + }, + { + "epoch": 3.08, + "learning_rate": 0.00032952858575727184, + "loss": 2.5794, + "theoretical_loss": 3.6024300634546726, + "tokens_seen": 1147122688 + }, + { + "epoch": 3.08, + "learning_rate": 0.000329518555667001, + "loss": 2.5395, + "theoretical_loss": 3.6024109716864654, + "tokens_seen": 1147188224 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003295085255767302, + "loss": 2.5811, + "theoretical_loss": 3.602391881314258, + "tokens_seen": 1147253760 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003294984954864594, + "loss": 2.8294, + "theoretical_loss": 3.602372792337869, + "tokens_seen": 1147319296 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003294884653961886, + "loss": 2.4544, + "theoretical_loss": 3.6023537047571166, + "tokens_seen": 1147384832 + }, + { + "epoch": 3.08, + "learning_rate": 0.00032947843530591774, + "loss": 2.4459, + "theoretical_loss": 3.6023346185718195, + "tokens_seen": 1147450368 + }, + { + "epoch": 3.08, + "learning_rate": 0.000329468405215647, + "loss": 2.7234, + "theoretical_loss": 3.6023155337817956, + "tokens_seen": 1147515904 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003294583751253761, + "loss": 2.7293, + "theoretical_loss": 3.602296450386863, + "tokens_seen": 1147581440 + }, + { + "epoch": 3.08, + "learning_rate": 0.00032944834503510534, + "loss": 2.6907, + "theoretical_loss": 3.60227736838684, + "tokens_seen": 1147646976 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003294383149448345, + "loss": 2.7077, + "theoretical_loss": 3.602258287781545, + "tokens_seen": 1147712512 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003294282848545637, + "loss": 2.979, + "theoretical_loss": 3.6022392085707975, + "tokens_seen": 1147778048 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003294182547642929, + "loss": 2.687, + "theoretical_loss": 3.602220130754415, + "tokens_seen": 1147843584 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003294082246740221, + "loss": 2.7995, + "theoretical_loss": 3.602201054332216, + "tokens_seen": 1147909120 + }, + { + "epoch": 3.08, + "learning_rate": 0.00032939819458375124, + "loss": 2.847, + "theoretical_loss": 3.602181979304019, + "tokens_seen": 1147974656 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003293881644934805, + "loss": 2.6341, + "theoretical_loss": 3.602162905669643, + "tokens_seen": 1148040192 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003293781344032096, + "loss": 2.773, + "theoretical_loss": 3.602143833428906, + "tokens_seen": 1148105728 + }, + { + "epoch": 3.08, + "learning_rate": 0.00032936810431293884, + "loss": 2.7618, + "theoretical_loss": 3.6021247625816275, + "tokens_seen": 1148171264 + }, + { + "epoch": 3.08, + "learning_rate": 0.000329358074222668, + "loss": 2.6518, + "theoretical_loss": 3.6021056931276254, + "tokens_seen": 1148236800 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003293480441323972, + "loss": 2.7529, + "theoretical_loss": 3.6020866250667183, + "tokens_seen": 1148302336 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003293380140421264, + "loss": 2.71, + "theoretical_loss": 3.602067558398726, + "tokens_seen": 1148367872 + }, + { + "epoch": 3.08, + "learning_rate": 0.00032932798395185557, + "loss": 2.8273, + "theoretical_loss": 3.6020484931234655, + "tokens_seen": 1148433408 + }, + { + "epoch": 3.08, + "learning_rate": 0.00032931795386158475, + "loss": 2.8462, + "theoretical_loss": 3.6020294292407575, + "tokens_seen": 1148498944 + }, + { + "epoch": 3.08, + "objective/train/docs_used": 1268910, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8044888973236084, + "objective/train/theoretical_loss": 3.6020246634876454, + "objective/train/tokens_used": 1168975328, + "theoretical_loss": 3.6020246634876454, + "tokens_seen": 1148515328 + }, + { + "epoch": 3.08, + "learning_rate": 0.000329307923771314, + "loss": 2.5725, + "theoretical_loss": 3.6020103667504193, + "tokens_seen": 1148564480 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003292978936810431, + "loss": 2.7818, + "theoretical_loss": 3.601991305652271, + "tokens_seen": 1148630016 + }, + { + "epoch": 3.08, + "learning_rate": 0.00032928786359077235, + "loss": 2.7852, + "theoretical_loss": 3.601972245946131, + "tokens_seen": 1148695552 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003292778335005015, + "loss": 2.8112, + "theoretical_loss": 3.6019531876318176, + "tokens_seen": 1148761088 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003292678034102307, + "loss": 2.7196, + "theoretical_loss": 3.60193413070915, + "tokens_seen": 1148826624 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003292577733199599, + "loss": 2.822, + "theoretical_loss": 3.6019150751779483, + "tokens_seen": 1148892160 + }, + { + "epoch": 3.08, + "learning_rate": 0.00032924774322968907, + "loss": 2.8823, + "theoretical_loss": 3.601896021038031, + "tokens_seen": 1148957696 + }, + { + "epoch": 3.08, + "learning_rate": 0.00032923771313941825, + "loss": 2.6873, + "theoretical_loss": 3.6018769682892158, + "tokens_seen": 1149023232 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003292276830491475, + "loss": 2.7185, + "theoretical_loss": 3.6018579169313236, + "tokens_seen": 1149088768 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003292176529588766, + "loss": 2.6924, + "theoretical_loss": 3.601838866964173, + "tokens_seen": 1149154304 + }, + { + "epoch": 3.08, + "learning_rate": 0.00032920762286860585, + "loss": 2.6421, + "theoretical_loss": 3.6018198183875825, + "tokens_seen": 1149219840 + }, + { + "epoch": 3.08, + "learning_rate": 0.000329197592778335, + "loss": 2.6667, + "theoretical_loss": 3.6018007712013724, + "tokens_seen": 1149285376 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003291875626880642, + "loss": 2.5585, + "theoretical_loss": 3.6017817254053615, + "tokens_seen": 1149350912 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003291775325977934, + "loss": 2.761, + "theoretical_loss": 3.6017626809993684, + "tokens_seen": 1149416448 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003291675025075226, + "loss": 2.5061, + "theoretical_loss": 3.6017436379832133, + "tokens_seen": 1149481984 + }, + { + "epoch": 3.08, + "learning_rate": 0.00032915747241725175, + "loss": 2.7733, + "theoretical_loss": 3.601724596356715, + "tokens_seen": 1149547520 + }, + { + "epoch": 3.08, + "learning_rate": 0.00032914744232698094, + "loss": 2.4415, + "theoretical_loss": 3.6017055561196933, + "tokens_seen": 1149613056 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003291374122367101, + "loss": 2.5899, + "theoretical_loss": 3.6016865172719674, + "tokens_seen": 1149678592 + }, + { + "epoch": 3.08, + "learning_rate": 0.00032912738214643935, + "loss": 2.5564, + "theoretical_loss": 3.601667479813356, + "tokens_seen": 1149744128 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003291173520561685, + "loss": 2.7516, + "theoretical_loss": 3.60164844374368, + "tokens_seen": 1149809664 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003291073219658977, + "loss": 2.7717, + "theoretical_loss": 3.6016294090627583, + "tokens_seen": 1149875200 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003290972918756269, + "loss": 2.81, + "theoretical_loss": 3.6016103757704103, + "tokens_seen": 1149940736 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003290872617853561, + "loss": 2.7642, + "theoretical_loss": 3.6015913438664553, + "tokens_seen": 1150006272 + }, + { + "epoch": 3.08, + "learning_rate": 0.00032907723169508526, + "loss": 2.5971, + "theoretical_loss": 3.6015723133507134, + "tokens_seen": 1150071808 + }, + { + "epoch": 3.08, + "learning_rate": 0.00032906720160481444, + "loss": 2.6199, + "theoretical_loss": 3.6015532842230042, + "tokens_seen": 1150137344 + }, + { + "epoch": 3.08, + "objective/train/docs_used": 1269509, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.0626049041748047, + "objective/train/theoretical_loss": 3.601548527157936, + "objective/train/tokens_used": 1170613728, + "theoretical_loss": 3.601548527157936, + "tokens_seen": 1150153728 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003290571715145436, + "loss": 2.6663, + "theoretical_loss": 3.6015342564831476, + "tokens_seen": 1150202880 + }, + { + "epoch": 3.08, + "learning_rate": 0.00032904714142427286, + "loss": 2.5544, + "theoretical_loss": 3.6015152301309623, + "tokens_seen": 1150268416 + }, + { + "epoch": 3.08, + "learning_rate": 0.000329037111334002, + "loss": 2.7881, + "theoretical_loss": 3.601496205166269, + "tokens_seen": 1150333952 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003290270812437312, + "loss": 2.8447, + "theoretical_loss": 3.601477181588887, + "tokens_seen": 1150399488 + }, + { + "epoch": 3.08, + "learning_rate": 0.00032901705115346034, + "loss": 2.8106, + "theoretical_loss": 3.6014581593986366, + "tokens_seen": 1150465024 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003290070210631896, + "loss": 2.6651, + "theoretical_loss": 3.6014391385953375, + "tokens_seen": 1150530560 + }, + { + "epoch": 3.08, + "learning_rate": 0.00032899699097291876, + "loss": 2.6976, + "theoretical_loss": 3.601420119178809, + "tokens_seen": 1150596096 + }, + { + "epoch": 3.08, + "learning_rate": 0.00032898696088264794, + "loss": 2.6538, + "theoretical_loss": 3.601401101148872, + "tokens_seen": 1150661632 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003289769307923771, + "loss": 2.5883, + "theoretical_loss": 3.6013820845053455, + "tokens_seen": 1150727168 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003289669007021063, + "loss": 2.7749, + "theoretical_loss": 3.60136306924805, + "tokens_seen": 1150792704 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003289568706118355, + "loss": 2.7311, + "theoretical_loss": 3.6013440553768055, + "tokens_seen": 1150858240 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003289468405215647, + "loss": 2.7103, + "theoretical_loss": 3.601325042891432, + "tokens_seen": 1150923776 + }, + { + "epoch": 3.08, + "learning_rate": 0.00032893681043129385, + "loss": 2.8096, + "theoretical_loss": 3.601306031791749, + "tokens_seen": 1150989312 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003289267803410231, + "loss": 2.6958, + "theoretical_loss": 3.6012870220775772, + "tokens_seen": 1151054848 + }, + { + "epoch": 3.08, + "learning_rate": 0.00032891675025075226, + "loss": 2.6214, + "theoretical_loss": 3.6012680137487374, + "tokens_seen": 1151120384 + }, + { + "epoch": 3.08, + "learning_rate": 0.00032890672016048145, + "loss": 2.5678, + "theoretical_loss": 3.6012490068050482, + "tokens_seen": 1151185920 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003288966900702106, + "loss": 2.8512, + "theoretical_loss": 3.6012300012463316, + "tokens_seen": 1151251456 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003288866599799398, + "loss": 2.8587, + "theoretical_loss": 3.601210997072406, + "tokens_seen": 1151316992 + }, + { + "epoch": 3.08, + "learning_rate": 0.000328876629889669, + "loss": 2.6661, + "theoretical_loss": 3.6011919942830932, + "tokens_seen": 1151382528 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003288665997993982, + "loss": 2.7665, + "theoretical_loss": 3.601172992878213, + "tokens_seen": 1151448064 + }, + { + "epoch": 3.08, + "learning_rate": 0.00032885656970912735, + "loss": 2.7371, + "theoretical_loss": 3.6011539928575855, + "tokens_seen": 1151513600 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003288465396188566, + "loss": 2.5777, + "theoretical_loss": 3.601134994221032, + "tokens_seen": 1151579136 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003288365095285857, + "loss": 2.7746, + "theoretical_loss": 3.601115996968371, + "tokens_seen": 1151644672 + }, + { + "epoch": 3.08, + "learning_rate": 0.00032882647943831495, + "loss": 2.6474, + "theoretical_loss": 3.6010970010994248, + "tokens_seen": 1151710208 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003288164493480442, + "loss": 2.5405, + "theoretical_loss": 3.6010780066140127, + "tokens_seen": 1151775744 + }, + { + "epoch": 3.08, + "objective/train/docs_used": 1270458, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7559614181518555, + "objective/train/theoretical_loss": 3.601073258208816, + "objective/train/tokens_used": 1172252128, + "theoretical_loss": 3.601073258208816, + "tokens_seen": 1151792128 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003288064192577733, + "loss": 2.6892, + "theoretical_loss": 3.6010590135119562, + "tokens_seen": 1151841280 + }, + { + "epoch": 3.08, + "learning_rate": 0.00032879638916750255, + "loss": 2.5914, + "theoretical_loss": 3.601040021793075, + "tokens_seen": 1151906816 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003287863590772317, + "loss": 2.6654, + "theoretical_loss": 3.601021031457191, + "tokens_seen": 1151972352 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003287763289869609, + "loss": 2.6697, + "theoretical_loss": 3.601002042504123, + "tokens_seen": 1152037888 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003287662988966901, + "loss": 2.6694, + "theoretical_loss": 3.600983054933693, + "tokens_seen": 1152103424 + }, + { + "epoch": 3.08, + "learning_rate": 0.00032875626880641927, + "loss": 2.8141, + "theoretical_loss": 3.600964068745721, + "tokens_seen": 1152168960 + }, + { + "epoch": 3.08, + "learning_rate": 0.00032874623871614845, + "loss": 2.6888, + "theoretical_loss": 3.600945083940028, + "tokens_seen": 1152234496 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003287362086258777, + "loss": 2.8488, + "theoretical_loss": 3.6009261005164346, + "tokens_seen": 1152300032 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003287261785356068, + "loss": 2.5548, + "theoretical_loss": 3.600907118474762, + "tokens_seen": 1152365568 + }, + { + "epoch": 3.08, + "learning_rate": 0.00032871614844533605, + "loss": 2.746, + "theoretical_loss": 3.600888137814831, + "tokens_seen": 1152431104 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003287061183550652, + "loss": 2.5498, + "theoretical_loss": 3.6008691585364616, + "tokens_seen": 1152496640 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003286960882647944, + "loss": 2.5718, + "theoretical_loss": 3.6008501806394753, + "tokens_seen": 1152562176 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003286860581745236, + "loss": 2.8185, + "theoretical_loss": 3.600831204123693, + "tokens_seen": 1152627712 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003286760280842528, + "loss": 2.7984, + "theoretical_loss": 3.6008122289889357, + "tokens_seen": 1152693248 + }, + { + "epoch": 3.08, + "learning_rate": 0.00032866599799398195, + "loss": 2.6887, + "theoretical_loss": 3.600793255235024, + "tokens_seen": 1152758784 + }, + { + "epoch": 3.08, + "learning_rate": 0.00032865596790371114, + "loss": 2.6701, + "theoretical_loss": 3.6007742828617797, + "tokens_seen": 1152824320 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003286459378134403, + "loss": 2.5288, + "theoretical_loss": 3.600755311869023, + "tokens_seen": 1152889856 + }, + { + "epoch": 3.08, + "learning_rate": 0.00032863590772316955, + "loss": 2.7081, + "theoretical_loss": 3.6007363422565755, + "tokens_seen": 1152955392 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003286258776328987, + "loss": 2.7438, + "theoretical_loss": 3.600717374024258, + "tokens_seen": 1153020928 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003286158475426279, + "loss": 2.6553, + "theoretical_loss": 3.600698407171892, + "tokens_seen": 1153086464 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003286058174523571, + "loss": 2.8296, + "theoretical_loss": 3.600679441699299, + "tokens_seen": 1153152000 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003285957873620863, + "loss": 2.7507, + "theoretical_loss": 3.6006604776062994, + "tokens_seen": 1153217536 + }, + { + "epoch": 3.08, + "learning_rate": 0.00032858575727181546, + "loss": 2.7123, + "theoretical_loss": 3.6006415148927147, + "tokens_seen": 1153283072 + }, + { + "epoch": 3.08, + "learning_rate": 0.00032857572718154464, + "loss": 2.5924, + "theoretical_loss": 3.6006225535583667, + "tokens_seen": 1153348608 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003285656970912738, + "loss": 2.8264, + "theoretical_loss": 3.6006035936030756, + "tokens_seen": 1153414144 + }, + { + "epoch": 3.08, + "objective/train/docs_used": 1271091, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.858920097351074, + "objective/train/theoretical_loss": 3.60059885382971, + "objective/train/tokens_used": 1173890528, + "theoretical_loss": 3.60059885382971, + "tokens_seen": 1153430528 + }, + { + "epoch": 3.08, + "learning_rate": 0.00032855566700100306, + "loss": 3.0625, + "theoretical_loss": 3.6005846350266637, + "tokens_seen": 1153479680 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003285456369107322, + "loss": 2.8801, + "theoretical_loss": 3.6005656778289525, + "tokens_seen": 1153545216 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003285356068204614, + "loss": 2.6872, + "theoretical_loss": 3.6005467220097627, + "tokens_seen": 1153610752 + }, + { + "epoch": 3.08, + "learning_rate": 0.00032852557673019054, + "loss": 2.8549, + "theoretical_loss": 3.6005277675689165, + "tokens_seen": 1153676288 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003285155466399198, + "loss": 2.833, + "theoretical_loss": 3.6005088145062345, + "tokens_seen": 1153741824 + }, + { + "epoch": 3.08, + "learning_rate": 0.00032850551654964896, + "loss": 2.7275, + "theoretical_loss": 3.600489862821539, + "tokens_seen": 1153807360 + }, + { + "epoch": 3.08, + "learning_rate": 0.00032849548645937814, + "loss": 2.7133, + "theoretical_loss": 3.600470912514651, + "tokens_seen": 1153872896 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003284854563691073, + "loss": 2.6046, + "theoretical_loss": 3.6004519635853924, + "tokens_seen": 1153938432 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003284754262788365, + "loss": 2.4431, + "theoretical_loss": 3.600433016033585, + "tokens_seen": 1154003968 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003284653961885657, + "loss": 2.5941, + "theoretical_loss": 3.6004140698590503, + "tokens_seen": 1154069504 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003284553660982949, + "loss": 2.6592, + "theoretical_loss": 3.6003951250616097, + "tokens_seen": 1154135040 + }, + { + "epoch": 3.08, + "learning_rate": 0.00032844533600802405, + "loss": 2.8864, + "theoretical_loss": 3.600376181641085, + "tokens_seen": 1154200576 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003284353059177533, + "loss": 2.7652, + "theoretical_loss": 3.6003572395972987, + "tokens_seen": 1154266112 + }, + { + "epoch": 3.08, + "learning_rate": 0.00032842527582748246, + "loss": 2.7161, + "theoretical_loss": 3.6003382989300716, + "tokens_seen": 1154331648 + }, + { + "epoch": 3.08, + "learning_rate": 0.00032841524573721165, + "loss": 2.7713, + "theoretical_loss": 3.600319359639226, + "tokens_seen": 1154397184 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003284052156469408, + "loss": 2.6318, + "theoretical_loss": 3.600300421724583, + "tokens_seen": 1154462720 + }, + { + "epoch": 3.08, + "learning_rate": 0.00032839518555667, + "loss": 2.5957, + "theoretical_loss": 3.600281485185966, + "tokens_seen": 1154528256 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003283851554663992, + "loss": 2.536, + "theoretical_loss": 3.6002625500231957, + "tokens_seen": 1154593792 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003283751253761284, + "loss": 2.5904, + "theoretical_loss": 3.6002436162360945, + "tokens_seen": 1154659328 + }, + { + "epoch": 3.08, + "learning_rate": 0.00032836509528585755, + "loss": 2.5183, + "theoretical_loss": 3.600224683824484, + "tokens_seen": 1154724864 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003283550651955868, + "loss": 2.7599, + "theoretical_loss": 3.6002057527881868, + "tokens_seen": 1154790400 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003283450351053159, + "loss": 2.7745, + "theoretical_loss": 3.600186823127024, + "tokens_seen": 1154855936 + }, + { + "epoch": 3.08, + "learning_rate": 0.00032833500501504515, + "loss": 2.6968, + "theoretical_loss": 3.600167894840819, + "tokens_seen": 1154921472 + }, + { + "epoch": 3.08, + "learning_rate": 0.00032832497492477433, + "loss": 2.7957, + "theoretical_loss": 3.6001489679293934, + "tokens_seen": 1154987008 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003283149448345035, + "loss": 2.8614, + "theoretical_loss": 3.6001300423925686, + "tokens_seen": 1155052544 + }, + { + "epoch": 3.08, + "objective/train/docs_used": 1272129, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.853865623474121, + "objective/train/theoretical_loss": 3.600125311223123, + "objective/train/tokens_used": 1175528928, + "theoretical_loss": 3.600125311223123, + "tokens_seen": 1155068928 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003283049147442327, + "loss": 2.8174, + "theoretical_loss": 3.600111118230168, + "tokens_seen": 1155118080 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003282948846539619, + "loss": 2.8958, + "theoretical_loss": 3.600092195442013, + "tokens_seen": 1155183616 + }, + { + "epoch": 3.08, + "learning_rate": 0.00032828485456369105, + "loss": 2.7905, + "theoretical_loss": 3.6000732740279258, + "tokens_seen": 1155249152 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003282748244734203, + "loss": 2.7081, + "theoretical_loss": 3.600054353987729, + "tokens_seen": 1155314688 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003282647943831494, + "loss": 2.413, + "theoretical_loss": 3.600035435321245, + "tokens_seen": 1155380224 + }, + { + "epoch": 3.08, + "learning_rate": 0.00032825476429287865, + "loss": 2.695, + "theoretical_loss": 3.6000165180282955, + "tokens_seen": 1155445760 + }, + { + "epoch": 3.08, + "learning_rate": 0.00032824473420260783, + "loss": 2.5683, + "theoretical_loss": 3.5999976021087035, + "tokens_seen": 1155511296 + }, + { + "epoch": 3.08, + "learning_rate": 0.000328234704112337, + "loss": 2.5468, + "theoretical_loss": 3.5999786875622917, + "tokens_seen": 1155576832 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003282246740220662, + "loss": 2.413, + "theoretical_loss": 3.599959774388882, + "tokens_seen": 1155642368 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003282146439317954, + "loss": 2.7004, + "theoretical_loss": 3.599940862588297, + "tokens_seen": 1155707904 + }, + { + "epoch": 3.08, + "learning_rate": 0.00032820461384152456, + "loss": 2.6909, + "theoretical_loss": 3.5999219521603587, + "tokens_seen": 1155773440 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003281945837512538, + "loss": 2.6059, + "theoretical_loss": 3.5999030431048906, + "tokens_seen": 1155838976 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003281845536609829, + "loss": 2.6892, + "theoretical_loss": 3.599884135421715, + "tokens_seen": 1155904512 + }, + { + "epoch": 3.08, + "learning_rate": 0.00032817452357071215, + "loss": 2.5906, + "theoretical_loss": 3.5998652291106543, + "tokens_seen": 1155970048 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003281644934804413, + "loss": 2.7893, + "theoretical_loss": 3.599846324171531, + "tokens_seen": 1156035584 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003281544633901705, + "loss": 2.6951, + "theoretical_loss": 3.5998274206041683, + "tokens_seen": 1156101120 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003281444332998997, + "loss": 2.6377, + "theoretical_loss": 3.5998085184083886, + "tokens_seen": 1156166656 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003281344032096289, + "loss": 2.5826, + "theoretical_loss": 3.5997896175840145, + "tokens_seen": 1156232192 + }, + { + "epoch": 3.08, + "learning_rate": 0.00032812437311935806, + "loss": 2.7148, + "theoretical_loss": 3.5997707181308685, + "tokens_seen": 1156297728 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003281143430290873, + "loss": 2.7672, + "theoretical_loss": 3.599751820048774, + "tokens_seen": 1156363264 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003281043129388164, + "loss": 2.8374, + "theoretical_loss": 3.5997329233375543, + "tokens_seen": 1156428800 + }, + { + "epoch": 3.08, + "learning_rate": 0.00032809428284854566, + "loss": 2.6762, + "theoretical_loss": 3.599714027997031, + "tokens_seen": 1156494336 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003280842527582748, + "loss": 2.3282, + "theoretical_loss": 3.599695134027028, + "tokens_seen": 1156559872 + }, + { + "epoch": 3.08, + "learning_rate": 0.000328074222668004, + "loss": 2.7864, + "theoretical_loss": 3.5996762414273675, + "tokens_seen": 1156625408 + }, + { + "epoch": 3.08, + "learning_rate": 0.00032806419257773326, + "loss": 2.8162, + "theoretical_loss": 3.599657350197873, + "tokens_seen": 1156690944 + }, + { + "epoch": 3.08, + "objective/train/docs_used": 1272864, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.16133189201355, + "objective/train/theoretical_loss": 3.5996526276045673, + "objective/train/tokens_used": 1177167328, + "theoretical_loss": 3.5996526276045673, + "tokens_seen": 1156707328 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003280541624874624, + "loss": 2.8585, + "theoretical_loss": 3.599638460338368, + "tokens_seen": 1156756480 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003280441323971916, + "loss": 2.5503, + "theoretical_loss": 3.599619571848674, + "tokens_seen": 1156822016 + }, + { + "epoch": 3.08, + "learning_rate": 0.00032803410230692074, + "loss": 2.7527, + "theoretical_loss": 3.5996006847286157, + "tokens_seen": 1156887552 + }, + { + "epoch": 3.08, + "learning_rate": 0.00032802407221665, + "loss": 2.6692, + "theoretical_loss": 3.599581798978015, + "tokens_seen": 1156953088 + }, + { + "epoch": 3.08, + "learning_rate": 0.00032801404212637916, + "loss": 2.475, + "theoretical_loss": 3.599562914596696, + "tokens_seen": 1157018624 + }, + { + "epoch": 3.08, + "learning_rate": 0.00032800401203610834, + "loss": 2.62, + "theoretical_loss": 3.5995440315844807, + "tokens_seen": 1157084160 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003279939819458375, + "loss": 2.8344, + "theoretical_loss": 3.5995251499411935, + "tokens_seen": 1157149696 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003279839518555667, + "loss": 2.6817, + "theoretical_loss": 3.5995062696666573, + "tokens_seen": 1157215232 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003279739217652959, + "loss": 2.7615, + "theoretical_loss": 3.599487390760695, + "tokens_seen": 1157280768 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003279638916750251, + "loss": 2.567, + "theoretical_loss": 3.5994685132231297, + "tokens_seen": 1157346304 + }, + { + "epoch": 3.08, + "learning_rate": 0.00032795386158475425, + "loss": 2.4982, + "theoretical_loss": 3.5994496370537856, + "tokens_seen": 1157411840 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003279438314944835, + "loss": 2.7333, + "theoretical_loss": 3.599430762252486, + "tokens_seen": 1157477376 + }, + { + "epoch": 3.08, + "learning_rate": 0.00032793380140421266, + "loss": 2.8056, + "theoretical_loss": 3.599411888819054, + "tokens_seen": 1157542912 + }, + { + "epoch": 3.08, + "learning_rate": 0.00032792377131394185, + "loss": 2.5637, + "theoretical_loss": 3.599393016753312, + "tokens_seen": 1157608448 + }, + { + "epoch": 3.08, + "learning_rate": 0.000327913741223671, + "loss": 2.7559, + "theoretical_loss": 3.599374146055085, + "tokens_seen": 1157673984 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003279037111334002, + "loss": 2.7122, + "theoretical_loss": 3.599355276724196, + "tokens_seen": 1157739520 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003278936810431294, + "loss": 2.7572, + "theoretical_loss": 3.5993364087604682, + "tokens_seen": 1157805056 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003278836509528586, + "loss": 2.5657, + "theoretical_loss": 3.5993175421637256, + "tokens_seen": 1157870592 + }, + { + "epoch": 3.08, + "learning_rate": 0.00032787362086258775, + "loss": 2.7728, + "theoretical_loss": 3.599298676933792, + "tokens_seen": 1157936128 + }, + { + "epoch": 3.08, + "learning_rate": 0.000327863590772317, + "loss": 2.2984, + "theoretical_loss": 3.59927981307049, + "tokens_seen": 1158001664 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003278535606820461, + "loss": 2.5368, + "theoretical_loss": 3.599260950573644, + "tokens_seen": 1158067200 + }, + { + "epoch": 3.08, + "learning_rate": 0.00032784353059177535, + "loss": 2.6498, + "theoretical_loss": 3.599242089443078, + "tokens_seen": 1158132736 + }, + { + "epoch": 3.08, + "learning_rate": 0.00032783350050150453, + "loss": 2.5435, + "theoretical_loss": 3.599223229678615, + "tokens_seen": 1158198272 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003278234704112337, + "loss": 2.6249, + "theoretical_loss": 3.5992043712800794, + "tokens_seen": 1158263808 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003278134403209629, + "loss": 2.8027, + "theoretical_loss": 3.599185514247295, + "tokens_seen": 1158329344 + }, + { + "epoch": 3.08, + "objective/train/docs_used": 1274329, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.622804641723633, + "objective/train/theoretical_loss": 3.5991808002024763, + "objective/train/tokens_used": 1178805728, + "theoretical_loss": 3.5991808002024763, + "tokens_seen": 1158345728 + }, + { + "epoch": 3.08, + "learning_rate": 0.00032780341023069213, + "loss": 2.5089, + "theoretical_loss": 3.5991666585800846, + "tokens_seen": 1158394880 + }, + { + "epoch": 3.08, + "learning_rate": 0.00032779338014042125, + "loss": 2.9317, + "theoretical_loss": 3.599147804278273, + "tokens_seen": 1158460416 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003277833500501505, + "loss": 2.7415, + "theoretical_loss": 3.599128951341684, + "tokens_seen": 1158525952 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003277733199598796, + "loss": 2.6007, + "theoretical_loss": 3.5991100997701415, + "tokens_seen": 1158591488 + }, + { + "epoch": 3.08, + "learning_rate": 0.00032776328986960885, + "loss": 2.5627, + "theoretical_loss": 3.599091249563469, + "tokens_seen": 1158657024 + }, + { + "epoch": 3.08, + "learning_rate": 0.00032775325977933803, + "loss": 2.5528, + "theoretical_loss": 3.5990724007214916, + "tokens_seen": 1158722560 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003277432296890672, + "loss": 2.4406, + "theoretical_loss": 3.5990535532440324, + "tokens_seen": 1158788096 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003277331995987964, + "loss": 2.4564, + "theoretical_loss": 3.5990347071309152, + "tokens_seen": 1158853632 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003277231695085256, + "loss": 2.8616, + "theoretical_loss": 3.5990158623819646, + "tokens_seen": 1158919168 + }, + { + "epoch": 3.08, + "learning_rate": 0.00032771313941825476, + "loss": 2.6851, + "theoretical_loss": 3.5989970189970046, + "tokens_seen": 1158984704 + }, + { + "epoch": 3.08, + "learning_rate": 0.000327703109327984, + "loss": 2.8071, + "theoretical_loss": 3.5989781769758595, + "tokens_seen": 1159050240 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003276930792377131, + "loss": 2.7617, + "theoretical_loss": 3.5989593363183534, + "tokens_seen": 1159115776 + }, + { + "epoch": 3.08, + "learning_rate": 0.00032768304914744235, + "loss": 2.6882, + "theoretical_loss": 3.598940497024311, + "tokens_seen": 1159181312 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003276730190571715, + "loss": 2.7579, + "theoretical_loss": 3.5989216590935555, + "tokens_seen": 1159246848 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003276629889669007, + "loss": 2.548, + "theoretical_loss": 3.5989028225259116, + "tokens_seen": 1159312384 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003276529588766299, + "loss": 2.6048, + "theoretical_loss": 3.598883987321204, + "tokens_seen": 1159377920 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003276429287863591, + "loss": 2.7181, + "theoretical_loss": 3.598865153479257, + "tokens_seen": 1159443456 + }, + { + "epoch": 3.08, + "learning_rate": 0.00032763289869608826, + "loss": 2.7681, + "theoretical_loss": 3.5988463209998947, + "tokens_seen": 1159508992 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003276228686058175, + "loss": 2.7062, + "theoretical_loss": 3.5988274898829413, + "tokens_seen": 1159574528 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003276128385155466, + "loss": 2.6341, + "theoretical_loss": 3.598808660128222, + "tokens_seen": 1159640064 + }, + { + "epoch": 3.08, + "learning_rate": 0.00032760280842527586, + "loss": 2.8696, + "theoretical_loss": 3.5987898317355604, + "tokens_seen": 1159705600 + }, + { + "epoch": 3.08, + "learning_rate": 0.000327592778335005, + "loss": 2.7675, + "theoretical_loss": 3.5987710047047816, + "tokens_seen": 1159771136 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003275827482447342, + "loss": 2.5033, + "theoretical_loss": 3.59875217903571, + "tokens_seen": 1159836672 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003275727181544634, + "loss": 2.7395, + "theoretical_loss": 3.5987333547281706, + "tokens_seen": 1159902208 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003275626880641926, + "loss": 2.6689, + "theoretical_loss": 3.598714531781987, + "tokens_seen": 1159967744 + }, + { + "epoch": 3.08, + "objective/train/docs_used": 1274819, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.889496326446533, + "objective/train/theoretical_loss": 3.5987098262581325, + "objective/train/tokens_used": 1180444128, + "theoretical_loss": 3.5987098262581325, + "tokens_seen": 1159984128 + }, + { + "epoch": 3.08, + "learning_rate": 0.00032755265797392176, + "loss": 2.6646, + "theoretical_loss": 3.598695710196985, + "tokens_seen": 1160033280 + }, + { + "epoch": 3.08, + "learning_rate": 0.00032754262788365094, + "loss": 2.621, + "theoretical_loss": 3.598676889972988, + "tokens_seen": 1160098816 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003275325977933801, + "loss": 2.7977, + "theoretical_loss": 3.598658071109822, + "tokens_seen": 1160164352 + }, + { + "epoch": 3.08, + "learning_rate": 0.00032752256770310936, + "loss": 2.6562, + "theoretical_loss": 3.5986392536073106, + "tokens_seen": 1160229888 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003275125376128385, + "loss": 2.7267, + "theoretical_loss": 3.5986204374652795, + "tokens_seen": 1160295424 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003275025075225677, + "loss": 2.7961, + "theoretical_loss": 3.598601622683553, + "tokens_seen": 1160360960 + }, + { + "epoch": 3.08, + "learning_rate": 0.00032749247743229685, + "loss": 2.5709, + "theoretical_loss": 3.598582809261956, + "tokens_seen": 1160426496 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003274824473420261, + "loss": 2.7136, + "theoretical_loss": 3.598563997200314, + "tokens_seen": 1160492032 + }, + { + "epoch": 3.08, + "learning_rate": 0.00032747241725175527, + "loss": 2.6793, + "theoretical_loss": 3.59854518649845, + "tokens_seen": 1160557568 + }, + { + "epoch": 3.08, + "learning_rate": 0.00032746238716148445, + "loss": 2.7906, + "theoretical_loss": 3.5985263771561913, + "tokens_seen": 1160623104 + }, + { + "epoch": 3.08, + "learning_rate": 0.00032745235707121363, + "loss": 2.5507, + "theoretical_loss": 3.5985075691733615, + "tokens_seen": 1160688640 + }, + { + "epoch": 3.08, + "learning_rate": 0.00032744232698094286, + "loss": 2.6493, + "theoretical_loss": 3.598488762549786, + "tokens_seen": 1160754176 + }, + { + "epoch": 3.08, + "learning_rate": 0.000327432296890672, + "loss": 2.5678, + "theoretical_loss": 3.59846995728529, + "tokens_seen": 1160819712 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003274222668004012, + "loss": 2.8937, + "theoretical_loss": 3.598451153379698, + "tokens_seen": 1160885248 + }, + { + "epoch": 3.08, + "learning_rate": 0.00032741223671013035, + "loss": 2.802, + "theoretical_loss": 3.5984323508328355, + "tokens_seen": 1160950784 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003274022066198596, + "loss": 2.7575, + "theoretical_loss": 3.598413549644528, + "tokens_seen": 1161016320 + }, + { + "epoch": 3.08, + "learning_rate": 0.00032739217652958877, + "loss": 2.5724, + "theoretical_loss": 3.5983947498145996, + "tokens_seen": 1161081856 + }, + { + "epoch": 3.08, + "learning_rate": 0.00032738214643931795, + "loss": 2.695, + "theoretical_loss": 3.5983759513428764, + "tokens_seen": 1161147392 + }, + { + "epoch": 3.08, + "learning_rate": 0.00032737211634904713, + "loss": 2.7675, + "theoretical_loss": 3.5983571542291832, + "tokens_seen": 1161212928 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003273620862587763, + "loss": 2.6807, + "theoretical_loss": 3.5983383584733453, + "tokens_seen": 1161278464 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003273520561685055, + "loss": 2.8354, + "theoretical_loss": 3.5983195640751884, + "tokens_seen": 1161344000 + }, + { + "epoch": 3.08, + "learning_rate": 0.00032734202607823473, + "loss": 2.6488, + "theoretical_loss": 3.598300771034537, + "tokens_seen": 1161409536 + }, + { + "epoch": 3.08, + "learning_rate": 0.00032733199598796386, + "loss": 2.4031, + "theoretical_loss": 3.598281979351217, + "tokens_seen": 1161475072 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003273219658976931, + "loss": 2.4916, + "theoretical_loss": 3.5982631890250545, + "tokens_seen": 1161540608 + }, + { + "epoch": 3.08, + "learning_rate": 0.00032731193580742233, + "loss": 2.7317, + "theoretical_loss": 3.5982444000558735, + "tokens_seen": 1161606144 + }, + { + "epoch": 3.08, + "objective/train/docs_used": 1275406, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6565632820129395, + "objective/train/theoretical_loss": 3.5982397030255866, + "objective/train/tokens_used": 1182082528, + "theoretical_loss": 3.5982397030255866, + "tokens_seen": 1161622528 + }, + { + "epoch": 3.08, + "learning_rate": 0.00032730190571715145, + "loss": 2.3889, + "theoretical_loss": 3.5982256124435006, + "tokens_seen": 1161671680 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003272918756268807, + "loss": 2.7799, + "theoretical_loss": 3.59820682618776, + "tokens_seen": 1161737216 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003272818455366098, + "loss": 2.7662, + "theoretical_loss": 3.598188041288479, + "tokens_seen": 1161802752 + }, + { + "epoch": 3.08, + "learning_rate": 0.00032727181544633905, + "loss": 2.337, + "theoretical_loss": 3.5981692577454814, + "tokens_seen": 1161868288 + }, + { + "epoch": 3.08, + "learning_rate": 0.00032726178535606823, + "loss": 2.8643, + "theoretical_loss": 3.598150475558594, + "tokens_seen": 1161933824 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003272517552657974, + "loss": 2.6224, + "theoretical_loss": 3.5981316947276416, + "tokens_seen": 1161999360 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003272417251755266, + "loss": 2.4621, + "theoretical_loss": 3.5981129152524503, + "tokens_seen": 1162064896 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003272316950852558, + "loss": 2.322, + "theoretical_loss": 3.598094137132846, + "tokens_seen": 1162130432 + }, + { + "epoch": 3.08, + "learning_rate": 0.00032722166499498496, + "loss": 2.8936, + "theoretical_loss": 3.5980753603686537, + "tokens_seen": 1162195968 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003272116349047142, + "loss": 2.8057, + "theoretical_loss": 3.5980565849597, + "tokens_seen": 1162261504 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003272016048144433, + "loss": 2.7092, + "theoretical_loss": 3.5980378109058098, + "tokens_seen": 1162327040 + }, + { + "epoch": 3.08, + "learning_rate": 0.00032719157472417256, + "loss": 2.6417, + "theoretical_loss": 3.598019038206809, + "tokens_seen": 1162392576 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003271815446339017, + "loss": 2.4224, + "theoretical_loss": 3.598000266862525, + "tokens_seen": 1162458112 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003271715145436309, + "loss": 2.4214, + "theoretical_loss": 3.597981496872781, + "tokens_seen": 1162523648 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003271614844533601, + "loss": 2.5387, + "theoretical_loss": 3.5979627282374054, + "tokens_seen": 1162589184 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003271514543630893, + "loss": 2.6946, + "theoretical_loss": 3.5979439609562225, + "tokens_seen": 1162654720 + }, + { + "epoch": 3.08, + "learning_rate": 0.00032714142427281846, + "loss": 2.6044, + "theoretical_loss": 3.5979251950290587, + "tokens_seen": 1162720256 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003271313941825477, + "loss": 2.516, + "theoretical_loss": 3.5979064304557404, + "tokens_seen": 1162785792 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003271213640922768, + "loss": 2.5136, + "theoretical_loss": 3.5978876672360935, + "tokens_seen": 1162851328 + }, + { + "epoch": 3.08, + "learning_rate": 0.00032711133400200606, + "loss": 2.625, + "theoretical_loss": 3.597868905369944, + "tokens_seen": 1162916864 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003271013039117352, + "loss": 2.4939, + "theoretical_loss": 3.5978501448571176, + "tokens_seen": 1162982400 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003270912738214644, + "loss": 2.684, + "theoretical_loss": 3.5978313856974413, + "tokens_seen": 1163047936 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003270812437311936, + "loss": 2.8434, + "theoretical_loss": 3.59781262789074, + "tokens_seen": 1163113472 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003270712136409228, + "loss": 2.5827, + "theoretical_loss": 3.597793871436841, + "tokens_seen": 1163179008 + }, + { + "epoch": 3.08, + "learning_rate": 0.00032706118355065196, + "loss": 2.7304, + "theoretical_loss": 3.59777511633557, + "tokens_seen": 1163244544 + }, + { + "epoch": 3.08, + "objective/train/docs_used": 1276640, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.588357448577881, + "objective/train/theoretical_loss": 3.5977704277715805, + "objective/train/tokens_used": 1183720928, + "theoretical_loss": 3.5977704277715805, + "tokens_seen": 1163260928 + }, + { + "epoch": 3.08, + "learning_rate": 0.00032705115346038114, + "loss": 2.678, + "theoretical_loss": 3.597756362586754, + "tokens_seen": 1163310080 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003270411233701103, + "loss": 2.7792, + "theoretical_loss": 3.597737610190218, + "tokens_seen": 1163375616 + }, + { + "epoch": 3.08, + "learning_rate": 0.00032703109327983956, + "loss": 2.67, + "theoretical_loss": 3.597718859145789, + "tokens_seen": 1163441152 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003270210631895687, + "loss": 2.6397, + "theoretical_loss": 3.5977001094532937, + "tokens_seen": 1163506688 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003270110330992979, + "loss": 2.4046, + "theoretical_loss": 3.597681361112558, + "tokens_seen": 1163572224 + }, + { + "epoch": 3.08, + "learning_rate": 0.00032700100300902705, + "loss": 2.55, + "theoretical_loss": 3.5976626141234087, + "tokens_seen": 1163637760 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003269909729187563, + "loss": 2.4804, + "theoretical_loss": 3.5976438684856715, + "tokens_seen": 1163703296 + }, + { + "epoch": 3.08, + "learning_rate": 0.00032698094282848547, + "loss": 2.6268, + "theoretical_loss": 3.597625124199174, + "tokens_seen": 1163768832 + }, + { + "epoch": 3.08, + "learning_rate": 0.00032697091273821465, + "loss": 2.5692, + "theoretical_loss": 3.5976063812637418, + "tokens_seen": 1163834368 + }, + { + "epoch": 3.08, + "learning_rate": 0.00032696088264794383, + "loss": 2.5237, + "theoretical_loss": 3.5975876396792015, + "tokens_seen": 1163899904 + }, + { + "epoch": 3.08, + "learning_rate": 0.00032695085255767306, + "loss": 2.6462, + "theoretical_loss": 3.5975688994453803, + "tokens_seen": 1163965440 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003269408224674022, + "loss": 2.7789, + "theoretical_loss": 3.597550160562104, + "tokens_seen": 1164030976 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003269307923771314, + "loss": 2.5586, + "theoretical_loss": 3.5975314230292, + "tokens_seen": 1164096512 + }, + { + "epoch": 3.08, + "learning_rate": 0.00032692076228686055, + "loss": 2.6079, + "theoretical_loss": 3.5975126868464953, + "tokens_seen": 1164162048 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003269107321965898, + "loss": 2.5372, + "theoretical_loss": 3.5974939520138154, + "tokens_seen": 1164227584 + }, + { + "epoch": 3.08, + "learning_rate": 0.00032690070210631897, + "loss": 2.4518, + "theoretical_loss": 3.5974752185309873, + "tokens_seen": 1164293120 + }, + { + "epoch": 3.08, + "learning_rate": 0.00032689067201604815, + "loss": 2.4755, + "theoretical_loss": 3.5974564863978387, + "tokens_seen": 1164358656 + }, + { + "epoch": 3.08, + "learning_rate": 0.00032688064192577733, + "loss": 2.6264, + "theoretical_loss": 3.5974377556141954, + "tokens_seen": 1164424192 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003268706118355065, + "loss": 2.4069, + "theoretical_loss": 3.5974190261798844, + "tokens_seen": 1164489728 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003268605817452357, + "loss": 2.5928, + "theoretical_loss": 3.5974002980947333, + "tokens_seen": 1164555264 + }, + { + "epoch": 3.08, + "learning_rate": 0.00032685055165496493, + "loss": 2.5782, + "theoretical_loss": 3.597381571358568, + "tokens_seen": 1164620800 + }, + { + "epoch": 3.08, + "learning_rate": 0.00032684052156469406, + "loss": 2.4116, + "theoretical_loss": 3.597362845971216, + "tokens_seen": 1164686336 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003268304914744233, + "loss": 2.8265, + "theoretical_loss": 3.597344121932504, + "tokens_seen": 1164751872 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003268204613841524, + "loss": 2.7104, + "theoretical_loss": 3.59732539924226, + "tokens_seen": 1164817408 + }, + { + "epoch": 3.08, + "learning_rate": 0.00032681043129388165, + "loss": 2.5426, + "theoretical_loss": 3.5973066779003093, + "tokens_seen": 1164882944 + }, + { + "epoch": 3.08, + "objective/train/docs_used": 1277429, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5294029712677, + "objective/train/theoretical_loss": 3.5973019977754723, + "objective/train/tokens_used": 1185359328, + "theoretical_loss": 3.5973019977754723, + "tokens_seen": 1164899328 + }, + { + "epoch": 3.08, + "learning_rate": 0.00032680040120361084, + "loss": 2.485, + "theoretical_loss": 3.59728795790648, + "tokens_seen": 1164948480 + }, + { + "epoch": 3.08, + "learning_rate": 0.00032679037111334, + "loss": 2.661, + "theoretical_loss": 3.5972692392605996, + "tokens_seen": 1165014016 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003267803410230692, + "loss": 2.813, + "theoretical_loss": 3.5972505219624944, + "tokens_seen": 1165079552 + }, + { + "epoch": 3.08, + "learning_rate": 0.00032677031093279843, + "loss": 2.4742, + "theoretical_loss": 3.597231806011991, + "tokens_seen": 1165145088 + }, + { + "epoch": 3.08, + "learning_rate": 0.00032676028084252756, + "loss": 2.6719, + "theoretical_loss": 3.5972130914089187, + "tokens_seen": 1165210624 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003267502507522568, + "loss": 2.5704, + "theoretical_loss": 3.5971943781531026, + "tokens_seen": 1165276160 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003267402206619859, + "loss": 2.6898, + "theoretical_loss": 3.597175666244371, + "tokens_seen": 1165341696 + }, + { + "epoch": 3.08, + "learning_rate": 0.00032673019057171516, + "loss": 2.3992, + "theoretical_loss": 3.597156955682551, + "tokens_seen": 1165407232 + }, + { + "epoch": 3.08, + "learning_rate": 0.00032672016048144434, + "loss": 2.4696, + "theoretical_loss": 3.5971382464674697, + "tokens_seen": 1165472768 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003267101303911735, + "loss": 2.7106, + "theoretical_loss": 3.5971195385989545, + "tokens_seen": 1165538304 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003267001003009027, + "loss": 2.4775, + "theoretical_loss": 3.597100832076833, + "tokens_seen": 1165603840 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003266900702106319, + "loss": 2.4111, + "theoretical_loss": 3.5970821269009328, + "tokens_seen": 1165669376 + }, + { + "epoch": 3.08, + "learning_rate": 0.00032668004012036106, + "loss": 2.6574, + "theoretical_loss": 3.5970634230710803, + "tokens_seen": 1165734912 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003266700100300903, + "loss": 2.5731, + "theoretical_loss": 3.597044720587104, + "tokens_seen": 1165800448 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003266599799398194, + "loss": 2.7557, + "theoretical_loss": 3.5970260194488315, + "tokens_seen": 1165865984 + }, + { + "epoch": 3.08, + "learning_rate": 0.00032664994984954866, + "loss": 2.6017, + "theoretical_loss": 3.5970073196560897, + "tokens_seen": 1165931520 + }, + { + "epoch": 3.08, + "learning_rate": 0.00032663991975927784, + "loss": 2.6644, + "theoretical_loss": 3.5969886212087063, + "tokens_seen": 1165997056 + }, + { + "epoch": 3.08, + "learning_rate": 0.000326629889669007, + "loss": 2.6361, + "theoretical_loss": 3.596969924106509, + "tokens_seen": 1166062592 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003266198595787362, + "loss": 2.7494, + "theoretical_loss": 3.596951228349325, + "tokens_seen": 1166128128 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003266098294884654, + "loss": 2.7595, + "theoretical_loss": 3.5969325339369833, + "tokens_seen": 1166193664 + }, + { + "epoch": 3.08, + "learning_rate": 0.00032659979939819457, + "loss": 2.6563, + "theoretical_loss": 3.59691384086931, + "tokens_seen": 1166259200 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003265897693079238, + "loss": 2.7333, + "theoretical_loss": 3.5968951491461336, + "tokens_seen": 1166324736 + }, + { + "epoch": 3.08, + "learning_rate": 0.00032657973921765293, + "loss": 2.7429, + "theoretical_loss": 3.596876458767282, + "tokens_seen": 1166390272 + }, + { + "epoch": 3.08, + "learning_rate": 0.00032656970912738216, + "loss": 2.4125, + "theoretical_loss": 3.596857769732583, + "tokens_seen": 1166455808 + }, + { + "epoch": 3.08, + "learning_rate": 0.00032655967903711135, + "loss": 2.5437, + "theoretical_loss": 3.596839082041864, + "tokens_seen": 1166521344 + }, + { + "epoch": 3.08, + "objective/train/docs_used": 1282424, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.0159366130828857, + "objective/train/theoretical_loss": 3.5968344103291607, + "objective/train/tokens_used": 1186997728, + "theoretical_loss": 3.5968344103291607, + "tokens_seen": 1166537728 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003265496489468405, + "loss": 2.6358, + "theoretical_loss": 3.596820395694953, + "tokens_seen": 1166586880 + }, + { + "epoch": 3.08, + "learning_rate": 0.00032653961885656976, + "loss": 2.3213, + "theoretical_loss": 3.5968017106916776, + "tokens_seen": 1166652416 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003265295887662989, + "loss": 2.6772, + "theoretical_loss": 3.5967830270318664, + "tokens_seen": 1166717952 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003265195586760281, + "loss": 2.7451, + "theoretical_loss": 3.596764344715347, + "tokens_seen": 1166783488 + }, + { + "epoch": 3.08, + "learning_rate": 0.00032650952858575725, + "loss": 2.7173, + "theoretical_loss": 3.596745663741947, + "tokens_seen": 1166849024 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003264994984954865, + "loss": 2.751, + "theoretical_loss": 3.596726984111495, + "tokens_seen": 1166914560 + }, + { + "epoch": 3.08, + "learning_rate": 0.00032648946840521567, + "loss": 2.6119, + "theoretical_loss": 3.5967083058238187, + "tokens_seen": 1166980096 + }, + { + "epoch": 3.08, + "learning_rate": 0.00032647943831494485, + "loss": 2.7055, + "theoretical_loss": 3.5966896288787464, + "tokens_seen": 1167045632 + }, + { + "epoch": 3.08, + "learning_rate": 0.00032646940822467403, + "loss": 2.4983, + "theoretical_loss": 3.596670953276106, + "tokens_seen": 1167111168 + }, + { + "epoch": 3.08, + "learning_rate": 0.00032645937813440326, + "loss": 2.6833, + "theoretical_loss": 3.596652279015726, + "tokens_seen": 1167176704 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003264493480441324, + "loss": 2.5952, + "theoretical_loss": 3.596633606097434, + "tokens_seen": 1167242240 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003264393179538616, + "loss": 2.6169, + "theoretical_loss": 3.596614934521059, + "tokens_seen": 1167307776 + }, + { + "epoch": 3.08, + "learning_rate": 0.00032642928786359075, + "loss": 2.6897, + "theoretical_loss": 3.596596264286428, + "tokens_seen": 1167373312 + }, + { + "epoch": 3.08, + "learning_rate": 0.00032641925777332, + "loss": 2.4475, + "theoretical_loss": 3.596577595393371, + "tokens_seen": 1167438848 + }, + { + "epoch": 3.08, + "learning_rate": 0.00032640922768304917, + "loss": 2.7334, + "theoretical_loss": 3.5965589278417145, + "tokens_seen": 1167504384 + }, + { + "epoch": 3.08, + "learning_rate": 0.00032639919759277835, + "loss": 2.5306, + "theoretical_loss": 3.596540261631288, + "tokens_seen": 1167569920 + }, + { + "epoch": 3.08, + "learning_rate": 0.00032638916750250753, + "loss": 2.5529, + "theoretical_loss": 3.596521596761919, + "tokens_seen": 1167635456 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003263791374122367, + "loss": 2.9282, + "theoretical_loss": 3.5965029332334373, + "tokens_seen": 1167700992 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003263691073219659, + "loss": 2.487, + "theoretical_loss": 3.5964842710456697, + "tokens_seen": 1167766528 + }, + { + "epoch": 3.08, + "learning_rate": 0.00032635907723169513, + "loss": 2.6882, + "theoretical_loss": 3.5964656101984454, + "tokens_seen": 1167832064 + }, + { + "epoch": 3.08, + "learning_rate": 0.00032634904714142426, + "loss": 2.8419, + "theoretical_loss": 3.5964469506915933, + "tokens_seen": 1167897600 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003263390170511535, + "loss": 2.5661, + "theoretical_loss": 3.5964282925249407, + "tokens_seen": 1167963136 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003263289869608826, + "loss": 2.821, + "theoretical_loss": 3.5964096356983175, + "tokens_seen": 1168028672 + }, + { + "epoch": 3.08, + "learning_rate": 0.00032631895687061185, + "loss": 2.7562, + "theoretical_loss": 3.5963909802115515, + "tokens_seen": 1168094208 + }, + { + "epoch": 3.08, + "learning_rate": 0.00032630892678034104, + "loss": 2.3968, + "theoretical_loss": 3.5963723260644715, + "tokens_seen": 1168159744 + }, + { + "epoch": 3.08, + "objective/train/docs_used": 1282424, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.3768837451934814, + "objective/train/theoretical_loss": 3.5963676627370074, + "objective/train/tokens_used": 1187029472, + "theoretical_loss": 3.5963676627370074, + "tokens_seen": 1168176128 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003262988966900702, + "loss": 2.6501, + "theoretical_loss": 3.596353673256906, + "tokens_seen": 1168225280 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003262888665997994, + "loss": 2.4647, + "theoretical_loss": 3.5963350217886845, + "tokens_seen": 1168290816 + }, + { + "epoch": 3.08, + "learning_rate": 0.00032627883650952863, + "loss": 2.7632, + "theoretical_loss": 3.5963163716596345, + "tokens_seen": 1168356352 + }, + { + "epoch": 3.08, + "learning_rate": 0.00032626880641925776, + "loss": 2.441, + "theoretical_loss": 3.5962977228695854, + "tokens_seen": 1168421888 + }, + { + "epoch": 3.08, + "learning_rate": 0.000326258776328987, + "loss": 2.7156, + "theoretical_loss": 3.596279075418366, + "tokens_seen": 1168487424 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003262487462387161, + "loss": 2.5267, + "theoretical_loss": 3.5962604293058043, + "tokens_seen": 1168552960 + }, + { + "epoch": 3.08, + "learning_rate": 0.00032623871614844536, + "loss": 2.628, + "theoretical_loss": 3.5962417845317303, + "tokens_seen": 1168618496 + }, + { + "epoch": 3.08, + "learning_rate": 0.00032622868605817454, + "loss": 2.5469, + "theoretical_loss": 3.596223141095972, + "tokens_seen": 1168684032 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003262186559679037, + "loss": 2.6036, + "theoretical_loss": 3.5962044989983593, + "tokens_seen": 1168749568 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003262086258776329, + "loss": 2.3583, + "theoretical_loss": 3.5961858582387194, + "tokens_seen": 1168815104 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003261985957873621, + "loss": 2.6966, + "theoretical_loss": 3.596167218816883, + "tokens_seen": 1168880640 + }, + { + "epoch": 3.08, + "learning_rate": 0.00032618856569709126, + "loss": 2.6841, + "theoretical_loss": 3.5961485807326783, + "tokens_seen": 1168946176 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003261785356068205, + "loss": 2.6477, + "theoretical_loss": 3.5961299439859347, + "tokens_seen": 1169011712 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003261685055165496, + "loss": 2.7623, + "theoretical_loss": 3.596111308576481, + "tokens_seen": 1169077248 + }, + { + "epoch": 3.08, + "learning_rate": 0.00032615847542627886, + "loss": 2.5495, + "theoretical_loss": 3.5960926745041455, + "tokens_seen": 1169142784 + }, + { + "epoch": 3.08, + "learning_rate": 0.00032614844533600804, + "loss": 2.7324, + "theoretical_loss": 3.596074041768759, + "tokens_seen": 1169208320 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003261384152457372, + "loss": 2.5712, + "theoretical_loss": 3.596055410370149, + "tokens_seen": 1169273856 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003261283851554664, + "loss": 2.6998, + "theoretical_loss": 3.596036780308146, + "tokens_seen": 1169339392 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003261183550651956, + "loss": 2.6135, + "theoretical_loss": 3.5960181515825784, + "tokens_seen": 1169404928 + }, + { + "epoch": 3.08, + "learning_rate": 0.00032610832497492477, + "loss": 2.577, + "theoretical_loss": 3.595999524193276, + "tokens_seen": 1169470464 + }, + { + "epoch": 3.08, + "learning_rate": 0.000326098294884654, + "loss": 2.8685, + "theoretical_loss": 3.595980898140067, + "tokens_seen": 1169536000 + }, + { + "epoch": 3.08, + "learning_rate": 0.00032608826479438313, + "loss": 2.5749, + "theoretical_loss": 3.5959622734227823, + "tokens_seen": 1169601536 + }, + { + "epoch": 3.08, + "learning_rate": 0.00032607823470411236, + "loss": 2.7083, + "theoretical_loss": 3.5959436500412494, + "tokens_seen": 1169667072 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003260682046138415, + "loss": 2.5683, + "theoretical_loss": 3.5959250279952997, + "tokens_seen": 1169732608 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003260581745235707, + "loss": 2.7942, + "theoretical_loss": 3.595906407284761, + "tokens_seen": 1169798144 + }, + { + "epoch": 3.08, + "objective/train/docs_used": 1282424, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.560084581375122, + "objective/train/theoretical_loss": 3.595901752315764, + "objective/train/tokens_used": 1187029472, + "theoretical_loss": 3.595901752315764, + "tokens_seen": 1169814528 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003260481444332999, + "loss": 2.7187, + "theoretical_loss": 3.595887787909463, + "tokens_seen": 1169863680 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003260381143430291, + "loss": 2.909, + "theoretical_loss": 3.5958691698692355, + "tokens_seen": 1169929216 + }, + { + "epoch": 3.08, + "learning_rate": 0.00032602808425275827, + "loss": 2.7557, + "theoretical_loss": 3.595850553163908, + "tokens_seen": 1169994752 + }, + { + "epoch": 3.08, + "learning_rate": 0.00032601805416248745, + "loss": 2.5457, + "theoretical_loss": 3.59583193779331, + "tokens_seen": 1170060288 + }, + { + "epoch": 3.08, + "learning_rate": 0.00032600802407221663, + "loss": 2.6619, + "theoretical_loss": 3.5958133237572714, + "tokens_seen": 1170125824 + }, + { + "epoch": 3.08, + "learning_rate": 0.00032599799398194587, + "loss": 2.5647, + "theoretical_loss": 3.595794711055621, + "tokens_seen": 1170191360 + }, + { + "epoch": 3.08, + "learning_rate": 0.000325987963891675, + "loss": 2.7163, + "theoretical_loss": 3.595776099688189, + "tokens_seen": 1170256896 + }, + { + "epoch": 3.08, + "learning_rate": 0.00032597793380140423, + "loss": 2.4849, + "theoretical_loss": 3.5957574896548046, + "tokens_seen": 1170322432 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003259679037111334, + "loss": 2.8802, + "theoretical_loss": 3.595738880955298, + "tokens_seen": 1170387968 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003259578736208626, + "loss": 2.296, + "theoretical_loss": 3.5957202735894986, + "tokens_seen": 1170453504 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003259478435305918, + "loss": 2.6945, + "theoretical_loss": 3.595701667557236, + "tokens_seen": 1170519040 + }, + { + "epoch": 3.08, + "learning_rate": 0.00032593781344032095, + "loss": 2.6302, + "theoretical_loss": 3.5956830628583405, + "tokens_seen": 1170584576 + }, + { + "epoch": 3.08, + "learning_rate": 0.00032592778335005014, + "loss": 2.4871, + "theoretical_loss": 3.5956644594926415, + "tokens_seen": 1170650112 + }, + { + "epoch": 3.08, + "learning_rate": 0.00032591775325977937, + "loss": 2.6558, + "theoretical_loss": 3.5956458574599686, + "tokens_seen": 1170715648 + }, + { + "epoch": 3.08, + "learning_rate": 0.0003259077231695085, + "loss": 2.5413, + "theoretical_loss": 3.595627256760152, + "tokens_seen": 1170781184 + }, + { + "epoch": 3.09, + "learning_rate": 0.00032589769307923773, + "loss": 2.4552, + "theoretical_loss": 3.5956086573930217, + "tokens_seen": 1170846720 + }, + { + "epoch": 3.09, + "learning_rate": 0.00032588766298896686, + "loss": 2.4495, + "theoretical_loss": 3.5955900593584076, + "tokens_seen": 1170912256 + }, + { + "epoch": 3.09, + "learning_rate": 0.0003258776328986961, + "loss": 2.7012, + "theoretical_loss": 3.5955714626561397, + "tokens_seen": 1170977792 + }, + { + "epoch": 3.09, + "learning_rate": 0.0003258676028084253, + "loss": 2.4193, + "theoretical_loss": 3.5955528672860475, + "tokens_seen": 1171043328 + }, + { + "epoch": 3.09, + "learning_rate": 0.00032585757271815446, + "loss": 2.7626, + "theoretical_loss": 3.5955342732479614, + "tokens_seen": 1171108864 + }, + { + "epoch": 3.09, + "learning_rate": 0.00032584754262788364, + "loss": 2.6797, + "theoretical_loss": 3.5955156805417117, + "tokens_seen": 1171174400 + }, + { + "epoch": 3.09, + "learning_rate": 0.0003258375125376128, + "loss": 2.3862, + "theoretical_loss": 3.595497089167128, + "tokens_seen": 1171239936 + }, + { + "epoch": 3.09, + "learning_rate": 0.000325827482447342, + "loss": 2.4888, + "theoretical_loss": 3.5954784991240407, + "tokens_seen": 1171305472 + }, + { + "epoch": 3.09, + "learning_rate": 0.00032581745235707124, + "loss": 2.5438, + "theoretical_loss": 3.5954599104122797, + "tokens_seen": 1171371008 + }, + { + "epoch": 3.09, + "learning_rate": 0.0003258074222668004, + "loss": 2.3983, + "theoretical_loss": 3.5954413230316757, + "tokens_seen": 1171436544 + }, + { + "epoch": 3.09, + "objective/train/docs_used": 1282424, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6180059909820557, + "objective/train/theoretical_loss": 3.595436676394498, + "objective/train/tokens_used": 1187029472, + "theoretical_loss": 3.595436676394498, + "tokens_seen": 1171452928 + }, + { + "epoch": 3.09, + "learning_rate": 0.0003257973921765296, + "loss": 2.658, + "theoretical_loss": 3.5954227369820586, + "tokens_seen": 1171502080 + }, + { + "epoch": 3.09, + "learning_rate": 0.00032578736208625883, + "loss": 2.4644, + "theoretical_loss": 3.5954041522632583, + "tokens_seen": 1171567616 + }, + { + "epoch": 3.09, + "learning_rate": 0.00032577733199598796, + "loss": 2.6907, + "theoretical_loss": 3.5953855688751055, + "tokens_seen": 1171633152 + }, + { + "epoch": 3.09, + "learning_rate": 0.0003257673019057172, + "loss": 2.7324, + "theoretical_loss": 3.5953669868174307, + "tokens_seen": 1171698688 + }, + { + "epoch": 3.09, + "learning_rate": 0.0003257572718154463, + "loss": 2.2496, + "theoretical_loss": 3.595348406090064, + "tokens_seen": 1171764224 + }, + { + "epoch": 3.09, + "learning_rate": 0.00032574724172517556, + "loss": 2.7043, + "theoretical_loss": 3.595329826692835, + "tokens_seen": 1171829760 + }, + { + "epoch": 3.09, + "learning_rate": 0.00032573721163490474, + "loss": 2.5151, + "theoretical_loss": 3.5953112486255754, + "tokens_seen": 1171895296 + }, + { + "epoch": 3.09, + "learning_rate": 0.0003257271815446339, + "loss": 2.5465, + "theoretical_loss": 3.5952926718881146, + "tokens_seen": 1171960832 + }, + { + "epoch": 3.09, + "learning_rate": 0.0003257171514543631, + "loss": 2.6868, + "theoretical_loss": 3.595274096480284, + "tokens_seen": 1172026368 + }, + { + "epoch": 3.09, + "learning_rate": 0.0003257071213640923, + "loss": 2.7925, + "theoretical_loss": 3.5952555224019136, + "tokens_seen": 1172091904 + }, + { + "epoch": 3.09, + "learning_rate": 0.00032569709127382146, + "loss": 2.7214, + "theoretical_loss": 3.595236949652833, + "tokens_seen": 1172157440 + }, + { + "epoch": 3.09, + "learning_rate": 0.0003256870611835507, + "loss": 2.564, + "theoretical_loss": 3.5952183782328744, + "tokens_seen": 1172222976 + }, + { + "epoch": 3.09, + "learning_rate": 0.0003256770310932798, + "loss": 2.4106, + "theoretical_loss": 3.5951998081418672, + "tokens_seen": 1172288512 + }, + { + "epoch": 3.09, + "learning_rate": 0.00032566700100300906, + "loss": 2.505, + "theoretical_loss": 3.5951812393796434, + "tokens_seen": 1172354048 + }, + { + "epoch": 3.09, + "learning_rate": 0.00032565697091273824, + "loss": 2.5024, + "theoretical_loss": 3.595162671946032, + "tokens_seen": 1172419584 + }, + { + "epoch": 3.09, + "learning_rate": 0.0003256469408224674, + "loss": 2.4508, + "theoretical_loss": 3.5951441058408644, + "tokens_seen": 1172485120 + }, + { + "epoch": 3.09, + "learning_rate": 0.0003256369107321966, + "loss": 2.7553, + "theoretical_loss": 3.5951255410639713, + "tokens_seen": 1172550656 + }, + { + "epoch": 3.09, + "learning_rate": 0.0003256268806419258, + "loss": 2.5269, + "theoretical_loss": 3.595106977615184, + "tokens_seen": 1172616192 + }, + { + "epoch": 3.09, + "learning_rate": 0.00032561685055165497, + "loss": 2.6031, + "theoretical_loss": 3.595088415494332, + "tokens_seen": 1172681728 + }, + { + "epoch": 3.09, + "learning_rate": 0.0003256068204613842, + "loss": 2.5579, + "theoretical_loss": 3.5950698547012463, + "tokens_seen": 1172747264 + }, + { + "epoch": 3.09, + "learning_rate": 0.00032559679037111333, + "loss": 2.374, + "theoretical_loss": 3.595051295235759, + "tokens_seen": 1172812800 + }, + { + "epoch": 3.09, + "learning_rate": 0.00032558676028084256, + "loss": 2.6482, + "theoretical_loss": 3.5950327370977, + "tokens_seen": 1172878336 + }, + { + "epoch": 3.09, + "learning_rate": 0.0003255767301905717, + "loss": 2.7096, + "theoretical_loss": 3.595014180286901, + "tokens_seen": 1172943872 + }, + { + "epoch": 3.09, + "learning_rate": 0.0003255667001003009, + "loss": 2.4358, + "theoretical_loss": 3.5949956248031913, + "tokens_seen": 1173009408 + }, + { + "epoch": 3.09, + "learning_rate": 0.0003255566700100301, + "loss": 2.5746, + "theoretical_loss": 3.5949770706464035, + "tokens_seen": 1173074944 + }, + { + "epoch": 3.09, + "objective/train/docs_used": 1282424, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.576780080795288, + "objective/train/theoretical_loss": 3.5949724323145182, + "objective/train/tokens_used": 1187029472, + "theoretical_loss": 3.5949724323145182, + "tokens_seen": 1173091328 + }, + { + "epoch": 3.09, + "learning_rate": 0.0003255466399197593, + "loss": 2.6818, + "theoretical_loss": 3.594958517816368, + "tokens_seen": 1173140480 + }, + { + "epoch": 3.09, + "learning_rate": 0.00032553660982948847, + "loss": 2.6777, + "theoretical_loss": 3.5949399663129156, + "tokens_seen": 1173206016 + }, + { + "epoch": 3.09, + "learning_rate": 0.00032552657973921765, + "loss": 2.7612, + "theoretical_loss": 3.5949214161358776, + "tokens_seen": 1173271552 + }, + { + "epoch": 3.09, + "learning_rate": 0.00032551654964894683, + "loss": 2.5333, + "theoretical_loss": 3.594902867285085, + "tokens_seen": 1173337088 + }, + { + "epoch": 3.09, + "learning_rate": 0.00032550651955867607, + "loss": 2.6658, + "theoretical_loss": 3.594884319760369, + "tokens_seen": 1173402624 + }, + { + "epoch": 3.09, + "learning_rate": 0.0003254964894684052, + "loss": 2.8645, + "theoretical_loss": 3.59486577356156, + "tokens_seen": 1173468160 + }, + { + "epoch": 3.09, + "learning_rate": 0.00032548645937813443, + "loss": 2.7192, + "theoretical_loss": 3.594847228688491, + "tokens_seen": 1173533696 + }, + { + "epoch": 3.09, + "learning_rate": 0.0003254764292878636, + "loss": 2.7719, + "theoretical_loss": 3.594828685140991, + "tokens_seen": 1173599232 + }, + { + "epoch": 3.09, + "learning_rate": 0.0003254663991975928, + "loss": 2.535, + "theoretical_loss": 3.5948101429188926, + "tokens_seen": 1173664768 + }, + { + "epoch": 3.09, + "learning_rate": 0.000325456369107322, + "loss": 2.5644, + "theoretical_loss": 3.5947916020220267, + "tokens_seen": 1173730304 + }, + { + "epoch": 3.09, + "learning_rate": 0.00032544633901705115, + "loss": 2.6096, + "theoretical_loss": 3.594773062450225, + "tokens_seen": 1173795840 + }, + { + "epoch": 3.09, + "learning_rate": 0.00032543630892678034, + "loss": 2.14, + "theoretical_loss": 3.594754524203318, + "tokens_seen": 1173861376 + }, + { + "epoch": 3.09, + "learning_rate": 0.00032542627883650957, + "loss": 2.4617, + "theoretical_loss": 3.5947359872811377, + "tokens_seen": 1173926912 + }, + { + "epoch": 3.09, + "learning_rate": 0.0003254162487462387, + "loss": 2.5123, + "theoretical_loss": 3.5947174516835156, + "tokens_seen": 1173992448 + }, + { + "epoch": 3.09, + "learning_rate": 0.00032540621865596793, + "loss": 2.6792, + "theoretical_loss": 3.5946989174102826, + "tokens_seen": 1174057984 + }, + { + "epoch": 3.09, + "learning_rate": 0.00032539618856569706, + "loss": 2.5653, + "theoretical_loss": 3.5946803844612703, + "tokens_seen": 1174123520 + }, + { + "epoch": 3.09, + "learning_rate": 0.0003253861584754263, + "loss": 2.5136, + "theoretical_loss": 3.5946618528363103, + "tokens_seen": 1174189056 + }, + { + "epoch": 3.09, + "learning_rate": 0.0003253761283851555, + "loss": 2.6376, + "theoretical_loss": 3.594643322535234, + "tokens_seen": 1174254592 + }, + { + "epoch": 3.09, + "learning_rate": 0.00032536609829488466, + "loss": 2.4409, + "theoretical_loss": 3.5946247935578732, + "tokens_seen": 1174320128 + }, + { + "epoch": 3.09, + "learning_rate": 0.00032535606820461384, + "loss": 2.6197, + "theoretical_loss": 3.5946062659040594, + "tokens_seen": 1174385664 + }, + { + "epoch": 3.09, + "learning_rate": 0.000325346038114343, + "loss": 2.6027, + "theoretical_loss": 3.5945877395736243, + "tokens_seen": 1174451200 + }, + { + "epoch": 3.09, + "learning_rate": 0.0003253360080240722, + "loss": 2.634, + "theoretical_loss": 3.5945692145663988, + "tokens_seen": 1174516736 + }, + { + "epoch": 3.09, + "learning_rate": 0.00032532597793380144, + "loss": 2.8002, + "theoretical_loss": 3.5945506908822153, + "tokens_seen": 1174582272 + }, + { + "epoch": 3.09, + "learning_rate": 0.00032531594784353056, + "loss": 2.607, + "theoretical_loss": 3.5945321685209053, + "tokens_seen": 1174647808 + }, + { + "epoch": 3.09, + "learning_rate": 0.0003253059177532598, + "loss": 2.6358, + "theoretical_loss": 3.5945136474823007, + "tokens_seen": 1174713344 + }, + { + "epoch": 3.09, + "objective/train/docs_used": 1282424, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.806351900100708, + "objective/train/theoretical_loss": 3.5945090174293024, + "objective/train/tokens_used": 1187029472, + "theoretical_loss": 3.5945090174293024, + "tokens_seen": 1174729728 + }, + { + "epoch": 3.09, + "learning_rate": 0.000325295887662989, + "loss": 2.546, + "theoretical_loss": 3.594495127766233, + "tokens_seen": 1174778880 + }, + { + "epoch": 3.09, + "learning_rate": 0.00032528585757271816, + "loss": 2.735, + "theoretical_loss": 3.594476609372534, + "tokens_seen": 1174844416 + }, + { + "epoch": 3.09, + "learning_rate": 0.00032527582748244734, + "loss": 2.649, + "theoretical_loss": 3.5944580923010356, + "tokens_seen": 1174909952 + }, + { + "epoch": 3.09, + "learning_rate": 0.0003252657973921765, + "loss": 2.1022, + "theoretical_loss": 3.5944395765515695, + "tokens_seen": 1174975488 + }, + { + "epoch": 3.09, + "learning_rate": 0.0003252557673019057, + "loss": 2.6525, + "theoretical_loss": 3.594421062123968, + "tokens_seen": 1175041024 + }, + { + "epoch": 3.09, + "learning_rate": 0.00032524573721163494, + "loss": 2.6477, + "theoretical_loss": 3.5944025490180627, + "tokens_seen": 1175106560 + }, + { + "epoch": 3.09, + "learning_rate": 0.00032523570712136407, + "loss": 2.6208, + "theoretical_loss": 3.5943840372336853, + "tokens_seen": 1175172096 + }, + { + "epoch": 3.09, + "learning_rate": 0.0003252256770310933, + "loss": 2.603, + "theoretical_loss": 3.5943655267706687, + "tokens_seen": 1175237632 + }, + { + "epoch": 3.09, + "learning_rate": 0.00032521564694082243, + "loss": 2.6506, + "theoretical_loss": 3.594347017628844, + "tokens_seen": 1175303168 + }, + { + "epoch": 3.09, + "learning_rate": 0.00032520561685055166, + "loss": 2.7226, + "theoretical_loss": 3.5943285098080433, + "tokens_seen": 1175368704 + }, + { + "epoch": 3.09, + "learning_rate": 0.00032519558676028084, + "loss": 2.6465, + "theoretical_loss": 3.594310003308099, + "tokens_seen": 1175434240 + }, + { + "epoch": 3.09, + "learning_rate": 0.00032518555667001, + "loss": 2.893, + "theoretical_loss": 3.594291498128843, + "tokens_seen": 1175499776 + }, + { + "epoch": 3.09, + "learning_rate": 0.0003251755265797392, + "loss": 2.506, + "theoretical_loss": 3.5942729942701077, + "tokens_seen": 1175565312 + }, + { + "epoch": 3.09, + "learning_rate": 0.00032516549648946844, + "loss": 2.6732, + "theoretical_loss": 3.594254491731725, + "tokens_seen": 1175630848 + }, + { + "epoch": 3.09, + "learning_rate": 0.00032515546639919757, + "loss": 2.6036, + "theoretical_loss": 3.5942359905135275, + "tokens_seen": 1175696384 + }, + { + "epoch": 3.09, + "learning_rate": 0.0003251454363089268, + "loss": 2.6352, + "theoretical_loss": 3.5942174906153466, + "tokens_seen": 1175761920 + }, + { + "epoch": 3.09, + "learning_rate": 0.00032513540621865593, + "loss": 2.4589, + "theoretical_loss": 3.5941989920370148, + "tokens_seen": 1175827456 + }, + { + "epoch": 3.09, + "learning_rate": 0.00032512537612838517, + "loss": 2.6806, + "theoretical_loss": 3.594180494778365, + "tokens_seen": 1175892992 + }, + { + "epoch": 3.09, + "learning_rate": 0.00032511534603811435, + "loss": 2.6492, + "theoretical_loss": 3.5941619988392293, + "tokens_seen": 1175958528 + }, + { + "epoch": 3.09, + "learning_rate": 0.00032510531594784353, + "loss": 2.6898, + "theoretical_loss": 3.5941435042194394, + "tokens_seen": 1176024064 + }, + { + "epoch": 3.09, + "learning_rate": 0.0003250952858575727, + "loss": 2.6345, + "theoretical_loss": 3.5941250109188285, + "tokens_seen": 1176089600 + }, + { + "epoch": 3.09, + "learning_rate": 0.0003250852557673019, + "loss": 2.4554, + "theoretical_loss": 3.5941065189372288, + "tokens_seen": 1176155136 + }, + { + "epoch": 3.09, + "learning_rate": 0.00032507522567703107, + "loss": 2.2642, + "theoretical_loss": 3.594088028274472, + "tokens_seen": 1176220672 + }, + { + "epoch": 3.09, + "learning_rate": 0.0003250651955867603, + "loss": 2.7183, + "theoretical_loss": 3.5940695389303916, + "tokens_seen": 1176286208 + }, + { + "epoch": 3.09, + "learning_rate": 0.0003250551654964895, + "loss": 2.3044, + "theoretical_loss": 3.5940510509048194, + "tokens_seen": 1176351744 + }, + { + "epoch": 3.09, + "objective/train/docs_used": 1282424, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.1171867847442627, + "objective/train/theoretical_loss": 3.5940464291044236, + "objective/train/tokens_used": 1187029472, + "theoretical_loss": 3.5940464291044236, + "tokens_seen": 1176368128 + }, + { + "epoch": 3.09, + "learning_rate": 0.00032504513540621867, + "loss": 2.7721, + "theoretical_loss": 3.5940325641975885, + "tokens_seen": 1176417280 + }, + { + "epoch": 3.09, + "learning_rate": 0.00032503510531594785, + "loss": 2.6341, + "theoretical_loss": 3.594014078808531, + "tokens_seen": 1176482816 + }, + { + "epoch": 3.09, + "learning_rate": 0.00032502507522567703, + "loss": 2.5146, + "theoretical_loss": 3.5939955947374793, + "tokens_seen": 1176548352 + }, + { + "epoch": 3.09, + "learning_rate": 0.00032501504513540627, + "loss": 2.6868, + "theoretical_loss": 3.5939771119842665, + "tokens_seen": 1176613888 + }, + { + "epoch": 3.09, + "learning_rate": 0.0003250050150451354, + "loss": 2.3658, + "theoretical_loss": 3.593958630548725, + "tokens_seen": 1176679424 + }, + { + "epoch": 3.09, + "learning_rate": 0.00032499498495486463, + "loss": 2.6863, + "theoretical_loss": 3.593940150430688, + "tokens_seen": 1176744960 + }, + { + "epoch": 3.09, + "learning_rate": 0.0003249849548645938, + "loss": 2.6961, + "theoretical_loss": 3.5939216716299875, + "tokens_seen": 1176810496 + }, + { + "epoch": 3.09, + "learning_rate": 0.000324974924774323, + "loss": 2.5861, + "theoretical_loss": 3.5939031941464563, + "tokens_seen": 1176876032 + }, + { + "epoch": 3.09, + "learning_rate": 0.0003249648946840522, + "loss": 2.667, + "theoretical_loss": 3.5938847179799276, + "tokens_seen": 1176941568 + }, + { + "epoch": 3.09, + "learning_rate": 0.00032495486459378135, + "loss": 2.5671, + "theoretical_loss": 3.593866243130234, + "tokens_seen": 1177007104 + }, + { + "epoch": 3.09, + "learning_rate": 0.00032494483450351054, + "loss": 2.2381, + "theoretical_loss": 3.5938477695972084, + "tokens_seen": 1177072640 + }, + { + "epoch": 3.09, + "learning_rate": 0.00032493480441323977, + "loss": 2.5901, + "theoretical_loss": 3.593829297380683, + "tokens_seen": 1177138176 + }, + { + "epoch": 3.09, + "learning_rate": 0.0003249247743229689, + "loss": 2.635, + "theoretical_loss": 3.593810826480492, + "tokens_seen": 1177203712 + }, + { + "epoch": 3.09, + "learning_rate": 0.00032491474423269813, + "loss": 2.5594, + "theoretical_loss": 3.5937923568964676, + "tokens_seen": 1177269248 + }, + { + "epoch": 3.09, + "learning_rate": 0.00032490471414242726, + "loss": 2.6957, + "theoretical_loss": 3.5937738886284425, + "tokens_seen": 1177334784 + }, + { + "epoch": 3.09, + "learning_rate": 0.0003248946840521565, + "loss": 2.7526, + "theoretical_loss": 3.5937554216762497, + "tokens_seen": 1177400320 + }, + { + "epoch": 3.09, + "learning_rate": 0.0003248846539618857, + "loss": 2.8548, + "theoretical_loss": 3.5937369560397228, + "tokens_seen": 1177465856 + }, + { + "epoch": 3.09, + "learning_rate": 0.00032487462387161486, + "loss": 2.6545, + "theoretical_loss": 3.593718491718694, + "tokens_seen": 1177531392 + }, + { + "epoch": 3.09, + "learning_rate": 0.00032486459378134404, + "loss": 2.6746, + "theoretical_loss": 3.5937000287129974, + "tokens_seen": 1177596928 + }, + { + "epoch": 3.09, + "learning_rate": 0.0003248545636910732, + "loss": 2.6491, + "theoretical_loss": 3.5936815670224656, + "tokens_seen": 1177662464 + }, + { + "epoch": 3.09, + "learning_rate": 0.0003248445336008024, + "loss": 2.5586, + "theoretical_loss": 3.5936631066469316, + "tokens_seen": 1177728000 + }, + { + "epoch": 3.09, + "learning_rate": 0.00032483450351053164, + "loss": 2.8073, + "theoretical_loss": 3.5936446475862285, + "tokens_seen": 1177793536 + }, + { + "epoch": 3.09, + "learning_rate": 0.00032482447342026076, + "loss": 2.401, + "theoretical_loss": 3.59362618984019, + "tokens_seen": 1177859072 + }, + { + "epoch": 3.09, + "learning_rate": 0.00032481444332999, + "loss": 2.5078, + "theoretical_loss": 3.593607733408649, + "tokens_seen": 1177924608 + }, + { + "epoch": 3.09, + "learning_rate": 0.0003248044132397192, + "loss": 2.5333, + "theoretical_loss": 3.5935892782914385, + "tokens_seen": 1177990144 + }, + { + "epoch": 3.09, + "objective/train/docs_used": 1282424, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5997087955474854, + "objective/train/theoretical_loss": 3.5935846647174805, + "objective/train/tokens_used": 1187029472, + "theoretical_loss": 3.5935846647174805, + "tokens_seen": 1178006528 + }, + { + "epoch": 3.09, + "learning_rate": 0.00032479438314944836, + "loss": 2.6478, + "theoretical_loss": 3.593570824488392, + "tokens_seen": 1178055680 + }, + { + "epoch": 3.09, + "learning_rate": 0.00032478435305917754, + "loss": 2.6641, + "theoretical_loss": 3.593552371999343, + "tokens_seen": 1178121216 + }, + { + "epoch": 3.09, + "learning_rate": 0.0003247743229689067, + "loss": 2.6132, + "theoretical_loss": 3.5935339208241253, + "tokens_seen": 1178186752 + }, + { + "epoch": 3.09, + "learning_rate": 0.0003247642928786359, + "loss": 2.5248, + "theoretical_loss": 3.593515470962571, + "tokens_seen": 1178252288 + }, + { + "epoch": 3.09, + "learning_rate": 0.00032475426278836514, + "loss": 2.4717, + "theoretical_loss": 3.5934970224145144, + "tokens_seen": 1178317824 + }, + { + "epoch": 3.09, + "learning_rate": 0.00032474423269809427, + "loss": 2.4565, + "theoretical_loss": 3.5934785751797893, + "tokens_seen": 1178383360 + }, + { + "epoch": 3.09, + "learning_rate": 0.0003247342026078235, + "loss": 2.5356, + "theoretical_loss": 3.593460129258228, + "tokens_seen": 1178448896 + }, + { + "epoch": 3.09, + "learning_rate": 0.00032472417251755263, + "loss": 2.4481, + "theoretical_loss": 3.5934416846496644, + "tokens_seen": 1178514432 + }, + { + "epoch": 3.09, + "learning_rate": 0.00032471414242728186, + "loss": 2.679, + "theoretical_loss": 3.593423241353933, + "tokens_seen": 1178579968 + }, + { + "epoch": 3.09, + "learning_rate": 0.00032470411233701104, + "loss": 2.4818, + "theoretical_loss": 3.593404799370867, + "tokens_seen": 1178645504 + }, + { + "epoch": 3.09, + "learning_rate": 0.0003246940822467402, + "loss": 2.6474, + "theoretical_loss": 3.5933863587002985, + "tokens_seen": 1178711040 + }, + { + "epoch": 3.09, + "learning_rate": 0.0003246840521564694, + "loss": 2.5939, + "theoretical_loss": 3.5933679193420627, + "tokens_seen": 1178776576 + }, + { + "epoch": 3.09, + "learning_rate": 0.00032467402206619864, + "loss": 2.6959, + "theoretical_loss": 3.593349481295993, + "tokens_seen": 1178842112 + }, + { + "epoch": 3.09, + "learning_rate": 0.00032466399197592777, + "loss": 2.6948, + "theoretical_loss": 3.5933310445619226, + "tokens_seen": 1178907648 + }, + { + "epoch": 3.09, + "learning_rate": 0.000324653961885657, + "loss": 2.3513, + "theoretical_loss": 3.593312609139686, + "tokens_seen": 1178973184 + }, + { + "epoch": 3.09, + "learning_rate": 0.00032464393179538613, + "loss": 2.438, + "theoretical_loss": 3.593294175029116, + "tokens_seen": 1179038720 + }, + { + "epoch": 3.09, + "learning_rate": 0.00032463390170511537, + "loss": 2.4049, + "theoretical_loss": 3.593275742230047, + "tokens_seen": 1179104256 + }, + { + "epoch": 3.09, + "learning_rate": 0.00032462387161484455, + "loss": 2.7499, + "theoretical_loss": 3.5932573107423127, + "tokens_seen": 1179169792 + }, + { + "epoch": 3.09, + "learning_rate": 0.00032461384152457373, + "loss": 2.6122, + "theoretical_loss": 3.593238880565747, + "tokens_seen": 1179235328 + }, + { + "epoch": 3.09, + "learning_rate": 0.0003246038114343029, + "loss": 2.5427, + "theoretical_loss": 3.5932204517001836, + "tokens_seen": 1179300864 + }, + { + "epoch": 3.09, + "learning_rate": 0.0003245937813440321, + "loss": 2.5502, + "theoretical_loss": 3.593202024145456, + "tokens_seen": 1179366400 + }, + { + "epoch": 3.09, + "learning_rate": 0.00032458375125376127, + "loss": 2.5535, + "theoretical_loss": 3.593183597901399, + "tokens_seen": 1179431936 + }, + { + "epoch": 3.09, + "learning_rate": 0.0003245737211634905, + "loss": 2.6025, + "theoretical_loss": 3.593165172967846, + "tokens_seen": 1179497472 + }, + { + "epoch": 3.09, + "learning_rate": 0.00032456369107321963, + "loss": 2.736, + "theoretical_loss": 3.5931467493446307, + "tokens_seen": 1179563008 + }, + { + "epoch": 3.09, + "learning_rate": 0.00032455366098294887, + "loss": 2.3614, + "theoretical_loss": 3.593128327031588, + "tokens_seen": 1179628544 + }, + { + "epoch": 3.09, + "objective/train/docs_used": 1282424, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.648859977722168, + "objective/train/theoretical_loss": 3.5931237216580225, + "objective/train/tokens_used": 1187029472, + "theoretical_loss": 3.5931237216580225, + "tokens_seen": 1179644928 + }, + { + "epoch": 3.09, + "learning_rate": 0.000324543630892678, + "loss": 2.65, + "theoretical_loss": 3.593109906028552, + "tokens_seen": 1179694080 + }, + { + "epoch": 3.09, + "learning_rate": 0.00032453360080240723, + "loss": 2.6016, + "theoretical_loss": 3.5930914863353554, + "tokens_seen": 1179759616 + }, + { + "epoch": 3.09, + "learning_rate": 0.0003245235707121364, + "loss": 2.6044, + "theoretical_loss": 3.5930730679518335, + "tokens_seen": 1179825152 + }, + { + "epoch": 3.09, + "learning_rate": 0.0003245135406218656, + "loss": 2.4911, + "theoretical_loss": 3.5930546508778196, + "tokens_seen": 1179890688 + }, + { + "epoch": 3.09, + "learning_rate": 0.0003245035105315948, + "loss": 2.5624, + "theoretical_loss": 3.593036235113149, + "tokens_seen": 1179956224 + }, + { + "epoch": 3.09, + "learning_rate": 0.000324493480441324, + "loss": 2.6682, + "theoretical_loss": 3.5930178206576553, + "tokens_seen": 1180021760 + }, + { + "epoch": 3.09, + "learning_rate": 0.00032448345035105314, + "loss": 2.8837, + "theoretical_loss": 3.592999407511172, + "tokens_seen": 1180087296 + }, + { + "epoch": 3.09, + "learning_rate": 0.0003244734202607824, + "loss": 2.643, + "theoretical_loss": 3.592980995673535, + "tokens_seen": 1180152832 + }, + { + "epoch": 3.09, + "learning_rate": 0.0003244633901705115, + "loss": 2.408, + "theoretical_loss": 3.5929625851445772, + "tokens_seen": 1180218368 + }, + { + "epoch": 3.09, + "learning_rate": 0.00032445336008024074, + "loss": 2.7895, + "theoretical_loss": 3.592944175924133, + "tokens_seen": 1180283904 + }, + { + "epoch": 3.09, + "learning_rate": 0.0003244433299899699, + "loss": 2.629, + "theoretical_loss": 3.592925768012038, + "tokens_seen": 1180349440 + }, + { + "epoch": 3.09, + "learning_rate": 0.0003244332998996991, + "loss": 2.3755, + "theoretical_loss": 3.592907361408125, + "tokens_seen": 1180414976 + }, + { + "epoch": 3.09, + "learning_rate": 0.0003244232698094283, + "loss": 2.7623, + "theoretical_loss": 3.5928889561122292, + "tokens_seen": 1180480512 + }, + { + "epoch": 3.09, + "learning_rate": 0.00032441323971915746, + "loss": 2.7398, + "theoretical_loss": 3.592870552124185, + "tokens_seen": 1180546048 + }, + { + "epoch": 3.09, + "learning_rate": 0.00032440320962888664, + "loss": 2.6753, + "theoretical_loss": 3.592852149443827, + "tokens_seen": 1180611584 + }, + { + "epoch": 3.09, + "learning_rate": 0.0003243931795386159, + "loss": 2.5979, + "theoretical_loss": 3.5928337480709898, + "tokens_seen": 1180677120 + }, + { + "epoch": 3.09, + "learning_rate": 0.000324383149448345, + "loss": 2.5014, + "theoretical_loss": 3.592815348005507, + "tokens_seen": 1180742656 + }, + { + "epoch": 3.09, + "learning_rate": 0.00032437311935807424, + "loss": 2.2792, + "theoretical_loss": 3.592796949247214, + "tokens_seen": 1180808192 + }, + { + "epoch": 3.09, + "learning_rate": 0.00032436308926780337, + "loss": 2.755, + "theoretical_loss": 3.592778551795945, + "tokens_seen": 1180873728 + }, + { + "epoch": 3.09, + "learning_rate": 0.0003243530591775326, + "loss": 2.6092, + "theoretical_loss": 3.5927601556515354, + "tokens_seen": 1180939264 + }, + { + "epoch": 3.09, + "learning_rate": 0.0003243430290872618, + "loss": 2.592, + "theoretical_loss": 3.5927417608138192, + "tokens_seen": 1181004800 + }, + { + "epoch": 3.09, + "learning_rate": 0.00032433299899699096, + "loss": 2.7324, + "theoretical_loss": 3.5927233672826304, + "tokens_seen": 1181070336 + }, + { + "epoch": 3.09, + "learning_rate": 0.0003243229689067202, + "loss": 2.6515, + "theoretical_loss": 3.5927049750578055, + "tokens_seen": 1181135872 + }, + { + "epoch": 3.09, + "learning_rate": 0.0003243129388164494, + "loss": 2.7052, + "theoretical_loss": 3.5926865841391775, + "tokens_seen": 1181201408 + }, + { + "epoch": 3.09, + "learning_rate": 0.00032430290872617856, + "loss": 2.8725, + "theoretical_loss": 3.592668194526582, + "tokens_seen": 1181266944 + }, + { + "epoch": 3.09, + "objective/train/docs_used": 1282424, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.2769789695739746, + "objective/train/theoretical_loss": 3.5926635973274816, + "objective/train/tokens_used": 1187029472, + "theoretical_loss": 3.5926635973274816, + "tokens_seen": 1181283328 + }, + { + "epoch": 3.09, + "learning_rate": 0.00032429287863590774, + "loss": 2.4443, + "theoretical_loss": 3.592649806219854, + "tokens_seen": 1181332480 + }, + { + "epoch": 3.09, + "learning_rate": 0.0003242828485456369, + "loss": 2.4296, + "theoretical_loss": 3.5926314192188276, + "tokens_seen": 1181398016 + }, + { + "epoch": 3.09, + "learning_rate": 0.0003242728184553661, + "loss": 2.5088, + "theoretical_loss": 3.592613033523339, + "tokens_seen": 1181463552 + }, + { + "epoch": 3.09, + "learning_rate": 0.00032426278836509534, + "loss": 2.3114, + "theoretical_loss": 3.592594649133221, + "tokens_seen": 1181529088 + }, + { + "epoch": 3.09, + "learning_rate": 0.00032425275827482447, + "loss": 2.7753, + "theoretical_loss": 3.59257626604831, + "tokens_seen": 1181594624 + }, + { + "epoch": 3.09, + "learning_rate": 0.0003242427281845537, + "loss": 2.4749, + "theoretical_loss": 3.592557884268441, + "tokens_seen": 1181660160 + }, + { + "epoch": 3.09, + "learning_rate": 0.00032423269809428283, + "loss": 2.7379, + "theoretical_loss": 3.592539503793448, + "tokens_seen": 1181725696 + }, + { + "epoch": 3.09, + "learning_rate": 0.00032422266800401206, + "loss": 2.6699, + "theoretical_loss": 3.5925211246231674, + "tokens_seen": 1181791232 + }, + { + "epoch": 3.09, + "learning_rate": 0.00032421263791374125, + "loss": 2.4808, + "theoretical_loss": 3.5925027467574333, + "tokens_seen": 1181856768 + }, + { + "epoch": 3.09, + "learning_rate": 0.0003242026078234704, + "loss": 2.5653, + "theoretical_loss": 3.5924843701960807, + "tokens_seen": 1181922304 + }, + { + "epoch": 3.09, + "learning_rate": 0.0003241925777331996, + "loss": 2.6213, + "theoretical_loss": 3.5924659949389453, + "tokens_seen": 1181987840 + }, + { + "epoch": 3.09, + "learning_rate": 0.00032418254764292884, + "loss": 2.4843, + "theoretical_loss": 3.592447620985862, + "tokens_seen": 1182053376 + }, + { + "epoch": 3.09, + "learning_rate": 0.00032417251755265797, + "loss": 2.5115, + "theoretical_loss": 3.592429248336665, + "tokens_seen": 1182118912 + }, + { + "epoch": 3.09, + "learning_rate": 0.0003241624874623872, + "loss": 2.8637, + "theoretical_loss": 3.592410876991191, + "tokens_seen": 1182184448 + }, + { + "epoch": 3.09, + "learning_rate": 0.00032415245737211633, + "loss": 2.6917, + "theoretical_loss": 3.5923925069492744, + "tokens_seen": 1182249984 + }, + { + "epoch": 3.09, + "learning_rate": 0.00032414242728184557, + "loss": 2.5962, + "theoretical_loss": 3.592374138210751, + "tokens_seen": 1182315520 + }, + { + "epoch": 3.09, + "learning_rate": 0.00032413239719157475, + "loss": 2.2916, + "theoretical_loss": 3.5923557707754554, + "tokens_seen": 1182381056 + }, + { + "epoch": 3.09, + "learning_rate": 0.00032412236710130393, + "loss": 2.2378, + "theoretical_loss": 3.5923374046432226, + "tokens_seen": 1182446592 + }, + { + "epoch": 3.09, + "learning_rate": 0.0003241123370110331, + "loss": 2.6255, + "theoretical_loss": 3.5923190398138893, + "tokens_seen": 1182512128 + }, + { + "epoch": 3.09, + "learning_rate": 0.0003241023069207623, + "loss": 2.641, + "theoretical_loss": 3.59230067628729, + "tokens_seen": 1182577664 + }, + { + "epoch": 3.09, + "learning_rate": 0.00032409227683049147, + "loss": 2.6365, + "theoretical_loss": 3.59228231406326, + "tokens_seen": 1182643200 + }, + { + "epoch": 3.09, + "learning_rate": 0.0003240822467402207, + "loss": 2.683, + "theoretical_loss": 3.5922639531416354, + "tokens_seen": 1182708736 + }, + { + "epoch": 3.09, + "learning_rate": 0.00032407221664994984, + "loss": 2.7049, + "theoretical_loss": 3.592245593522251, + "tokens_seen": 1182774272 + }, + { + "epoch": 3.09, + "learning_rate": 0.00032406218655967907, + "loss": 2.6362, + "theoretical_loss": 3.592227235204942, + "tokens_seen": 1182839808 + }, + { + "epoch": 3.09, + "learning_rate": 0.0003240521564694082, + "loss": 2.5488, + "theoretical_loss": 3.5922088781895454, + "tokens_seen": 1182905344 + }, + { + "epoch": 3.09, + "objective/train/docs_used": 1282424, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5676214694976807, + "objective/train/theoretical_loss": 3.5922042891391004, + "objective/train/tokens_used": 1187029472, + "theoretical_loss": 3.5922042891391004, + "tokens_seen": 1182921728 + }, + { + "epoch": 3.09, + "learning_rate": 0.00032404212637913743, + "loss": 2.6056, + "theoretical_loss": 3.592190522475895, + "tokens_seen": 1182970880 + }, + { + "epoch": 3.09, + "learning_rate": 0.0003240320962888666, + "loss": 2.6205, + "theoretical_loss": 3.5921721680638274, + "tokens_seen": 1183036416 + }, + { + "epoch": 3.09, + "learning_rate": 0.0003240220661985958, + "loss": 2.6659, + "theoretical_loss": 3.5921538149531784, + "tokens_seen": 1183101952 + }, + { + "epoch": 3.09, + "learning_rate": 0.000324012036108325, + "loss": 2.4522, + "theoretical_loss": 3.5921354631437827, + "tokens_seen": 1183167488 + }, + { + "epoch": 3.09, + "learning_rate": 0.0003240020060180542, + "loss": 2.6169, + "theoretical_loss": 3.5921171126354765, + "tokens_seen": 1183233024 + }, + { + "epoch": 3.09, + "learning_rate": 0.00032399197592778334, + "loss": 2.527, + "theoretical_loss": 3.592098763428096, + "tokens_seen": 1183298560 + }, + { + "epoch": 3.09, + "learning_rate": 0.0003239819458375126, + "loss": 2.5272, + "theoretical_loss": 3.5920804155214756, + "tokens_seen": 1183364096 + }, + { + "epoch": 3.09, + "learning_rate": 0.0003239719157472417, + "loss": 2.5678, + "theoretical_loss": 3.5920620689154523, + "tokens_seen": 1183429632 + }, + { + "epoch": 3.09, + "learning_rate": 0.00032396188565697094, + "loss": 2.5987, + "theoretical_loss": 3.5920437236098612, + "tokens_seen": 1183495168 + }, + { + "epoch": 3.09, + "learning_rate": 0.0003239518555667001, + "loss": 2.8055, + "theoretical_loss": 3.592025379604539, + "tokens_seen": 1183560704 + }, + { + "epoch": 3.09, + "learning_rate": 0.0003239418254764293, + "loss": 2.6494, + "theoretical_loss": 3.5920070368993207, + "tokens_seen": 1183626240 + }, + { + "epoch": 3.09, + "learning_rate": 0.0003239317953861585, + "loss": 2.6893, + "theoretical_loss": 3.5919886954940425, + "tokens_seen": 1183691776 + }, + { + "epoch": 3.09, + "learning_rate": 0.00032392176529588766, + "loss": 2.1192, + "theoretical_loss": 3.59197035538854, + "tokens_seen": 1183757312 + }, + { + "epoch": 3.09, + "learning_rate": 0.00032391173520561684, + "loss": 2.5386, + "theoretical_loss": 3.591952016582649, + "tokens_seen": 1183822848 + }, + { + "epoch": 3.09, + "learning_rate": 0.0003239017051153461, + "loss": 2.7836, + "theoretical_loss": 3.591933679076207, + "tokens_seen": 1183888384 + }, + { + "epoch": 3.09, + "learning_rate": 0.0003238916750250752, + "loss": 2.5445, + "theoretical_loss": 3.5919153428690476, + "tokens_seen": 1183953920 + }, + { + "epoch": 3.09, + "learning_rate": 0.00032388164493480444, + "loss": 2.7289, + "theoretical_loss": 3.591897007961008, + "tokens_seen": 1184019456 + }, + { + "epoch": 3.09, + "learning_rate": 0.00032387161484453357, + "loss": 2.8839, + "theoretical_loss": 3.5918786743519258, + "tokens_seen": 1184084992 + }, + { + "epoch": 3.09, + "learning_rate": 0.0003238615847542628, + "loss": 2.4107, + "theoretical_loss": 3.5918603420416346, + "tokens_seen": 1184150528 + }, + { + "epoch": 3.09, + "learning_rate": 0.000323851554663992, + "loss": 2.5578, + "theoretical_loss": 3.5918420110299714, + "tokens_seen": 1184216064 + }, + { + "epoch": 3.09, + "learning_rate": 0.00032384152457372116, + "loss": 2.6188, + "theoretical_loss": 3.5918236813167725, + "tokens_seen": 1184281600 + }, + { + "epoch": 3.09, + "learning_rate": 0.00032383149448345034, + "loss": 2.7958, + "theoretical_loss": 3.591805352901874, + "tokens_seen": 1184347136 + }, + { + "epoch": 3.09, + "learning_rate": 0.0003238214643931796, + "loss": 2.8208, + "theoretical_loss": 3.5917870257851128, + "tokens_seen": 1184412672 + }, + { + "epoch": 3.09, + "learning_rate": 0.0003238114343029087, + "loss": 2.4828, + "theoretical_loss": 3.591768699966324, + "tokens_seen": 1184478208 + }, + { + "epoch": 3.09, + "learning_rate": 0.00032380140421263794, + "loss": 2.5075, + "theoretical_loss": 3.5917503754453444, + "tokens_seen": 1184543744 + }, + { + "epoch": 3.09, + "objective/train/docs_used": 1282424, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8725781440734863, + "objective/train/theoretical_loss": 3.5917457945178626, + "objective/train/tokens_used": 1187029472, + "theoretical_loss": 3.5917457945178626, + "tokens_seen": 1184560128 + }, + { + "epoch": 3.09, + "learning_rate": 0.00032379137412236707, + "loss": 2.8068, + "theoretical_loss": 3.59173205222201, + "tokens_seen": 1184609280 + }, + { + "epoch": 3.09, + "learning_rate": 0.0003237813440320963, + "loss": 2.456, + "theoretical_loss": 3.591713730296158, + "tokens_seen": 1184674816 + }, + { + "epoch": 3.09, + "learning_rate": 0.0003237713139418255, + "loss": 2.7284, + "theoretical_loss": 3.591695409667624, + "tokens_seen": 1184740352 + }, + { + "epoch": 3.09, + "learning_rate": 0.00032376128385155467, + "loss": 2.3995, + "theoretical_loss": 3.591677090336244, + "tokens_seen": 1184805888 + }, + { + "epoch": 3.09, + "learning_rate": 0.00032375125376128385, + "loss": 2.7092, + "theoretical_loss": 3.591658772301855, + "tokens_seen": 1184871424 + }, + { + "epoch": 3.09, + "learning_rate": 0.00032374122367101303, + "loss": 2.6974, + "theoretical_loss": 3.591640455564294, + "tokens_seen": 1184936960 + }, + { + "epoch": 3.09, + "learning_rate": 0.0003237311935807422, + "loss": 2.286, + "theoretical_loss": 3.5916221401233965, + "tokens_seen": 1185002496 + }, + { + "epoch": 3.09, + "learning_rate": 0.00032372116349047145, + "loss": 2.3438, + "theoretical_loss": 3.591603825978999, + "tokens_seen": 1185068032 + }, + { + "epoch": 3.09, + "learning_rate": 0.00032371113340020057, + "loss": 2.4572, + "theoretical_loss": 3.591585513130939, + "tokens_seen": 1185133568 + }, + { + "epoch": 3.09, + "learning_rate": 0.0003237011033099298, + "loss": 2.7635, + "theoretical_loss": 3.5915672015790525, + "tokens_seen": 1185199104 + }, + { + "epoch": 3.09, + "learning_rate": 0.00032369107321965893, + "loss": 2.781, + "theoretical_loss": 3.5915488913231757, + "tokens_seen": 1185264640 + }, + { + "epoch": 3.09, + "learning_rate": 0.00032368104312938817, + "loss": 2.649, + "theoretical_loss": 3.5915305823631454, + "tokens_seen": 1185330176 + }, + { + "epoch": 3.09, + "learning_rate": 0.00032367101303911735, + "loss": 2.5419, + "theoretical_loss": 3.5915122746987986, + "tokens_seen": 1185395712 + }, + { + "epoch": 3.09, + "learning_rate": 0.00032366098294884653, + "loss": 2.563, + "theoretical_loss": 3.5914939683299716, + "tokens_seen": 1185461248 + }, + { + "epoch": 3.09, + "learning_rate": 0.0003236509528585757, + "loss": 2.7375, + "theoretical_loss": 3.591475663256502, + "tokens_seen": 1185526784 + }, + { + "epoch": 3.09, + "learning_rate": 0.00032364092276830495, + "loss": 2.3776, + "theoretical_loss": 3.591457359478225, + "tokens_seen": 1185592320 + }, + { + "epoch": 3.09, + "learning_rate": 0.0003236308926780341, + "loss": 2.5519, + "theoretical_loss": 3.591439056994979, + "tokens_seen": 1185657856 + }, + { + "epoch": 3.09, + "learning_rate": 0.0003236208625877633, + "loss": 2.798, + "theoretical_loss": 3.5914207558066, + "tokens_seen": 1185723392 + }, + { + "epoch": 3.09, + "learning_rate": 0.00032361083249749244, + "loss": 2.7447, + "theoretical_loss": 3.591402455912924, + "tokens_seen": 1185788928 + }, + { + "epoch": 3.09, + "learning_rate": 0.0003236008024072217, + "loss": 2.5904, + "theoretical_loss": 3.5913841573137892, + "tokens_seen": 1185854464 + }, + { + "epoch": 3.09, + "learning_rate": 0.00032359077231695085, + "loss": 2.6773, + "theoretical_loss": 3.5913658600090326, + "tokens_seen": 1185920000 + }, + { + "epoch": 3.09, + "learning_rate": 0.00032358074222668004, + "loss": 2.6649, + "theoretical_loss": 3.5913475639984895, + "tokens_seen": 1185985536 + }, + { + "epoch": 3.09, + "learning_rate": 0.00032357071213640927, + "loss": 2.5266, + "theoretical_loss": 3.5913292692819985, + "tokens_seen": 1186051072 + }, + { + "epoch": 3.09, + "learning_rate": 0.0003235606820461384, + "loss": 2.7652, + "theoretical_loss": 3.591310975859396, + "tokens_seen": 1186116608 + }, + { + "epoch": 3.09, + "learning_rate": 0.00032355065195586763, + "loss": 2.641, + "theoretical_loss": 3.5912926837305186, + "tokens_seen": 1186182144 + }, + { + "epoch": 3.09, + "objective/train/docs_used": 1282424, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.350708246231079, + "objective/train/theoretical_loss": 3.5912881109004244, + "objective/train/tokens_used": 1187029472, + "theoretical_loss": 3.5912881109004244, + "tokens_seen": 1186198528 + }, + { + "epoch": 3.09, + "learning_rate": 0.0003235406218655968, + "loss": 2.4416, + "theoretical_loss": 3.591274392895204, + "tokens_seen": 1186247680 + }, + { + "epoch": 3.09, + "learning_rate": 0.000323530591775326, + "loss": 2.4279, + "theoretical_loss": 3.5912561033532886, + "tokens_seen": 1186313216 + }, + { + "epoch": 3.09, + "learning_rate": 0.0003235205616850552, + "loss": 2.6713, + "theoretical_loss": 3.59123781510461, + "tokens_seen": 1186378752 + }, + { + "epoch": 3.09, + "learning_rate": 0.0003235105315947844, + "loss": 2.8526, + "theoretical_loss": 3.591219528149005, + "tokens_seen": 1186444288 + }, + { + "epoch": 3.09, + "learning_rate": 0.00032350050150451354, + "loss": 2.4494, + "theoretical_loss": 3.5912012424863113, + "tokens_seen": 1186509824 + }, + { + "epoch": 3.09, + "learning_rate": 0.0003234904714142428, + "loss": 2.661, + "theoretical_loss": 3.5911829581163657, + "tokens_seen": 1186575360 + }, + { + "epoch": 3.09, + "learning_rate": 0.0003234804413239719, + "loss": 2.5418, + "theoretical_loss": 3.591164675039005, + "tokens_seen": 1186640896 + }, + { + "epoch": 3.09, + "learning_rate": 0.00032347041123370114, + "loss": 2.596, + "theoretical_loss": 3.5911463932540677, + "tokens_seen": 1186706432 + }, + { + "epoch": 3.09, + "learning_rate": 0.0003234603811434303, + "loss": 2.421, + "theoretical_loss": 3.5911281127613894, + "tokens_seen": 1186771968 + }, + { + "epoch": 3.09, + "learning_rate": 0.0003234503510531595, + "loss": 2.4505, + "theoretical_loss": 3.591109833560809, + "tokens_seen": 1186837504 + }, + { + "epoch": 3.09, + "learning_rate": 0.0003234403209628887, + "loss": 2.5962, + "theoretical_loss": 3.591091555652162, + "tokens_seen": 1186903040 + }, + { + "epoch": 3.09, + "learning_rate": 0.00032343029087261786, + "loss": 2.823, + "theoretical_loss": 3.591073279035288, + "tokens_seen": 1186968576 + }, + { + "epoch": 3.09, + "learning_rate": 0.00032342026078234704, + "loss": 2.4926, + "theoretical_loss": 3.591055003710023, + "tokens_seen": 1187034112 + }, + { + "epoch": 4.0, + "learning_rate": 0.0003234102306920763, + "loss": 3.3894, + "theoretical_loss": 3.5910324468863224, + "tokens_seen": 1187115008 + }, + { + "epoch": 4.0, + "learning_rate": 0.0003234002006018054, + "loss": 2.6997, + "theoretical_loss": 3.5910141744464092, + "tokens_seen": 1187180544 + }, + { + "epoch": 4.0, + "learning_rate": 0.00032339017051153464, + "loss": 2.7738, + "theoretical_loss": 3.5909959032975793, + "tokens_seen": 1187246080 + }, + { + "epoch": 4.0, + "learning_rate": 0.00032338014042126377, + "loss": 2.723, + "theoretical_loss": 3.590977633439671, + "tokens_seen": 1187311616 + }, + { + "epoch": 4.0, + "learning_rate": 0.000323370110330993, + "loss": 2.6097, + "theoretical_loss": 3.5909593648725204, + "tokens_seen": 1187377152 + }, + { + "epoch": 4.0, + "learning_rate": 0.0003233600802407222, + "loss": 2.744, + "theoretical_loss": 3.5909410975959664, + "tokens_seen": 1187442688 + }, + { + "epoch": 4.0, + "learning_rate": 0.00032335005015045136, + "loss": 2.8156, + "theoretical_loss": 3.590922831609846, + "tokens_seen": 1187508224 + }, + { + "epoch": 4.0, + "learning_rate": 0.00032334002006018054, + "loss": 2.8714, + "theoretical_loss": 3.5909045669139967, + "tokens_seen": 1187573760 + }, + { + "epoch": 4.0, + "learning_rate": 0.0003233299899699098, + "loss": 2.5544, + "theoretical_loss": 3.590886303508257, + "tokens_seen": 1187639296 + }, + { + "epoch": 4.0, + "learning_rate": 0.0003233199598796389, + "loss": 2.7742, + "theoretical_loss": 3.5908680413924636, + "tokens_seen": 1187704832 + }, + { + "epoch": 4.0, + "learning_rate": 0.00032330992978936814, + "loss": 2.8224, + "theoretical_loss": 3.5908497805664545, + "tokens_seen": 1187770368 + }, + { + "epoch": 4.0, + "objective/train/docs_used": 1347233, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8168835639953613, + "objective/train/theoretical_loss": 3.5908315210300685, + "objective/train/tokens_used": 1208295904, + "theoretical_loss": 3.5908315210300685, + "tokens_seen": 1187835904 + }, + { + "epoch": 4.0, + "learning_rate": 0.00032329989969909727, + "loss": 2.6189, + "theoretical_loss": 3.5908315210300685, + "tokens_seen": 1187835904 + }, + { + "epoch": 4.0, + "learning_rate": 0.0003232898696088265, + "loss": 2.699, + "theoretical_loss": 3.5908132627831417, + "tokens_seen": 1187901440 + }, + { + "epoch": 4.0, + "learning_rate": 0.0003232798395185557, + "loss": 2.7227, + "theoretical_loss": 3.5907950058255125, + "tokens_seen": 1187966976 + }, + { + "epoch": 4.0, + "learning_rate": 0.00032326980942828487, + "loss": 2.7502, + "theoretical_loss": 3.590776750157019, + "tokens_seen": 1188032512 + }, + { + "epoch": 4.0, + "learning_rate": 0.00032325977933801405, + "loss": 2.6845, + "theoretical_loss": 3.5907584957774996, + "tokens_seen": 1188098048 + }, + { + "epoch": 4.0, + "learning_rate": 0.00032324974924774323, + "loss": 2.8323, + "theoretical_loss": 3.590740242686791, + "tokens_seen": 1188163584 + }, + { + "epoch": 4.0, + "learning_rate": 0.0003232397191574724, + "loss": 2.6911, + "theoretical_loss": 3.590721990884732, + "tokens_seen": 1188229120 + }, + { + "epoch": 4.0, + "learning_rate": 0.00032322968906720165, + "loss": 2.8486, + "theoretical_loss": 3.5907037403711604, + "tokens_seen": 1188294656 + }, + { + "epoch": 4.0, + "learning_rate": 0.00032321965897693077, + "loss": 2.7744, + "theoretical_loss": 3.590685491145914, + "tokens_seen": 1188360192 + }, + { + "epoch": 4.0, + "learning_rate": 0.00032320962888666, + "loss": 2.7593, + "theoretical_loss": 3.5906672432088302, + "tokens_seen": 1188425728 + }, + { + "epoch": 4.0, + "learning_rate": 0.00032319959879638913, + "loss": 2.7644, + "theoretical_loss": 3.5906489965597483, + "tokens_seen": 1188491264 + }, + { + "epoch": 4.0, + "learning_rate": 0.00032318956870611837, + "loss": 2.7176, + "theoretical_loss": 3.5906307511985056, + "tokens_seen": 1188556800 + }, + { + "epoch": 4.0, + "learning_rate": 0.00032317953861584755, + "loss": 2.8788, + "theoretical_loss": 3.5906125071249404, + "tokens_seen": 1188622336 + }, + { + "epoch": 4.0, + "learning_rate": 0.00032316950852557673, + "loss": 2.7639, + "theoretical_loss": 3.5905942643388906, + "tokens_seen": 1188687872 + }, + { + "epoch": 4.0, + "learning_rate": 0.0003231594784353059, + "loss": 2.5787, + "theoretical_loss": 3.590576022840195, + "tokens_seen": 1188753408 + }, + { + "epoch": 4.0, + "learning_rate": 0.00032314944834503515, + "loss": 2.7243, + "theoretical_loss": 3.590557782628691, + "tokens_seen": 1188818944 + }, + { + "epoch": 4.0, + "learning_rate": 0.0003231394182547643, + "loss": 2.7841, + "theoretical_loss": 3.5905395437042174, + "tokens_seen": 1188884480 + }, + { + "epoch": 4.0, + "learning_rate": 0.0003231293881644935, + "loss": 2.5921, + "theoretical_loss": 3.5905213060666124, + "tokens_seen": 1188950016 + }, + { + "epoch": 4.0, + "learning_rate": 0.00032311935807422264, + "loss": 2.8621, + "theoretical_loss": 3.590503069715714, + "tokens_seen": 1189015552 + }, + { + "epoch": 4.0, + "learning_rate": 0.0003231093279839519, + "loss": 2.7728, + "theoretical_loss": 3.5904848346513605, + "tokens_seen": 1189081088 + }, + { + "epoch": 4.0, + "learning_rate": 0.00032309929789368105, + "loss": 2.8418, + "theoretical_loss": 3.5904666008733903, + "tokens_seen": 1189146624 + }, + { + "epoch": 4.0, + "learning_rate": 0.00032308926780341024, + "loss": 2.7546, + "theoretical_loss": 3.590448368381642, + "tokens_seen": 1189212160 + }, + { + "epoch": 4.0, + "learning_rate": 0.0003230792377131394, + "loss": 2.4472, + "theoretical_loss": 3.5904301371759533, + "tokens_seen": 1189277696 + }, + { + "epoch": 4.0, + "learning_rate": 0.0003230692076228686, + "loss": 2.7317, + "theoretical_loss": 3.590411907256164, + "tokens_seen": 1189343232 + }, + { + "epoch": 4.0, + "learning_rate": 0.0003230591775325978, + "loss": 2.7694, + "theoretical_loss": 3.590393678622111, + "tokens_seen": 1189408768 + }, + { + "epoch": 4.0, + "objective/train/docs_used": 1352239, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.0367116928100586, + "objective/train/theoretical_loss": 3.5903754512736334, + "objective/train/tokens_used": 1209934304, + "theoretical_loss": 3.5903754512736334, + "tokens_seen": 1189474304 + }, + { + "epoch": 4.0, + "learning_rate": 0.000323049147442327, + "loss": 2.7382, + "theoretical_loss": 3.5903754512736334, + "tokens_seen": 1189474304 + }, + { + "epoch": 4.0, + "learning_rate": 0.00032303911735205614, + "loss": 2.4576, + "theoretical_loss": 3.59035722521057, + "tokens_seen": 1189539840 + }, + { + "epoch": 4.0, + "learning_rate": 0.0003230290872617854, + "loss": 2.648, + "theoretical_loss": 3.5903390004327593, + "tokens_seen": 1189605376 + }, + { + "epoch": 4.0, + "learning_rate": 0.0003230190571715145, + "loss": 2.7522, + "theoretical_loss": 3.5903207769400396, + "tokens_seen": 1189670912 + }, + { + "epoch": 4.0, + "learning_rate": 0.00032300902708124374, + "loss": 2.87, + "theoretical_loss": 3.5903025547322494, + "tokens_seen": 1189736448 + }, + { + "epoch": 4.0, + "learning_rate": 0.0003229989969909729, + "loss": 2.7883, + "theoretical_loss": 3.590284333809228, + "tokens_seen": 1189801984 + }, + { + "epoch": 4.0, + "learning_rate": 0.0003229889669007021, + "loss": 2.8014, + "theoretical_loss": 3.590266114170813, + "tokens_seen": 1189867520 + }, + { + "epoch": 4.0, + "learning_rate": 0.0003229789368104313, + "loss": 2.9751, + "theoretical_loss": 3.5902478958168436, + "tokens_seen": 1189933056 + }, + { + "epoch": 4.0, + "learning_rate": 0.0003229689067201605, + "loss": 2.8034, + "theoretical_loss": 3.590229678747159, + "tokens_seen": 1189998592 + }, + { + "epoch": 4.0, + "learning_rate": 0.00032295887662988964, + "loss": 2.6914, + "theoretical_loss": 3.590211462961597, + "tokens_seen": 1190064128 + }, + { + "epoch": 4.0, + "learning_rate": 0.0003229488465396189, + "loss": 2.7211, + "theoretical_loss": 3.5901932484599977, + "tokens_seen": 1190129664 + }, + { + "epoch": 4.0, + "learning_rate": 0.000322938816449348, + "loss": 2.7461, + "theoretical_loss": 3.590175035242199, + "tokens_seen": 1190195200 + }, + { + "epoch": 4.0, + "learning_rate": 0.00032292878635907724, + "loss": 2.6821, + "theoretical_loss": 3.5901568233080394, + "tokens_seen": 1190260736 + }, + { + "epoch": 4.0, + "learning_rate": 0.0003229187562688064, + "loss": 2.7034, + "theoretical_loss": 3.5901386126573582, + "tokens_seen": 1190326272 + }, + { + "epoch": 4.0, + "learning_rate": 0.0003229087261785356, + "loss": 2.6984, + "theoretical_loss": 3.590120403289994, + "tokens_seen": 1190391808 + }, + { + "epoch": 4.0, + "learning_rate": 0.0003228986960882648, + "loss": 2.8506, + "theoretical_loss": 3.5901021952057866, + "tokens_seen": 1190457344 + }, + { + "epoch": 4.0, + "learning_rate": 0.00032288866599799397, + "loss": 2.8324, + "theoretical_loss": 3.590083988404574, + "tokens_seen": 1190522880 + }, + { + "epoch": 4.0, + "learning_rate": 0.00032287863590772315, + "loss": 2.7436, + "theoretical_loss": 3.590065782886196, + "tokens_seen": 1190588416 + }, + { + "epoch": 4.0, + "learning_rate": 0.0003228686058174524, + "loss": 2.669, + "theoretical_loss": 3.590047578650491, + "tokens_seen": 1190653952 + }, + { + "epoch": 4.0, + "learning_rate": 0.0003228585757271815, + "loss": 2.7477, + "theoretical_loss": 3.5900293756972976, + "tokens_seen": 1190719488 + }, + { + "epoch": 4.0, + "learning_rate": 0.00032284854563691074, + "loss": 2.7421, + "theoretical_loss": 3.590011174026456, + "tokens_seen": 1190785024 + }, + { + "epoch": 4.0, + "learning_rate": 0.0003228385155466399, + "loss": 2.7889, + "theoretical_loss": 3.5899929736378047, + "tokens_seen": 1190850560 + }, + { + "epoch": 4.0, + "learning_rate": 0.0003228284854563691, + "loss": 2.8669, + "theoretical_loss": 3.589974774531182, + "tokens_seen": 1190916096 + }, + { + "epoch": 4.0, + "learning_rate": 0.00032281845536609834, + "loss": 2.7807, + "theoretical_loss": 3.5899565767064288, + "tokens_seen": 1190981632 + }, + { + "epoch": 4.0, + "learning_rate": 0.00032280842527582747, + "loss": 2.707, + "theoretical_loss": 3.5899383801633835, + "tokens_seen": 1191047168 + }, + { + "epoch": 4.0, + "objective/train/docs_used": 1357258, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.0113937854766846, + "objective/train/theoretical_loss": 3.5899201849018842, + "objective/train/tokens_used": 1211572704, + "theoretical_loss": 3.5899201849018842, + "tokens_seen": 1191112704 + }, + { + "epoch": 4.0, + "learning_rate": 0.0003227983951855567, + "loss": 2.8706, + "theoretical_loss": 3.5899201849018842, + "tokens_seen": 1191112704 + }, + { + "epoch": 4.0, + "learning_rate": 0.0003227883650952859, + "loss": 2.815, + "theoretical_loss": 3.5899019909217724, + "tokens_seen": 1191178240 + }, + { + "epoch": 4.0, + "learning_rate": 0.00032277833500501507, + "loss": 2.82, + "theoretical_loss": 3.589883798222885, + "tokens_seen": 1191243776 + }, + { + "epoch": 4.0, + "learning_rate": 0.00032276830491474425, + "loss": 2.4651, + "theoretical_loss": 3.5898656068050627, + "tokens_seen": 1191309312 + }, + { + "epoch": 4.0, + "learning_rate": 0.00032275827482447343, + "loss": 2.6, + "theoretical_loss": 3.589847416668145, + "tokens_seen": 1191374848 + }, + { + "epoch": 4.0, + "learning_rate": 0.0003227482447342026, + "loss": 2.7682, + "theoretical_loss": 3.58982922781197, + "tokens_seen": 1191440384 + }, + { + "epoch": 4.0, + "learning_rate": 0.00032273821464393185, + "loss": 2.7798, + "theoretical_loss": 3.5898110402363788, + "tokens_seen": 1191505920 + }, + { + "epoch": 4.0, + "learning_rate": 0.00032272818455366097, + "loss": 2.6923, + "theoretical_loss": 3.5897928539412094, + "tokens_seen": 1191571456 + }, + { + "epoch": 4.0, + "learning_rate": 0.0003227181544633902, + "loss": 2.6961, + "theoretical_loss": 3.5897746689263017, + "tokens_seen": 1191636992 + }, + { + "epoch": 4.0, + "learning_rate": 0.00032270812437311933, + "loss": 2.7842, + "theoretical_loss": 3.5897564851914954, + "tokens_seen": 1191702528 + }, + { + "epoch": 4.0, + "learning_rate": 0.00032269809428284857, + "loss": 2.6957, + "theoretical_loss": 3.589738302736629, + "tokens_seen": 1191768064 + }, + { + "epoch": 4.0, + "learning_rate": 0.00032268806419257775, + "loss": 2.8474, + "theoretical_loss": 3.5897201215615437, + "tokens_seen": 1191833600 + }, + { + "epoch": 4.0, + "learning_rate": 0.00032267803410230693, + "loss": 2.754, + "theoretical_loss": 3.589701941666078, + "tokens_seen": 1191899136 + }, + { + "epoch": 4.0, + "learning_rate": 0.0003226680040120361, + "loss": 2.6007, + "theoretical_loss": 3.589683763050071, + "tokens_seen": 1191964672 + }, + { + "epoch": 4.0, + "learning_rate": 0.00032265797392176535, + "loss": 2.9095, + "theoretical_loss": 3.5896655857133637, + "tokens_seen": 1192030208 + }, + { + "epoch": 4.0, + "learning_rate": 0.0003226479438314945, + "loss": 2.7656, + "theoretical_loss": 3.589647409655795, + "tokens_seen": 1192095744 + }, + { + "epoch": 4.0, + "learning_rate": 0.0003226379137412237, + "loss": 2.7206, + "theoretical_loss": 3.5896292348772043, + "tokens_seen": 1192161280 + }, + { + "epoch": 4.0, + "learning_rate": 0.00032262788365095284, + "loss": 2.9005, + "theoretical_loss": 3.5896110613774317, + "tokens_seen": 1192226816 + }, + { + "epoch": 4.0, + "learning_rate": 0.0003226178535606821, + "loss": 2.8052, + "theoretical_loss": 3.5895928891563167, + "tokens_seen": 1192292352 + }, + { + "epoch": 4.0, + "learning_rate": 0.00032260782347041125, + "loss": 2.7781, + "theoretical_loss": 3.589574718213699, + "tokens_seen": 1192357888 + }, + { + "epoch": 4.0, + "learning_rate": 0.00032259779338014044, + "loss": 2.7395, + "theoretical_loss": 3.589556548549419, + "tokens_seen": 1192423424 + }, + { + "epoch": 4.0, + "learning_rate": 0.0003225877632898696, + "loss": 2.931, + "theoretical_loss": 3.589538380163316, + "tokens_seen": 1192488960 + }, + { + "epoch": 4.0, + "learning_rate": 0.0003225777331995988, + "loss": 2.7551, + "theoretical_loss": 3.5895202130552297, + "tokens_seen": 1192554496 + }, + { + "epoch": 4.0, + "learning_rate": 0.000322567703109328, + "loss": 2.7762, + "theoretical_loss": 3.5895020472250003, + "tokens_seen": 1192620032 + }, + { + "epoch": 4.0, + "learning_rate": 0.0003225576730190572, + "loss": 2.6019, + "theoretical_loss": 3.589483882672468, + "tokens_seen": 1192685568 + }, + { + "epoch": 4.0, + "objective/train/docs_used": 1362192, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9670515060424805, + "objective/train/theoretical_loss": 3.589465719397472, + "objective/train/tokens_used": 1213211104, + "theoretical_loss": 3.589465719397472, + "tokens_seen": 1192751104 + }, + { + "epoch": 4.0, + "learning_rate": 0.00032254764292878634, + "loss": 2.8065, + "theoretical_loss": 3.589465719397472, + "tokens_seen": 1192751104 + }, + { + "epoch": 4.0, + "learning_rate": 0.0003225376128385156, + "loss": 2.6473, + "theoretical_loss": 3.5894475573998528, + "tokens_seen": 1192816640 + }, + { + "epoch": 4.0, + "learning_rate": 0.0003225275827482447, + "loss": 2.7868, + "theoretical_loss": 3.58942939667945, + "tokens_seen": 1192882176 + }, + { + "epoch": 4.0, + "learning_rate": 0.00032251755265797394, + "loss": 2.7406, + "theoretical_loss": 3.589411237236104, + "tokens_seen": 1192947712 + }, + { + "epoch": 4.0, + "learning_rate": 0.0003225075225677031, + "loss": 2.8387, + "theoretical_loss": 3.589393079069654, + "tokens_seen": 1193013248 + }, + { + "epoch": 4.0, + "learning_rate": 0.0003224974924774323, + "loss": 2.6082, + "theoretical_loss": 3.5893749221799416, + "tokens_seen": 1193078784 + }, + { + "epoch": 4.0, + "learning_rate": 0.0003224874623871615, + "loss": 2.7082, + "theoretical_loss": 3.589356766566806, + "tokens_seen": 1193144320 + }, + { + "epoch": 4.0, + "learning_rate": 0.0003224774322968907, + "loss": 2.7234, + "theoretical_loss": 3.5893386122300868, + "tokens_seen": 1193209856 + }, + { + "epoch": 4.0, + "learning_rate": 0.00032246740220661984, + "loss": 2.6437, + "theoretical_loss": 3.5893204591696253, + "tokens_seen": 1193275392 + }, + { + "epoch": 4.0, + "learning_rate": 0.0003224573721163491, + "loss": 2.6607, + "theoretical_loss": 3.5893023073852612, + "tokens_seen": 1193340928 + }, + { + "epoch": 4.0, + "learning_rate": 0.0003224473420260782, + "loss": 2.6923, + "theoretical_loss": 3.5892841568768343, + "tokens_seen": 1193406464 + }, + { + "epoch": 4.0, + "learning_rate": 0.00032243731193580744, + "loss": 2.758, + "theoretical_loss": 3.589266007644185, + "tokens_seen": 1193472000 + }, + { + "epoch": 4.0, + "learning_rate": 0.0003224272818455366, + "loss": 2.7518, + "theoretical_loss": 3.5892478596871547, + "tokens_seen": 1193537536 + }, + { + "epoch": 4.0, + "learning_rate": 0.0003224172517552658, + "loss": 2.7189, + "theoretical_loss": 3.589229713005582, + "tokens_seen": 1193603072 + }, + { + "epoch": 4.0, + "learning_rate": 0.000322407221664995, + "loss": 2.7066, + "theoretical_loss": 3.5892115675993077, + "tokens_seen": 1193668608 + }, + { + "epoch": 4.0, + "learning_rate": 0.00032239719157472417, + "loss": 2.6636, + "theoretical_loss": 3.5891934234681733, + "tokens_seen": 1193734144 + }, + { + "epoch": 4.0, + "learning_rate": 0.00032238716148445335, + "loss": 2.6847, + "theoretical_loss": 3.589175280612018, + "tokens_seen": 1193799680 + }, + { + "epoch": 4.0, + "learning_rate": 0.0003223771313941826, + "loss": 2.7743, + "theoretical_loss": 3.589157139030683, + "tokens_seen": 1193865216 + }, + { + "epoch": 4.0, + "learning_rate": 0.0003223671013039117, + "loss": 2.8292, + "theoretical_loss": 3.589138998724008, + "tokens_seen": 1193930752 + }, + { + "epoch": 4.0, + "learning_rate": 0.00032235707121364095, + "loss": 2.8121, + "theoretical_loss": 3.5891208596918345, + "tokens_seen": 1193996288 + }, + { + "epoch": 4.0, + "learning_rate": 0.0003223470411233701, + "loss": 2.7214, + "theoretical_loss": 3.589102721934002, + "tokens_seen": 1194061824 + }, + { + "epoch": 4.0, + "learning_rate": 0.0003223370110330993, + "loss": 2.7198, + "theoretical_loss": 3.589084585450351, + "tokens_seen": 1194127360 + }, + { + "epoch": 4.0, + "learning_rate": 0.0003223269809428285, + "loss": 2.7321, + "theoretical_loss": 3.589066450240723, + "tokens_seen": 1194192896 + }, + { + "epoch": 4.0, + "learning_rate": 0.00032231695085255767, + "loss": 2.669, + "theoretical_loss": 3.5890483163049574, + "tokens_seen": 1194258432 + }, + { + "epoch": 4.0, + "learning_rate": 0.00032230692076228685, + "loss": 2.8583, + "theoretical_loss": 3.5890301836428957, + "tokens_seen": 1194323968 + }, + { + "epoch": 4.0, + "objective/train/docs_used": 1367200, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5147886276245117, + "objective/train/theoretical_loss": 3.589012052254379, + "objective/train/tokens_used": 1214849504, + "theoretical_loss": 3.589012052254379, + "tokens_seen": 1194389504 + }, + { + "epoch": 4.0, + "learning_rate": 0.0003222968906720161, + "loss": 2.6539, + "theoretical_loss": 3.589012052254379, + "tokens_seen": 1194389504 + }, + { + "epoch": 4.0, + "learning_rate": 0.0003222868605817452, + "loss": 2.8133, + "theoretical_loss": 3.5889939221392466, + "tokens_seen": 1194455040 + }, + { + "epoch": 4.0, + "learning_rate": 0.00032227683049147445, + "loss": 2.7431, + "theoretical_loss": 3.5889757932973403, + "tokens_seen": 1194520576 + }, + { + "epoch": 4.0, + "learning_rate": 0.0003222668004012036, + "loss": 2.7153, + "theoretical_loss": 3.5889576657285, + "tokens_seen": 1194586112 + }, + { + "epoch": 4.0, + "learning_rate": 0.0003222567703109328, + "loss": 2.691, + "theoretical_loss": 3.5889395394325674, + "tokens_seen": 1194651648 + }, + { + "epoch": 4.0, + "learning_rate": 0.000322246740220662, + "loss": 2.784, + "theoretical_loss": 3.5889214144093824, + "tokens_seen": 1194717184 + }, + { + "epoch": 4.0, + "learning_rate": 0.00032223671013039117, + "loss": 2.6861, + "theoretical_loss": 3.588903290658787, + "tokens_seen": 1194782720 + }, + { + "epoch": 4.0, + "learning_rate": 0.00032222668004012035, + "loss": 2.567, + "theoretical_loss": 3.5888851681806204, + "tokens_seen": 1194848256 + }, + { + "epoch": 4.0, + "learning_rate": 0.00032221664994984953, + "loss": 2.6833, + "theoretical_loss": 3.588867046974725, + "tokens_seen": 1194913792 + }, + { + "epoch": 4.0, + "learning_rate": 0.0003222066198595787, + "loss": 2.7416, + "theoretical_loss": 3.588848927040941, + "tokens_seen": 1194979328 + }, + { + "epoch": 4.0, + "learning_rate": 0.00032219658976930795, + "loss": 2.6881, + "theoretical_loss": 3.588830808379109, + "tokens_seen": 1195044864 + }, + { + "epoch": 4.0, + "learning_rate": 0.0003221865596790371, + "loss": 2.5894, + "theoretical_loss": 3.588812690989071, + "tokens_seen": 1195110400 + }, + { + "epoch": 4.0, + "learning_rate": 0.0003221765295887663, + "loss": 2.8525, + "theoretical_loss": 3.588794574870667, + "tokens_seen": 1195175936 + }, + { + "epoch": 4.0, + "learning_rate": 0.0003221664994984955, + "loss": 2.6549, + "theoretical_loss": 3.588776460023739, + "tokens_seen": 1195241472 + }, + { + "epoch": 4.0, + "learning_rate": 0.0003221564694082247, + "loss": 2.8704, + "theoretical_loss": 3.5887583464481265, + "tokens_seen": 1195307008 + }, + { + "epoch": 4.0, + "learning_rate": 0.00032214643931795386, + "loss": 2.7525, + "theoretical_loss": 3.5887402341436725, + "tokens_seen": 1195372544 + }, + { + "epoch": 4.0, + "learning_rate": 0.00032213640922768304, + "loss": 2.745, + "theoretical_loss": 3.588722123110217, + "tokens_seen": 1195438080 + }, + { + "epoch": 4.0, + "learning_rate": 0.0003221263791374122, + "loss": 2.8724, + "theoretical_loss": 3.588704013347601, + "tokens_seen": 1195503616 + }, + { + "epoch": 4.0, + "learning_rate": 0.00032211634904714145, + "loss": 2.6772, + "theoretical_loss": 3.588685904855666, + "tokens_seen": 1195569152 + }, + { + "epoch": 4.0, + "learning_rate": 0.0003221063189568706, + "loss": 2.5685, + "theoretical_loss": 3.588667797634253, + "tokens_seen": 1195634688 + }, + { + "epoch": 4.0, + "learning_rate": 0.0003220962888665998, + "loss": 2.6183, + "theoretical_loss": 3.5886496916832034, + "tokens_seen": 1195700224 + }, + { + "epoch": 4.0, + "learning_rate": 0.00032208625877632894, + "loss": 2.8578, + "theoretical_loss": 3.588631587002359, + "tokens_seen": 1195765760 + }, + { + "epoch": 4.0, + "learning_rate": 0.0003220762286860582, + "loss": 2.4787, + "theoretical_loss": 3.58861348359156, + "tokens_seen": 1195831296 + }, + { + "epoch": 4.0, + "learning_rate": 0.0003220661985957874, + "loss": 2.4697, + "theoretical_loss": 3.5885953814506486, + "tokens_seen": 1195896832 + }, + { + "epoch": 4.0, + "learning_rate": 0.00032205616850551654, + "loss": 2.6127, + "theoretical_loss": 3.588577280579466, + "tokens_seen": 1195962368 + }, + { + "epoch": 4.0, + "objective/train/docs_used": 1372299, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.829193592071533, + "objective/train/theoretical_loss": 3.5885591809778523, + "objective/train/tokens_used": 1216487904, + "theoretical_loss": 3.5885591809778523, + "tokens_seen": 1196027904 + }, + { + "epoch": 4.0, + "learning_rate": 0.0003220461384152458, + "loss": 2.8615, + "theoretical_loss": 3.5885591809778523, + "tokens_seen": 1196027904 + }, + { + "epoch": 4.0, + "learning_rate": 0.0003220361083249749, + "loss": 2.649, + "theoretical_loss": 3.588541082645651, + "tokens_seen": 1196093440 + }, + { + "epoch": 4.0, + "learning_rate": 0.00032202607823470414, + "loss": 2.816, + "theoretical_loss": 3.588522985582702, + "tokens_seen": 1196158976 + }, + { + "epoch": 4.0, + "learning_rate": 0.0003220160481444333, + "loss": 2.7417, + "theoretical_loss": 3.5885048897888474, + "tokens_seen": 1196224512 + }, + { + "epoch": 4.0, + "learning_rate": 0.0003220060180541625, + "loss": 2.7062, + "theoretical_loss": 3.588486795263928, + "tokens_seen": 1196290048 + }, + { + "epoch": 4.0, + "learning_rate": 0.0003219959879638917, + "loss": 2.618, + "theoretical_loss": 3.588468702007787, + "tokens_seen": 1196355584 + }, + { + "epoch": 4.0, + "learning_rate": 0.0003219859578736209, + "loss": 2.4866, + "theoretical_loss": 3.5884506100202636, + "tokens_seen": 1196421120 + }, + { + "epoch": 4.0, + "learning_rate": 0.00032197592778335004, + "loss": 2.5016, + "theoretical_loss": 3.5884325193012008, + "tokens_seen": 1196486656 + }, + { + "epoch": 4.0, + "learning_rate": 0.0003219658976930793, + "loss": 2.5037, + "theoretical_loss": 3.58841442985044, + "tokens_seen": 1196552192 + }, + { + "epoch": 4.0, + "learning_rate": 0.0003219558676028084, + "loss": 2.5967, + "theoretical_loss": 3.588396341667823, + "tokens_seen": 1196617728 + }, + { + "epoch": 4.0, + "learning_rate": 0.00032194583751253764, + "loss": 2.6184, + "theoretical_loss": 3.5883782547531906, + "tokens_seen": 1196683264 + }, + { + "epoch": 4.0, + "learning_rate": 0.0003219358074222668, + "loss": 2.8586, + "theoretical_loss": 3.5883601691063856, + "tokens_seen": 1196748800 + }, + { + "epoch": 4.0, + "learning_rate": 0.000321925777331996, + "loss": 2.7405, + "theoretical_loss": 3.5883420847272487, + "tokens_seen": 1196814336 + }, + { + "epoch": 4.0, + "learning_rate": 0.0003219157472417252, + "loss": 2.8422, + "theoretical_loss": 3.5883240016156224, + "tokens_seen": 1196879872 + }, + { + "epoch": 4.0, + "learning_rate": 0.00032190571715145437, + "loss": 2.7227, + "theoretical_loss": 3.588305919771348, + "tokens_seen": 1196945408 + }, + { + "epoch": 4.0, + "learning_rate": 0.00032189568706118355, + "loss": 2.8192, + "theoretical_loss": 3.5882878391942676, + "tokens_seen": 1197010944 + }, + { + "epoch": 4.0, + "learning_rate": 0.0003218856569709128, + "loss": 2.6226, + "theoretical_loss": 3.5882697598842226, + "tokens_seen": 1197076480 + }, + { + "epoch": 4.0, + "learning_rate": 0.0003218756268806419, + "loss": 2.6056, + "theoretical_loss": 3.5882516818410557, + "tokens_seen": 1197142016 + }, + { + "epoch": 4.0, + "learning_rate": 0.00032186559679037115, + "loss": 2.751, + "theoretical_loss": 3.588233605064608, + "tokens_seen": 1197207552 + }, + { + "epoch": 4.0, + "learning_rate": 0.0003218555667001003, + "loss": 2.5782, + "theoretical_loss": 3.588215529554722, + "tokens_seen": 1197273088 + }, + { + "epoch": 4.0, + "learning_rate": 0.0003218455366098295, + "loss": 2.5359, + "theoretical_loss": 3.5881974553112386, + "tokens_seen": 1197338624 + }, + { + "epoch": 4.0, + "learning_rate": 0.0003218355065195587, + "loss": 2.6353, + "theoretical_loss": 3.5881793823340007, + "tokens_seen": 1197404160 + }, + { + "epoch": 4.0, + "learning_rate": 0.00032182547642928787, + "loss": 2.4617, + "theoretical_loss": 3.58816131062285, + "tokens_seen": 1197469696 + }, + { + "epoch": 4.0, + "learning_rate": 0.00032181544633901705, + "loss": 2.8249, + "theoretical_loss": 3.5881432401776285, + "tokens_seen": 1197535232 + }, + { + "epoch": 4.0, + "learning_rate": 0.0003218054162487463, + "loss": 2.753, + "theoretical_loss": 3.588125170998178, + "tokens_seen": 1197600768 + }, + { + "epoch": 4.0, + "objective/train/docs_used": 1377331, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7911863327026367, + "objective/train/theoretical_loss": 3.588107103084342, + "objective/train/tokens_used": 1218126304, + "theoretical_loss": 3.588107103084342, + "tokens_seen": 1197666304 + }, + { + "epoch": 4.0, + "learning_rate": 0.0003217953861584754, + "loss": 2.8607, + "theoretical_loss": 3.588107103084342, + "tokens_seen": 1197666304 + }, + { + "epoch": 4.0, + "learning_rate": 0.00032178535606820465, + "loss": 2.597, + "theoretical_loss": 3.58808903643596, + "tokens_seen": 1197731840 + }, + { + "epoch": 4.0, + "learning_rate": 0.0003217753259779338, + "loss": 2.684, + "theoretical_loss": 3.5880709710528764, + "tokens_seen": 1197797376 + }, + { + "epoch": 4.0, + "learning_rate": 0.000321765295887663, + "loss": 2.819, + "theoretical_loss": 3.5880529069349327, + "tokens_seen": 1197862912 + }, + { + "epoch": 4.0, + "learning_rate": 0.0003217552657973922, + "loss": 2.6415, + "theoretical_loss": 3.588034844081971, + "tokens_seen": 1197928448 + }, + { + "epoch": 4.0, + "learning_rate": 0.00032174523570712137, + "loss": 2.6505, + "theoretical_loss": 3.588016782493833, + "tokens_seen": 1197993984 + }, + { + "epoch": 4.0, + "learning_rate": 0.00032173520561685055, + "loss": 2.799, + "theoretical_loss": 3.587998722170362, + "tokens_seen": 1198059520 + }, + { + "epoch": 4.0, + "learning_rate": 0.00032172517552657974, + "loss": 2.7641, + "theoretical_loss": 3.5879806631113995, + "tokens_seen": 1198125056 + }, + { + "epoch": 4.0, + "learning_rate": 0.0003217151454363089, + "loss": 2.7987, + "theoretical_loss": 3.587962605316788, + "tokens_seen": 1198190592 + }, + { + "epoch": 4.0, + "learning_rate": 0.00032170511534603815, + "loss": 2.6003, + "theoretical_loss": 3.5879445487863695, + "tokens_seen": 1198256128 + }, + { + "epoch": 4.0, + "learning_rate": 0.0003216950852557673, + "loss": 2.8261, + "theoretical_loss": 3.587926493519987, + "tokens_seen": 1198321664 + }, + { + "epoch": 4.0, + "learning_rate": 0.0003216850551654965, + "loss": 2.6754, + "theoretical_loss": 3.5879084395174825, + "tokens_seen": 1198387200 + }, + { + "epoch": 4.0, + "learning_rate": 0.0003216750250752257, + "loss": 2.7249, + "theoretical_loss": 3.5878903867786986, + "tokens_seen": 1198452736 + }, + { + "epoch": 4.0, + "learning_rate": 0.0003216649949849549, + "loss": 2.6604, + "theoretical_loss": 3.5878723353034783, + "tokens_seen": 1198518272 + }, + { + "epoch": 4.0, + "learning_rate": 0.00032165496489468406, + "loss": 2.925, + "theoretical_loss": 3.5878542850916624, + "tokens_seen": 1198583808 + }, + { + "epoch": 4.0, + "learning_rate": 0.00032164493480441324, + "loss": 2.7317, + "theoretical_loss": 3.587836236143095, + "tokens_seen": 1198649344 + }, + { + "epoch": 4.0, + "learning_rate": 0.0003216349047141424, + "loss": 2.8008, + "theoretical_loss": 3.587818188457618, + "tokens_seen": 1198714880 + }, + { + "epoch": 4.0, + "learning_rate": 0.00032162487462387165, + "loss": 2.689, + "theoretical_loss": 3.587800142035074, + "tokens_seen": 1198780416 + }, + { + "epoch": 4.0, + "learning_rate": 0.0003216148445336008, + "loss": 2.7237, + "theoretical_loss": 3.5877820968753054, + "tokens_seen": 1198845952 + }, + { + "epoch": 4.0, + "learning_rate": 0.00032160481444333, + "loss": 2.6511, + "theoretical_loss": 3.587764052978155, + "tokens_seen": 1198911488 + }, + { + "epoch": 4.0, + "learning_rate": 0.00032159478435305914, + "loss": 2.5991, + "theoretical_loss": 3.587746010343466, + "tokens_seen": 1198977024 + }, + { + "epoch": 4.0, + "learning_rate": 0.0003215847542627884, + "loss": 2.7523, + "theoretical_loss": 3.5877279689710804, + "tokens_seen": 1199042560 + }, + { + "epoch": 4.0, + "learning_rate": 0.00032157472417251756, + "loss": 2.8926, + "theoretical_loss": 3.587709928860841, + "tokens_seen": 1199108096 + }, + { + "epoch": 4.0, + "learning_rate": 0.00032156469408224674, + "loss": 2.6584, + "theoretical_loss": 3.5876918900125903, + "tokens_seen": 1199173632 + }, + { + "epoch": 4.0, + "learning_rate": 0.0003215546639919759, + "loss": 2.5422, + "theoretical_loss": 3.5876738524261715, + "tokens_seen": 1199239168 + }, + { + "epoch": 4.0, + "objective/train/docs_used": 1382178, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.804316282272339, + "objective/train/theoretical_loss": 3.587655816101427, + "objective/train/tokens_used": 1219764704, + "theoretical_loss": 3.587655816101427, + "tokens_seen": 1199304704 + }, + { + "epoch": 4.0, + "learning_rate": 0.0003215446339017051, + "loss": 2.7885, + "theoretical_loss": 3.587655816101427, + "tokens_seen": 1199304704 + }, + { + "epoch": 4.0, + "learning_rate": 0.0003215346038114343, + "loss": 2.8044, + "theoretical_loss": 3.5876377810382003, + "tokens_seen": 1199370240 + }, + { + "epoch": 4.0, + "learning_rate": 0.0003215245737211635, + "loss": 2.7155, + "theoretical_loss": 3.5876197472363334, + "tokens_seen": 1199435776 + }, + { + "epoch": 4.0, + "learning_rate": 0.00032151454363089265, + "loss": 2.5803, + "theoretical_loss": 3.58760171469567, + "tokens_seen": 1199501312 + }, + { + "epoch": 4.0, + "learning_rate": 0.0003215045135406219, + "loss": 2.6165, + "theoretical_loss": 3.587583683416052, + "tokens_seen": 1199566848 + }, + { + "epoch": 4.0, + "learning_rate": 0.00032149448345035106, + "loss": 2.7678, + "theoretical_loss": 3.5875656533973235, + "tokens_seen": 1199632384 + }, + { + "epoch": 4.0, + "learning_rate": 0.00032148445336008024, + "loss": 2.8196, + "theoretical_loss": 3.5875476246393263, + "tokens_seen": 1199697920 + }, + { + "epoch": 4.0, + "learning_rate": 0.0003214744232698094, + "loss": 2.549, + "theoretical_loss": 3.587529597141904, + "tokens_seen": 1199763456 + }, + { + "epoch": 4.0, + "learning_rate": 0.0003214643931795386, + "loss": 2.85, + "theoretical_loss": 3.5875115709048995, + "tokens_seen": 1199828992 + }, + { + "epoch": 4.0, + "learning_rate": 0.0003214543630892678, + "loss": 2.6543, + "theoretical_loss": 3.587493545928156, + "tokens_seen": 1199894528 + }, + { + "epoch": 4.0, + "learning_rate": 0.000321444332998997, + "loss": 2.7943, + "theoretical_loss": 3.587475522211516, + "tokens_seen": 1199960064 + }, + { + "epoch": 4.0, + "learning_rate": 0.00032143430290872615, + "loss": 2.6293, + "theoretical_loss": 3.587457499754823, + "tokens_seen": 1200025600 + }, + { + "epoch": 4.0, + "learning_rate": 0.0003214242728184554, + "loss": 2.7308, + "theoretical_loss": 3.5874394785579207, + "tokens_seen": 1200091136 + }, + { + "epoch": 4.0, + "learning_rate": 0.0003214142427281845, + "loss": 2.6226, + "theoretical_loss": 3.587421458620651, + "tokens_seen": 1200156672 + }, + { + "epoch": 4.0, + "learning_rate": 0.00032140421263791375, + "loss": 2.615, + "theoretical_loss": 3.5874034399428583, + "tokens_seen": 1200222208 + }, + { + "epoch": 4.0, + "learning_rate": 0.00032139418254764293, + "loss": 2.7029, + "theoretical_loss": 3.5873854225243855, + "tokens_seen": 1200287744 + }, + { + "epoch": 4.0, + "learning_rate": 0.0003213841524573721, + "loss": 2.6413, + "theoretical_loss": 3.587367406365075, + "tokens_seen": 1200353280 + }, + { + "epoch": 4.0, + "learning_rate": 0.0003213741223671013, + "loss": 2.6256, + "theoretical_loss": 3.5873493914647705, + "tokens_seen": 1200418816 + }, + { + "epoch": 4.0, + "learning_rate": 0.0003213640922768305, + "loss": 2.756, + "theoretical_loss": 3.5873313778233165, + "tokens_seen": 1200484352 + }, + { + "epoch": 4.0, + "learning_rate": 0.00032135406218655965, + "loss": 2.6721, + "theoretical_loss": 3.5873133654405542, + "tokens_seen": 1200549888 + }, + { + "epoch": 4.0, + "learning_rate": 0.0003213440320962889, + "loss": 2.6894, + "theoretical_loss": 3.5872953543163284, + "tokens_seen": 1200615424 + }, + { + "epoch": 4.0, + "learning_rate": 0.000321334002006018, + "loss": 2.7949, + "theoretical_loss": 3.587277344450482, + "tokens_seen": 1200680960 + }, + { + "epoch": 4.0, + "learning_rate": 0.00032132397191574725, + "loss": 2.8017, + "theoretical_loss": 3.587259335842859, + "tokens_seen": 1200746496 + }, + { + "epoch": 4.0, + "learning_rate": 0.0003213139418254765, + "loss": 2.7304, + "theoretical_loss": 3.5872413284933016, + "tokens_seen": 1200812032 + }, + { + "epoch": 4.0, + "learning_rate": 0.0003213039117352056, + "loss": 2.8873, + "theoretical_loss": 3.5872233224016545, + "tokens_seen": 1200877568 + }, + { + "epoch": 4.0, + "objective/train/docs_used": 1387236, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.934062957763672, + "objective/train/theoretical_loss": 3.5872053175677605, + "objective/train/tokens_used": 1221403104, + "theoretical_loss": 3.5872053175677605, + "tokens_seen": 1200943104 + }, + { + "epoch": 4.0, + "learning_rate": 0.00032129388164493485, + "loss": 2.7567, + "theoretical_loss": 3.5872053175677605, + "tokens_seen": 1200943104 + }, + { + "epoch": 4.0, + "learning_rate": 0.000321283851554664, + "loss": 2.6779, + "theoretical_loss": 3.587187313991463, + "tokens_seen": 1201008640 + }, + { + "epoch": 4.0, + "learning_rate": 0.0003212738214643932, + "loss": 2.7564, + "theoretical_loss": 3.587169311672606, + "tokens_seen": 1201074176 + }, + { + "epoch": 4.0, + "learning_rate": 0.0003212637913741224, + "loss": 2.8282, + "theoretical_loss": 3.5871513106110333, + "tokens_seen": 1201139712 + }, + { + "epoch": 4.0, + "learning_rate": 0.0003212537612838516, + "loss": 2.5829, + "theoretical_loss": 3.587133310806588, + "tokens_seen": 1201205248 + }, + { + "epoch": 4.0, + "learning_rate": 0.00032124373119358075, + "loss": 2.6074, + "theoretical_loss": 3.5871153122591135, + "tokens_seen": 1201270784 + }, + { + "epoch": 4.0, + "learning_rate": 0.00032123370110330994, + "loss": 2.5451, + "theoretical_loss": 3.587097314968454, + "tokens_seen": 1201336320 + }, + { + "epoch": 4.0, + "learning_rate": 0.0003212236710130391, + "loss": 2.7113, + "theoretical_loss": 3.5870793189344523, + "tokens_seen": 1201401856 + }, + { + "epoch": 4.0, + "learning_rate": 0.00032121364092276835, + "loss": 2.4835, + "theoretical_loss": 3.587061324156954, + "tokens_seen": 1201467392 + }, + { + "epoch": 4.0, + "learning_rate": 0.0003212036108324975, + "loss": 2.5967, + "theoretical_loss": 3.587043330635801, + "tokens_seen": 1201532928 + }, + { + "epoch": 4.0, + "learning_rate": 0.0003211935807422267, + "loss": 2.6397, + "theoretical_loss": 3.5870253383708377, + "tokens_seen": 1201598464 + }, + { + "epoch": 4.0, + "learning_rate": 0.0003211835506519559, + "loss": 2.7351, + "theoretical_loss": 3.5870073473619075, + "tokens_seen": 1201664000 + }, + { + "epoch": 4.0, + "learning_rate": 0.0003211735205616851, + "loss": 2.6823, + "theoretical_loss": 3.5869893576088545, + "tokens_seen": 1201729536 + }, + { + "epoch": 4.0, + "learning_rate": 0.00032116349047141426, + "loss": 2.5033, + "theoretical_loss": 3.5869713691115237, + "tokens_seen": 1201795072 + }, + { + "epoch": 4.0, + "learning_rate": 0.00032115346038114344, + "loss": 2.7195, + "theoretical_loss": 3.586953381869757, + "tokens_seen": 1201860608 + }, + { + "epoch": 4.0, + "learning_rate": 0.0003211434302908726, + "loss": 2.691, + "theoretical_loss": 3.5869353958833994, + "tokens_seen": 1201926144 + }, + { + "epoch": 4.0, + "learning_rate": 0.00032113340020060185, + "loss": 2.7591, + "theoretical_loss": 3.586917411152295, + "tokens_seen": 1201991680 + }, + { + "epoch": 4.0, + "learning_rate": 0.000321123370110331, + "loss": 2.7759, + "theoretical_loss": 3.586899427676287, + "tokens_seen": 1202057216 + }, + { + "epoch": 4.0, + "learning_rate": 0.0003211133400200602, + "loss": 2.5341, + "theoretical_loss": 3.5868814454552203, + "tokens_seen": 1202122752 + }, + { + "epoch": 4.0, + "learning_rate": 0.00032110330992978934, + "loss": 2.7228, + "theoretical_loss": 3.586863464488938, + "tokens_seen": 1202188288 + }, + { + "epoch": 4.0, + "learning_rate": 0.0003210932798395186, + "loss": 2.7055, + "theoretical_loss": 3.5868454847772853, + "tokens_seen": 1202253824 + }, + { + "epoch": 4.0, + "learning_rate": 0.00032108324974924776, + "loss": 2.6004, + "theoretical_loss": 3.586827506320105, + "tokens_seen": 1202319360 + }, + { + "epoch": 4.0, + "learning_rate": 0.00032107321965897694, + "loss": 2.7184, + "theoretical_loss": 3.5868095291172413, + "tokens_seen": 1202384896 + }, + { + "epoch": 4.0, + "learning_rate": 0.0003210631895687061, + "loss": 2.7188, + "theoretical_loss": 3.5867915531685397, + "tokens_seen": 1202450432 + }, + { + "epoch": 4.0, + "learning_rate": 0.0003210531594784353, + "loss": 2.7433, + "theoretical_loss": 3.586773578473843, + "tokens_seen": 1202515968 + }, + { + "epoch": 4.0, + "objective/train/docs_used": 1392256, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8471696376800537, + "objective/train/theoretical_loss": 3.586755605032996, + "objective/train/tokens_used": 1223041504, + "theoretical_loss": 3.586755605032996, + "tokens_seen": 1202581504 + }, + { + "epoch": 4.0, + "learning_rate": 0.0003210431293881645, + "loss": 2.7683, + "theoretical_loss": 3.586755605032996, + "tokens_seen": 1202581504 + }, + { + "epoch": 4.0, + "learning_rate": 0.0003210330992978937, + "loss": 2.6809, + "theoretical_loss": 3.5867376328458427, + "tokens_seen": 1202647040 + }, + { + "epoch": 4.0, + "learning_rate": 0.00032102306920762285, + "loss": 2.7461, + "theoretical_loss": 3.5867196619122272, + "tokens_seen": 1202712576 + }, + { + "epoch": 4.0, + "learning_rate": 0.0003210130391173521, + "loss": 2.7872, + "theoretical_loss": 3.586701692231994, + "tokens_seen": 1202778112 + }, + { + "epoch": 4.0, + "learning_rate": 0.00032100300902708126, + "loss": 2.7223, + "theoretical_loss": 3.5866837238049873, + "tokens_seen": 1202843648 + }, + { + "epoch": 4.0, + "learning_rate": 0.00032099297893681044, + "loss": 2.6087, + "theoretical_loss": 3.586665756631052, + "tokens_seen": 1202909184 + }, + { + "epoch": 4.0, + "learning_rate": 0.0003209829488465396, + "loss": 2.6453, + "theoretical_loss": 3.5866477907100314, + "tokens_seen": 1202974720 + }, + { + "epoch": 4.0, + "learning_rate": 0.0003209729187562688, + "loss": 2.6654, + "theoretical_loss": 3.5866298260417704, + "tokens_seen": 1203040256 + }, + { + "epoch": 4.0, + "learning_rate": 0.000320962888665998, + "loss": 2.6581, + "theoretical_loss": 3.586611862626114, + "tokens_seen": 1203105792 + }, + { + "epoch": 4.0, + "learning_rate": 0.0003209528585757272, + "loss": 2.6549, + "theoretical_loss": 3.5865939004629057, + "tokens_seen": 1203171328 + }, + { + "epoch": 4.0, + "learning_rate": 0.00032094282848545635, + "loss": 2.7228, + "theoretical_loss": 3.58657593955199, + "tokens_seen": 1203236864 + }, + { + "epoch": 4.0, + "learning_rate": 0.0003209327983951856, + "loss": 2.8043, + "theoretical_loss": 3.586557979893212, + "tokens_seen": 1203302400 + }, + { + "epoch": 4.0, + "learning_rate": 0.0003209227683049147, + "loss": 2.5721, + "theoretical_loss": 3.5865400214864156, + "tokens_seen": 1203367936 + }, + { + "epoch": 4.0, + "learning_rate": 0.00032091273821464395, + "loss": 2.6316, + "theoretical_loss": 3.5865220643314464, + "tokens_seen": 1203433472 + }, + { + "epoch": 4.0, + "learning_rate": 0.00032090270812437313, + "loss": 2.6906, + "theoretical_loss": 3.586504108428148, + "tokens_seen": 1203499008 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003208926780341023, + "loss": 2.6847, + "theoretical_loss": 3.586486153776365, + "tokens_seen": 1203564544 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003208826479438315, + "loss": 2.7254, + "theoretical_loss": 3.586468200375942, + "tokens_seen": 1203630080 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003208726178535607, + "loss": 2.7631, + "theoretical_loss": 3.5864502482267246, + "tokens_seen": 1203695616 + }, + { + "epoch": 4.01, + "learning_rate": 0.00032086258776328985, + "loss": 2.6306, + "theoretical_loss": 3.5864322973285567, + "tokens_seen": 1203761152 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003208525576730191, + "loss": 2.806, + "theoretical_loss": 3.5864143476812824, + "tokens_seen": 1203826688 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003208425275827482, + "loss": 2.6084, + "theoretical_loss": 3.586396399284748, + "tokens_seen": 1203892224 + }, + { + "epoch": 4.01, + "learning_rate": 0.00032083249749247745, + "loss": 2.7025, + "theoretical_loss": 3.5863784521387974, + "tokens_seen": 1203957760 + }, + { + "epoch": 4.01, + "learning_rate": 0.00032082246740220663, + "loss": 2.7901, + "theoretical_loss": 3.5863605062432753, + "tokens_seen": 1204023296 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003208124373119358, + "loss": 2.7009, + "theoretical_loss": 3.586342561598027, + "tokens_seen": 1204088832 + }, + { + "epoch": 4.01, + "learning_rate": 0.000320802407221665, + "loss": 2.6494, + "theoretical_loss": 3.5863246182028963, + "tokens_seen": 1204154368 + }, + { + "epoch": 4.01, + "objective/train/docs_used": 1397361, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8406195640563965, + "objective/train/theoretical_loss": 3.5863066760577293, + "objective/train/tokens_used": 1224679904, + "theoretical_loss": 3.5863066760577293, + "tokens_seen": 1204219904 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003207923771313942, + "loss": 2.6758, + "theoretical_loss": 3.5863066760577293, + "tokens_seen": 1204219904 + }, + { + "epoch": 4.01, + "learning_rate": 0.00032078234704112336, + "loss": 2.831, + "theoretical_loss": 3.5862887351623702, + "tokens_seen": 1204285440 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003207723169508526, + "loss": 2.8496, + "theoretical_loss": 3.5862707955166644, + "tokens_seen": 1204350976 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003207622868605817, + "loss": 2.6733, + "theoretical_loss": 3.5862528571204564, + "tokens_seen": 1204416512 + }, + { + "epoch": 4.01, + "learning_rate": 0.00032075225677031095, + "loss": 2.7109, + "theoretical_loss": 3.5862349199735912, + "tokens_seen": 1204482048 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003207422266800401, + "loss": 2.7204, + "theoretical_loss": 3.5862169840759144, + "tokens_seen": 1204547584 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003207321965897693, + "loss": 2.6926, + "theoretical_loss": 3.586199049427271, + "tokens_seen": 1204613120 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003207221664994985, + "loss": 2.6064, + "theoretical_loss": 3.586181116027505, + "tokens_seen": 1204678656 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003207121364092277, + "loss": 2.5865, + "theoretical_loss": 3.5861631838764625, + "tokens_seen": 1204744192 + }, + { + "epoch": 4.01, + "learning_rate": 0.00032070210631895686, + "loss": 2.5053, + "theoretical_loss": 3.5861452529739886, + "tokens_seen": 1204809728 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003206920762286861, + "loss": 2.751, + "theoretical_loss": 3.5861273233199276, + "tokens_seen": 1204875264 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003206820461384152, + "loss": 2.8412, + "theoretical_loss": 3.586109394914126, + "tokens_seen": 1204940800 + }, + { + "epoch": 4.01, + "learning_rate": 0.00032067201604814446, + "loss": 2.79, + "theoretical_loss": 3.586091467756428, + "tokens_seen": 1205006336 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003206619859578736, + "loss": 2.7373, + "theoretical_loss": 3.586073541846679, + "tokens_seen": 1205071872 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003206519558676028, + "loss": 2.7581, + "theoretical_loss": 3.5860556171847247, + "tokens_seen": 1205137408 + }, + { + "epoch": 4.01, + "learning_rate": 0.000320641925777332, + "loss": 2.9763, + "theoretical_loss": 3.58603769377041, + "tokens_seen": 1205202944 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003206318956870612, + "loss": 2.7726, + "theoretical_loss": 3.5860197716035804, + "tokens_seen": 1205268480 + }, + { + "epoch": 4.01, + "learning_rate": 0.00032062186559679036, + "loss": 2.6571, + "theoretical_loss": 3.586001850684081, + "tokens_seen": 1205334016 + }, + { + "epoch": 4.01, + "learning_rate": 0.00032061183550651954, + "loss": 2.6722, + "theoretical_loss": 3.5859839310117576, + "tokens_seen": 1205399552 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003206018054162487, + "loss": 2.6794, + "theoretical_loss": 3.5859660125864554, + "tokens_seen": 1205465088 + }, + { + "epoch": 4.01, + "learning_rate": 0.00032059177532597796, + "loss": 2.7808, + "theoretical_loss": 3.5859480954080194, + "tokens_seen": 1205530624 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003205817452357071, + "loss": 2.7927, + "theoretical_loss": 3.585930179476296, + "tokens_seen": 1205596160 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003205717151454363, + "loss": 2.6525, + "theoretical_loss": 3.58591226479113, + "tokens_seen": 1205661696 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003205616850551655, + "loss": 2.7671, + "theoretical_loss": 3.5858943513523664, + "tokens_seen": 1205727232 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003205516549648947, + "loss": 2.771, + "theoretical_loss": 3.5858764391598523, + "tokens_seen": 1205792768 + }, + { + "epoch": 4.01, + "objective/train/docs_used": 1402482, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.825957775115967, + "objective/train/theoretical_loss": 3.585858528213432, + "objective/train/tokens_used": 1226318304, + "theoretical_loss": 3.585858528213432, + "tokens_seen": 1205858304 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003205416248746239, + "loss": 2.6716, + "theoretical_loss": 3.585858528213432, + "tokens_seen": 1205858304 + }, + { + "epoch": 4.01, + "learning_rate": 0.00032053159478435305, + "loss": 2.5004, + "theoretical_loss": 3.5858406185129517, + "tokens_seen": 1205923840 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003205215646940823, + "loss": 2.6681, + "theoretical_loss": 3.585822710058257, + "tokens_seen": 1205989376 + }, + { + "epoch": 4.01, + "learning_rate": 0.00032051153460381146, + "loss": 2.6805, + "theoretical_loss": 3.585804802849193, + "tokens_seen": 1206054912 + }, + { + "epoch": 4.01, + "learning_rate": 0.00032050150451354064, + "loss": 2.557, + "theoretical_loss": 3.5857868968856055, + "tokens_seen": 1206120448 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003204914744232698, + "loss": 2.7872, + "theoretical_loss": 3.585768992167341, + "tokens_seen": 1206185984 + }, + { + "epoch": 4.01, + "learning_rate": 0.000320481444332999, + "loss": 2.696, + "theoretical_loss": 3.585751088694244, + "tokens_seen": 1206251520 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003204714142427282, + "loss": 2.8453, + "theoretical_loss": 3.585733186466161, + "tokens_seen": 1206317056 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003204613841524574, + "loss": 2.7537, + "theoretical_loss": 3.585715285482938, + "tokens_seen": 1206382592 + }, + { + "epoch": 4.01, + "learning_rate": 0.00032045135406218655, + "loss": 2.7335, + "theoretical_loss": 3.585697385744421, + "tokens_seen": 1206448128 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003204413239719158, + "loss": 2.2961, + "theoretical_loss": 3.585679487250455, + "tokens_seen": 1206513664 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003204312938816449, + "loss": 2.6176, + "theoretical_loss": 3.5856615900008864, + "tokens_seen": 1206579200 + }, + { + "epoch": 4.01, + "learning_rate": 0.00032042126379137415, + "loss": 2.553, + "theoretical_loss": 3.5856436939955607, + "tokens_seen": 1206644736 + }, + { + "epoch": 4.01, + "learning_rate": 0.00032041123370110333, + "loss": 2.5838, + "theoretical_loss": 3.5856257992343243, + "tokens_seen": 1206710272 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003204012036108325, + "loss": 2.813, + "theoretical_loss": 3.5856079057170227, + "tokens_seen": 1206775808 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003203911735205617, + "loss": 2.719, + "theoretical_loss": 3.585590013443502, + "tokens_seen": 1206841344 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003203811434302909, + "loss": 2.7019, + "theoretical_loss": 3.585572122413609, + "tokens_seen": 1206906880 + }, + { + "epoch": 4.01, + "learning_rate": 0.00032037111334002005, + "loss": 2.4981, + "theoretical_loss": 3.5855542326271888, + "tokens_seen": 1206972416 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003203610832497493, + "loss": 2.6035, + "theoretical_loss": 3.5855363440840873, + "tokens_seen": 1207037952 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003203510531594784, + "loss": 2.6704, + "theoretical_loss": 3.5855184567841514, + "tokens_seen": 1207103488 + }, + { + "epoch": 4.01, + "learning_rate": 0.00032034102306920765, + "loss": 2.5376, + "theoretical_loss": 3.585500570727227, + "tokens_seen": 1207169024 + }, + { + "epoch": 4.01, + "learning_rate": 0.00032033099297893683, + "loss": 2.5535, + "theoretical_loss": 3.58548268591316, + "tokens_seen": 1207234560 + }, + { + "epoch": 4.01, + "learning_rate": 0.000320320962888666, + "loss": 2.7184, + "theoretical_loss": 3.5854648023417965, + "tokens_seen": 1207300096 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003203109327983952, + "loss": 2.746, + "theoretical_loss": 3.585446920012983, + "tokens_seen": 1207365632 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003203009027081244, + "loss": 2.7058, + "theoretical_loss": 3.5854290389265655, + "tokens_seen": 1207431168 + }, + { + "epoch": 4.01, + "objective/train/docs_used": 1407494, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6062726974487305, + "objective/train/theoretical_loss": 3.5854111590823905, + "objective/train/tokens_used": 1227956704, + "theoretical_loss": 3.5854111590823905, + "tokens_seen": 1207496704 + }, + { + "epoch": 4.01, + "learning_rate": 0.00032029087261785356, + "loss": 2.688, + "theoretical_loss": 3.5854111590823905, + "tokens_seen": 1207496704 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003202808425275828, + "loss": 2.8863, + "theoretical_loss": 3.5853932804803037, + "tokens_seen": 1207562240 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003202708124373119, + "loss": 2.6487, + "theoretical_loss": 3.5853754031201523, + "tokens_seen": 1207627776 + }, + { + "epoch": 4.01, + "learning_rate": 0.00032026078234704115, + "loss": 2.7036, + "theoretical_loss": 3.585357527001782, + "tokens_seen": 1207693312 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003202507522567703, + "loss": 2.7126, + "theoretical_loss": 3.5853396521250396, + "tokens_seen": 1207758848 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003202407221664995, + "loss": 2.543, + "theoretical_loss": 3.585321778489771, + "tokens_seen": 1207824384 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003202306920762287, + "loss": 2.7911, + "theoretical_loss": 3.585303906095823, + "tokens_seen": 1207889920 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003202206619859579, + "loss": 2.6455, + "theoretical_loss": 3.585286034943042, + "tokens_seen": 1207955456 + }, + { + "epoch": 4.01, + "learning_rate": 0.00032021063189568706, + "loss": 2.6103, + "theoretical_loss": 3.585268165031274, + "tokens_seen": 1208020992 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003202006018054163, + "loss": 2.6886, + "theoretical_loss": 3.5852502963603667, + "tokens_seen": 1208086528 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003201905717151454, + "loss": 2.6231, + "theoretical_loss": 3.585232428930165, + "tokens_seen": 1208152064 + }, + { + "epoch": 4.01, + "learning_rate": 0.00032018054162487466, + "loss": 2.7009, + "theoretical_loss": 3.585214562740517, + "tokens_seen": 1208217600 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003201705115346038, + "loss": 2.8047, + "theoretical_loss": 3.585196697791268, + "tokens_seen": 1208283136 + }, + { + "epoch": 4.01, + "learning_rate": 0.000320160481444333, + "loss": 2.7623, + "theoretical_loss": 3.5851788340822655, + "tokens_seen": 1208348672 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003201504513540622, + "loss": 2.7484, + "theoretical_loss": 3.5851609716133552, + "tokens_seen": 1208414208 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003201404212637914, + "loss": 2.6339, + "theoretical_loss": 3.585143110384385, + "tokens_seen": 1208479744 + }, + { + "epoch": 4.01, + "learning_rate": 0.00032013039117352056, + "loss": 2.6358, + "theoretical_loss": 3.585125250395201, + "tokens_seen": 1208545280 + }, + { + "epoch": 4.01, + "learning_rate": 0.00032012036108324974, + "loss": 2.6491, + "theoretical_loss": 3.58510739164565, + "tokens_seen": 1208610816 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003201103309929789, + "loss": 2.7829, + "theoretical_loss": 3.585089534135578, + "tokens_seen": 1208676352 + }, + { + "epoch": 4.01, + "learning_rate": 0.00032010030090270816, + "loss": 2.5954, + "theoretical_loss": 3.585071677864833, + "tokens_seen": 1208741888 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003200902708124373, + "loss": 2.6858, + "theoretical_loss": 3.585053822833261, + "tokens_seen": 1208807424 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003200802407221665, + "loss": 2.7955, + "theoretical_loss": 3.585035969040709, + "tokens_seen": 1208872960 + }, + { + "epoch": 4.01, + "learning_rate": 0.00032007021063189565, + "loss": 2.7785, + "theoretical_loss": 3.5850181164870243, + "tokens_seen": 1208938496 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003200601805416249, + "loss": 2.6245, + "theoretical_loss": 3.5850002651720527, + "tokens_seen": 1209004032 + }, + { + "epoch": 4.01, + "learning_rate": 0.00032005015045135407, + "loss": 2.5587, + "theoretical_loss": 3.5849824150956424, + "tokens_seen": 1209069568 + }, + { + "epoch": 4.01, + "objective/train/docs_used": 1410378, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7569308280944824, + "objective/train/theoretical_loss": 3.5849645662576393, + "objective/train/tokens_used": 1229595104, + "theoretical_loss": 3.5849645662576393, + "tokens_seen": 1209135104 + }, + { + "epoch": 4.01, + "learning_rate": 0.00032004012036108325, + "loss": 2.7375, + "theoretical_loss": 3.5849645662576393, + "tokens_seen": 1209135104 + }, + { + "epoch": 4.01, + "learning_rate": 0.00032003009027081243, + "loss": 2.5791, + "theoretical_loss": 3.584946718657891, + "tokens_seen": 1209200640 + }, + { + "epoch": 4.01, + "learning_rate": 0.00032002006018054166, + "loss": 2.6087, + "theoretical_loss": 3.5849288722962447, + "tokens_seen": 1209266176 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003200100300902708, + "loss": 2.8597, + "theoretical_loss": 3.584911027172547, + "tokens_seen": 1209331712 + }, + { + "epoch": 4.01, + "learning_rate": 0.00032, + "loss": 2.6192, + "theoretical_loss": 3.584893183286644, + "tokens_seen": 1209397248 + }, + { + "epoch": 4.01, + "learning_rate": 0.00031998996990972915, + "loss": 2.6508, + "theoretical_loss": 3.5848753406383844, + "tokens_seen": 1209462784 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003199799398194584, + "loss": 2.564, + "theoretical_loss": 3.5848574992276148, + "tokens_seen": 1209528320 + }, + { + "epoch": 4.01, + "learning_rate": 0.00031996990972918757, + "loss": 2.6464, + "theoretical_loss": 3.584839659054182, + "tokens_seen": 1209593856 + }, + { + "epoch": 4.01, + "learning_rate": 0.00031995987963891675, + "loss": 2.6648, + "theoretical_loss": 3.5848218201179334, + "tokens_seen": 1209659392 + }, + { + "epoch": 4.01, + "learning_rate": 0.00031994984954864593, + "loss": 2.698, + "theoretical_loss": 3.584803982418716, + "tokens_seen": 1209724928 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003199398194583751, + "loss": 2.6806, + "theoretical_loss": 3.584786145956377, + "tokens_seen": 1209790464 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003199297893681043, + "loss": 2.7552, + "theoretical_loss": 3.584768310730764, + "tokens_seen": 1209856000 + }, + { + "epoch": 4.01, + "learning_rate": 0.00031991975927783353, + "loss": 2.7092, + "theoretical_loss": 3.584750476741724, + "tokens_seen": 1209921536 + }, + { + "epoch": 4.01, + "learning_rate": 0.00031990972918756266, + "loss": 2.6622, + "theoretical_loss": 3.5847326439891045, + "tokens_seen": 1209987072 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003198996990972919, + "loss": 2.8497, + "theoretical_loss": 3.584714812472752, + "tokens_seen": 1210052608 + }, + { + "epoch": 4.01, + "learning_rate": 0.000319889669007021, + "loss": 2.592, + "theoretical_loss": 3.584696982192515, + "tokens_seen": 1210118144 + }, + { + "epoch": 4.01, + "learning_rate": 0.00031987963891675025, + "loss": 2.5142, + "theoretical_loss": 3.5846791531482403, + "tokens_seen": 1210183680 + }, + { + "epoch": 4.01, + "learning_rate": 0.00031986960882647943, + "loss": 2.7154, + "theoretical_loss": 3.584661325339775, + "tokens_seen": 1210249216 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003198595787362086, + "loss": 2.7834, + "theoretical_loss": 3.584643498766968, + "tokens_seen": 1210314752 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003198495486459378, + "loss": 2.5158, + "theoretical_loss": 3.584625673429665, + "tokens_seen": 1210380288 + }, + { + "epoch": 4.01, + "learning_rate": 0.00031983951855566703, + "loss": 2.6208, + "theoretical_loss": 3.584607849327714, + "tokens_seen": 1210445824 + }, + { + "epoch": 4.01, + "learning_rate": 0.00031982948846539616, + "loss": 2.6352, + "theoretical_loss": 3.5845900264609623, + "tokens_seen": 1210511360 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003198194583751254, + "loss": 2.7371, + "theoretical_loss": 3.584572204829258, + "tokens_seen": 1210576896 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003198094282848546, + "loss": 2.5, + "theoretical_loss": 3.5845543844324483, + "tokens_seen": 1210642432 + }, + { + "epoch": 4.01, + "learning_rate": 0.00031979939819458376, + "loss": 2.6279, + "theoretical_loss": 3.5845365652703816, + "tokens_seen": 1210707968 + }, + { + "epoch": 4.01, + "objective/train/docs_used": 1413162, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.503354787826538, + "objective/train/theoretical_loss": 3.584518747342904, + "objective/train/tokens_used": 1231233504, + "theoretical_loss": 3.584518747342904, + "tokens_seen": 1210773504 + }, + { + "epoch": 4.01, + "learning_rate": 0.000319789368104313, + "loss": 2.5154, + "theoretical_loss": 3.584518747342904, + "tokens_seen": 1210773504 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003197793380140421, + "loss": 2.8354, + "theoretical_loss": 3.5845009306498645, + "tokens_seen": 1210839040 + }, + { + "epoch": 4.01, + "learning_rate": 0.00031976930792377135, + "loss": 2.9319, + "theoretical_loss": 3.58448311519111, + "tokens_seen": 1210904576 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003197592778335005, + "loss": 2.6632, + "theoretical_loss": 3.584465300966489, + "tokens_seen": 1210970112 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003197492477432297, + "loss": 2.6491, + "theoretical_loss": 3.584447487975848, + "tokens_seen": 1211035648 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003197392176529589, + "loss": 2.7853, + "theoretical_loss": 3.5844296762190355, + "tokens_seen": 1211101184 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003197291875626881, + "loss": 2.7017, + "theoretical_loss": 3.5844118656958996, + "tokens_seen": 1211166720 + }, + { + "epoch": 4.01, + "learning_rate": 0.00031971915747241726, + "loss": 2.7758, + "theoretical_loss": 3.5843940564062873, + "tokens_seen": 1211232256 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003197091273821465, + "loss": 2.8573, + "theoretical_loss": 3.584376248350047, + "tokens_seen": 1211297792 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003196990972918756, + "loss": 2.6947, + "theoretical_loss": 3.5843584415270264, + "tokens_seen": 1211363328 + }, + { + "epoch": 4.01, + "learning_rate": 0.00031968906720160486, + "loss": 2.6019, + "theoretical_loss": 3.5843406359370737, + "tokens_seen": 1211428864 + }, + { + "epoch": 4.01, + "learning_rate": 0.000319679037111334, + "loss": 2.7002, + "theoretical_loss": 3.584322831580036, + "tokens_seen": 1211494400 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003196690070210632, + "loss": 2.4958, + "theoretical_loss": 3.584305028455762, + "tokens_seen": 1211559936 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003196589769307924, + "loss": 2.7174, + "theoretical_loss": 3.5842872265640993, + "tokens_seen": 1211625472 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003196489468405216, + "loss": 2.6841, + "theoretical_loss": 3.584269425904896, + "tokens_seen": 1211691008 + }, + { + "epoch": 4.01, + "learning_rate": 0.00031963891675025076, + "loss": 2.6407, + "theoretical_loss": 3.584251626478, + "tokens_seen": 1211756544 + }, + { + "epoch": 4.01, + "learning_rate": 0.00031962888665997994, + "loss": 2.8254, + "theoretical_loss": 3.58423382828326, + "tokens_seen": 1211822080 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003196188565697091, + "loss": 2.7335, + "theoretical_loss": 3.5842160313205236, + "tokens_seen": 1211887616 + }, + { + "epoch": 4.01, + "learning_rate": 0.00031960882647943836, + "loss": 2.6786, + "theoretical_loss": 3.5841982355896382, + "tokens_seen": 1211953152 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003195987963891675, + "loss": 2.8014, + "theoretical_loss": 3.584180441090453, + "tokens_seen": 1212018688 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003195887662988967, + "loss": 2.6308, + "theoretical_loss": 3.584162647822816, + "tokens_seen": 1212084224 + }, + { + "epoch": 4.01, + "learning_rate": 0.00031957873620862585, + "loss": 2.566, + "theoretical_loss": 3.5841448557865743, + "tokens_seen": 1212149760 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003195687061183551, + "loss": 2.5022, + "theoretical_loss": 3.584127064981578, + "tokens_seen": 1212215296 + }, + { + "epoch": 4.01, + "learning_rate": 0.00031955867602808427, + "loss": 2.831, + "theoretical_loss": 3.5841092754076733, + "tokens_seen": 1212280832 + }, + { + "epoch": 4.01, + "learning_rate": 0.00031954864593781345, + "loss": 2.3898, + "theoretical_loss": 3.5840914870647103, + "tokens_seen": 1212346368 + }, + { + "epoch": 4.01, + "objective/train/docs_used": 1414282, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5722250938415527, + "objective/train/theoretical_loss": 3.5840736999525356, + "objective/train/tokens_used": 1232871904, + "theoretical_loss": 3.5840736999525356, + "tokens_seen": 1212411904 + }, + { + "epoch": 4.01, + "learning_rate": 0.00031953861584754263, + "loss": 2.7378, + "theoretical_loss": 3.5840736999525356, + "tokens_seen": 1212411904 + }, + { + "epoch": 4.01, + "learning_rate": 0.00031952858575727186, + "loss": 2.5702, + "theoretical_loss": 3.5840559140709987, + "tokens_seen": 1212477440 + }, + { + "epoch": 4.01, + "learning_rate": 0.000319518555667001, + "loss": 2.7093, + "theoretical_loss": 3.5840381294199477, + "tokens_seen": 1212542976 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003195085255767302, + "loss": 2.8855, + "theoretical_loss": 3.584020345999231, + "tokens_seen": 1212608512 + }, + { + "epoch": 4.01, + "learning_rate": 0.00031949849548645935, + "loss": 2.7143, + "theoretical_loss": 3.5840025638086965, + "tokens_seen": 1212674048 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003194884653961886, + "loss": 2.7684, + "theoretical_loss": 3.5839847828481934, + "tokens_seen": 1212739584 + }, + { + "epoch": 4.01, + "learning_rate": 0.00031947843530591777, + "loss": 2.6283, + "theoretical_loss": 3.5839670031175697, + "tokens_seen": 1212805120 + }, + { + "epoch": 4.01, + "learning_rate": 0.00031946840521564695, + "loss": 2.4975, + "theoretical_loss": 3.5839492246166733, + "tokens_seen": 1212870656 + }, + { + "epoch": 4.01, + "learning_rate": 0.00031945837512537613, + "loss": 2.7088, + "theoretical_loss": 3.583931447345354, + "tokens_seen": 1212936192 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003194483450351053, + "loss": 2.6488, + "theoretical_loss": 3.5839136713034594, + "tokens_seen": 1213001728 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003194383149448345, + "loss": 2.6018, + "theoretical_loss": 3.583895896490838, + "tokens_seen": 1213067264 + }, + { + "epoch": 4.01, + "learning_rate": 0.00031942828485456373, + "loss": 2.7103, + "theoretical_loss": 3.5838781229073398, + "tokens_seen": 1213132800 + }, + { + "epoch": 4.01, + "learning_rate": 0.00031941825476429286, + "loss": 2.6472, + "theoretical_loss": 3.5838603505528113, + "tokens_seen": 1213198336 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003194082246740221, + "loss": 2.5772, + "theoretical_loss": 3.5838425794271025, + "tokens_seen": 1213263872 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003193981945837512, + "loss": 2.4455, + "theoretical_loss": 3.5838248095300616, + "tokens_seen": 1213329408 + }, + { + "epoch": 4.01, + "learning_rate": 0.00031938816449348045, + "loss": 2.7764, + "theoretical_loss": 3.583807040861538, + "tokens_seen": 1213394944 + }, + { + "epoch": 4.01, + "learning_rate": 0.00031937813440320964, + "loss": 2.7773, + "theoretical_loss": 3.583789273421379, + "tokens_seen": 1213460480 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003193681043129388, + "loss": 2.7144, + "theoretical_loss": 3.583771507209435, + "tokens_seen": 1213526016 + }, + { + "epoch": 4.01, + "learning_rate": 0.000319358074222668, + "loss": 2.6453, + "theoretical_loss": 3.5837537422255537, + "tokens_seen": 1213591552 + }, + { + "epoch": 4.01, + "learning_rate": 0.00031934804413239723, + "loss": 2.6259, + "theoretical_loss": 3.583735978469584, + "tokens_seen": 1213657088 + }, + { + "epoch": 4.01, + "learning_rate": 0.00031933801404212636, + "loss": 2.6677, + "theoretical_loss": 3.5837182159413747, + "tokens_seen": 1213722624 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003193279839518556, + "loss": 2.5048, + "theoretical_loss": 3.583700454640775, + "tokens_seen": 1213788160 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003193179538615847, + "loss": 2.6941, + "theoretical_loss": 3.583682694567634, + "tokens_seen": 1213853696 + }, + { + "epoch": 4.01, + "learning_rate": 0.00031930792377131396, + "loss": 2.821, + "theoretical_loss": 3.5836649357218002, + "tokens_seen": 1213919232 + }, + { + "epoch": 4.01, + "learning_rate": 0.00031929789368104314, + "loss": 2.8494, + "theoretical_loss": 3.5836471781031225, + "tokens_seen": 1213984768 + }, + { + "epoch": 4.01, + "objective/train/docs_used": 1415070, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7974677085876465, + "objective/train/theoretical_loss": 3.58362942171145, + "objective/train/tokens_used": 1234510304, + "theoretical_loss": 3.58362942171145, + "tokens_seen": 1214050304 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003192878635907723, + "loss": 2.8067, + "theoretical_loss": 3.58362942171145, + "tokens_seen": 1214050304 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003192778335005015, + "loss": 2.7528, + "theoretical_loss": 3.5836116665466315, + "tokens_seen": 1214115840 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003192678034102307, + "loss": 2.6348, + "theoretical_loss": 3.583593912608516, + "tokens_seen": 1214181376 + }, + { + "epoch": 4.01, + "learning_rate": 0.00031925777331995986, + "loss": 2.5474, + "theoretical_loss": 3.5835761598969524, + "tokens_seen": 1214246912 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003192477432296891, + "loss": 2.6987, + "theoretical_loss": 3.5835584084117906, + "tokens_seen": 1214312448 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003192377131394182, + "loss": 2.6957, + "theoretical_loss": 3.583540658152879, + "tokens_seen": 1214377984 + }, + { + "epoch": 4.01, + "learning_rate": 0.00031922768304914746, + "loss": 2.6414, + "theoretical_loss": 3.583522909120067, + "tokens_seen": 1214443520 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003192176529588766, + "loss": 2.6054, + "theoretical_loss": 3.583505161313204, + "tokens_seen": 1214509056 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003192076228686058, + "loss": 2.7971, + "theoretical_loss": 3.5834874147321383, + "tokens_seen": 1214574592 + }, + { + "epoch": 4.01, + "learning_rate": 0.000319197592778335, + "loss": 2.8113, + "theoretical_loss": 3.5834696693767194, + "tokens_seen": 1214640128 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003191875626880642, + "loss": 2.7911, + "theoretical_loss": 3.583451925246797, + "tokens_seen": 1214705664 + }, + { + "epoch": 4.01, + "learning_rate": 0.00031917753259779337, + "loss": 2.7277, + "theoretical_loss": 3.58343418234222, + "tokens_seen": 1214771200 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003191675025075226, + "loss": 2.6605, + "theoretical_loss": 3.583416440662838, + "tokens_seen": 1214836736 + }, + { + "epoch": 4.01, + "learning_rate": 0.00031915747241725173, + "loss": 2.6873, + "theoretical_loss": 3.5833987002085, + "tokens_seen": 1214902272 + }, + { + "epoch": 4.01, + "learning_rate": 0.00031914744232698096, + "loss": 2.6275, + "theoretical_loss": 3.5833809609790555, + "tokens_seen": 1214967808 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003191374122367101, + "loss": 2.6332, + "theoretical_loss": 3.5833632229743535, + "tokens_seen": 1215033344 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003191273821464393, + "loss": 2.6789, + "theoretical_loss": 3.5833454861942435, + "tokens_seen": 1215098880 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003191173520561685, + "loss": 2.7704, + "theoretical_loss": 3.583327750638576, + "tokens_seen": 1215164416 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003191073219658977, + "loss": 2.7698, + "theoretical_loss": 3.5833100163071987, + "tokens_seen": 1215229952 + }, + { + "epoch": 4.01, + "learning_rate": 0.00031909729187562687, + "loss": 2.8002, + "theoretical_loss": 3.5832922831999623, + "tokens_seen": 1215295488 + }, + { + "epoch": 4.01, + "learning_rate": 0.00031908726178535605, + "loss": 2.6626, + "theoretical_loss": 3.5832745513167152, + "tokens_seen": 1215361024 + }, + { + "epoch": 4.01, + "learning_rate": 0.00031907723169508523, + "loss": 2.6527, + "theoretical_loss": 3.5832568206573088, + "tokens_seen": 1215426560 + }, + { + "epoch": 4.01, + "learning_rate": 0.00031906720160481447, + "loss": 2.6216, + "theoretical_loss": 3.58323909122159, + "tokens_seen": 1215492096 + }, + { + "epoch": 4.01, + "learning_rate": 0.00031905717151454365, + "loss": 2.6442, + "theoretical_loss": 3.583221363009411, + "tokens_seen": 1215557632 + }, + { + "epoch": 4.01, + "learning_rate": 0.00031904714142427283, + "loss": 2.6907, + "theoretical_loss": 3.5832036360206203, + "tokens_seen": 1215623168 + }, + { + "epoch": 4.01, + "objective/train/docs_used": 1416232, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.866744041442871, + "objective/train/theoretical_loss": 3.5831859102550667, + "objective/train/tokens_used": 1236148704, + "theoretical_loss": 3.5831859102550667, + "tokens_seen": 1215688704 + }, + { + "epoch": 4.01, + "learning_rate": 0.00031903711133400206, + "loss": 2.7375, + "theoretical_loss": 3.5831859102550667, + "tokens_seen": 1215688704 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003190270812437312, + "loss": 2.9198, + "theoretical_loss": 3.583168185712601, + "tokens_seen": 1215754240 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003190170511534604, + "loss": 2.5112, + "theoretical_loss": 3.5831504623930726, + "tokens_seen": 1215819776 + }, + { + "epoch": 4.01, + "learning_rate": 0.00031900702106318955, + "loss": 2.8218, + "theoretical_loss": 3.583132740296331, + "tokens_seen": 1215885312 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003189969909729188, + "loss": 2.6373, + "theoretical_loss": 3.583115019422226, + "tokens_seen": 1215950848 + }, + { + "epoch": 4.01, + "learning_rate": 0.00031898696088264797, + "loss": 2.6703, + "theoretical_loss": 3.5830972997706074, + "tokens_seen": 1216016384 + }, + { + "epoch": 4.01, + "learning_rate": 0.00031897693079237715, + "loss": 2.6533, + "theoretical_loss": 3.583079581341325, + "tokens_seen": 1216081920 + }, + { + "epoch": 4.01, + "learning_rate": 0.00031896690070210633, + "loss": 2.705, + "theoretical_loss": 3.583061864134229, + "tokens_seen": 1216147456 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003189568706118355, + "loss": 2.8154, + "theoretical_loss": 3.583044148149168, + "tokens_seen": 1216212992 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003189468405215647, + "loss": 2.7847, + "theoretical_loss": 3.583026433385993, + "tokens_seen": 1216278528 + }, + { + "epoch": 4.01, + "learning_rate": 0.00031893681043129393, + "loss": 2.7064, + "theoretical_loss": 3.583008719844554, + "tokens_seen": 1216344064 + }, + { + "epoch": 4.01, + "learning_rate": 0.00031892678034102306, + "loss": 2.677, + "theoretical_loss": 3.5829910075247007, + "tokens_seen": 1216409600 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003189167502507523, + "loss": 2.5897, + "theoretical_loss": 3.5829732964262826, + "tokens_seen": 1216475136 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003189067201604814, + "loss": 2.6573, + "theoretical_loss": 3.58295558654915, + "tokens_seen": 1216540672 + }, + { + "epoch": 4.01, + "learning_rate": 0.00031889669007021065, + "loss": 2.7557, + "theoretical_loss": 3.582937877893153, + "tokens_seen": 1216606208 + }, + { + "epoch": 4.01, + "learning_rate": 0.00031888665997993984, + "loss": 2.7416, + "theoretical_loss": 3.582920170458141, + "tokens_seen": 1216671744 + }, + { + "epoch": 4.01, + "learning_rate": 0.000318876629889669, + "loss": 2.6819, + "theoretical_loss": 3.582902464243965, + "tokens_seen": 1216737280 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003188665997993982, + "loss": 2.6083, + "theoretical_loss": 3.582884759250474, + "tokens_seen": 1216802816 + }, + { + "epoch": 4.01, + "learning_rate": 0.00031885656970912743, + "loss": 2.7381, + "theoretical_loss": 3.5828670554775197, + "tokens_seen": 1216868352 + }, + { + "epoch": 4.01, + "learning_rate": 0.00031884653961885656, + "loss": 2.8159, + "theoretical_loss": 3.5828493529249505, + "tokens_seen": 1216933888 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003188365095285858, + "loss": 2.5582, + "theoretical_loss": 3.5828316515926177, + "tokens_seen": 1216999424 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003188264794383149, + "loss": 2.7293, + "theoretical_loss": 3.5828139514803707, + "tokens_seen": 1217064960 + }, + { + "epoch": 4.01, + "learning_rate": 0.00031881644934804416, + "loss": 2.5561, + "theoretical_loss": 3.582796252588061, + "tokens_seen": 1217130496 + }, + { + "epoch": 4.01, + "learning_rate": 0.00031880641925777334, + "loss": 2.8685, + "theoretical_loss": 3.582778554915537, + "tokens_seen": 1217196032 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003187963891675025, + "loss": 2.5221, + "theoretical_loss": 3.5827608584626502, + "tokens_seen": 1217261568 + }, + { + "epoch": 4.01, + "objective/train/docs_used": 1416846, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6486992835998535, + "objective/train/theoretical_loss": 3.5827431632292503, + "objective/train/tokens_used": 1237787104, + "theoretical_loss": 3.5827431632292503, + "tokens_seen": 1217327104 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003187863590772317, + "loss": 2.7341, + "theoretical_loss": 3.5827431632292503, + "tokens_seen": 1217327104 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003187763289869609, + "loss": 2.8365, + "theoretical_loss": 3.5827254692151884, + "tokens_seen": 1217392640 + }, + { + "epoch": 4.01, + "learning_rate": 0.00031876629889669006, + "loss": 2.6498, + "theoretical_loss": 3.582707776420315, + "tokens_seen": 1217458176 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003187562688064193, + "loss": 2.532, + "theoretical_loss": 3.5826900848444785, + "tokens_seen": 1217523712 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003187462387161484, + "loss": 2.4932, + "theoretical_loss": 3.582672394487531, + "tokens_seen": 1217589248 + }, + { + "epoch": 4.01, + "learning_rate": 0.00031873620862587766, + "loss": 2.7149, + "theoretical_loss": 3.582654705349323, + "tokens_seen": 1217654784 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003187261785356068, + "loss": 2.8224, + "theoretical_loss": 3.5826370174297044, + "tokens_seen": 1217720320 + }, + { + "epoch": 4.01, + "learning_rate": 0.000318716148445336, + "loss": 2.4685, + "theoretical_loss": 3.5826193307285257, + "tokens_seen": 1217785856 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003187061183550652, + "loss": 2.5885, + "theoretical_loss": 3.5826016452456377, + "tokens_seen": 1217851392 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003186960882647944, + "loss": 2.7836, + "theoretical_loss": 3.5825839609808905, + "tokens_seen": 1217916928 + }, + { + "epoch": 4.01, + "learning_rate": 0.00031868605817452357, + "loss": 2.8991, + "theoretical_loss": 3.5825662779341347, + "tokens_seen": 1217982464 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003186760280842528, + "loss": 2.5322, + "theoretical_loss": 3.5825485961052213, + "tokens_seen": 1218048000 + }, + { + "epoch": 4.01, + "learning_rate": 0.00031866599799398193, + "loss": 2.5996, + "theoretical_loss": 3.582530915494001, + "tokens_seen": 1218113536 + }, + { + "epoch": 4.01, + "learning_rate": 0.00031865596790371116, + "loss": 2.7052, + "theoretical_loss": 3.5825132361003234, + "tokens_seen": 1218179072 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003186459378134403, + "loss": 2.6446, + "theoretical_loss": 3.58249555792404, + "tokens_seen": 1218244608 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003186359077231695, + "loss": 2.7679, + "theoretical_loss": 3.5824778809650017, + "tokens_seen": 1218310144 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003186258776328987, + "loss": 2.6478, + "theoretical_loss": 3.582460205223059, + "tokens_seen": 1218375680 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003186158475426279, + "loss": 2.5404, + "theoretical_loss": 3.582442530698062, + "tokens_seen": 1218441216 + }, + { + "epoch": 4.01, + "learning_rate": 0.00031860581745235707, + "loss": 2.7493, + "theoretical_loss": 3.5824248573898623, + "tokens_seen": 1218506752 + }, + { + "epoch": 4.01, + "learning_rate": 0.00031859578736208625, + "loss": 2.6604, + "theoretical_loss": 3.58240718529831, + "tokens_seen": 1218572288 + }, + { + "epoch": 4.01, + "learning_rate": 0.00031858575727181543, + "loss": 2.6678, + "theoretical_loss": 3.582389514423257, + "tokens_seen": 1218637824 + }, + { + "epoch": 4.01, + "learning_rate": 0.00031857572718154467, + "loss": 2.9431, + "theoretical_loss": 3.5823718447645527, + "tokens_seen": 1218703360 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003185656970912738, + "loss": 2.8017, + "theoretical_loss": 3.582354176322049, + "tokens_seen": 1218768896 + }, + { + "epoch": 4.01, + "learning_rate": 0.00031855566700100303, + "loss": 2.5855, + "theoretical_loss": 3.5823365090955965, + "tokens_seen": 1218834432 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003185456369107322, + "loss": 2.7062, + "theoretical_loss": 3.582318843085046, + "tokens_seen": 1218899968 + }, + { + "epoch": 4.01, + "objective/train/docs_used": 1418472, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8614342212677, + "objective/train/theoretical_loss": 3.5823011782902485, + "objective/train/tokens_used": 1239425504, + "theoretical_loss": 3.5823011782902485, + "tokens_seen": 1218965504 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003185356068204614, + "loss": 2.7619, + "theoretical_loss": 3.5823011782902485, + "tokens_seen": 1218965504 + }, + { + "epoch": 4.01, + "learning_rate": 0.00031852557673019057, + "loss": 2.8181, + "theoretical_loss": 3.5822835147110554, + "tokens_seen": 1219031040 + }, + { + "epoch": 4.01, + "learning_rate": 0.00031851554663991975, + "loss": 2.5943, + "theoretical_loss": 3.582265852347317, + "tokens_seen": 1219096576 + }, + { + "epoch": 4.01, + "learning_rate": 0.00031850551654964893, + "loss": 2.6939, + "theoretical_loss": 3.582248191198885, + "tokens_seen": 1219162112 + }, + { + "epoch": 4.01, + "learning_rate": 0.00031849548645937817, + "loss": 2.5154, + "theoretical_loss": 3.58223053126561, + "tokens_seen": 1219227648 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003184854563691073, + "loss": 2.5464, + "theoretical_loss": 3.5822128725473434, + "tokens_seen": 1219293184 + }, + { + "epoch": 4.01, + "learning_rate": 0.00031847542627883653, + "loss": 2.5821, + "theoretical_loss": 3.582195215043936, + "tokens_seen": 1219358720 + }, + { + "epoch": 4.01, + "learning_rate": 0.00031846539618856566, + "loss": 2.4363, + "theoretical_loss": 3.582177558755239, + "tokens_seen": 1219424256 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003184553660982949, + "loss": 2.6333, + "theoretical_loss": 3.5821599036811036, + "tokens_seen": 1219489792 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003184453360080241, + "loss": 2.7076, + "theoretical_loss": 3.582142249821381, + "tokens_seen": 1219555328 + }, + { + "epoch": 4.01, + "learning_rate": 0.00031843530591775326, + "loss": 2.626, + "theoretical_loss": 3.5821245971759224, + "tokens_seen": 1219620864 + }, + { + "epoch": 4.01, + "learning_rate": 0.00031842527582748244, + "loss": 2.4653, + "theoretical_loss": 3.582106945744579, + "tokens_seen": 1219686400 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003184152457372116, + "loss": 2.5816, + "theoretical_loss": 3.5820892955272026, + "tokens_seen": 1219751936 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003184052156469408, + "loss": 2.7463, + "theoretical_loss": 3.5820716465236435, + "tokens_seen": 1219817472 + }, + { + "epoch": 4.01, + "learning_rate": 0.00031839518555667004, + "loss": 2.65, + "theoretical_loss": 3.5820539987337536, + "tokens_seen": 1219883008 + }, + { + "epoch": 4.01, + "learning_rate": 0.00031838515546639916, + "loss": 2.8067, + "theoretical_loss": 3.5820363521573846, + "tokens_seen": 1219948544 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003183751253761284, + "loss": 2.7794, + "theoretical_loss": 3.582018706794387, + "tokens_seen": 1220014080 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003183650952858576, + "loss": 2.6482, + "theoretical_loss": 3.582001062644613, + "tokens_seen": 1220079616 + }, + { + "epoch": 4.01, + "learning_rate": 0.00031835506519558676, + "loss": 2.542, + "theoretical_loss": 3.5819834197079135, + "tokens_seen": 1220145152 + }, + { + "epoch": 4.01, + "learning_rate": 0.00031834503510531594, + "loss": 2.5878, + "theoretical_loss": 3.58196577798414, + "tokens_seen": 1220210688 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003183350050150451, + "loss": 2.4764, + "theoretical_loss": 3.581948137473144, + "tokens_seen": 1220276224 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003183249749247743, + "loss": 2.6947, + "theoretical_loss": 3.5819304981747777, + "tokens_seen": 1220341760 + }, + { + "epoch": 4.01, + "learning_rate": 0.00031831494483450354, + "loss": 2.6588, + "theoretical_loss": 3.5819128600888916, + "tokens_seen": 1220407296 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003183049147442327, + "loss": 2.7199, + "theoretical_loss": 3.581895223215337, + "tokens_seen": 1220472832 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003182948846539619, + "loss": 2.6414, + "theoretical_loss": 3.5818775875539672, + "tokens_seen": 1220538368 + }, + { + "epoch": 4.01, + "objective/train/docs_used": 1419245, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.898358106613159, + "objective/train/theoretical_loss": 3.5818599531046322, + "objective/train/tokens_used": 1241063904, + "theoretical_loss": 3.5818599531046322, + "tokens_seen": 1220603904 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003182848545636911, + "loss": 2.807, + "theoretical_loss": 3.5818599531046322, + "tokens_seen": 1220603904 + }, + { + "epoch": 4.01, + "learning_rate": 0.00031827482447342026, + "loss": 2.5729, + "theoretical_loss": 3.5818423198671847, + "tokens_seen": 1220669440 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003182647943831495, + "loss": 2.6056, + "theoretical_loss": 3.5818246878414755, + "tokens_seen": 1220734976 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003182547642928786, + "loss": 2.5781, + "theoretical_loss": 3.5818070570273566, + "tokens_seen": 1220800512 + }, + { + "epoch": 4.01, + "learning_rate": 0.00031824473420260786, + "loss": 2.726, + "theoretical_loss": 3.58178942742468, + "tokens_seen": 1220866048 + }, + { + "epoch": 4.01, + "learning_rate": 0.000318234704112337, + "loss": 2.7488, + "theoretical_loss": 3.581771799033297, + "tokens_seen": 1220931584 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003182246740220662, + "loss": 2.4787, + "theoretical_loss": 3.58175417185306, + "tokens_seen": 1220997120 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003182146439317954, + "loss": 2.6335, + "theoretical_loss": 3.5817365458838193, + "tokens_seen": 1221062656 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003182046138415246, + "loss": 2.6146, + "theoretical_loss": 3.581718921125429, + "tokens_seen": 1221128192 + }, + { + "epoch": 4.01, + "learning_rate": 0.00031819458375125377, + "loss": 2.8097, + "theoretical_loss": 3.5817012975777387, + "tokens_seen": 1221193728 + }, + { + "epoch": 4.01, + "learning_rate": 0.000318184553660983, + "loss": 2.6973, + "theoretical_loss": 3.5816836752406016, + "tokens_seen": 1221259264 + }, + { + "epoch": 4.01, + "learning_rate": 0.00031817452357071213, + "loss": 2.677, + "theoretical_loss": 3.581666054113869, + "tokens_seen": 1221324800 + }, + { + "epoch": 4.01, + "learning_rate": 0.00031816449348044136, + "loss": 2.6547, + "theoretical_loss": 3.5816484341973935, + "tokens_seen": 1221390336 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003181544633901705, + "loss": 2.465, + "theoretical_loss": 3.5816308154910264, + "tokens_seen": 1221455872 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003181444332998997, + "loss": 2.6044, + "theoretical_loss": 3.58161319799462, + "tokens_seen": 1221521408 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003181344032096289, + "loss": 2.7337, + "theoretical_loss": 3.5815955817080263, + "tokens_seen": 1221586944 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003181243731193581, + "loss": 2.8371, + "theoretical_loss": 3.581577966631097, + "tokens_seen": 1221652480 + }, + { + "epoch": 4.01, + "learning_rate": 0.00031811434302908727, + "loss": 2.5997, + "theoretical_loss": 3.5815603527636846, + "tokens_seen": 1221718016 + }, + { + "epoch": 4.01, + "learning_rate": 0.00031810431293881645, + "loss": 2.5927, + "theoretical_loss": 3.5815427401056406, + "tokens_seen": 1221783552 + }, + { + "epoch": 4.01, + "learning_rate": 0.00031809428284854563, + "loss": 2.7329, + "theoretical_loss": 3.5815251286568177, + "tokens_seen": 1221849088 + }, + { + "epoch": 4.01, + "learning_rate": 0.00031808425275827487, + "loss": 2.6258, + "theoretical_loss": 3.5815075184170673, + "tokens_seen": 1221914624 + }, + { + "epoch": 4.01, + "learning_rate": 0.000318074222668004, + "loss": 2.8222, + "theoretical_loss": 3.5814899093862422, + "tokens_seen": 1221980160 + }, + { + "epoch": 4.01, + "learning_rate": 0.00031806419257773323, + "loss": 3.0164, + "theoretical_loss": 3.5814723015641947, + "tokens_seen": 1222045696 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003180541624874624, + "loss": 2.7932, + "theoretical_loss": 3.5814546949507764, + "tokens_seen": 1222111232 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003180441323971916, + "loss": 2.6886, + "theoretical_loss": 3.5814370895458403, + "tokens_seen": 1222176768 + }, + { + "epoch": 4.01, + "objective/train/docs_used": 1420806, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.511540174484253, + "objective/train/theoretical_loss": 3.5814194853492376, + "objective/train/tokens_used": 1242702304, + "theoretical_loss": 3.5814194853492376, + "tokens_seen": 1222242304 + }, + { + "epoch": 4.01, + "learning_rate": 0.00031803410230692077, + "loss": 2.626, + "theoretical_loss": 3.5814194853492376, + "tokens_seen": 1222242304 + }, + { + "epoch": 4.01, + "learning_rate": 0.00031802407221664995, + "loss": 2.7816, + "theoretical_loss": 3.5814018823608214, + "tokens_seen": 1222307840 + }, + { + "epoch": 4.01, + "learning_rate": 0.00031801404212637913, + "loss": 2.6152, + "theoretical_loss": 3.5813842805804437, + "tokens_seen": 1222373376 + }, + { + "epoch": 4.01, + "learning_rate": 0.00031800401203610837, + "loss": 2.6969, + "theoretical_loss": 3.581366680007957, + "tokens_seen": 1222438912 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003179939819458375, + "loss": 2.663, + "theoretical_loss": 3.581349080643214, + "tokens_seen": 1222504448 + }, + { + "epoch": 4.01, + "learning_rate": 0.00031798395185556673, + "loss": 2.8343, + "theoretical_loss": 3.581331482486066, + "tokens_seen": 1222569984 + }, + { + "epoch": 4.01, + "learning_rate": 0.00031797392176529586, + "loss": 2.7702, + "theoretical_loss": 3.581313885536366, + "tokens_seen": 1222635520 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003179638916750251, + "loss": 2.7763, + "theoretical_loss": 3.581296289793967, + "tokens_seen": 1222701056 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003179538615847543, + "loss": 2.8597, + "theoretical_loss": 3.581278695258721, + "tokens_seen": 1222766592 + }, + { + "epoch": 4.01, + "learning_rate": 0.00031794383149448346, + "loss": 2.7061, + "theoretical_loss": 3.58126110193048, + "tokens_seen": 1222832128 + }, + { + "epoch": 4.01, + "learning_rate": 0.00031793380140421264, + "loss": 2.643, + "theoretical_loss": 3.5812435098090973, + "tokens_seen": 1222897664 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003179237713139418, + "loss": 2.85, + "theoretical_loss": 3.5812259188944253, + "tokens_seen": 1222963200 + }, + { + "epoch": 4.01, + "learning_rate": 0.000317913741223671, + "loss": 2.6904, + "theoretical_loss": 3.5812083291863157, + "tokens_seen": 1223028736 + }, + { + "epoch": 4.01, + "learning_rate": 0.00031790371113340024, + "loss": 2.6822, + "theoretical_loss": 3.5811907406846224, + "tokens_seen": 1223094272 + }, + { + "epoch": 4.01, + "learning_rate": 0.00031789368104312936, + "loss": 2.7356, + "theoretical_loss": 3.5811731533891975, + "tokens_seen": 1223159808 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003178836509528586, + "loss": 2.7586, + "theoretical_loss": 3.581155567299893, + "tokens_seen": 1223225344 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003178736208625878, + "loss": 2.8288, + "theoretical_loss": 3.581137982416563, + "tokens_seen": 1223290880 + }, + { + "epoch": 4.01, + "learning_rate": 0.00031786359077231696, + "loss": 2.5823, + "theoretical_loss": 3.581120398739059, + "tokens_seen": 1223356416 + }, + { + "epoch": 4.01, + "learning_rate": 0.00031785356068204614, + "loss": 2.5514, + "theoretical_loss": 3.581102816267234, + "tokens_seen": 1223421952 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003178435305917753, + "loss": 2.6135, + "theoretical_loss": 3.5810852350009403, + "tokens_seen": 1223487488 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003178335005015045, + "loss": 2.6177, + "theoretical_loss": 3.5810676549400315, + "tokens_seen": 1223553024 + }, + { + "epoch": 4.01, + "learning_rate": 0.00031782347041123374, + "loss": 2.7085, + "theoretical_loss": 3.5810500760843604, + "tokens_seen": 1223618560 + }, + { + "epoch": 4.01, + "learning_rate": 0.00031781344032096287, + "loss": 2.257, + "theoretical_loss": 3.581032498433779, + "tokens_seen": 1223684096 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003178034102306921, + "loss": 2.7905, + "theoretical_loss": 3.581014921988141, + "tokens_seen": 1223749632 + }, + { + "epoch": 4.01, + "learning_rate": 0.00031779338014042123, + "loss": 2.6344, + "theoretical_loss": 3.580997346747299, + "tokens_seen": 1223815168 + }, + { + "epoch": 4.01, + "objective/train/docs_used": 1421310, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.000223159790039, + "objective/train/theoretical_loss": 3.580979772711106, + "objective/train/tokens_used": 1244340704, + "theoretical_loss": 3.580979772711106, + "tokens_seen": 1223880704 + }, + { + "epoch": 4.01, + "learning_rate": 0.00031778335005015046, + "loss": 2.6517, + "theoretical_loss": 3.580979772711106, + "tokens_seen": 1223880704 + }, + { + "epoch": 4.01, + "learning_rate": 0.00031777331995987964, + "loss": 2.7078, + "theoretical_loss": 3.580962199879414, + "tokens_seen": 1223946240 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003177632898696088, + "loss": 2.8244, + "theoretical_loss": 3.580944628252077, + "tokens_seen": 1224011776 + }, + { + "epoch": 4.01, + "learning_rate": 0.000317753259779338, + "loss": 2.7608, + "theoretical_loss": 3.5809270578289483, + "tokens_seen": 1224077312 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003177432296890672, + "loss": 2.671, + "theoretical_loss": 3.58090948860988, + "tokens_seen": 1224142848 + }, + { + "epoch": 4.01, + "learning_rate": 0.00031773319959879637, + "loss": 2.6531, + "theoretical_loss": 3.580891920594725, + "tokens_seen": 1224208384 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003177231695085256, + "loss": 2.5167, + "theoretical_loss": 3.5808743537833374, + "tokens_seen": 1224273920 + }, + { + "epoch": 4.01, + "learning_rate": 0.00031771313941825473, + "loss": 2.7415, + "theoretical_loss": 3.58085678817557, + "tokens_seen": 1224339456 + }, + { + "epoch": 4.01, + "learning_rate": 0.00031770310932798397, + "loss": 2.5674, + "theoretical_loss": 3.580839223771275, + "tokens_seen": 1224404992 + }, + { + "epoch": 4.01, + "learning_rate": 0.00031769307923771315, + "loss": 2.7093, + "theoretical_loss": 3.5808216605703063, + "tokens_seen": 1224470528 + }, + { + "epoch": 4.01, + "learning_rate": 0.00031768304914744233, + "loss": 2.5677, + "theoretical_loss": 3.580804098572517, + "tokens_seen": 1224536064 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003176730190571715, + "loss": 2.8175, + "theoretical_loss": 3.58078653777776, + "tokens_seen": 1224601600 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003176629889669007, + "loss": 2.701, + "theoretical_loss": 3.580768978185889, + "tokens_seen": 1224667136 + }, + { + "epoch": 4.01, + "learning_rate": 0.00031765295887662987, + "loss": 2.7692, + "theoretical_loss": 3.5807514197967567, + "tokens_seen": 1224732672 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003176429287863591, + "loss": 2.7475, + "theoretical_loss": 3.580733862610217, + "tokens_seen": 1224798208 + }, + { + "epoch": 4.01, + "learning_rate": 0.00031763289869608823, + "loss": 2.8054, + "theoretical_loss": 3.5807163066261225, + "tokens_seen": 1224863744 + }, + { + "epoch": 4.01, + "learning_rate": 0.00031762286860581747, + "loss": 2.6037, + "theoretical_loss": 3.580698751844327, + "tokens_seen": 1224929280 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003176128385155466, + "loss": 2.5773, + "theoretical_loss": 3.580681198264684, + "tokens_seen": 1224994816 + }, + { + "epoch": 4.01, + "learning_rate": 0.00031760280842527583, + "loss": 2.7677, + "theoretical_loss": 3.580663645887046, + "tokens_seen": 1225060352 + }, + { + "epoch": 4.01, + "learning_rate": 0.000317592778335005, + "loss": 2.6867, + "theoretical_loss": 3.580646094711267, + "tokens_seen": 1225125888 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003175827482447342, + "loss": 2.708, + "theoretical_loss": 3.5806285447372006, + "tokens_seen": 1225191424 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003175727181544634, + "loss": 2.8949, + "theoretical_loss": 3.5806109959647, + "tokens_seen": 1225256960 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003175626880641926, + "loss": 2.3915, + "theoretical_loss": 3.5805934483936186, + "tokens_seen": 1225322496 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003175526579739218, + "loss": 2.7247, + "theoretical_loss": 3.58057590202381, + "tokens_seen": 1225388032 + }, + { + "epoch": 4.01, + "learning_rate": 0.00031754262788365097, + "loss": 2.745, + "theoretical_loss": 3.5805583568551276, + "tokens_seen": 1225453568 + }, + { + "epoch": 4.01, + "objective/train/docs_used": 1422763, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5950467586517334, + "objective/train/theoretical_loss": 3.5805408128874254, + "objective/train/tokens_used": 1245979104, + "theoretical_loss": 3.5805408128874254, + "tokens_seen": 1225519104 + }, + { + "epoch": 4.01, + "learning_rate": 0.00031753259779338015, + "loss": 2.619, + "theoretical_loss": 3.5805408128874254, + "tokens_seen": 1225519104 + }, + { + "epoch": 4.01, + "learning_rate": 0.00031752256770310933, + "loss": 2.767, + "theoretical_loss": 3.5805232701205565, + "tokens_seen": 1225584640 + }, + { + "epoch": 4.01, + "learning_rate": 0.00031751253761283857, + "loss": 2.8689, + "theoretical_loss": 3.580505728554374, + "tokens_seen": 1225650176 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003175025075225677, + "loss": 2.6551, + "theoretical_loss": 3.580488188188733, + "tokens_seen": 1225715712 + }, + { + "epoch": 4.01, + "learning_rate": 0.00031749247743229693, + "loss": 2.7369, + "theoretical_loss": 3.5804706490234857, + "tokens_seen": 1225781248 + }, + { + "epoch": 4.01, + "learning_rate": 0.00031748244734202606, + "loss": 2.4896, + "theoretical_loss": 3.580453111058487, + "tokens_seen": 1225846784 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003174724172517553, + "loss": 2.6095, + "theoretical_loss": 3.58043557429359, + "tokens_seen": 1225912320 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003174623871614845, + "loss": 2.8006, + "theoretical_loss": 3.5804180387286477, + "tokens_seen": 1225977856 + }, + { + "epoch": 4.01, + "learning_rate": 0.00031745235707121366, + "loss": 2.6738, + "theoretical_loss": 3.5804005043635154, + "tokens_seen": 1226043392 + }, + { + "epoch": 4.01, + "learning_rate": 0.00031744232698094284, + "loss": 2.7993, + "theoretical_loss": 3.5803829711980453, + "tokens_seen": 1226108928 + }, + { + "epoch": 4.01, + "learning_rate": 0.000317432296890672, + "loss": 2.5852, + "theoretical_loss": 3.580365439232092, + "tokens_seen": 1226174464 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003174222668004012, + "loss": 2.6334, + "theoretical_loss": 3.5803479084655097, + "tokens_seen": 1226240000 + }, + { + "epoch": 4.01, + "learning_rate": 0.00031741223671013044, + "loss": 2.8156, + "theoretical_loss": 3.580330378898152, + "tokens_seen": 1226305536 + }, + { + "epoch": 4.01, + "learning_rate": 0.00031740220661985956, + "loss": 2.5313, + "theoretical_loss": 3.5803128505298725, + "tokens_seen": 1226371072 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003173921765295888, + "loss": 2.848, + "theoretical_loss": 3.5802953233605246, + "tokens_seen": 1226436608 + }, + { + "epoch": 4.01, + "learning_rate": 0.000317382146439318, + "loss": 2.7893, + "theoretical_loss": 3.5802777973899635, + "tokens_seen": 1226502144 + }, + { + "epoch": 4.01, + "learning_rate": 0.00031737211634904716, + "loss": 2.5006, + "theoretical_loss": 3.580260272618043, + "tokens_seen": 1226567680 + }, + { + "epoch": 4.01, + "learning_rate": 0.00031736208625877634, + "loss": 2.6533, + "theoretical_loss": 3.580242749044616, + "tokens_seen": 1226633216 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003173520561685055, + "loss": 2.6162, + "theoretical_loss": 3.580225226669537, + "tokens_seen": 1226698752 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003173420260782347, + "loss": 2.7069, + "theoretical_loss": 3.5802077054926604, + "tokens_seen": 1226764288 + }, + { + "epoch": 4.01, + "learning_rate": 0.00031733199598796394, + "loss": 2.8578, + "theoretical_loss": 3.58019018551384, + "tokens_seen": 1226829824 + }, + { + "epoch": 4.01, + "learning_rate": 0.00031732196589769307, + "loss": 2.5383, + "theoretical_loss": 3.5801726667329294, + "tokens_seen": 1226895360 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003173119358074223, + "loss": 2.6596, + "theoretical_loss": 3.5801551491497836, + "tokens_seen": 1226960896 + }, + { + "epoch": 4.01, + "learning_rate": 0.00031730190571715143, + "loss": 2.6207, + "theoretical_loss": 3.5801376327642567, + "tokens_seen": 1227026432 + }, + { + "epoch": 4.01, + "learning_rate": 0.00031729187562688066, + "loss": 2.6281, + "theoretical_loss": 3.5801201175762025, + "tokens_seen": 1227091968 + }, + { + "epoch": 4.01, + "objective/train/docs_used": 1423509, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.779162645339966, + "objective/train/theoretical_loss": 3.580102603585475, + "objective/train/tokens_used": 1247617504, + "theoretical_loss": 3.580102603585475, + "tokens_seen": 1227157504 + }, + { + "epoch": 4.01, + "learning_rate": 0.00031728184553660984, + "loss": 2.7531, + "theoretical_loss": 3.580102603585475, + "tokens_seen": 1227157504 + }, + { + "epoch": 4.01, + "learning_rate": 0.000317271815446339, + "loss": 2.6181, + "theoretical_loss": 3.5800850907919286, + "tokens_seen": 1227223040 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003172617853560682, + "loss": 2.6957, + "theoretical_loss": 3.5800675791954175, + "tokens_seen": 1227288576 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003172517552657974, + "loss": 2.5925, + "theoretical_loss": 3.5800500687957966, + "tokens_seen": 1227354112 + }, + { + "epoch": 4.01, + "learning_rate": 0.00031724172517552657, + "loss": 2.5597, + "theoretical_loss": 3.5800325595929188, + "tokens_seen": 1227419648 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003172316950852558, + "loss": 2.7134, + "theoretical_loss": 3.58001505158664, + "tokens_seen": 1227485184 + }, + { + "epoch": 4.01, + "learning_rate": 0.00031722166499498493, + "loss": 2.4009, + "theoretical_loss": 3.5799975447768135, + "tokens_seen": 1227550720 + }, + { + "epoch": 4.01, + "learning_rate": 0.00031721163490471417, + "loss": 2.7466, + "theoretical_loss": 3.579980039163294, + "tokens_seen": 1227616256 + }, + { + "epoch": 4.01, + "learning_rate": 0.00031720160481444335, + "loss": 2.7712, + "theoretical_loss": 3.5799625347459356, + "tokens_seen": 1227681792 + }, + { + "epoch": 4.01, + "learning_rate": 0.00031719157472417253, + "loss": 2.5761, + "theoretical_loss": 3.579945031524593, + "tokens_seen": 1227747328 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003171815446339017, + "loss": 2.5698, + "theoretical_loss": 3.5799275294991206, + "tokens_seen": 1227812864 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003171715145436309, + "loss": 2.7248, + "theoretical_loss": 3.5799100286693735, + "tokens_seen": 1227878400 + }, + { + "epoch": 4.01, + "learning_rate": 0.00031716148445336007, + "loss": 2.7414, + "theoretical_loss": 3.579892529035205, + "tokens_seen": 1227943936 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003171514543630893, + "loss": 2.6472, + "theoretical_loss": 3.57987503059647, + "tokens_seen": 1228009472 + }, + { + "epoch": 4.01, + "learning_rate": 0.00031714142427281843, + "loss": 2.7689, + "theoretical_loss": 3.579857533353024, + "tokens_seen": 1228075008 + }, + { + "epoch": 4.01, + "learning_rate": 0.00031713139418254767, + "loss": 2.5172, + "theoretical_loss": 3.57984003730472, + "tokens_seen": 1228140544 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003171213640922768, + "loss": 2.7424, + "theoretical_loss": 3.5798225424514145, + "tokens_seen": 1228206080 + }, + { + "epoch": 4.01, + "learning_rate": 0.00031711133400200603, + "loss": 2.6671, + "theoretical_loss": 3.5798050487929602, + "tokens_seen": 1228271616 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003171013039117352, + "loss": 2.4961, + "theoretical_loss": 3.5797875563292125, + "tokens_seen": 1228337152 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003170912738214644, + "loss": 2.8536, + "theoretical_loss": 3.5797700650600266, + "tokens_seen": 1228402688 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003170812437311936, + "loss": 2.6944, + "theoretical_loss": 3.579752574985257, + "tokens_seen": 1228468224 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003170712136409228, + "loss": 2.6289, + "theoretical_loss": 3.5797350861047574, + "tokens_seen": 1228533760 + }, + { + "epoch": 4.01, + "learning_rate": 0.00031706118355065194, + "loss": 2.8619, + "theoretical_loss": 3.5797175984183838, + "tokens_seen": 1228599296 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003170511534603812, + "loss": 2.6812, + "theoretical_loss": 3.5797001119259906, + "tokens_seen": 1228664832 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003170411233701103, + "loss": 2.5994, + "theoretical_loss": 3.5796826266274326, + "tokens_seen": 1228730368 + }, + { + "epoch": 4.01, + "objective/train/docs_used": 1425069, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5875046253204346, + "objective/train/theoretical_loss": 3.5796651425225643, + "objective/train/tokens_used": 1249255904, + "theoretical_loss": 3.5796651425225643, + "tokens_seen": 1228795904 + }, + { + "epoch": 4.01, + "learning_rate": 0.00031703109327983954, + "loss": 2.6018, + "theoretical_loss": 3.5796651425225643, + "tokens_seen": 1228795904 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003170210631895687, + "loss": 2.8186, + "theoretical_loss": 3.5796476596112408, + "tokens_seen": 1228861440 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003170110330992979, + "loss": 2.7534, + "theoretical_loss": 3.5796301778933173, + "tokens_seen": 1228926976 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003170010030090271, + "loss": 2.7221, + "theoretical_loss": 3.579612697368648, + "tokens_seen": 1228992512 + }, + { + "epoch": 4.01, + "learning_rate": 0.00031699097291875626, + "loss": 2.6292, + "theoretical_loss": 3.5795952180370882, + "tokens_seen": 1229058048 + }, + { + "epoch": 4.01, + "learning_rate": 0.00031698094282848544, + "loss": 2.5586, + "theoretical_loss": 3.579577739898493, + "tokens_seen": 1229123584 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003169709127382147, + "loss": 2.5425, + "theoretical_loss": 3.5795602629527172, + "tokens_seen": 1229189120 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003169608826479438, + "loss": 2.4003, + "theoretical_loss": 3.5795427871996157, + "tokens_seen": 1229254656 + }, + { + "epoch": 4.01, + "learning_rate": 0.00031695085255767304, + "loss": 2.7679, + "theoretical_loss": 3.579525312639044, + "tokens_seen": 1229320192 + }, + { + "epoch": 4.01, + "learning_rate": 0.00031694082246740217, + "loss": 2.8011, + "theoretical_loss": 3.579507839270857, + "tokens_seen": 1229385728 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003169307923771314, + "loss": 2.7587, + "theoretical_loss": 3.579490367094909, + "tokens_seen": 1229451264 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003169207622868606, + "loss": 2.7321, + "theoretical_loss": 3.579472896111056, + "tokens_seen": 1229516800 + }, + { + "epoch": 4.01, + "learning_rate": 0.00031691073219658976, + "loss": 2.8625, + "theoretical_loss": 3.5794554263191527, + "tokens_seen": 1229582336 + }, + { + "epoch": 4.01, + "learning_rate": 0.00031690070210631894, + "loss": 2.5965, + "theoretical_loss": 3.5794379577190547, + "tokens_seen": 1229647872 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003168906720160482, + "loss": 2.7067, + "theoretical_loss": 3.579420490310617, + "tokens_seen": 1229713408 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003168806419257773, + "loss": 2.8471, + "theoretical_loss": 3.5794030240936943, + "tokens_seen": 1229778944 + }, + { + "epoch": 4.01, + "learning_rate": 0.00031687061183550654, + "loss": 2.7518, + "theoretical_loss": 3.5793855590681423, + "tokens_seen": 1229844480 + }, + { + "epoch": 4.01, + "learning_rate": 0.00031686058174523567, + "loss": 2.8173, + "theoretical_loss": 3.579368095233816, + "tokens_seen": 1229910016 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003168505516549649, + "loss": 2.6816, + "theoretical_loss": 3.579350632590571, + "tokens_seen": 1229975552 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003168405215646941, + "loss": 2.7107, + "theoretical_loss": 3.5793331711382628, + "tokens_seen": 1230041088 + }, + { + "epoch": 4.01, + "learning_rate": 0.00031683049147442327, + "loss": 2.7466, + "theoretical_loss": 3.5793157108767457, + "tokens_seen": 1230106624 + }, + { + "epoch": 4.01, + "learning_rate": 0.00031682046138415245, + "loss": 2.6286, + "theoretical_loss": 3.5792982518058762, + "tokens_seen": 1230172160 + }, + { + "epoch": 4.01, + "learning_rate": 0.00031681043129388163, + "loss": 2.683, + "theoretical_loss": 3.579280793925509, + "tokens_seen": 1230237696 + }, + { + "epoch": 4.01, + "learning_rate": 0.00031680040120361086, + "loss": 2.5633, + "theoretical_loss": 3.5792633372355, + "tokens_seen": 1230303232 + }, + { + "epoch": 4.01, + "learning_rate": 0.00031679037111334004, + "loss": 2.626, + "theoretical_loss": 3.5792458817357042, + "tokens_seen": 1230368768 + }, + { + "epoch": 4.01, + "objective/train/docs_used": 1425869, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.124709367752075, + "objective/train/theoretical_loss": 3.579228427425978, + "objective/train/tokens_used": 1250894304, + "theoretical_loss": 3.579228427425978, + "tokens_seen": 1230434304 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003167803410230692, + "loss": 2.5447, + "theoretical_loss": 3.579228427425978, + "tokens_seen": 1230434304 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003167703109327984, + "loss": 2.7377, + "theoretical_loss": 3.579210974306175, + "tokens_seen": 1230499840 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003167602808425276, + "loss": 2.8242, + "theoretical_loss": 3.5791935223761526, + "tokens_seen": 1230565376 + }, + { + "epoch": 4.01, + "learning_rate": 0.00031675025075225677, + "loss": 2.8878, + "theoretical_loss": 3.5791760716357652, + "tokens_seen": 1230630912 + }, + { + "epoch": 4.01, + "learning_rate": 0.000316740220661986, + "loss": 2.6552, + "theoretical_loss": 3.579158622084869, + "tokens_seen": 1230696448 + }, + { + "epoch": 4.01, + "learning_rate": 0.00031673019057171513, + "loss": 2.7781, + "theoretical_loss": 3.579141173723319, + "tokens_seen": 1230761984 + }, + { + "epoch": 4.01, + "learning_rate": 0.00031672016048144437, + "loss": 2.572, + "theoretical_loss": 3.5791237265509714, + "tokens_seen": 1230827520 + }, + { + "epoch": 4.01, + "learning_rate": 0.00031671013039117355, + "loss": 2.5123, + "theoretical_loss": 3.5791062805676814, + "tokens_seen": 1230893056 + }, + { + "epoch": 4.01, + "learning_rate": 0.00031670010030090273, + "loss": 2.4936, + "theoretical_loss": 3.5790888357733053, + "tokens_seen": 1230958592 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003166900702106319, + "loss": 2.703, + "theoretical_loss": 3.5790713921676978, + "tokens_seen": 1231024128 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003166800401203611, + "loss": 2.8052, + "theoretical_loss": 3.5790539497507154, + "tokens_seen": 1231089664 + }, + { + "epoch": 4.01, + "learning_rate": 0.00031667001003009027, + "loss": 2.6364, + "theoretical_loss": 3.579036508522213, + "tokens_seen": 1231155200 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003166599799398195, + "loss": 2.4755, + "theoretical_loss": 3.5790190684820473, + "tokens_seen": 1231220736 + }, + { + "epoch": 4.01, + "learning_rate": 0.00031664994984954863, + "loss": 2.8016, + "theoretical_loss": 3.579001629630074, + "tokens_seen": 1231286272 + }, + { + "epoch": 4.01, + "learning_rate": 0.00031663991975927787, + "loss": 2.4563, + "theoretical_loss": 3.5789841919661485, + "tokens_seen": 1231351808 + }, + { + "epoch": 4.01, + "learning_rate": 0.000316629889669007, + "loss": 2.9177, + "theoretical_loss": 3.5789667554901268, + "tokens_seen": 1231417344 + }, + { + "epoch": 4.01, + "learning_rate": 0.00031661985957873623, + "loss": 2.4134, + "theoretical_loss": 3.5789493202018643, + "tokens_seen": 1231482880 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003166098294884654, + "loss": 2.7808, + "theoretical_loss": 3.5789318861012176, + "tokens_seen": 1231548416 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003165997993981946, + "loss": 2.8649, + "theoretical_loss": 3.5789144531880424, + "tokens_seen": 1231613952 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003165897693079238, + "loss": 2.8881, + "theoretical_loss": 3.5788970214621942, + "tokens_seen": 1231679488 + }, + { + "epoch": 4.01, + "learning_rate": 0.000316579739217653, + "loss": 2.8825, + "theoretical_loss": 3.57887959092353, + "tokens_seen": 1231745024 + }, + { + "epoch": 4.01, + "learning_rate": 0.00031656970912738214, + "loss": 2.644, + "theoretical_loss": 3.5788621615719043, + "tokens_seen": 1231810560 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003165596790371114, + "loss": 2.6932, + "theoretical_loss": 3.5788447334071747, + "tokens_seen": 1231876096 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003165496489468405, + "loss": 2.7675, + "theoretical_loss": 3.578827306429196, + "tokens_seen": 1231941632 + }, + { + "epoch": 4.01, + "learning_rate": 0.00031653961885656974, + "loss": 2.6962, + "theoretical_loss": 3.5788098806378246, + "tokens_seen": 1232007168 + }, + { + "epoch": 4.01, + "objective/train/docs_used": 1427266, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6803057193756104, + "objective/train/theoretical_loss": 3.578792456032917, + "objective/train/tokens_used": 1252532704, + "theoretical_loss": 3.578792456032917, + "tokens_seen": 1232072704 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003165295887662989, + "loss": 2.6417, + "theoretical_loss": 3.578792456032917, + "tokens_seen": 1232072704 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003165195586760281, + "loss": 2.4745, + "theoretical_loss": 3.5787750326143293, + "tokens_seen": 1232138240 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003165095285857573, + "loss": 2.571, + "theoretical_loss": 3.578757610381917, + "tokens_seen": 1232203776 + }, + { + "epoch": 4.01, + "learning_rate": 0.00031649949849548646, + "loss": 2.7561, + "theoretical_loss": 3.578740189335537, + "tokens_seen": 1232269312 + }, + { + "epoch": 4.01, + "learning_rate": 0.00031648946840521564, + "loss": 2.5433, + "theoretical_loss": 3.5787227694750445, + "tokens_seen": 1232334848 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003164794383149449, + "loss": 3.0216, + "theoretical_loss": 3.578705350800297, + "tokens_seen": 1232400384 + }, + { + "epoch": 4.01, + "learning_rate": 0.000316469408224674, + "loss": 2.4573, + "theoretical_loss": 3.5786879333111496, + "tokens_seen": 1232465920 + }, + { + "epoch": 4.01, + "learning_rate": 0.00031645937813440324, + "loss": 2.8524, + "theoretical_loss": 3.5786705170074598, + "tokens_seen": 1232531456 + }, + { + "epoch": 4.01, + "learning_rate": 0.00031644934804413237, + "loss": 2.7487, + "theoretical_loss": 3.578653101889082, + "tokens_seen": 1232596992 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003164393179538616, + "loss": 2.6817, + "theoretical_loss": 3.5786356879558747, + "tokens_seen": 1232662528 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003164292878635908, + "loss": 2.6149, + "theoretical_loss": 3.5786182752076927, + "tokens_seen": 1232728064 + }, + { + "epoch": 4.01, + "learning_rate": 0.00031641925777331996, + "loss": 2.6015, + "theoretical_loss": 3.578600863644393, + "tokens_seen": 1232793600 + }, + { + "epoch": 4.01, + "learning_rate": 0.00031640922768304914, + "loss": 2.5019, + "theoretical_loss": 3.5785834532658316, + "tokens_seen": 1232859136 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003163991975927784, + "loss": 2.8141, + "theoretical_loss": 3.5785660440718656, + "tokens_seen": 1232924672 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003163891675025075, + "loss": 2.5939, + "theoretical_loss": 3.5785486360623504, + "tokens_seen": 1232990208 + }, + { + "epoch": 4.01, + "learning_rate": 0.00031637913741223674, + "loss": 2.7359, + "theoretical_loss": 3.5785312292371434, + "tokens_seen": 1233055744 + }, + { + "epoch": 4.01, + "learning_rate": 0.00031636910732196587, + "loss": 2.5487, + "theoretical_loss": 3.5785138235961007, + "tokens_seen": 1233121280 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003163590772316951, + "loss": 2.5529, + "theoretical_loss": 3.578496419139079, + "tokens_seen": 1233186816 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003163490471414243, + "loss": 3.0039, + "theoretical_loss": 3.5784790158659345, + "tokens_seen": 1233252352 + }, + { + "epoch": 4.01, + "learning_rate": 0.00031633901705115347, + "loss": 2.5339, + "theoretical_loss": 3.578461613776524, + "tokens_seen": 1233317888 + }, + { + "epoch": 4.01, + "learning_rate": 0.00031632898696088265, + "loss": 2.7298, + "theoretical_loss": 3.5784442128707044, + "tokens_seen": 1233383424 + }, + { + "epoch": 4.01, + "learning_rate": 0.00031631895687061183, + "loss": 2.8187, + "theoretical_loss": 3.578426813148332, + "tokens_seen": 1233448960 + }, + { + "epoch": 4.01, + "learning_rate": 0.000316308926780341, + "loss": 2.7148, + "theoretical_loss": 3.578409414609263, + "tokens_seen": 1233514496 + }, + { + "epoch": 4.01, + "learning_rate": 0.00031629889669007024, + "loss": 2.6501, + "theoretical_loss": 3.5783920172533543, + "tokens_seen": 1233580032 + }, + { + "epoch": 4.01, + "learning_rate": 0.00031628886659979937, + "loss": 2.741, + "theoretical_loss": 3.5783746210804637, + "tokens_seen": 1233645568 + }, + { + "epoch": 4.01, + "objective/train/docs_used": 1427860, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9305169582366943, + "objective/train/theoretical_loss": 3.5783572260904464, + "objective/train/tokens_used": 1254171104, + "theoretical_loss": 3.5783572260904464, + "tokens_seen": 1233711104 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003162788365095286, + "loss": 2.7442, + "theoretical_loss": 3.5783572260904464, + "tokens_seen": 1233711104 + }, + { + "epoch": 4.01, + "learning_rate": 0.00031626880641925773, + "loss": 2.633, + "theoretical_loss": 3.57833983228316, + "tokens_seen": 1233776640 + }, + { + "epoch": 4.01, + "learning_rate": 0.00031625877632898697, + "loss": 2.5109, + "theoretical_loss": 3.5783224396584608, + "tokens_seen": 1233842176 + }, + { + "epoch": 4.01, + "learning_rate": 0.00031624874623871615, + "loss": 2.4862, + "theoretical_loss": 3.5783050482162055, + "tokens_seen": 1233907712 + }, + { + "epoch": 4.01, + "learning_rate": 0.00031623871614844533, + "loss": 2.6074, + "theoretical_loss": 3.5782876579562517, + "tokens_seen": 1233973248 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003162286860581745, + "loss": 2.5857, + "theoretical_loss": 3.5782702688784553, + "tokens_seen": 1234038784 + }, + { + "epoch": 4.01, + "learning_rate": 0.00031621865596790375, + "loss": 2.5521, + "theoretical_loss": 3.578252880982674, + "tokens_seen": 1234104320 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003162086258776329, + "loss": 2.809, + "theoretical_loss": 3.578235494268764, + "tokens_seen": 1234169856 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003161985957873621, + "loss": 2.7293, + "theoretical_loss": 3.5782181087365825, + "tokens_seen": 1234235392 + }, + { + "epoch": 4.01, + "learning_rate": 0.00031618856569709124, + "loss": 2.6321, + "theoretical_loss": 3.5782007243859866, + "tokens_seen": 1234300928 + }, + { + "epoch": 4.01, + "learning_rate": 0.00031617853560682047, + "loss": 2.3845, + "theoretical_loss": 3.5781833412168336, + "tokens_seen": 1234366464 + }, + { + "epoch": 4.01, + "learning_rate": 0.00031616850551654965, + "loss": 2.6426, + "theoretical_loss": 3.578165959228979, + "tokens_seen": 1234432000 + }, + { + "epoch": 4.01, + "learning_rate": 0.00031615847542627883, + "loss": 2.6782, + "theoretical_loss": 3.5781485784222813, + "tokens_seen": 1234497536 + }, + { + "epoch": 4.01, + "learning_rate": 0.000316148445336008, + "loss": 2.6913, + "theoretical_loss": 3.5781311987965974, + "tokens_seen": 1234563072 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003161384152457372, + "loss": 2.6628, + "theoretical_loss": 3.578113820351784, + "tokens_seen": 1234628608 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003161283851554664, + "loss": 2.8064, + "theoretical_loss": 3.5780964430876976, + "tokens_seen": 1234694144 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003161183550651956, + "loss": 2.513, + "theoretical_loss": 3.5780790670041966, + "tokens_seen": 1234759680 + }, + { + "epoch": 4.01, + "learning_rate": 0.00031610832497492474, + "loss": 2.624, + "theoretical_loss": 3.578061692101137, + "tokens_seen": 1234825216 + }, + { + "epoch": 4.01, + "learning_rate": 0.000316098294884654, + "loss": 2.6117, + "theoretical_loss": 3.5780443183783763, + "tokens_seen": 1234890752 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003160882647943831, + "loss": 2.6111, + "theoretical_loss": 3.5780269458357723, + "tokens_seen": 1234956288 + }, + { + "epoch": 4.01, + "learning_rate": 0.00031607823470411234, + "loss": 2.5321, + "theoretical_loss": 3.5780095744731817, + "tokens_seen": 1235021824 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003160682046138415, + "loss": 2.5517, + "theoretical_loss": 3.5779922042904615, + "tokens_seen": 1235087360 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003160581745235707, + "loss": 2.3564, + "theoretical_loss": 3.5779748352874696, + "tokens_seen": 1235152896 + }, + { + "epoch": 4.01, + "learning_rate": 0.00031604814443329994, + "loss": 2.561, + "theoretical_loss": 3.577957467464062, + "tokens_seen": 1235218432 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003160381143430291, + "loss": 2.8883, + "theoretical_loss": 3.577940100820098, + "tokens_seen": 1235283968 + }, + { + "epoch": 4.01, + "objective/train/docs_used": 1429255, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.2701942920684814, + "objective/train/theoretical_loss": 3.5779227353554335, + "objective/train/tokens_used": 1255809504, + "theoretical_loss": 3.5779227353554335, + "tokens_seen": 1235349504 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003160280842527583, + "loss": 2.5859, + "theoretical_loss": 3.5779227353554335, + "tokens_seen": 1235349504 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003160180541624875, + "loss": 2.9854, + "theoretical_loss": 3.5779053710699262, + "tokens_seen": 1235415040 + }, + { + "epoch": 4.01, + "learning_rate": 0.00031600802407221666, + "loss": 2.5211, + "theoretical_loss": 3.577888007963433, + "tokens_seen": 1235480576 + }, + { + "epoch": 4.01, + "learning_rate": 0.00031599799398194584, + "loss": 2.6688, + "theoretical_loss": 3.5778706460358127, + "tokens_seen": 1235546112 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003159879638916751, + "loss": 2.6484, + "theoretical_loss": 3.5778532852869214, + "tokens_seen": 1235611648 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003159779338014042, + "loss": 2.6478, + "theoretical_loss": 3.5778359257166175, + "tokens_seen": 1235677184 + }, + { + "epoch": 4.01, + "learning_rate": 0.00031596790371113344, + "loss": 2.5795, + "theoretical_loss": 3.5778185673247576, + "tokens_seen": 1235742720 + }, + { + "epoch": 4.01, + "learning_rate": 0.00031595787362086257, + "loss": 2.5562, + "theoretical_loss": 3.5778012101111996, + "tokens_seen": 1235808256 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003159478435305918, + "loss": 2.8139, + "theoretical_loss": 3.577783854075801, + "tokens_seen": 1235873792 + }, + { + "epoch": 4.01, + "learning_rate": 0.000315937813440321, + "loss": 2.7729, + "theoretical_loss": 3.5777664992184195, + "tokens_seen": 1235939328 + }, + { + "epoch": 4.01, + "learning_rate": 0.00031592778335005016, + "loss": 2.8157, + "theoretical_loss": 3.5777491455389123, + "tokens_seen": 1236004864 + }, + { + "epoch": 4.01, + "learning_rate": 0.00031591775325977934, + "loss": 2.5978, + "theoretical_loss": 3.577731793037138, + "tokens_seen": 1236070400 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003159077231695086, + "loss": 2.8192, + "theoretical_loss": 3.577714441712953, + "tokens_seen": 1236135936 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003158976930792377, + "loss": 2.5671, + "theoretical_loss": 3.577697091566216, + "tokens_seen": 1236201472 + }, + { + "epoch": 4.01, + "learning_rate": 0.00031588766298896694, + "loss": 2.6757, + "theoretical_loss": 3.577679742596784, + "tokens_seen": 1236267008 + }, + { + "epoch": 4.01, + "learning_rate": 0.00031587763289869607, + "loss": 2.6728, + "theoretical_loss": 3.5776623948045145, + "tokens_seen": 1236332544 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003158676028084253, + "loss": 2.5845, + "theoretical_loss": 3.5776450481892663, + "tokens_seen": 1236398080 + }, + { + "epoch": 4.01, + "learning_rate": 0.0003158575727181545, + "loss": 2.5337, + "theoretical_loss": 3.5776277027508963, + "tokens_seen": 1236463616 + }, + { + "epoch": 4.01, + "learning_rate": 0.00031584754262788367, + "loss": 2.8174, + "theoretical_loss": 3.5776103584892622, + "tokens_seen": 1236529152 + }, + { + "epoch": 4.02, + "learning_rate": 0.00031583751253761285, + "loss": 2.6758, + "theoretical_loss": 3.577593015404222, + "tokens_seen": 1236594688 + }, + { + "epoch": 4.02, + "learning_rate": 0.00031582748244734203, + "loss": 2.5004, + "theoretical_loss": 3.577575673495634, + "tokens_seen": 1236660224 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003158174523570712, + "loss": 2.5351, + "theoretical_loss": 3.5775583327633553, + "tokens_seen": 1236725760 + }, + { + "epoch": 4.02, + "learning_rate": 0.00031580742226680044, + "loss": 2.7012, + "theoretical_loss": 3.577540993207245, + "tokens_seen": 1236791296 + }, + { + "epoch": 4.02, + "learning_rate": 0.00031579739217652957, + "loss": 2.4612, + "theoretical_loss": 3.5775236548271594, + "tokens_seen": 1236856832 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003157873620862588, + "loss": 2.6514, + "theoretical_loss": 3.5775063176229573, + "tokens_seen": 1236922368 + }, + { + "epoch": 4.02, + "objective/train/docs_used": 1430103, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.250220537185669, + "objective/train/theoretical_loss": 3.577488981594497, + "objective/train/tokens_used": 1257447904, + "theoretical_loss": 3.577488981594497, + "tokens_seen": 1236987904 + }, + { + "epoch": 4.02, + "learning_rate": 0.00031577733199598793, + "loss": 2.5234, + "theoretical_loss": 3.577488981594497, + "tokens_seen": 1236987904 + }, + { + "epoch": 4.02, + "learning_rate": 0.00031576730190571717, + "loss": 2.7337, + "theoretical_loss": 3.5774716467416354, + "tokens_seen": 1237053440 + }, + { + "epoch": 4.02, + "learning_rate": 0.00031575727181544635, + "loss": 2.7496, + "theoretical_loss": 3.5774543130642313, + "tokens_seen": 1237118976 + }, + { + "epoch": 4.02, + "learning_rate": 0.00031574724172517553, + "loss": 2.5515, + "theoretical_loss": 3.577436980562143, + "tokens_seen": 1237184512 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003157372116349047, + "loss": 2.8153, + "theoretical_loss": 3.5774196492352277, + "tokens_seen": 1237250048 + }, + { + "epoch": 4.02, + "learning_rate": 0.00031572718154463395, + "loss": 2.5813, + "theoretical_loss": 3.577402319083344, + "tokens_seen": 1237315584 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003157171514543631, + "loss": 2.5498, + "theoretical_loss": 3.57738499010635, + "tokens_seen": 1237381120 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003157071213640923, + "loss": 2.7301, + "theoretical_loss": 3.5773676623041037, + "tokens_seen": 1237446656 + }, + { + "epoch": 4.02, + "learning_rate": 0.00031569709127382144, + "loss": 2.7963, + "theoretical_loss": 3.5773503356764635, + "tokens_seen": 1237512192 + }, + { + "epoch": 4.02, + "learning_rate": 0.00031568706118355067, + "loss": 2.4556, + "theoretical_loss": 3.577333010223287, + "tokens_seen": 1237577728 + }, + { + "epoch": 4.02, + "learning_rate": 0.00031567703109327985, + "loss": 2.4602, + "theoretical_loss": 3.5773156859444333, + "tokens_seen": 1237643264 + }, + { + "epoch": 4.02, + "learning_rate": 0.00031566700100300903, + "loss": 2.5788, + "theoretical_loss": 3.5772983628397594, + "tokens_seen": 1237708800 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003156569709127382, + "loss": 2.7927, + "theoretical_loss": 3.5772810409091247, + "tokens_seen": 1237774336 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003156469408224674, + "loss": 2.7631, + "theoretical_loss": 3.577263720152387, + "tokens_seen": 1237839872 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003156369107321966, + "loss": 2.6811, + "theoretical_loss": 3.577246400569405, + "tokens_seen": 1237905408 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003156268806419258, + "loss": 2.5648, + "theoretical_loss": 3.5772290821600357, + "tokens_seen": 1237970944 + }, + { + "epoch": 4.02, + "learning_rate": 0.00031561685055165494, + "loss": 2.5189, + "theoretical_loss": 3.5772117649241393, + "tokens_seen": 1238036480 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003156068204613842, + "loss": 2.9144, + "theoretical_loss": 3.577194448861573, + "tokens_seen": 1238102016 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003155967903711133, + "loss": 2.7917, + "theoretical_loss": 3.5771771339721954, + "tokens_seen": 1238167552 + }, + { + "epoch": 4.02, + "learning_rate": 0.00031558676028084254, + "loss": 2.8326, + "theoretical_loss": 3.577159820255865, + "tokens_seen": 1238233088 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003155767301905717, + "loss": 2.6731, + "theoretical_loss": 3.5771425077124404, + "tokens_seen": 1238298624 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003155667001003009, + "loss": 2.6652, + "theoretical_loss": 3.57712519634178, + "tokens_seen": 1238364160 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003155566700100301, + "loss": 2.5107, + "theoretical_loss": 3.5771078861437418, + "tokens_seen": 1238429696 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003155466399197593, + "loss": 2.5704, + "theoretical_loss": 3.577090577118185, + "tokens_seen": 1238495232 + }, + { + "epoch": 4.02, + "learning_rate": 0.00031553660982948844, + "loss": 2.736, + "theoretical_loss": 3.577073269264968, + "tokens_seen": 1238560768 + }, + { + "epoch": 4.02, + "objective/train/docs_used": 1431596, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.694431781768799, + "objective/train/theoretical_loss": 3.577055962583949, + "objective/train/tokens_used": 1259086304, + "theoretical_loss": 3.577055962583949, + "tokens_seen": 1238626304 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003155265797392177, + "loss": 2.7258, + "theoretical_loss": 3.577055962583949, + "tokens_seen": 1238626304 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003155165496489468, + "loss": 2.8068, + "theoretical_loss": 3.5770386570749872, + "tokens_seen": 1238691840 + }, + { + "epoch": 4.02, + "learning_rate": 0.00031550651955867604, + "loss": 2.5443, + "theoretical_loss": 3.5770213527379404, + "tokens_seen": 1238757376 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003154964894684052, + "loss": 2.6994, + "theoretical_loss": 3.5770040495726687, + "tokens_seen": 1238822912 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003154864593781344, + "loss": 2.7348, + "theoretical_loss": 3.576986747579029, + "tokens_seen": 1238888448 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003154764292878636, + "loss": 2.727, + "theoretical_loss": 3.5769694467568813, + "tokens_seen": 1238953984 + }, + { + "epoch": 4.02, + "learning_rate": 0.00031546639919759277, + "loss": 2.8486, + "theoretical_loss": 3.5769521471060832, + "tokens_seen": 1239019520 + }, + { + "epoch": 4.02, + "learning_rate": 0.00031545636910732195, + "loss": 2.7562, + "theoretical_loss": 3.5769348486264945, + "tokens_seen": 1239085056 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003154463390170512, + "loss": 2.8638, + "theoretical_loss": 3.5769175513179734, + "tokens_seen": 1239150592 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003154363089267803, + "loss": 2.577, + "theoretical_loss": 3.5769002551803792, + "tokens_seen": 1239216128 + }, + { + "epoch": 4.02, + "learning_rate": 0.00031542627883650954, + "loss": 2.6711, + "theoretical_loss": 3.5768829602135694, + "tokens_seen": 1239281664 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003154162487462387, + "loss": 2.6202, + "theoretical_loss": 3.5768656664174046, + "tokens_seen": 1239347200 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003154062186559679, + "loss": 2.7357, + "theoretical_loss": 3.5768483737917425, + "tokens_seen": 1239412736 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003153961885656971, + "loss": 2.7097, + "theoretical_loss": 3.576831082336443, + "tokens_seen": 1239478272 + }, + { + "epoch": 4.02, + "learning_rate": 0.00031538615847542627, + "loss": 2.6539, + "theoretical_loss": 3.5768137920513636, + "tokens_seen": 1239543808 + }, + { + "epoch": 4.02, + "learning_rate": 0.00031537612838515545, + "loss": 2.5496, + "theoretical_loss": 3.5767965029363644, + "tokens_seen": 1239609344 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003153660982948847, + "loss": 2.8014, + "theoretical_loss": 3.5767792149913036, + "tokens_seen": 1239674880 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003153560682046138, + "loss": 2.5942, + "theoretical_loss": 3.5767619282160403, + "tokens_seen": 1239740416 + }, + { + "epoch": 4.02, + "learning_rate": 0.00031534603811434305, + "loss": 2.5592, + "theoretical_loss": 3.576744642610435, + "tokens_seen": 1239805952 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003153360080240722, + "loss": 2.4833, + "theoretical_loss": 3.5767273581743444, + "tokens_seen": 1239871488 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003153259779338014, + "loss": 2.8076, + "theoretical_loss": 3.576710074907629, + "tokens_seen": 1239937024 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003153159478435306, + "loss": 2.751, + "theoretical_loss": 3.576692792810147, + "tokens_seen": 1240002560 + }, + { + "epoch": 4.02, + "learning_rate": 0.00031530591775325977, + "loss": 2.8131, + "theoretical_loss": 3.576675511881759, + "tokens_seen": 1240068096 + }, + { + "epoch": 4.02, + "learning_rate": 0.000315295887662989, + "loss": 2.7056, + "theoretical_loss": 3.5766582321223224, + "tokens_seen": 1240133632 + }, + { + "epoch": 4.02, + "learning_rate": 0.00031528585757271813, + "loss": 2.6935, + "theoretical_loss": 3.5766409535316974, + "tokens_seen": 1240199168 + }, + { + "epoch": 4.02, + "objective/train/docs_used": 1432295, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.533489227294922, + "objective/train/theoretical_loss": 3.576623676109743, + "objective/train/tokens_used": 1260724704, + "theoretical_loss": 3.576623676109743, + "tokens_seen": 1240264704 + }, + { + "epoch": 4.02, + "learning_rate": 0.00031527582748244737, + "loss": 2.8536, + "theoretical_loss": 3.576623676109743, + "tokens_seen": 1240264704 + }, + { + "epoch": 4.02, + "learning_rate": 0.00031526579739217655, + "loss": 2.5282, + "theoretical_loss": 3.5766063998563187, + "tokens_seen": 1240330240 + }, + { + "epoch": 4.02, + "learning_rate": 0.00031525576730190573, + "loss": 2.5231, + "theoretical_loss": 3.5765891247712824, + "tokens_seen": 1240395776 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003152457372116349, + "loss": 2.6814, + "theoretical_loss": 3.5765718508544952, + "tokens_seen": 1240461312 + }, + { + "epoch": 4.02, + "learning_rate": 0.00031523570712136415, + "loss": 2.5898, + "theoretical_loss": 3.5765545781058155, + "tokens_seen": 1240526848 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003152256770310933, + "loss": 2.8252, + "theoretical_loss": 3.5765373065251023, + "tokens_seen": 1240592384 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003152156469408225, + "loss": 2.6618, + "theoretical_loss": 3.576520036112215, + "tokens_seen": 1240657920 + }, + { + "epoch": 4.02, + "learning_rate": 0.00031520561685055164, + "loss": 2.8285, + "theoretical_loss": 3.5765027668670135, + "tokens_seen": 1240723456 + }, + { + "epoch": 4.02, + "learning_rate": 0.00031519558676028087, + "loss": 2.5839, + "theoretical_loss": 3.576485498789357, + "tokens_seen": 1240788992 + }, + { + "epoch": 4.02, + "learning_rate": 0.00031518555667001005, + "loss": 2.6935, + "theoretical_loss": 3.5764682318791046, + "tokens_seen": 1240854528 + }, + { + "epoch": 4.02, + "learning_rate": 0.00031517552657973924, + "loss": 2.8678, + "theoretical_loss": 3.576450966136116, + "tokens_seen": 1240920064 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003151654964894684, + "loss": 2.66, + "theoretical_loss": 3.5764337015602505, + "tokens_seen": 1240985600 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003151554663991976, + "loss": 2.5715, + "theoretical_loss": 3.5764164381513677, + "tokens_seen": 1241051136 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003151454363089268, + "loss": 2.6386, + "theoretical_loss": 3.5763991759093265, + "tokens_seen": 1241116672 + }, + { + "epoch": 4.02, + "learning_rate": 0.000315135406218656, + "loss": 2.6935, + "theoretical_loss": 3.5763819148339877, + "tokens_seen": 1241182208 + }, + { + "epoch": 4.02, + "learning_rate": 0.00031512537612838514, + "loss": 2.6195, + "theoretical_loss": 3.57636465492521, + "tokens_seen": 1241247744 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003151153460381144, + "loss": 2.6906, + "theoretical_loss": 3.576347396182853, + "tokens_seen": 1241313280 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003151053159478435, + "loss": 2.5668, + "theoretical_loss": 3.5763301386067763, + "tokens_seen": 1241378816 + }, + { + "epoch": 4.02, + "learning_rate": 0.00031509528585757274, + "loss": 2.9009, + "theoretical_loss": 3.5763128821968393, + "tokens_seen": 1241444352 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003150852557673019, + "loss": 2.7263, + "theoretical_loss": 3.576295626952902, + "tokens_seen": 1241509888 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003150752256770311, + "loss": 2.7097, + "theoretical_loss": 3.5762783728748246, + "tokens_seen": 1241575424 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003150651955867603, + "loss": 2.636, + "theoretical_loss": 3.576261119962466, + "tokens_seen": 1241640960 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003150551654964895, + "loss": 2.792, + "theoretical_loss": 3.576243868215686, + "tokens_seen": 1241706496 + }, + { + "epoch": 4.02, + "learning_rate": 0.00031504513540621864, + "loss": 2.7841, + "theoretical_loss": 3.5762266176343447, + "tokens_seen": 1241772032 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003150351053159479, + "loss": 2.8194, + "theoretical_loss": 3.576209368218301, + "tokens_seen": 1241837568 + }, + { + "epoch": 4.02, + "objective/train/docs_used": 1433476, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.985485315322876, + "objective/train/theoretical_loss": 3.576192119967416, + "objective/train/tokens_used": 1262363104, + "theoretical_loss": 3.576192119967416, + "tokens_seen": 1241903104 + }, + { + "epoch": 4.02, + "learning_rate": 0.000315025075225677, + "loss": 2.8603, + "theoretical_loss": 3.576192119967416, + "tokens_seen": 1241903104 + }, + { + "epoch": 4.02, + "learning_rate": 0.00031501504513540624, + "loss": 2.7702, + "theoretical_loss": 3.5761748728815483, + "tokens_seen": 1241968640 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003150050150451354, + "loss": 2.7794, + "theoretical_loss": 3.576157626960559, + "tokens_seen": 1242034176 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003149949849548646, + "loss": 2.6736, + "theoretical_loss": 3.576140382204307, + "tokens_seen": 1242099712 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003149849548645938, + "loss": 2.6884, + "theoretical_loss": 3.576123138612652, + "tokens_seen": 1242165248 + }, + { + "epoch": 4.02, + "learning_rate": 0.00031497492477432297, + "loss": 2.4335, + "theoretical_loss": 3.576105896185455, + "tokens_seen": 1242230784 + }, + { + "epoch": 4.02, + "learning_rate": 0.00031496489468405215, + "loss": 2.4949, + "theoretical_loss": 3.5760886549225748, + "tokens_seen": 1242296320 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003149548645937814, + "loss": 2.5471, + "theoretical_loss": 3.576071414823872, + "tokens_seen": 1242361856 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003149448345035105, + "loss": 2.7882, + "theoretical_loss": 3.576054175889207, + "tokens_seen": 1242427392 + }, + { + "epoch": 4.02, + "learning_rate": 0.00031493480441323974, + "loss": 2.6353, + "theoretical_loss": 3.5760369381184383, + "tokens_seen": 1242492928 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003149247743229689, + "loss": 2.764, + "theoretical_loss": 3.5760197015114277, + "tokens_seen": 1242558464 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003149147442326981, + "loss": 2.7535, + "theoretical_loss": 3.576002466068034, + "tokens_seen": 1242624000 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003149047141424273, + "loss": 2.5901, + "theoretical_loss": 3.575985231788118, + "tokens_seen": 1242689536 + }, + { + "epoch": 4.02, + "learning_rate": 0.00031489468405215647, + "loss": 2.6292, + "theoretical_loss": 3.575967998671539, + "tokens_seen": 1242755072 + }, + { + "epoch": 4.02, + "learning_rate": 0.00031488465396188565, + "loss": 2.3893, + "theoretical_loss": 3.575950766718158, + "tokens_seen": 1242820608 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003148746238716149, + "loss": 2.6584, + "theoretical_loss": 3.5759335359278355, + "tokens_seen": 1242886144 + }, + { + "epoch": 4.02, + "learning_rate": 0.000314864593781344, + "loss": 2.6964, + "theoretical_loss": 3.5759163063004302, + "tokens_seen": 1242951680 + }, + { + "epoch": 4.02, + "learning_rate": 0.00031485456369107325, + "loss": 2.7812, + "theoretical_loss": 3.5758990778358033, + "tokens_seen": 1243017216 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003148445336008024, + "loss": 2.7795, + "theoretical_loss": 3.575881850533815, + "tokens_seen": 1243082752 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003148345035105316, + "loss": 2.6807, + "theoretical_loss": 3.5758646243943253, + "tokens_seen": 1243148288 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003148244734202608, + "loss": 2.715, + "theoretical_loss": 3.5758473994171944, + "tokens_seen": 1243213824 + }, + { + "epoch": 4.02, + "learning_rate": 0.00031481444332998997, + "loss": 2.5597, + "theoretical_loss": 3.575830175602283, + "tokens_seen": 1243279360 + }, + { + "epoch": 4.02, + "learning_rate": 0.00031480441323971915, + "loss": 2.8623, + "theoretical_loss": 3.575812952949451, + "tokens_seen": 1243344896 + }, + { + "epoch": 4.02, + "learning_rate": 0.00031479438314944833, + "loss": 2.3986, + "theoretical_loss": 3.575795731458559, + "tokens_seen": 1243410432 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003147843530591775, + "loss": 2.5205, + "theoretical_loss": 3.5757785111294673, + "tokens_seen": 1243475968 + }, + { + "epoch": 4.02, + "objective/train/docs_used": 1434244, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7484705448150635, + "objective/train/theoretical_loss": 3.5757612919620367, + "objective/train/tokens_used": 1264001504, + "theoretical_loss": 3.5757612919620367, + "tokens_seen": 1243541504 + }, + { + "epoch": 4.02, + "learning_rate": 0.00031477432296890675, + "loss": 2.7885, + "theoretical_loss": 3.5757612919620367, + "tokens_seen": 1243541504 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003147642928786359, + "loss": 2.8115, + "theoretical_loss": 3.575744073956127, + "tokens_seen": 1243607040 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003147542627883651, + "loss": 2.6593, + "theoretical_loss": 3.575726857111598, + "tokens_seen": 1243672576 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003147442326980943, + "loss": 2.73, + "theoretical_loss": 3.575709641428312, + "tokens_seen": 1243738112 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003147342026078235, + "loss": 2.6826, + "theoretical_loss": 3.5756924269061288, + "tokens_seen": 1243803648 + }, + { + "epoch": 4.02, + "learning_rate": 0.00031472417251755266, + "loss": 2.8426, + "theoretical_loss": 3.575675213544908, + "tokens_seen": 1243869184 + }, + { + "epoch": 4.02, + "learning_rate": 0.00031471414242728184, + "loss": 2.8811, + "theoretical_loss": 3.575658001344511, + "tokens_seen": 1243934720 + }, + { + "epoch": 4.02, + "learning_rate": 0.000314704112337011, + "loss": 2.726, + "theoretical_loss": 3.5756407903047984, + "tokens_seen": 1244000256 + }, + { + "epoch": 4.02, + "learning_rate": 0.00031469408224674025, + "loss": 2.6208, + "theoretical_loss": 3.5756235804256304, + "tokens_seen": 1244065792 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003146840521564694, + "loss": 2.5834, + "theoretical_loss": 3.5756063717068676, + "tokens_seen": 1244131328 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003146740220661986, + "loss": 2.7418, + "theoretical_loss": 3.5755891641483712, + "tokens_seen": 1244196864 + }, + { + "epoch": 4.02, + "learning_rate": 0.00031466399197592774, + "loss": 2.7675, + "theoretical_loss": 3.575571957750001, + "tokens_seen": 1244262400 + }, + { + "epoch": 4.02, + "learning_rate": 0.000314653961885657, + "loss": 2.6723, + "theoretical_loss": 3.5755547525116187, + "tokens_seen": 1244327936 + }, + { + "epoch": 4.02, + "learning_rate": 0.00031464393179538616, + "loss": 2.5414, + "theoretical_loss": 3.575537548433084, + "tokens_seen": 1244393472 + }, + { + "epoch": 4.02, + "learning_rate": 0.00031463390170511534, + "loss": 2.5029, + "theoretical_loss": 3.575520345514258, + "tokens_seen": 1244459008 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003146238716148445, + "loss": 2.8787, + "theoretical_loss": 3.575503143755002, + "tokens_seen": 1244524544 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003146138415245737, + "loss": 2.6157, + "theoretical_loss": 3.575485943155176, + "tokens_seen": 1244590080 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003146038114343029, + "loss": 2.671, + "theoretical_loss": 3.5754687437146417, + "tokens_seen": 1244655616 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003145937813440321, + "loss": 2.527, + "theoretical_loss": 3.5754515454332587, + "tokens_seen": 1244721152 + }, + { + "epoch": 4.02, + "learning_rate": 0.00031458375125376125, + "loss": 2.3568, + "theoretical_loss": 3.575434348310889, + "tokens_seen": 1244786688 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003145737211634905, + "loss": 2.4167, + "theoretical_loss": 3.575417152347393, + "tokens_seen": 1244852224 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003145636910732197, + "loss": 2.5593, + "theoretical_loss": 3.5753999575426314, + "tokens_seen": 1244917760 + }, + { + "epoch": 4.02, + "learning_rate": 0.00031455366098294884, + "loss": 2.78, + "theoretical_loss": 3.5753827638964655, + "tokens_seen": 1244983296 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003145436308926781, + "loss": 2.7571, + "theoretical_loss": 3.575365571408756, + "tokens_seen": 1245048832 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003145336008024072, + "loss": 2.4951, + "theoretical_loss": 3.5753483800793644, + "tokens_seen": 1245114368 + }, + { + "epoch": 4.02, + "objective/train/docs_used": 1435669, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.428680419921875, + "objective/train/theoretical_loss": 3.575331189908151, + "objective/train/tokens_used": 1265639904, + "theoretical_loss": 3.575331189908151, + "tokens_seen": 1245179904 + }, + { + "epoch": 4.02, + "learning_rate": 0.00031452357071213644, + "loss": 2.7965, + "theoretical_loss": 3.575331189908151, + "tokens_seen": 1245179904 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003145135406218656, + "loss": 2.6784, + "theoretical_loss": 3.575314000894976, + "tokens_seen": 1245245440 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003145035105315948, + "loss": 2.4244, + "theoretical_loss": 3.575296813039703, + "tokens_seen": 1245310976 + }, + { + "epoch": 4.02, + "learning_rate": 0.000314493480441324, + "loss": 2.7376, + "theoretical_loss": 3.575279626342191, + "tokens_seen": 1245376512 + }, + { + "epoch": 4.02, + "learning_rate": 0.00031448345035105317, + "loss": 2.7399, + "theoretical_loss": 3.5752624408023017, + "tokens_seen": 1245442048 + }, + { + "epoch": 4.02, + "learning_rate": 0.00031447342026078235, + "loss": 2.7588, + "theoretical_loss": 3.5752452564198967, + "tokens_seen": 1245507584 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003144633901705116, + "loss": 2.6889, + "theoretical_loss": 3.575228073194836, + "tokens_seen": 1245573120 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003144533600802407, + "loss": 2.6662, + "theoretical_loss": 3.575210891126982, + "tokens_seen": 1245638656 + }, + { + "epoch": 4.02, + "learning_rate": 0.00031444332998996994, + "loss": 2.5576, + "theoretical_loss": 3.575193710216195, + "tokens_seen": 1245704192 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003144332998996991, + "loss": 2.7878, + "theoretical_loss": 3.575176530462337, + "tokens_seen": 1245769728 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003144232698094283, + "loss": 2.7706, + "theoretical_loss": 3.5751593518652682, + "tokens_seen": 1245835264 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003144132397191575, + "loss": 2.7004, + "theoretical_loss": 3.575142174424851, + "tokens_seen": 1245900800 + }, + { + "epoch": 4.02, + "learning_rate": 0.00031440320962888667, + "loss": 2.8504, + "theoretical_loss": 3.5751249981409456, + "tokens_seen": 1245966336 + }, + { + "epoch": 4.02, + "learning_rate": 0.00031439317953861585, + "loss": 2.6992, + "theoretical_loss": 3.575107823013414, + "tokens_seen": 1246031872 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003143831494483451, + "loss": 2.6023, + "theoretical_loss": 3.5750906490421173, + "tokens_seen": 1246097408 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003143731193580742, + "loss": 2.5395, + "theoretical_loss": 3.575073476226917, + "tokens_seen": 1246162944 + }, + { + "epoch": 4.02, + "learning_rate": 0.00031436308926780345, + "loss": 2.5867, + "theoretical_loss": 3.575056304567674, + "tokens_seen": 1246228480 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003143530591775326, + "loss": 2.572, + "theoretical_loss": 3.5750391340642507, + "tokens_seen": 1246294016 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003143430290872618, + "loss": 2.6351, + "theoretical_loss": 3.575021964716508, + "tokens_seen": 1246359552 + }, + { + "epoch": 4.02, + "learning_rate": 0.000314332998996991, + "loss": 2.4288, + "theoretical_loss": 3.5750047965243072, + "tokens_seen": 1246425088 + }, + { + "epoch": 4.02, + "learning_rate": 0.00031432296890672017, + "loss": 2.5788, + "theoretical_loss": 3.57498762948751, + "tokens_seen": 1246490624 + }, + { + "epoch": 4.02, + "learning_rate": 0.00031431293881644935, + "loss": 2.7888, + "theoretical_loss": 3.574970463605977, + "tokens_seen": 1246556160 + }, + { + "epoch": 4.02, + "learning_rate": 0.00031430290872617853, + "loss": 2.682, + "theoretical_loss": 3.5749532988795716, + "tokens_seen": 1246621696 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003142928786359077, + "loss": 2.5778, + "theoretical_loss": 3.5749361353081537, + "tokens_seen": 1246687232 + }, + { + "epoch": 4.02, + "learning_rate": 0.00031428284854563695, + "loss": 2.5332, + "theoretical_loss": 3.5749189728915853, + "tokens_seen": 1246752768 + }, + { + "epoch": 4.02, + "objective/train/docs_used": 1436465, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.797041893005371, + "objective/train/theoretical_loss": 3.5749018116297284, + "objective/train/tokens_used": 1267278304, + "theoretical_loss": 3.5749018116297284, + "tokens_seen": 1246818304 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003142728184553661, + "loss": 2.7708, + "theoretical_loss": 3.5749018116297284, + "tokens_seen": 1246818304 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003142627883650953, + "loss": 2.3856, + "theoretical_loss": 3.574884651522444, + "tokens_seen": 1246883840 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003142527582748245, + "loss": 2.7342, + "theoretical_loss": 3.5748674925695942, + "tokens_seen": 1246949376 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003142427281845537, + "loss": 2.5475, + "theoretical_loss": 3.5748503347710407, + "tokens_seen": 1247014912 + }, + { + "epoch": 4.02, + "learning_rate": 0.00031423269809428286, + "loss": 2.4742, + "theoretical_loss": 3.574833178126645, + "tokens_seen": 1247080448 + }, + { + "epoch": 4.02, + "learning_rate": 0.00031422266800401204, + "loss": 2.6937, + "theoretical_loss": 3.574816022636269, + "tokens_seen": 1247145984 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003142126379137412, + "loss": 2.6389, + "theoretical_loss": 3.5747988682997747, + "tokens_seen": 1247211520 + }, + { + "epoch": 4.02, + "learning_rate": 0.00031420260782347045, + "loss": 2.6282, + "theoretical_loss": 3.5747817151170227, + "tokens_seen": 1247277056 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003141925777331996, + "loss": 2.6403, + "theoretical_loss": 3.574764563087876, + "tokens_seen": 1247342592 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003141825476429288, + "loss": 2.8371, + "theoretical_loss": 3.5747474122121954, + "tokens_seen": 1247408128 + }, + { + "epoch": 4.02, + "learning_rate": 0.00031417251755265794, + "loss": 2.5975, + "theoretical_loss": 3.574730262489844, + "tokens_seen": 1247473664 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003141624874623872, + "loss": 2.5355, + "theoretical_loss": 3.5747131139206827, + "tokens_seen": 1247539200 + }, + { + "epoch": 4.02, + "learning_rate": 0.00031415245737211636, + "loss": 2.6023, + "theoretical_loss": 3.5746959665045734, + "tokens_seen": 1247604736 + }, + { + "epoch": 4.02, + "learning_rate": 0.00031414242728184554, + "loss": 2.7003, + "theoretical_loss": 3.5746788202413784, + "tokens_seen": 1247670272 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003141323971915747, + "loss": 2.3663, + "theoretical_loss": 3.57466167513096, + "tokens_seen": 1247735808 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003141223671013039, + "loss": 2.7268, + "theoretical_loss": 3.5746445311731794, + "tokens_seen": 1247801344 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003141123370110331, + "loss": 2.6805, + "theoretical_loss": 3.5746273883678983, + "tokens_seen": 1247866880 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003141023069207623, + "loss": 2.7947, + "theoretical_loss": 3.57461024671498, + "tokens_seen": 1247932416 + }, + { + "epoch": 4.02, + "learning_rate": 0.00031409227683049145, + "loss": 2.8079, + "theoretical_loss": 3.574593106214285, + "tokens_seen": 1247997952 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003140822467402207, + "loss": 2.6496, + "theoretical_loss": 3.5745759668656767, + "tokens_seen": 1248063488 + }, + { + "epoch": 4.02, + "learning_rate": 0.00031407221664994986, + "loss": 2.6264, + "theoretical_loss": 3.574558828669016, + "tokens_seen": 1248129024 + }, + { + "epoch": 4.02, + "learning_rate": 0.00031406218655967904, + "loss": 2.2906, + "theoretical_loss": 3.574541691624166, + "tokens_seen": 1248194560 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003140521564694082, + "loss": 2.902, + "theoretical_loss": 3.574524555730988, + "tokens_seen": 1248260096 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003140421263791374, + "loss": 2.7177, + "theoretical_loss": 3.5745074209893453, + "tokens_seen": 1248325632 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003140320962888666, + "loss": 2.7631, + "theoretical_loss": 3.5744902873990982, + "tokens_seen": 1248391168 + }, + { + "epoch": 4.02, + "objective/train/docs_used": 1437026, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7030203342437744, + "objective/train/theoretical_loss": 3.5744731549601108, + "objective/train/tokens_used": 1268916704, + "theoretical_loss": 3.5744731549601108, + "tokens_seen": 1248456704 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003140220661985958, + "loss": 2.6616, + "theoretical_loss": 3.5744731549601108, + "tokens_seen": 1248456704 + }, + { + "epoch": 4.02, + "learning_rate": 0.00031401203610832495, + "loss": 2.6589, + "theoretical_loss": 3.5744560236722442, + "tokens_seen": 1248522240 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003140020060180542, + "loss": 2.6114, + "theoretical_loss": 3.574438893535361, + "tokens_seen": 1248587776 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003139919759277833, + "loss": 2.7657, + "theoretical_loss": 3.5744217645493235, + "tokens_seen": 1248653312 + }, + { + "epoch": 4.02, + "learning_rate": 0.00031398194583751255, + "loss": 2.6823, + "theoretical_loss": 3.574404636713994, + "tokens_seen": 1248718848 + }, + { + "epoch": 4.02, + "learning_rate": 0.00031397191574724173, + "loss": 2.5717, + "theoretical_loss": 3.5743875100292346, + "tokens_seen": 1248784384 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003139618856569709, + "loss": 2.6765, + "theoretical_loss": 3.574370384494908, + "tokens_seen": 1248849920 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003139518555667001, + "loss": 2.6439, + "theoretical_loss": 3.574353260110876, + "tokens_seen": 1248915456 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003139418254764293, + "loss": 2.5065, + "theoretical_loss": 3.574336136877001, + "tokens_seen": 1248980992 + }, + { + "epoch": 4.02, + "learning_rate": 0.00031393179538615845, + "loss": 2.713, + "theoretical_loss": 3.5743190147931463, + "tokens_seen": 1249046528 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003139217652958877, + "loss": 2.5401, + "theoretical_loss": 3.5743018938591735, + "tokens_seen": 1249112064 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003139117352056168, + "loss": 2.6175, + "theoretical_loss": 3.574284774074945, + "tokens_seen": 1249177600 + }, + { + "epoch": 4.02, + "learning_rate": 0.00031390170511534605, + "loss": 2.6046, + "theoretical_loss": 3.574267655440323, + "tokens_seen": 1249243136 + }, + { + "epoch": 4.02, + "learning_rate": 0.00031389167502507523, + "loss": 2.6326, + "theoretical_loss": 3.574250537955172, + "tokens_seen": 1249308672 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003138816449348044, + "loss": 2.4136, + "theoretical_loss": 3.5742334216193523, + "tokens_seen": 1249374208 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003138716148445336, + "loss": 2.4814, + "theoretical_loss": 3.574216306432727, + "tokens_seen": 1249439744 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003138615847542628, + "loss": 2.6682, + "theoretical_loss": 3.5741991923951595, + "tokens_seen": 1249505280 + }, + { + "epoch": 4.02, + "learning_rate": 0.00031385155466399196, + "loss": 2.7518, + "theoretical_loss": 3.574182079506511, + "tokens_seen": 1249570816 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003138415245737212, + "loss": 2.5976, + "theoretical_loss": 3.5741649677666456, + "tokens_seen": 1249636352 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003138314944834503, + "loss": 2.7658, + "theoretical_loss": 3.574147857175425, + "tokens_seen": 1249701888 + }, + { + "epoch": 4.02, + "learning_rate": 0.00031382146439317955, + "loss": 2.8964, + "theoretical_loss": 3.574130747732712, + "tokens_seen": 1249767424 + }, + { + "epoch": 4.02, + "learning_rate": 0.00031381143430290873, + "loss": 2.6613, + "theoretical_loss": 3.5741136394383695, + "tokens_seen": 1249832960 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003138014042126379, + "loss": 2.5456, + "theoretical_loss": 3.57409653229226, + "tokens_seen": 1249898496 + }, + { + "epoch": 4.02, + "learning_rate": 0.00031379137412236715, + "loss": 2.7456, + "theoretical_loss": 3.5740794262942464, + "tokens_seen": 1249964032 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003137813440320963, + "loss": 2.5863, + "theoretical_loss": 3.574062321444192, + "tokens_seen": 1250029568 + }, + { + "epoch": 4.02, + "objective/train/docs_used": 1438333, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5628535747528076, + "objective/train/theoretical_loss": 3.5740452177419577, + "objective/train/tokens_used": 1270555104, + "theoretical_loss": 3.5740452177419577, + "tokens_seen": 1250095104 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003137713139418255, + "loss": 2.7452, + "theoretical_loss": 3.5740452177419577, + "tokens_seen": 1250095104 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003137612838515547, + "loss": 2.5718, + "theoretical_loss": 3.5740281151874087, + "tokens_seen": 1250160640 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003137512537612839, + "loss": 2.5954, + "theoretical_loss": 3.5740110137804066, + "tokens_seen": 1250226176 + }, + { + "epoch": 4.02, + "learning_rate": 0.00031374122367101306, + "loss": 2.577, + "theoretical_loss": 3.5739939135208143, + "tokens_seen": 1250291712 + }, + { + "epoch": 4.02, + "learning_rate": 0.00031373119358074224, + "loss": 2.5118, + "theoretical_loss": 3.5739768144084945, + "tokens_seen": 1250357248 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003137211634904714, + "loss": 2.7736, + "theoretical_loss": 3.5739597164433103, + "tokens_seen": 1250422784 + }, + { + "epoch": 4.02, + "learning_rate": 0.00031371113340020065, + "loss": 2.8973, + "theoretical_loss": 3.5739426196251247, + "tokens_seen": 1250488320 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003137011033099298, + "loss": 2.9025, + "theoretical_loss": 3.5739255239538013, + "tokens_seen": 1250553856 + }, + { + "epoch": 4.02, + "learning_rate": 0.000313691073219659, + "loss": 2.8072, + "theoretical_loss": 3.573908429429202, + "tokens_seen": 1250619392 + }, + { + "epoch": 4.02, + "learning_rate": 0.00031368104312938814, + "loss": 2.5561, + "theoretical_loss": 3.5738913360511897, + "tokens_seen": 1250684928 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003136710130391174, + "loss": 2.9685, + "theoretical_loss": 3.5738742438196285, + "tokens_seen": 1250750464 + }, + { + "epoch": 4.02, + "learning_rate": 0.00031366098294884656, + "loss": 2.7795, + "theoretical_loss": 3.5738571527343805, + "tokens_seen": 1250816000 + }, + { + "epoch": 4.02, + "learning_rate": 0.00031365095285857574, + "loss": 2.7314, + "theoretical_loss": 3.573840062795309, + "tokens_seen": 1250881536 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003136409227683049, + "loss": 2.8282, + "theoretical_loss": 3.5738229740022778, + "tokens_seen": 1250947072 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003136308926780341, + "loss": 2.6157, + "theoretical_loss": 3.5738058863551494, + "tokens_seen": 1251012608 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003136208625877633, + "loss": 2.8974, + "theoretical_loss": 3.5737887998537867, + "tokens_seen": 1251078144 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003136108324974925, + "loss": 2.7745, + "theoretical_loss": 3.5737717144980534, + "tokens_seen": 1251143680 + }, + { + "epoch": 4.02, + "learning_rate": 0.00031360080240722165, + "loss": 2.7687, + "theoretical_loss": 3.573754630287812, + "tokens_seen": 1251209216 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003135907723169509, + "loss": 2.6226, + "theoretical_loss": 3.573737547222926, + "tokens_seen": 1251274752 + }, + { + "epoch": 4.02, + "learning_rate": 0.00031358074222668006, + "loss": 2.568, + "theoretical_loss": 3.5737204653032597, + "tokens_seen": 1251340288 + }, + { + "epoch": 4.02, + "learning_rate": 0.00031357071213640924, + "loss": 2.5222, + "theoretical_loss": 3.5737033845286748, + "tokens_seen": 1251405824 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003135606820461384, + "loss": 2.7934, + "theoretical_loss": 3.573686304899035, + "tokens_seen": 1251471360 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003135506519558676, + "loss": 2.5712, + "theoretical_loss": 3.5736692264142036, + "tokens_seen": 1251536896 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003135406218655968, + "loss": 2.4537, + "theoretical_loss": 3.5736521490740447, + "tokens_seen": 1251602432 + }, + { + "epoch": 4.02, + "learning_rate": 0.000313530591775326, + "loss": 2.4182, + "theoretical_loss": 3.573635072878421, + "tokens_seen": 1251667968 + }, + { + "epoch": 4.02, + "objective/train/docs_used": 1439071, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.880246639251709, + "objective/train/theoretical_loss": 3.5736179978271956, + "objective/train/tokens_used": 1272193504, + "theoretical_loss": 3.5736179978271956, + "tokens_seen": 1251733504 + }, + { + "epoch": 4.02, + "learning_rate": 0.00031352056168505515, + "loss": 2.4664, + "theoretical_loss": 3.5736179978271956, + "tokens_seen": 1251733504 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003135105315947844, + "loss": 2.862, + "theoretical_loss": 3.5736009239202327, + "tokens_seen": 1251799040 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003135005015045135, + "loss": 2.8991, + "theoretical_loss": 3.5735838511573945, + "tokens_seen": 1251864576 + }, + { + "epoch": 4.02, + "learning_rate": 0.00031349047141424275, + "loss": 2.9548, + "theoretical_loss": 3.573566779538546, + "tokens_seen": 1251930112 + }, + { + "epoch": 4.02, + "learning_rate": 0.00031348044132397193, + "loss": 2.5068, + "theoretical_loss": 3.5735497090635495, + "tokens_seen": 1251995648 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003134704112337011, + "loss": 2.6644, + "theoretical_loss": 3.573532639732269, + "tokens_seen": 1252061184 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003134603811434303, + "loss": 2.6921, + "theoretical_loss": 3.5735155715445677, + "tokens_seen": 1252126720 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003134503510531595, + "loss": 2.7023, + "theoretical_loss": 3.573498504500309, + "tokens_seen": 1252192256 + }, + { + "epoch": 4.02, + "learning_rate": 0.00031344032096288865, + "loss": 2.725, + "theoretical_loss": 3.573481438599357, + "tokens_seen": 1252257792 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003134302908726179, + "loss": 2.7769, + "theoretical_loss": 3.5734643738415754, + "tokens_seen": 1252323328 + }, + { + "epoch": 4.02, + "learning_rate": 0.000313420260782347, + "loss": 2.5722, + "theoretical_loss": 3.573447310226827, + "tokens_seen": 1252388864 + }, + { + "epoch": 4.02, + "learning_rate": 0.00031341023069207625, + "loss": 2.7007, + "theoretical_loss": 3.5734302477549758, + "tokens_seen": 1252454400 + }, + { + "epoch": 4.02, + "learning_rate": 0.00031340020060180543, + "loss": 2.6761, + "theoretical_loss": 3.573413186425886, + "tokens_seen": 1252519936 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003133901705115346, + "loss": 2.5791, + "theoretical_loss": 3.5733961262394205, + "tokens_seen": 1252585472 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003133801404212638, + "loss": 2.5603, + "theoretical_loss": 3.5733790671954435, + "tokens_seen": 1252651008 + }, + { + "epoch": 4.02, + "learning_rate": 0.000313370110330993, + "loss": 2.617, + "theoretical_loss": 3.5733620092938185, + "tokens_seen": 1252716544 + }, + { + "epoch": 4.02, + "learning_rate": 0.00031336008024072216, + "loss": 2.7296, + "theoretical_loss": 3.573344952534409, + "tokens_seen": 1252782080 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003133500501504514, + "loss": 2.5422, + "theoretical_loss": 3.573327896917079, + "tokens_seen": 1252847616 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003133400200601805, + "loss": 2.6829, + "theoretical_loss": 3.5733108424416926, + "tokens_seen": 1252913152 + }, + { + "epoch": 4.02, + "learning_rate": 0.00031332998996990975, + "loss": 2.48, + "theoretical_loss": 3.5732937891081136, + "tokens_seen": 1252978688 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003133199598796389, + "loss": 2.9448, + "theoretical_loss": 3.573276736916205, + "tokens_seen": 1253044224 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003133099297893681, + "loss": 2.6787, + "theoretical_loss": 3.5732596858658314, + "tokens_seen": 1253109760 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003132998996990973, + "loss": 2.6809, + "theoretical_loss": 3.5732426359568565, + "tokens_seen": 1253175296 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003132898696088265, + "loss": 2.4739, + "theoretical_loss": 3.5732255871891443, + "tokens_seen": 1253240832 + }, + { + "epoch": 4.02, + "learning_rate": 0.00031327983951855566, + "loss": 2.7654, + "theoretical_loss": 3.5732085395625592, + "tokens_seen": 1253306368 + }, + { + "epoch": 4.02, + "objective/train/docs_used": 1440666, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5984394550323486, + "objective/train/theoretical_loss": 3.573191493076964, + "objective/train/tokens_used": 1273831904, + "theoretical_loss": 3.573191493076964, + "tokens_seen": 1253371904 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003132698094282849, + "loss": 2.6108, + "theoretical_loss": 3.573191493076964, + "tokens_seen": 1253371904 + }, + { + "epoch": 4.02, + "learning_rate": 0.000313259779338014, + "loss": 2.734, + "theoretical_loss": 3.5731744477322236, + "tokens_seen": 1253437440 + }, + { + "epoch": 4.02, + "learning_rate": 0.00031324974924774326, + "loss": 2.7576, + "theoretical_loss": 3.5731574035282017, + "tokens_seen": 1253502976 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003132397191574724, + "loss": 2.8248, + "theoretical_loss": 3.5731403604647625, + "tokens_seen": 1253568512 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003132296890672016, + "loss": 2.7149, + "theoretical_loss": 3.5731233185417697, + "tokens_seen": 1253634048 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003132196589769308, + "loss": 2.5068, + "theoretical_loss": 3.5731062777590874, + "tokens_seen": 1253699584 + }, + { + "epoch": 4.02, + "learning_rate": 0.00031320962888666, + "loss": 2.4724, + "theoretical_loss": 3.57308923811658, + "tokens_seen": 1253765120 + }, + { + "epoch": 4.02, + "learning_rate": 0.00031319959879638916, + "loss": 2.5921, + "theoretical_loss": 3.5730721996141117, + "tokens_seen": 1253830656 + }, + { + "epoch": 4.02, + "learning_rate": 0.00031318956870611834, + "loss": 2.7723, + "theoretical_loss": 3.5730551622515465, + "tokens_seen": 1253896192 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003131795386158475, + "loss": 2.5817, + "theoretical_loss": 3.5730381260287483, + "tokens_seen": 1253961728 + }, + { + "epoch": 4.02, + "learning_rate": 0.00031316950852557676, + "loss": 2.4434, + "theoretical_loss": 3.5730210909455815, + "tokens_seen": 1254027264 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003131594784353059, + "loss": 2.8277, + "theoretical_loss": 3.57300405700191, + "tokens_seen": 1254092800 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003131494483450351, + "loss": 2.4262, + "theoretical_loss": 3.572987024197599, + "tokens_seen": 1254158336 + }, + { + "epoch": 4.02, + "learning_rate": 0.00031313941825476425, + "loss": 2.32, + "theoretical_loss": 3.572969992532512, + "tokens_seen": 1254223872 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003131293881644935, + "loss": 2.4445, + "theoretical_loss": 3.5729529620065126, + "tokens_seen": 1254289408 + }, + { + "epoch": 4.02, + "learning_rate": 0.00031311935807422267, + "loss": 2.7825, + "theoretical_loss": 3.5729359326194663, + "tokens_seen": 1254354944 + }, + { + "epoch": 4.02, + "learning_rate": 0.00031310932798395185, + "loss": 2.6967, + "theoretical_loss": 3.572918904371237, + "tokens_seen": 1254420480 + }, + { + "epoch": 4.02, + "learning_rate": 0.00031309929789368103, + "loss": 2.5315, + "theoretical_loss": 3.57290187726169, + "tokens_seen": 1254486016 + }, + { + "epoch": 4.02, + "learning_rate": 0.00031308926780341026, + "loss": 2.3605, + "theoretical_loss": 3.5728848512906874, + "tokens_seen": 1254551552 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003130792377131394, + "loss": 2.5921, + "theoretical_loss": 3.5728678264580958, + "tokens_seen": 1254617088 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003130692076228686, + "loss": 2.8032, + "theoretical_loss": 3.572850802763778, + "tokens_seen": 1254682624 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003130591775325978, + "loss": 2.7727, + "theoretical_loss": 3.5728337802076, + "tokens_seen": 1254748160 + }, + { + "epoch": 4.02, + "learning_rate": 0.000313049147442327, + "loss": 2.5223, + "theoretical_loss": 3.5728167587894255, + "tokens_seen": 1254813696 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003130391173520562, + "loss": 2.7716, + "theoretical_loss": 3.5727997385091186, + "tokens_seen": 1254879232 + }, + { + "epoch": 4.02, + "learning_rate": 0.00031302908726178535, + "loss": 2.7501, + "theoretical_loss": 3.5727827193665442, + "tokens_seen": 1254944768 + }, + { + "epoch": 4.02, + "objective/train/docs_used": 1441890, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5261950492858887, + "objective/train/theoretical_loss": 3.5727657013615666, + "objective/train/tokens_used": 1275470304, + "theoretical_loss": 3.5727657013615666, + "tokens_seen": 1255010304 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003130190571715146, + "loss": 2.4896, + "theoretical_loss": 3.5727657013615666, + "tokens_seen": 1255010304 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003130090270812437, + "loss": 2.6838, + "theoretical_loss": 3.572748684494051, + "tokens_seen": 1255075840 + }, + { + "epoch": 4.02, + "learning_rate": 0.00031299899699097295, + "loss": 2.6958, + "theoretical_loss": 3.5727316687638613, + "tokens_seen": 1255141376 + }, + { + "epoch": 4.02, + "learning_rate": 0.00031298896690070213, + "loss": 2.7371, + "theoretical_loss": 3.5727146541708628, + "tokens_seen": 1255206912 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003129789368104313, + "loss": 2.6208, + "theoretical_loss": 3.572697640714919, + "tokens_seen": 1255272448 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003129689067201605, + "loss": 2.7077, + "theoretical_loss": 3.5726806283958963, + "tokens_seen": 1255337984 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003129588766298897, + "loss": 2.81, + "theoretical_loss": 3.5726636172136574, + "tokens_seen": 1255403520 + }, + { + "epoch": 4.02, + "learning_rate": 0.00031294884653961885, + "loss": 2.9799, + "theoretical_loss": 3.5726466071680685, + "tokens_seen": 1255469056 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003129388164493481, + "loss": 2.8607, + "theoretical_loss": 3.5726295982589935, + "tokens_seen": 1255534592 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003129287863590772, + "loss": 2.9026, + "theoretical_loss": 3.5726125904862975, + "tokens_seen": 1255600128 + }, + { + "epoch": 4.02, + "learning_rate": 0.00031291875626880645, + "loss": 2.6078, + "theoretical_loss": 3.5725955838498455, + "tokens_seen": 1255665664 + }, + { + "epoch": 4.02, + "learning_rate": 0.00031290872617853563, + "loss": 2.4904, + "theoretical_loss": 3.5725785783495017, + "tokens_seen": 1255731200 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003128986960882648, + "loss": 2.7038, + "theoretical_loss": 3.572561573985131, + "tokens_seen": 1255796736 + }, + { + "epoch": 4.02, + "learning_rate": 0.000312888665997994, + "loss": 2.6649, + "theoretical_loss": 3.5725445707565986, + "tokens_seen": 1255862272 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003128786359077232, + "loss": 2.5088, + "theoretical_loss": 3.5725275686637694, + "tokens_seen": 1255927808 + }, + { + "epoch": 4.02, + "learning_rate": 0.00031286860581745236, + "loss": 2.8002, + "theoretical_loss": 3.572510567706508, + "tokens_seen": 1255993344 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003128585757271816, + "loss": 2.9148, + "theoretical_loss": 3.572493567884679, + "tokens_seen": 1256058880 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003128485456369107, + "loss": 2.6956, + "theoretical_loss": 3.572476569198148, + "tokens_seen": 1256124416 + }, + { + "epoch": 4.02, + "learning_rate": 0.00031283851554663995, + "loss": 2.6343, + "theoretical_loss": 3.57245957164678, + "tokens_seen": 1256189952 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003128284854563691, + "loss": 2.7083, + "theoretical_loss": 3.5724425752304394, + "tokens_seen": 1256255488 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003128184553660983, + "loss": 2.7523, + "theoretical_loss": 3.5724255799489915, + "tokens_seen": 1256321024 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003128084252758275, + "loss": 2.9487, + "theoretical_loss": 3.572408585802301, + "tokens_seen": 1256386560 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003127983951855567, + "loss": 2.5786, + "theoretical_loss": 3.5723915927902334, + "tokens_seen": 1256452096 + }, + { + "epoch": 4.02, + "learning_rate": 0.00031278836509528586, + "loss": 2.5373, + "theoretical_loss": 3.572374600912654, + "tokens_seen": 1256517632 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003127783350050151, + "loss": 2.825, + "theoretical_loss": 3.572357610169427, + "tokens_seen": 1256583168 + }, + { + "epoch": 4.02, + "objective/train/docs_used": 1442594, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8732402324676514, + "objective/train/theoretical_loss": 3.5723406205604182, + "objective/train/tokens_used": 1277108704, + "theoretical_loss": 3.5723406205604182, + "tokens_seen": 1256648704 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003127683049147442, + "loss": 2.6564, + "theoretical_loss": 3.5723406205604182, + "tokens_seen": 1256648704 + }, + { + "epoch": 4.02, + "learning_rate": 0.00031275827482447346, + "loss": 2.6124, + "theoretical_loss": 3.572323632085493, + "tokens_seen": 1256714240 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003127482447342026, + "loss": 2.7265, + "theoretical_loss": 3.5723066447445158, + "tokens_seen": 1256779776 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003127382146439318, + "loss": 2.6958, + "theoretical_loss": 3.572289658537352, + "tokens_seen": 1256845312 + }, + { + "epoch": 4.02, + "learning_rate": 0.000312728184553661, + "loss": 2.7697, + "theoretical_loss": 3.572272673463867, + "tokens_seen": 1256910848 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003127181544633902, + "loss": 2.5176, + "theoretical_loss": 3.5722556895239257, + "tokens_seen": 1256976384 + }, + { + "epoch": 4.02, + "learning_rate": 0.00031270812437311936, + "loss": 2.8485, + "theoretical_loss": 3.5722387067173935, + "tokens_seen": 1257041920 + }, + { + "epoch": 4.02, + "learning_rate": 0.00031269809428284854, + "loss": 2.9032, + "theoretical_loss": 3.5722217250441366, + "tokens_seen": 1257107456 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003126880641925777, + "loss": 2.7659, + "theoretical_loss": 3.5722047445040186, + "tokens_seen": 1257172992 + }, + { + "epoch": 4.02, + "learning_rate": 0.00031267803410230696, + "loss": 2.6285, + "theoretical_loss": 3.5721877650969063, + "tokens_seen": 1257238528 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003126680040120361, + "loss": 2.7843, + "theoretical_loss": 3.572170786822664, + "tokens_seen": 1257304064 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003126579739217653, + "loss": 2.5797, + "theoretical_loss": 3.572153809681158, + "tokens_seen": 1257369600 + }, + { + "epoch": 4.02, + "learning_rate": 0.00031264794383149445, + "loss": 2.5359, + "theoretical_loss": 3.572136833672253, + "tokens_seen": 1257435136 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003126379137412237, + "loss": 2.5155, + "theoretical_loss": 3.5721198587958147, + "tokens_seen": 1257500672 + }, + { + "epoch": 4.02, + "learning_rate": 0.00031262788365095287, + "loss": 2.6478, + "theoretical_loss": 3.5721028850517085, + "tokens_seen": 1257566208 + }, + { + "epoch": 4.02, + "learning_rate": 0.00031261785356068205, + "loss": 2.7653, + "theoretical_loss": 3.5720859124397997, + "tokens_seen": 1257631744 + }, + { + "epoch": 4.02, + "learning_rate": 0.00031260782347041123, + "loss": 2.6135, + "theoretical_loss": 3.572068940959954, + "tokens_seen": 1257697280 + }, + { + "epoch": 4.02, + "learning_rate": 0.00031259779338014046, + "loss": 2.6955, + "theoretical_loss": 3.572051970612037, + "tokens_seen": 1257762816 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003125877632898696, + "loss": 2.7319, + "theoretical_loss": 3.5720350013959137, + "tokens_seen": 1257828352 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003125777331995988, + "loss": 2.7793, + "theoretical_loss": 3.5720180333114504, + "tokens_seen": 1257893888 + }, + { + "epoch": 4.02, + "learning_rate": 0.00031256770310932795, + "loss": 2.6664, + "theoretical_loss": 3.572001066358512, + "tokens_seen": 1257959424 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003125576730190572, + "loss": 2.7865, + "theoretical_loss": 3.571984100536964, + "tokens_seen": 1258024960 + }, + { + "epoch": 4.02, + "learning_rate": 0.00031254764292878637, + "loss": 2.5398, + "theoretical_loss": 3.5719671358466734, + "tokens_seen": 1258090496 + }, + { + "epoch": 4.02, + "learning_rate": 0.00031253761283851555, + "loss": 2.8289, + "theoretical_loss": 3.5719501722875044, + "tokens_seen": 1258156032 + }, + { + "epoch": 4.02, + "learning_rate": 0.00031252758274824473, + "loss": 2.6467, + "theoretical_loss": 3.5719332098593233, + "tokens_seen": 1258221568 + }, + { + "epoch": 4.02, + "objective/train/docs_used": 1443926, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4860124588012695, + "objective/train/theoretical_loss": 3.5719162485619953, + "objective/train/tokens_used": 1278747104, + "theoretical_loss": 3.5719162485619953, + "tokens_seen": 1258287104 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003125175526579739, + "loss": 2.571, + "theoretical_loss": 3.5719162485619953, + "tokens_seen": 1258287104 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003125075225677031, + "loss": 2.7123, + "theoretical_loss": 3.5718992883953864, + "tokens_seen": 1258352640 + }, + { + "epoch": 4.02, + "learning_rate": 0.00031249749247743233, + "loss": 2.6302, + "theoretical_loss": 3.571882329359363, + "tokens_seen": 1258418176 + }, + { + "epoch": 4.02, + "learning_rate": 0.00031248746238716146, + "loss": 2.7074, + "theoretical_loss": 3.57186537145379, + "tokens_seen": 1258483712 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003124774322968907, + "loss": 2.6631, + "theoretical_loss": 3.571848414678533, + "tokens_seen": 1258549248 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003124674022066198, + "loss": 2.6379, + "theoretical_loss": 3.5718314590334583, + "tokens_seen": 1258614784 + }, + { + "epoch": 4.02, + "learning_rate": 0.00031245737211634905, + "loss": 2.8167, + "theoretical_loss": 3.571814504518432, + "tokens_seen": 1258680320 + }, + { + "epoch": 4.02, + "learning_rate": 0.00031244734202607823, + "loss": 2.6393, + "theoretical_loss": 3.5717975511333195, + "tokens_seen": 1258745856 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003124373119358074, + "loss": 2.586, + "theoretical_loss": 3.5717805988779867, + "tokens_seen": 1258811392 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003124272818455366, + "loss": 2.6975, + "theoretical_loss": 3.5717636477522996, + "tokens_seen": 1258876928 + }, + { + "epoch": 4.02, + "learning_rate": 0.00031241725175526583, + "loss": 2.8738, + "theoretical_loss": 3.5717466977561245, + "tokens_seen": 1258942464 + }, + { + "epoch": 4.02, + "learning_rate": 0.00031240722166499496, + "loss": 2.5827, + "theoretical_loss": 3.5717297488893265, + "tokens_seen": 1259008000 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003123971915747242, + "loss": 2.835, + "theoretical_loss": 3.5717128011517723, + "tokens_seen": 1259073536 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003123871614844533, + "loss": 2.6795, + "theoretical_loss": 3.5716958545433277, + "tokens_seen": 1259139072 + }, + { + "epoch": 4.02, + "learning_rate": 0.00031237713139418256, + "loss": 2.5723, + "theoretical_loss": 3.5716789090638583, + "tokens_seen": 1259204608 + }, + { + "epoch": 4.02, + "learning_rate": 0.00031236710130391174, + "loss": 2.5694, + "theoretical_loss": 3.5716619647132304, + "tokens_seen": 1259270144 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003123570712136409, + "loss": 2.7332, + "theoretical_loss": 3.5716450214913102, + "tokens_seen": 1259335680 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003123470411233701, + "loss": 2.7726, + "theoretical_loss": 3.5716280793979642, + "tokens_seen": 1259401216 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003123370110330993, + "loss": 2.6838, + "theoretical_loss": 3.5716111384330578, + "tokens_seen": 1259466752 + }, + { + "epoch": 4.02, + "learning_rate": 0.00031232698094282846, + "loss": 2.7907, + "theoretical_loss": 3.5715941985964568, + "tokens_seen": 1259532288 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003123169508525577, + "loss": 2.6441, + "theoretical_loss": 3.571577259888028, + "tokens_seen": 1259597824 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003123069207622869, + "loss": 2.5449, + "theoretical_loss": 3.5715603223076378, + "tokens_seen": 1259663360 + }, + { + "epoch": 4.02, + "learning_rate": 0.00031229689067201606, + "loss": 2.6923, + "theoretical_loss": 3.5715433858551524, + "tokens_seen": 1259728896 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003122868605817453, + "loss": 2.7129, + "theoretical_loss": 3.571526450530437, + "tokens_seen": 1259794432 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003122768304914744, + "loss": 2.7213, + "theoretical_loss": 3.571509516333359, + "tokens_seen": 1259859968 + }, + { + "epoch": 4.02, + "objective/train/docs_used": 1444504, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6051039695739746, + "objective/train/theoretical_loss": 3.5714925832637836, + "objective/train/tokens_used": 1280385504, + "theoretical_loss": 3.5714925832637836, + "tokens_seen": 1259925504 + }, + { + "epoch": 4.02, + "learning_rate": 0.00031226680040120366, + "loss": 2.6518, + "theoretical_loss": 3.5714925832637836, + "tokens_seen": 1259925504 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003122567703109328, + "loss": 2.6353, + "theoretical_loss": 3.571475651321578, + "tokens_seen": 1259991040 + }, + { + "epoch": 4.02, + "learning_rate": 0.000312246740220662, + "loss": 2.8753, + "theoretical_loss": 3.571458720506608, + "tokens_seen": 1260056576 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003122367101303912, + "loss": 2.6656, + "theoretical_loss": 3.5714417908187404, + "tokens_seen": 1260122112 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003122266800401204, + "loss": 2.6615, + "theoretical_loss": 3.5714248622578415, + "tokens_seen": 1260187648 + }, + { + "epoch": 4.02, + "learning_rate": 0.00031221664994984956, + "loss": 2.6877, + "theoretical_loss": 3.571407934823777, + "tokens_seen": 1260253184 + }, + { + "epoch": 4.02, + "learning_rate": 0.00031220661985957874, + "loss": 2.7432, + "theoretical_loss": 3.5713910085164136, + "tokens_seen": 1260318720 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003121965897693079, + "loss": 2.6483, + "theoretical_loss": 3.571374083335618, + "tokens_seen": 1260384256 + }, + { + "epoch": 4.02, + "learning_rate": 0.00031218655967903716, + "loss": 2.4913, + "theoretical_loss": 3.5713571592812565, + "tokens_seen": 1260449792 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003121765295887663, + "loss": 2.5252, + "theoretical_loss": 3.571340236353196, + "tokens_seen": 1260515328 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003121664994984955, + "loss": 2.9449, + "theoretical_loss": 3.5713233145513024, + "tokens_seen": 1260580864 + }, + { + "epoch": 4.02, + "learning_rate": 0.00031215646940822465, + "loss": 2.8003, + "theoretical_loss": 3.5713063938754424, + "tokens_seen": 1260646400 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003121464393179539, + "loss": 2.5984, + "theoretical_loss": 3.571289474325482, + "tokens_seen": 1260711936 + }, + { + "epoch": 4.02, + "learning_rate": 0.00031213640922768307, + "loss": 2.7581, + "theoretical_loss": 3.571272555901289, + "tokens_seen": 1260777472 + }, + { + "epoch": 4.02, + "learning_rate": 0.00031212637913741225, + "loss": 2.8671, + "theoretical_loss": 3.5712556386027288, + "tokens_seen": 1260843008 + }, + { + "epoch": 4.02, + "learning_rate": 0.00031211634904714143, + "loss": 2.5729, + "theoretical_loss": 3.571238722429669, + "tokens_seen": 1260908544 + }, + { + "epoch": 4.02, + "learning_rate": 0.00031210631895687066, + "loss": 2.7379, + "theoretical_loss": 3.571221807381975, + "tokens_seen": 1260974080 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003120962888665998, + "loss": 2.4806, + "theoretical_loss": 3.571204893459515, + "tokens_seen": 1261039616 + }, + { + "epoch": 4.02, + "learning_rate": 0.000312086258776329, + "loss": 2.725, + "theoretical_loss": 3.5711879806621543, + "tokens_seen": 1261105152 + }, + { + "epoch": 4.02, + "learning_rate": 0.00031207622868605815, + "loss": 2.7716, + "theoretical_loss": 3.5711710689897602, + "tokens_seen": 1261170688 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003120661985957874, + "loss": 2.5052, + "theoretical_loss": 3.5711541584421993, + "tokens_seen": 1261236224 + }, + { + "epoch": 4.02, + "learning_rate": 0.00031205616850551657, + "loss": 2.6935, + "theoretical_loss": 3.5711372490193383, + "tokens_seen": 1261301760 + }, + { + "epoch": 4.02, + "learning_rate": 0.00031204613841524575, + "loss": 2.5666, + "theoretical_loss": 3.571120340721044, + "tokens_seen": 1261367296 + }, + { + "epoch": 4.02, + "learning_rate": 0.00031203610832497493, + "loss": 2.6823, + "theoretical_loss": 3.571103433547184, + "tokens_seen": 1261432832 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003120260782347041, + "loss": 2.7387, + "theoretical_loss": 3.5710865274976236, + "tokens_seen": 1261498368 + }, + { + "epoch": 4.02, + "objective/train/docs_used": 1445499, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4663262367248535, + "objective/train/theoretical_loss": 3.5710696225722307, + "objective/train/tokens_used": 1282023904, + "theoretical_loss": 3.5710696225722307, + "tokens_seen": 1261563904 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003120160481444333, + "loss": 2.7363, + "theoretical_loss": 3.5710696225722307, + "tokens_seen": 1261563904 + }, + { + "epoch": 4.02, + "learning_rate": 0.00031200601805416253, + "loss": 2.6551, + "theoretical_loss": 3.571052718770871, + "tokens_seen": 1261629440 + }, + { + "epoch": 4.02, + "learning_rate": 0.00031199598796389166, + "loss": 2.9558, + "theoretical_loss": 3.5710358160934135, + "tokens_seen": 1261694976 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003119859578736209, + "loss": 2.7521, + "theoretical_loss": 3.571018914539723, + "tokens_seen": 1261760512 + }, + { + "epoch": 4.02, + "learning_rate": 0.00031197592778335, + "loss": 2.6555, + "theoretical_loss": 3.5710020141096677, + "tokens_seen": 1261826048 + }, + { + "epoch": 4.02, + "learning_rate": 0.00031196589769307925, + "loss": 2.6724, + "theoretical_loss": 3.570985114803114, + "tokens_seen": 1261891584 + }, + { + "epoch": 4.02, + "learning_rate": 0.00031195586760280843, + "loss": 2.766, + "theoretical_loss": 3.570968216619929, + "tokens_seen": 1261957120 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003119458375125376, + "loss": 2.7561, + "theoretical_loss": 3.5709513195599794, + "tokens_seen": 1262022656 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003119358074222668, + "loss": 2.7715, + "theoretical_loss": 3.5709344236231324, + "tokens_seen": 1262088192 + }, + { + "epoch": 4.02, + "learning_rate": 0.00031192577733199603, + "loss": 2.7651, + "theoretical_loss": 3.5709175288092556, + "tokens_seen": 1262153728 + }, + { + "epoch": 4.02, + "learning_rate": 0.00031191574724172516, + "loss": 2.5331, + "theoretical_loss": 3.5709006351182153, + "tokens_seen": 1262219264 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003119057171514544, + "loss": 2.7933, + "theoretical_loss": 3.5708837425498787, + "tokens_seen": 1262284800 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003118956870611835, + "loss": 2.5899, + "theoretical_loss": 3.570866851104113, + "tokens_seen": 1262350336 + }, + { + "epoch": 4.02, + "learning_rate": 0.00031188565697091276, + "loss": 2.7355, + "theoretical_loss": 3.570849960780786, + "tokens_seen": 1262415872 + }, + { + "epoch": 4.02, + "learning_rate": 0.00031187562688064194, + "loss": 2.7772, + "theoretical_loss": 3.570833071579764, + "tokens_seen": 1262481408 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003118655967903711, + "loss": 2.3021, + "theoretical_loss": 3.570816183500914, + "tokens_seen": 1262546944 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003118555667001003, + "loss": 2.7408, + "theoretical_loss": 3.570799296544104, + "tokens_seen": 1262612480 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003118455366098295, + "loss": 2.6958, + "theoretical_loss": 3.5707824107092003, + "tokens_seen": 1262678016 + }, + { + "epoch": 4.02, + "learning_rate": 0.00031183550651955866, + "loss": 3.0141, + "theoretical_loss": 3.5707655259960713, + "tokens_seen": 1262743552 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003118254764292879, + "loss": 2.7682, + "theoretical_loss": 3.5707486424045833, + "tokens_seen": 1262809088 + }, + { + "epoch": 4.02, + "learning_rate": 0.000311815446339017, + "loss": 2.6249, + "theoretical_loss": 3.5707317599346036, + "tokens_seen": 1262874624 + }, + { + "epoch": 4.02, + "learning_rate": 0.00031180541624874626, + "loss": 2.6679, + "theoretical_loss": 3.5707148785860006, + "tokens_seen": 1262940160 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003117953861584754, + "loss": 2.719, + "theoretical_loss": 3.57069799835864, + "tokens_seen": 1263005696 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003117853560682046, + "loss": 2.9154, + "theoretical_loss": 3.5706811192523906, + "tokens_seen": 1263071232 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003117753259779338, + "loss": 2.6642, + "theoretical_loss": 3.570664241267119, + "tokens_seen": 1263136768 + }, + { + "epoch": 4.02, + "objective/train/docs_used": 1446182, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.874471664428711, + "objective/train/theoretical_loss": 3.570647364402693, + "objective/train/tokens_used": 1283662304, + "theoretical_loss": 3.570647364402693, + "tokens_seen": 1263202304 + }, + { + "epoch": 4.02, + "learning_rate": 0.000311765295887663, + "loss": 2.6386, + "theoretical_loss": 3.570647364402693, + "tokens_seen": 1263202304 + }, + { + "epoch": 4.02, + "learning_rate": 0.00031175526579739217, + "loss": 2.6591, + "theoretical_loss": 3.5706304886589795, + "tokens_seen": 1263267840 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003117452357071214, + "loss": 2.8144, + "theoretical_loss": 3.5706136140358464, + "tokens_seen": 1263333376 + }, + { + "epoch": 4.02, + "learning_rate": 0.00031173520561685053, + "loss": 2.7972, + "theoretical_loss": 3.570596740533161, + "tokens_seen": 1263398912 + }, + { + "epoch": 4.02, + "learning_rate": 0.00031172517552657976, + "loss": 2.7657, + "theoretical_loss": 3.5705798681507908, + "tokens_seen": 1263464448 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003117151454363089, + "loss": 2.8032, + "theoretical_loss": 3.5705629968886035, + "tokens_seen": 1263529984 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003117051153460381, + "loss": 2.6886, + "theoretical_loss": 3.570546126746466, + "tokens_seen": 1263595520 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003116950852557673, + "loss": 2.5839, + "theoretical_loss": 3.570529257724247, + "tokens_seen": 1263661056 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003116850551654965, + "loss": 2.6547, + "theoretical_loss": 3.5705123898218125, + "tokens_seen": 1263726592 + }, + { + "epoch": 4.02, + "learning_rate": 0.00031167502507522567, + "loss": 2.67, + "theoretical_loss": 3.5704955230390314, + "tokens_seen": 1263792128 + }, + { + "epoch": 4.02, + "learning_rate": 0.00031166499498495485, + "loss": 2.6661, + "theoretical_loss": 3.570478657375771, + "tokens_seen": 1263857664 + }, + { + "epoch": 4.02, + "learning_rate": 0.00031165496489468403, + "loss": 2.6153, + "theoretical_loss": 3.5704617928318987, + "tokens_seen": 1263923200 + }, + { + "epoch": 4.02, + "learning_rate": 0.00031164493480441327, + "loss": 2.671, + "theoretical_loss": 3.5704449294072824, + "tokens_seen": 1263988736 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003116349047141424, + "loss": 2.8392, + "theoretical_loss": 3.5704280671017896, + "tokens_seen": 1264054272 + }, + { + "epoch": 4.02, + "learning_rate": 0.00031162487462387163, + "loss": 2.9227, + "theoretical_loss": 3.5704112059152884, + "tokens_seen": 1264119808 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003116148445336008, + "loss": 2.6377, + "theoretical_loss": 3.570394345847646, + "tokens_seen": 1264185344 + }, + { + "epoch": 4.02, + "learning_rate": 0.00031160481444333, + "loss": 2.5467, + "theoretical_loss": 3.570377486898731, + "tokens_seen": 1264250880 + }, + { + "epoch": 4.02, + "learning_rate": 0.00031159478435305917, + "loss": 2.7076, + "theoretical_loss": 3.57036062906841, + "tokens_seen": 1264316416 + }, + { + "epoch": 4.02, + "learning_rate": 0.00031158475426278835, + "loss": 2.6186, + "theoretical_loss": 3.5703437723565514, + "tokens_seen": 1264381952 + }, + { + "epoch": 4.02, + "learning_rate": 0.00031157472417251753, + "loss": 2.7294, + "theoretical_loss": 3.5703269167630234, + "tokens_seen": 1264447488 + }, + { + "epoch": 4.02, + "learning_rate": 0.00031156469408224677, + "loss": 2.7033, + "theoretical_loss": 3.5703100622876933, + "tokens_seen": 1264513024 + }, + { + "epoch": 4.02, + "learning_rate": 0.00031155466399197595, + "loss": 2.4419, + "theoretical_loss": 3.570293208930429, + "tokens_seen": 1264578560 + }, + { + "epoch": 4.02, + "learning_rate": 0.00031154463390170513, + "loss": 2.6084, + "theoretical_loss": 3.5702763566910987, + "tokens_seen": 1264644096 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003115346038114343, + "loss": 2.5783, + "theoretical_loss": 3.5702595055695703, + "tokens_seen": 1264709632 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003115245737211635, + "loss": 2.7979, + "theoretical_loss": 3.5702426555657114, + "tokens_seen": 1264775168 + }, + { + "epoch": 4.02, + "objective/train/docs_used": 1446866, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.502274751663208, + "objective/train/theoretical_loss": 3.57022580667939, + "objective/train/tokens_used": 1285300704, + "theoretical_loss": 3.57022580667939, + "tokens_seen": 1264840704 + }, + { + "epoch": 4.02, + "learning_rate": 0.00031151454363089273, + "loss": 2.7241, + "theoretical_loss": 3.57022580667939, + "tokens_seen": 1264840704 + }, + { + "epoch": 4.02, + "learning_rate": 0.00031150451354062186, + "loss": 2.6458, + "theoretical_loss": 3.5702089589104746, + "tokens_seen": 1264906240 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003114944834503511, + "loss": 2.719, + "theoretical_loss": 3.570192112258833, + "tokens_seen": 1264971776 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003114844533600802, + "loss": 2.7096, + "theoretical_loss": 3.5701752667243327, + "tokens_seen": 1265037312 + }, + { + "epoch": 4.02, + "learning_rate": 0.00031147442326980945, + "loss": 2.929, + "theoretical_loss": 3.570158422306842, + "tokens_seen": 1265102848 + }, + { + "epoch": 4.02, + "learning_rate": 0.00031146439317953863, + "loss": 3.0498, + "theoretical_loss": 3.57014157900623, + "tokens_seen": 1265168384 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003114543630892678, + "loss": 2.6962, + "theoretical_loss": 3.570124736822363, + "tokens_seen": 1265233920 + }, + { + "epoch": 4.02, + "learning_rate": 0.000311444332998997, + "loss": 2.5755, + "theoretical_loss": 3.5701078957551107, + "tokens_seen": 1265299456 + }, + { + "epoch": 4.02, + "learning_rate": 0.00031143430290872623, + "loss": 2.7734, + "theoretical_loss": 3.5700910558043404, + "tokens_seen": 1265364992 + }, + { + "epoch": 4.02, + "learning_rate": 0.00031142427281845536, + "loss": 2.714, + "theoretical_loss": 3.5700742169699202, + "tokens_seen": 1265430528 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003114142427281846, + "loss": 2.6307, + "theoretical_loss": 3.5700573792517187, + "tokens_seen": 1265496064 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003114042126379137, + "loss": 2.5619, + "theoretical_loss": 3.570040542649604, + "tokens_seen": 1265561600 + }, + { + "epoch": 4.02, + "learning_rate": 0.00031139418254764296, + "loss": 2.8593, + "theoretical_loss": 3.570023707163444, + "tokens_seen": 1265627136 + }, + { + "epoch": 4.02, + "learning_rate": 0.00031138415245737214, + "loss": 2.5305, + "theoretical_loss": 3.5700068727931074, + "tokens_seen": 1265692672 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003113741223671013, + "loss": 2.7596, + "theoretical_loss": 3.569990039538462, + "tokens_seen": 1265758208 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003113640922768305, + "loss": 2.7541, + "theoretical_loss": 3.569973207399377, + "tokens_seen": 1265823744 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003113540621865597, + "loss": 2.6803, + "theoretical_loss": 3.5699563763757194, + "tokens_seen": 1265889280 + }, + { + "epoch": 4.02, + "learning_rate": 0.00031134403209628886, + "loss": 2.9374, + "theoretical_loss": 3.5699395464673587, + "tokens_seen": 1265954816 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003113340020060181, + "loss": 2.5742, + "theoretical_loss": 3.569922717674163, + "tokens_seen": 1266020352 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003113239719157472, + "loss": 2.4029, + "theoretical_loss": 3.569905889996, + "tokens_seen": 1266085888 + }, + { + "epoch": 4.02, + "learning_rate": 0.00031131394182547646, + "loss": 2.7946, + "theoretical_loss": 3.5698890634327385, + "tokens_seen": 1266151424 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003113039117352056, + "loss": 2.6271, + "theoretical_loss": 3.5698722379842476, + "tokens_seen": 1266216960 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003112938816449348, + "loss": 2.8781, + "theoretical_loss": 3.569855413650395, + "tokens_seen": 1266282496 + }, + { + "epoch": 4.02, + "learning_rate": 0.000311283851554664, + "loss": 2.6544, + "theoretical_loss": 3.569838590431049, + "tokens_seen": 1266348032 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003112738214643932, + "loss": 2.8127, + "theoretical_loss": 3.5698217683260784, + "tokens_seen": 1266413568 + }, + { + "epoch": 4.02, + "objective/train/docs_used": 1448419, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8012731075286865, + "objective/train/theoretical_loss": 3.569804947335353, + "objective/train/tokens_used": 1286939104, + "theoretical_loss": 3.569804947335353, + "tokens_seen": 1266479104 + }, + { + "epoch": 4.02, + "learning_rate": 0.00031126379137412237, + "loss": 2.6329, + "theoretical_loss": 3.569804947335353, + "tokens_seen": 1266479104 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003112537612838516, + "loss": 2.7451, + "theoretical_loss": 3.5697881274587386, + "tokens_seen": 1266544640 + }, + { + "epoch": 4.02, + "learning_rate": 0.00031124373119358073, + "loss": 2.5991, + "theoretical_loss": 3.5697713086961063, + "tokens_seen": 1266610176 + }, + { + "epoch": 4.02, + "learning_rate": 0.00031123370110330996, + "loss": 2.6133, + "theoretical_loss": 3.5697544910473233, + "tokens_seen": 1266675712 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003112236710130391, + "loss": 2.5417, + "theoretical_loss": 3.569737674512259, + "tokens_seen": 1266741248 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003112136409227683, + "loss": 2.6459, + "theoretical_loss": 3.569720859090781, + "tokens_seen": 1266806784 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003112036108324975, + "loss": 2.6508, + "theoretical_loss": 3.569704044782759, + "tokens_seen": 1266872320 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003111935807422267, + "loss": 2.8711, + "theoretical_loss": 3.5696872315880612, + "tokens_seen": 1266937856 + }, + { + "epoch": 4.02, + "learning_rate": 0.00031118355065195587, + "loss": 2.6125, + "theoretical_loss": 3.5696704195065565, + "tokens_seen": 1267003392 + }, + { + "epoch": 4.02, + "learning_rate": 0.00031117352056168505, + "loss": 2.6055, + "theoretical_loss": 3.569653608538113, + "tokens_seen": 1267068928 + }, + { + "epoch": 4.02, + "learning_rate": 0.00031116349047141423, + "loss": 2.6969, + "theoretical_loss": 3.5696367986826005, + "tokens_seen": 1267134464 + }, + { + "epoch": 4.02, + "learning_rate": 0.00031115346038114347, + "loss": 2.6464, + "theoretical_loss": 3.569619989939887, + "tokens_seen": 1267200000 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003111434302908726, + "loss": 2.5475, + "theoretical_loss": 3.5696031823098413, + "tokens_seen": 1267265536 + }, + { + "epoch": 4.02, + "learning_rate": 0.00031113340020060183, + "loss": 2.7453, + "theoretical_loss": 3.5695863757923325, + "tokens_seen": 1267331072 + }, + { + "epoch": 4.02, + "learning_rate": 0.000311123370110331, + "loss": 2.5926, + "theoretical_loss": 3.5695695703872294, + "tokens_seen": 1267396608 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003111133400200602, + "loss": 2.687, + "theoretical_loss": 3.5695527660944006, + "tokens_seen": 1267462144 + }, + { + "epoch": 4.02, + "learning_rate": 0.00031110330992978937, + "loss": 2.6837, + "theoretical_loss": 3.569535962913715, + "tokens_seen": 1267527680 + }, + { + "epoch": 4.02, + "learning_rate": 0.00031109327983951855, + "loss": 2.942, + "theoretical_loss": 3.5695191608450423, + "tokens_seen": 1267593216 + }, + { + "epoch": 4.02, + "learning_rate": 0.00031108324974924773, + "loss": 2.4684, + "theoretical_loss": 3.5695023598882507, + "tokens_seen": 1267658752 + }, + { + "epoch": 4.02, + "learning_rate": 0.00031107321965897697, + "loss": 2.6351, + "theoretical_loss": 3.5694855600432085, + "tokens_seen": 1267724288 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003110631895687061, + "loss": 2.4623, + "theoretical_loss": 3.569468761309786, + "tokens_seen": 1267789824 + }, + { + "epoch": 4.02, + "learning_rate": 0.00031105315947843533, + "loss": 2.663, + "theoretical_loss": 3.5694519636878512, + "tokens_seen": 1267855360 + }, + { + "epoch": 4.02, + "learning_rate": 0.00031104312938816446, + "loss": 2.5563, + "theoretical_loss": 3.569435167177274, + "tokens_seen": 1267920896 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003110330992978937, + "loss": 2.6268, + "theoretical_loss": 3.569418371777923, + "tokens_seen": 1267986432 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003110230692076229, + "loss": 2.6797, + "theoretical_loss": 3.5694015774896664, + "tokens_seen": 1268051968 + }, + { + "epoch": 4.02, + "objective/train/docs_used": 1449162, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7662107944488525, + "objective/train/theoretical_loss": 3.5693847843123745, + "objective/train/tokens_used": 1288577504, + "theoretical_loss": 3.5693847843123745, + "tokens_seen": 1268117504 + }, + { + "epoch": 4.02, + "learning_rate": 0.00031101303911735206, + "loss": 2.8176, + "theoretical_loss": 3.5693847843123745, + "tokens_seen": 1268117504 + }, + { + "epoch": 4.02, + "learning_rate": 0.00031100300902708124, + "loss": 2.7783, + "theoretical_loss": 3.569367992245916, + "tokens_seen": 1268183040 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003109929789368104, + "loss": 2.613, + "theoretical_loss": 3.56935120129016, + "tokens_seen": 1268248576 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003109829488465396, + "loss": 2.6012, + "theoretical_loss": 3.5693344114449754, + "tokens_seen": 1268314112 + }, + { + "epoch": 4.02, + "learning_rate": 0.00031097291875626883, + "loss": 2.6034, + "theoretical_loss": 3.5693176227102317, + "tokens_seen": 1268379648 + }, + { + "epoch": 4.02, + "learning_rate": 0.00031096288866599796, + "loss": 2.8255, + "theoretical_loss": 3.569300835085798, + "tokens_seen": 1268445184 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003109528585757272, + "loss": 2.5776, + "theoretical_loss": 3.5692840485715434, + "tokens_seen": 1268510720 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003109428284854564, + "loss": 2.5631, + "theoretical_loss": 3.5692672631673372, + "tokens_seen": 1268576256 + }, + { + "epoch": 4.02, + "learning_rate": 0.00031093279839518556, + "loss": 2.9934, + "theoretical_loss": 3.569250478873049, + "tokens_seen": 1268641792 + }, + { + "epoch": 4.02, + "learning_rate": 0.00031092276830491474, + "loss": 2.7714, + "theoretical_loss": 3.5692336956885473, + "tokens_seen": 1268707328 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003109127382146439, + "loss": 2.7042, + "theoretical_loss": 3.569216913613702, + "tokens_seen": 1268772864 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003109027081243731, + "loss": 2.6858, + "theoretical_loss": 3.5692001326483824, + "tokens_seen": 1268838400 + }, + { + "epoch": 4.02, + "learning_rate": 0.00031089267803410234, + "loss": 2.4343, + "theoretical_loss": 3.569183352792458, + "tokens_seen": 1268903936 + }, + { + "epoch": 4.02, + "learning_rate": 0.00031088264794383147, + "loss": 2.6434, + "theoretical_loss": 3.569166574045797, + "tokens_seen": 1268969472 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003108726178535607, + "loss": 2.7497, + "theoretical_loss": 3.5691497964082703, + "tokens_seen": 1269035008 + }, + { + "epoch": 4.02, + "learning_rate": 0.00031086258776328983, + "loss": 2.7075, + "theoretical_loss": 3.5691330198797466, + "tokens_seen": 1269100544 + }, + { + "epoch": 4.02, + "learning_rate": 0.00031085255767301906, + "loss": 2.7228, + "theoretical_loss": 3.569116244460096, + "tokens_seen": 1269166080 + }, + { + "epoch": 4.02, + "learning_rate": 0.00031084252758274824, + "loss": 2.5925, + "theoretical_loss": 3.5690994701491863, + "tokens_seen": 1269231616 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003108324974924774, + "loss": 2.6519, + "theoretical_loss": 3.5690826969468885, + "tokens_seen": 1269297152 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003108224674022066, + "loss": 2.727, + "theoretical_loss": 3.5690659248530716, + "tokens_seen": 1269362688 + }, + { + "epoch": 4.02, + "learning_rate": 0.0003108124373119358, + "loss": 2.7352, + "theoretical_loss": 3.5690491538676055, + "tokens_seen": 1269428224 + }, + { + "epoch": 4.02, + "learning_rate": 0.000310802407221665, + "loss": 2.61, + "theoretical_loss": 3.5690323839903586, + "tokens_seen": 1269493760 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003107923771313942, + "loss": 2.6601, + "theoretical_loss": 3.5690156152212023, + "tokens_seen": 1269559296 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003107823470411234, + "loss": 2.7277, + "theoretical_loss": 3.568998847560004, + "tokens_seen": 1269624832 + }, + { + "epoch": 4.03, + "learning_rate": 0.00031077231695085257, + "loss": 2.6063, + "theoretical_loss": 3.5689820810066353, + "tokens_seen": 1269690368 + }, + { + "epoch": 4.03, + "objective/train/docs_used": 1449667, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5712544918060303, + "objective/train/theoretical_loss": 3.568965315560965, + "objective/train/tokens_used": 1290215904, + "theoretical_loss": 3.568965315560965, + "tokens_seen": 1269755904 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003107622868605818, + "loss": 2.55, + "theoretical_loss": 3.568965315560965, + "tokens_seen": 1269755904 + }, + { + "epoch": 4.03, + "learning_rate": 0.00031075225677031093, + "loss": 2.6712, + "theoretical_loss": 3.5689485512228623, + "tokens_seen": 1269821440 + }, + { + "epoch": 4.03, + "learning_rate": 0.00031074222668004016, + "loss": 2.435, + "theoretical_loss": 3.568931787992198, + "tokens_seen": 1269886976 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003107321965897693, + "loss": 2.6075, + "theoretical_loss": 3.56891502586884, + "tokens_seen": 1269952512 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003107221664994985, + "loss": 2.5393, + "theoretical_loss": 3.5688982648526597, + "tokens_seen": 1270018048 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003107121364092277, + "loss": 2.5827, + "theoretical_loss": 3.5688815049435263, + "tokens_seen": 1270083584 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003107021063189569, + "loss": 2.6393, + "theoretical_loss": 3.5688647461413097, + "tokens_seen": 1270149120 + }, + { + "epoch": 4.03, + "learning_rate": 0.00031069207622868607, + "loss": 2.7534, + "theoretical_loss": 3.5688479884458797, + "tokens_seen": 1270214656 + }, + { + "epoch": 4.03, + "learning_rate": 0.00031068204613841525, + "loss": 2.713, + "theoretical_loss": 3.5688312318571054, + "tokens_seen": 1270280192 + }, + { + "epoch": 4.03, + "learning_rate": 0.00031067201604814443, + "loss": 2.6002, + "theoretical_loss": 3.5688144763748575, + "tokens_seen": 1270345728 + }, + { + "epoch": 4.03, + "learning_rate": 0.00031066198595787367, + "loss": 2.7751, + "theoretical_loss": 3.5687977219990055, + "tokens_seen": 1270411264 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003106519558676028, + "loss": 2.5598, + "theoretical_loss": 3.5687809687294187, + "tokens_seen": 1270476800 + }, + { + "epoch": 4.03, + "learning_rate": 0.00031064192577733203, + "loss": 2.7079, + "theoretical_loss": 3.5687642165659685, + "tokens_seen": 1270542336 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003106318956870612, + "loss": 2.467, + "theoretical_loss": 3.5687474655085234, + "tokens_seen": 1270607872 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003106218655967904, + "loss": 2.8467, + "theoretical_loss": 3.5687307155569536, + "tokens_seen": 1270673408 + }, + { + "epoch": 4.03, + "learning_rate": 0.00031061183550651957, + "loss": 2.9576, + "theoretical_loss": 3.56871396671113, + "tokens_seen": 1270738944 + }, + { + "epoch": 4.03, + "learning_rate": 0.00031060180541624875, + "loss": 2.5267, + "theoretical_loss": 3.5686972189709216, + "tokens_seen": 1270804480 + }, + { + "epoch": 4.03, + "learning_rate": 0.00031059177532597793, + "loss": 2.7555, + "theoretical_loss": 3.5686804723361982, + "tokens_seen": 1270870016 + }, + { + "epoch": 4.03, + "learning_rate": 0.00031058174523570717, + "loss": 2.6888, + "theoretical_loss": 3.5686637268068306, + "tokens_seen": 1270935552 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003105717151454363, + "loss": 2.7137, + "theoretical_loss": 3.568646982382689, + "tokens_seen": 1271001088 + }, + { + "epoch": 4.03, + "learning_rate": 0.00031056168505516553, + "loss": 2.8593, + "theoretical_loss": 3.5686302390636424, + "tokens_seen": 1271066624 + }, + { + "epoch": 4.03, + "learning_rate": 0.00031055165496489466, + "loss": 2.7043, + "theoretical_loss": 3.568613496849562, + "tokens_seen": 1271132160 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003105416248746239, + "loss": 2.8863, + "theoretical_loss": 3.568596755740317, + "tokens_seen": 1271197696 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003105315947843531, + "loss": 2.6993, + "theoretical_loss": 3.5685800157357788, + "tokens_seen": 1271263232 + }, + { + "epoch": 4.03, + "learning_rate": 0.00031052156469408226, + "loss": 2.6819, + "theoretical_loss": 3.5685632768358158, + "tokens_seen": 1271328768 + }, + { + "epoch": 4.03, + "objective/train/docs_used": 1450856, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6388139724731445, + "objective/train/theoretical_loss": 3.5685465390402995, + "objective/train/tokens_used": 1291854304, + "theoretical_loss": 3.5685465390402995, + "tokens_seen": 1271394304 + }, + { + "epoch": 4.03, + "learning_rate": 0.00031051153460381144, + "loss": 2.5599, + "theoretical_loss": 3.5685465390402995, + "tokens_seen": 1271394304 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003105015045135406, + "loss": 2.6993, + "theoretical_loss": 3.5685298023491, + "tokens_seen": 1271459840 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003104914744232698, + "loss": 2.7917, + "theoretical_loss": 3.568513066762087, + "tokens_seen": 1271525376 + }, + { + "epoch": 4.03, + "learning_rate": 0.00031048144433299904, + "loss": 2.7555, + "theoretical_loss": 3.5684963322791305, + "tokens_seen": 1271590912 + }, + { + "epoch": 4.03, + "learning_rate": 0.00031047141424272816, + "loss": 2.6375, + "theoretical_loss": 3.568479598900102, + "tokens_seen": 1271656448 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003104613841524574, + "loss": 2.581, + "theoretical_loss": 3.5684628666248708, + "tokens_seen": 1271721984 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003104513540621866, + "loss": 2.6204, + "theoretical_loss": 3.568446135453307, + "tokens_seen": 1271787520 + }, + { + "epoch": 4.03, + "learning_rate": 0.00031044132397191576, + "loss": 2.7401, + "theoretical_loss": 3.568429405385282, + "tokens_seen": 1271853056 + }, + { + "epoch": 4.03, + "learning_rate": 0.00031043129388164494, + "loss": 2.6455, + "theoretical_loss": 3.5684126764206656, + "tokens_seen": 1271918592 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003104212637913741, + "loss": 2.6757, + "theoretical_loss": 3.5683959485593277, + "tokens_seen": 1271984128 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003104112337011033, + "loss": 2.7583, + "theoretical_loss": 3.5683792218011394, + "tokens_seen": 1272049664 + }, + { + "epoch": 4.03, + "learning_rate": 0.00031040120361083254, + "loss": 2.589, + "theoretical_loss": 3.568362496145971, + "tokens_seen": 1272115200 + }, + { + "epoch": 4.03, + "learning_rate": 0.00031039117352056167, + "loss": 2.8462, + "theoretical_loss": 3.5683457715936924, + "tokens_seen": 1272180736 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003103811434302909, + "loss": 2.777, + "theoretical_loss": 3.5683290481441747, + "tokens_seen": 1272246272 + }, + { + "epoch": 4.03, + "learning_rate": 0.00031037111334002003, + "loss": 2.5992, + "theoretical_loss": 3.5683123257972884, + "tokens_seen": 1272311808 + }, + { + "epoch": 4.03, + "learning_rate": 0.00031036108324974926, + "loss": 2.7123, + "theoretical_loss": 3.568295604552903, + "tokens_seen": 1272377344 + }, + { + "epoch": 4.03, + "learning_rate": 0.00031035105315947844, + "loss": 2.9384, + "theoretical_loss": 3.5682788844108906, + "tokens_seen": 1272442880 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003103410230692076, + "loss": 2.812, + "theoretical_loss": 3.5682621653711206, + "tokens_seen": 1272508416 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003103309929789368, + "loss": 2.7367, + "theoretical_loss": 3.5682454474334637, + "tokens_seen": 1272573952 + }, + { + "epoch": 4.03, + "learning_rate": 0.000310320962888666, + "loss": 2.761, + "theoretical_loss": 3.568228730597791, + "tokens_seen": 1272639488 + }, + { + "epoch": 4.03, + "learning_rate": 0.00031031093279839517, + "loss": 2.8438, + "theoretical_loss": 3.568212014863973, + "tokens_seen": 1272705024 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003103009027081244, + "loss": 3.026, + "theoretical_loss": 3.56819530023188, + "tokens_seen": 1272770560 + }, + { + "epoch": 4.03, + "learning_rate": 0.00031029087261785353, + "loss": 2.6869, + "theoretical_loss": 3.568178586701383, + "tokens_seen": 1272836096 + }, + { + "epoch": 4.03, + "learning_rate": 0.00031028084252758277, + "loss": 2.7515, + "theoretical_loss": 3.568161874272352, + "tokens_seen": 1272901632 + }, + { + "epoch": 4.03, + "learning_rate": 0.00031027081243731195, + "loss": 2.6262, + "theoretical_loss": 3.568145162944659, + "tokens_seen": 1272967168 + }, + { + "epoch": 4.03, + "objective/train/docs_used": 1451655, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.884730577468872, + "objective/train/theoretical_loss": 3.568128452718174, + "objective/train/tokens_used": 1293492704, + "theoretical_loss": 3.568128452718174, + "tokens_seen": 1273032704 + }, + { + "epoch": 4.03, + "learning_rate": 0.00031026078234704113, + "loss": 2.7017, + "theoretical_loss": 3.568128452718174, + "tokens_seen": 1273032704 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003102507522567703, + "loss": 2.817, + "theoretical_loss": 3.568111743592767, + "tokens_seen": 1273098240 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003102407221664995, + "loss": 2.6407, + "theoretical_loss": 3.5680950355683096, + "tokens_seen": 1273163776 + }, + { + "epoch": 4.03, + "learning_rate": 0.00031023069207622867, + "loss": 2.6872, + "theoretical_loss": 3.5680783286446727, + "tokens_seen": 1273229312 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003102206619859579, + "loss": 2.9628, + "theoretical_loss": 3.568061622821727, + "tokens_seen": 1273294848 + }, + { + "epoch": 4.03, + "learning_rate": 0.00031021063189568703, + "loss": 2.7832, + "theoretical_loss": 3.568044918099343, + "tokens_seen": 1273360384 + }, + { + "epoch": 4.03, + "learning_rate": 0.00031020060180541627, + "loss": 2.6355, + "theoretical_loss": 3.5680282144773923, + "tokens_seen": 1273425920 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003101905717151454, + "loss": 2.9272, + "theoretical_loss": 3.5680115119557447, + "tokens_seen": 1273491456 + }, + { + "epoch": 4.03, + "learning_rate": 0.00031018054162487463, + "loss": 2.7187, + "theoretical_loss": 3.5679948105342714, + "tokens_seen": 1273556992 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003101705115346038, + "loss": 2.4965, + "theoretical_loss": 3.5679781102128443, + "tokens_seen": 1273622528 + }, + { + "epoch": 4.03, + "learning_rate": 0.000310160481444333, + "loss": 2.6559, + "theoretical_loss": 3.5679614109913333, + "tokens_seen": 1273688064 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003101504513540622, + "loss": 2.5532, + "theoretical_loss": 3.5679447128696102, + "tokens_seen": 1273753600 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003101404212637914, + "loss": 2.6703, + "theoretical_loss": 3.567928015847545, + "tokens_seen": 1273819136 + }, + { + "epoch": 4.03, + "learning_rate": 0.00031013039117352054, + "loss": 2.7403, + "theoretical_loss": 3.567911319925009, + "tokens_seen": 1273884672 + }, + { + "epoch": 4.03, + "learning_rate": 0.00031012036108324977, + "loss": 2.8405, + "theoretical_loss": 3.567894625101874, + "tokens_seen": 1273950208 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003101103309929789, + "loss": 2.8008, + "theoretical_loss": 3.5678779313780105, + "tokens_seen": 1274015744 + }, + { + "epoch": 4.03, + "learning_rate": 0.00031010030090270813, + "loss": 2.6953, + "theoretical_loss": 3.567861238753289, + "tokens_seen": 1274081280 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003100902708124373, + "loss": 2.7145, + "theoretical_loss": 3.5678445472275815, + "tokens_seen": 1274146816 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003100802407221665, + "loss": 2.7383, + "theoretical_loss": 3.567827856800759, + "tokens_seen": 1274212352 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003100702106318957, + "loss": 2.6356, + "theoretical_loss": 3.5678111674726924, + "tokens_seen": 1274277888 + }, + { + "epoch": 4.03, + "learning_rate": 0.00031006018054162486, + "loss": 2.8844, + "theoretical_loss": 3.5677944792432528, + "tokens_seen": 1274343424 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003100501504513541, + "loss": 2.789, + "theoretical_loss": 3.5677777921123113, + "tokens_seen": 1274408960 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003100401203610833, + "loss": 2.8134, + "theoretical_loss": 3.5677611060797396, + "tokens_seen": 1274474496 + }, + { + "epoch": 4.03, + "learning_rate": 0.00031003009027081246, + "loss": 2.5342, + "theoretical_loss": 3.5677444211454086, + "tokens_seen": 1274540032 + }, + { + "epoch": 4.03, + "learning_rate": 0.00031002006018054164, + "loss": 2.6298, + "theoretical_loss": 3.5677277373091894, + "tokens_seen": 1274605568 + }, + { + "epoch": 4.03, + "objective/train/docs_used": 1452721, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9962716102600098, + "objective/train/theoretical_loss": 3.5677110545709536, + "objective/train/tokens_used": 1295131104, + "theoretical_loss": 3.5677110545709536, + "tokens_seen": 1274671104 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003100100300902708, + "loss": 2.7639, + "theoretical_loss": 3.5677110545709536, + "tokens_seen": 1274671104 + }, + { + "epoch": 4.03, + "learning_rate": 0.00031, + "loss": 2.8508, + "theoretical_loss": 3.5676943729305726, + "tokens_seen": 1274736640 + }, + { + "epoch": 4.03, + "learning_rate": 0.00030998996990972924, + "loss": 2.6672, + "theoretical_loss": 3.567677692387917, + "tokens_seen": 1274802176 + }, + { + "epoch": 4.03, + "learning_rate": 0.00030997993981945836, + "loss": 2.6414, + "theoretical_loss": 3.5676610129428585, + "tokens_seen": 1274867712 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003099699097291876, + "loss": 2.8492, + "theoretical_loss": 3.5676443345952693, + "tokens_seen": 1274933248 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003099598796389168, + "loss": 2.8881, + "theoretical_loss": 3.567627657345019, + "tokens_seen": 1274998784 + }, + { + "epoch": 4.03, + "learning_rate": 0.00030994984954864596, + "loss": 2.8469, + "theoretical_loss": 3.567610981191981, + "tokens_seen": 1275064320 + }, + { + "epoch": 4.03, + "learning_rate": 0.00030993981945837514, + "loss": 2.5649, + "theoretical_loss": 3.5675943061360256, + "tokens_seen": 1275129856 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003099297893681043, + "loss": 2.7739, + "theoretical_loss": 3.5675776321770236, + "tokens_seen": 1275195392 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003099197592778335, + "loss": 2.5517, + "theoretical_loss": 3.5675609593148483, + "tokens_seen": 1275260928 + }, + { + "epoch": 4.03, + "learning_rate": 0.00030990972918756274, + "loss": 2.7727, + "theoretical_loss": 3.56754428754937, + "tokens_seen": 1275326464 + }, + { + "epoch": 4.03, + "learning_rate": 0.00030989969909729187, + "loss": 2.8845, + "theoretical_loss": 3.5675276168804597, + "tokens_seen": 1275392000 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003098896690070211, + "loss": 2.5749, + "theoretical_loss": 3.5675109473079902, + "tokens_seen": 1275457536 + }, + { + "epoch": 4.03, + "learning_rate": 0.00030987963891675023, + "loss": 2.7791, + "theoretical_loss": 3.5674942788318322, + "tokens_seen": 1275523072 + }, + { + "epoch": 4.03, + "learning_rate": 0.00030986960882647946, + "loss": 2.5838, + "theoretical_loss": 3.567477611451858, + "tokens_seen": 1275588608 + }, + { + "epoch": 4.03, + "learning_rate": 0.00030985957873620864, + "loss": 2.7737, + "theoretical_loss": 3.567460945167938, + "tokens_seen": 1275654144 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003098495486459378, + "loss": 2.6846, + "theoretical_loss": 3.567444279979945, + "tokens_seen": 1275719680 + }, + { + "epoch": 4.03, + "learning_rate": 0.000309839518555667, + "loss": 2.7138, + "theoretical_loss": 3.56742761588775, + "tokens_seen": 1275785216 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003098294884653962, + "loss": 2.6993, + "theoretical_loss": 3.5674109528912252, + "tokens_seen": 1275850752 + }, + { + "epoch": 4.03, + "learning_rate": 0.00030981945837512537, + "loss": 2.8416, + "theoretical_loss": 3.567394290990242, + "tokens_seen": 1275916288 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003098094282848546, + "loss": 2.7672, + "theoretical_loss": 3.567377630184672, + "tokens_seen": 1275981824 + }, + { + "epoch": 4.03, + "learning_rate": 0.00030979939819458373, + "loss": 2.9553, + "theoretical_loss": 3.567360970474387, + "tokens_seen": 1276047360 + }, + { + "epoch": 4.03, + "learning_rate": 0.00030978936810431297, + "loss": 2.5491, + "theoretical_loss": 3.567344311859258, + "tokens_seen": 1276112896 + }, + { + "epoch": 4.03, + "learning_rate": 0.00030977933801404215, + "loss": 2.9679, + "theoretical_loss": 3.567327654339158, + "tokens_seen": 1276178432 + }, + { + "epoch": 4.03, + "learning_rate": 0.00030976930792377133, + "loss": 2.7076, + "theoretical_loss": 3.5673109979139586, + "tokens_seen": 1276243968 + }, + { + "epoch": 4.03, + "objective/train/docs_used": 1453859, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.2308290004730225, + "objective/train/theoretical_loss": 3.567294342583531, + "objective/train/tokens_used": 1296769504, + "theoretical_loss": 3.567294342583531, + "tokens_seen": 1276309504 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003097592778335005, + "loss": 2.9256, + "theoretical_loss": 3.567294342583531, + "tokens_seen": 1276309504 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003097492477432297, + "loss": 2.9109, + "theoretical_loss": 3.5672776883477475, + "tokens_seen": 1276375040 + }, + { + "epoch": 4.03, + "learning_rate": 0.00030973921765295887, + "loss": 2.7835, + "theoretical_loss": 3.5672610352064797, + "tokens_seen": 1276440576 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003097291875626881, + "loss": 2.6977, + "theoretical_loss": 3.5672443831595997, + "tokens_seen": 1276506112 + }, + { + "epoch": 4.03, + "learning_rate": 0.00030971915747241723, + "loss": 2.9232, + "theoretical_loss": 3.567227732206979, + "tokens_seen": 1276571648 + }, + { + "epoch": 4.03, + "learning_rate": 0.00030970912738214647, + "loss": 2.8935, + "theoretical_loss": 3.56721108234849, + "tokens_seen": 1276637184 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003096990972918756, + "loss": 3.0577, + "theoretical_loss": 3.5671944335840045, + "tokens_seen": 1276702720 + }, + { + "epoch": 4.03, + "learning_rate": 0.00030968906720160483, + "loss": 2.892, + "theoretical_loss": 3.567177785913394, + "tokens_seen": 1276768256 + }, + { + "epoch": 4.03, + "learning_rate": 0.000309679037111334, + "loss": 2.8298, + "theoretical_loss": 3.5671611393365312, + "tokens_seen": 1276833792 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003096690070210632, + "loss": 2.7139, + "theoretical_loss": 3.567144493853288, + "tokens_seen": 1276899328 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003096589769307924, + "loss": 2.8356, + "theoretical_loss": 3.567127849463536, + "tokens_seen": 1276964864 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003096489468405216, + "loss": 2.8401, + "theoretical_loss": 3.5671112061671475, + "tokens_seen": 1277030400 + }, + { + "epoch": 4.03, + "learning_rate": 0.00030963891675025074, + "loss": 2.8479, + "theoretical_loss": 3.5670945639639946, + "tokens_seen": 1277095936 + }, + { + "epoch": 4.03, + "learning_rate": 0.00030962888665997997, + "loss": 2.8155, + "theoretical_loss": 3.567077922853949, + "tokens_seen": 1277161472 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003096188565697091, + "loss": 2.6403, + "theoretical_loss": 3.567061282836884, + "tokens_seen": 1277227008 + }, + { + "epoch": 4.03, + "learning_rate": 0.00030960882647943833, + "loss": 2.7317, + "theoretical_loss": 3.56704464391267, + "tokens_seen": 1277292544 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003095987963891675, + "loss": 2.6416, + "theoretical_loss": 3.5670280060811805, + "tokens_seen": 1277358080 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003095887662988967, + "loss": 2.8647, + "theoretical_loss": 3.5670113693422874, + "tokens_seen": 1277423616 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003095787362086259, + "loss": 2.7733, + "theoretical_loss": 3.5669947336958625, + "tokens_seen": 1277489152 + }, + { + "epoch": 4.03, + "learning_rate": 0.00030956870611835506, + "loss": 2.9185, + "theoretical_loss": 3.566978099141778, + "tokens_seen": 1277554688 + }, + { + "epoch": 4.03, + "learning_rate": 0.00030955867602808424, + "loss": 2.8959, + "theoretical_loss": 3.566961465679907, + "tokens_seen": 1277620224 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003095486459378135, + "loss": 2.9382, + "theoretical_loss": 3.566944833310121, + "tokens_seen": 1277685760 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003095386158475426, + "loss": 2.7745, + "theoretical_loss": 3.566928202032292, + "tokens_seen": 1277751296 + }, + { + "epoch": 4.03, + "learning_rate": 0.00030952858575727184, + "loss": 2.6371, + "theoretical_loss": 3.566911571846293, + "tokens_seen": 1277816832 + }, + { + "epoch": 4.03, + "learning_rate": 0.00030951855566700096, + "loss": 2.8015, + "theoretical_loss": 3.566894942751997, + "tokens_seen": 1277882368 + }, + { + "epoch": 4.03, + "objective/train/docs_used": 1454164, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9620399475097656, + "objective/train/theoretical_loss": 3.5668783147492746, + "objective/train/tokens_used": 1298407904, + "theoretical_loss": 3.5668783147492746, + "tokens_seen": 1277947904 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003095085255767302, + "loss": 2.9085, + "theoretical_loss": 3.5668783147492746, + "tokens_seen": 1277947904 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003094984954864594, + "loss": 2.9253, + "theoretical_loss": 3.5668616878379993, + "tokens_seen": 1278013440 + }, + { + "epoch": 4.03, + "learning_rate": 0.00030948846539618856, + "loss": 2.6875, + "theoretical_loss": 3.5668450620180425, + "tokens_seen": 1278078976 + }, + { + "epoch": 4.03, + "learning_rate": 0.00030947843530591774, + "loss": 2.8784, + "theoretical_loss": 3.566828437289278, + "tokens_seen": 1278144512 + }, + { + "epoch": 4.03, + "learning_rate": 0.000309468405215647, + "loss": 2.7315, + "theoretical_loss": 3.5668118136515776, + "tokens_seen": 1278210048 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003094583751253761, + "loss": 2.7856, + "theoretical_loss": 3.5667951911048137, + "tokens_seen": 1278275584 + }, + { + "epoch": 4.03, + "learning_rate": 0.00030944834503510534, + "loss": 2.8026, + "theoretical_loss": 3.5667785696488585, + "tokens_seen": 1278341120 + }, + { + "epoch": 4.03, + "learning_rate": 0.00030943831494483447, + "loss": 2.7517, + "theoretical_loss": 3.566761949283585, + "tokens_seen": 1278406656 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003094282848545637, + "loss": 2.9017, + "theoretical_loss": 3.5667453300088656, + "tokens_seen": 1278472192 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003094182547642929, + "loss": 2.6789, + "theoretical_loss": 3.566728711824573, + "tokens_seen": 1278537728 + }, + { + "epoch": 4.03, + "learning_rate": 0.00030940822467402207, + "loss": 2.7979, + "theoretical_loss": 3.5667120947305793, + "tokens_seen": 1278603264 + }, + { + "epoch": 4.03, + "learning_rate": 0.00030939819458375125, + "loss": 2.7914, + "theoretical_loss": 3.5666954787267575, + "tokens_seen": 1278668800 + }, + { + "epoch": 4.03, + "learning_rate": 0.00030938816449348043, + "loss": 2.665, + "theoretical_loss": 3.56667886381298, + "tokens_seen": 1278734336 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003093781344032096, + "loss": 2.9661, + "theoretical_loss": 3.566662249989119, + "tokens_seen": 1278799872 + }, + { + "epoch": 4.03, + "learning_rate": 0.00030936810431293884, + "loss": 2.9095, + "theoretical_loss": 3.566645637255048, + "tokens_seen": 1278865408 + }, + { + "epoch": 4.03, + "learning_rate": 0.00030935807422266797, + "loss": 2.8724, + "theoretical_loss": 3.5666290256106397, + "tokens_seen": 1278930944 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003093480441323972, + "loss": 2.8794, + "theoretical_loss": 3.566612415055766, + "tokens_seen": 1278996480 + }, + { + "epoch": 4.03, + "learning_rate": 0.00030933801404212633, + "loss": 2.7036, + "theoretical_loss": 3.5665958055903, + "tokens_seen": 1279062016 + }, + { + "epoch": 4.03, + "learning_rate": 0.00030932798395185557, + "loss": 2.6173, + "theoretical_loss": 3.5665791972141148, + "tokens_seen": 1279127552 + }, + { + "epoch": 4.03, + "learning_rate": 0.00030931795386158475, + "loss": 2.891, + "theoretical_loss": 3.5665625899270825, + "tokens_seen": 1279193088 + }, + { + "epoch": 4.03, + "learning_rate": 0.00030930792377131393, + "loss": 3.0057, + "theoretical_loss": 3.5665459837290765, + "tokens_seen": 1279258624 + }, + { + "epoch": 4.03, + "learning_rate": 0.00030929789368104317, + "loss": 2.7406, + "theoretical_loss": 3.566529378619969, + "tokens_seen": 1279324160 + }, + { + "epoch": 4.03, + "learning_rate": 0.00030928786359077235, + "loss": 2.5874, + "theoretical_loss": 3.5665127745996332, + "tokens_seen": 1279389696 + }, + { + "epoch": 4.03, + "learning_rate": 0.00030927783350050153, + "loss": 2.8983, + "theoretical_loss": 3.566496171667942, + "tokens_seen": 1279455232 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003092678034102307, + "loss": 2.8011, + "theoretical_loss": 3.566479569824768, + "tokens_seen": 1279520768 + }, + { + "epoch": 4.03, + "objective/train/docs_used": 1454164, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.791790723800659, + "objective/train/theoretical_loss": 3.566462969069984, + "objective/train/tokens_used": 1300046304, + "theoretical_loss": 3.566462969069984, + "tokens_seen": 1279586304 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003092577733199599, + "loss": 3.0594, + "theoretical_loss": 3.566462969069984, + "tokens_seen": 1279586304 + }, + { + "epoch": 4.03, + "learning_rate": 0.00030924774322968907, + "loss": 2.856, + "theoretical_loss": 3.5664463694034634, + "tokens_seen": 1279651840 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003092377131394183, + "loss": 2.4826, + "theoretical_loss": 3.5664297708250787, + "tokens_seen": 1279717376 + }, + { + "epoch": 4.03, + "learning_rate": 0.00030922768304914743, + "loss": 2.989, + "theoretical_loss": 3.5664131733347033, + "tokens_seen": 1279782912 + }, + { + "epoch": 4.03, + "learning_rate": 0.00030921765295887667, + "loss": 2.8264, + "theoretical_loss": 3.5663965769322097, + "tokens_seen": 1279848448 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003092076228686058, + "loss": 2.9237, + "theoretical_loss": 3.5663799816174713, + "tokens_seen": 1279913984 + }, + { + "epoch": 4.03, + "learning_rate": 0.00030919759277833503, + "loss": 2.7125, + "theoretical_loss": 3.5663633873903606, + "tokens_seen": 1279979520 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003091875626880642, + "loss": 2.7873, + "theoretical_loss": 3.566346794250751, + "tokens_seen": 1280045056 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003091775325977934, + "loss": 2.9169, + "theoretical_loss": 3.5663302021985155, + "tokens_seen": 1280110592 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003091675025075226, + "loss": 3.0449, + "theoretical_loss": 3.5663136112335274, + "tokens_seen": 1280176128 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003091574724172518, + "loss": 3.1691, + "theoretical_loss": 3.566297021355659, + "tokens_seen": 1280241664 + }, + { + "epoch": 4.03, + "learning_rate": 0.00030914744232698094, + "loss": 2.9161, + "theoretical_loss": 3.5662804325647848, + "tokens_seen": 1280307200 + }, + { + "epoch": 4.03, + "learning_rate": 0.00030913741223671017, + "loss": 2.8993, + "theoretical_loss": 3.5662638448607766, + "tokens_seen": 1280372736 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003091273821464393, + "loss": 3.1612, + "theoretical_loss": 3.566247258243508, + "tokens_seen": 1280438272 + }, + { + "epoch": 4.03, + "learning_rate": 0.00030911735205616853, + "loss": 2.7969, + "theoretical_loss": 3.5662306727128525, + "tokens_seen": 1280503808 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003091073219658977, + "loss": 2.802, + "theoretical_loss": 3.5662140882686835, + "tokens_seen": 1280569344 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003090972918756269, + "loss": 2.9697, + "theoretical_loss": 3.5661975049108734, + "tokens_seen": 1280634880 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003090872617853561, + "loss": 2.8291, + "theoretical_loss": 3.5661809226392958, + "tokens_seen": 1280700416 + }, + { + "epoch": 4.03, + "learning_rate": 0.00030907723169508526, + "loss": 2.9147, + "theoretical_loss": 3.566164341453824, + "tokens_seen": 1280765952 + }, + { + "epoch": 4.03, + "learning_rate": 0.00030906720160481444, + "loss": 3.0401, + "theoretical_loss": 3.5661477613543315, + "tokens_seen": 1280831488 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003090571715145437, + "loss": 3.0241, + "theoretical_loss": 3.5661311823406914, + "tokens_seen": 1280897024 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003090471414242728, + "loss": 2.6549, + "theoretical_loss": 3.5661146044127774, + "tokens_seen": 1280962560 + }, + { + "epoch": 4.03, + "learning_rate": 0.00030903711133400204, + "loss": 2.9967, + "theoretical_loss": 3.566098027570462, + "tokens_seen": 1281028096 + }, + { + "epoch": 4.03, + "learning_rate": 0.00030902708124373116, + "loss": 2.6381, + "theoretical_loss": 3.5660814518136195, + "tokens_seen": 1281093632 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003090170511534604, + "loss": 2.6968, + "theoretical_loss": 3.566064877142123, + "tokens_seen": 1281159168 + }, + { + "epoch": 4.03, + "objective/train/docs_used": 1454920, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6659634113311768, + "objective/train/theoretical_loss": 3.5660483035558457, + "objective/train/tokens_used": 1301684704, + "theoretical_loss": 3.5660483035558457, + "tokens_seen": 1281224704 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003090070210631896, + "loss": 2.4981, + "theoretical_loss": 3.5660483035558457, + "tokens_seen": 1281224704 + }, + { + "epoch": 4.03, + "learning_rate": 0.00030899699097291876, + "loss": 2.7639, + "theoretical_loss": 3.566031731054661, + "tokens_seen": 1281290240 + }, + { + "epoch": 4.03, + "learning_rate": 0.00030898696088264794, + "loss": 2.7521, + "theoretical_loss": 3.566015159638442, + "tokens_seen": 1281355776 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003089769307923772, + "loss": 2.7621, + "theoretical_loss": 3.565998589307064, + "tokens_seen": 1281421312 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003089669007021063, + "loss": 3.0341, + "theoretical_loss": 3.5659820200603987, + "tokens_seen": 1281486848 + }, + { + "epoch": 4.03, + "learning_rate": 0.00030895687061183554, + "loss": 2.8547, + "theoretical_loss": 3.56596545189832, + "tokens_seen": 1281552384 + }, + { + "epoch": 4.03, + "learning_rate": 0.00030894684052156467, + "loss": 2.8013, + "theoretical_loss": 3.5659488848207013, + "tokens_seen": 1281617920 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003089368104312939, + "loss": 2.8409, + "theoretical_loss": 3.565932318827417, + "tokens_seen": 1281683456 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003089267803410231, + "loss": 2.8523, + "theoretical_loss": 3.5659157539183397, + "tokens_seen": 1281748992 + }, + { + "epoch": 4.03, + "learning_rate": 0.00030891675025075227, + "loss": 3.1319, + "theoretical_loss": 3.5658991900933437, + "tokens_seen": 1281814528 + }, + { + "epoch": 4.03, + "learning_rate": 0.00030890672016048145, + "loss": 2.5981, + "theoretical_loss": 3.5658826273523028, + "tokens_seen": 1281880064 + }, + { + "epoch": 4.03, + "learning_rate": 0.00030889669007021063, + "loss": 2.8412, + "theoretical_loss": 3.56586606569509, + "tokens_seen": 1281945600 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003088866599799398, + "loss": 2.7512, + "theoretical_loss": 3.565849505121579, + "tokens_seen": 1282011136 + }, + { + "epoch": 4.03, + "learning_rate": 0.00030887662988966904, + "loss": 2.7909, + "theoretical_loss": 3.5658329456316435, + "tokens_seen": 1282076672 + }, + { + "epoch": 4.03, + "learning_rate": 0.00030886659979939817, + "loss": 2.9902, + "theoretical_loss": 3.5658163872251576, + "tokens_seen": 1282142208 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003088565697091274, + "loss": 3.0112, + "theoretical_loss": 3.5657998299019953, + "tokens_seen": 1282207744 + }, + { + "epoch": 4.03, + "learning_rate": 0.00030884653961885653, + "loss": 2.8936, + "theoretical_loss": 3.5657832736620296, + "tokens_seen": 1282273280 + }, + { + "epoch": 4.03, + "learning_rate": 0.00030883650952858577, + "loss": 2.9248, + "theoretical_loss": 3.5657667185051354, + "tokens_seen": 1282338816 + }, + { + "epoch": 4.03, + "learning_rate": 0.00030882647943831495, + "loss": 2.8855, + "theoretical_loss": 3.5657501644311846, + "tokens_seen": 1282404352 + }, + { + "epoch": 4.03, + "learning_rate": 0.00030881644934804413, + "loss": 2.9276, + "theoretical_loss": 3.5657336114400526, + "tokens_seen": 1282469888 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003088064192577733, + "loss": 2.8715, + "theoretical_loss": 3.5657170595316128, + "tokens_seen": 1282535424 + }, + { + "epoch": 4.03, + "learning_rate": 0.00030879638916750255, + "loss": 2.7902, + "theoretical_loss": 3.565700508705739, + "tokens_seen": 1282600960 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003087863590772317, + "loss": 2.809, + "theoretical_loss": 3.565683958962305, + "tokens_seen": 1282666496 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003087763289869609, + "loss": 2.7277, + "theoretical_loss": 3.5656674103011854, + "tokens_seen": 1282732032 + }, + { + "epoch": 4.03, + "learning_rate": 0.00030876629889669004, + "loss": 2.9848, + "theoretical_loss": 3.5656508627222534, + "tokens_seen": 1282797568 + }, + { + "epoch": 4.03, + "objective/train/docs_used": 1455734, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.738227367401123, + "objective/train/theoretical_loss": 3.5656343162253825, + "objective/train/tokens_used": 1303323104, + "theoretical_loss": 3.5656343162253825, + "tokens_seen": 1282863104 + }, + { + "epoch": 4.03, + "learning_rate": 0.00030875626880641927, + "loss": 2.8359, + "theoretical_loss": 3.5656343162253825, + "tokens_seen": 1282863104 + }, + { + "epoch": 4.03, + "learning_rate": 0.00030874623871614845, + "loss": 2.641, + "theoretical_loss": 3.565617770810448, + "tokens_seen": 1282928640 + }, + { + "epoch": 4.03, + "learning_rate": 0.00030873620862587763, + "loss": 2.9164, + "theoretical_loss": 3.5656012264773227, + "tokens_seen": 1282994176 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003087261785356068, + "loss": 2.7244, + "theoretical_loss": 3.5655846832258815, + "tokens_seen": 1283059712 + }, + { + "epoch": 4.03, + "learning_rate": 0.000308716148445336, + "loss": 3.0738, + "theoretical_loss": 3.565568141055998, + "tokens_seen": 1283125248 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003087061183550652, + "loss": 2.7018, + "theoretical_loss": 3.565551599967546, + "tokens_seen": 1283190784 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003086960882647944, + "loss": 2.5522, + "theoretical_loss": 3.5655350599604, + "tokens_seen": 1283256320 + }, + { + "epoch": 4.03, + "learning_rate": 0.00030868605817452354, + "loss": 2.6565, + "theoretical_loss": 3.5655185210344333, + "tokens_seen": 1283321856 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003086760280842528, + "loss": 2.8782, + "theoretical_loss": 3.5655019831895216, + "tokens_seen": 1283387392 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003086659979939819, + "loss": 2.9636, + "theoretical_loss": 3.5654854464255377, + "tokens_seen": 1283452928 + }, + { + "epoch": 4.03, + "learning_rate": 0.00030865596790371114, + "loss": 2.8601, + "theoretical_loss": 3.5654689107423563, + "tokens_seen": 1283518464 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003086459378134403, + "loss": 2.5827, + "theoretical_loss": 3.5654523761398513, + "tokens_seen": 1283584000 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003086359077231695, + "loss": 3.0475, + "theoretical_loss": 3.565435842617897, + "tokens_seen": 1283649536 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003086258776328987, + "loss": 2.9249, + "theoretical_loss": 3.5654193101763685, + "tokens_seen": 1283715072 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003086158475426279, + "loss": 2.855, + "theoretical_loss": 3.5654027788151383, + "tokens_seen": 1283780608 + }, + { + "epoch": 4.03, + "learning_rate": 0.00030860581745235704, + "loss": 3.0105, + "theoretical_loss": 3.5653862485340824, + "tokens_seen": 1283846144 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003085957873620863, + "loss": 2.9229, + "theoretical_loss": 3.5653697193330736, + "tokens_seen": 1283911680 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003085857572718154, + "loss": 2.8418, + "theoretical_loss": 3.565353191211987, + "tokens_seen": 1283977216 + }, + { + "epoch": 4.03, + "learning_rate": 0.00030857572718154464, + "loss": 2.8892, + "theoretical_loss": 3.565336664170697, + "tokens_seen": 1284042752 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003085656970912738, + "loss": 2.7801, + "theoretical_loss": 3.565320138209078, + "tokens_seen": 1284108288 + }, + { + "epoch": 4.03, + "learning_rate": 0.000308555667001003, + "loss": 2.8704, + "theoretical_loss": 3.5653036133270035, + "tokens_seen": 1284173824 + }, + { + "epoch": 4.03, + "learning_rate": 0.00030854563691073224, + "loss": 2.884, + "theoretical_loss": 3.565287089524349, + "tokens_seen": 1284239360 + }, + { + "epoch": 4.03, + "learning_rate": 0.00030853560682046137, + "loss": 2.8085, + "theoretical_loss": 3.5652705668009883, + "tokens_seen": 1284304896 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003085255767301906, + "loss": 2.5732, + "theoretical_loss": 3.565254045156796, + "tokens_seen": 1284370432 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003085155466399198, + "loss": 2.8816, + "theoretical_loss": 3.5652375245916463, + "tokens_seen": 1284435968 + }, + { + "epoch": 4.03, + "objective/train/docs_used": 1456993, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8912301063537598, + "objective/train/theoretical_loss": 3.565221005105414, + "objective/train/tokens_used": 1304961504, + "theoretical_loss": 3.565221005105414, + "tokens_seen": 1284501504 + }, + { + "epoch": 4.03, + "learning_rate": 0.00030850551654964896, + "loss": 2.9473, + "theoretical_loss": 3.565221005105414, + "tokens_seen": 1284501504 + }, + { + "epoch": 4.03, + "learning_rate": 0.00030849548645937814, + "loss": 2.8586, + "theoretical_loss": 3.5652044866979735, + "tokens_seen": 1284567040 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003084854563691074, + "loss": 2.6153, + "theoretical_loss": 3.5651879693692, + "tokens_seen": 1284632576 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003084754262788365, + "loss": 2.9459, + "theoretical_loss": 3.565171453118966, + "tokens_seen": 1284698112 + }, + { + "epoch": 4.03, + "learning_rate": 0.00030846539618856574, + "loss": 2.7047, + "theoretical_loss": 3.5651549379471486, + "tokens_seen": 1284763648 + }, + { + "epoch": 4.03, + "learning_rate": 0.00030845536609829487, + "loss": 2.8943, + "theoretical_loss": 3.565138423853621, + "tokens_seen": 1284829184 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003084453360080241, + "loss": 2.6728, + "theoretical_loss": 3.5651219108382577, + "tokens_seen": 1284894720 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003084353059177533, + "loss": 2.8053, + "theoretical_loss": 3.5651053989009336, + "tokens_seen": 1284960256 + }, + { + "epoch": 4.03, + "learning_rate": 0.00030842527582748247, + "loss": 2.7473, + "theoretical_loss": 3.5650888880415237, + "tokens_seen": 1285025792 + }, + { + "epoch": 4.03, + "learning_rate": 0.00030841524573721165, + "loss": 2.9395, + "theoretical_loss": 3.5650723782599023, + "tokens_seen": 1285091328 + }, + { + "epoch": 4.03, + "learning_rate": 0.00030840521564694083, + "loss": 2.6765, + "theoretical_loss": 3.565055869555944, + "tokens_seen": 1285156864 + }, + { + "epoch": 4.03, + "learning_rate": 0.00030839518555667, + "loss": 2.8818, + "theoretical_loss": 3.5650393619295233, + "tokens_seen": 1285222400 + }, + { + "epoch": 4.03, + "learning_rate": 0.00030838515546639924, + "loss": 2.7072, + "theoretical_loss": 3.5650228553805157, + "tokens_seen": 1285287936 + }, + { + "epoch": 4.03, + "learning_rate": 0.00030837512537612837, + "loss": 2.7556, + "theoretical_loss": 3.5650063499087956, + "tokens_seen": 1285353472 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003083650952858576, + "loss": 3.0278, + "theoretical_loss": 3.564989845514237, + "tokens_seen": 1285419008 + }, + { + "epoch": 4.03, + "learning_rate": 0.00030835506519558673, + "loss": 2.9658, + "theoretical_loss": 3.564973342196716, + "tokens_seen": 1285484544 + }, + { + "epoch": 4.03, + "learning_rate": 0.00030834503510531597, + "loss": 3.0249, + "theoretical_loss": 3.5649568399561065, + "tokens_seen": 1285550080 + }, + { + "epoch": 4.03, + "learning_rate": 0.00030833500501504515, + "loss": 2.7389, + "theoretical_loss": 3.564940338792284, + "tokens_seen": 1285615616 + }, + { + "epoch": 4.03, + "learning_rate": 0.00030832497492477433, + "loss": 2.589, + "theoretical_loss": 3.5649238387051225, + "tokens_seen": 1285681152 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003083149448345035, + "loss": 2.8527, + "theoretical_loss": 3.5649073396944972, + "tokens_seen": 1285746688 + }, + { + "epoch": 4.03, + "learning_rate": 0.00030830491474423275, + "loss": 2.6517, + "theoretical_loss": 3.564890841760284, + "tokens_seen": 1285812224 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003082948846539619, + "loss": 2.85, + "theoretical_loss": 3.564874344902356, + "tokens_seen": 1285877760 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003082848545636911, + "loss": 2.8013, + "theoretical_loss": 3.56485784912059, + "tokens_seen": 1285943296 + }, + { + "epoch": 4.03, + "learning_rate": 0.00030827482447342024, + "loss": 2.8096, + "theoretical_loss": 3.564841354414859, + "tokens_seen": 1286008832 + }, + { + "epoch": 4.03, + "learning_rate": 0.00030826479438314947, + "loss": 2.8349, + "theoretical_loss": 3.5648248607850395, + "tokens_seen": 1286074368 + }, + { + "epoch": 4.03, + "objective/train/docs_used": 1457561, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.149444818496704, + "objective/train/theoretical_loss": 3.5648083682310063, + "objective/train/tokens_used": 1306599904, + "theoretical_loss": 3.5648083682310063, + "tokens_seen": 1286139904 + }, + { + "epoch": 4.03, + "learning_rate": 0.00030825476429287865, + "loss": 2.9877, + "theoretical_loss": 3.5648083682310063, + "tokens_seen": 1286139904 + }, + { + "epoch": 4.03, + "learning_rate": 0.00030824473420260783, + "loss": 2.9448, + "theoretical_loss": 3.5647918767526336, + "tokens_seen": 1286205440 + }, + { + "epoch": 4.03, + "learning_rate": 0.000308234704112337, + "loss": 2.8879, + "theoretical_loss": 3.5647753863497975, + "tokens_seen": 1286270976 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003082246740220662, + "loss": 2.8146, + "theoretical_loss": 3.5647588970223723, + "tokens_seen": 1286336512 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003082146439317954, + "loss": 2.9556, + "theoretical_loss": 3.5647424087702335, + "tokens_seen": 1286402048 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003082046138415246, + "loss": 2.9042, + "theoretical_loss": 3.564725921593256, + "tokens_seen": 1286467584 + }, + { + "epoch": 4.03, + "learning_rate": 0.00030819458375125374, + "loss": 2.7621, + "theoretical_loss": 3.5647094354913147, + "tokens_seen": 1286533120 + }, + { + "epoch": 4.03, + "learning_rate": 0.000308184553660983, + "loss": 2.7691, + "theoretical_loss": 3.564692950464285, + "tokens_seen": 1286598656 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003081745235707121, + "loss": 2.7588, + "theoretical_loss": 3.5646764665120427, + "tokens_seen": 1286664192 + }, + { + "epoch": 4.03, + "learning_rate": 0.00030816449348044134, + "loss": 2.8401, + "theoretical_loss": 3.5646599836344617, + "tokens_seen": 1286729728 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003081544633901705, + "loss": 2.8366, + "theoretical_loss": 3.564643501831418, + "tokens_seen": 1286795264 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003081444332998997, + "loss": 2.7475, + "theoretical_loss": 3.564627021102787, + "tokens_seen": 1286860800 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003081344032096289, + "loss": 2.7572, + "theoretical_loss": 3.5646105414484435, + "tokens_seen": 1286926336 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003081243731193581, + "loss": 2.779, + "theoretical_loss": 3.5645940628682626, + "tokens_seen": 1286991872 + }, + { + "epoch": 4.03, + "learning_rate": 0.00030811434302908724, + "loss": 2.9851, + "theoretical_loss": 3.5645775853621204, + "tokens_seen": 1287057408 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003081043129388165, + "loss": 2.7908, + "theoretical_loss": 3.564561108929891, + "tokens_seen": 1287122944 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003080942828485456, + "loss": 2.9733, + "theoretical_loss": 3.564544633571451, + "tokens_seen": 1287188480 + }, + { + "epoch": 4.03, + "learning_rate": 0.00030808425275827484, + "loss": 2.7524, + "theoretical_loss": 3.5645281592866755, + "tokens_seen": 1287254016 + }, + { + "epoch": 4.03, + "learning_rate": 0.000308074222668004, + "loss": 2.7138, + "theoretical_loss": 3.5645116860754387, + "tokens_seen": 1287319552 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003080641925777332, + "loss": 2.8349, + "theoretical_loss": 3.564495213937618, + "tokens_seen": 1287385088 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003080541624874624, + "loss": 2.5351, + "theoretical_loss": 3.564478742873087, + "tokens_seen": 1287450624 + }, + { + "epoch": 4.03, + "learning_rate": 0.00030804413239719157, + "loss": 2.5995, + "theoretical_loss": 3.5644622728817215, + "tokens_seen": 1287516160 + }, + { + "epoch": 4.03, + "learning_rate": 0.00030803410230692075, + "loss": 2.8158, + "theoretical_loss": 3.564445803963398, + "tokens_seen": 1287581696 + }, + { + "epoch": 4.03, + "learning_rate": 0.00030802407221665, + "loss": 2.7517, + "theoretical_loss": 3.564429336117991, + "tokens_seen": 1287647232 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003080140421263791, + "loss": 2.9679, + "theoretical_loss": 3.564412869345376, + "tokens_seen": 1287712768 + }, + { + "epoch": 4.03, + "objective/train/docs_used": 1459012, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.3446168899536133, + "objective/train/theoretical_loss": 3.5643964036454294, + "objective/train/tokens_used": 1308238304, + "theoretical_loss": 3.5643964036454294, + "tokens_seen": 1287778304 + }, + { + "epoch": 4.03, + "learning_rate": 0.00030800401203610834, + "loss": 2.78, + "theoretical_loss": 3.5643964036454294, + "tokens_seen": 1287778304 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003079939819458375, + "loss": 2.8933, + "theoretical_loss": 3.5643799390180257, + "tokens_seen": 1287843840 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003079839518555667, + "loss": 2.6891, + "theoretical_loss": 3.564363475463041, + "tokens_seen": 1287909376 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003079739217652959, + "loss": 2.7915, + "theoretical_loss": 3.5643470129803507, + "tokens_seen": 1287974912 + }, + { + "epoch": 4.03, + "learning_rate": 0.00030796389167502507, + "loss": 2.6668, + "theoretical_loss": 3.5643305515698307, + "tokens_seen": 1288040448 + }, + { + "epoch": 4.03, + "learning_rate": 0.00030795386158475425, + "loss": 2.6905, + "theoretical_loss": 3.5643140912313562, + "tokens_seen": 1288105984 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003079438314944835, + "loss": 2.9587, + "theoretical_loss": 3.5642976319648034, + "tokens_seen": 1288171520 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003079338014042126, + "loss": 2.9989, + "theoretical_loss": 3.5642811737700475, + "tokens_seen": 1288237056 + }, + { + "epoch": 4.03, + "learning_rate": 0.00030792377131394185, + "loss": 2.8329, + "theoretical_loss": 3.5642647166469636, + "tokens_seen": 1288302592 + }, + { + "epoch": 4.03, + "learning_rate": 0.000307913741223671, + "loss": 2.7683, + "theoretical_loss": 3.564248260595429, + "tokens_seen": 1288368128 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003079037111334002, + "loss": 2.8708, + "theoretical_loss": 3.5642318056153184, + "tokens_seen": 1288433664 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003078936810431294, + "loss": 2.6001, + "theoretical_loss": 3.5642153517065074, + "tokens_seen": 1288499200 + }, + { + "epoch": 4.03, + "learning_rate": 0.00030788365095285857, + "loss": 2.8064, + "theoretical_loss": 3.564198898868872, + "tokens_seen": 1288564736 + }, + { + "epoch": 4.03, + "learning_rate": 0.00030787362086258775, + "loss": 2.7796, + "theoretical_loss": 3.5641824471022883, + "tokens_seen": 1288630272 + }, + { + "epoch": 4.03, + "learning_rate": 0.00030786359077231693, + "loss": 2.6772, + "theoretical_loss": 3.564165996406632, + "tokens_seen": 1288695808 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003078535606820461, + "loss": 3.1331, + "theoretical_loss": 3.5641495467817785, + "tokens_seen": 1288761344 + }, + { + "epoch": 4.03, + "learning_rate": 0.00030784353059177535, + "loss": 2.9419, + "theoretical_loss": 3.564133098227604, + "tokens_seen": 1288826880 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003078335005015045, + "loss": 2.7909, + "theoretical_loss": 3.5641166507439843, + "tokens_seen": 1288892416 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003078234704112337, + "loss": 2.5918, + "theoretical_loss": 3.5641002043307957, + "tokens_seen": 1288957952 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003078134403209629, + "loss": 2.8002, + "theoretical_loss": 3.564083758987913, + "tokens_seen": 1289023488 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003078034102306921, + "loss": 3.0434, + "theoretical_loss": 3.5640673147152135, + "tokens_seen": 1289089024 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003077933801404213, + "loss": 2.5849, + "theoretical_loss": 3.5640508715125723, + "tokens_seen": 1289154560 + }, + { + "epoch": 4.03, + "learning_rate": 0.00030778335005015044, + "loss": 2.9531, + "theoretical_loss": 3.5640344293798654, + "tokens_seen": 1289220096 + }, + { + "epoch": 4.03, + "learning_rate": 0.00030777331995987967, + "loss": 2.8258, + "theoretical_loss": 3.5640179883169694, + "tokens_seen": 1289285632 + }, + { + "epoch": 4.03, + "learning_rate": 0.00030776328986960885, + "loss": 2.7369, + "theoretical_loss": 3.5640015483237595, + "tokens_seen": 1289351168 + }, + { + "epoch": 4.03, + "objective/train/docs_used": 1461873, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.972330093383789, + "objective/train/theoretical_loss": 3.5639851094001127, + "objective/train/tokens_used": 1309876704, + "theoretical_loss": 3.5639851094001127, + "tokens_seen": 1289416704 + }, + { + "epoch": 4.03, + "learning_rate": 0.00030775325977933803, + "loss": 2.8289, + "theoretical_loss": 3.5639851094001127, + "tokens_seen": 1289416704 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003077432296890672, + "loss": 2.6794, + "theoretical_loss": 3.5639686715459042, + "tokens_seen": 1289482240 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003077331995987964, + "loss": 2.9203, + "theoretical_loss": 3.56395223476101, + "tokens_seen": 1289547776 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003077231695085256, + "loss": 2.7811, + "theoretical_loss": 3.563935799045307, + "tokens_seen": 1289613312 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003077131394182548, + "loss": 2.8663, + "theoretical_loss": 3.5639193643986706, + "tokens_seen": 1289678848 + }, + { + "epoch": 4.03, + "learning_rate": 0.00030770310932798394, + "loss": 2.8262, + "theoretical_loss": 3.5639029308209778, + "tokens_seen": 1289744384 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003076930792377132, + "loss": 2.7288, + "theoretical_loss": 3.5638864983121037, + "tokens_seen": 1289809920 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003076830491474423, + "loss": 2.7878, + "theoretical_loss": 3.5638700668719254, + "tokens_seen": 1289875456 + }, + { + "epoch": 4.03, + "learning_rate": 0.00030767301905717154, + "loss": 3.0314, + "theoretical_loss": 3.5638536365003186, + "tokens_seen": 1289940992 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003076629889669007, + "loss": 2.5894, + "theoretical_loss": 3.5638372071971594, + "tokens_seen": 1290006528 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003076529588766299, + "loss": 2.6758, + "theoretical_loss": 3.5638207789623246, + "tokens_seen": 1290072064 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003076429287863591, + "loss": 2.7737, + "theoretical_loss": 3.5638043517956897, + "tokens_seen": 1290137600 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003076328986960883, + "loss": 2.7539, + "theoretical_loss": 3.563787925697132, + "tokens_seen": 1290203136 + }, + { + "epoch": 4.03, + "learning_rate": 0.00030762286860581744, + "loss": 2.8386, + "theoretical_loss": 3.563771500666527, + "tokens_seen": 1290268672 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003076128385155467, + "loss": 2.6868, + "theoretical_loss": 3.5637550767037514, + "tokens_seen": 1290334208 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003076028084252758, + "loss": 2.832, + "theoretical_loss": 3.563738653808681, + "tokens_seen": 1290399744 + }, + { + "epoch": 4.03, + "learning_rate": 0.00030759277833500504, + "loss": 2.8009, + "theoretical_loss": 3.5637222319811928, + "tokens_seen": 1290465280 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003075827482447342, + "loss": 2.6638, + "theoretical_loss": 3.563705811221163, + "tokens_seen": 1290530816 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003075727181544634, + "loss": 2.754, + "theoretical_loss": 3.5636893915284675, + "tokens_seen": 1290596352 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003075626880641926, + "loss": 2.8253, + "theoretical_loss": 3.563672972902984, + "tokens_seen": 1290661888 + }, + { + "epoch": 4.03, + "learning_rate": 0.00030755265797392177, + "loss": 2.7206, + "theoretical_loss": 3.563656555344588, + "tokens_seen": 1290727424 + }, + { + "epoch": 4.03, + "learning_rate": 0.00030754262788365095, + "loss": 2.8326, + "theoretical_loss": 3.563640138853156, + "tokens_seen": 1290792960 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003075325977933802, + "loss": 2.8658, + "theoretical_loss": 3.5636237234285644, + "tokens_seen": 1290858496 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003075225677031093, + "loss": 2.9978, + "theoretical_loss": 3.56360730907069, + "tokens_seen": 1290924032 + }, + { + "epoch": 4.03, + "learning_rate": 0.00030751253761283854, + "loss": 2.8136, + "theoretical_loss": 3.5635908957794094, + "tokens_seen": 1290989568 + }, + { + "epoch": 4.03, + "objective/train/docs_used": 1467150, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8942673206329346, + "objective/train/theoretical_loss": 3.563574483554599, + "objective/train/tokens_used": 1311515104, + "theoretical_loss": 3.563574483554599, + "tokens_seen": 1291055104 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003075025075225677, + "loss": 2.7327, + "theoretical_loss": 3.563574483554599, + "tokens_seen": 1291055104 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003074924774322969, + "loss": 2.8427, + "theoretical_loss": 3.5635580723961353, + "tokens_seen": 1291120640 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003074824473420261, + "loss": 2.8505, + "theoretical_loss": 3.563541662303895, + "tokens_seen": 1291186176 + }, + { + "epoch": 4.03, + "learning_rate": 0.00030747241725175527, + "loss": 2.923, + "theoretical_loss": 3.563525253277755, + "tokens_seen": 1291251712 + }, + { + "epoch": 4.03, + "learning_rate": 0.00030746238716148445, + "loss": 2.7578, + "theoretical_loss": 3.5635088453175916, + "tokens_seen": 1291317248 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003074523570712137, + "loss": 2.9013, + "theoretical_loss": 3.563492438423281, + "tokens_seen": 1291382784 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003074423269809428, + "loss": 2.6727, + "theoretical_loss": 3.563476032594701, + "tokens_seen": 1291448320 + }, + { + "epoch": 4.03, + "learning_rate": 0.00030743229689067205, + "loss": 2.8296, + "theoretical_loss": 3.563459627831727, + "tokens_seen": 1291513856 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003074222668004012, + "loss": 2.6678, + "theoretical_loss": 3.563443224134237, + "tokens_seen": 1291579392 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003074122367101304, + "loss": 2.6133, + "theoretical_loss": 3.563426821502107, + "tokens_seen": 1291644928 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003074022066198596, + "loss": 2.7212, + "theoretical_loss": 3.563410419935214, + "tokens_seen": 1291710464 + }, + { + "epoch": 4.03, + "learning_rate": 0.00030739217652958877, + "loss": 2.7894, + "theoretical_loss": 3.563394019433434, + "tokens_seen": 1291776000 + }, + { + "epoch": 4.03, + "learning_rate": 0.00030738214643931795, + "loss": 2.9254, + "theoretical_loss": 3.563377619996645, + "tokens_seen": 1291841536 + }, + { + "epoch": 4.03, + "learning_rate": 0.00030737211634904713, + "loss": 3.0763, + "theoretical_loss": 3.5633612216247235, + "tokens_seen": 1291907072 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003073620862587763, + "loss": 2.9024, + "theoretical_loss": 3.563344824317546, + "tokens_seen": 1291972608 + }, + { + "epoch": 4.03, + "learning_rate": 0.00030735205616850555, + "loss": 3.1284, + "theoretical_loss": 3.5633284280749895, + "tokens_seen": 1292038144 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003073420260782347, + "loss": 2.679, + "theoretical_loss": 3.563312032896931, + "tokens_seen": 1292103680 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003073319959879639, + "loss": 2.7707, + "theoretical_loss": 3.563295638783247, + "tokens_seen": 1292169216 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003073219658976931, + "loss": 2.778, + "theoretical_loss": 3.5632792457338147, + "tokens_seen": 1292234752 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003073119358074223, + "loss": 2.8644, + "theoretical_loss": 3.563262853748511, + "tokens_seen": 1292300288 + }, + { + "epoch": 4.03, + "learning_rate": 0.00030730190571715146, + "loss": 2.8502, + "theoretical_loss": 3.5632464628272134, + "tokens_seen": 1292365824 + }, + { + "epoch": 4.03, + "learning_rate": 0.00030729187562688064, + "loss": 2.6877, + "theoretical_loss": 3.563230072969798, + "tokens_seen": 1292431360 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003072818455366098, + "loss": 2.8643, + "theoretical_loss": 3.563213684176142, + "tokens_seen": 1292496896 + }, + { + "epoch": 4.03, + "learning_rate": 0.00030727181544633905, + "loss": 2.8738, + "theoretical_loss": 3.5631972964461234, + "tokens_seen": 1292562432 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003072617853560682, + "loss": 2.7507, + "theoretical_loss": 3.563180909779618, + "tokens_seen": 1292627968 + }, + { + "epoch": 4.03, + "objective/train/docs_used": 1472020, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7531261444091797, + "objective/train/theoretical_loss": 3.5631645241765026, + "objective/train/tokens_used": 1313153504, + "theoretical_loss": 3.5631645241765026, + "tokens_seen": 1292693504 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003072517552657974, + "loss": 2.6529, + "theoretical_loss": 3.5631645241765026, + "tokens_seen": 1292693504 + }, + { + "epoch": 4.03, + "learning_rate": 0.00030724172517552654, + "loss": 2.8054, + "theoretical_loss": 3.563148139636656, + "tokens_seen": 1292759040 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003072316950852558, + "loss": 2.904, + "theoretical_loss": 3.5631317561599545, + "tokens_seen": 1292824576 + }, + { + "epoch": 4.03, + "learning_rate": 0.00030722166499498496, + "loss": 2.7999, + "theoretical_loss": 3.5631153737462746, + "tokens_seen": 1292890112 + }, + { + "epoch": 4.03, + "learning_rate": 0.00030721163490471414, + "loss": 2.7538, + "theoretical_loss": 3.563098992395494, + "tokens_seen": 1292955648 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003072016048144433, + "loss": 2.7512, + "theoretical_loss": 3.5630826121074897, + "tokens_seen": 1293021184 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003071915747241725, + "loss": 2.4014, + "theoretical_loss": 3.5630662328821385, + "tokens_seen": 1293086720 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003071815446339017, + "loss": 2.5846, + "theoretical_loss": 3.563049854719319, + "tokens_seen": 1293152256 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003071715145436309, + "loss": 2.75, + "theoretical_loss": 3.563033477618907, + "tokens_seen": 1293217792 + }, + { + "epoch": 4.03, + "learning_rate": 0.00030716148445336005, + "loss": 2.8021, + "theoretical_loss": 3.5630171015807806, + "tokens_seen": 1293283328 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003071514543630893, + "loss": 2.7755, + "theoretical_loss": 3.563000726604816, + "tokens_seen": 1293348864 + }, + { + "epoch": 4.03, + "learning_rate": 0.00030714142427281846, + "loss": 2.8792, + "theoretical_loss": 3.5629843526908918, + "tokens_seen": 1293414400 + }, + { + "epoch": 4.03, + "learning_rate": 0.00030713139418254764, + "loss": 2.8276, + "theoretical_loss": 3.562967979838885, + "tokens_seen": 1293479936 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003071213640922768, + "loss": 2.8245, + "theoretical_loss": 3.562951608048672, + "tokens_seen": 1293545472 + }, + { + "epoch": 4.03, + "learning_rate": 0.000307111334002006, + "loss": 2.7646, + "theoretical_loss": 3.562935237320131, + "tokens_seen": 1293611008 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003071013039117352, + "loss": 2.6934, + "theoretical_loss": 3.5629188676531394, + "tokens_seen": 1293676544 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003070912738214644, + "loss": 2.8052, + "theoretical_loss": 3.5629024990475746, + "tokens_seen": 1293742080 + }, + { + "epoch": 4.03, + "learning_rate": 0.00030708124373119355, + "loss": 2.7833, + "theoretical_loss": 3.562886131503314, + "tokens_seen": 1293807616 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003070712136409228, + "loss": 2.8311, + "theoretical_loss": 3.562869765020234, + "tokens_seen": 1293873152 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003070611835506519, + "loss": 2.8201, + "theoretical_loss": 3.562853399598213, + "tokens_seen": 1293938688 + }, + { + "epoch": 4.03, + "learning_rate": 0.00030705115346038115, + "loss": 2.7745, + "theoretical_loss": 3.562837035237129, + "tokens_seen": 1294004224 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003070411233701104, + "loss": 3.03, + "theoretical_loss": 3.5628206719368585, + "tokens_seen": 1294069760 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003070310932798395, + "loss": 2.877, + "theoretical_loss": 3.5628043096972792, + "tokens_seen": 1294135296 + }, + { + "epoch": 4.03, + "learning_rate": 0.00030702106318956874, + "loss": 2.8951, + "theoretical_loss": 3.5627879485182685, + "tokens_seen": 1294200832 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003070110330992979, + "loss": 2.7721, + "theoretical_loss": 3.562771588399705, + "tokens_seen": 1294266368 + }, + { + "epoch": 4.03, + "objective/train/docs_used": 1477198, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.093344211578369, + "objective/train/theoretical_loss": 3.562755229341465, + "objective/train/tokens_used": 1314791904, + "theoretical_loss": 3.562755229341465, + "tokens_seen": 1294331904 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003070010030090271, + "loss": 2.9489, + "theoretical_loss": 3.562755229341465, + "tokens_seen": 1294331904 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003069909729187563, + "loss": 2.6401, + "theoretical_loss": 3.562738871343427, + "tokens_seen": 1294397440 + }, + { + "epoch": 4.03, + "learning_rate": 0.00030698094282848547, + "loss": 2.7694, + "theoretical_loss": 3.5627225144054684, + "tokens_seen": 1294462976 + }, + { + "epoch": 4.03, + "learning_rate": 0.00030697091273821465, + "loss": 2.7897, + "theoretical_loss": 3.562706158527466, + "tokens_seen": 1294528512 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003069608826479439, + "loss": 2.6688, + "theoretical_loss": 3.562689803709299, + "tokens_seen": 1294594048 + }, + { + "epoch": 4.03, + "learning_rate": 0.000306950852557673, + "loss": 2.8461, + "theoretical_loss": 3.562673449950844, + "tokens_seen": 1294659584 + }, + { + "epoch": 4.03, + "learning_rate": 0.00030694082246740225, + "loss": 3.0205, + "theoretical_loss": 3.5626570972519787, + "tokens_seen": 1294725120 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003069307923771314, + "loss": 2.9597, + "theoretical_loss": 3.5626407456125806, + "tokens_seen": 1294790656 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003069207622868606, + "loss": 2.7252, + "theoretical_loss": 3.562624395032528, + "tokens_seen": 1294856192 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003069107321965898, + "loss": 2.7873, + "theoretical_loss": 3.5626080455116993, + "tokens_seen": 1294921728 + }, + { + "epoch": 4.03, + "learning_rate": 0.00030690070210631897, + "loss": 2.7035, + "theoretical_loss": 3.562591697049971, + "tokens_seen": 1294987264 + }, + { + "epoch": 4.03, + "learning_rate": 0.00030689067201604815, + "loss": 2.8921, + "theoretical_loss": 3.5625753496472212, + "tokens_seen": 1295052800 + }, + { + "epoch": 4.03, + "learning_rate": 0.00030688064192577733, + "loss": 2.6662, + "theoretical_loss": 3.5625590033033285, + "tokens_seen": 1295118336 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003068706118355065, + "loss": 2.9771, + "theoretical_loss": 3.5625426580181703, + "tokens_seen": 1295183872 + }, + { + "epoch": 4.03, + "learning_rate": 0.00030686058174523575, + "loss": 2.6901, + "theoretical_loss": 3.562526313791624, + "tokens_seen": 1295249408 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003068505516549649, + "loss": 2.9169, + "theoretical_loss": 3.5625099706235677, + "tokens_seen": 1295314944 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003068405215646941, + "loss": 2.6764, + "theoretical_loss": 3.562493628513879, + "tokens_seen": 1295380480 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003068304914744233, + "loss": 2.6589, + "theoretical_loss": 3.5624772874624373, + "tokens_seen": 1295446016 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003068204613841525, + "loss": 2.3541, + "theoretical_loss": 3.5624609474691185, + "tokens_seen": 1295511552 + }, + { + "epoch": 4.03, + "learning_rate": 0.00030681043129388166, + "loss": 2.6614, + "theoretical_loss": 3.562444608533802, + "tokens_seen": 1295577088 + }, + { + "epoch": 4.03, + "learning_rate": 0.00030680040120361084, + "loss": 2.6918, + "theoretical_loss": 3.5624282706563655, + "tokens_seen": 1295642624 + }, + { + "epoch": 4.03, + "learning_rate": 0.00030679037111334, + "loss": 2.4951, + "theoretical_loss": 3.562411933836686, + "tokens_seen": 1295708160 + }, + { + "epoch": 4.03, + "learning_rate": 0.00030678034102306925, + "loss": 2.7433, + "theoretical_loss": 3.5623955980746436, + "tokens_seen": 1295773696 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003067703109327984, + "loss": 2.8738, + "theoretical_loss": 3.562379263370114, + "tokens_seen": 1295839232 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003067602808425276, + "loss": 2.6308, + "theoretical_loss": 3.562362929722977, + "tokens_seen": 1295904768 + }, + { + "epoch": 4.03, + "objective/train/docs_used": 1480267, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.763779640197754, + "objective/train/theoretical_loss": 3.56234659713311, + "objective/train/tokens_used": 1316430304, + "theoretical_loss": 3.56234659713311, + "tokens_seen": 1295970304 + }, + { + "epoch": 4.03, + "learning_rate": 0.00030675025075225674, + "loss": 2.5947, + "theoretical_loss": 3.56234659713311, + "tokens_seen": 1295970304 + }, + { + "epoch": 4.03, + "learning_rate": 0.000306740220661986, + "loss": 2.6591, + "theoretical_loss": 3.562330265600391, + "tokens_seen": 1296035840 + }, + { + "epoch": 4.03, + "learning_rate": 0.00030673019057171516, + "loss": 2.8535, + "theoretical_loss": 3.562313935124698, + "tokens_seen": 1296101376 + }, + { + "epoch": 4.03, + "learning_rate": 0.00030672016048144434, + "loss": 2.7508, + "theoretical_loss": 3.56229760570591, + "tokens_seen": 1296166912 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003067101303911735, + "loss": 2.7883, + "theoretical_loss": 3.562281277343904, + "tokens_seen": 1296232448 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003067001003009027, + "loss": 2.8509, + "theoretical_loss": 3.5622649500385593, + "tokens_seen": 1296297984 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003066900702106319, + "loss": 2.8038, + "theoretical_loss": 3.562248623789753, + "tokens_seen": 1296363520 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003066800401203611, + "loss": 2.7852, + "theoretical_loss": 3.5622322985973645, + "tokens_seen": 1296429056 + }, + { + "epoch": 4.03, + "learning_rate": 0.00030667001003009025, + "loss": 2.7057, + "theoretical_loss": 3.562215974461271, + "tokens_seen": 1296494592 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003066599799398195, + "loss": 2.6778, + "theoretical_loss": 3.562199651381351, + "tokens_seen": 1296560128 + }, + { + "epoch": 4.03, + "learning_rate": 0.00030664994984954866, + "loss": 2.7529, + "theoretical_loss": 3.5621833293574836, + "tokens_seen": 1296625664 + }, + { + "epoch": 4.03, + "learning_rate": 0.00030663991975927784, + "loss": 2.5541, + "theoretical_loss": 3.562167008389546, + "tokens_seen": 1296691200 + }, + { + "epoch": 4.03, + "learning_rate": 0.000306629889669007, + "loss": 2.7384, + "theoretical_loss": 3.5621506884774172, + "tokens_seen": 1296756736 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003066198595787362, + "loss": 2.9487, + "theoretical_loss": 3.562134369620975, + "tokens_seen": 1296822272 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003066098294884654, + "loss": 2.71, + "theoretical_loss": 3.5621180518200988, + "tokens_seen": 1296887808 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003065997993981946, + "loss": 2.8281, + "theoretical_loss": 3.562101735074666, + "tokens_seen": 1296953344 + }, + { + "epoch": 4.03, + "learning_rate": 0.00030658976930792375, + "loss": 2.6336, + "theoretical_loss": 3.5620854193845553, + "tokens_seen": 1297018880 + }, + { + "epoch": 4.03, + "learning_rate": 0.000306579739217653, + "loss": 2.8688, + "theoretical_loss": 3.562069104749645, + "tokens_seen": 1297084416 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003065697091273821, + "loss": 2.8225, + "theoretical_loss": 3.5620527911698137, + "tokens_seen": 1297149952 + }, + { + "epoch": 4.03, + "learning_rate": 0.00030655967903711135, + "loss": 2.664, + "theoretical_loss": 3.56203647864494, + "tokens_seen": 1297215488 + }, + { + "epoch": 4.03, + "learning_rate": 0.00030654964894684053, + "loss": 2.7885, + "theoretical_loss": 3.5620201671749023, + "tokens_seen": 1297281024 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003065396188565697, + "loss": 2.8634, + "theoretical_loss": 3.562003856759579, + "tokens_seen": 1297346560 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003065295887662989, + "loss": 2.7129, + "theoretical_loss": 3.5619875473988487, + "tokens_seen": 1297412096 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003065195586760281, + "loss": 2.7556, + "theoretical_loss": 3.56197123909259, + "tokens_seen": 1297477632 + }, + { + "epoch": 4.03, + "learning_rate": 0.00030650952858575725, + "loss": 2.7793, + "theoretical_loss": 3.561954931840681, + "tokens_seen": 1297543168 + }, + { + "epoch": 4.03, + "objective/train/docs_used": 1480743, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7817656993865967, + "objective/train/theoretical_loss": 3.561938625643001, + "objective/train/tokens_used": 1318068704, + "theoretical_loss": 3.561938625643001, + "tokens_seen": 1297608704 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003064994984954865, + "loss": 2.6356, + "theoretical_loss": 3.561938625643001, + "tokens_seen": 1297608704 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003064894684052156, + "loss": 2.5468, + "theoretical_loss": 3.5619223204994284, + "tokens_seen": 1297674240 + }, + { + "epoch": 4.03, + "learning_rate": 0.00030647943831494485, + "loss": 2.9011, + "theoretical_loss": 3.5619060164098415, + "tokens_seen": 1297739776 + }, + { + "epoch": 4.03, + "learning_rate": 0.00030646940822467403, + "loss": 3.0608, + "theoretical_loss": 3.5618897133741196, + "tokens_seen": 1297805312 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003064593781344032, + "loss": 2.73, + "theoretical_loss": 3.56187341139214, + "tokens_seen": 1297870848 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003064493480441324, + "loss": 2.7114, + "theoretical_loss": 3.5618571104637833, + "tokens_seen": 1297936384 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003064393179538616, + "loss": 2.8433, + "theoretical_loss": 3.5618408105889268, + "tokens_seen": 1298001920 + }, + { + "epoch": 4.03, + "learning_rate": 0.00030642928786359076, + "loss": 2.943, + "theoretical_loss": 3.5618245117674494, + "tokens_seen": 1298067456 + }, + { + "epoch": 4.03, + "learning_rate": 0.00030641925777332, + "loss": 2.9349, + "theoretical_loss": 3.5618082139992304, + "tokens_seen": 1298132992 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003064092276830491, + "loss": 2.8906, + "theoretical_loss": 3.561791917284148, + "tokens_seen": 1298198528 + }, + { + "epoch": 4.03, + "learning_rate": 0.00030639919759277835, + "loss": 2.8165, + "theoretical_loss": 3.5617756216220817, + "tokens_seen": 1298264064 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003063891675025075, + "loss": 2.6757, + "theoretical_loss": 3.5617593270129095, + "tokens_seen": 1298329600 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003063791374122367, + "loss": 2.7742, + "theoretical_loss": 3.561743033456511, + "tokens_seen": 1298395136 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003063691073219659, + "loss": 2.8762, + "theoretical_loss": 3.561726740952764, + "tokens_seen": 1298460672 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003063590772316951, + "loss": 2.4558, + "theoretical_loss": 3.5617104495015486, + "tokens_seen": 1298526208 + }, + { + "epoch": 4.03, + "learning_rate": 0.00030634904714142426, + "loss": 2.8145, + "theoretical_loss": 3.561694159102743, + "tokens_seen": 1298591744 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003063390170511535, + "loss": 2.7078, + "theoretical_loss": 3.561677869756226, + "tokens_seen": 1298657280 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003063289869608826, + "loss": 2.8624, + "theoretical_loss": 3.561661581461877, + "tokens_seen": 1298722816 + }, + { + "epoch": 4.03, + "learning_rate": 0.00030631895687061186, + "loss": 2.8161, + "theoretical_loss": 3.5616452942195744, + "tokens_seen": 1298788352 + }, + { + "epoch": 4.03, + "learning_rate": 0.000306308926780341, + "loss": 2.606, + "theoretical_loss": 3.5616290080291977, + "tokens_seen": 1298853888 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003062988966900702, + "loss": 2.7127, + "theoretical_loss": 3.561612722890626, + "tokens_seen": 1298919424 + }, + { + "epoch": 4.03, + "learning_rate": 0.00030628886659979945, + "loss": 2.884, + "theoretical_loss": 3.561596438803737, + "tokens_seen": 1298984960 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003062788365095286, + "loss": 2.7747, + "theoretical_loss": 3.5615801557684117, + "tokens_seen": 1299050496 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003062688064192578, + "loss": 2.6148, + "theoretical_loss": 3.5615638737845274, + "tokens_seen": 1299116032 + }, + { + "epoch": 4.03, + "learning_rate": 0.00030625877632898694, + "loss": 2.895, + "theoretical_loss": 3.561547592851964, + "tokens_seen": 1299181568 + }, + { + "epoch": 4.03, + "objective/train/docs_used": 1482030, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.784689426422119, + "objective/train/theoretical_loss": 3.5615313129706005, + "objective/train/tokens_used": 1319707104, + "theoretical_loss": 3.5615313129706005, + "tokens_seen": 1299247104 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003062487462387162, + "loss": 2.6131, + "theoretical_loss": 3.5615313129706005, + "tokens_seen": 1299247104 + }, + { + "epoch": 4.03, + "learning_rate": 0.00030623871614844536, + "loss": 2.7551, + "theoretical_loss": 3.5615150341403163, + "tokens_seen": 1299312640 + }, + { + "epoch": 4.03, + "learning_rate": 0.00030622868605817454, + "loss": 2.7255, + "theoretical_loss": 3.56149875636099, + "tokens_seen": 1299378176 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003062186559679037, + "loss": 2.6936, + "theoretical_loss": 3.5614824796325006, + "tokens_seen": 1299443712 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003062086258776329, + "loss": 2.6426, + "theoretical_loss": 3.5614662039547285, + "tokens_seen": 1299509248 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003061985957873621, + "loss": 2.7086, + "theoretical_loss": 3.5614499293275514, + "tokens_seen": 1299574784 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003061885656970913, + "loss": 2.8353, + "theoretical_loss": 3.5614336557508492, + "tokens_seen": 1299640320 + }, + { + "epoch": 4.03, + "learning_rate": 0.00030617853560682045, + "loss": 2.8087, + "theoretical_loss": 3.5614173832245015, + "tokens_seen": 1299705856 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003061685055165497, + "loss": 2.8922, + "theoretical_loss": 3.5614011117483866, + "tokens_seen": 1299771392 + }, + { + "epoch": 4.03, + "learning_rate": 0.00030615847542627886, + "loss": 2.7872, + "theoretical_loss": 3.5613848413223845, + "tokens_seen": 1299836928 + }, + { + "epoch": 4.03, + "learning_rate": 0.00030614844533600804, + "loss": 2.7919, + "theoretical_loss": 3.5613685719463746, + "tokens_seen": 1299902464 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003061384152457372, + "loss": 2.6497, + "theoretical_loss": 3.5613523036202355, + "tokens_seen": 1299968000 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003061283851554664, + "loss": 2.6103, + "theoretical_loss": 3.561336036343847, + "tokens_seen": 1300033536 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003061183550651956, + "loss": 2.468, + "theoretical_loss": 3.5613197701170884, + "tokens_seen": 1300099072 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003061083249749248, + "loss": 2.9235, + "theoretical_loss": 3.561303504939839, + "tokens_seen": 1300164608 + }, + { + "epoch": 4.03, + "learning_rate": 0.00030609829488465395, + "loss": 2.8431, + "theoretical_loss": 3.5612872408119784, + "tokens_seen": 1300230144 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003060882647943832, + "loss": 2.9829, + "theoretical_loss": 3.5612709777333853, + "tokens_seen": 1300295680 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003060782347041123, + "loss": 2.7384, + "theoretical_loss": 3.5612547157039405, + "tokens_seen": 1300361216 + }, + { + "epoch": 4.03, + "learning_rate": 0.00030606820461384155, + "loss": 2.9586, + "theoretical_loss": 3.561238454723522, + "tokens_seen": 1300426752 + }, + { + "epoch": 4.03, + "learning_rate": 0.00030605817452357073, + "loss": 2.8804, + "theoretical_loss": 3.56122219479201, + "tokens_seen": 1300492288 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003060481444332999, + "loss": 2.764, + "theoretical_loss": 3.5612059359092836, + "tokens_seen": 1300557824 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003060381143430291, + "loss": 2.8745, + "theoretical_loss": 3.5611896780752232, + "tokens_seen": 1300623360 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003060280842527583, + "loss": 2.8413, + "theoretical_loss": 3.5611734212897073, + "tokens_seen": 1300688896 + }, + { + "epoch": 4.03, + "learning_rate": 0.00030601805416248745, + "loss": 2.6772, + "theoretical_loss": 3.5611571655526157, + "tokens_seen": 1300754432 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003060080240722167, + "loss": 2.6801, + "theoretical_loss": 3.5611409108638283, + "tokens_seen": 1300819968 + }, + { + "epoch": 4.03, + "objective/train/docs_used": 1482698, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8064839839935303, + "objective/train/theoretical_loss": 3.5611246572232242, + "objective/train/tokens_used": 1321345504, + "theoretical_loss": 3.5611246572232242, + "tokens_seen": 1300885504 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003059979939819458, + "loss": 2.7375, + "theoretical_loss": 3.5611246572232242, + "tokens_seen": 1300885504 + }, + { + "epoch": 4.03, + "learning_rate": 0.00030598796389167505, + "loss": 2.8112, + "theoretical_loss": 3.5611084046306836, + "tokens_seen": 1300951040 + }, + { + "epoch": 4.03, + "learning_rate": 0.00030597793380140423, + "loss": 2.9242, + "theoretical_loss": 3.561092153086086, + "tokens_seen": 1301016576 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003059679037111334, + "loss": 2.624, + "theoretical_loss": 3.5610759025893106, + "tokens_seen": 1301082112 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003059578736208626, + "loss": 2.7119, + "theoretical_loss": 3.5610596531402376, + "tokens_seen": 1301147648 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003059478435305918, + "loss": 2.8299, + "theoretical_loss": 3.5610434047387463, + "tokens_seen": 1301213184 + }, + { + "epoch": 4.03, + "learning_rate": 0.00030593781344032096, + "loss": 2.7453, + "theoretical_loss": 3.5610271573847165, + "tokens_seen": 1301278720 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003059277833500502, + "loss": 2.6769, + "theoretical_loss": 3.561010911078028, + "tokens_seen": 1301344256 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003059177532597793, + "loss": 2.5735, + "theoretical_loss": 3.5609946658185607, + "tokens_seen": 1301409792 + }, + { + "epoch": 4.03, + "learning_rate": 0.00030590772316950855, + "loss": 2.814, + "theoretical_loss": 3.560978421606194, + "tokens_seen": 1301475328 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003058976930792377, + "loss": 2.7553, + "theoretical_loss": 3.560962178440808, + "tokens_seen": 1301540864 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003058876629889669, + "loss": 3.0141, + "theoretical_loss": 3.560945936322282, + "tokens_seen": 1301606400 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003058776328986961, + "loss": 2.7595, + "theoretical_loss": 3.560929695250497, + "tokens_seen": 1301671936 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003058676028084253, + "loss": 2.778, + "theoretical_loss": 3.5609134552253314, + "tokens_seen": 1301737472 + }, + { + "epoch": 4.03, + "learning_rate": 0.00030585757271815446, + "loss": 2.8669, + "theoretical_loss": 3.5608972162466657, + "tokens_seen": 1301803008 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003058475426278837, + "loss": 2.7928, + "theoretical_loss": 3.5608809783143798, + "tokens_seen": 1301868544 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003058375125376128, + "loss": 2.6076, + "theoretical_loss": 3.560864741428354, + "tokens_seen": 1301934080 + }, + { + "epoch": 4.03, + "learning_rate": 0.00030582748244734206, + "loss": 2.8176, + "theoretical_loss": 3.560848505588468, + "tokens_seen": 1301999616 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003058174523570712, + "loss": 2.7657, + "theoretical_loss": 3.560832270794601, + "tokens_seen": 1302065152 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003058074222668004, + "loss": 2.6445, + "theoretical_loss": 3.5608160370466333, + "tokens_seen": 1302130688 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003057973921765296, + "loss": 2.8125, + "theoretical_loss": 3.560799804344446, + "tokens_seen": 1302196224 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003057873620862588, + "loss": 2.5955, + "theoretical_loss": 3.5607835726879173, + "tokens_seen": 1302261760 + }, + { + "epoch": 4.03, + "learning_rate": 0.00030577733199598796, + "loss": 2.7631, + "theoretical_loss": 3.560767342076929, + "tokens_seen": 1302327296 + }, + { + "epoch": 4.03, + "learning_rate": 0.00030576730190571714, + "loss": 2.9111, + "theoretical_loss": 3.56075111251136, + "tokens_seen": 1302392832 + }, + { + "epoch": 4.03, + "learning_rate": 0.0003057572718154463, + "loss": 2.7766, + "theoretical_loss": 3.5607348839910906, + "tokens_seen": 1302458368 + }, + { + "epoch": 4.03, + "objective/train/docs_used": 1483979, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.209777355194092, + "objective/train/theoretical_loss": 3.560718656516001, + "objective/train/tokens_used": 1322983904, + "theoretical_loss": 3.560718656516001, + "tokens_seen": 1302523904 + }, + { + "epoch": 4.03, + "learning_rate": 0.00030574724172517556, + "loss": 2.6928, + "theoretical_loss": 3.560718656516001, + "tokens_seen": 1302523904 + }, + { + "epoch": 4.04, + "learning_rate": 0.0003057372116349047, + "loss": 2.7563, + "theoretical_loss": 3.5607024300859713, + "tokens_seen": 1302589440 + }, + { + "epoch": 4.04, + "learning_rate": 0.0003057271815446339, + "loss": 2.7126, + "theoretical_loss": 3.5606862047008816, + "tokens_seen": 1302654976 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030571715145436305, + "loss": 2.735, + "theoretical_loss": 3.5606699803606117, + "tokens_seen": 1302720512 + }, + { + "epoch": 4.04, + "learning_rate": 0.0003057071213640923, + "loss": 2.7698, + "theoretical_loss": 3.560653757065042, + "tokens_seen": 1302786048 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030569709127382147, + "loss": 2.9375, + "theoretical_loss": 3.5606375348140533, + "tokens_seen": 1302851584 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030568706118355065, + "loss": 2.8591, + "theoretical_loss": 3.560621313607525, + "tokens_seen": 1302917120 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030567703109327983, + "loss": 2.5649, + "theoretical_loss": 3.560605093445338, + "tokens_seen": 1302982656 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030566700100300906, + "loss": 2.9389, + "theoretical_loss": 3.5605888743273715, + "tokens_seen": 1303048192 + }, + { + "epoch": 4.04, + "learning_rate": 0.0003056569709127382, + "loss": 2.5524, + "theoretical_loss": 3.5605726562535067, + "tokens_seen": 1303113728 + }, + { + "epoch": 4.04, + "learning_rate": 0.0003056469408224674, + "loss": 2.9435, + "theoretical_loss": 3.5605564392236237, + "tokens_seen": 1303179264 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030563691073219655, + "loss": 2.9062, + "theoretical_loss": 3.560540223237602, + "tokens_seen": 1303244800 + }, + { + "epoch": 4.04, + "learning_rate": 0.0003056268806419258, + "loss": 2.752, + "theoretical_loss": 3.5605240082953236, + "tokens_seen": 1303310336 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030561685055165497, + "loss": 2.8643, + "theoretical_loss": 3.5605077943966674, + "tokens_seen": 1303375872 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030560682046138415, + "loss": 2.9482, + "theoretical_loss": 3.5604915815415143, + "tokens_seen": 1303441408 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030559679037111333, + "loss": 2.8014, + "theoretical_loss": 3.560475369729744, + "tokens_seen": 1303506944 + }, + { + "epoch": 4.04, + "learning_rate": 0.0003055867602808425, + "loss": 2.4184, + "theoretical_loss": 3.560459158961238, + "tokens_seen": 1303572480 + }, + { + "epoch": 4.04, + "learning_rate": 0.0003055767301905717, + "loss": 2.7457, + "theoretical_loss": 3.5604429492358767, + "tokens_seen": 1303638016 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030556670010030093, + "loss": 2.69, + "theoretical_loss": 3.560426740553539, + "tokens_seen": 1303703552 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030555667001003006, + "loss": 2.5995, + "theoretical_loss": 3.5604105329141067, + "tokens_seen": 1303769088 + }, + { + "epoch": 4.04, + "learning_rate": 0.0003055466399197593, + "loss": 2.9198, + "theoretical_loss": 3.5603943263174607, + "tokens_seen": 1303834624 + }, + { + "epoch": 4.04, + "learning_rate": 0.0003055366098294885, + "loss": 2.5802, + "theoretical_loss": 3.56037812076348, + "tokens_seen": 1303900160 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030552657973921765, + "loss": 2.797, + "theoretical_loss": 3.560361916252046, + "tokens_seen": 1303965696 + }, + { + "epoch": 4.04, + "learning_rate": 0.0003055165496489469, + "loss": 2.8847, + "theoretical_loss": 3.5603457127830396, + "tokens_seen": 1304031232 + }, + { + "epoch": 4.04, + "learning_rate": 0.000305506519558676, + "loss": 2.694, + "theoretical_loss": 3.5603295103563406, + "tokens_seen": 1304096768 + }, + { + "epoch": 4.04, + "objective/train/docs_used": 1484701, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.840122699737549, + "objective/train/theoretical_loss": 3.5603133089718293, + "objective/train/tokens_used": 1324622304, + "theoretical_loss": 3.5603133089718293, + "tokens_seen": 1304162304 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030549648946840525, + "loss": 2.7326, + "theoretical_loss": 3.5603133089718293, + "tokens_seen": 1304162304 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030548645937813443, + "loss": 2.8274, + "theoretical_loss": 3.5602971086293875, + "tokens_seen": 1304227840 + }, + { + "epoch": 4.04, + "learning_rate": 0.0003054764292878636, + "loss": 2.5019, + "theoretical_loss": 3.5602809093288954, + "tokens_seen": 1304293376 + }, + { + "epoch": 4.04, + "learning_rate": 0.0003054663991975928, + "loss": 2.6509, + "theoretical_loss": 3.560264711070233, + "tokens_seen": 1304358912 + }, + { + "epoch": 4.04, + "learning_rate": 0.000305456369107322, + "loss": 2.5558, + "theoretical_loss": 3.5602485138532813, + "tokens_seen": 1304424448 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030544633901705116, + "loss": 2.5316, + "theoretical_loss": 3.560232317677921, + "tokens_seen": 1304489984 + }, + { + "epoch": 4.04, + "learning_rate": 0.0003054363089267804, + "loss": 2.8007, + "theoretical_loss": 3.560216122544033, + "tokens_seen": 1304555520 + }, + { + "epoch": 4.04, + "learning_rate": 0.0003054262788365095, + "loss": 2.9023, + "theoretical_loss": 3.560199928451498, + "tokens_seen": 1304621056 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030541624874623875, + "loss": 2.6966, + "theoretical_loss": 3.560183735400196, + "tokens_seen": 1304686592 + }, + { + "epoch": 4.04, + "learning_rate": 0.0003054062186559679, + "loss": 2.8067, + "theoretical_loss": 3.5601675433900093, + "tokens_seen": 1304752128 + }, + { + "epoch": 4.04, + "learning_rate": 0.0003053961885656971, + "loss": 2.7268, + "theoretical_loss": 3.5601513524208173, + "tokens_seen": 1304817664 + }, + { + "epoch": 4.04, + "learning_rate": 0.0003053861584754263, + "loss": 2.7663, + "theoretical_loss": 3.560135162492501, + "tokens_seen": 1304883200 + }, + { + "epoch": 4.04, + "learning_rate": 0.0003053761283851555, + "loss": 2.6635, + "theoretical_loss": 3.5601189736049417, + "tokens_seen": 1304948736 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030536609829488466, + "loss": 2.8489, + "theoretical_loss": 3.5601027857580196, + "tokens_seen": 1305014272 + }, + { + "epoch": 4.04, + "learning_rate": 0.0003053560682046139, + "loss": 2.8466, + "theoretical_loss": 3.5600865989516164, + "tokens_seen": 1305079808 + }, + { + "epoch": 4.04, + "learning_rate": 0.000305346038114343, + "loss": 2.8473, + "theoretical_loss": 3.5600704131856125, + "tokens_seen": 1305145344 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030533600802407226, + "loss": 2.6758, + "theoretical_loss": 3.5600542284598884, + "tokens_seen": 1305210880 + }, + { + "epoch": 4.04, + "learning_rate": 0.0003053259779338014, + "loss": 2.598, + "theoretical_loss": 3.560038044774325, + "tokens_seen": 1305276416 + }, + { + "epoch": 4.04, + "learning_rate": 0.0003053159478435306, + "loss": 2.6272, + "theoretical_loss": 3.5600218621288047, + "tokens_seen": 1305341952 + }, + { + "epoch": 4.04, + "learning_rate": 0.0003053059177532598, + "loss": 2.7393, + "theoretical_loss": 3.5600056805232065, + "tokens_seen": 1305407488 + }, + { + "epoch": 4.04, + "learning_rate": 0.000305295887662989, + "loss": 2.8424, + "theoretical_loss": 3.5599894999574127, + "tokens_seen": 1305473024 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030528585757271816, + "loss": 2.7768, + "theoretical_loss": 3.559973320431304, + "tokens_seen": 1305538560 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030527582748244734, + "loss": 2.6083, + "theoretical_loss": 3.5599571419447607, + "tokens_seen": 1305604096 + }, + { + "epoch": 4.04, + "learning_rate": 0.0003052657973921765, + "loss": 2.8214, + "theoretical_loss": 3.559940964497664, + "tokens_seen": 1305669632 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030525576730190576, + "loss": 2.8903, + "theoretical_loss": 3.559924788089896, + "tokens_seen": 1305735168 + }, + { + "epoch": 4.04, + "objective/train/docs_used": 1485880, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7324297428131104, + "objective/train/theoretical_loss": 3.5599086127213373, + "objective/train/tokens_used": 1326260704, + "theoretical_loss": 3.5599086127213373, + "tokens_seen": 1305800704 + }, + { + "epoch": 4.04, + "learning_rate": 0.0003052457372116349, + "loss": 2.7493, + "theoretical_loss": 3.5599086127213373, + "tokens_seen": 1305800704 + }, + { + "epoch": 4.04, + "learning_rate": 0.0003052357071213641, + "loss": 2.8219, + "theoretical_loss": 3.559892438391868, + "tokens_seen": 1305866240 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030522567703109325, + "loss": 3.01, + "theoretical_loss": 3.559876265101371, + "tokens_seen": 1305931776 + }, + { + "epoch": 4.04, + "learning_rate": 0.0003052156469408225, + "loss": 2.8747, + "theoretical_loss": 3.5598600928497257, + "tokens_seen": 1305997312 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030520561685055167, + "loss": 2.5935, + "theoretical_loss": 3.559843921636814, + "tokens_seen": 1306062848 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030519558676028085, + "loss": 2.9397, + "theoretical_loss": 3.5598277514625174, + "tokens_seen": 1306128384 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030518555667001003, + "loss": 2.9605, + "theoretical_loss": 3.5598115823267165, + "tokens_seen": 1306193920 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030517552657973926, + "loss": 2.6671, + "theoretical_loss": 3.5597954142292925, + "tokens_seen": 1306259456 + }, + { + "epoch": 4.04, + "learning_rate": 0.0003051654964894684, + "loss": 2.8636, + "theoretical_loss": 3.559779247170127, + "tokens_seen": 1306324992 + }, + { + "epoch": 4.04, + "learning_rate": 0.0003051554663991976, + "loss": 2.6677, + "theoretical_loss": 3.559763081149101, + "tokens_seen": 1306390528 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030514543630892675, + "loss": 2.8608, + "theoretical_loss": 3.5597469161660955, + "tokens_seen": 1306456064 + }, + { + "epoch": 4.04, + "learning_rate": 0.000305135406218656, + "loss": 2.7773, + "theoretical_loss": 3.559730752220993, + "tokens_seen": 1306521600 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030512537612838517, + "loss": 2.5519, + "theoretical_loss": 3.559714589313673, + "tokens_seen": 1306587136 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030511534603811435, + "loss": 2.752, + "theoretical_loss": 3.5596984274440184, + "tokens_seen": 1306652672 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030510531594784353, + "loss": 2.8003, + "theoretical_loss": 3.55968226661191, + "tokens_seen": 1306718208 + }, + { + "epoch": 4.04, + "learning_rate": 0.0003050952858575727, + "loss": 2.7637, + "theoretical_loss": 3.5596661068172284, + "tokens_seen": 1306783744 + }, + { + "epoch": 4.04, + "learning_rate": 0.0003050852557673019, + "loss": 2.5725, + "theoretical_loss": 3.559649948059856, + "tokens_seen": 1306849280 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030507522567703113, + "loss": 2.9491, + "theoretical_loss": 3.5596337903396735, + "tokens_seen": 1306914816 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030506519558676026, + "loss": 3.0898, + "theoretical_loss": 3.559617633656563, + "tokens_seen": 1306980352 + }, + { + "epoch": 4.04, + "learning_rate": 0.0003050551654964895, + "loss": 2.7713, + "theoretical_loss": 3.559601478010406, + "tokens_seen": 1307045888 + }, + { + "epoch": 4.04, + "learning_rate": 0.0003050451354062186, + "loss": 2.734, + "theoretical_loss": 3.559585323401083, + "tokens_seen": 1307111424 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030503510531594785, + "loss": 2.5877, + "theoretical_loss": 3.559569169828476, + "tokens_seen": 1307176960 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030502507522567703, + "loss": 2.5828, + "theoretical_loss": 3.5595530172924663, + "tokens_seen": 1307242496 + }, + { + "epoch": 4.04, + "learning_rate": 0.0003050150451354062, + "loss": 2.6721, + "theoretical_loss": 3.5595368657929365, + "tokens_seen": 1307308032 + }, + { + "epoch": 4.04, + "learning_rate": 0.0003050050150451354, + "loss": 2.7474, + "theoretical_loss": 3.559520715329767, + "tokens_seen": 1307373568 + }, + { + "epoch": 4.04, + "objective/train/docs_used": 1486544, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7242846488952637, + "objective/train/theoretical_loss": 3.559504565902839, + "objective/train/tokens_used": 1327899104, + "theoretical_loss": 3.559504565902839, + "tokens_seen": 1307439104 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030499498495486463, + "loss": 2.6497, + "theoretical_loss": 3.559504565902839, + "tokens_seen": 1307439104 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030498495486459376, + "loss": 2.7098, + "theoretical_loss": 3.559488417512035, + "tokens_seen": 1307504640 + }, + { + "epoch": 4.04, + "learning_rate": 0.000304974924774323, + "loss": 2.7243, + "theoretical_loss": 3.5594722701572366, + "tokens_seen": 1307570176 + }, + { + "epoch": 4.04, + "learning_rate": 0.0003049648946840521, + "loss": 2.7246, + "theoretical_loss": 3.559456123838325, + "tokens_seen": 1307635712 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030495486459378136, + "loss": 2.7058, + "theoretical_loss": 3.559439978555182, + "tokens_seen": 1307701248 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030494483450351054, + "loss": 2.8295, + "theoretical_loss": 3.5594238343076894, + "tokens_seen": 1307766784 + }, + { + "epoch": 4.04, + "learning_rate": 0.0003049348044132397, + "loss": 2.8496, + "theoretical_loss": 3.5594076910957284, + "tokens_seen": 1307832320 + }, + { + "epoch": 4.04, + "learning_rate": 0.0003049247743229689, + "loss": 2.6376, + "theoretical_loss": 3.559391548919181, + "tokens_seen": 1307897856 + }, + { + "epoch": 4.04, + "learning_rate": 0.0003049147442326981, + "loss": 2.869, + "theoretical_loss": 3.559375407777929, + "tokens_seen": 1307963392 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030490471414242726, + "loss": 2.6927, + "theoretical_loss": 3.5593592676718533, + "tokens_seen": 1308028928 + }, + { + "epoch": 4.04, + "learning_rate": 0.0003048946840521565, + "loss": 2.7517, + "theoretical_loss": 3.5593431286008372, + "tokens_seen": 1308094464 + }, + { + "epoch": 4.04, + "learning_rate": 0.0003048846539618856, + "loss": 2.6785, + "theoretical_loss": 3.5593269905647613, + "tokens_seen": 1308160000 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030487462387161486, + "loss": 2.5399, + "theoretical_loss": 3.559310853563508, + "tokens_seen": 1308225536 + }, + { + "epoch": 4.04, + "learning_rate": 0.000304864593781344, + "loss": 2.8535, + "theoretical_loss": 3.5592947175969587, + "tokens_seen": 1308291072 + }, + { + "epoch": 4.04, + "learning_rate": 0.0003048545636910732, + "loss": 2.7204, + "theoretical_loss": 3.5592785826649953, + "tokens_seen": 1308356608 + }, + { + "epoch": 4.04, + "learning_rate": 0.0003048445336008024, + "loss": 2.5783, + "theoretical_loss": 3.5592624487675, + "tokens_seen": 1308422144 + }, + { + "epoch": 4.04, + "learning_rate": 0.0003048345035105316, + "loss": 2.5697, + "theoretical_loss": 3.559246315904354, + "tokens_seen": 1308487680 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030482447342026076, + "loss": 2.7008, + "theoretical_loss": 3.55923018407544, + "tokens_seen": 1308553216 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030481444332999, + "loss": 2.6793, + "theoretical_loss": 3.5592140532806393, + "tokens_seen": 1308618752 + }, + { + "epoch": 4.04, + "learning_rate": 0.0003048044132397192, + "loss": 2.6712, + "theoretical_loss": 3.559197923519834, + "tokens_seen": 1308684288 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030479438314944836, + "loss": 2.9295, + "theoretical_loss": 3.559181794792906, + "tokens_seen": 1308749824 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030478435305917754, + "loss": 2.8828, + "theoretical_loss": 3.5591656670997374, + "tokens_seen": 1308815360 + }, + { + "epoch": 4.04, + "learning_rate": 0.0003047743229689067, + "loss": 2.6114, + "theoretical_loss": 3.5591495404402096, + "tokens_seen": 1308880896 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030476429287863596, + "loss": 2.6058, + "theoretical_loss": 3.5591334148142058, + "tokens_seen": 1308946432 + }, + { + "epoch": 4.04, + "learning_rate": 0.0003047542627883651, + "loss": 2.7359, + "theoretical_loss": 3.559117290221607, + "tokens_seen": 1309011968 + }, + { + "epoch": 4.04, + "objective/train/docs_used": 1488092, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8345699310302734, + "objective/train/theoretical_loss": 3.5591011666622956, + "objective/train/tokens_used": 1329537504, + "theoretical_loss": 3.5591011666622956, + "tokens_seen": 1309077504 + }, + { + "epoch": 4.04, + "learning_rate": 0.0003047442326980943, + "loss": 2.831, + "theoretical_loss": 3.5591011666622956, + "tokens_seen": 1309077504 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030473420260782345, + "loss": 2.7527, + "theoretical_loss": 3.5590850441361535, + "tokens_seen": 1309143040 + }, + { + "epoch": 4.04, + "learning_rate": 0.0003047241725175527, + "loss": 2.9376, + "theoretical_loss": 3.559068922643063, + "tokens_seen": 1309208576 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030471414242728187, + "loss": 2.8627, + "theoretical_loss": 3.559052802182906, + "tokens_seen": 1309274112 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030470411233701105, + "loss": 2.8548, + "theoretical_loss": 3.559036682755565, + "tokens_seen": 1309339648 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030469408224674023, + "loss": 2.5759, + "theoretical_loss": 3.559020564360922, + "tokens_seen": 1309405184 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030468405215646946, + "loss": 2.7288, + "theoretical_loss": 3.559004446998858, + "tokens_seen": 1309470720 + }, + { + "epoch": 4.04, + "learning_rate": 0.0003046740220661986, + "loss": 2.814, + "theoretical_loss": 3.5589883306692576, + "tokens_seen": 1309536256 + }, + { + "epoch": 4.04, + "learning_rate": 0.0003046639919759278, + "loss": 2.7851, + "theoretical_loss": 3.558972215372001, + "tokens_seen": 1309601792 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030465396188565695, + "loss": 2.7324, + "theoretical_loss": 3.5589561011069706, + "tokens_seen": 1309667328 + }, + { + "epoch": 4.04, + "learning_rate": 0.0003046439317953862, + "loss": 2.5386, + "theoretical_loss": 3.5589399878740493, + "tokens_seen": 1309732864 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030463390170511537, + "loss": 2.7966, + "theoretical_loss": 3.558923875673119, + "tokens_seen": 1309798400 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030462387161484455, + "loss": 2.8575, + "theoretical_loss": 3.5589077645040623, + "tokens_seen": 1309863936 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030461384152457373, + "loss": 2.7993, + "theoretical_loss": 3.5588916543667617, + "tokens_seen": 1309929472 + }, + { + "epoch": 4.04, + "learning_rate": 0.0003046038114343029, + "loss": 2.8267, + "theoretical_loss": 3.558875545261098, + "tokens_seen": 1309995008 + }, + { + "epoch": 4.04, + "learning_rate": 0.0003045937813440321, + "loss": 2.6302, + "theoretical_loss": 3.558859437186956, + "tokens_seen": 1310060544 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030458375125376133, + "loss": 2.8977, + "theoretical_loss": 3.5588433301442155, + "tokens_seen": 1310126080 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030457372116349046, + "loss": 2.5146, + "theoretical_loss": 3.5588272241327603, + "tokens_seen": 1310191616 + }, + { + "epoch": 4.04, + "learning_rate": 0.0003045636910732197, + "loss": 2.5984, + "theoretical_loss": 3.5588111191524727, + "tokens_seen": 1310257152 + }, + { + "epoch": 4.04, + "learning_rate": 0.0003045536609829488, + "loss": 2.7796, + "theoretical_loss": 3.558795015203235, + "tokens_seen": 1310322688 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030454363089267805, + "loss": 2.5794, + "theoretical_loss": 3.5587789122849296, + "tokens_seen": 1310388224 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030453360080240723, + "loss": 2.601, + "theoretical_loss": 3.5587628103974387, + "tokens_seen": 1310453760 + }, + { + "epoch": 4.04, + "learning_rate": 0.0003045235707121364, + "loss": 2.7363, + "theoretical_loss": 3.558746709540645, + "tokens_seen": 1310519296 + }, + { + "epoch": 4.04, + "learning_rate": 0.0003045135406218656, + "loss": 2.8367, + "theoretical_loss": 3.5587306097144307, + "tokens_seen": 1310584832 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030450351053159483, + "loss": 2.9193, + "theoretical_loss": 3.558714510918679, + "tokens_seen": 1310650368 + }, + { + "debugging/Self-BLEU-5": 0.5335497100843765, + "debugging/distinct-1-grams": 0.7520819374414864, + "debugging/distinct-2-grams": 0.9491809766666045, + "debugging/entropy-1-grams": 5.943300109412918, + "debugging/entropy-2-grams": 7.027315581762769, + "debugging/length": 477.3888888888889, + "debugging/num_segments": 18, + "debugging/score": 0.0018735238636605285, + "debugging/score_std": 0.0038943973989434763, + "epoch": 4.04, + "objective/train/docs_used": 1488756, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.642608642578125, + "objective/train/theoretical_loss": 3.558698413153272, + "objective/train/tokens_used": 1331175904, + "theoretical_loss": 3.558698413153272, + "tokens_seen": 1310715904 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030449348044132396, + "loss": 2.7676, + "theoretical_loss": 3.558698413153272, + "tokens_seen": 1310715904 + }, + { + "epoch": 4.04, + "learning_rate": 0.0003044834503510532, + "loss": 2.6587, + "theoretical_loss": 3.558682316418092, + "tokens_seen": 1310781440 + }, + { + "epoch": 4.04, + "learning_rate": 0.0003044734202607823, + "loss": 2.7785, + "theoretical_loss": 3.5586662207130217, + "tokens_seen": 1310846976 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030446339017051156, + "loss": 2.6989, + "theoretical_loss": 3.558650126037944, + "tokens_seen": 1310912512 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030445336008024074, + "loss": 2.6195, + "theoretical_loss": 3.5586340323927415, + "tokens_seen": 1310978048 + }, + { + "epoch": 4.04, + "learning_rate": 0.0003044433299899699, + "loss": 2.8218, + "theoretical_loss": 3.558617939777297, + "tokens_seen": 1311043584 + }, + { + "epoch": 4.04, + "learning_rate": 0.0003044332998996991, + "loss": 2.6592, + "theoretical_loss": 3.5586018481914916, + "tokens_seen": 1311109120 + }, + { + "epoch": 4.04, + "learning_rate": 0.0003044232698094283, + "loss": 2.3855, + "theoretical_loss": 3.5585857576352096, + "tokens_seen": 1311174656 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030441323971915746, + "loss": 2.6516, + "theoretical_loss": 3.5585696681083334, + "tokens_seen": 1311240192 + }, + { + "epoch": 4.04, + "learning_rate": 0.0003044032096288867, + "loss": 2.6568, + "theoretical_loss": 3.558553579610745, + "tokens_seen": 1311305728 + }, + { + "epoch": 4.04, + "learning_rate": 0.0003043931795386158, + "loss": 2.5956, + "theoretical_loss": 3.558537492142328, + "tokens_seen": 1311371264 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030438314944834506, + "loss": 2.7413, + "theoretical_loss": 3.558521405702965, + "tokens_seen": 1311436800 + }, + { + "epoch": 4.04, + "learning_rate": 0.0003043731193580742, + "loss": 2.4865, + "theoretical_loss": 3.558505320292538, + "tokens_seen": 1311502336 + }, + { + "epoch": 4.04, + "learning_rate": 0.0003043630892678034, + "loss": 2.5645, + "theoretical_loss": 3.5584892359109306, + "tokens_seen": 1311567872 + }, + { + "epoch": 4.04, + "learning_rate": 0.0003043530591775326, + "loss": 2.7924, + "theoretical_loss": 3.558473152558025, + "tokens_seen": 1311633408 + }, + { + "epoch": 4.04, + "learning_rate": 0.0003043430290872618, + "loss": 2.6965, + "theoretical_loss": 3.558457070233704, + "tokens_seen": 1311698944 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030433299899699097, + "loss": 2.5144, + "theoretical_loss": 3.558440988937851, + "tokens_seen": 1311764480 + }, + { + "epoch": 4.04, + "learning_rate": 0.0003043229689067202, + "loss": 2.6407, + "theoretical_loss": 3.558424908670349, + "tokens_seen": 1311830016 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030431293881644933, + "loss": 2.6782, + "theoretical_loss": 3.5584088294310803, + "tokens_seen": 1311895552 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030430290872617856, + "loss": 2.6771, + "theoretical_loss": 3.558392751219927, + "tokens_seen": 1311961088 + }, + { + "epoch": 4.04, + "learning_rate": 0.0003042928786359077, + "loss": 2.7834, + "theoretical_loss": 3.558376674036774, + "tokens_seen": 1312026624 + }, + { + "epoch": 4.04, + "learning_rate": 0.0003042828485456369, + "loss": 2.7159, + "theoretical_loss": 3.558360597881503, + "tokens_seen": 1312092160 + }, + { + "epoch": 4.04, + "learning_rate": 0.0003042728184553661, + "loss": 2.6537, + "theoretical_loss": 3.5583445227539965, + "tokens_seen": 1312157696 + }, + { + "epoch": 4.04, + "learning_rate": 0.0003042627883650953, + "loss": 2.79, + "theoretical_loss": 3.5583284486541387, + "tokens_seen": 1312223232 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030425275827482447, + "loss": 2.9484, + "theoretical_loss": 3.558312375581812, + "tokens_seen": 1312288768 + }, + { + "epoch": 4.04, + "objective/train/docs_used": 1490224, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.775263786315918, + "objective/train/theoretical_loss": 3.5582963035368986, + "objective/train/tokens_used": 1332814304, + "theoretical_loss": 3.5582963035368986, + "tokens_seen": 1312354304 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030424272818455365, + "loss": 2.5093, + "theoretical_loss": 3.5582963035368986, + "tokens_seen": 1312354304 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030423269809428283, + "loss": 2.9231, + "theoretical_loss": 3.558280232519283, + "tokens_seen": 1312419840 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030422266800401207, + "loss": 2.684, + "theoretical_loss": 3.558264162528847, + "tokens_seen": 1312485376 + }, + { + "epoch": 4.04, + "learning_rate": 0.0003042126379137412, + "loss": 2.7024, + "theoretical_loss": 3.5582480935654743, + "tokens_seen": 1312550912 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030420260782347043, + "loss": 2.7182, + "theoretical_loss": 3.5582320256290485, + "tokens_seen": 1312616448 + }, + { + "epoch": 4.04, + "learning_rate": 0.0003041925777331996, + "loss": 2.6307, + "theoretical_loss": 3.5582159587194515, + "tokens_seen": 1312681984 + }, + { + "epoch": 4.04, + "learning_rate": 0.0003041825476429288, + "loss": 2.626, + "theoretical_loss": 3.5581998928365675, + "tokens_seen": 1312747520 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030417251755265797, + "loss": 2.7163, + "theoretical_loss": 3.5581838279802787, + "tokens_seen": 1312813056 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030416248746238715, + "loss": 3.0121, + "theoretical_loss": 3.5581677641504683, + "tokens_seen": 1312878592 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030415245737211633, + "loss": 2.7012, + "theoretical_loss": 3.558151701347021, + "tokens_seen": 1312944128 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030414242728184557, + "loss": 2.6484, + "theoretical_loss": 3.5581356395698176, + "tokens_seen": 1313009664 + }, + { + "epoch": 4.04, + "learning_rate": 0.0003041323971915747, + "loss": 2.6568, + "theoretical_loss": 3.5581195788187436, + "tokens_seen": 1313075200 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030412236710130393, + "loss": 2.6755, + "theoretical_loss": 3.558103519093681, + "tokens_seen": 1313140736 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030411233701103306, + "loss": 2.6254, + "theoretical_loss": 3.558087460394513, + "tokens_seen": 1313206272 + }, + { + "epoch": 4.04, + "learning_rate": 0.0003041023069207623, + "loss": 2.6578, + "theoretical_loss": 3.558071402721123, + "tokens_seen": 1313271808 + }, + { + "epoch": 4.04, + "learning_rate": 0.0003040922768304915, + "loss": 2.8261, + "theoretical_loss": 3.558055346073395, + "tokens_seen": 1313337344 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030408224674022066, + "loss": 2.8016, + "theoretical_loss": 3.5580392904512115, + "tokens_seen": 1313402880 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030407221664994984, + "loss": 2.7408, + "theoretical_loss": 3.558023235854456, + "tokens_seen": 1313468416 + }, + { + "epoch": 4.04, + "learning_rate": 0.000304062186559679, + "loss": 2.6432, + "theoretical_loss": 3.5580071822830117, + "tokens_seen": 1313533952 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030405215646940825, + "loss": 2.6584, + "theoretical_loss": 3.5579911297367626, + "tokens_seen": 1313599488 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030404212637913743, + "loss": 2.5022, + "theoretical_loss": 3.5579750782155917, + "tokens_seen": 1313665024 + }, + { + "epoch": 4.04, + "learning_rate": 0.0003040320962888666, + "loss": 2.6291, + "theoretical_loss": 3.557959027719382, + "tokens_seen": 1313730560 + }, + { + "epoch": 4.04, + "learning_rate": 0.0003040220661985958, + "loss": 2.7724, + "theoretical_loss": 3.557942978248018, + "tokens_seen": 1313796096 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030401203610832503, + "loss": 2.5832, + "theoretical_loss": 3.557926929801382, + "tokens_seen": 1313861632 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030400200601805416, + "loss": 2.4701, + "theoretical_loss": 3.5579108823793577, + "tokens_seen": 1313927168 + }, + { + "epoch": 4.04, + "objective/train/docs_used": 1490984, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.301514148712158, + "objective/train/theoretical_loss": 3.5578948359818297, + "objective/train/tokens_used": 1334452704, + "theoretical_loss": 3.5578948359818297, + "tokens_seen": 1313992704 + }, + { + "epoch": 4.04, + "learning_rate": 0.0003039919759277834, + "loss": 2.5106, + "theoretical_loss": 3.5578948359818297, + "tokens_seen": 1313992704 + }, + { + "epoch": 4.04, + "learning_rate": 0.0003039819458375125, + "loss": 2.7581, + "theoretical_loss": 3.5578787906086795, + "tokens_seen": 1314058240 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030397191574724176, + "loss": 2.7226, + "theoretical_loss": 3.5578627462597927, + "tokens_seen": 1314123776 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030396188565697094, + "loss": 2.9072, + "theoretical_loss": 3.557846702935051, + "tokens_seen": 1314189312 + }, + { + "epoch": 4.04, + "learning_rate": 0.0003039518555667001, + "loss": 2.5841, + "theoretical_loss": 3.5578306606343393, + "tokens_seen": 1314254848 + }, + { + "epoch": 4.04, + "learning_rate": 0.0003039418254764293, + "loss": 2.3466, + "theoretical_loss": 3.557814619357541, + "tokens_seen": 1314320384 + }, + { + "epoch": 4.04, + "learning_rate": 0.0003039317953861585, + "loss": 2.5913, + "theoretical_loss": 3.557798579104539, + "tokens_seen": 1314385920 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030392176529588766, + "loss": 2.6581, + "theoretical_loss": 3.5577825398752174, + "tokens_seen": 1314451456 + }, + { + "epoch": 4.04, + "learning_rate": 0.0003039117352056169, + "loss": 2.61, + "theoretical_loss": 3.5577665016694597, + "tokens_seen": 1314516992 + }, + { + "epoch": 4.04, + "learning_rate": 0.000303901705115346, + "loss": 2.9948, + "theoretical_loss": 3.5577504644871496, + "tokens_seen": 1314582528 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030389167502507526, + "loss": 2.6986, + "theoretical_loss": 3.557734428328171, + "tokens_seen": 1314648064 + }, + { + "epoch": 4.04, + "learning_rate": 0.0003038816449348044, + "loss": 2.808, + "theoretical_loss": 3.557718393192407, + "tokens_seen": 1314713600 + }, + { + "epoch": 4.04, + "learning_rate": 0.0003038716148445336, + "loss": 2.6255, + "theoretical_loss": 3.5577023590797423, + "tokens_seen": 1314779136 + }, + { + "epoch": 4.04, + "learning_rate": 0.0003038615847542628, + "loss": 2.5779, + "theoretical_loss": 3.5576863259900597, + "tokens_seen": 1314844672 + }, + { + "epoch": 4.04, + "learning_rate": 0.000303851554663992, + "loss": 2.6617, + "theoretical_loss": 3.5576702939232434, + "tokens_seen": 1314910208 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030384152457372117, + "loss": 2.8716, + "theoretical_loss": 3.5576542628791765, + "tokens_seen": 1314975744 + }, + { + "epoch": 4.04, + "learning_rate": 0.0003038314944834504, + "loss": 2.8262, + "theoretical_loss": 3.557638232857744, + "tokens_seen": 1315041280 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030382146439317953, + "loss": 2.7641, + "theoretical_loss": 3.5576222038588288, + "tokens_seen": 1315106816 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030381143430290876, + "loss": 2.8674, + "theoretical_loss": 3.557606175882315, + "tokens_seen": 1315172352 + }, + { + "epoch": 4.04, + "learning_rate": 0.0003038014042126379, + "loss": 2.8339, + "theoretical_loss": 3.5575901489280866, + "tokens_seen": 1315237888 + }, + { + "epoch": 4.04, + "learning_rate": 0.0003037913741223671, + "loss": 2.6716, + "theoretical_loss": 3.557574122996027, + "tokens_seen": 1315303424 + }, + { + "epoch": 4.04, + "learning_rate": 0.0003037813440320963, + "loss": 2.8439, + "theoretical_loss": 3.5575580980860204, + "tokens_seen": 1315368960 + }, + { + "epoch": 4.04, + "learning_rate": 0.0003037713139418255, + "loss": 2.7056, + "theoretical_loss": 3.5575420741979507, + "tokens_seen": 1315434496 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030376128385155467, + "loss": 2.7797, + "theoretical_loss": 3.5575260513317017, + "tokens_seen": 1315500032 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030375125376128385, + "loss": 2.5237, + "theoretical_loss": 3.5575100294871573, + "tokens_seen": 1315565568 + }, + { + "epoch": 4.04, + "objective/train/docs_used": 1492577, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.998716354370117, + "objective/train/theoretical_loss": 3.557494008664202, + "objective/train/tokens_used": 1336091104, + "theoretical_loss": 3.557494008664202, + "tokens_seen": 1315631104 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030374122367101303, + "loss": 2.6291, + "theoretical_loss": 3.557494008664202, + "tokens_seen": 1315631104 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030373119358074227, + "loss": 2.5316, + "theoretical_loss": 3.557477988862719, + "tokens_seen": 1315696640 + }, + { + "epoch": 4.04, + "learning_rate": 0.0003037211634904714, + "loss": 2.768, + "theoretical_loss": 3.5574619700825925, + "tokens_seen": 1315762176 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030371113340020063, + "loss": 2.7363, + "theoretical_loss": 3.557445952323707, + "tokens_seen": 1315827712 + }, + { + "epoch": 4.04, + "learning_rate": 0.0003037011033099298, + "loss": 2.671, + "theoretical_loss": 3.557429935585946, + "tokens_seen": 1315893248 + }, + { + "epoch": 4.04, + "learning_rate": 0.000303691073219659, + "loss": 2.6929, + "theoretical_loss": 3.5574139198691936, + "tokens_seen": 1315958784 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030368104312938817, + "loss": 2.8983, + "theoretical_loss": 3.5573979051733344, + "tokens_seen": 1316024320 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030367101303911735, + "loss": 2.7451, + "theoretical_loss": 3.5573818914982525, + "tokens_seen": 1316089856 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030366098294884653, + "loss": 2.506, + "theoretical_loss": 3.557365878843831, + "tokens_seen": 1316155392 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030365095285857577, + "loss": 2.5843, + "theoretical_loss": 3.557349867209955, + "tokens_seen": 1316220928 + }, + { + "epoch": 4.04, + "learning_rate": 0.0003036409227683049, + "loss": 2.8426, + "theoretical_loss": 3.5573338565965074, + "tokens_seen": 1316286464 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030363089267803413, + "loss": 2.8309, + "theoretical_loss": 3.5573178470033744, + "tokens_seen": 1316352000 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030362086258776326, + "loss": 2.8832, + "theoretical_loss": 3.5573018384304382, + "tokens_seen": 1316417536 + }, + { + "epoch": 4.04, + "learning_rate": 0.0003036108324974925, + "loss": 2.7057, + "theoretical_loss": 3.5572858308775843, + "tokens_seen": 1316483072 + }, + { + "epoch": 4.04, + "learning_rate": 0.0003036008024072217, + "loss": 2.7707, + "theoretical_loss": 3.5572698243446963, + "tokens_seen": 1316548608 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030359077231695086, + "loss": 2.3975, + "theoretical_loss": 3.5572538188316587, + "tokens_seen": 1316614144 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030358074222668004, + "loss": 2.8102, + "theoretical_loss": 3.5572378143383556, + "tokens_seen": 1316679680 + }, + { + "epoch": 4.04, + "learning_rate": 0.0003035707121364092, + "loss": 2.6688, + "theoretical_loss": 3.5572218108646716, + "tokens_seen": 1316745216 + }, + { + "epoch": 4.04, + "learning_rate": 0.0003035606820461384, + "loss": 2.969, + "theoretical_loss": 3.5572058084104907, + "tokens_seen": 1316810752 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030355065195586763, + "loss": 2.5797, + "theoretical_loss": 3.557189806975697, + "tokens_seen": 1316876288 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030354062186559676, + "loss": 2.4623, + "theoretical_loss": 3.5571738065601757, + "tokens_seen": 1316941824 + }, + { + "epoch": 4.04, + "learning_rate": 0.000303530591775326, + "loss": 2.8467, + "theoretical_loss": 3.55715780716381, + "tokens_seen": 1317007360 + }, + { + "epoch": 4.04, + "learning_rate": 0.0003035205616850552, + "loss": 2.8461, + "theoretical_loss": 3.5571418087864854, + "tokens_seen": 1317072896 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030351053159478436, + "loss": 2.6863, + "theoretical_loss": 3.5571258114280853, + "tokens_seen": 1317138432 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030350050150451354, + "loss": 2.8625, + "theoretical_loss": 3.557109815088495, + "tokens_seen": 1317203968 + }, + { + "epoch": 4.04, + "objective/train/docs_used": 1493455, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.698178291320801, + "objective/train/theoretical_loss": 3.557093819767598, + "objective/train/tokens_used": 1337729504, + "theoretical_loss": 3.557093819767598, + "tokens_seen": 1317269504 + }, + { + "epoch": 4.04, + "learning_rate": 0.0003034904714142427, + "loss": 2.6329, + "theoretical_loss": 3.557093819767598, + "tokens_seen": 1317269504 + }, + { + "epoch": 4.04, + "learning_rate": 0.0003034804413239719, + "loss": 2.5847, + "theoretical_loss": 3.5570778254652793, + "tokens_seen": 1317335040 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030347041123370114, + "loss": 2.5977, + "theoretical_loss": 3.5570618321814234, + "tokens_seen": 1317400576 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030346038114343026, + "loss": 2.711, + "theoretical_loss": 3.5570458399159146, + "tokens_seen": 1317466112 + }, + { + "epoch": 4.04, + "learning_rate": 0.0003034503510531595, + "loss": 2.7197, + "theoretical_loss": 3.5570298486686376, + "tokens_seen": 1317531648 + }, + { + "epoch": 4.04, + "learning_rate": 0.0003034403209628886, + "loss": 2.5191, + "theoretical_loss": 3.557013858439477, + "tokens_seen": 1317597184 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030343029087261786, + "loss": 2.568, + "theoretical_loss": 3.556997869228317, + "tokens_seen": 1317662720 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030342026078234704, + "loss": 2.6477, + "theoretical_loss": 3.5569818810350418, + "tokens_seen": 1317728256 + }, + { + "epoch": 4.04, + "learning_rate": 0.0003034102306920762, + "loss": 2.7502, + "theoretical_loss": 3.5569658938595374, + "tokens_seen": 1317793792 + }, + { + "epoch": 4.04, + "learning_rate": 0.0003034002006018054, + "loss": 2.7715, + "theoretical_loss": 3.5569499077016866, + "tokens_seen": 1317859328 + }, + { + "epoch": 4.04, + "learning_rate": 0.0003033901705115346, + "loss": 2.7943, + "theoretical_loss": 3.5569339225613756, + "tokens_seen": 1317924864 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030338014042126377, + "loss": 2.6835, + "theoretical_loss": 3.556917938438488, + "tokens_seen": 1317990400 + }, + { + "epoch": 4.04, + "learning_rate": 0.000303370110330993, + "loss": 2.6136, + "theoretical_loss": 3.5569019553329095, + "tokens_seen": 1318055936 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030336008024072213, + "loss": 2.7573, + "theoretical_loss": 3.5568859732445235, + "tokens_seen": 1318121472 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030335005015045137, + "loss": 2.6267, + "theoretical_loss": 3.5568699921732154, + "tokens_seen": 1318187008 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030334002006018055, + "loss": 2.652, + "theoretical_loss": 3.5568540121188703, + "tokens_seen": 1318252544 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030332998996990973, + "loss": 2.6998, + "theoretical_loss": 3.556838033081372, + "tokens_seen": 1318318080 + }, + { + "epoch": 4.04, + "learning_rate": 0.0003033199598796389, + "loss": 2.7622, + "theoretical_loss": 3.5568220550606053, + "tokens_seen": 1318383616 + }, + { + "epoch": 4.04, + "learning_rate": 0.0003033099297893681, + "loss": 2.6623, + "theoretical_loss": 3.5568060780564563, + "tokens_seen": 1318449152 + }, + { + "epoch": 4.04, + "learning_rate": 0.0003032998996990973, + "loss": 2.5731, + "theoretical_loss": 3.556790102068808, + "tokens_seen": 1318514688 + }, + { + "epoch": 4.04, + "learning_rate": 0.0003032898696088265, + "loss": 2.7561, + "theoretical_loss": 3.5567741270975466, + "tokens_seen": 1318580224 + }, + { + "epoch": 4.04, + "learning_rate": 0.0003032798395185557, + "loss": 2.7818, + "theoretical_loss": 3.5567581531425563, + "tokens_seen": 1318645760 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030326980942828487, + "loss": 2.5645, + "theoretical_loss": 3.556742180203722, + "tokens_seen": 1318711296 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030325977933801405, + "loss": 2.8654, + "theoretical_loss": 3.5567262082809283, + "tokens_seen": 1318776832 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030324974924774323, + "loss": 2.4856, + "theoretical_loss": 3.5567102373740607, + "tokens_seen": 1318842368 + }, + { + "epoch": 4.04, + "objective/train/docs_used": 1494825, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.679054021835327, + "objective/train/theoretical_loss": 3.5566942674830035, + "objective/train/tokens_used": 1339367904, + "theoretical_loss": 3.5566942674830035, + "tokens_seen": 1318907904 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030323971915747247, + "loss": 2.5742, + "theoretical_loss": 3.5566942674830035, + "tokens_seen": 1318907904 + }, + { + "epoch": 4.04, + "learning_rate": 0.0003032296890672016, + "loss": 2.674, + "theoretical_loss": 3.5566782986076424, + "tokens_seen": 1318973440 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030321965897693083, + "loss": 2.6931, + "theoretical_loss": 3.5566623307478613, + "tokens_seen": 1319038976 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030320962888666, + "loss": 2.5451, + "theoretical_loss": 3.556646363903546, + "tokens_seen": 1319104512 + }, + { + "epoch": 4.04, + "learning_rate": 0.0003031995987963892, + "loss": 2.7607, + "theoretical_loss": 3.556630398074581, + "tokens_seen": 1319170048 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030318956870611837, + "loss": 2.6781, + "theoretical_loss": 3.556614433260852, + "tokens_seen": 1319235584 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030317953861584755, + "loss": 2.9642, + "theoretical_loss": 3.5565984694622426, + "tokens_seen": 1319301120 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030316950852557673, + "loss": 2.8698, + "theoretical_loss": 3.556582506678639, + "tokens_seen": 1319366656 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030315947843530597, + "loss": 2.4901, + "theoretical_loss": 3.556566544909926, + "tokens_seen": 1319432192 + }, + { + "epoch": 4.04, + "learning_rate": 0.0003031494483450351, + "loss": 2.6169, + "theoretical_loss": 3.556550584155989, + "tokens_seen": 1319497728 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030313941825476433, + "loss": 2.7552, + "theoretical_loss": 3.5565346244167126, + "tokens_seen": 1319563264 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030312938816449346, + "loss": 2.5759, + "theoretical_loss": 3.556518665691982, + "tokens_seen": 1319628800 + }, + { + "epoch": 4.04, + "learning_rate": 0.0003031193580742227, + "loss": 2.8975, + "theoretical_loss": 3.556502707981682, + "tokens_seen": 1319694336 + }, + { + "epoch": 4.04, + "learning_rate": 0.0003031093279839519, + "loss": 2.7001, + "theoretical_loss": 3.556486751285699, + "tokens_seen": 1319759872 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030309929789368106, + "loss": 2.5405, + "theoretical_loss": 3.556470795603916, + "tokens_seen": 1319825408 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030308926780341024, + "loss": 2.7051, + "theoretical_loss": 3.5564548409362207, + "tokens_seen": 1319890944 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030308926780341024, + "loss": 2.6966, + "theoretical_loss": 3.5564388872824964, + "tokens_seen": 1319956480 + }, + { + "epoch": 4.04, + "learning_rate": 0.0003030792377131394, + "loss": 2.593, + "theoretical_loss": 3.556422934642629, + "tokens_seen": 1320022016 + }, + { + "epoch": 4.04, + "learning_rate": 0.0003030692076228686, + "loss": 2.5576, + "theoretical_loss": 3.5564069830165037, + "tokens_seen": 1320087552 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030305917753259783, + "loss": 2.7134, + "theoretical_loss": 3.5563910324040062, + "tokens_seen": 1320153088 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030304914744232696, + "loss": 2.7155, + "theoretical_loss": 3.5563750828050207, + "tokens_seen": 1320218624 + }, + { + "epoch": 4.04, + "learning_rate": 0.0003030391173520562, + "loss": 2.4824, + "theoretical_loss": 3.5563591342194334, + "tokens_seen": 1320284160 + }, + { + "epoch": 4.04, + "learning_rate": 0.0003030290872617854, + "loss": 2.7657, + "theoretical_loss": 3.5563431866471293, + "tokens_seen": 1320349696 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030301905717151456, + "loss": 2.6831, + "theoretical_loss": 3.556327240087994, + "tokens_seen": 1320415232 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030300902708124374, + "loss": 2.77, + "theoretical_loss": 3.5563112945419126, + "tokens_seen": 1320480768 + }, + { + "epoch": 4.04, + "objective/train/docs_used": 1495445, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9724247455596924, + "objective/train/theoretical_loss": 3.55629535000877, + "objective/train/tokens_used": 1341006304, + "theoretical_loss": 3.55629535000877, + "tokens_seen": 1320546304 + }, + { + "epoch": 4.04, + "learning_rate": 0.0003029989969909729, + "loss": 2.9013, + "theoretical_loss": 3.55629535000877, + "tokens_seen": 1320546304 + }, + { + "epoch": 4.04, + "learning_rate": 0.0003029889669007021, + "loss": 2.7225, + "theoretical_loss": 3.5562794064884526, + "tokens_seen": 1320611840 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030297893681043134, + "loss": 2.4485, + "theoretical_loss": 3.5562634639808453, + "tokens_seen": 1320677376 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030296890672016046, + "loss": 2.7318, + "theoretical_loss": 3.556247522485833, + "tokens_seen": 1320742912 + }, + { + "epoch": 4.04, + "learning_rate": 0.0003029588766298897, + "loss": 2.7806, + "theoretical_loss": 3.556231582003303, + "tokens_seen": 1320808448 + }, + { + "epoch": 4.04, + "learning_rate": 0.0003029488465396188, + "loss": 2.6104, + "theoretical_loss": 3.5562156425331377, + "tokens_seen": 1320873984 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030293881644934806, + "loss": 2.7759, + "theoretical_loss": 3.5561997040752256, + "tokens_seen": 1320939520 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030292878635907724, + "loss": 2.5906, + "theoretical_loss": 3.5561837666294505, + "tokens_seen": 1321005056 + }, + { + "epoch": 4.04, + "learning_rate": 0.0003029187562688064, + "loss": 2.8147, + "theoretical_loss": 3.5561678301956983, + "tokens_seen": 1321070592 + }, + { + "epoch": 4.04, + "learning_rate": 0.0003029087261785356, + "loss": 2.6234, + "theoretical_loss": 3.556151894773854, + "tokens_seen": 1321136128 + }, + { + "epoch": 4.04, + "learning_rate": 0.0003028986960882648, + "loss": 2.8506, + "theoretical_loss": 3.5561359603638047, + "tokens_seen": 1321201664 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030288866599799397, + "loss": 2.7474, + "theoretical_loss": 3.5561200269654343, + "tokens_seen": 1321267200 + }, + { + "epoch": 4.04, + "learning_rate": 0.0003028786359077232, + "loss": 2.8021, + "theoretical_loss": 3.5561040945786293, + "tokens_seen": 1321332736 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030286860581745233, + "loss": 2.818, + "theoretical_loss": 3.556088163203275, + "tokens_seen": 1321398272 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030285857572718157, + "loss": 2.7076, + "theoretical_loss": 3.5560722328392576, + "tokens_seen": 1321463808 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030284854563691075, + "loss": 2.6369, + "theoretical_loss": 3.5560563034864616, + "tokens_seen": 1321529344 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030283851554663993, + "loss": 2.5183, + "theoretical_loss": 3.556040375144774, + "tokens_seen": 1321594880 + }, + { + "epoch": 4.04, + "learning_rate": 0.0003028284854563691, + "loss": 2.545, + "theoretical_loss": 3.556024447814079, + "tokens_seen": 1321660416 + }, + { + "epoch": 4.04, + "learning_rate": 0.0003028184553660983, + "loss": 2.4651, + "theoretical_loss": 3.556008521494264, + "tokens_seen": 1321725952 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030280842527582747, + "loss": 2.6089, + "theoretical_loss": 3.555992596185213, + "tokens_seen": 1321791488 + }, + { + "epoch": 4.04, + "learning_rate": 0.0003027983951855567, + "loss": 2.6557, + "theoretical_loss": 3.555976671886813, + "tokens_seen": 1321857024 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030278836509528583, + "loss": 2.6763, + "theoretical_loss": 3.5559607485989497, + "tokens_seen": 1321922560 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030277833500501507, + "loss": 2.6129, + "theoretical_loss": 3.5559448263215083, + "tokens_seen": 1321988096 + }, + { + "epoch": 4.04, + "learning_rate": 0.0003027683049147442, + "loss": 2.5999, + "theoretical_loss": 3.555928905054375, + "tokens_seen": 1322053632 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030275827482447343, + "loss": 2.923, + "theoretical_loss": 3.555912984797435, + "tokens_seen": 1322119168 + }, + { + "epoch": 4.04, + "objective/train/docs_used": 1496938, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7434074878692627, + "objective/train/theoretical_loss": 3.555897065550575, + "objective/train/tokens_used": 1342644704, + "theoretical_loss": 3.555897065550575, + "tokens_seen": 1322184704 + }, + { + "epoch": 4.04, + "learning_rate": 0.0003027482447342026, + "loss": 2.7297, + "theoretical_loss": 3.555897065550575, + "tokens_seen": 1322184704 + }, + { + "epoch": 4.04, + "learning_rate": 0.0003027382146439318, + "loss": 2.804, + "theoretical_loss": 3.5558811473136807, + "tokens_seen": 1322250240 + }, + { + "epoch": 4.04, + "learning_rate": 0.000302728184553661, + "loss": 2.3758, + "theoretical_loss": 3.5558652300866367, + "tokens_seen": 1322315776 + }, + { + "epoch": 4.04, + "learning_rate": 0.0003027181544633902, + "loss": 2.5547, + "theoretical_loss": 3.555849313869331, + "tokens_seen": 1322381312 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030270812437311934, + "loss": 2.663, + "theoretical_loss": 3.5558333986616475, + "tokens_seen": 1322446848 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030269809428284857, + "loss": 2.5285, + "theoretical_loss": 3.555817484463474, + "tokens_seen": 1322512384 + }, + { + "epoch": 4.04, + "learning_rate": 0.0003026880641925777, + "loss": 2.6559, + "theoretical_loss": 3.555801571274695, + "tokens_seen": 1322577920 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030267803410230693, + "loss": 2.7535, + "theoretical_loss": 3.5557856590951964, + "tokens_seen": 1322643456 + }, + { + "epoch": 4.04, + "learning_rate": 0.0003026680040120361, + "loss": 2.8497, + "theoretical_loss": 3.5557697479248653, + "tokens_seen": 1322708992 + }, + { + "epoch": 4.04, + "learning_rate": 0.0003026579739217653, + "loss": 2.6702, + "theoretical_loss": 3.555753837763587, + "tokens_seen": 1322774528 + }, + { + "epoch": 4.04, + "learning_rate": 0.0003026479438314945, + "loss": 2.6117, + "theoretical_loss": 3.555737928611248, + "tokens_seen": 1322840064 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030263791374122366, + "loss": 2.5529, + "theoretical_loss": 3.5557220204677336, + "tokens_seen": 1322905600 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030262788365095284, + "loss": 2.6863, + "theoretical_loss": 3.5557061133329304, + "tokens_seen": 1322971136 + }, + { + "epoch": 4.04, + "learning_rate": 0.0003026178535606821, + "loss": 2.7612, + "theoretical_loss": 3.555690207206724, + "tokens_seen": 1323036672 + }, + { + "epoch": 4.04, + "learning_rate": 0.0003026078234704112, + "loss": 2.8795, + "theoretical_loss": 3.5556743020890007, + "tokens_seen": 1323102208 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030259779338014044, + "loss": 2.7482, + "theoretical_loss": 3.5556583979796477, + "tokens_seen": 1323167744 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030258776328986956, + "loss": 2.6481, + "theoretical_loss": 3.5556424948785494, + "tokens_seen": 1323233280 + }, + { + "epoch": 4.04, + "learning_rate": 0.0003025777331995988, + "loss": 2.561, + "theoretical_loss": 3.555626592785593, + "tokens_seen": 1323298816 + }, + { + "epoch": 4.04, + "learning_rate": 0.000302567703109328, + "loss": 2.6666, + "theoretical_loss": 3.555610691700664, + "tokens_seen": 1323364352 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030255767301905716, + "loss": 2.6272, + "theoretical_loss": 3.5555947916236494, + "tokens_seen": 1323429888 + }, + { + "epoch": 4.04, + "learning_rate": 0.0003025476429287864, + "loss": 2.755, + "theoretical_loss": 3.5555788925544345, + "tokens_seen": 1323495424 + }, + { + "epoch": 4.04, + "learning_rate": 0.0003025376128385156, + "loss": 2.7486, + "theoretical_loss": 3.5555629944929064, + "tokens_seen": 1323560960 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030252758274824476, + "loss": 2.768, + "theoretical_loss": 3.555547097438951, + "tokens_seen": 1323626496 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030251755265797394, + "loss": 2.7912, + "theoretical_loss": 3.555531201392454, + "tokens_seen": 1323692032 + }, + { + "epoch": 4.04, + "learning_rate": 0.0003025075225677031, + "loss": 2.7348, + "theoretical_loss": 3.5555153063533025, + "tokens_seen": 1323757568 + }, + { + "epoch": 4.04, + "objective/train/docs_used": 1497393, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.852140426635742, + "objective/train/theoretical_loss": 3.5554994123213826, + "objective/train/tokens_used": 1344283104, + "theoretical_loss": 3.5554994123213826, + "tokens_seen": 1323823104 + }, + { + "epoch": 4.04, + "learning_rate": 0.0003024974924774323, + "loss": 2.8627, + "theoretical_loss": 3.5554994123213826, + "tokens_seen": 1323823104 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030248746238716154, + "loss": 2.9006, + "theoretical_loss": 3.55548351929658, + "tokens_seen": 1323888640 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030247743229689066, + "loss": 2.5953, + "theoretical_loss": 3.555467627278782, + "tokens_seen": 1323954176 + }, + { + "epoch": 4.04, + "learning_rate": 0.0003024674022066199, + "loss": 2.5436, + "theoretical_loss": 3.555451736267874, + "tokens_seen": 1324019712 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030245737211634903, + "loss": 2.6353, + "theoretical_loss": 3.5554358462637436, + "tokens_seen": 1324085248 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030244734202607826, + "loss": 2.6937, + "theoretical_loss": 3.5554199572662757, + "tokens_seen": 1324150784 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030243731193580744, + "loss": 2.849, + "theoretical_loss": 3.5554040692753577, + "tokens_seen": 1324216320 + }, + { + "epoch": 4.04, + "learning_rate": 0.0003024272818455366, + "loss": 2.7065, + "theoretical_loss": 3.555388182290876, + "tokens_seen": 1324281856 + }, + { + "epoch": 4.04, + "learning_rate": 0.0003024172517552658, + "loss": 2.8507, + "theoretical_loss": 3.555372296312717, + "tokens_seen": 1324347392 + }, + { + "epoch": 4.04, + "learning_rate": 0.000302407221664995, + "loss": 2.6622, + "theoretical_loss": 3.555356411340766, + "tokens_seen": 1324412928 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030239719157472417, + "loss": 2.7269, + "theoretical_loss": 3.555340527374911, + "tokens_seen": 1324478464 + }, + { + "epoch": 4.04, + "learning_rate": 0.0003023871614844534, + "loss": 2.6154, + "theoretical_loss": 3.5553246444150384, + "tokens_seen": 1324544000 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030237713139418253, + "loss": 2.501, + "theoretical_loss": 3.5553087624610336, + "tokens_seen": 1324609536 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030236710130391177, + "loss": 2.6562, + "theoretical_loss": 3.555292881512784, + "tokens_seen": 1324675072 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030235707121364095, + "loss": 2.5376, + "theoretical_loss": 3.5552770015701762, + "tokens_seen": 1324740608 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030234704112337013, + "loss": 2.8098, + "theoretical_loss": 3.5552611226330963, + "tokens_seen": 1324806144 + }, + { + "epoch": 4.04, + "learning_rate": 0.0003023370110330993, + "loss": 2.767, + "theoretical_loss": 3.555245244701431, + "tokens_seen": 1324871680 + }, + { + "epoch": 4.04, + "learning_rate": 0.0003023269809428285, + "loss": 2.5934, + "theoretical_loss": 3.5552293677750675, + "tokens_seen": 1324937216 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030231695085255767, + "loss": 2.7604, + "theoretical_loss": 3.5552134918538916, + "tokens_seen": 1325002752 + }, + { + "epoch": 4.04, + "learning_rate": 0.0003023069207622869, + "loss": 2.7105, + "theoretical_loss": 3.5551976169377904, + "tokens_seen": 1325068288 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030229689067201603, + "loss": 2.5927, + "theoretical_loss": 3.5551817430266506, + "tokens_seen": 1325133824 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030228686058174527, + "loss": 2.7396, + "theoretical_loss": 3.5551658701203586, + "tokens_seen": 1325199360 + }, + { + "epoch": 4.04, + "learning_rate": 0.0003022768304914744, + "loss": 2.7238, + "theoretical_loss": 3.5551499982188015, + "tokens_seen": 1325264896 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030226680040120363, + "loss": 2.7457, + "theoretical_loss": 3.5551341273218657, + "tokens_seen": 1325330432 + }, + { + "epoch": 4.04, + "learning_rate": 0.0003022567703109328, + "loss": 2.7258, + "theoretical_loss": 3.555118257429438, + "tokens_seen": 1325395968 + }, + { + "epoch": 4.04, + "objective/train/docs_used": 1498967, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8756134510040283, + "objective/train/theoretical_loss": 3.555102388541405, + "objective/train/tokens_used": 1345921504, + "theoretical_loss": 3.555102388541405, + "tokens_seen": 1325461504 + }, + { + "epoch": 4.04, + "learning_rate": 0.000302246740220662, + "loss": 2.9374, + "theoretical_loss": 3.555102388541405, + "tokens_seen": 1325461504 + }, + { + "epoch": 4.04, + "learning_rate": 0.0003022367101303912, + "loss": 2.7265, + "theoretical_loss": 3.555086520657654, + "tokens_seen": 1325527040 + }, + { + "epoch": 4.04, + "learning_rate": 0.0003022266800401204, + "loss": 2.7696, + "theoretical_loss": 3.5550706537780714, + "tokens_seen": 1325592576 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030221664994984954, + "loss": 2.6507, + "theoretical_loss": 3.5550547879025443, + "tokens_seen": 1325658112 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030220661985957877, + "loss": 2.7139, + "theoretical_loss": 3.555038923030959, + "tokens_seen": 1325723648 + }, + { + "epoch": 4.04, + "learning_rate": 0.0003021965897693079, + "loss": 2.8103, + "theoretical_loss": 3.5550230591632026, + "tokens_seen": 1325789184 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030218655967903713, + "loss": 2.656, + "theoretical_loss": 3.5550071962991616, + "tokens_seen": 1325854720 + }, + { + "epoch": 4.04, + "learning_rate": 0.0003021765295887663, + "loss": 2.6585, + "theoretical_loss": 3.554991334438724, + "tokens_seen": 1325920256 + }, + { + "epoch": 4.04, + "learning_rate": 0.0003021664994984955, + "loss": 2.7376, + "theoretical_loss": 3.554975473581776, + "tokens_seen": 1325985792 + }, + { + "epoch": 4.04, + "learning_rate": 0.0003021564694082247, + "loss": 2.6548, + "theoretical_loss": 3.5549596137282045, + "tokens_seen": 1326051328 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030214643931795386, + "loss": 2.7974, + "theoretical_loss": 3.554943754877896, + "tokens_seen": 1326116864 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030213640922768304, + "loss": 2.6282, + "theoretical_loss": 3.5549278970307383, + "tokens_seen": 1326182400 + }, + { + "epoch": 4.04, + "learning_rate": 0.0003021263791374123, + "loss": 2.695, + "theoretical_loss": 3.554912040186618, + "tokens_seen": 1326247936 + }, + { + "epoch": 4.04, + "learning_rate": 0.0003021163490471414, + "loss": 2.7362, + "theoretical_loss": 3.554896184345422, + "tokens_seen": 1326313472 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030210631895687064, + "loss": 2.5531, + "theoretical_loss": 3.5548803295070375, + "tokens_seen": 1326379008 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030209628886659976, + "loss": 2.6168, + "theoretical_loss": 3.5548644756713514, + "tokens_seen": 1326444544 + }, + { + "epoch": 4.04, + "learning_rate": 0.000302086258776329, + "loss": 2.9134, + "theoretical_loss": 3.554848622838251, + "tokens_seen": 1326510080 + }, + { + "epoch": 4.04, + "learning_rate": 0.0003020762286860582, + "loss": 2.7424, + "theoretical_loss": 3.5548327710076233, + "tokens_seen": 1326575616 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030206619859578736, + "loss": 2.648, + "theoretical_loss": 3.5548169201793547, + "tokens_seen": 1326641152 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030205616850551654, + "loss": 2.8463, + "theoretical_loss": 3.5548010703533333, + "tokens_seen": 1326706688 + }, + { + "epoch": 4.04, + "learning_rate": 0.0003020461384152458, + "loss": 2.5416, + "theoretical_loss": 3.5547852215294453, + "tokens_seen": 1326772224 + }, + { + "epoch": 4.04, + "learning_rate": 0.0003020361083249749, + "loss": 2.6082, + "theoretical_loss": 3.554769373707579, + "tokens_seen": 1326837760 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030202607823470414, + "loss": 2.6569, + "theoretical_loss": 3.5547535268876205, + "tokens_seen": 1326903296 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030201604814443327, + "loss": 2.6231, + "theoretical_loss": 3.554737681069458, + "tokens_seen": 1326968832 + }, + { + "epoch": 4.04, + "learning_rate": 0.0003020060180541625, + "loss": 2.7764, + "theoretical_loss": 3.5547218362529773, + "tokens_seen": 1327034368 + }, + { + "epoch": 4.04, + "objective/train/docs_used": 1499694, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.877124547958374, + "objective/train/theoretical_loss": 3.5547059924380666, + "objective/train/tokens_used": 1347559904, + "theoretical_loss": 3.5547059924380666, + "tokens_seen": 1327099904 + }, + { + "epoch": 4.04, + "learning_rate": 0.0003019959879638917, + "loss": 2.8643, + "theoretical_loss": 3.5547059924380666, + "tokens_seen": 1327099904 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030198595787362087, + "loss": 2.7271, + "theoretical_loss": 3.554690149624613, + "tokens_seen": 1327165440 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030197592778335005, + "loss": 2.7456, + "theoretical_loss": 3.554674307812504, + "tokens_seen": 1327230976 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030196589769307923, + "loss": 2.6898, + "theoretical_loss": 3.5546584670016257, + "tokens_seen": 1327296512 + }, + { + "epoch": 4.04, + "learning_rate": 0.0003019558676028084, + "loss": 2.5976, + "theoretical_loss": 3.5546426271918667, + "tokens_seen": 1327362048 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030194583751253764, + "loss": 2.7791, + "theoretical_loss": 3.554626788383114, + "tokens_seen": 1327427584 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030193580742226677, + "loss": 2.7436, + "theoretical_loss": 3.554610950575255, + "tokens_seen": 1327493120 + }, + { + "epoch": 4.04, + "learning_rate": 0.000301925777331996, + "loss": 2.3796, + "theoretical_loss": 3.554595113768176, + "tokens_seen": 1327558656 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030191574724172513, + "loss": 2.3177, + "theoretical_loss": 3.554579277961766, + "tokens_seen": 1327624192 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030190571715145437, + "loss": 2.7672, + "theoretical_loss": 3.554563443155911, + "tokens_seen": 1327689728 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030189568706118355, + "loss": 2.8033, + "theoretical_loss": 3.5545476093504993, + "tokens_seen": 1327755264 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030188565697091273, + "loss": 2.5326, + "theoretical_loss": 3.554531776545418, + "tokens_seen": 1327820800 + }, + { + "epoch": 4.04, + "learning_rate": 0.0003018756268806419, + "loss": 2.7771, + "theoretical_loss": 3.554515944740554, + "tokens_seen": 1327886336 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030186559679037115, + "loss": 2.9061, + "theoretical_loss": 3.554500113935796, + "tokens_seen": 1327951872 + }, + { + "epoch": 4.04, + "learning_rate": 0.0003018555667001003, + "loss": 2.6614, + "theoretical_loss": 3.5544842841310302, + "tokens_seen": 1328017408 + }, + { + "epoch": 4.04, + "learning_rate": 0.0003018455366098295, + "loss": 2.7331, + "theoretical_loss": 3.5544684553261447, + "tokens_seen": 1328082944 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030183550651955864, + "loss": 2.5361, + "theoretical_loss": 3.5544526275210266, + "tokens_seen": 1328148480 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030182547642928787, + "loss": 2.7823, + "theoretical_loss": 3.554436800715564, + "tokens_seen": 1328214016 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030181544633901705, + "loss": 2.5981, + "theoretical_loss": 3.5544209749096445, + "tokens_seen": 1328279552 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030180541624874623, + "loss": 2.6959, + "theoretical_loss": 3.554405150103155, + "tokens_seen": 1328345088 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030179538615847547, + "loss": 2.7439, + "theoretical_loss": 3.5543893262959836, + "tokens_seen": 1328410624 + }, + { + "epoch": 4.04, + "learning_rate": 0.0003017853560682046, + "loss": 2.6138, + "theoretical_loss": 3.5543735034880175, + "tokens_seen": 1328476160 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030177532597793383, + "loss": 2.5255, + "theoretical_loss": 3.5543576816791447, + "tokens_seen": 1328541696 + }, + { + "epoch": 4.04, + "learning_rate": 0.000301765295887663, + "loss": 2.6184, + "theoretical_loss": 3.554341860869253, + "tokens_seen": 1328607232 + }, + { + "epoch": 4.04, + "learning_rate": 0.0003017552657973922, + "loss": 2.557, + "theoretical_loss": 3.5543260410582294, + "tokens_seen": 1328672768 + }, + { + "epoch": 4.04, + "objective/train/docs_used": 1500360, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.750994920730591, + "objective/train/theoretical_loss": 3.554310222245962, + "objective/train/tokens_used": 1349198304, + "theoretical_loss": 3.554310222245962, + "tokens_seen": 1328738304 + }, + { + "epoch": 4.04, + "learning_rate": 0.0003017452357071214, + "loss": 2.6728, + "theoretical_loss": 3.554310222245962, + "tokens_seen": 1328738304 + }, + { + "epoch": 4.04, + "learning_rate": 0.0003017352056168506, + "loss": 2.6421, + "theoretical_loss": 3.5542944044323384, + "tokens_seen": 1328803840 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030172517552657974, + "loss": 2.787, + "theoretical_loss": 3.5542785876172465, + "tokens_seen": 1328869376 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030171514543630897, + "loss": 2.7608, + "theoretical_loss": 3.5542627718005733, + "tokens_seen": 1328934912 + }, + { + "epoch": 4.04, + "learning_rate": 0.0003017051153460381, + "loss": 2.5684, + "theoretical_loss": 3.5542469569822073, + "tokens_seen": 1329000448 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030169508525576733, + "loss": 2.539, + "theoretical_loss": 3.554231143162036, + "tokens_seen": 1329065984 + }, + { + "epoch": 4.04, + "learning_rate": 0.0003016850551654965, + "loss": 2.5904, + "theoretical_loss": 3.5542153303399475, + "tokens_seen": 1329131520 + }, + { + "epoch": 4.04, + "learning_rate": 0.0003016750250752257, + "loss": 2.5785, + "theoretical_loss": 3.554199518515829, + "tokens_seen": 1329197056 + }, + { + "epoch": 4.04, + "learning_rate": 0.0003016649949849549, + "loss": 2.8468, + "theoretical_loss": 3.554183707689569, + "tokens_seen": 1329262592 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030165496489468406, + "loss": 2.7968, + "theoretical_loss": 3.554167897861055, + "tokens_seen": 1329328128 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030164493480441324, + "loss": 2.4569, + "theoretical_loss": 3.5541520890301745, + "tokens_seen": 1329393664 + }, + { + "epoch": 4.04, + "learning_rate": 0.0003016349047141425, + "loss": 2.6965, + "theoretical_loss": 3.554136281196816, + "tokens_seen": 1329459200 + }, + { + "epoch": 4.04, + "learning_rate": 0.0003016248746238716, + "loss": 2.555, + "theoretical_loss": 3.5541204743608663, + "tokens_seen": 1329524736 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030161484453360084, + "loss": 2.7436, + "theoretical_loss": 3.5541046685222146, + "tokens_seen": 1329590272 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030160481444332996, + "loss": 2.76, + "theoretical_loss": 3.5540888636807484, + "tokens_seen": 1329655808 + }, + { + "epoch": 4.04, + "learning_rate": 0.0003015947843530592, + "loss": 2.6591, + "theoretical_loss": 3.5540730598363552, + "tokens_seen": 1329721344 + }, + { + "epoch": 4.04, + "learning_rate": 0.0003015847542627884, + "loss": 2.5531, + "theoretical_loss": 3.5540572569889237, + "tokens_seen": 1329786880 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030157472417251756, + "loss": 2.7457, + "theoretical_loss": 3.554041455138342, + "tokens_seen": 1329852416 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030156469408224674, + "loss": 2.6176, + "theoretical_loss": 3.5540256542844966, + "tokens_seen": 1329917952 + }, + { + "epoch": 4.04, + "learning_rate": 0.000301554663991976, + "loss": 2.508, + "theoretical_loss": 3.554009854427277, + "tokens_seen": 1329983488 + }, + { + "epoch": 4.04, + "learning_rate": 0.0003015446339017051, + "loss": 2.8276, + "theoretical_loss": 3.5539940555665703, + "tokens_seen": 1330049024 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030153460381143434, + "loss": 2.7321, + "theoretical_loss": 3.5539782577022656, + "tokens_seen": 1330114560 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030152457372116347, + "loss": 2.5992, + "theoretical_loss": 3.55396246083425, + "tokens_seen": 1330180096 + }, + { + "epoch": 4.04, + "learning_rate": 0.0003015145436308927, + "loss": 2.5204, + "theoretical_loss": 3.553946664962412, + "tokens_seen": 1330245632 + }, + { + "epoch": 4.04, + "learning_rate": 0.0003015045135406219, + "loss": 2.8735, + "theoretical_loss": 3.5539308700866394, + "tokens_seen": 1330311168 + }, + { + "epoch": 4.04, + "objective/train/docs_used": 1501699, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.85687518119812, + "objective/train/theoretical_loss": 3.5539150762068212, + "objective/train/tokens_used": 1350836704, + "theoretical_loss": 3.5539150762068212, + "tokens_seen": 1330376704 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030149448345035107, + "loss": 2.6929, + "theoretical_loss": 3.5539150762068212, + "tokens_seen": 1330376704 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030148445336008025, + "loss": 2.5348, + "theoretical_loss": 3.5538992833228447, + "tokens_seen": 1330442240 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030147442326980943, + "loss": 2.8463, + "theoretical_loss": 3.553883491434598, + "tokens_seen": 1330507776 + }, + { + "epoch": 4.04, + "learning_rate": 0.0003014643931795386, + "loss": 2.6109, + "theoretical_loss": 3.5538677005419697, + "tokens_seen": 1330573312 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030145436308926784, + "loss": 2.8107, + "theoretical_loss": 3.5538519106448483, + "tokens_seen": 1330638848 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030144433299899697, + "loss": 2.5365, + "theoretical_loss": 3.553836121743121, + "tokens_seen": 1330704384 + }, + { + "epoch": 4.04, + "learning_rate": 0.0003014343029087262, + "loss": 2.7058, + "theoretical_loss": 3.5538203338366765, + "tokens_seen": 1330769920 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030142427281845533, + "loss": 2.6364, + "theoretical_loss": 3.553804546925404, + "tokens_seen": 1330835456 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030141424272818457, + "loss": 2.2382, + "theoretical_loss": 3.5537887610091903, + "tokens_seen": 1330900992 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030140421263791375, + "loss": 2.6748, + "theoretical_loss": 3.553772976087924, + "tokens_seen": 1330966528 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030139418254764293, + "loss": 2.7097, + "theoretical_loss": 3.5537571921614943, + "tokens_seen": 1331032064 + }, + { + "epoch": 4.04, + "learning_rate": 0.0003013841524573721, + "loss": 2.6677, + "theoretical_loss": 3.5537414092297888, + "tokens_seen": 1331097600 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030137412236710135, + "loss": 2.5247, + "theoretical_loss": 3.553725627292696, + "tokens_seen": 1331163136 + }, + { + "epoch": 4.04, + "learning_rate": 0.0003013640922768305, + "loss": 2.4289, + "theoretical_loss": 3.5537098463501042, + "tokens_seen": 1331228672 + }, + { + "epoch": 4.04, + "learning_rate": 0.0003013540621865597, + "loss": 2.6032, + "theoretical_loss": 3.553694066401902, + "tokens_seen": 1331294208 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030134403209628884, + "loss": 2.7124, + "theoretical_loss": 3.553678287447977, + "tokens_seen": 1331359744 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030133400200601807, + "loss": 2.6053, + "theoretical_loss": 3.553662509488219, + "tokens_seen": 1331425280 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030132397191574725, + "loss": 2.6833, + "theoretical_loss": 3.553646732522515, + "tokens_seen": 1331490816 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030131394182547643, + "loss": 2.7973, + "theoretical_loss": 3.5536309565507542, + "tokens_seen": 1331556352 + }, + { + "epoch": 4.04, + "learning_rate": 0.0003013039117352056, + "loss": 2.5321, + "theoretical_loss": 3.5536151815728254, + "tokens_seen": 1331621888 + }, + { + "epoch": 4.04, + "learning_rate": 0.0003012938816449348, + "loss": 2.7239, + "theoretical_loss": 3.5535994075886164, + "tokens_seen": 1331687424 + }, + { + "epoch": 4.04, + "learning_rate": 0.000301283851554664, + "loss": 2.8171, + "theoretical_loss": 3.5535836345980165, + "tokens_seen": 1331752960 + }, + { + "epoch": 4.04, + "learning_rate": 0.0003012738214643932, + "loss": 2.7051, + "theoretical_loss": 3.5535678626009126, + "tokens_seen": 1331818496 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030126379137412234, + "loss": 2.6428, + "theoretical_loss": 3.5535520915971954, + "tokens_seen": 1331884032 + }, + { + "epoch": 4.04, + "learning_rate": 0.0003012537612838516, + "loss": 2.857, + "theoretical_loss": 3.5535363215867517, + "tokens_seen": 1331949568 + }, + { + "epoch": 4.04, + "objective/train/docs_used": 1502342, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4824728965759277, + "objective/train/theoretical_loss": 3.553520552569471, + "objective/train/tokens_used": 1352475104, + "theoretical_loss": 3.553520552569471, + "tokens_seen": 1332015104 + }, + { + "epoch": 4.04, + "learning_rate": 0.0003012437311935807, + "loss": 2.8718, + "theoretical_loss": 3.553520552569471, + "tokens_seen": 1332015104 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030123370110330994, + "loss": 2.7301, + "theoretical_loss": 3.5535047845452414, + "tokens_seen": 1332080640 + }, + { + "epoch": 4.04, + "learning_rate": 0.0003012236710130391, + "loss": 2.6574, + "theoretical_loss": 3.553489017513952, + "tokens_seen": 1332146176 + }, + { + "epoch": 4.04, + "learning_rate": 0.0003012136409227683, + "loss": 2.6469, + "theoretical_loss": 3.5534732514754914, + "tokens_seen": 1332211712 + }, + { + "epoch": 4.04, + "learning_rate": 0.0003012036108324975, + "loss": 2.8646, + "theoretical_loss": 3.5534574864297475, + "tokens_seen": 1332277248 + }, + { + "epoch": 4.04, + "learning_rate": 0.0003011935807422267, + "loss": 2.8386, + "theoretical_loss": 3.5534417223766095, + "tokens_seen": 1332342784 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030118355065195584, + "loss": 2.7916, + "theoretical_loss": 3.553425959315967, + "tokens_seen": 1332408320 + }, + { + "epoch": 4.04, + "learning_rate": 0.0003011735205616851, + "loss": 2.7231, + "theoretical_loss": 3.5534101972477066, + "tokens_seen": 1332473856 + }, + { + "epoch": 4.04, + "learning_rate": 0.0003011634904714142, + "loss": 2.6288, + "theoretical_loss": 3.553394436171719, + "tokens_seen": 1332539392 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030115346038114344, + "loss": 2.6364, + "theoretical_loss": 3.553378676087892, + "tokens_seen": 1332604928 + }, + { + "epoch": 4.04, + "learning_rate": 0.0003011434302908726, + "loss": 2.5164, + "theoretical_loss": 3.553362916996114, + "tokens_seen": 1332670464 + }, + { + "epoch": 4.04, + "learning_rate": 0.0003011334002006018, + "loss": 2.5531, + "theoretical_loss": 3.553347158896275, + "tokens_seen": 1332736000 + }, + { + "epoch": 4.04, + "learning_rate": 0.000301123370110331, + "loss": 2.8268, + "theoretical_loss": 3.5533314017882622, + "tokens_seen": 1332801536 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030111334002006016, + "loss": 2.658, + "theoretical_loss": 3.553315645671966, + "tokens_seen": 1332867072 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030110330992978935, + "loss": 2.6564, + "theoretical_loss": 3.553299890547274, + "tokens_seen": 1332932608 + }, + { + "epoch": 4.04, + "learning_rate": 0.0003010932798395186, + "loss": 2.4595, + "theoretical_loss": 3.553284136414076, + "tokens_seen": 1332998144 + }, + { + "epoch": 4.04, + "learning_rate": 0.0003010832497492477, + "loss": 2.5884, + "theoretical_loss": 3.5532683832722602, + "tokens_seen": 1333063680 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030107321965897694, + "loss": 2.6104, + "theoretical_loss": 3.553252631121716, + "tokens_seen": 1333129216 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030106318956870607, + "loss": 2.8035, + "theoretical_loss": 3.5532368799623315, + "tokens_seen": 1333194752 + }, + { + "epoch": 4.04, + "learning_rate": 0.0003010531594784353, + "loss": 2.641, + "theoretical_loss": 3.553221129793996, + "tokens_seen": 1333260288 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030104312938816454, + "loss": 2.4992, + "theoretical_loss": 3.553205380616599, + "tokens_seen": 1333325824 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030103309929789367, + "loss": 2.7852, + "theoretical_loss": 3.5531896324300285, + "tokens_seen": 1333391360 + }, + { + "epoch": 4.04, + "learning_rate": 0.0003010230692076229, + "loss": 2.8122, + "theoretical_loss": 3.553173885234174, + "tokens_seen": 1333456896 + }, + { + "epoch": 4.04, + "learning_rate": 0.0003010130391173521, + "loss": 2.769, + "theoretical_loss": 3.553158139028925, + "tokens_seen": 1333522432 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030100300902708127, + "loss": 2.7795, + "theoretical_loss": 3.5531423938141695, + "tokens_seen": 1333587968 + }, + { + "epoch": 4.04, + "objective/train/docs_used": 1503973, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.984467029571533, + "objective/train/theoretical_loss": 3.553126649589797, + "objective/train/tokens_used": 1354113504, + "theoretical_loss": 3.553126649589797, + "tokens_seen": 1333653504 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030099297893681045, + "loss": 2.9471, + "theoretical_loss": 3.553126649589797, + "tokens_seen": 1333653504 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030098294884653963, + "loss": 2.8916, + "theoretical_loss": 3.5531109063556965, + "tokens_seen": 1333719040 + }, + { + "epoch": 4.04, + "learning_rate": 0.0003009729187562688, + "loss": 2.3241, + "theoretical_loss": 3.553095164111757, + "tokens_seen": 1333784576 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030096288866599804, + "loss": 2.926, + "theoretical_loss": 3.553079422857868, + "tokens_seen": 1333850112 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030095285857572717, + "loss": 2.8648, + "theoretical_loss": 3.553063682593918, + "tokens_seen": 1333915648 + }, + { + "epoch": 4.04, + "learning_rate": 0.0003009428284854564, + "loss": 2.6997, + "theoretical_loss": 3.5530479433197963, + "tokens_seen": 1333981184 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030093279839518553, + "loss": 2.6958, + "theoretical_loss": 3.553032205035392, + "tokens_seen": 1334046720 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030092276830491477, + "loss": 2.6967, + "theoretical_loss": 3.553016467740594, + "tokens_seen": 1334112256 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030091273821464395, + "loss": 2.5381, + "theoretical_loss": 3.5530007314352927, + "tokens_seen": 1334177792 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030090270812437313, + "loss": 2.7095, + "theoretical_loss": 3.552984996119376, + "tokens_seen": 1334243328 + }, + { + "epoch": 4.04, + "learning_rate": 0.0003008926780341023, + "loss": 2.9267, + "theoretical_loss": 3.5529692617927333, + "tokens_seen": 1334308864 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030088264794383155, + "loss": 2.9963, + "theoretical_loss": 3.5529535284552543, + "tokens_seen": 1334374400 + }, + { + "epoch": 4.04, + "learning_rate": 0.0003008726178535607, + "loss": 2.7728, + "theoretical_loss": 3.5529377961068276, + "tokens_seen": 1334439936 + }, + { + "epoch": 4.04, + "learning_rate": 0.0003008625877632899, + "loss": 2.7666, + "theoretical_loss": 3.5529220647473427, + "tokens_seen": 1334505472 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030085255767301904, + "loss": 2.7262, + "theoretical_loss": 3.552906334376689, + "tokens_seen": 1334571008 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030084252758274827, + "loss": 2.5032, + "theoretical_loss": 3.552890604994756, + "tokens_seen": 1334636544 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030083249749247745, + "loss": 2.8243, + "theoretical_loss": 3.5528748766014333, + "tokens_seen": 1334702080 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030082246740220663, + "loss": 2.6245, + "theoretical_loss": 3.5528591491966086, + "tokens_seen": 1334767616 + }, + { + "epoch": 4.04, + "learning_rate": 0.0003008124373119358, + "loss": 2.6308, + "theoretical_loss": 3.552843422780173, + "tokens_seen": 1334833152 + }, + { + "epoch": 4.04, + "learning_rate": 0.000300802407221665, + "loss": 2.6874, + "theoretical_loss": 3.552827697352015, + "tokens_seen": 1334898688 + }, + { + "epoch": 4.04, + "learning_rate": 0.0003007923771313942, + "loss": 2.753, + "theoretical_loss": 3.5528119729120244, + "tokens_seen": 1334964224 + }, + { + "epoch": 4.04, + "learning_rate": 0.0003007823470411234, + "loss": 2.6477, + "theoretical_loss": 3.5527962494600906, + "tokens_seen": 1335029760 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030077231695085254, + "loss": 2.7341, + "theoretical_loss": 3.552780526996102, + "tokens_seen": 1335095296 + }, + { + "epoch": 4.04, + "learning_rate": 0.0003007622868605818, + "loss": 2.6776, + "theoretical_loss": 3.552764805519949, + "tokens_seen": 1335160832 + }, + { + "epoch": 4.04, + "learning_rate": 0.0003007522567703109, + "loss": 2.6165, + "theoretical_loss": 3.5527490850315218, + "tokens_seen": 1335226368 + }, + { + "epoch": 4.04, + "objective/train/docs_used": 1504757, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.535205125808716, + "objective/train/theoretical_loss": 3.552733365530708, + "objective/train/tokens_used": 1355751904, + "theoretical_loss": 3.552733365530708, + "tokens_seen": 1335291904 + }, + { + "epoch": 4.04, + "learning_rate": 0.00030074222668004014, + "loss": 2.5726, + "theoretical_loss": 3.552733365530708, + "tokens_seen": 1335291904 + }, + { + "epoch": 4.04, + "learning_rate": 0.0003007321965897693, + "loss": 2.7941, + "theoretical_loss": 3.5527176470173982, + "tokens_seen": 1335357440 + }, + { + "epoch": 4.04, + "learning_rate": 0.0003007221664994985, + "loss": 2.77, + "theoretical_loss": 3.552701929491482, + "tokens_seen": 1335422976 + }, + { + "epoch": 4.04, + "learning_rate": 0.0003007121364092277, + "loss": 2.4994, + "theoretical_loss": 3.5526862129528487, + "tokens_seen": 1335488512 + }, + { + "epoch": 4.05, + "learning_rate": 0.0003007021063189569, + "loss": 2.6934, + "theoretical_loss": 3.5526704974013876, + "tokens_seen": 1335554048 + }, + { + "epoch": 4.05, + "learning_rate": 0.00030069207622868604, + "loss": 2.7728, + "theoretical_loss": 3.5526547828369885, + "tokens_seen": 1335619584 + }, + { + "epoch": 4.05, + "learning_rate": 0.0003006820461384153, + "loss": 2.7509, + "theoretical_loss": 3.5526390692595413, + "tokens_seen": 1335685120 + }, + { + "epoch": 4.05, + "learning_rate": 0.0003006720160481444, + "loss": 2.7018, + "theoretical_loss": 3.552623356668935, + "tokens_seen": 1335750656 + }, + { + "epoch": 4.05, + "learning_rate": 0.00030066198595787364, + "loss": 2.6671, + "theoretical_loss": 3.552607645065059, + "tokens_seen": 1335816192 + }, + { + "epoch": 4.05, + "learning_rate": 0.0003006519558676028, + "loss": 2.7556, + "theoretical_loss": 3.552591934447804, + "tokens_seen": 1335881728 + }, + { + "epoch": 4.05, + "learning_rate": 0.000300641925777332, + "loss": 2.8379, + "theoretical_loss": 3.5525762248170585, + "tokens_seen": 1335947264 + }, + { + "epoch": 4.05, + "learning_rate": 0.0003006318956870612, + "loss": 2.5638, + "theoretical_loss": 3.552560516172713, + "tokens_seen": 1336012800 + }, + { + "epoch": 4.05, + "learning_rate": 0.00030062186559679036, + "loss": 2.7779, + "theoretical_loss": 3.552544808514657, + "tokens_seen": 1336078336 + }, + { + "epoch": 4.05, + "learning_rate": 0.00030061183550651955, + "loss": 2.6395, + "theoretical_loss": 3.5525291018427803, + "tokens_seen": 1336143872 + }, + { + "epoch": 4.05, + "learning_rate": 0.0003006018054162488, + "loss": 2.7103, + "theoretical_loss": 3.5525133961569715, + "tokens_seen": 1336209408 + }, + { + "epoch": 4.05, + "learning_rate": 0.0003005917753259779, + "loss": 2.5746, + "theoretical_loss": 3.5524976914571216, + "tokens_seen": 1336274944 + }, + { + "epoch": 4.05, + "learning_rate": 0.00030058174523570714, + "loss": 2.5816, + "theoretical_loss": 3.5524819877431204, + "tokens_seen": 1336340480 + }, + { + "epoch": 4.05, + "learning_rate": 0.00030057171514543627, + "loss": 2.7479, + "theoretical_loss": 3.552466285014857, + "tokens_seen": 1336406016 + }, + { + "epoch": 4.05, + "learning_rate": 0.0003005616850551655, + "loss": 2.7497, + "theoretical_loss": 3.5524505832722215, + "tokens_seen": 1336471552 + }, + { + "epoch": 4.05, + "learning_rate": 0.0003005516549648947, + "loss": 2.7455, + "theoretical_loss": 3.5524348825151035, + "tokens_seen": 1336537088 + }, + { + "epoch": 4.05, + "learning_rate": 0.00030054162487462387, + "loss": 2.5357, + "theoretical_loss": 3.5524191827433933, + "tokens_seen": 1336602624 + }, + { + "epoch": 4.05, + "learning_rate": 0.00030053159478435305, + "loss": 2.6763, + "theoretical_loss": 3.5524034839569802, + "tokens_seen": 1336668160 + }, + { + "epoch": 4.05, + "learning_rate": 0.0003005215646940823, + "loss": 2.6913, + "theoretical_loss": 3.5523877861557542, + "tokens_seen": 1336733696 + }, + { + "epoch": 4.05, + "learning_rate": 0.0003005115346038114, + "loss": 2.693, + "theoretical_loss": 3.552372089339605, + "tokens_seen": 1336799232 + }, + { + "epoch": 4.05, + "learning_rate": 0.00030050150451354065, + "loss": 2.6484, + "theoretical_loss": 3.5523563935084232, + "tokens_seen": 1336864768 + }, + { + "epoch": 4.05, + "objective/train/docs_used": 1506128, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.197422742843628, + "objective/train/theoretical_loss": 3.552340698662098, + "objective/train/tokens_used": 1357390304, + "theoretical_loss": 3.552340698662098, + "tokens_seen": 1336930304 + }, + { + "epoch": 4.05, + "learning_rate": 0.0003004914744232698, + "loss": 2.6837, + "theoretical_loss": 3.552340698662098, + "tokens_seen": 1336930304 + }, + { + "epoch": 4.05, + "learning_rate": 0.000300481444332999, + "loss": 2.4243, + "theoretical_loss": 3.5523250048005197, + "tokens_seen": 1336995840 + }, + { + "epoch": 4.05, + "learning_rate": 0.0003004714142427282, + "loss": 2.6001, + "theoretical_loss": 3.5523093119235782, + "tokens_seen": 1337061376 + }, + { + "epoch": 4.05, + "learning_rate": 0.00030046138415245737, + "loss": 2.7931, + "theoretical_loss": 3.5522936200311634, + "tokens_seen": 1337126912 + }, + { + "epoch": 4.05, + "learning_rate": 0.00030045135406218655, + "loss": 2.5818, + "theoretical_loss": 3.552277929123165, + "tokens_seen": 1337192448 + }, + { + "epoch": 4.05, + "learning_rate": 0.00030044132397191573, + "loss": 2.7045, + "theoretical_loss": 3.5522622391994734, + "tokens_seen": 1337257984 + }, + { + "epoch": 4.05, + "learning_rate": 0.0003004312938816449, + "loss": 2.6828, + "theoretical_loss": 3.552246550259979, + "tokens_seen": 1337323520 + }, + { + "epoch": 4.05, + "learning_rate": 0.00030042126379137415, + "loss": 2.4758, + "theoretical_loss": 3.552230862304571, + "tokens_seen": 1337389056 + }, + { + "epoch": 4.05, + "learning_rate": 0.0003004112337011033, + "loss": 2.7366, + "theoretical_loss": 3.55221517533314, + "tokens_seen": 1337454592 + }, + { + "epoch": 4.05, + "learning_rate": 0.0003004012036108325, + "loss": 2.8237, + "theoretical_loss": 3.5521994893455755, + "tokens_seen": 1337520128 + }, + { + "epoch": 4.05, + "learning_rate": 0.0003003911735205617, + "loss": 2.7176, + "theoretical_loss": 3.5521838043417686, + "tokens_seen": 1337585664 + }, + { + "epoch": 4.05, + "learning_rate": 0.0003003811434302909, + "loss": 2.6029, + "theoretical_loss": 3.552168120321608, + "tokens_seen": 1337651200 + }, + { + "epoch": 4.05, + "learning_rate": 0.00030037111334002006, + "loss": 2.7121, + "theoretical_loss": 3.5521524372849846, + "tokens_seen": 1337716736 + }, + { + "epoch": 4.05, + "learning_rate": 0.00030036108324974924, + "loss": 2.8973, + "theoretical_loss": 3.552136755231789, + "tokens_seen": 1337782272 + }, + { + "epoch": 4.05, + "learning_rate": 0.0003003510531594784, + "loss": 2.8893, + "theoretical_loss": 3.552121074161911, + "tokens_seen": 1337847808 + }, + { + "epoch": 4.05, + "learning_rate": 0.00030034102306920765, + "loss": 2.7755, + "theoretical_loss": 3.5521053940752405, + "tokens_seen": 1337913344 + }, + { + "epoch": 4.05, + "learning_rate": 0.0003003309929789368, + "loss": 2.8843, + "theoretical_loss": 3.552089714971668, + "tokens_seen": 1337978880 + }, + { + "epoch": 4.05, + "learning_rate": 0.000300320962888666, + "loss": 2.6604, + "theoretical_loss": 3.5520740368510832, + "tokens_seen": 1338044416 + }, + { + "epoch": 4.05, + "learning_rate": 0.00030031093279839514, + "loss": 2.7421, + "theoretical_loss": 3.552058359713377, + "tokens_seen": 1338109952 + }, + { + "epoch": 4.05, + "learning_rate": 0.0003003009027081244, + "loss": 2.6681, + "theoretical_loss": 3.5520426835584393, + "tokens_seen": 1338175488 + }, + { + "epoch": 4.05, + "learning_rate": 0.0003002908726178536, + "loss": 2.7552, + "theoretical_loss": 3.552027008386161, + "tokens_seen": 1338241024 + }, + { + "epoch": 4.05, + "learning_rate": 0.00030028084252758274, + "loss": 2.633, + "theoretical_loss": 3.552011334196431, + "tokens_seen": 1338306560 + }, + { + "epoch": 4.05, + "learning_rate": 0.000300270812437312, + "loss": 2.6558, + "theoretical_loss": 3.5519956609891405, + "tokens_seen": 1338372096 + }, + { + "epoch": 4.05, + "learning_rate": 0.0003002607823470411, + "loss": 2.635, + "theoretical_loss": 3.55197998876418, + "tokens_seen": 1338437632 + }, + { + "epoch": 4.05, + "learning_rate": 0.00030025075225677034, + "loss": 2.8198, + "theoretical_loss": 3.5519643175214397, + "tokens_seen": 1338503168 + }, + { + "epoch": 4.05, + "objective/train/docs_used": 1506680, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8979873657226562, + "objective/train/theoretical_loss": 3.5519486472608097, + "objective/train/tokens_used": 1359028704, + "theoretical_loss": 3.5519486472608097, + "tokens_seen": 1338568704 + }, + { + "epoch": 4.05, + "learning_rate": 0.0003002407221664995, + "loss": 3.014, + "theoretical_loss": 3.5519486472608097, + "tokens_seen": 1338568704 + }, + { + "epoch": 4.05, + "learning_rate": 0.0003002306920762287, + "loss": 2.7011, + "theoretical_loss": 3.5519329779821804, + "tokens_seen": 1338634240 + }, + { + "epoch": 4.05, + "learning_rate": 0.0003002206619859579, + "loss": 2.7054, + "theoretical_loss": 3.5519173096854426, + "tokens_seen": 1338699776 + }, + { + "epoch": 4.05, + "learning_rate": 0.0003002106318956871, + "loss": 2.6003, + "theoretical_loss": 3.551901642370486, + "tokens_seen": 1338765312 + }, + { + "epoch": 4.05, + "learning_rate": 0.00030020060180541624, + "loss": 2.5723, + "theoretical_loss": 3.5518859760372017, + "tokens_seen": 1338830848 + }, + { + "epoch": 4.05, + "learning_rate": 0.0003001905717151455, + "loss": 2.5742, + "theoretical_loss": 3.55187031068548, + "tokens_seen": 1338896384 + }, + { + "epoch": 4.05, + "learning_rate": 0.0003001805416248746, + "loss": 2.903, + "theoretical_loss": 3.5518546463152108, + "tokens_seen": 1338961920 + }, + { + "epoch": 4.05, + "learning_rate": 0.00030017051153460384, + "loss": 2.753, + "theoretical_loss": 3.5518389829262853, + "tokens_seen": 1339027456 + }, + { + "epoch": 4.05, + "learning_rate": 0.000300160481444333, + "loss": 2.5019, + "theoretical_loss": 3.5518233205185936, + "tokens_seen": 1339092992 + }, + { + "epoch": 4.05, + "learning_rate": 0.0003001504513540622, + "loss": 2.7562, + "theoretical_loss": 3.5518076590920264, + "tokens_seen": 1339158528 + }, + { + "epoch": 4.05, + "learning_rate": 0.0003001404212637914, + "loss": 2.602, + "theoretical_loss": 3.5517919986464745, + "tokens_seen": 1339224064 + }, + { + "epoch": 4.05, + "learning_rate": 0.00030013039117352056, + "loss": 2.6566, + "theoretical_loss": 3.551776339181828, + "tokens_seen": 1339289600 + }, + { + "epoch": 4.05, + "learning_rate": 0.00030012036108324975, + "loss": 2.5779, + "theoretical_loss": 3.551760680697977, + "tokens_seen": 1339355136 + }, + { + "epoch": 4.05, + "learning_rate": 0.000300110330992979, + "loss": 2.7822, + "theoretical_loss": 3.5517450231948127, + "tokens_seen": 1339420672 + }, + { + "epoch": 4.05, + "learning_rate": 0.0003001003009027081, + "loss": 2.9438, + "theoretical_loss": 3.551729366672226, + "tokens_seen": 1339486208 + }, + { + "epoch": 4.05, + "learning_rate": 0.00030009027081243734, + "loss": 2.7497, + "theoretical_loss": 3.551713711130107, + "tokens_seen": 1339551744 + }, + { + "epoch": 4.05, + "learning_rate": 0.00030008024072216647, + "loss": 2.8555, + "theoretical_loss": 3.5516980565683465, + "tokens_seen": 1339617280 + }, + { + "epoch": 4.05, + "learning_rate": 0.0003000702106318957, + "loss": 2.4172, + "theoretical_loss": 3.5516824029868355, + "tokens_seen": 1339682816 + }, + { + "epoch": 4.05, + "learning_rate": 0.0003000601805416249, + "loss": 2.5359, + "theoretical_loss": 3.551666750385464, + "tokens_seen": 1339748352 + }, + { + "epoch": 4.05, + "learning_rate": 0.00030005015045135407, + "loss": 2.7497, + "theoretical_loss": 3.5516510987641228, + "tokens_seen": 1339813888 + }, + { + "epoch": 4.05, + "learning_rate": 0.00030004012036108325, + "loss": 2.9051, + "theoretical_loss": 3.5516354481227035, + "tokens_seen": 1339879424 + }, + { + "epoch": 4.05, + "learning_rate": 0.0003000300902708125, + "loss": 2.5491, + "theoretical_loss": 3.551619798461095, + "tokens_seen": 1339944960 + }, + { + "epoch": 4.05, + "learning_rate": 0.0003000200601805416, + "loss": 2.9152, + "theoretical_loss": 3.55160414977919, + "tokens_seen": 1340010496 + }, + { + "epoch": 4.05, + "learning_rate": 0.00030001003009027085, + "loss": 2.7579, + "theoretical_loss": 3.5515885020768785, + "tokens_seen": 1340076032 + }, + { + "epoch": 4.05, + "learning_rate": 0.0003, + "loss": 2.7868, + "theoretical_loss": 3.551572855354051, + "tokens_seen": 1340141568 + }, + { + "epoch": 4.05, + "objective/train/docs_used": 1506682, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5435173511505127, + "objective/train/theoretical_loss": 3.551557209610599, + "objective/train/tokens_used": 1360667104, + "theoretical_loss": 3.551557209610599, + "tokens_seen": 1340207104 + }, + { + "epoch": 4.05, + "learning_rate": 0.0002999899699097292, + "loss": 2.5745, + "theoretical_loss": 3.551557209610599, + "tokens_seen": 1340207104 + }, + { + "epoch": 4.05, + "learning_rate": 0.0002999799398194584, + "loss": 2.315, + "theoretical_loss": 3.551541564846412, + "tokens_seen": 1340272640 + }, + { + "epoch": 4.05, + "learning_rate": 0.00029996990972918757, + "loss": 2.8512, + "theoretical_loss": 3.551525921061382, + "tokens_seen": 1340338176 + }, + { + "epoch": 4.05, + "learning_rate": 0.00029995987963891675, + "loss": 2.7299, + "theoretical_loss": 3.5515102782553996, + "tokens_seen": 1340403712 + }, + { + "epoch": 4.05, + "learning_rate": 0.00029994984954864593, + "loss": 2.7094, + "theoretical_loss": 3.5514946364283557, + "tokens_seen": 1340469248 + }, + { + "epoch": 4.05, + "learning_rate": 0.0002999398194583751, + "loss": 2.6636, + "theoretical_loss": 3.5514789955801405, + "tokens_seen": 1340534784 + }, + { + "epoch": 4.05, + "learning_rate": 0.00029992978936810435, + "loss": 2.7232, + "theoretical_loss": 3.551463355710646, + "tokens_seen": 1340600320 + }, + { + "epoch": 4.05, + "learning_rate": 0.0002999197592778335, + "loss": 2.6671, + "theoretical_loss": 3.5514477168197622, + "tokens_seen": 1340665856 + }, + { + "epoch": 4.05, + "learning_rate": 0.0002999097291875627, + "loss": 2.8143, + "theoretical_loss": 3.5514320789073808, + "tokens_seen": 1340731392 + }, + { + "epoch": 4.05, + "learning_rate": 0.0002998996990972919, + "loss": 2.4735, + "theoretical_loss": 3.551416441973392, + "tokens_seen": 1340796928 + }, + { + "epoch": 4.05, + "learning_rate": 0.0002998896690070211, + "loss": 2.7152, + "theoretical_loss": 3.5514008060176874, + "tokens_seen": 1340862464 + }, + { + "epoch": 4.05, + "learning_rate": 0.00029987963891675026, + "loss": 2.6733, + "theoretical_loss": 3.5513851710401574, + "tokens_seen": 1340928000 + }, + { + "epoch": 4.05, + "learning_rate": 0.00029986960882647944, + "loss": 2.8356, + "theoretical_loss": 3.551369537040693, + "tokens_seen": 1340993536 + }, + { + "epoch": 4.05, + "learning_rate": 0.0002998595787362086, + "loss": 2.7239, + "theoretical_loss": 3.5513539040191864, + "tokens_seen": 1341059072 + }, + { + "epoch": 4.05, + "learning_rate": 0.00029984954864593785, + "loss": 2.6926, + "theoretical_loss": 3.5513382719755273, + "tokens_seen": 1341124608 + }, + { + "epoch": 4.05, + "learning_rate": 0.000299839518555667, + "loss": 2.537, + "theoretical_loss": 3.551322640909607, + "tokens_seen": 1341190144 + }, + { + "epoch": 4.05, + "learning_rate": 0.0002998294884653962, + "loss": 2.826, + "theoretical_loss": 3.551307010821317, + "tokens_seen": 1341255680 + }, + { + "epoch": 4.05, + "learning_rate": 0.00029981945837512534, + "loss": 2.705, + "theoretical_loss": 3.5512913817105485, + "tokens_seen": 1341321216 + }, + { + "epoch": 4.05, + "learning_rate": 0.0002998094282848546, + "loss": 2.6646, + "theoretical_loss": 3.5512757535771917, + "tokens_seen": 1341386752 + }, + { + "epoch": 4.05, + "learning_rate": 0.00029979939819458376, + "loss": 2.6163, + "theoretical_loss": 3.5512601264211385, + "tokens_seen": 1341452288 + }, + { + "epoch": 4.05, + "learning_rate": 0.00029978936810431294, + "loss": 2.6182, + "theoretical_loss": 3.55124450024228, + "tokens_seen": 1341517824 + }, + { + "epoch": 4.05, + "learning_rate": 0.0002997793380140421, + "loss": 2.6714, + "theoretical_loss": 3.551228875040507, + "tokens_seen": 1341583360 + }, + { + "epoch": 4.05, + "learning_rate": 0.0002997693079237713, + "loss": 2.7248, + "theoretical_loss": 3.551213250815711, + "tokens_seen": 1341648896 + }, + { + "epoch": 4.05, + "learning_rate": 0.0002997592778335005, + "loss": 2.832, + "theoretical_loss": 3.5511976275677837, + "tokens_seen": 1341714432 + }, + { + "epoch": 4.05, + "learning_rate": 0.0002997492477432297, + "loss": 2.7048, + "theoretical_loss": 3.551182005296615, + "tokens_seen": 1341779968 + }, + { + "epoch": 4.05, + "objective/train/docs_used": 1507446, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8023407459259033, + "objective/train/theoretical_loss": 3.551166384002097, + "objective/train/tokens_used": 1362305504, + "theoretical_loss": 3.551166384002097, + "tokens_seen": 1341845504 + }, + { + "epoch": 4.05, + "learning_rate": 0.00029973921765295885, + "loss": 2.6677, + "theoretical_loss": 3.551166384002097, + "tokens_seen": 1341845504 + }, + { + "epoch": 4.05, + "learning_rate": 0.0002997291875626881, + "loss": 2.635, + "theoretical_loss": 3.5511507636841206, + "tokens_seen": 1341911040 + }, + { + "epoch": 4.05, + "learning_rate": 0.00029971915747241726, + "loss": 2.8049, + "theoretical_loss": 3.5511351443425774, + "tokens_seen": 1341976576 + }, + { + "epoch": 4.05, + "learning_rate": 0.00029970912738214644, + "loss": 2.6777, + "theoretical_loss": 3.5511195259773585, + "tokens_seen": 1342042112 + }, + { + "epoch": 4.05, + "learning_rate": 0.0002996990972918756, + "loss": 2.8039, + "theoretical_loss": 3.551103908588355, + "tokens_seen": 1342107648 + }, + { + "epoch": 4.05, + "learning_rate": 0.0002996890672016048, + "loss": 2.7576, + "theoretical_loss": 3.551088292175459, + "tokens_seen": 1342173184 + }, + { + "epoch": 4.05, + "learning_rate": 0.000299679037111334, + "loss": 2.5265, + "theoretical_loss": 3.551072676738561, + "tokens_seen": 1342238720 + }, + { + "epoch": 4.05, + "learning_rate": 0.0002996690070210632, + "loss": 2.6238, + "theoretical_loss": 3.551057062277553, + "tokens_seen": 1342304256 + }, + { + "epoch": 4.05, + "learning_rate": 0.00029965897693079235, + "loss": 2.8529, + "theoretical_loss": 3.5510414487923256, + "tokens_seen": 1342369792 + }, + { + "epoch": 4.05, + "learning_rate": 0.0002996489468405216, + "loss": 2.7188, + "theoretical_loss": 3.5510258362827707, + "tokens_seen": 1342435328 + }, + { + "epoch": 4.05, + "learning_rate": 0.0002996389167502507, + "loss": 2.469, + "theoretical_loss": 3.5510102247487794, + "tokens_seen": 1342500864 + }, + { + "epoch": 4.05, + "learning_rate": 0.00029962888665997995, + "loss": 2.737, + "theoretical_loss": 3.5509946141902433, + "tokens_seen": 1342566400 + }, + { + "epoch": 4.05, + "learning_rate": 0.00029961885656970913, + "loss": 2.7072, + "theoretical_loss": 3.550979004607054, + "tokens_seen": 1342631936 + }, + { + "epoch": 4.05, + "learning_rate": 0.0002996088264794383, + "loss": 2.8882, + "theoretical_loss": 3.550963395999103, + "tokens_seen": 1342697472 + }, + { + "epoch": 4.05, + "learning_rate": 0.0002995987963891675, + "loss": 2.6852, + "theoretical_loss": 3.5509477883662814, + "tokens_seen": 1342763008 + }, + { + "epoch": 4.05, + "learning_rate": 0.00029958876629889667, + "loss": 2.8004, + "theoretical_loss": 3.550932181708481, + "tokens_seen": 1342828544 + }, + { + "epoch": 4.05, + "learning_rate": 0.00029957873620862585, + "loss": 2.566, + "theoretical_loss": 3.5509165760255934, + "tokens_seen": 1342894080 + }, + { + "epoch": 4.05, + "learning_rate": 0.0002995687061183551, + "loss": 2.4521, + "theoretical_loss": 3.5509009713175095, + "tokens_seen": 1342959616 + }, + { + "epoch": 4.05, + "learning_rate": 0.0002995586760280842, + "loss": 2.9367, + "theoretical_loss": 3.5508853675841214, + "tokens_seen": 1343025152 + }, + { + "epoch": 4.05, + "learning_rate": 0.00029954864593781345, + "loss": 2.634, + "theoretical_loss": 3.5508697648253205, + "tokens_seen": 1343090688 + }, + { + "epoch": 4.05, + "learning_rate": 0.0002995386158475427, + "loss": 2.6086, + "theoretical_loss": 3.5508541630409987, + "tokens_seen": 1343156224 + }, + { + "epoch": 4.05, + "learning_rate": 0.0002995285857572718, + "loss": 2.542, + "theoretical_loss": 3.5508385622310468, + "tokens_seen": 1343221760 + }, + { + "epoch": 4.05, + "learning_rate": 0.00029951855566700105, + "loss": 2.6771, + "theoretical_loss": 3.550822962395357, + "tokens_seen": 1343287296 + }, + { + "epoch": 4.05, + "learning_rate": 0.0002995085255767302, + "loss": 2.6823, + "theoretical_loss": 3.5508073635338206, + "tokens_seen": 1343352832 + }, + { + "epoch": 4.05, + "learning_rate": 0.0002994984954864594, + "loss": 2.4627, + "theoretical_loss": 3.55079176564633, + "tokens_seen": 1343418368 + }, + { + "epoch": 4.05, + "objective/train/docs_used": 1508187, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9692583084106445, + "objective/train/theoretical_loss": 3.550776168732776, + "objective/train/tokens_used": 1363943904, + "theoretical_loss": 3.550776168732776, + "tokens_seen": 1343483904 + }, + { + "epoch": 4.05, + "learning_rate": 0.0002994884653961886, + "loss": 2.5772, + "theoretical_loss": 3.550776168732776, + "tokens_seen": 1343483904 + }, + { + "epoch": 4.05, + "learning_rate": 0.00029947843530591777, + "loss": 2.6239, + "theoretical_loss": 3.550760572793051, + "tokens_seen": 1343549440 + }, + { + "epoch": 4.05, + "learning_rate": 0.00029946840521564695, + "loss": 2.4927, + "theoretical_loss": 3.5507449778270463, + "tokens_seen": 1343614976 + }, + { + "epoch": 4.05, + "learning_rate": 0.00029945837512537613, + "loss": 2.5679, + "theoretical_loss": 3.5507293838346534, + "tokens_seen": 1343680512 + }, + { + "epoch": 4.05, + "learning_rate": 0.0002994483450351053, + "loss": 2.9101, + "theoretical_loss": 3.550713790815765, + "tokens_seen": 1343746048 + }, + { + "epoch": 4.05, + "learning_rate": 0.00029943831494483455, + "loss": 2.6945, + "theoretical_loss": 3.550698198770271, + "tokens_seen": 1343811584 + }, + { + "epoch": 4.05, + "learning_rate": 0.0002994282848545637, + "loss": 2.7543, + "theoretical_loss": 3.550682607698065, + "tokens_seen": 1343877120 + }, + { + "epoch": 4.05, + "learning_rate": 0.0002994182547642929, + "loss": 2.6513, + "theoretical_loss": 3.5506670175990376, + "tokens_seen": 1343942656 + }, + { + "epoch": 4.05, + "learning_rate": 0.0002994082246740221, + "loss": 2.7973, + "theoretical_loss": 3.5506514284730812, + "tokens_seen": 1344008192 + }, + { + "epoch": 4.05, + "learning_rate": 0.0002993981945837513, + "loss": 2.803, + "theoretical_loss": 3.550635840320088, + "tokens_seen": 1344073728 + }, + { + "epoch": 4.05, + "learning_rate": 0.00029938816449348046, + "loss": 2.5209, + "theoretical_loss": 3.5506202531399493, + "tokens_seen": 1344139264 + }, + { + "epoch": 4.05, + "learning_rate": 0.00029937813440320964, + "loss": 2.601, + "theoretical_loss": 3.5506046669325566, + "tokens_seen": 1344204800 + }, + { + "epoch": 4.05, + "learning_rate": 0.0002993681043129388, + "loss": 2.5979, + "theoretical_loss": 3.5505890816978027, + "tokens_seen": 1344270336 + }, + { + "epoch": 4.05, + "learning_rate": 0.00029935807422266805, + "loss": 2.7865, + "theoretical_loss": 3.550573497435578, + "tokens_seen": 1344335872 + }, + { + "epoch": 4.05, + "learning_rate": 0.0002993480441323972, + "loss": 2.6167, + "theoretical_loss": 3.5505579141457764, + "tokens_seen": 1344401408 + }, + { + "epoch": 4.05, + "learning_rate": 0.0002993380140421264, + "loss": 2.5938, + "theoretical_loss": 3.5505423318282885, + "tokens_seen": 1344466944 + }, + { + "epoch": 4.05, + "learning_rate": 0.00029932798395185554, + "loss": 2.6116, + "theoretical_loss": 3.5505267504830065, + "tokens_seen": 1344532480 + }, + { + "epoch": 4.05, + "learning_rate": 0.0002993179538615848, + "loss": 2.6737, + "theoretical_loss": 3.5505111701098224, + "tokens_seen": 1344598016 + }, + { + "epoch": 4.05, + "learning_rate": 0.00029930792377131396, + "loss": 2.6748, + "theoretical_loss": 3.550495590708628, + "tokens_seen": 1344663552 + }, + { + "epoch": 4.05, + "learning_rate": 0.00029929789368104314, + "loss": 2.6336, + "theoretical_loss": 3.5504800122793156, + "tokens_seen": 1344729088 + }, + { + "epoch": 4.05, + "learning_rate": 0.0002992878635907723, + "loss": 2.6307, + "theoretical_loss": 3.550464434821777, + "tokens_seen": 1344794624 + }, + { + "epoch": 4.05, + "learning_rate": 0.0002992778335005015, + "loss": 2.6766, + "theoretical_loss": 3.5504488583359044, + "tokens_seen": 1344860160 + }, + { + "epoch": 4.05, + "learning_rate": 0.0002992678034102307, + "loss": 2.5858, + "theoretical_loss": 3.55043328282159, + "tokens_seen": 1344925696 + }, + { + "epoch": 4.05, + "learning_rate": 0.0002992577733199599, + "loss": 2.8844, + "theoretical_loss": 3.5504177082787254, + "tokens_seen": 1344991232 + }, + { + "epoch": 4.05, + "learning_rate": 0.00029924774322968905, + "loss": 2.7651, + "theoretical_loss": 3.550402134707203, + "tokens_seen": 1345056768 + }, + { + "epoch": 4.05, + "objective/train/docs_used": 1509426, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.554690361022949, + "objective/train/theoretical_loss": 3.5503865621069144, + "objective/train/tokens_used": 1365582304, + "theoretical_loss": 3.5503865621069144, + "tokens_seen": 1345122304 + }, + { + "epoch": 4.05, + "learning_rate": 0.0002992377131394183, + "loss": 2.6236, + "theoretical_loss": 3.5503865621069144, + "tokens_seen": 1345122304 + }, + { + "epoch": 4.05, + "learning_rate": 0.00029922768304914746, + "loss": 2.7252, + "theoretical_loss": 3.5503709904777523, + "tokens_seen": 1345187840 + }, + { + "epoch": 4.05, + "learning_rate": 0.00029921765295887664, + "loss": 2.8637, + "theoretical_loss": 3.550355419819609, + "tokens_seen": 1345253376 + }, + { + "epoch": 4.05, + "learning_rate": 0.0002992076228686058, + "loss": 2.5164, + "theoretical_loss": 3.550339850132376, + "tokens_seen": 1345318912 + }, + { + "epoch": 4.05, + "learning_rate": 0.000299197592778335, + "loss": 2.7611, + "theoretical_loss": 3.550324281415946, + "tokens_seen": 1345384448 + }, + { + "epoch": 4.05, + "learning_rate": 0.0002991875626880642, + "loss": 2.8538, + "theoretical_loss": 3.5503087136702103, + "tokens_seen": 1345449984 + }, + { + "epoch": 4.05, + "learning_rate": 0.0002991775325977934, + "loss": 2.5655, + "theoretical_loss": 3.5502931468950623, + "tokens_seen": 1345515520 + }, + { + "epoch": 4.05, + "learning_rate": 0.00029916750250752255, + "loss": 2.8136, + "theoretical_loss": 3.550277581090394, + "tokens_seen": 1345581056 + }, + { + "epoch": 4.05, + "learning_rate": 0.0002991574724172518, + "loss": 2.8745, + "theoretical_loss": 3.5502620162560965, + "tokens_seen": 1345646592 + }, + { + "epoch": 4.05, + "learning_rate": 0.0002991474423269809, + "loss": 2.728, + "theoretical_loss": 3.550246452392063, + "tokens_seen": 1345712128 + }, + { + "epoch": 4.05, + "learning_rate": 0.00029913741223671015, + "loss": 2.765, + "theoretical_loss": 3.550230889498186, + "tokens_seen": 1345777664 + }, + { + "epoch": 4.05, + "learning_rate": 0.00029912738214643933, + "loss": 2.8291, + "theoretical_loss": 3.550215327574357, + "tokens_seen": 1345843200 + }, + { + "epoch": 4.05, + "learning_rate": 0.0002991173520561685, + "loss": 2.4863, + "theoretical_loss": 3.550199766620469, + "tokens_seen": 1345908736 + }, + { + "epoch": 4.05, + "learning_rate": 0.0002991073219658977, + "loss": 2.7966, + "theoretical_loss": 3.550184206636414, + "tokens_seen": 1345974272 + }, + { + "epoch": 4.05, + "learning_rate": 0.00029909729187562687, + "loss": 2.6954, + "theoretical_loss": 3.550168647622084, + "tokens_seen": 1346039808 + }, + { + "epoch": 4.05, + "learning_rate": 0.00029908726178535605, + "loss": 2.8217, + "theoretical_loss": 3.550153089577372, + "tokens_seen": 1346105344 + }, + { + "epoch": 4.05, + "learning_rate": 0.0002990772316950853, + "loss": 2.6436, + "theoretical_loss": 3.5501375325021707, + "tokens_seen": 1346170880 + }, + { + "epoch": 4.05, + "learning_rate": 0.0002990672016048144, + "loss": 2.8473, + "theoretical_loss": 3.550121976396371, + "tokens_seen": 1346236416 + }, + { + "epoch": 4.05, + "learning_rate": 0.00029905717151454365, + "loss": 2.7741, + "theoretical_loss": 3.5501064212598665, + "tokens_seen": 1346301952 + }, + { + "epoch": 4.05, + "learning_rate": 0.00029904714142427283, + "loss": 2.477, + "theoretical_loss": 3.5500908670925493, + "tokens_seen": 1346367488 + }, + { + "epoch": 4.05, + "learning_rate": 0.000299037111334002, + "loss": 2.5492, + "theoretical_loss": 3.550075313894312, + "tokens_seen": 1346433024 + }, + { + "epoch": 4.05, + "learning_rate": 0.0002990270812437312, + "loss": 2.7839, + "theoretical_loss": 3.5500597616650467, + "tokens_seen": 1346498560 + }, + { + "epoch": 4.05, + "learning_rate": 0.0002990170511534604, + "loss": 2.6982, + "theoretical_loss": 3.5500442104046463, + "tokens_seen": 1346564096 + }, + { + "epoch": 4.05, + "learning_rate": 0.00029900702106318956, + "loss": 2.6283, + "theoretical_loss": 3.5500286601130027, + "tokens_seen": 1346629632 + }, + { + "epoch": 4.05, + "learning_rate": 0.0002989969909729188, + "loss": 2.6279, + "theoretical_loss": 3.550013110790009, + "tokens_seen": 1346695168 + }, + { + "epoch": 4.05, + "objective/train/docs_used": 1510013, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7701847553253174, + "objective/train/theoretical_loss": 3.549997562435558, + "objective/train/tokens_used": 1367220704, + "theoretical_loss": 3.549997562435558, + "tokens_seen": 1346760704 + }, + { + "epoch": 4.05, + "learning_rate": 0.0002989869608826479, + "loss": 2.6528, + "theoretical_loss": 3.549997562435558, + "tokens_seen": 1346760704 + }, + { + "epoch": 4.05, + "learning_rate": 0.00029897693079237715, + "loss": 2.8473, + "theoretical_loss": 3.549982015049541, + "tokens_seen": 1346826240 + }, + { + "epoch": 4.05, + "learning_rate": 0.0002989669007021063, + "loss": 2.8033, + "theoretical_loss": 3.549966468631852, + "tokens_seen": 1346891776 + }, + { + "epoch": 4.05, + "learning_rate": 0.0002989568706118355, + "loss": 2.8413, + "theoretical_loss": 3.5499509231823825, + "tokens_seen": 1346957312 + }, + { + "epoch": 4.05, + "learning_rate": 0.0002989468405215647, + "loss": 2.7241, + "theoretical_loss": 3.5499353787010257, + "tokens_seen": 1347022848 + }, + { + "epoch": 4.05, + "learning_rate": 0.0002989368104312939, + "loss": 2.5255, + "theoretical_loss": 3.5499198351876737, + "tokens_seen": 1347088384 + }, + { + "epoch": 4.05, + "learning_rate": 0.00029892678034102306, + "loss": 2.782, + "theoretical_loss": 3.5499042926422195, + "tokens_seen": 1347153920 + }, + { + "epoch": 4.05, + "learning_rate": 0.0002989167502507523, + "loss": 2.6674, + "theoretical_loss": 3.549888751064556, + "tokens_seen": 1347219456 + }, + { + "epoch": 4.05, + "learning_rate": 0.0002989067201604814, + "loss": 2.7379, + "theoretical_loss": 3.5498732104545754, + "tokens_seen": 1347284992 + }, + { + "epoch": 4.05, + "learning_rate": 0.00029889669007021066, + "loss": 2.7572, + "theoretical_loss": 3.5498576708121705, + "tokens_seen": 1347350528 + }, + { + "epoch": 4.05, + "learning_rate": 0.0002988866599799398, + "loss": 2.8585, + "theoretical_loss": 3.549842132137234, + "tokens_seen": 1347416064 + }, + { + "epoch": 4.05, + "learning_rate": 0.000298876629889669, + "loss": 2.6141, + "theoretical_loss": 3.5498265944296588, + "tokens_seen": 1347481600 + }, + { + "epoch": 4.05, + "learning_rate": 0.0002988665997993982, + "loss": 2.6897, + "theoretical_loss": 3.549811057689337, + "tokens_seen": 1347547136 + }, + { + "epoch": 4.05, + "learning_rate": 0.0002988565697091274, + "loss": 2.9036, + "theoretical_loss": 3.5497955219161623, + "tokens_seen": 1347612672 + }, + { + "epoch": 4.05, + "learning_rate": 0.00029884653961885656, + "loss": 2.8636, + "theoretical_loss": 3.549779987110027, + "tokens_seen": 1347678208 + }, + { + "epoch": 4.05, + "learning_rate": 0.00029883650952858574, + "loss": 2.7638, + "theoretical_loss": 3.549764453270824, + "tokens_seen": 1347743744 + }, + { + "epoch": 4.05, + "learning_rate": 0.0002988264794383149, + "loss": 2.8871, + "theoretical_loss": 3.5497489203984456, + "tokens_seen": 1347809280 + }, + { + "epoch": 4.05, + "learning_rate": 0.00029881644934804416, + "loss": 2.513, + "theoretical_loss": 3.5497333884927853, + "tokens_seen": 1347874816 + }, + { + "epoch": 4.05, + "learning_rate": 0.0002988064192577733, + "loss": 2.6083, + "theoretical_loss": 3.549717857553736, + "tokens_seen": 1347940352 + }, + { + "epoch": 4.05, + "learning_rate": 0.0002987963891675025, + "loss": 2.7726, + "theoretical_loss": 3.549702327581189, + "tokens_seen": 1348005888 + }, + { + "epoch": 4.05, + "learning_rate": 0.0002987863590772317, + "loss": 2.8613, + "theoretical_loss": 3.5496867985750393, + "tokens_seen": 1348071424 + }, + { + "epoch": 4.05, + "learning_rate": 0.0002987763289869609, + "loss": 2.7175, + "theoretical_loss": 3.5496712705351787, + "tokens_seen": 1348136960 + }, + { + "epoch": 4.05, + "learning_rate": 0.0002987662988966901, + "loss": 2.8827, + "theoretical_loss": 3.5496557434615, + "tokens_seen": 1348202496 + }, + { + "epoch": 4.05, + "learning_rate": 0.00029875626880641925, + "loss": 2.7725, + "theoretical_loss": 3.549640217353897, + "tokens_seen": 1348268032 + }, + { + "epoch": 4.05, + "learning_rate": 0.0002987462387161485, + "loss": 2.8003, + "theoretical_loss": 3.5496246922122614, + "tokens_seen": 1348333568 + }, + { + "epoch": 4.05, + "objective/train/docs_used": 1511229, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.716949939727783, + "objective/train/theoretical_loss": 3.549609168036487, + "objective/train/tokens_used": 1368859104, + "theoretical_loss": 3.549609168036487, + "tokens_seen": 1348399104 + }, + { + "epoch": 4.05, + "learning_rate": 0.00029873620862587766, + "loss": 2.5916, + "theoretical_loss": 3.549609168036487, + "tokens_seen": 1348399104 + }, + { + "epoch": 4.05, + "learning_rate": 0.00029872617853560684, + "loss": 2.5237, + "theoretical_loss": 3.5495936448264667, + "tokens_seen": 1348464640 + }, + { + "epoch": 4.05, + "learning_rate": 0.000298716148445336, + "loss": 2.7218, + "theoretical_loss": 3.549578122582093, + "tokens_seen": 1348530176 + }, + { + "epoch": 4.05, + "learning_rate": 0.0002987061183550652, + "loss": 2.7952, + "theoretical_loss": 3.549562601303259, + "tokens_seen": 1348595712 + }, + { + "epoch": 4.05, + "learning_rate": 0.0002986960882647944, + "loss": 2.7038, + "theoretical_loss": 3.5495470809898584, + "tokens_seen": 1348661248 + }, + { + "epoch": 4.05, + "learning_rate": 0.0002986860581745236, + "loss": 2.7883, + "theoretical_loss": 3.5495315616417837, + "tokens_seen": 1348726784 + }, + { + "epoch": 4.05, + "learning_rate": 0.00029867602808425275, + "loss": 2.5982, + "theoretical_loss": 3.549516043258928, + "tokens_seen": 1348792320 + }, + { + "epoch": 4.05, + "learning_rate": 0.000298665997993982, + "loss": 2.3621, + "theoretical_loss": 3.549500525841185, + "tokens_seen": 1348857856 + }, + { + "epoch": 4.05, + "learning_rate": 0.0002986559679037111, + "loss": 2.8927, + "theoretical_loss": 3.5494850093884462, + "tokens_seen": 1348923392 + }, + { + "epoch": 4.05, + "learning_rate": 0.00029864593781344035, + "loss": 2.95, + "theoretical_loss": 3.5494694939006064, + "tokens_seen": 1348988928 + }, + { + "epoch": 4.05, + "learning_rate": 0.00029863590772316953, + "loss": 2.5967, + "theoretical_loss": 3.549453979377558, + "tokens_seen": 1349054464 + }, + { + "epoch": 4.05, + "learning_rate": 0.0002986258776328987, + "loss": 2.8127, + "theoretical_loss": 3.549438465819194, + "tokens_seen": 1349120000 + }, + { + "epoch": 4.05, + "learning_rate": 0.0002986158475426279, + "loss": 2.6664, + "theoretical_loss": 3.549422953225408, + "tokens_seen": 1349185536 + }, + { + "epoch": 4.05, + "learning_rate": 0.00029860581745235707, + "loss": 2.7774, + "theoretical_loss": 3.5494074415960926, + "tokens_seen": 1349251072 + }, + { + "epoch": 4.05, + "learning_rate": 0.00029859578736208625, + "loss": 2.5631, + "theoretical_loss": 3.5493919309311415, + "tokens_seen": 1349316608 + }, + { + "epoch": 4.05, + "learning_rate": 0.0002985857572718155, + "loss": 2.6669, + "theoretical_loss": 3.5493764212304475, + "tokens_seen": 1349382144 + }, + { + "epoch": 4.05, + "learning_rate": 0.0002985757271815446, + "loss": 2.6485, + "theoretical_loss": 3.5493609124939045, + "tokens_seen": 1349447680 + }, + { + "epoch": 4.05, + "learning_rate": 0.00029856569709127385, + "loss": 2.6763, + "theoretical_loss": 3.549345404721405, + "tokens_seen": 1349513216 + }, + { + "epoch": 4.05, + "learning_rate": 0.00029855566700100303, + "loss": 2.6191, + "theoretical_loss": 3.549329897912843, + "tokens_seen": 1349578752 + }, + { + "epoch": 4.05, + "learning_rate": 0.0002985456369107322, + "loss": 2.8153, + "theoretical_loss": 3.5493143920681107, + "tokens_seen": 1349644288 + }, + { + "epoch": 4.05, + "learning_rate": 0.0002985356068204614, + "loss": 2.9487, + "theoretical_loss": 3.5492988871871023, + "tokens_seen": 1349709824 + }, + { + "epoch": 4.05, + "learning_rate": 0.0002985255767301906, + "loss": 2.7807, + "theoretical_loss": 3.549283383269711, + "tokens_seen": 1349775360 + }, + { + "epoch": 4.05, + "learning_rate": 0.00029851554663991976, + "loss": 2.4485, + "theoretical_loss": 3.5492678803158295, + "tokens_seen": 1349840896 + }, + { + "epoch": 4.05, + "learning_rate": 0.000298505516549649, + "loss": 2.6957, + "theoretical_loss": 3.5492523783253525, + "tokens_seen": 1349906432 + }, + { + "epoch": 4.05, + "learning_rate": 0.0002984954864593781, + "loss": 2.695, + "theoretical_loss": 3.549236877298172, + "tokens_seen": 1349971968 + }, + { + "epoch": 4.05, + "objective/train/docs_used": 1512083, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.1851322650909424, + "objective/train/theoretical_loss": 3.549221377234182, + "objective/train/tokens_used": 1370497504, + "theoretical_loss": 3.549221377234182, + "tokens_seen": 1350037504 + }, + { + "epoch": 4.05, + "learning_rate": 0.00029848545636910735, + "loss": 2.5258, + "theoretical_loss": 3.549221377234182, + "tokens_seen": 1350037504 + }, + { + "epoch": 4.05, + "learning_rate": 0.0002984754262788365, + "loss": 2.8254, + "theoretical_loss": 3.5492058781332756, + "tokens_seen": 1350103040 + }, + { + "epoch": 4.05, + "learning_rate": 0.0002984653961885657, + "loss": 2.7243, + "theoretical_loss": 3.5491903799953466, + "tokens_seen": 1350168576 + }, + { + "epoch": 4.05, + "learning_rate": 0.0002984553660982949, + "loss": 2.6613, + "theoretical_loss": 3.549174882820288, + "tokens_seen": 1350234112 + }, + { + "epoch": 4.05, + "learning_rate": 0.0002984453360080241, + "loss": 2.676, + "theoretical_loss": 3.549159386607993, + "tokens_seen": 1350299648 + }, + { + "epoch": 4.05, + "learning_rate": 0.00029843530591775326, + "loss": 2.7808, + "theoretical_loss": 3.549143891358357, + "tokens_seen": 1350365184 + }, + { + "epoch": 4.05, + "learning_rate": 0.0002984252758274825, + "loss": 2.6886, + "theoretical_loss": 3.5491283970712706, + "tokens_seen": 1350430720 + }, + { + "epoch": 4.05, + "learning_rate": 0.0002984152457372116, + "loss": 2.4495, + "theoretical_loss": 3.549112903746629, + "tokens_seen": 1350496256 + }, + { + "epoch": 4.05, + "learning_rate": 0.00029840521564694086, + "loss": 2.6895, + "theoretical_loss": 3.549097411384326, + "tokens_seen": 1350561792 + }, + { + "epoch": 4.05, + "learning_rate": 0.00029839518555667, + "loss": 2.8342, + "theoretical_loss": 3.5490819199842543, + "tokens_seen": 1350627328 + }, + { + "epoch": 4.05, + "learning_rate": 0.0002983851554663992, + "loss": 2.4988, + "theoretical_loss": 3.5490664295463077, + "tokens_seen": 1350692864 + }, + { + "epoch": 4.05, + "learning_rate": 0.0002983751253761284, + "loss": 2.7996, + "theoretical_loss": 3.549050940070379, + "tokens_seen": 1350758400 + }, + { + "epoch": 4.05, + "learning_rate": 0.0002983650952858576, + "loss": 2.7887, + "theoretical_loss": 3.549035451556364, + "tokens_seen": 1350823936 + }, + { + "epoch": 4.05, + "learning_rate": 0.00029835506519558676, + "loss": 2.7817, + "theoretical_loss": 3.549019964004154, + "tokens_seen": 1350889472 + }, + { + "epoch": 4.05, + "learning_rate": 0.00029834503510531594, + "loss": 2.747, + "theoretical_loss": 3.5490044774136433, + "tokens_seen": 1350955008 + }, + { + "epoch": 4.05, + "learning_rate": 0.0002983350050150451, + "loss": 2.7483, + "theoretical_loss": 3.548988991784726, + "tokens_seen": 1351020544 + }, + { + "epoch": 4.05, + "learning_rate": 0.00029832497492477436, + "loss": 2.6019, + "theoretical_loss": 3.548973507117296, + "tokens_seen": 1351086080 + }, + { + "epoch": 4.05, + "learning_rate": 0.0002983149448345035, + "loss": 2.6762, + "theoretical_loss": 3.5489580234112457, + "tokens_seen": 1351151616 + }, + { + "epoch": 4.05, + "learning_rate": 0.0002983049147442327, + "loss": 2.8147, + "theoretical_loss": 3.54894254066647, + "tokens_seen": 1351217152 + }, + { + "epoch": 4.05, + "learning_rate": 0.00029829488465396185, + "loss": 2.6882, + "theoretical_loss": 3.5489270588828616, + "tokens_seen": 1351282688 + }, + { + "epoch": 4.05, + "learning_rate": 0.0002982848545636911, + "loss": 2.5956, + "theoretical_loss": 3.548911578060315, + "tokens_seen": 1351348224 + }, + { + "epoch": 4.05, + "learning_rate": 0.00029827482447342026, + "loss": 2.7254, + "theoretical_loss": 3.548896098198724, + "tokens_seen": 1351413760 + }, + { + "epoch": 4.05, + "learning_rate": 0.00029826479438314945, + "loss": 2.5003, + "theoretical_loss": 3.5488806192979814, + "tokens_seen": 1351479296 + }, + { + "epoch": 4.05, + "learning_rate": 0.0002982547642928786, + "loss": 2.9524, + "theoretical_loss": 3.5488651413579824, + "tokens_seen": 1351544832 + }, + { + "epoch": 4.05, + "learning_rate": 0.00029824473420260786, + "loss": 2.7396, + "theoretical_loss": 3.5488496643786194, + "tokens_seen": 1351610368 + }, + { + "epoch": 4.05, + "objective/train/docs_used": 1513423, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7058820724487305, + "objective/train/theoretical_loss": 3.5488341883597867, + "objective/train/tokens_used": 1372135904, + "theoretical_loss": 3.5488341883597867, + "tokens_seen": 1351675904 + }, + { + "epoch": 4.05, + "learning_rate": 0.000298234704112337, + "loss": 2.7002, + "theoretical_loss": 3.5488341883597867, + "tokens_seen": 1351675904 + }, + { + "epoch": 4.05, + "learning_rate": 0.0002982246740220662, + "loss": 2.4233, + "theoretical_loss": 3.5488187133013787, + "tokens_seen": 1351741440 + }, + { + "epoch": 4.05, + "learning_rate": 0.00029821464393179535, + "loss": 2.6596, + "theoretical_loss": 3.5488032392032887, + "tokens_seen": 1351806976 + }, + { + "epoch": 4.05, + "learning_rate": 0.0002982046138415246, + "loss": 2.768, + "theoretical_loss": 3.54878776606541, + "tokens_seen": 1351872512 + }, + { + "epoch": 4.05, + "learning_rate": 0.00029819458375125377, + "loss": 2.6433, + "theoretical_loss": 3.5487722938876374, + "tokens_seen": 1351938048 + }, + { + "epoch": 4.05, + "learning_rate": 0.00029818455366098295, + "loss": 2.6283, + "theoretical_loss": 3.5487568226698647, + "tokens_seen": 1352003584 + }, + { + "epoch": 4.05, + "learning_rate": 0.00029817452357071213, + "loss": 2.8337, + "theoretical_loss": 3.5487413524119855, + "tokens_seen": 1352069120 + }, + { + "epoch": 4.05, + "learning_rate": 0.0002981644934804413, + "loss": 2.7834, + "theoretical_loss": 3.5487258831138937, + "tokens_seen": 1352134656 + }, + { + "epoch": 4.05, + "learning_rate": 0.0002981544633901705, + "loss": 2.5967, + "theoretical_loss": 3.548710414775483, + "tokens_seen": 1352200192 + }, + { + "epoch": 4.05, + "learning_rate": 0.00029814443329989973, + "loss": 2.69, + "theoretical_loss": 3.5486949473966485, + "tokens_seen": 1352265728 + }, + { + "epoch": 4.05, + "learning_rate": 0.00029813440320962885, + "loss": 2.7136, + "theoretical_loss": 3.548679480977283, + "tokens_seen": 1352331264 + }, + { + "epoch": 4.05, + "learning_rate": 0.0002981243731193581, + "loss": 2.4809, + "theoretical_loss": 3.5486640155172804, + "tokens_seen": 1352396800 + }, + { + "epoch": 4.05, + "learning_rate": 0.0002981143430290872, + "loss": 2.668, + "theoretical_loss": 3.5486485510165355, + "tokens_seen": 1352462336 + }, + { + "epoch": 4.05, + "learning_rate": 0.00029810431293881645, + "loss": 2.6052, + "theoretical_loss": 3.5486330874749425, + "tokens_seen": 1352527872 + }, + { + "epoch": 4.05, + "learning_rate": 0.00029809428284854563, + "loss": 2.697, + "theoretical_loss": 3.5486176248923944, + "tokens_seen": 1352593408 + }, + { + "epoch": 4.05, + "learning_rate": 0.0002980842527582748, + "loss": 2.6051, + "theoretical_loss": 3.548602163268786, + "tokens_seen": 1352658944 + }, + { + "epoch": 4.05, + "learning_rate": 0.000298074222668004, + "loss": 2.6487, + "theoretical_loss": 3.5485867026040108, + "tokens_seen": 1352724480 + }, + { + "epoch": 4.05, + "learning_rate": 0.00029806419257773323, + "loss": 2.7828, + "theoretical_loss": 3.5485712428979634, + "tokens_seen": 1352790016 + }, + { + "epoch": 4.05, + "learning_rate": 0.00029805416248746236, + "loss": 2.5762, + "theoretical_loss": 3.5485557841505377, + "tokens_seen": 1352855552 + }, + { + "epoch": 4.05, + "learning_rate": 0.0002980441323971916, + "loss": 2.6515, + "theoretical_loss": 3.548540326361628, + "tokens_seen": 1352921088 + }, + { + "epoch": 4.05, + "learning_rate": 0.0002980341023069208, + "loss": 2.8252, + "theoretical_loss": 3.5485248695311284, + "tokens_seen": 1352986624 + }, + { + "epoch": 4.05, + "learning_rate": 0.00029802407221664996, + "loss": 2.6945, + "theoretical_loss": 3.5485094136589326, + "tokens_seen": 1353052160 + }, + { + "epoch": 4.05, + "learning_rate": 0.0002980140421263792, + "loss": 2.8939, + "theoretical_loss": 3.5484939587449356, + "tokens_seen": 1353117696 + }, + { + "epoch": 4.05, + "learning_rate": 0.0002980040120361083, + "loss": 2.7708, + "theoretical_loss": 3.5484785047890304, + "tokens_seen": 1353183232 + }, + { + "epoch": 4.05, + "learning_rate": 0.00029799398194583755, + "loss": 2.4756, + "theoretical_loss": 3.5484630517911127, + "tokens_seen": 1353248768 + }, + { + "epoch": 4.05, + "objective/train/docs_used": 1514073, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4900929927825928, + "objective/train/theoretical_loss": 3.548447599751076, + "objective/train/tokens_used": 1373774304, + "theoretical_loss": 3.548447599751076, + "tokens_seen": 1353314304 + }, + { + "epoch": 4.05, + "learning_rate": 0.0002979839518555667, + "loss": 2.7386, + "theoretical_loss": 3.548447599751076, + "tokens_seen": 1353314304 + }, + { + "epoch": 4.05, + "learning_rate": 0.0002979739217652959, + "loss": 2.6159, + "theoretical_loss": 3.548432148668814, + "tokens_seen": 1353379840 + }, + { + "epoch": 4.05, + "learning_rate": 0.0002979638916750251, + "loss": 2.7703, + "theoretical_loss": 3.548416698544222, + "tokens_seen": 1353445376 + }, + { + "epoch": 4.05, + "learning_rate": 0.0002979538615847543, + "loss": 2.6948, + "theoretical_loss": 3.5484012493771933, + "tokens_seen": 1353510912 + }, + { + "epoch": 4.05, + "learning_rate": 0.00029794383149448346, + "loss": 2.7864, + "theoretical_loss": 3.5483858011676226, + "tokens_seen": 1353576448 + }, + { + "epoch": 4.05, + "learning_rate": 0.0002979338014042127, + "loss": 2.5321, + "theoretical_loss": 3.548370353915405, + "tokens_seen": 1353641984 + }, + { + "epoch": 4.05, + "learning_rate": 0.0002979237713139418, + "loss": 2.614, + "theoretical_loss": 3.548354907620433, + "tokens_seen": 1353707520 + }, + { + "epoch": 4.05, + "learning_rate": 0.00029791374122367106, + "loss": 2.539, + "theoretical_loss": 3.5483394622826028, + "tokens_seen": 1353773056 + }, + { + "epoch": 4.05, + "learning_rate": 0.0002979037111334002, + "loss": 2.6795, + "theoretical_loss": 3.548324017901807, + "tokens_seen": 1353838592 + }, + { + "epoch": 4.05, + "learning_rate": 0.0002978936810431294, + "loss": 2.4953, + "theoretical_loss": 3.548308574477942, + "tokens_seen": 1353904128 + }, + { + "epoch": 4.05, + "learning_rate": 0.0002978836509528586, + "loss": 2.658, + "theoretical_loss": 3.548293132010901, + "tokens_seen": 1353969664 + }, + { + "epoch": 4.05, + "learning_rate": 0.0002978736208625878, + "loss": 2.5627, + "theoretical_loss": 3.548277690500578, + "tokens_seen": 1354035200 + }, + { + "epoch": 4.05, + "learning_rate": 0.00029786359077231696, + "loss": 2.6826, + "theoretical_loss": 3.548262249946869, + "tokens_seen": 1354100736 + }, + { + "epoch": 4.05, + "learning_rate": 0.00029785356068204614, + "loss": 2.5905, + "theoretical_loss": 3.5482468103496663, + "tokens_seen": 1354166272 + }, + { + "epoch": 4.05, + "learning_rate": 0.0002978435305917753, + "loss": 2.7626, + "theoretical_loss": 3.548231371708866, + "tokens_seen": 1354231808 + }, + { + "epoch": 4.05, + "learning_rate": 0.00029783350050150456, + "loss": 2.8008, + "theoretical_loss": 3.5482159340243618, + "tokens_seen": 1354297344 + }, + { + "epoch": 4.05, + "learning_rate": 0.0002978234704112337, + "loss": 2.6798, + "theoretical_loss": 3.548200497296049, + "tokens_seen": 1354362880 + }, + { + "epoch": 4.05, + "learning_rate": 0.0002978134403209629, + "loss": 2.7777, + "theoretical_loss": 3.548185061523821, + "tokens_seen": 1354428416 + }, + { + "epoch": 4.05, + "learning_rate": 0.00029780341023069205, + "loss": 2.4268, + "theoretical_loss": 3.548169626707573, + "tokens_seen": 1354493952 + }, + { + "epoch": 4.05, + "learning_rate": 0.0002977933801404213, + "loss": 2.4828, + "theoretical_loss": 3.5481541928472, + "tokens_seen": 1354559488 + }, + { + "epoch": 4.05, + "learning_rate": 0.00029778335005015046, + "loss": 2.6172, + "theoretical_loss": 3.5481387599425953, + "tokens_seen": 1354625024 + }, + { + "epoch": 4.05, + "learning_rate": 0.00029777331995987965, + "loss": 2.6898, + "theoretical_loss": 3.5481233279936544, + "tokens_seen": 1354690560 + }, + { + "epoch": 4.05, + "learning_rate": 0.00029776328986960883, + "loss": 2.8771, + "theoretical_loss": 3.5481078970002713, + "tokens_seen": 1354756096 + }, + { + "epoch": 4.05, + "learning_rate": 0.00029775325977933806, + "loss": 2.8269, + "theoretical_loss": 3.5480924669623413, + "tokens_seen": 1354821632 + }, + { + "epoch": 4.05, + "learning_rate": 0.0002977432296890672, + "loss": 2.6392, + "theoretical_loss": 3.548077037879759, + "tokens_seen": 1354887168 + }, + { + "epoch": 4.05, + "objective/train/docs_used": 1515331, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.898995876312256, + "objective/train/theoretical_loss": 3.548061609752419, + "objective/train/tokens_used": 1375412704, + "theoretical_loss": 3.548061609752419, + "tokens_seen": 1354952704 + }, + { + "epoch": 4.05, + "learning_rate": 0.0002977331995987964, + "loss": 2.8321, + "theoretical_loss": 3.548061609752419, + "tokens_seen": 1354952704 + }, + { + "epoch": 4.05, + "learning_rate": 0.00029772316950852555, + "loss": 2.7478, + "theoretical_loss": 3.5480461825802148, + "tokens_seen": 1355018240 + }, + { + "epoch": 4.05, + "learning_rate": 0.0002977131394182548, + "loss": 2.7196, + "theoretical_loss": 3.548030756363042, + "tokens_seen": 1355083776 + }, + { + "epoch": 4.05, + "learning_rate": 0.00029770310932798397, + "loss": 2.8133, + "theoretical_loss": 3.548015331100796, + "tokens_seen": 1355149312 + }, + { + "epoch": 4.05, + "learning_rate": 0.00029769307923771315, + "loss": 2.7607, + "theoretical_loss": 3.5479999067933705, + "tokens_seen": 1355214848 + }, + { + "epoch": 4.05, + "learning_rate": 0.00029768304914744233, + "loss": 2.879, + "theoretical_loss": 3.5479844834406604, + "tokens_seen": 1355280384 + }, + { + "epoch": 4.05, + "learning_rate": 0.0002976730190571715, + "loss": 2.8117, + "theoretical_loss": 3.5479690610425605, + "tokens_seen": 1355345920 + }, + { + "epoch": 4.05, + "learning_rate": 0.0002976629889669007, + "loss": 2.6956, + "theoretical_loss": 3.547953639598966, + "tokens_seen": 1355411456 + }, + { + "epoch": 4.05, + "learning_rate": 0.00029765295887662993, + "loss": 2.6193, + "theoretical_loss": 3.5479382191097706, + "tokens_seen": 1355476992 + }, + { + "epoch": 4.05, + "learning_rate": 0.00029764292878635905, + "loss": 2.5348, + "theoretical_loss": 3.5479227995748706, + "tokens_seen": 1355542528 + }, + { + "epoch": 4.05, + "learning_rate": 0.0002976328986960883, + "loss": 2.741, + "theoretical_loss": 3.5479073809941593, + "tokens_seen": 1355608064 + }, + { + "epoch": 4.05, + "learning_rate": 0.0002976228686058174, + "loss": 2.6227, + "theoretical_loss": 3.5478919633675328, + "tokens_seen": 1355673600 + }, + { + "epoch": 4.05, + "learning_rate": 0.00029761283851554665, + "loss": 2.7048, + "theoretical_loss": 3.5478765466948854, + "tokens_seen": 1355739136 + }, + { + "epoch": 4.05, + "learning_rate": 0.00029760280842527583, + "loss": 2.6778, + "theoretical_loss": 3.5478611309761114, + "tokens_seen": 1355804672 + }, + { + "epoch": 4.05, + "learning_rate": 0.000297592778335005, + "loss": 2.7497, + "theoretical_loss": 3.5478457162111066, + "tokens_seen": 1355870208 + }, + { + "epoch": 4.05, + "learning_rate": 0.0002975827482447342, + "loss": 2.9066, + "theoretical_loss": 3.547830302399765, + "tokens_seen": 1355935744 + }, + { + "epoch": 4.05, + "learning_rate": 0.00029757271815446343, + "loss": 2.7068, + "theoretical_loss": 3.547814889541983, + "tokens_seen": 1356001280 + }, + { + "epoch": 4.05, + "learning_rate": 0.00029756268806419256, + "loss": 2.4788, + "theoretical_loss": 3.547799477637654, + "tokens_seen": 1356066816 + }, + { + "epoch": 4.05, + "learning_rate": 0.0002975526579739218, + "loss": 2.5697, + "theoretical_loss": 3.547784066686673, + "tokens_seen": 1356132352 + }, + { + "epoch": 4.05, + "learning_rate": 0.0002975426278836509, + "loss": 2.4808, + "theoretical_loss": 3.547768656688936, + "tokens_seen": 1356197888 + }, + { + "epoch": 4.05, + "learning_rate": 0.00029753259779338016, + "loss": 2.8969, + "theoretical_loss": 3.5477532476443376, + "tokens_seen": 1356263424 + }, + { + "epoch": 4.05, + "learning_rate": 0.00029752256770310934, + "loss": 2.5684, + "theoretical_loss": 3.5477378395527723, + "tokens_seen": 1356328960 + }, + { + "epoch": 4.05, + "learning_rate": 0.0002975125376128385, + "loss": 2.6905, + "theoretical_loss": 3.5477224324141354, + "tokens_seen": 1356394496 + }, + { + "epoch": 4.05, + "learning_rate": 0.0002975025075225677, + "loss": 2.6665, + "theoretical_loss": 3.547707026228322, + "tokens_seen": 1356460032 + }, + { + "epoch": 4.05, + "learning_rate": 0.0002974924774322969, + "loss": 2.6812, + "theoretical_loss": 3.547691620995227, + "tokens_seen": 1356525568 + }, + { + "epoch": 4.05, + "objective/train/docs_used": 1516059, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.887085437774658, + "objective/train/theoretical_loss": 3.5476762167147458, + "objective/train/tokens_used": 1377051104, + "theoretical_loss": 3.5476762167147458, + "tokens_seen": 1356591104 + }, + { + "epoch": 4.05, + "learning_rate": 0.00029748244734202606, + "loss": 2.7841, + "theoretical_loss": 3.5476762167147458, + "tokens_seen": 1356591104 + }, + { + "epoch": 4.05, + "learning_rate": 0.0002974724172517553, + "loss": 2.8691, + "theoretical_loss": 3.5476608133867735, + "tokens_seen": 1356656640 + }, + { + "epoch": 4.05, + "learning_rate": 0.0002974623871614844, + "loss": 2.8379, + "theoretical_loss": 3.547645411011205, + "tokens_seen": 1356722176 + }, + { + "epoch": 4.05, + "learning_rate": 0.00029745235707121366, + "loss": 2.6562, + "theoretical_loss": 3.5476300095879347, + "tokens_seen": 1356787712 + }, + { + "epoch": 4.05, + "learning_rate": 0.0002974423269809428, + "loss": 2.8677, + "theoretical_loss": 3.5476146091168586, + "tokens_seen": 1356853248 + }, + { + "epoch": 4.05, + "learning_rate": 0.000297432296890672, + "loss": 2.5165, + "theoretical_loss": 3.5475992095978715, + "tokens_seen": 1356918784 + }, + { + "epoch": 4.05, + "learning_rate": 0.0002974222668004012, + "loss": 2.6703, + "theoretical_loss": 3.547583811030869, + "tokens_seen": 1356984320 + }, + { + "epoch": 4.05, + "learning_rate": 0.0002974122367101304, + "loss": 2.6626, + "theoretical_loss": 3.5475684134157457, + "tokens_seen": 1357049856 + }, + { + "epoch": 4.05, + "learning_rate": 0.00029740220661985956, + "loss": 2.6634, + "theoretical_loss": 3.5475530167523974, + "tokens_seen": 1357115392 + }, + { + "epoch": 4.05, + "learning_rate": 0.0002973921765295888, + "loss": 2.6589, + "theoretical_loss": 3.5475376210407186, + "tokens_seen": 1357180928 + }, + { + "epoch": 4.05, + "learning_rate": 0.0002973821464393179, + "loss": 2.7632, + "theoretical_loss": 3.5475222262806048, + "tokens_seen": 1357246464 + }, + { + "epoch": 4.05, + "learning_rate": 0.00029737211634904716, + "loss": 2.8114, + "theoretical_loss": 3.5475068324719516, + "tokens_seen": 1357312000 + }, + { + "epoch": 4.05, + "learning_rate": 0.0002973620862587763, + "loss": 2.6815, + "theoretical_loss": 3.547491439614654, + "tokens_seen": 1357377536 + }, + { + "epoch": 4.05, + "learning_rate": 0.0002973520561685055, + "loss": 2.6894, + "theoretical_loss": 3.5474760477086074, + "tokens_seen": 1357443072 + }, + { + "epoch": 4.05, + "learning_rate": 0.0002973420260782347, + "loss": 2.8562, + "theoretical_loss": 3.5474606567537066, + "tokens_seen": 1357508608 + }, + { + "epoch": 4.05, + "learning_rate": 0.0002973319959879639, + "loss": 2.5874, + "theoretical_loss": 3.5474452667498477, + "tokens_seen": 1357574144 + }, + { + "epoch": 4.05, + "learning_rate": 0.00029732196589769307, + "loss": 2.5583, + "theoretical_loss": 3.547429877696925, + "tokens_seen": 1357639680 + }, + { + "epoch": 4.05, + "learning_rate": 0.00029731193580742225, + "loss": 2.5643, + "theoretical_loss": 3.5474144895948347, + "tokens_seen": 1357705216 + }, + { + "epoch": 4.05, + "learning_rate": 0.00029730190571715143, + "loss": 2.6758, + "theoretical_loss": 3.547399102443472, + "tokens_seen": 1357770752 + }, + { + "epoch": 4.05, + "learning_rate": 0.00029729187562688067, + "loss": 2.7373, + "theoretical_loss": 3.547383716242732, + "tokens_seen": 1357836288 + }, + { + "epoch": 4.05, + "learning_rate": 0.00029728184553660985, + "loss": 2.7633, + "theoretical_loss": 3.5473683309925104, + "tokens_seen": 1357901824 + }, + { + "epoch": 4.05, + "learning_rate": 0.00029727181544633903, + "loss": 2.643, + "theoretical_loss": 3.547352946692703, + "tokens_seen": 1357967360 + }, + { + "epoch": 4.05, + "learning_rate": 0.00029726178535606826, + "loss": 2.4956, + "theoretical_loss": 3.5473375633432043, + "tokens_seen": 1358032896 + }, + { + "epoch": 4.05, + "learning_rate": 0.0002972517552657974, + "loss": 2.7176, + "theoretical_loss": 3.54732218094391, + "tokens_seen": 1358098432 + }, + { + "epoch": 4.05, + "learning_rate": 0.0002972417251755266, + "loss": 2.9503, + "theoretical_loss": 3.5473067994947156, + "tokens_seen": 1358163968 + }, + { + "epoch": 4.05, + "objective/train/docs_used": 1517513, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5391719341278076, + "objective/train/theoretical_loss": 3.547291418995517, + "objective/train/tokens_used": 1378689504, + "theoretical_loss": 3.547291418995517, + "tokens_seen": 1358229504 + }, + { + "epoch": 4.05, + "learning_rate": 0.00029723169508525575, + "loss": 2.6743, + "theoretical_loss": 3.547291418995517, + "tokens_seen": 1358229504 + }, + { + "epoch": 4.05, + "learning_rate": 0.000297221664994985, + "loss": 2.8176, + "theoretical_loss": 3.547276039446209, + "tokens_seen": 1358295040 + }, + { + "epoch": 4.05, + "learning_rate": 0.00029721163490471417, + "loss": 2.7804, + "theoretical_loss": 3.547260660846688, + "tokens_seen": 1358360576 + }, + { + "epoch": 4.05, + "learning_rate": 0.00029720160481444335, + "loss": 2.6297, + "theoretical_loss": 3.5472452831968484, + "tokens_seen": 1358426112 + }, + { + "epoch": 4.05, + "learning_rate": 0.00029719157472417253, + "loss": 2.7697, + "theoretical_loss": 3.547229906496587, + "tokens_seen": 1358491648 + }, + { + "epoch": 4.05, + "learning_rate": 0.0002971815446339017, + "loss": 2.6438, + "theoretical_loss": 3.547214530745798, + "tokens_seen": 1358557184 + }, + { + "epoch": 4.05, + "learning_rate": 0.0002971715145436309, + "loss": 2.8875, + "theoretical_loss": 3.5471991559443783, + "tokens_seen": 1358622720 + }, + { + "epoch": 4.05, + "learning_rate": 0.00029716148445336013, + "loss": 2.5986, + "theoretical_loss": 3.5471837820922225, + "tokens_seen": 1358688256 + }, + { + "epoch": 4.05, + "learning_rate": 0.00029715145436308926, + "loss": 2.7038, + "theoretical_loss": 3.5471684091892266, + "tokens_seen": 1358753792 + }, + { + "epoch": 4.05, + "learning_rate": 0.0002971414242728185, + "loss": 2.7325, + "theoretical_loss": 3.5471530372352866, + "tokens_seen": 1358819328 + }, + { + "epoch": 4.05, + "learning_rate": 0.0002971313941825476, + "loss": 2.5486, + "theoretical_loss": 3.5471376662302974, + "tokens_seen": 1358884864 + }, + { + "epoch": 4.05, + "learning_rate": 0.00029712136409227685, + "loss": 2.9332, + "theoretical_loss": 3.547122296174155, + "tokens_seen": 1358950400 + }, + { + "epoch": 4.05, + "learning_rate": 0.00029711133400200603, + "loss": 2.789, + "theoretical_loss": 3.5471069270667552, + "tokens_seen": 1359015936 + }, + { + "epoch": 4.05, + "learning_rate": 0.0002971013039117352, + "loss": 2.6935, + "theoretical_loss": 3.5470915589079937, + "tokens_seen": 1359081472 + }, + { + "epoch": 4.05, + "learning_rate": 0.0002970912738214644, + "loss": 2.6713, + "theoretical_loss": 3.547076191697766, + "tokens_seen": 1359147008 + }, + { + "epoch": 4.05, + "learning_rate": 0.00029708124373119363, + "loss": 2.8167, + "theoretical_loss": 3.5470608254359672, + "tokens_seen": 1359212544 + }, + { + "epoch": 4.05, + "learning_rate": 0.00029707121364092276, + "loss": 2.4853, + "theoretical_loss": 3.5470454601224946, + "tokens_seen": 1359278080 + }, + { + "epoch": 4.05, + "learning_rate": 0.000297061183550652, + "loss": 2.7376, + "theoretical_loss": 3.5470300957572425, + "tokens_seen": 1359343616 + }, + { + "epoch": 4.05, + "learning_rate": 0.0002970511534603811, + "loss": 2.7228, + "theoretical_loss": 3.5470147323401076, + "tokens_seen": 1359409152 + }, + { + "epoch": 4.05, + "learning_rate": 0.00029704112337011036, + "loss": 2.958, + "theoretical_loss": 3.546999369870985, + "tokens_seen": 1359474688 + }, + { + "epoch": 4.05, + "learning_rate": 0.00029703109327983954, + "loss": 2.7322, + "theoretical_loss": 3.5469840083497717, + "tokens_seen": 1359540224 + }, + { + "epoch": 4.05, + "learning_rate": 0.0002970210631895687, + "loss": 2.7878, + "theoretical_loss": 3.546968647776362, + "tokens_seen": 1359605760 + }, + { + "epoch": 4.05, + "learning_rate": 0.0002970110330992979, + "loss": 2.3531, + "theoretical_loss": 3.5469532881506525, + "tokens_seen": 1359671296 + }, + { + "epoch": 4.05, + "learning_rate": 0.0002970010030090271, + "loss": 2.8062, + "theoretical_loss": 3.5469379294725387, + "tokens_seen": 1359736832 + }, + { + "epoch": 4.05, + "learning_rate": 0.00029699097291875626, + "loss": 2.582, + "theoretical_loss": 3.5469225717419173, + "tokens_seen": 1359802368 + }, + { + "epoch": 4.05, + "objective/train/docs_used": 1518071, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.3069539070129395, + "objective/train/theoretical_loss": 3.5469072149586833, + "objective/train/tokens_used": 1380327904, + "theoretical_loss": 3.5469072149586833, + "tokens_seen": 1359867904 + }, + { + "epoch": 4.05, + "learning_rate": 0.0002969809428284855, + "loss": 2.5244, + "theoretical_loss": 3.5469072149586833, + "tokens_seen": 1359867904 + }, + { + "epoch": 4.05, + "learning_rate": 0.0002969709127382146, + "loss": 2.8248, + "theoretical_loss": 3.5468918591227334, + "tokens_seen": 1359933440 + }, + { + "epoch": 4.05, + "learning_rate": 0.00029696088264794386, + "loss": 2.7735, + "theoretical_loss": 3.5468765042339623, + "tokens_seen": 1359998976 + }, + { + "epoch": 4.05, + "learning_rate": 0.000296950852557673, + "loss": 2.522, + "theoretical_loss": 3.5468611502922673, + "tokens_seen": 1360064512 + }, + { + "epoch": 4.05, + "learning_rate": 0.0002969408224674022, + "loss": 2.7694, + "theoretical_loss": 3.546845797297544, + "tokens_seen": 1360130048 + }, + { + "epoch": 4.05, + "learning_rate": 0.0002969408224674022, + "loss": 2.6656, + "theoretical_loss": 3.5468304452496873, + "tokens_seen": 1360195584 + }, + { + "epoch": 4.05, + "learning_rate": 0.0002969307923771314, + "loss": 2.591, + "theoretical_loss": 3.5468150941485943, + "tokens_seen": 1360261120 + }, + { + "epoch": 4.05, + "learning_rate": 0.0002969207622868606, + "loss": 2.7995, + "theoretical_loss": 3.5467997439941605, + "tokens_seen": 1360326656 + }, + { + "epoch": 4.05, + "learning_rate": 0.00029691073219658976, + "loss": 2.8339, + "theoretical_loss": 3.5467843947862825, + "tokens_seen": 1360392192 + }, + { + "epoch": 4.05, + "learning_rate": 0.000296900702106319, + "loss": 2.5979, + "theoretical_loss": 3.5467690465248563, + "tokens_seen": 1360457728 + }, + { + "epoch": 4.05, + "learning_rate": 0.0002968906720160481, + "loss": 2.6479, + "theoretical_loss": 3.546753699209777, + "tokens_seen": 1360523264 + }, + { + "epoch": 4.05, + "learning_rate": 0.00029688064192577736, + "loss": 2.5995, + "theoretical_loss": 3.5467383528409417, + "tokens_seen": 1360588800 + }, + { + "epoch": 4.05, + "learning_rate": 0.0002968706118355065, + "loss": 2.6558, + "theoretical_loss": 3.5467230074182456, + "tokens_seen": 1360654336 + }, + { + "epoch": 4.05, + "learning_rate": 0.0002968605817452357, + "loss": 2.7648, + "theoretical_loss": 3.5467076629415857, + "tokens_seen": 1360719872 + }, + { + "epoch": 4.05, + "learning_rate": 0.0002968505516549649, + "loss": 2.6445, + "theoretical_loss": 3.5466923194108575, + "tokens_seen": 1360785408 + }, + { + "epoch": 4.05, + "learning_rate": 0.0002968405215646941, + "loss": 2.3767, + "theoretical_loss": 3.546676976825957, + "tokens_seen": 1360850944 + }, + { + "epoch": 4.05, + "learning_rate": 0.00029683049147442327, + "loss": 2.5866, + "theoretical_loss": 3.5466616351867812, + "tokens_seen": 1360916480 + }, + { + "epoch": 4.05, + "learning_rate": 0.00029682046138415245, + "loss": 2.742, + "theoretical_loss": 3.5466462944932253, + "tokens_seen": 1360982016 + }, + { + "epoch": 4.05, + "learning_rate": 0.00029681043129388163, + "loss": 2.731, + "theoretical_loss": 3.5466309547451864, + "tokens_seen": 1361047552 + }, + { + "epoch": 4.05, + "learning_rate": 0.00029680040120361087, + "loss": 2.913, + "theoretical_loss": 3.54661561594256, + "tokens_seen": 1361113088 + }, + { + "epoch": 4.05, + "learning_rate": 0.00029679037111334, + "loss": 2.7811, + "theoretical_loss": 3.5466002780852426, + "tokens_seen": 1361178624 + }, + { + "epoch": 4.05, + "learning_rate": 0.00029678034102306923, + "loss": 2.7598, + "theoretical_loss": 3.5465849411731307, + "tokens_seen": 1361244160 + }, + { + "epoch": 4.05, + "learning_rate": 0.0002967703109327984, + "loss": 2.6426, + "theoretical_loss": 3.5465696052061197, + "tokens_seen": 1361309696 + }, + { + "epoch": 4.05, + "learning_rate": 0.0002967602808425276, + "loss": 2.5915, + "theoretical_loss": 3.5465542701841066, + "tokens_seen": 1361375232 + }, + { + "epoch": 4.05, + "learning_rate": 0.00029675025075225677, + "loss": 2.855, + "theoretical_loss": 3.5465389361069874, + "tokens_seen": 1361440768 + }, + { + "epoch": 4.05, + "objective/train/docs_used": 1519309, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.060181140899658, + "objective/train/theoretical_loss": 3.5465236029746583, + "objective/train/tokens_used": 1381966304, + "theoretical_loss": 3.5465236029746583, + "tokens_seen": 1361506304 + }, + { + "epoch": 4.05, + "learning_rate": 0.00029674022066198595, + "loss": 2.7874, + "theoretical_loss": 3.5465236029746583, + "tokens_seen": 1361506304 + }, + { + "epoch": 4.05, + "learning_rate": 0.00029673019057171513, + "loss": 2.7999, + "theoretical_loss": 3.546508270787016, + "tokens_seen": 1361571840 + }, + { + "epoch": 4.05, + "learning_rate": 0.00029672016048144437, + "loss": 2.5814, + "theoretical_loss": 3.546492939543957, + "tokens_seen": 1361637376 + }, + { + "epoch": 4.05, + "learning_rate": 0.0002967101303911735, + "loss": 2.6692, + "theoretical_loss": 3.5464776092453767, + "tokens_seen": 1361702912 + }, + { + "epoch": 4.05, + "learning_rate": 0.00029670010030090273, + "loss": 2.9344, + "theoretical_loss": 3.5464622798911725, + "tokens_seen": 1361768448 + }, + { + "epoch": 4.05, + "learning_rate": 0.00029669007021063186, + "loss": 2.7264, + "theoretical_loss": 3.5464469514812396, + "tokens_seen": 1361833984 + }, + { + "epoch": 4.05, + "learning_rate": 0.0002966800401203611, + "loss": 2.8269, + "theoretical_loss": 3.546431624015476, + "tokens_seen": 1361899520 + }, + { + "epoch": 4.05, + "learning_rate": 0.0002966700100300903, + "loss": 2.7462, + "theoretical_loss": 3.5464162974937765, + "tokens_seen": 1361965056 + }, + { + "epoch": 4.05, + "learning_rate": 0.00029665997993981946, + "loss": 2.7305, + "theoretical_loss": 3.5464009719160385, + "tokens_seen": 1362030592 + }, + { + "epoch": 4.05, + "learning_rate": 0.00029664994984954864, + "loss": 2.5155, + "theoretical_loss": 3.546385647282158, + "tokens_seen": 1362096128 + }, + { + "epoch": 4.05, + "learning_rate": 0.0002966399197592778, + "loss": 2.6736, + "theoretical_loss": 3.546370323592032, + "tokens_seen": 1362161664 + }, + { + "epoch": 4.05, + "learning_rate": 0.000296629889669007, + "loss": 2.7749, + "theoretical_loss": 3.5463550008455567, + "tokens_seen": 1362227200 + }, + { + "epoch": 4.05, + "learning_rate": 0.00029661985957873623, + "loss": 2.802, + "theoretical_loss": 3.546339679042628, + "tokens_seen": 1362292736 + }, + { + "epoch": 4.05, + "learning_rate": 0.00029660982948846536, + "loss": 2.684, + "theoretical_loss": 3.546324358183143, + "tokens_seen": 1362358272 + }, + { + "epoch": 4.05, + "learning_rate": 0.0002965997993981946, + "loss": 2.5993, + "theoretical_loss": 3.5463090382669984, + "tokens_seen": 1362423808 + }, + { + "epoch": 4.05, + "learning_rate": 0.0002965897693079238, + "loss": 2.6696, + "theoretical_loss": 3.5462937192940904, + "tokens_seen": 1362489344 + }, + { + "epoch": 4.05, + "learning_rate": 0.00029657973921765296, + "loss": 2.8625, + "theoretical_loss": 3.546278401264316, + "tokens_seen": 1362554880 + }, + { + "epoch": 4.05, + "learning_rate": 0.00029656970912738214, + "loss": 2.7273, + "theoretical_loss": 3.5462630841775713, + "tokens_seen": 1362620416 + }, + { + "epoch": 4.05, + "learning_rate": 0.0002965596790371113, + "loss": 2.6861, + "theoretical_loss": 3.546247768033753, + "tokens_seen": 1362685952 + }, + { + "epoch": 4.05, + "learning_rate": 0.0002965496489468405, + "loss": 2.6665, + "theoretical_loss": 3.546232452832758, + "tokens_seen": 1362751488 + }, + { + "epoch": 4.05, + "learning_rate": 0.00029653961885656974, + "loss": 2.6598, + "theoretical_loss": 3.546217138574482, + "tokens_seen": 1362817024 + }, + { + "epoch": 4.05, + "learning_rate": 0.0002965295887662989, + "loss": 2.8421, + "theoretical_loss": 3.546201825258823, + "tokens_seen": 1362882560 + }, + { + "epoch": 4.05, + "learning_rate": 0.0002965195586760281, + "loss": 2.7694, + "theoretical_loss": 3.546186512885676, + "tokens_seen": 1362948096 + }, + { + "epoch": 4.05, + "learning_rate": 0.0002965095285857573, + "loss": 2.7711, + "theoretical_loss": 3.5461712014549396, + "tokens_seen": 1363013632 + }, + { + "epoch": 4.05, + "learning_rate": 0.00029649949849548646, + "loss": 2.6412, + "theoretical_loss": 3.546155890966509, + "tokens_seen": 1363079168 + }, + { + "epoch": 4.05, + "objective/train/docs_used": 1519912, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8462095260620117, + "objective/train/theoretical_loss": 3.546140581420282, + "objective/train/tokens_used": 1383604704, + "theoretical_loss": 3.546140581420282, + "tokens_seen": 1363144704 + }, + { + "epoch": 4.05, + "learning_rate": 0.0002964894684052157, + "loss": 2.7918, + "theoretical_loss": 3.546140581420282, + "tokens_seen": 1363144704 + }, + { + "epoch": 4.05, + "learning_rate": 0.0002964794383149448, + "loss": 2.6971, + "theoretical_loss": 3.5461252728161545, + "tokens_seen": 1363210240 + }, + { + "epoch": 4.05, + "learning_rate": 0.00029646940822467406, + "loss": 2.6667, + "theoretical_loss": 3.5461099651540233, + "tokens_seen": 1363275776 + }, + { + "epoch": 4.05, + "learning_rate": 0.0002964593781344032, + "loss": 2.8574, + "theoretical_loss": 3.546094658433786, + "tokens_seen": 1363341312 + }, + { + "epoch": 4.05, + "learning_rate": 0.0002964493480441324, + "loss": 2.8651, + "theoretical_loss": 3.5460793526553376, + "tokens_seen": 1363406848 + }, + { + "epoch": 4.05, + "learning_rate": 0.0002964393179538616, + "loss": 2.6643, + "theoretical_loss": 3.546064047818577, + "tokens_seen": 1363472384 + }, + { + "epoch": 4.05, + "learning_rate": 0.0002964292878635908, + "loss": 2.831, + "theoretical_loss": 3.5460487439233996, + "tokens_seen": 1363537920 + }, + { + "epoch": 4.05, + "learning_rate": 0.00029641925777331996, + "loss": 2.587, + "theoretical_loss": 3.546033440969703, + "tokens_seen": 1363603456 + }, + { + "epoch": 4.05, + "learning_rate": 0.0002964092276830492, + "loss": 2.6461, + "theoretical_loss": 3.546018138957383, + "tokens_seen": 1363668992 + }, + { + "epoch": 4.05, + "learning_rate": 0.0002963991975927783, + "loss": 2.7813, + "theoretical_loss": 3.5460028378863377, + "tokens_seen": 1363734528 + }, + { + "epoch": 4.05, + "learning_rate": 0.00029638916750250756, + "loss": 2.617, + "theoretical_loss": 3.5459875377564636, + "tokens_seen": 1363800064 + }, + { + "epoch": 4.05, + "learning_rate": 0.0002963791374122367, + "loss": 2.8451, + "theoretical_loss": 3.5459722385676566, + "tokens_seen": 1363865600 + }, + { + "epoch": 4.05, + "learning_rate": 0.0002963691073219659, + "loss": 2.7216, + "theoretical_loss": 3.5459569403198152, + "tokens_seen": 1363931136 + }, + { + "epoch": 4.05, + "learning_rate": 0.0002963590772316951, + "loss": 2.6646, + "theoretical_loss": 3.545941643012835, + "tokens_seen": 1363996672 + }, + { + "epoch": 4.05, + "learning_rate": 0.0002963490471414243, + "loss": 2.5081, + "theoretical_loss": 3.5459263466466133, + "tokens_seen": 1364062208 + }, + { + "epoch": 4.05, + "learning_rate": 0.00029633901705115347, + "loss": 2.8276, + "theoretical_loss": 3.5459110512210477, + "tokens_seen": 1364127744 + }, + { + "epoch": 4.05, + "learning_rate": 0.00029632898696088265, + "loss": 2.5364, + "theoretical_loss": 3.5458957567360345, + "tokens_seen": 1364193280 + }, + { + "epoch": 4.05, + "learning_rate": 0.00029631895687061183, + "loss": 2.8552, + "theoretical_loss": 3.5458804631914704, + "tokens_seen": 1364258816 + }, + { + "epoch": 4.05, + "learning_rate": 0.00029630892678034107, + "loss": 2.7965, + "theoretical_loss": 3.5458651705872537, + "tokens_seen": 1364324352 + }, + { + "epoch": 4.05, + "learning_rate": 0.0002962988966900702, + "loss": 2.8659, + "theoretical_loss": 3.54584987892328, + "tokens_seen": 1364389888 + }, + { + "epoch": 4.05, + "learning_rate": 0.00029628886659979943, + "loss": 2.7637, + "theoretical_loss": 3.5458345881994466, + "tokens_seen": 1364455424 + }, + { + "epoch": 4.05, + "learning_rate": 0.0002962788365095286, + "loss": 2.7808, + "theoretical_loss": 3.5458192984156516, + "tokens_seen": 1364520960 + }, + { + "epoch": 4.05, + "learning_rate": 0.0002962688064192578, + "loss": 2.6831, + "theoretical_loss": 3.545804009571791, + "tokens_seen": 1364586496 + }, + { + "epoch": 4.05, + "learning_rate": 0.00029625877632898697, + "loss": 2.7481, + "theoretical_loss": 3.545788721667762, + "tokens_seen": 1364652032 + }, + { + "epoch": 4.05, + "learning_rate": 0.00029624874623871615, + "loss": 2.6898, + "theoretical_loss": 3.545773434703462, + "tokens_seen": 1364717568 + }, + { + "epoch": 4.05, + "objective/train/docs_used": 1521305, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.574596881866455, + "objective/train/theoretical_loss": 3.5457581486787877, + "objective/train/tokens_used": 1385243104, + "theoretical_loss": 3.5457581486787877, + "tokens_seen": 1364783104 + }, + { + "epoch": 4.05, + "learning_rate": 0.00029623871614844533, + "loss": 2.6539, + "theoretical_loss": 3.5457581486787877, + "tokens_seen": 1364783104 + }, + { + "epoch": 4.05, + "learning_rate": 0.00029622868605817457, + "loss": 2.8171, + "theoretical_loss": 3.545742863593637, + "tokens_seen": 1364848640 + }, + { + "epoch": 4.05, + "learning_rate": 0.0002962186559679037, + "loss": 2.6755, + "theoretical_loss": 3.5457275794479064, + "tokens_seen": 1364914176 + }, + { + "epoch": 4.05, + "learning_rate": 0.00029620862587763293, + "loss": 2.5866, + "theoretical_loss": 3.5457122962414935, + "tokens_seen": 1364979712 + }, + { + "epoch": 4.05, + "learning_rate": 0.00029619859578736206, + "loss": 2.6154, + "theoretical_loss": 3.5456970139742947, + "tokens_seen": 1365045248 + }, + { + "epoch": 4.05, + "learning_rate": 0.0002961885656970913, + "loss": 2.6771, + "theoretical_loss": 3.545681732646208, + "tokens_seen": 1365110784 + }, + { + "epoch": 4.05, + "learning_rate": 0.0002961785356068205, + "loss": 2.8346, + "theoretical_loss": 3.54566645225713, + "tokens_seen": 1365176320 + }, + { + "epoch": 4.05, + "learning_rate": 0.00029616850551654966, + "loss": 2.6756, + "theoretical_loss": 3.5456511728069584, + "tokens_seen": 1365241856 + }, + { + "epoch": 4.05, + "learning_rate": 0.00029615847542627884, + "loss": 2.7741, + "theoretical_loss": 3.5456358942955903, + "tokens_seen": 1365307392 + }, + { + "epoch": 4.05, + "learning_rate": 0.000296148445336008, + "loss": 2.8655, + "theoretical_loss": 3.545620616722923, + "tokens_seen": 1365372928 + }, + { + "epoch": 4.05, + "learning_rate": 0.0002961384152457372, + "loss": 2.7293, + "theoretical_loss": 3.5456053400888536, + "tokens_seen": 1365438464 + }, + { + "epoch": 4.05, + "learning_rate": 0.00029612838515546643, + "loss": 2.7781, + "theoretical_loss": 3.54559006439328, + "tokens_seen": 1365504000 + }, + { + "epoch": 4.05, + "learning_rate": 0.00029611835506519556, + "loss": 2.8338, + "theoretical_loss": 3.545574789636098, + "tokens_seen": 1365569536 + }, + { + "epoch": 4.05, + "learning_rate": 0.0002961083249749248, + "loss": 2.719, + "theoretical_loss": 3.545559515817207, + "tokens_seen": 1365635072 + }, + { + "epoch": 4.05, + "learning_rate": 0.000296098294884654, + "loss": 2.5393, + "theoretical_loss": 3.5455442429365025, + "tokens_seen": 1365700608 + }, + { + "epoch": 4.05, + "learning_rate": 0.00029608826479438316, + "loss": 2.82, + "theoretical_loss": 3.545528970993883, + "tokens_seen": 1365766144 + }, + { + "epoch": 4.05, + "learning_rate": 0.00029607823470411234, + "loss": 2.5854, + "theoretical_loss": 3.545513699989246, + "tokens_seen": 1365831680 + }, + { + "epoch": 4.05, + "learning_rate": 0.0002960682046138415, + "loss": 2.6055, + "theoretical_loss": 3.545498429922487, + "tokens_seen": 1365897216 + }, + { + "epoch": 4.05, + "learning_rate": 0.0002960581745235707, + "loss": 2.7797, + "theoretical_loss": 3.545483160793506, + "tokens_seen": 1365962752 + }, + { + "epoch": 4.05, + "learning_rate": 0.00029604814443329994, + "loss": 2.7801, + "theoretical_loss": 3.5454678926021987, + "tokens_seen": 1366028288 + }, + { + "epoch": 4.05, + "learning_rate": 0.00029603811434302906, + "loss": 2.6195, + "theoretical_loss": 3.5454526253484633, + "tokens_seen": 1366093824 + }, + { + "epoch": 4.05, + "learning_rate": 0.0002960280842527583, + "loss": 2.8992, + "theoretical_loss": 3.5454373590321966, + "tokens_seen": 1366159360 + }, + { + "epoch": 4.05, + "learning_rate": 0.0002960180541624874, + "loss": 2.5448, + "theoretical_loss": 3.545422093653297, + "tokens_seen": 1366224896 + }, + { + "epoch": 4.05, + "learning_rate": 0.00029600802407221666, + "loss": 3.0915, + "theoretical_loss": 3.545406829211661, + "tokens_seen": 1366290432 + }, + { + "epoch": 4.05, + "learning_rate": 0.00029599799398194584, + "loss": 2.6512, + "theoretical_loss": 3.545391565707187, + "tokens_seen": 1366355968 + }, + { + "epoch": 4.05, + "objective/train/docs_used": 1521877, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7869224548339844, + "objective/train/theoretical_loss": 3.5453763031397716, + "objective/train/tokens_used": 1386881504, + "theoretical_loss": 3.5453763031397716, + "tokens_seen": 1366421504 + }, + { + "epoch": 4.05, + "learning_rate": 0.000295987963891675, + "loss": 2.62, + "theoretical_loss": 3.5453763031397716, + "tokens_seen": 1366421504 + }, + { + "epoch": 4.05, + "learning_rate": 0.0002959779338014042, + "loss": 2.8634, + "theoretical_loss": 3.545361041509313, + "tokens_seen": 1366487040 + }, + { + "epoch": 4.05, + "learning_rate": 0.0002959679037111334, + "loss": 2.8177, + "theoretical_loss": 3.5453457808157087, + "tokens_seen": 1366552576 + }, + { + "epoch": 4.05, + "learning_rate": 0.00029595787362086257, + "loss": 2.8319, + "theoretical_loss": 3.545330521058856, + "tokens_seen": 1366618112 + }, + { + "epoch": 4.05, + "learning_rate": 0.0002959478435305918, + "loss": 2.8518, + "theoretical_loss": 3.5453152622386526, + "tokens_seen": 1366683648 + }, + { + "epoch": 4.05, + "learning_rate": 0.00029593781344032093, + "loss": 2.6757, + "theoretical_loss": 3.545300004354996, + "tokens_seen": 1366749184 + }, + { + "epoch": 4.05, + "learning_rate": 0.00029592778335005016, + "loss": 2.5993, + "theoretical_loss": 3.545284747407784, + "tokens_seen": 1366814720 + }, + { + "epoch": 4.05, + "learning_rate": 0.00029591775325977935, + "loss": 2.7115, + "theoretical_loss": 3.5452694913969145, + "tokens_seen": 1366880256 + }, + { + "epoch": 4.05, + "learning_rate": 0.00029590772316950853, + "loss": 2.9373, + "theoretical_loss": 3.5452542363222843, + "tokens_seen": 1366945792 + }, + { + "epoch": 4.05, + "learning_rate": 0.0002958976930792377, + "loss": 2.7642, + "theoretical_loss": 3.5452389821837915, + "tokens_seen": 1367011328 + }, + { + "epoch": 4.05, + "learning_rate": 0.0002958876629889669, + "loss": 2.6147, + "theoretical_loss": 3.545223728981334, + "tokens_seen": 1367076864 + }, + { + "epoch": 4.05, + "learning_rate": 0.00029587763289869607, + "loss": 2.5695, + "theoretical_loss": 3.5452084767148095, + "tokens_seen": 1367142400 + }, + { + "epoch": 4.05, + "learning_rate": 0.0002958676028084253, + "loss": 2.5337, + "theoretical_loss": 3.545193225384115, + "tokens_seen": 1367207936 + }, + { + "epoch": 4.05, + "learning_rate": 0.00029585757271815443, + "loss": 2.5786, + "theoretical_loss": 3.545177974989149, + "tokens_seen": 1367273472 + }, + { + "epoch": 4.05, + "learning_rate": 0.00029584754262788367, + "loss": 2.6016, + "theoretical_loss": 3.545162725529809, + "tokens_seen": 1367339008 + }, + { + "epoch": 4.05, + "learning_rate": 0.0002958375125376128, + "loss": 2.6825, + "theoretical_loss": 3.545147477005993, + "tokens_seen": 1367404544 + }, + { + "epoch": 4.05, + "learning_rate": 0.00029582748244734203, + "loss": 2.7878, + "theoretical_loss": 3.545132229417599, + "tokens_seen": 1367470080 + }, + { + "epoch": 4.05, + "learning_rate": 0.0002958174523570712, + "loss": 2.369, + "theoretical_loss": 3.5451169827645233, + "tokens_seen": 1367535616 + }, + { + "epoch": 4.05, + "learning_rate": 0.0002958074222668004, + "loss": 2.4724, + "theoretical_loss": 3.5451017370466653, + "tokens_seen": 1367601152 + }, + { + "epoch": 4.05, + "learning_rate": 0.0002957973921765296, + "loss": 2.6906, + "theoretical_loss": 3.5450864922639225, + "tokens_seen": 1367666688 + }, + { + "epoch": 4.05, + "learning_rate": 0.0002957873620862588, + "loss": 2.7822, + "theoretical_loss": 3.5450712484161917, + "tokens_seen": 1367732224 + }, + { + "epoch": 4.05, + "learning_rate": 0.000295777331995988, + "loss": 2.6861, + "theoretical_loss": 3.5450560055033717, + "tokens_seen": 1367797760 + }, + { + "epoch": 4.05, + "learning_rate": 0.00029576730190571717, + "loss": 2.868, + "theoretical_loss": 3.5450407635253605, + "tokens_seen": 1367863296 + }, + { + "epoch": 4.05, + "learning_rate": 0.00029575727181544635, + "loss": 2.5099, + "theoretical_loss": 3.5450255224820557, + "tokens_seen": 1367928832 + }, + { + "epoch": 4.05, + "learning_rate": 0.00029574724172517553, + "loss": 2.6857, + "theoretical_loss": 3.5450102823733554, + "tokens_seen": 1367994368 + }, + { + "epoch": 4.05, + "objective/train/docs_used": 1523240, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.509470224380493, + "objective/train/theoretical_loss": 3.5449950431991573, + "objective/train/tokens_used": 1388519904, + "theoretical_loss": 3.5449950431991573, + "tokens_seen": 1368059904 + }, + { + "epoch": 4.05, + "learning_rate": 0.00029573721163490477, + "loss": 2.6579, + "theoretical_loss": 3.5449950431991573, + "tokens_seen": 1368059904 + }, + { + "epoch": 4.05, + "learning_rate": 0.0002957271815446339, + "loss": 2.673, + "theoretical_loss": 3.544979804959359, + "tokens_seen": 1368125440 + }, + { + "epoch": 4.05, + "learning_rate": 0.00029571715145436313, + "loss": 2.5067, + "theoretical_loss": 3.544964567653859, + "tokens_seen": 1368190976 + }, + { + "epoch": 4.05, + "learning_rate": 0.00029570712136409226, + "loss": 2.6669, + "theoretical_loss": 3.544949331282555, + "tokens_seen": 1368256512 + }, + { + "epoch": 4.05, + "learning_rate": 0.0002956970912738215, + "loss": 2.496, + "theoretical_loss": 3.544934095845345, + "tokens_seen": 1368322048 + }, + { + "epoch": 4.05, + "learning_rate": 0.0002956870611835507, + "loss": 2.662, + "theoretical_loss": 3.5449188613421274, + "tokens_seen": 1368387584 + }, + { + "epoch": 4.05, + "learning_rate": 0.00029567703109327986, + "loss": 2.8064, + "theoretical_loss": 3.5449036277727997, + "tokens_seen": 1368453120 + }, + { + "epoch": 4.05, + "learning_rate": 0.00029566700100300904, + "loss": 2.6266, + "theoretical_loss": 3.54488839513726, + "tokens_seen": 1368518656 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002956569709127382, + "loss": 2.9046, + "theoretical_loss": 3.5448731634354065, + "tokens_seen": 1368584192 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002956469408224674, + "loss": 2.6641, + "theoretical_loss": 3.5448579326671372, + "tokens_seen": 1368649728 + }, + { + "epoch": 4.06, + "learning_rate": 0.00029563691073219663, + "loss": 2.8689, + "theoretical_loss": 3.5448427028323506, + "tokens_seen": 1368715264 + }, + { + "epoch": 4.06, + "learning_rate": 0.00029562688064192576, + "loss": 2.6824, + "theoretical_loss": 3.544827473930944, + "tokens_seen": 1368780800 + }, + { + "epoch": 4.06, + "learning_rate": 0.000295616850551655, + "loss": 2.6393, + "theoretical_loss": 3.5448122459628157, + "tokens_seen": 1368846336 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002956068204613842, + "loss": 2.5224, + "theoretical_loss": 3.544797018927864, + "tokens_seen": 1368911872 + }, + { + "epoch": 4.06, + "learning_rate": 0.00029559679037111336, + "loss": 2.7649, + "theoretical_loss": 3.5447817928259875, + "tokens_seen": 1368977408 + }, + { + "epoch": 4.06, + "learning_rate": 0.00029558676028084254, + "loss": 2.516, + "theoretical_loss": 3.544766567657084, + "tokens_seen": 1369042944 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002955767301905717, + "loss": 2.4305, + "theoretical_loss": 3.5447513434210514, + "tokens_seen": 1369108480 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002955667001003009, + "loss": 2.9265, + "theoretical_loss": 3.544736120117788, + "tokens_seen": 1369174016 + }, + { + "epoch": 4.06, + "learning_rate": 0.00029555667001003014, + "loss": 2.8476, + "theoretical_loss": 3.544720897747192, + "tokens_seen": 1369239552 + }, + { + "epoch": 4.06, + "learning_rate": 0.00029554663991975926, + "loss": 2.7277, + "theoretical_loss": 3.544705676309162, + "tokens_seen": 1369305088 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002955366098294885, + "loss": 2.6965, + "theoretical_loss": 3.544690455803596, + "tokens_seen": 1369370624 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002955265797392176, + "loss": 2.6146, + "theoretical_loss": 3.5446752362303924, + "tokens_seen": 1369436160 + }, + { + "epoch": 4.06, + "learning_rate": 0.00029551654964894686, + "loss": 2.7972, + "theoretical_loss": 3.5446600175894485, + "tokens_seen": 1369501696 + }, + { + "epoch": 4.06, + "learning_rate": 0.00029550651955867604, + "loss": 2.6935, + "theoretical_loss": 3.544644799880664, + "tokens_seen": 1369567232 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002954964894684052, + "loss": 2.7009, + "theoretical_loss": 3.5446295831039363, + "tokens_seen": 1369632768 + }, + { + "epoch": 4.06, + "objective/train/docs_used": 1523836, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.621335744857788, + "objective/train/theoretical_loss": 3.5446143672591646, + "objective/train/tokens_used": 1390158304, + "theoretical_loss": 3.5446143672591646, + "tokens_seen": 1369698304 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002954864593781344, + "loss": 2.811, + "theoretical_loss": 3.5446143672591646, + "tokens_seen": 1369698304 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002954764292878636, + "loss": 2.7063, + "theoretical_loss": 3.5445991523462457, + "tokens_seen": 1369763840 + }, + { + "epoch": 4.06, + "learning_rate": 0.00029546639919759277, + "loss": 2.5469, + "theoretical_loss": 3.5445839383650797, + "tokens_seen": 1369829376 + }, + { + "epoch": 4.06, + "learning_rate": 0.000295456369107322, + "loss": 2.6845, + "theoretical_loss": 3.544568725315564, + "tokens_seen": 1369894912 + }, + { + "epoch": 4.06, + "learning_rate": 0.00029544633901705113, + "loss": 2.4527, + "theoretical_loss": 3.544553513197597, + "tokens_seen": 1369960448 + }, + { + "epoch": 4.06, + "learning_rate": 0.00029543630892678037, + "loss": 2.7605, + "theoretical_loss": 3.5445383020110772, + "tokens_seen": 1370025984 + }, + { + "epoch": 4.06, + "learning_rate": 0.00029542627883650955, + "loss": 2.7853, + "theoretical_loss": 3.544523091755903, + "tokens_seen": 1370091520 + }, + { + "epoch": 4.06, + "learning_rate": 0.00029541624874623873, + "loss": 2.4475, + "theoretical_loss": 3.544507882431973, + "tokens_seen": 1370157056 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002954062186559679, + "loss": 2.8883, + "theoretical_loss": 3.5444926740391853, + "tokens_seen": 1370222592 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002953961885656971, + "loss": 2.7648, + "theoretical_loss": 3.5444774665774395, + "tokens_seen": 1370288128 + }, + { + "epoch": 4.06, + "learning_rate": 0.00029538615847542627, + "loss": 2.8952, + "theoretical_loss": 3.5444622600466325, + "tokens_seen": 1370353664 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002953761283851555, + "loss": 2.6118, + "theoretical_loss": 3.544447054446663, + "tokens_seen": 1370419200 + }, + { + "epoch": 4.06, + "learning_rate": 0.00029536609829488463, + "loss": 2.7484, + "theoretical_loss": 3.5444318497774305, + "tokens_seen": 1370484736 + }, + { + "epoch": 4.06, + "learning_rate": 0.00029535606820461387, + "loss": 2.6515, + "theoretical_loss": 3.544416646038833, + "tokens_seen": 1370550272 + }, + { + "epoch": 4.06, + "learning_rate": 0.000295346038114343, + "loss": 2.5422, + "theoretical_loss": 3.544401443230769, + "tokens_seen": 1370615808 + }, + { + "epoch": 4.06, + "learning_rate": 0.00029533600802407223, + "loss": 2.587, + "theoretical_loss": 3.544386241353137, + "tokens_seen": 1370681344 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002953259779338014, + "loss": 2.7654, + "theoretical_loss": 3.544371040405836, + "tokens_seen": 1370746880 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002953159478435306, + "loss": 2.4942, + "theoretical_loss": 3.5443558403887643, + "tokens_seen": 1370812416 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002953059177532598, + "loss": 2.6671, + "theoretical_loss": 3.5443406413018197, + "tokens_seen": 1370877952 + }, + { + "epoch": 4.06, + "learning_rate": 0.000295295887662989, + "loss": 2.5489, + "theoretical_loss": 3.544325443144902, + "tokens_seen": 1370943488 + }, + { + "epoch": 4.06, + "learning_rate": 0.00029528585757271814, + "loss": 2.5203, + "theoretical_loss": 3.5443102459179094, + "tokens_seen": 1371009024 + }, + { + "epoch": 4.06, + "learning_rate": 0.00029527582748244737, + "loss": 2.9006, + "theoretical_loss": 3.5442950496207413, + "tokens_seen": 1371074560 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002952657973921765, + "loss": 2.9251, + "theoretical_loss": 3.5442798542532947, + "tokens_seen": 1371140096 + }, + { + "epoch": 4.06, + "learning_rate": 0.00029525576730190573, + "loss": 2.6135, + "theoretical_loss": 3.54426465981547, + "tokens_seen": 1371205632 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002952457372116349, + "loss": 2.7752, + "theoretical_loss": 3.544249466307164, + "tokens_seen": 1371271168 + }, + { + "epoch": 4.06, + "objective/train/docs_used": 1525089, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.015775203704834, + "objective/train/theoretical_loss": 3.5442342737282777, + "objective/train/tokens_used": 1391796704, + "theoretical_loss": 3.5442342737282777, + "tokens_seen": 1371336704 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002952357071213641, + "loss": 2.9369, + "theoretical_loss": 3.5442342737282777, + "tokens_seen": 1371336704 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002952256770310933, + "loss": 2.5958, + "theoretical_loss": 3.5442190820787083, + "tokens_seen": 1371402240 + }, + { + "epoch": 4.06, + "learning_rate": 0.00029521564694082246, + "loss": 2.8745, + "theoretical_loss": 3.544203891358355, + "tokens_seen": 1371467776 + }, + { + "epoch": 4.06, + "learning_rate": 0.00029520561685055164, + "loss": 2.5521, + "theoretical_loss": 3.5441887015671165, + "tokens_seen": 1371533312 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002951955867602809, + "loss": 2.7852, + "theoretical_loss": 3.5441735127048917, + "tokens_seen": 1371598848 + }, + { + "epoch": 4.06, + "learning_rate": 0.00029518555667001, + "loss": 2.6479, + "theoretical_loss": 3.544158324771579, + "tokens_seen": 1371664384 + }, + { + "epoch": 4.06, + "learning_rate": 0.00029517552657973924, + "loss": 2.8104, + "theoretical_loss": 3.5441431377670773, + "tokens_seen": 1371729920 + }, + { + "epoch": 4.06, + "learning_rate": 0.00029516549648946836, + "loss": 2.7465, + "theoretical_loss": 3.5441279516912862, + "tokens_seen": 1371795456 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002951554663991976, + "loss": 2.5285, + "theoretical_loss": 3.5441127665441035, + "tokens_seen": 1371860992 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002951454363089268, + "loss": 2.6205, + "theoretical_loss": 3.5440975823254286, + "tokens_seen": 1371926528 + }, + { + "epoch": 4.06, + "learning_rate": 0.00029513540621865596, + "loss": 2.5794, + "theoretical_loss": 3.5440823990351604, + "tokens_seen": 1371992064 + }, + { + "epoch": 4.06, + "learning_rate": 0.00029512537612838514, + "loss": 2.6045, + "theoretical_loss": 3.5440672166731977, + "tokens_seen": 1372057600 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002951153460381144, + "loss": 2.8003, + "theoretical_loss": 3.5440520352394396, + "tokens_seen": 1372123136 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002951053159478435, + "loss": 2.9421, + "theoretical_loss": 3.5440368547337844, + "tokens_seen": 1372188672 + }, + { + "epoch": 4.06, + "learning_rate": 0.00029509528585757274, + "loss": 2.5751, + "theoretical_loss": 3.5440216751561318, + "tokens_seen": 1372254208 + }, + { + "epoch": 4.06, + "learning_rate": 0.00029508525576730187, + "loss": 2.8673, + "theoretical_loss": 3.5440064965063804, + "tokens_seen": 1372319744 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002950752256770311, + "loss": 2.8663, + "theoretical_loss": 3.543991318784429, + "tokens_seen": 1372385280 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002950651955867603, + "loss": 2.6738, + "theoretical_loss": 3.5439761419901767, + "tokens_seen": 1372450816 + }, + { + "epoch": 4.06, + "learning_rate": 0.00029505516549648946, + "loss": 2.6099, + "theoretical_loss": 3.5439609661235227, + "tokens_seen": 1372516352 + }, + { + "epoch": 4.06, + "learning_rate": 0.00029504513540621865, + "loss": 2.6654, + "theoretical_loss": 3.543945791184366, + "tokens_seen": 1372581888 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002950351053159478, + "loss": 2.9022, + "theoretical_loss": 3.5439306171726055, + "tokens_seen": 1372647424 + }, + { + "epoch": 4.06, + "learning_rate": 0.00029502507522567706, + "loss": 2.6579, + "theoretical_loss": 3.54391544408814, + "tokens_seen": 1372712960 + }, + { + "epoch": 4.06, + "learning_rate": 0.00029501504513540624, + "loss": 2.8065, + "theoretical_loss": 3.5439002719308696, + "tokens_seen": 1372778496 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002950050150451354, + "loss": 2.8156, + "theoretical_loss": 3.5438851007006917, + "tokens_seen": 1372844032 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002949949849548646, + "loss": 2.6603, + "theoretical_loss": 3.543869930397507, + "tokens_seen": 1372909568 + }, + { + "epoch": 4.06, + "objective/train/docs_used": 1525961, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.1202309131622314, + "objective/train/theoretical_loss": 3.5438547610212137, + "objective/train/tokens_used": 1393435104, + "theoretical_loss": 3.5438547610212137, + "tokens_seen": 1372975104 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002949849548645938, + "loss": 2.8852, + "theoretical_loss": 3.5438547610212137, + "tokens_seen": 1372975104 + }, + { + "epoch": 4.06, + "learning_rate": 0.00029497492477432297, + "loss": 2.841, + "theoretical_loss": 3.543839592571711, + "tokens_seen": 1373040640 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002949648946840522, + "loss": 2.7847, + "theoretical_loss": 3.543824425048898, + "tokens_seen": 1373106176 + }, + { + "epoch": 4.06, + "learning_rate": 0.00029495486459378133, + "loss": 2.6657, + "theoretical_loss": 3.543809258452675, + "tokens_seen": 1373171712 + }, + { + "epoch": 4.06, + "learning_rate": 0.00029494483450351057, + "loss": 2.8445, + "theoretical_loss": 3.5437940927829388, + "tokens_seen": 1373237248 + }, + { + "epoch": 4.06, + "learning_rate": 0.00029493480441323975, + "loss": 2.8022, + "theoretical_loss": 3.5437789280395915, + "tokens_seen": 1373302784 + }, + { + "epoch": 4.06, + "learning_rate": 0.00029492477432296893, + "loss": 2.6647, + "theoretical_loss": 3.54376376422253, + "tokens_seen": 1373368320 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002949147442326981, + "loss": 2.7687, + "theoretical_loss": 3.5437486013316546, + "tokens_seen": 1373433856 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002949047141424273, + "loss": 2.8542, + "theoretical_loss": 3.5437334393668642, + "tokens_seen": 1373499392 + }, + { + "epoch": 4.06, + "learning_rate": 0.00029489468405215647, + "loss": 2.4105, + "theoretical_loss": 3.543718278328058, + "tokens_seen": 1373564928 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002948846539618857, + "loss": 2.8407, + "theoretical_loss": 3.5437031182151357, + "tokens_seen": 1373630464 + }, + { + "epoch": 4.06, + "learning_rate": 0.00029487462387161483, + "loss": 2.7031, + "theoretical_loss": 3.543687959027996, + "tokens_seen": 1373696000 + }, + { + "epoch": 4.06, + "learning_rate": 0.00029486459378134407, + "loss": 2.8304, + "theoretical_loss": 3.5436728007665392, + "tokens_seen": 1373761536 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002948545636910732, + "loss": 2.7599, + "theoretical_loss": 3.543657643430663, + "tokens_seen": 1373827072 + }, + { + "epoch": 4.06, + "learning_rate": 0.00029484453360080243, + "loss": 2.7486, + "theoretical_loss": 3.5436424870202683, + "tokens_seen": 1373892608 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002948345035105316, + "loss": 2.7511, + "theoretical_loss": 3.5436273315352538, + "tokens_seen": 1373958144 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002948244734202608, + "loss": 2.7855, + "theoretical_loss": 3.543612176975519, + "tokens_seen": 1374023680 + }, + { + "epoch": 4.06, + "learning_rate": 0.00029481444332999, + "loss": 2.836, + "theoretical_loss": 3.543597023340963, + "tokens_seen": 1374089216 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002948044132397192, + "loss": 2.4758, + "theoretical_loss": 3.5435818706314848, + "tokens_seen": 1374154752 + }, + { + "epoch": 4.06, + "learning_rate": 0.00029479438314944834, + "loss": 2.6613, + "theoretical_loss": 3.5435667188469853, + "tokens_seen": 1374220288 + }, + { + "epoch": 4.06, + "learning_rate": 0.00029478435305917757, + "loss": 2.6733, + "theoretical_loss": 3.5435515679873624, + "tokens_seen": 1374285824 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002947743229689067, + "loss": 2.6872, + "theoretical_loss": 3.5435364180525166, + "tokens_seen": 1374351360 + }, + { + "epoch": 4.06, + "learning_rate": 0.00029476429287863593, + "loss": 3.1353, + "theoretical_loss": 3.5435212690423463, + "tokens_seen": 1374416896 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002947542627883651, + "loss": 2.7879, + "theoretical_loss": 3.543506120956752, + "tokens_seen": 1374482432 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002947442326980943, + "loss": 2.5917, + "theoretical_loss": 3.543490973795633, + "tokens_seen": 1374547968 + }, + { + "epoch": 4.06, + "objective/train/docs_used": 1531216, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.648439407348633, + "objective/train/theoretical_loss": 3.543475827558888, + "objective/train/tokens_used": 1395073504, + "theoretical_loss": 3.543475827558888, + "tokens_seen": 1374613504 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002947342026078235, + "loss": 2.9129, + "theoretical_loss": 3.543475827558888, + "tokens_seen": 1374613504 + }, + { + "epoch": 4.06, + "learning_rate": 0.00029472417251755266, + "loss": 2.5568, + "theoretical_loss": 3.543460682246417, + "tokens_seen": 1374679040 + }, + { + "epoch": 4.06, + "learning_rate": 0.00029471414242728184, + "loss": 2.6088, + "theoretical_loss": 3.54344553785812, + "tokens_seen": 1374744576 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002947041123370111, + "loss": 2.8646, + "theoretical_loss": 3.5434303943938965, + "tokens_seen": 1374810112 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002946940822467402, + "loss": 2.8851, + "theoretical_loss": 3.543415251853645, + "tokens_seen": 1374875648 + }, + { + "epoch": 4.06, + "learning_rate": 0.00029468405215646944, + "loss": 2.6553, + "theoretical_loss": 3.5434001102372665, + "tokens_seen": 1374941184 + }, + { + "epoch": 4.06, + "learning_rate": 0.00029467402206619856, + "loss": 2.9477, + "theoretical_loss": 3.54338496954466, + "tokens_seen": 1375006720 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002946639919759278, + "loss": 2.605, + "theoretical_loss": 3.543369829775725, + "tokens_seen": 1375072256 + }, + { + "epoch": 4.06, + "learning_rate": 0.000294653961885657, + "loss": 2.9116, + "theoretical_loss": 3.543354690930361, + "tokens_seen": 1375137792 + }, + { + "epoch": 4.06, + "learning_rate": 0.00029464393179538616, + "loss": 2.7982, + "theoretical_loss": 3.543339553008468, + "tokens_seen": 1375203328 + }, + { + "epoch": 4.06, + "learning_rate": 0.00029463390170511534, + "loss": 2.6475, + "theoretical_loss": 3.543324416009945, + "tokens_seen": 1375268864 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002946238716148446, + "loss": 2.7576, + "theoretical_loss": 3.5433092799346926, + "tokens_seen": 1375334400 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002946138415245737, + "loss": 2.9175, + "theoretical_loss": 3.5432941447826103, + "tokens_seen": 1375399936 + }, + { + "epoch": 4.06, + "learning_rate": 0.00029460381143430294, + "loss": 2.9014, + "theoretical_loss": 3.543279010553597, + "tokens_seen": 1375465472 + }, + { + "epoch": 4.06, + "learning_rate": 0.00029459378134403207, + "loss": 2.6803, + "theoretical_loss": 3.5432638772475533, + "tokens_seen": 1375531008 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002945837512537613, + "loss": 2.816, + "theoretical_loss": 3.543248744864379, + "tokens_seen": 1375596544 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002945737211634905, + "loss": 2.6571, + "theoretical_loss": 3.5432336134039732, + "tokens_seen": 1375662080 + }, + { + "epoch": 4.06, + "learning_rate": 0.00029456369107321966, + "loss": 2.6177, + "theoretical_loss": 3.5432184828662363, + "tokens_seen": 1375727616 + }, + { + "epoch": 4.06, + "learning_rate": 0.00029455366098294885, + "loss": 2.7331, + "theoretical_loss": 3.543203353251067, + "tokens_seen": 1375793152 + }, + { + "epoch": 4.06, + "learning_rate": 0.000294543630892678, + "loss": 2.8117, + "theoretical_loss": 3.5431882245583664, + "tokens_seen": 1375858688 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002945336008024072, + "loss": 2.6867, + "theoretical_loss": 3.543173096788034, + "tokens_seen": 1375924224 + }, + { + "epoch": 4.06, + "learning_rate": 0.00029452357071213644, + "loss": 2.5584, + "theoretical_loss": 3.543157969939969, + "tokens_seen": 1375989760 + }, + { + "epoch": 4.06, + "learning_rate": 0.00029451354062186557, + "loss": 2.6233, + "theoretical_loss": 3.5431428440140724, + "tokens_seen": 1376055296 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002945035105315948, + "loss": 2.7276, + "theoretical_loss": 3.5431277190102426, + "tokens_seen": 1376120832 + }, + { + "epoch": 4.06, + "learning_rate": 0.00029449348044132393, + "loss": 2.9551, + "theoretical_loss": 3.543112594928381, + "tokens_seen": 1376186368 + }, + { + "epoch": 4.06, + "objective/train/docs_used": 1536193, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.979797601699829, + "objective/train/theoretical_loss": 3.543097471768386, + "objective/train/tokens_used": 1396711904, + "theoretical_loss": 3.543097471768386, + "tokens_seen": 1376251904 + }, + { + "epoch": 4.06, + "learning_rate": 0.00029448345035105317, + "loss": 2.8658, + "theoretical_loss": 3.543097471768386, + "tokens_seen": 1376251904 + }, + { + "epoch": 4.06, + "learning_rate": 0.00029447342026078235, + "loss": 2.7535, + "theoretical_loss": 3.5430823495301587, + "tokens_seen": 1376317440 + }, + { + "epoch": 4.06, + "learning_rate": 0.00029446339017051153, + "loss": 2.8096, + "theoretical_loss": 3.5430672282135984, + "tokens_seen": 1376382976 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002944533600802407, + "loss": 2.5962, + "theoretical_loss": 3.5430521078186055, + "tokens_seen": 1376448512 + }, + { + "epoch": 4.06, + "learning_rate": 0.00029444332998996995, + "loss": 2.9369, + "theoretical_loss": 3.5430369883450794, + "tokens_seen": 1376514048 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002944332998996991, + "loss": 2.8077, + "theoretical_loss": 3.5430218697929208, + "tokens_seen": 1376579584 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002944232698094283, + "loss": 2.5626, + "theoretical_loss": 3.5430067521620288, + "tokens_seen": 1376645120 + }, + { + "epoch": 4.06, + "learning_rate": 0.00029441323971915744, + "loss": 2.7621, + "theoretical_loss": 3.542991635452304, + "tokens_seen": 1376710656 + }, + { + "epoch": 4.06, + "learning_rate": 0.00029440320962888667, + "loss": 2.5922, + "theoretical_loss": 3.5429765196636462, + "tokens_seen": 1376776192 + }, + { + "epoch": 4.06, + "learning_rate": 0.00029439317953861585, + "loss": 2.659, + "theoretical_loss": 3.542961404795956, + "tokens_seen": 1376841728 + }, + { + "epoch": 4.06, + "learning_rate": 0.00029438314944834503, + "loss": 2.6147, + "theoretical_loss": 3.542946290849132, + "tokens_seen": 1376907264 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002943731193580742, + "loss": 2.6262, + "theoretical_loss": 3.5429311778230765, + "tokens_seen": 1376972800 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002943630892678034, + "loss": 2.7149, + "theoretical_loss": 3.5429160657176877, + "tokens_seen": 1377038336 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002943530591775326, + "loss": 2.8174, + "theoretical_loss": 3.5429009545328665, + "tokens_seen": 1377103872 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002943430290872618, + "loss": 2.9529, + "theoretical_loss": 3.542885844268513, + "tokens_seen": 1377169408 + }, + { + "epoch": 4.06, + "learning_rate": 0.00029433299899699094, + "loss": 2.9298, + "theoretical_loss": 3.5428707349245263, + "tokens_seen": 1377234944 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002943229689067202, + "loss": 2.846, + "theoretical_loss": 3.5428556265008084, + "tokens_seen": 1377300480 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002943129388164493, + "loss": 2.5321, + "theoretical_loss": 3.5428405189972585, + "tokens_seen": 1377366016 + }, + { + "epoch": 4.06, + "learning_rate": 0.00029430290872617854, + "loss": 2.5836, + "theoretical_loss": 3.5428254124137766, + "tokens_seen": 1377431552 + }, + { + "epoch": 4.06, + "learning_rate": 0.00029429287863590777, + "loss": 2.6707, + "theoretical_loss": 3.5428103067502628, + "tokens_seen": 1377497088 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002942828485456369, + "loss": 2.6525, + "theoretical_loss": 3.5427952020066176, + "tokens_seen": 1377562624 + }, + { + "epoch": 4.06, + "learning_rate": 0.00029427281845536613, + "loss": 2.6044, + "theoretical_loss": 3.5427800981827415, + "tokens_seen": 1377628160 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002942627883650953, + "loss": 2.6824, + "theoretical_loss": 3.5427649952785343, + "tokens_seen": 1377693696 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002942527582748245, + "loss": 2.5064, + "theoretical_loss": 3.5427498932938972, + "tokens_seen": 1377759232 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002942427281845537, + "loss": 2.8199, + "theoretical_loss": 3.5427347922287287, + "tokens_seen": 1377824768 + }, + { + "epoch": 4.06, + "objective/train/docs_used": 1541285, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9072976112365723, + "objective/train/theoretical_loss": 3.5427196920829305, + "objective/train/tokens_used": 1398350304, + "theoretical_loss": 3.5427196920829305, + "tokens_seen": 1377890304 + }, + { + "epoch": 4.06, + "learning_rate": 0.00029423269809428286, + "loss": 2.781, + "theoretical_loss": 3.5427196920829305, + "tokens_seen": 1377890304 + }, + { + "epoch": 4.06, + "learning_rate": 0.00029422266800401204, + "loss": 2.6368, + "theoretical_loss": 3.5427045928564027, + "tokens_seen": 1377955840 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002942126379137413, + "loss": 2.9298, + "theoretical_loss": 3.5426894945490446, + "tokens_seen": 1378021376 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002942026078234704, + "loss": 2.6657, + "theoretical_loss": 3.5426743971607584, + "tokens_seen": 1378086912 + }, + { + "epoch": 4.06, + "learning_rate": 0.00029419257773319964, + "loss": 2.7524, + "theoretical_loss": 3.542659300691443, + "tokens_seen": 1378152448 + }, + { + "epoch": 4.06, + "learning_rate": 0.00029418254764292876, + "loss": 2.7688, + "theoretical_loss": 3.5426442051409985, + "tokens_seen": 1378217984 + }, + { + "epoch": 4.06, + "learning_rate": 0.000294172517552658, + "loss": 2.8809, + "theoretical_loss": 3.542629110509327, + "tokens_seen": 1378283520 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002941624874623872, + "loss": 2.8175, + "theoretical_loss": 3.542614016796327, + "tokens_seen": 1378349056 + }, + { + "epoch": 4.06, + "learning_rate": 0.00029415245737211636, + "loss": 2.5861, + "theoretical_loss": 3.5425989240019, + "tokens_seen": 1378414592 + }, + { + "epoch": 4.06, + "learning_rate": 0.00029414242728184554, + "loss": 2.547, + "theoretical_loss": 3.5425838321259464, + "tokens_seen": 1378480128 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002941323971915748, + "loss": 2.653, + "theoretical_loss": 3.542568741168366, + "tokens_seen": 1378545664 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002941223671013039, + "loss": 2.7472, + "theoretical_loss": 3.54255365112906, + "tokens_seen": 1378611200 + }, + { + "epoch": 4.06, + "learning_rate": 0.00029411233701103314, + "loss": 2.5158, + "theoretical_loss": 3.5425385620079286, + "tokens_seen": 1378676736 + }, + { + "epoch": 4.06, + "learning_rate": 0.00029410230692076227, + "loss": 2.7334, + "theoretical_loss": 3.5425234738048723, + "tokens_seen": 1378742272 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002940922768304915, + "loss": 2.7095, + "theoretical_loss": 3.5425083865197915, + "tokens_seen": 1378807808 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002940822467402207, + "loss": 2.6454, + "theoretical_loss": 3.5424933001525867, + "tokens_seen": 1378873344 + }, + { + "epoch": 4.06, + "learning_rate": 0.00029407221664994986, + "loss": 2.6793, + "theoretical_loss": 3.5424782147031584, + "tokens_seen": 1378938880 + }, + { + "epoch": 4.06, + "learning_rate": 0.00029406218655967905, + "loss": 2.7721, + "theoretical_loss": 3.5424631301714076, + "tokens_seen": 1379004416 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002940521564694082, + "loss": 2.8642, + "theoretical_loss": 3.5424480465572343, + "tokens_seen": 1379069952 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002940421263791374, + "loss": 2.719, + "theoretical_loss": 3.5424329638605396, + "tokens_seen": 1379135488 + }, + { + "epoch": 4.06, + "learning_rate": 0.00029403209628886664, + "loss": 2.6414, + "theoretical_loss": 3.5424178820812235, + "tokens_seen": 1379201024 + }, + { + "epoch": 4.06, + "learning_rate": 0.00029402206619859577, + "loss": 2.6062, + "theoretical_loss": 3.542402801219187, + "tokens_seen": 1379266560 + }, + { + "epoch": 4.06, + "learning_rate": 0.000294012036108325, + "loss": 2.721, + "theoretical_loss": 3.5423877212743307, + "tokens_seen": 1379332096 + }, + { + "epoch": 4.06, + "learning_rate": 0.00029400200601805413, + "loss": 2.6475, + "theoretical_loss": 3.542372642246555, + "tokens_seen": 1379397632 + }, + { + "epoch": 4.06, + "learning_rate": 0.00029399197592778337, + "loss": 2.6133, + "theoretical_loss": 3.542357564135761, + "tokens_seen": 1379463168 + }, + { + "epoch": 4.06, + "objective/train/docs_used": 1546362, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.725088119506836, + "objective/train/theoretical_loss": 3.542342486941849, + "objective/train/tokens_used": 1399988704, + "theoretical_loss": 3.542342486941849, + "tokens_seen": 1379528704 + }, + { + "epoch": 4.06, + "learning_rate": 0.00029398194583751255, + "loss": 2.7855, + "theoretical_loss": 3.542342486941849, + "tokens_seen": 1379528704 + }, + { + "epoch": 4.06, + "learning_rate": 0.00029397191574724173, + "loss": 2.7274, + "theoretical_loss": 3.5423274106647202, + "tokens_seen": 1379594240 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002939618856569709, + "loss": 2.7593, + "theoretical_loss": 3.542312335304275, + "tokens_seen": 1379659776 + }, + { + "epoch": 4.06, + "learning_rate": 0.00029395185556670015, + "loss": 2.5319, + "theoretical_loss": 3.5422972608604133, + "tokens_seen": 1379725312 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002939418254764293, + "loss": 2.5069, + "theoretical_loss": 3.5422821873330372, + "tokens_seen": 1379790848 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002939317953861585, + "loss": 2.8197, + "theoretical_loss": 3.542267114722047, + "tokens_seen": 1379856384 + }, + { + "epoch": 4.06, + "learning_rate": 0.00029392176529588764, + "loss": 2.6525, + "theoretical_loss": 3.542252043027343, + "tokens_seen": 1379921920 + }, + { + "epoch": 4.06, + "learning_rate": 0.00029391173520561687, + "loss": 2.783, + "theoretical_loss": 3.5422369722488263, + "tokens_seen": 1379987456 + }, + { + "epoch": 4.06, + "learning_rate": 0.00029390170511534605, + "loss": 2.641, + "theoretical_loss": 3.5422219023863977, + "tokens_seen": 1380052992 + }, + { + "epoch": 4.06, + "learning_rate": 0.00029389167502507523, + "loss": 2.7446, + "theoretical_loss": 3.542206833439958, + "tokens_seen": 1380118528 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002938816449348044, + "loss": 2.8127, + "theoretical_loss": 3.5421917654094086, + "tokens_seen": 1380184064 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002938716148445336, + "loss": 2.7673, + "theoretical_loss": 3.542176698294649, + "tokens_seen": 1380249600 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002938615847542628, + "loss": 2.7564, + "theoretical_loss": 3.5421616320955813, + "tokens_seen": 1380315136 + }, + { + "epoch": 4.06, + "learning_rate": 0.000293851554663992, + "loss": 2.6335, + "theoretical_loss": 3.5421465668121064, + "tokens_seen": 1380380672 + }, + { + "epoch": 4.06, + "learning_rate": 0.00029384152457372114, + "loss": 2.8381, + "theoretical_loss": 3.542131502444124, + "tokens_seen": 1380446208 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002938314944834504, + "loss": 2.9114, + "theoretical_loss": 3.542116438991536, + "tokens_seen": 1380511744 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002938214643931795, + "loss": 2.6381, + "theoretical_loss": 3.542101376454243, + "tokens_seen": 1380577280 + }, + { + "epoch": 4.06, + "learning_rate": 0.00029381143430290874, + "loss": 2.6397, + "theoretical_loss": 3.5420863148321464, + "tokens_seen": 1380642816 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002938014042126379, + "loss": 2.5404, + "theoretical_loss": 3.5420712541251462, + "tokens_seen": 1380708352 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002937913741223671, + "loss": 2.5423, + "theoretical_loss": 3.5420561943331443, + "tokens_seen": 1380773888 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002937813440320963, + "loss": 2.795, + "theoretical_loss": 3.5420411354560413, + "tokens_seen": 1380839424 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002937713139418255, + "loss": 2.565, + "theoretical_loss": 3.542026077493738, + "tokens_seen": 1380904960 + }, + { + "epoch": 4.06, + "learning_rate": 0.00029376128385155464, + "loss": 2.636, + "theoretical_loss": 3.542011020446136, + "tokens_seen": 1380970496 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002937512537612839, + "loss": 2.8553, + "theoretical_loss": 3.541995964313136, + "tokens_seen": 1381036032 + }, + { + "epoch": 4.06, + "learning_rate": 0.000293741223671013, + "loss": 2.5084, + "theoretical_loss": 3.5419809090946384, + "tokens_seen": 1381101568 + }, + { + "epoch": 4.06, + "objective/train/docs_used": 1551454, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5593550205230713, + "objective/train/theoretical_loss": 3.5419658547905453, + "objective/train/tokens_used": 1401627104, + "theoretical_loss": 3.5419658547905453, + "tokens_seen": 1381167104 + }, + { + "epoch": 4.06, + "learning_rate": 0.00029373119358074224, + "loss": 2.5186, + "theoretical_loss": 3.5419658547905453, + "tokens_seen": 1381167104 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002937211634904714, + "loss": 2.4587, + "theoretical_loss": 3.5419508014007572, + "tokens_seen": 1381232640 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002937111334002006, + "loss": 2.6895, + "theoretical_loss": 3.5419357489251757, + "tokens_seen": 1381298176 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002937011033099298, + "loss": 2.7143, + "theoretical_loss": 3.541920697363701, + "tokens_seen": 1381363712 + }, + { + "epoch": 4.06, + "learning_rate": 0.00029369107321965896, + "loss": 2.7308, + "theoretical_loss": 3.5419056467162355, + "tokens_seen": 1381429248 + }, + { + "epoch": 4.06, + "learning_rate": 0.00029368104312938815, + "loss": 2.8324, + "theoretical_loss": 3.5418905969826793, + "tokens_seen": 1381494784 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002936710130391174, + "loss": 2.6625, + "theoretical_loss": 3.541875548162934, + "tokens_seen": 1381560320 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002936609829488465, + "loss": 2.7212, + "theoretical_loss": 3.5418605002569006, + "tokens_seen": 1381625856 + }, + { + "epoch": 4.06, + "learning_rate": 0.00029365095285857574, + "loss": 2.706, + "theoretical_loss": 3.5418454532644796, + "tokens_seen": 1381691392 + }, + { + "epoch": 4.06, + "learning_rate": 0.00029364092276830487, + "loss": 2.4649, + "theoretical_loss": 3.5418304071855737, + "tokens_seen": 1381756928 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002936308926780341, + "loss": 2.7022, + "theoretical_loss": 3.5418153620200834, + "tokens_seen": 1381822464 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002936208625877633, + "loss": 2.6936, + "theoretical_loss": 3.5418003177679096, + "tokens_seen": 1381888000 + }, + { + "epoch": 4.06, + "learning_rate": 0.00029361083249749247, + "loss": 2.9936, + "theoretical_loss": 3.541785274428954, + "tokens_seen": 1381953536 + }, + { + "epoch": 4.06, + "learning_rate": 0.00029360080240722165, + "loss": 2.7206, + "theoretical_loss": 3.5417702320031177, + "tokens_seen": 1382019072 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002935907723169509, + "loss": 2.5303, + "theoretical_loss": 3.541755190490302, + "tokens_seen": 1382084608 + }, + { + "epoch": 4.06, + "learning_rate": 0.00029358074222668, + "loss": 2.9346, + "theoretical_loss": 3.541740149890408, + "tokens_seen": 1382150144 + }, + { + "epoch": 4.06, + "learning_rate": 0.00029357071213640925, + "loss": 2.3615, + "theoretical_loss": 3.541725110203337, + "tokens_seen": 1382215680 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002935606820461384, + "loss": 2.7336, + "theoretical_loss": 3.541710071428991, + "tokens_seen": 1382281216 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002935506519558676, + "loss": 2.5758, + "theoretical_loss": 3.5416950335672706, + "tokens_seen": 1382346752 + }, + { + "epoch": 4.06, + "learning_rate": 0.00029354062186559684, + "loss": 2.6175, + "theoretical_loss": 3.541679996618077, + "tokens_seen": 1382412288 + }, + { + "epoch": 4.06, + "learning_rate": 0.00029353059177532597, + "loss": 2.7783, + "theoretical_loss": 3.541664960581312, + "tokens_seen": 1382477824 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002935205616850552, + "loss": 2.6292, + "theoretical_loss": 3.541649925456878, + "tokens_seen": 1382543360 + }, + { + "epoch": 4.06, + "learning_rate": 0.00029351053159478433, + "loss": 2.7404, + "theoretical_loss": 3.5416348912446742, + "tokens_seen": 1382608896 + }, + { + "epoch": 4.06, + "learning_rate": 0.00029350050150451357, + "loss": 2.5828, + "theoretical_loss": 3.541619857944604, + "tokens_seen": 1382674432 + }, + { + "epoch": 4.06, + "learning_rate": 0.00029349047141424275, + "loss": 2.6829, + "theoretical_loss": 3.5416048255565675, + "tokens_seen": 1382739968 + }, + { + "epoch": 4.06, + "objective/train/docs_used": 1552054, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.0920627117156982, + "objective/train/theoretical_loss": 3.5415897940804664, + "objective/train/tokens_used": 1403265504, + "theoretical_loss": 3.5415897940804664, + "tokens_seen": 1382805504 + }, + { + "epoch": 4.06, + "learning_rate": 0.00029348044132397193, + "loss": 2.8501, + "theoretical_loss": 3.5415897940804664, + "tokens_seen": 1382805504 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002934704112337011, + "loss": 2.7506, + "theoretical_loss": 3.541574763516203, + "tokens_seen": 1382871040 + }, + { + "epoch": 4.06, + "learning_rate": 0.00029346038114343035, + "loss": 2.7521, + "theoretical_loss": 3.5415597338636777, + "tokens_seen": 1382936576 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002934503510531595, + "loss": 2.869, + "theoretical_loss": 3.5415447051227926, + "tokens_seen": 1383002112 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002934403209628887, + "loss": 2.6461, + "theoretical_loss": 3.5415296772934495, + "tokens_seen": 1383067648 + }, + { + "epoch": 4.06, + "learning_rate": 0.00029343029087261784, + "loss": 2.7489, + "theoretical_loss": 3.541514650375549, + "tokens_seen": 1383133184 + }, + { + "epoch": 4.06, + "learning_rate": 0.00029342026078234707, + "loss": 2.9143, + "theoretical_loss": 3.541499624368993, + "tokens_seen": 1383198720 + }, + { + "epoch": 4.06, + "learning_rate": 0.00029341023069207625, + "loss": 2.7198, + "theoretical_loss": 3.5414845992736836, + "tokens_seen": 1383264256 + }, + { + "epoch": 4.06, + "learning_rate": 0.00029340020060180543, + "loss": 2.4439, + "theoretical_loss": 3.541469575089522, + "tokens_seen": 1383329792 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002933901705115346, + "loss": 2.6594, + "theoretical_loss": 3.5414545518164093, + "tokens_seen": 1383395328 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002933801404212638, + "loss": 2.6616, + "theoretical_loss": 3.541439529454248, + "tokens_seen": 1383460864 + }, + { + "epoch": 4.06, + "learning_rate": 0.000293370110330993, + "loss": 2.879, + "theoretical_loss": 3.541424508002939, + "tokens_seen": 1383526400 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002933600802407222, + "loss": 2.6416, + "theoretical_loss": 3.541409487462384, + "tokens_seen": 1383591936 + }, + { + "epoch": 4.06, + "learning_rate": 0.00029335005015045134, + "loss": 2.6155, + "theoretical_loss": 3.541394467832485, + "tokens_seen": 1383657472 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002933400200601806, + "loss": 2.6164, + "theoretical_loss": 3.541379449113144, + "tokens_seen": 1383723008 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002933299899699097, + "loss": 2.8243, + "theoretical_loss": 3.5413644313042614, + "tokens_seen": 1383788544 + }, + { + "epoch": 4.06, + "learning_rate": 0.00029331995987963894, + "loss": 2.6085, + "theoretical_loss": 3.5413494144057402, + "tokens_seen": 1383854080 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002933099297893681, + "loss": 2.6678, + "theoretical_loss": 3.541334398417481, + "tokens_seen": 1383919616 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002932998996990973, + "loss": 2.6854, + "theoretical_loss": 3.541319383339387, + "tokens_seen": 1383985152 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002932898696088265, + "loss": 2.4524, + "theoretical_loss": 3.5413043691713586, + "tokens_seen": 1384050688 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002932798395185557, + "loss": 2.6441, + "theoretical_loss": 3.541289355913298, + "tokens_seen": 1384116224 + }, + { + "epoch": 4.06, + "learning_rate": 0.00029326980942828484, + "loss": 2.7086, + "theoretical_loss": 3.5412743435651066, + "tokens_seen": 1384181760 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002932597793380141, + "loss": 2.6357, + "theoretical_loss": 3.541259332126687, + "tokens_seen": 1384247296 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002932497492477432, + "loss": 2.5468, + "theoretical_loss": 3.54124432159794, + "tokens_seen": 1384312832 + }, + { + "epoch": 4.06, + "learning_rate": 0.00029323971915747244, + "loss": 2.7114, + "theoretical_loss": 3.5412293119787686, + "tokens_seen": 1384378368 + }, + { + "epoch": 4.06, + "objective/train/docs_used": 1553388, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.496983051300049, + "objective/train/theoretical_loss": 3.5412143032690735, + "objective/train/tokens_used": 1404903904, + "theoretical_loss": 3.5412143032690735, + "tokens_seen": 1384443904 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002932296890672016, + "loss": 2.5614, + "theoretical_loss": 3.5412143032690735, + "tokens_seen": 1384443904 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002932196589769308, + "loss": 2.6653, + "theoretical_loss": 3.5411992954687572, + "tokens_seen": 1384509440 + }, + { + "epoch": 4.06, + "learning_rate": 0.00029320962888666, + "loss": 2.656, + "theoretical_loss": 3.541184288577721, + "tokens_seen": 1384574976 + }, + { + "epoch": 4.06, + "learning_rate": 0.00029319959879638916, + "loss": 2.7702, + "theoretical_loss": 3.5411692825958676, + "tokens_seen": 1384640512 + }, + { + "epoch": 4.06, + "learning_rate": 0.00029318956870611835, + "loss": 2.6418, + "theoretical_loss": 3.5411542775230984, + "tokens_seen": 1384706048 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002931795386158476, + "loss": 2.6088, + "theoretical_loss": 3.541139273359315, + "tokens_seen": 1384771584 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002931695085255767, + "loss": 2.9696, + "theoretical_loss": 3.54112427010442, + "tokens_seen": 1384837120 + }, + { + "epoch": 4.06, + "learning_rate": 0.00029315947843530594, + "loss": 2.6929, + "theoretical_loss": 3.541109267758315, + "tokens_seen": 1384902656 + }, + { + "epoch": 4.06, + "learning_rate": 0.00029314944834503507, + "loss": 2.9294, + "theoretical_loss": 3.541094266320902, + "tokens_seen": 1384968192 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002931394182547643, + "loss": 2.5998, + "theoretical_loss": 3.541079265792083, + "tokens_seen": 1385033728 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002931293881644935, + "loss": 2.5983, + "theoretical_loss": 3.5410642661717597, + "tokens_seen": 1385099264 + }, + { + "epoch": 4.06, + "learning_rate": 0.00029311935807422267, + "loss": 2.7155, + "theoretical_loss": 3.541049267459834, + "tokens_seen": 1385164800 + }, + { + "epoch": 4.06, + "learning_rate": 0.00029310932798395185, + "loss": 2.8903, + "theoretical_loss": 3.5410342696562083, + "tokens_seen": 1385230336 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002930992978936811, + "loss": 2.6472, + "theoretical_loss": 3.541019272760785, + "tokens_seen": 1385295872 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002930892678034102, + "loss": 2.8402, + "theoretical_loss": 3.5410042767734655, + "tokens_seen": 1385361408 + }, + { + "epoch": 4.06, + "learning_rate": 0.00029307923771313945, + "loss": 2.6837, + "theoretical_loss": 3.5409892816941513, + "tokens_seen": 1385426944 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002930692076228686, + "loss": 2.6073, + "theoretical_loss": 3.5409742875227463, + "tokens_seen": 1385492480 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002930591775325978, + "loss": 2.6603, + "theoretical_loss": 3.540959294259151, + "tokens_seen": 1385558016 + }, + { + "epoch": 4.06, + "learning_rate": 0.000293049147442327, + "loss": 2.7129, + "theoretical_loss": 3.540944301903268, + "tokens_seen": 1385623552 + }, + { + "epoch": 4.06, + "learning_rate": 0.00029303911735205617, + "loss": 2.4107, + "theoretical_loss": 3.540929310454999, + "tokens_seen": 1385689088 + }, + { + "epoch": 4.06, + "learning_rate": 0.00029302908726178535, + "loss": 2.8114, + "theoretical_loss": 3.540914319914247, + "tokens_seen": 1385754624 + }, + { + "epoch": 4.06, + "learning_rate": 0.00029301905717151453, + "loss": 2.68, + "theoretical_loss": 3.5408993302809133, + "tokens_seen": 1385820160 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002930090270812437, + "loss": 2.6964, + "theoretical_loss": 3.540884341554901, + "tokens_seen": 1385885696 + }, + { + "epoch": 4.06, + "learning_rate": 0.00029299899699097295, + "loss": 2.6005, + "theoretical_loss": 3.5408693537361113, + "tokens_seen": 1385951232 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002929889669007021, + "loss": 2.7721, + "theoretical_loss": 3.540854366824447, + "tokens_seen": 1386016768 + }, + { + "epoch": 4.06, + "objective/train/docs_used": 1554184, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.679152250289917, + "objective/train/theoretical_loss": 3.5408393808198104, + "objective/train/tokens_used": 1406542304, + "theoretical_loss": 3.5408393808198104, + "tokens_seen": 1386082304 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002929789368104313, + "loss": 2.7258, + "theoretical_loss": 3.5408393808198104, + "tokens_seen": 1386082304 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002929689067201605, + "loss": 2.9701, + "theoretical_loss": 3.540824395722103, + "tokens_seen": 1386147840 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002929588766298897, + "loss": 2.6006, + "theoretical_loss": 3.540809411531228, + "tokens_seen": 1386213376 + }, + { + "epoch": 4.06, + "learning_rate": 0.00029294884653961885, + "loss": 2.8074, + "theoretical_loss": 3.5407944282470867, + "tokens_seen": 1386278912 + }, + { + "epoch": 4.06, + "learning_rate": 0.00029293881644934804, + "loss": 2.5889, + "theoretical_loss": 3.540779445869582, + "tokens_seen": 1386344448 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002929287863590772, + "loss": 2.7228, + "theoretical_loss": 3.540764464398616, + "tokens_seen": 1386409984 + }, + { + "epoch": 4.06, + "learning_rate": 0.00029291875626880645, + "loss": 2.6788, + "theoretical_loss": 3.540749483834091, + "tokens_seen": 1386475520 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002929087261785356, + "loss": 2.8537, + "theoretical_loss": 3.540734504175909, + "tokens_seen": 1386541056 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002928986960882648, + "loss": 2.8571, + "theoretical_loss": 3.540719525423973, + "tokens_seen": 1386606592 + }, + { + "epoch": 4.06, + "learning_rate": 0.00029288866599799394, + "loss": 2.8249, + "theoretical_loss": 3.540704547578185, + "tokens_seen": 1386672128 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002928786359077232, + "loss": 2.819, + "theoretical_loss": 3.540689570638447, + "tokens_seen": 1386737664 + }, + { + "epoch": 4.06, + "learning_rate": 0.00029286860581745236, + "loss": 2.8209, + "theoretical_loss": 3.5406745946046616, + "tokens_seen": 1386803200 + }, + { + "epoch": 4.06, + "learning_rate": 0.00029285857572718154, + "loss": 2.8291, + "theoretical_loss": 3.540659619476732, + "tokens_seen": 1386868736 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002928485456369107, + "loss": 2.6679, + "theoretical_loss": 3.5406446452545595, + "tokens_seen": 1386934272 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002928385155466399, + "loss": 2.8749, + "theoretical_loss": 3.5406296719380475, + "tokens_seen": 1386999808 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002928284854563691, + "loss": 2.7219, + "theoretical_loss": 3.5406146995270973, + "tokens_seen": 1387065344 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002928184553660983, + "loss": 2.7416, + "theoretical_loss": 3.540599728021612, + "tokens_seen": 1387130880 + }, + { + "epoch": 4.06, + "learning_rate": 0.00029280842527582744, + "loss": 2.7386, + "theoretical_loss": 3.540584757421494, + "tokens_seen": 1387196416 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002927983951855567, + "loss": 2.4624, + "theoretical_loss": 3.5405697877266458, + "tokens_seen": 1387261952 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002927883650952859, + "loss": 2.6384, + "theoretical_loss": 3.54055481893697, + "tokens_seen": 1387327488 + }, + { + "epoch": 4.06, + "learning_rate": 0.00029277833500501504, + "loss": 2.6868, + "theoretical_loss": 3.540539851052369, + "tokens_seen": 1387393024 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002927683049147443, + "loss": 2.5707, + "theoretical_loss": 3.5405248840727452, + "tokens_seen": 1387458560 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002927582748244734, + "loss": 2.7434, + "theoretical_loss": 3.5405099179980013, + "tokens_seen": 1387524096 + }, + { + "epoch": 4.06, + "learning_rate": 0.00029274824473420264, + "loss": 2.6813, + "theoretical_loss": 3.5404949528280394, + "tokens_seen": 1387589632 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002927382146439318, + "loss": 2.73, + "theoretical_loss": 3.5404799885627627, + "tokens_seen": 1387655168 + }, + { + "epoch": 4.06, + "objective/train/docs_used": 1555725, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6399734020233154, + "objective/train/theoretical_loss": 3.540465025202074, + "objective/train/tokens_used": 1408180704, + "theoretical_loss": 3.540465025202074, + "tokens_seen": 1387720704 + }, + { + "epoch": 4.06, + "learning_rate": 0.000292728184553661, + "loss": 2.8601, + "theoretical_loss": 3.540465025202074, + "tokens_seen": 1387720704 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002927181544633902, + "loss": 2.7116, + "theoretical_loss": 3.540450062745875, + "tokens_seen": 1387786240 + }, + { + "epoch": 4.06, + "learning_rate": 0.00029270812437311936, + "loss": 2.6407, + "theoretical_loss": 3.540435101194069, + "tokens_seen": 1387851776 + }, + { + "epoch": 4.06, + "learning_rate": 0.00029269809428284855, + "loss": 2.7734, + "theoretical_loss": 3.5404201405465585, + "tokens_seen": 1387917312 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002926880641925778, + "loss": 2.8065, + "theoretical_loss": 3.540405180803246, + "tokens_seen": 1387982848 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002926780341023069, + "loss": 2.4892, + "theoretical_loss": 3.540390221964034, + "tokens_seen": 1388048384 + }, + { + "epoch": 4.06, + "learning_rate": 0.00029266800401203614, + "loss": 2.615, + "theoretical_loss": 3.5403752640288255, + "tokens_seen": 1388113920 + }, + { + "epoch": 4.06, + "learning_rate": 0.00029265797392176527, + "loss": 2.6055, + "theoretical_loss": 3.540360306997523, + "tokens_seen": 1388179456 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002926479438314945, + "loss": 2.7447, + "theoretical_loss": 3.5403453508700298, + "tokens_seen": 1388244992 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002926379137412237, + "loss": 2.6932, + "theoretical_loss": 3.5403303956462477, + "tokens_seen": 1388310528 + }, + { + "epoch": 4.06, + "learning_rate": 0.00029262788365095287, + "loss": 2.5505, + "theoretical_loss": 3.5403154413260802, + "tokens_seen": 1388376064 + }, + { + "epoch": 4.06, + "learning_rate": 0.00029261785356068205, + "loss": 2.877, + "theoretical_loss": 3.5403004879094295, + "tokens_seen": 1388441600 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002926078234704113, + "loss": 2.5885, + "theoretical_loss": 3.5402855353961984, + "tokens_seen": 1388507136 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002925977933801404, + "loss": 2.6713, + "theoretical_loss": 3.54027058378629, + "tokens_seen": 1388572672 + }, + { + "epoch": 4.06, + "learning_rate": 0.00029258776328986965, + "loss": 2.496, + "theoretical_loss": 3.540255633079607, + "tokens_seen": 1388638208 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002925777331995988, + "loss": 2.5759, + "theoretical_loss": 3.5402406832760525, + "tokens_seen": 1388703744 + }, + { + "epoch": 4.06, + "learning_rate": 0.000292567703109328, + "loss": 2.5836, + "theoretical_loss": 3.540225734375529, + "tokens_seen": 1388769280 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002925576730190572, + "loss": 2.5895, + "theoretical_loss": 3.540210786377939, + "tokens_seen": 1388834816 + }, + { + "epoch": 4.06, + "learning_rate": 0.00029254764292878637, + "loss": 2.6178, + "theoretical_loss": 3.5401958392831854, + "tokens_seen": 1388900352 + }, + { + "epoch": 4.06, + "learning_rate": 0.00029253761283851555, + "loss": 2.4121, + "theoretical_loss": 3.540180893091172, + "tokens_seen": 1388965888 + }, + { + "epoch": 4.06, + "learning_rate": 0.00029252758274824473, + "loss": 2.6844, + "theoretical_loss": 3.5401659478018006, + "tokens_seen": 1389031424 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002925175526579739, + "loss": 2.6171, + "theoretical_loss": 3.540151003414975, + "tokens_seen": 1389096960 + }, + { + "epoch": 4.06, + "learning_rate": 0.00029250752256770315, + "loss": 2.6658, + "theoretical_loss": 3.5401360599305973, + "tokens_seen": 1389162496 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002924974924774323, + "loss": 2.774, + "theoretical_loss": 3.540121117348571, + "tokens_seen": 1389228032 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002924874623871615, + "loss": 2.7155, + "theoretical_loss": 3.540106175668799, + "tokens_seen": 1389293568 + }, + { + "epoch": 4.06, + "objective/train/docs_used": 1556470, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5523812770843506, + "objective/train/theoretical_loss": 3.540091234891184, + "objective/train/tokens_used": 1409819104, + "theoretical_loss": 3.540091234891184, + "tokens_seen": 1389359104 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002924774322968907, + "loss": 2.7275, + "theoretical_loss": 3.540091234891184, + "tokens_seen": 1389359104 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002924674022066199, + "loss": 2.6922, + "theoretical_loss": 3.540076295015629, + "tokens_seen": 1389424640 + }, + { + "epoch": 4.06, + "learning_rate": 0.00029245737211634906, + "loss": 2.7791, + "theoretical_loss": 3.5400613560420373, + "tokens_seen": 1389490176 + }, + { + "epoch": 4.06, + "learning_rate": 0.00029244734202607824, + "loss": 2.8218, + "theoretical_loss": 3.5400464179703115, + "tokens_seen": 1389555712 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002924373119358074, + "loss": 2.4633, + "theoretical_loss": 3.540031480800355, + "tokens_seen": 1389621248 + }, + { + "epoch": 4.06, + "learning_rate": 0.00029242728184553665, + "loss": 2.8368, + "theoretical_loss": 3.5400165445320706, + "tokens_seen": 1389686784 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002924172517552658, + "loss": 2.6233, + "theoretical_loss": 3.540001609165362, + "tokens_seen": 1389752320 + }, + { + "epoch": 4.06, + "learning_rate": 0.000292407221664995, + "loss": 2.7833, + "theoretical_loss": 3.539986674700131, + "tokens_seen": 1389817856 + }, + { + "epoch": 4.06, + "learning_rate": 0.00029239719157472414, + "loss": 2.6574, + "theoretical_loss": 3.5399717411362817, + "tokens_seen": 1389883392 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002923871614844534, + "loss": 2.7321, + "theoretical_loss": 3.5399568084737165, + "tokens_seen": 1389948928 + }, + { + "epoch": 4.06, + "learning_rate": 0.00029237713139418256, + "loss": 2.7022, + "theoretical_loss": 3.539941876712339, + "tokens_seen": 1390014464 + }, + { + "epoch": 4.06, + "learning_rate": 0.00029236710130391174, + "loss": 2.8063, + "theoretical_loss": 3.5399269458520526, + "tokens_seen": 1390080000 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002923570712136409, + "loss": 2.786, + "theoretical_loss": 3.53991201589276, + "tokens_seen": 1390145536 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002923470411233701, + "loss": 2.6066, + "theoretical_loss": 3.539897086834364, + "tokens_seen": 1390211072 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002923370110330993, + "loss": 2.6857, + "theoretical_loss": 3.539882158676768, + "tokens_seen": 1390276608 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002923269809428285, + "loss": 2.6082, + "theoretical_loss": 3.5398672314198762, + "tokens_seen": 1390342144 + }, + { + "epoch": 4.06, + "learning_rate": 0.00029231695085255765, + "loss": 2.6024, + "theoretical_loss": 3.539852305063591, + "tokens_seen": 1390407680 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002923069207622869, + "loss": 2.6098, + "theoretical_loss": 3.539837379607815, + "tokens_seen": 1390473216 + }, + { + "epoch": 4.06, + "learning_rate": 0.00029229689067201606, + "loss": 2.5219, + "theoretical_loss": 3.539822455052452, + "tokens_seen": 1390538752 + }, + { + "epoch": 4.06, + "learning_rate": 0.00029228686058174524, + "loss": 2.6146, + "theoretical_loss": 3.539807531397406, + "tokens_seen": 1390604288 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002922768304914744, + "loss": 2.6021, + "theoretical_loss": 3.539792608642579, + "tokens_seen": 1390669824 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002922668004012036, + "loss": 2.8935, + "theoretical_loss": 3.539777686787875, + "tokens_seen": 1390735360 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002922567703109328, + "loss": 2.566, + "theoretical_loss": 3.539762765833197, + "tokens_seen": 1390800896 + }, + { + "epoch": 4.06, + "learning_rate": 0.000292246740220662, + "loss": 2.7801, + "theoretical_loss": 3.5397478457784484, + "tokens_seen": 1390866432 + }, + { + "epoch": 4.06, + "learning_rate": 0.00029223671013039115, + "loss": 2.6456, + "theoretical_loss": 3.5397329266235324, + "tokens_seen": 1390931968 + }, + { + "epoch": 4.06, + "objective/train/docs_used": 1557630, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9702374935150146, + "objective/train/theoretical_loss": 3.539718008368353, + "objective/train/tokens_used": 1411457504, + "theoretical_loss": 3.539718008368353, + "tokens_seen": 1390997504 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002922266800401204, + "loss": 2.6979, + "theoretical_loss": 3.539718008368353, + "tokens_seen": 1390997504 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002922166499498495, + "loss": 2.7617, + "theoretical_loss": 3.5397030910128127, + "tokens_seen": 1391063040 + }, + { + "epoch": 4.06, + "learning_rate": 0.00029220661985957875, + "loss": 2.7569, + "theoretical_loss": 3.539688174556815, + "tokens_seen": 1391128576 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002921965897693079, + "loss": 2.6994, + "theoretical_loss": 3.5396732590002635, + "tokens_seen": 1391194112 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002921865596790371, + "loss": 2.7891, + "theoretical_loss": 3.5396583443430623, + "tokens_seen": 1391259648 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002921765295887663, + "loss": 2.6115, + "theoretical_loss": 3.539643430585113, + "tokens_seen": 1391325184 + }, + { + "epoch": 4.06, + "learning_rate": 0.00029216649949849547, + "loss": 2.9884, + "theoretical_loss": 3.5396285177263214, + "tokens_seen": 1391390720 + }, + { + "epoch": 4.06, + "learning_rate": 0.00029215646940822465, + "loss": 2.7357, + "theoretical_loss": 3.539613605766589, + "tokens_seen": 1391456256 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002921464393179539, + "loss": 2.7335, + "theoretical_loss": 3.53959869470582, + "tokens_seen": 1391521792 + }, + { + "epoch": 4.06, + "learning_rate": 0.000292136409227683, + "loss": 2.7324, + "theoretical_loss": 3.5395837845439173, + "tokens_seen": 1391587328 + }, + { + "epoch": 4.06, + "learning_rate": 0.00029212637913741225, + "loss": 2.8466, + "theoretical_loss": 3.5395688752807857, + "tokens_seen": 1391652864 + }, + { + "epoch": 4.06, + "learning_rate": 0.00029211634904714143, + "loss": 2.8641, + "theoretical_loss": 3.539553966916327, + "tokens_seen": 1391718400 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002921063189568706, + "loss": 2.6256, + "theoretical_loss": 3.539539059450447, + "tokens_seen": 1391783936 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002920962888665998, + "loss": 2.7322, + "theoretical_loss": 3.539524152883047, + "tokens_seen": 1391849472 + }, + { + "epoch": 4.06, + "learning_rate": 0.000292086258776329, + "loss": 2.6218, + "theoretical_loss": 3.539509247214031, + "tokens_seen": 1391915008 + }, + { + "epoch": 4.06, + "learning_rate": 0.00029207622868605815, + "loss": 2.6119, + "theoretical_loss": 3.539494342443303, + "tokens_seen": 1391980544 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002920661985957874, + "loss": 2.8052, + "theoretical_loss": 3.539479438570767, + "tokens_seen": 1392046080 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002920561685055165, + "loss": 2.4703, + "theoretical_loss": 3.5394645355963257, + "tokens_seen": 1392111616 + }, + { + "epoch": 4.06, + "learning_rate": 0.00029204613841524575, + "loss": 2.838, + "theoretical_loss": 3.539449633519883, + "tokens_seen": 1392177152 + }, + { + "epoch": 4.06, + "learning_rate": 0.00029203610832497493, + "loss": 2.8107, + "theoretical_loss": 3.539434732341343, + "tokens_seen": 1392242688 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002920260782347041, + "loss": 2.6331, + "theoretical_loss": 3.5394198320606094, + "tokens_seen": 1392308224 + }, + { + "epoch": 4.06, + "learning_rate": 0.00029201604814443335, + "loss": 2.962, + "theoretical_loss": 3.5394049326775847, + "tokens_seen": 1392373760 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002920060180541625, + "loss": 2.6808, + "theoretical_loss": 3.5393900341921736, + "tokens_seen": 1392439296 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002919959879638917, + "loss": 2.746, + "theoretical_loss": 3.5393751366042796, + "tokens_seen": 1392504832 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002919859578736209, + "loss": 2.7237, + "theoretical_loss": 3.5393602399138056, + "tokens_seen": 1392570368 + }, + { + "epoch": 4.06, + "objective/train/docs_used": 1558283, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4765679836273193, + "objective/train/theoretical_loss": 3.5393453441206564, + "objective/train/tokens_used": 1413095904, + "theoretical_loss": 3.5393453441206564, + "tokens_seen": 1392635904 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002919759277833501, + "loss": 2.6099, + "theoretical_loss": 3.5393453441206564, + "tokens_seen": 1392635904 + }, + { + "epoch": 4.06, + "learning_rate": 0.00029196589769307926, + "loss": 2.64, + "theoretical_loss": 3.5393304492247353, + "tokens_seen": 1392701440 + }, + { + "epoch": 4.06, + "learning_rate": 0.00029195586760280844, + "loss": 2.7599, + "theoretical_loss": 3.5393155552259463, + "tokens_seen": 1392766976 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002919458375125376, + "loss": 2.5263, + "theoretical_loss": 3.5393006621241927, + "tokens_seen": 1392832512 + }, + { + "epoch": 4.06, + "learning_rate": 0.00029193580742226685, + "loss": 2.4846, + "theoretical_loss": 3.5392857699193785, + "tokens_seen": 1392898048 + }, + { + "epoch": 4.06, + "learning_rate": 0.000291925777331996, + "loss": 2.7272, + "theoretical_loss": 3.539270878611407, + "tokens_seen": 1392963584 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002919157472417252, + "loss": 2.8948, + "theoretical_loss": 3.539255988200183, + "tokens_seen": 1393029120 + }, + { + "epoch": 4.06, + "learning_rate": 0.00029190571715145434, + "loss": 2.6318, + "theoretical_loss": 3.5392410986856095, + "tokens_seen": 1393094656 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002918956870611836, + "loss": 2.6671, + "theoretical_loss": 3.5392262100675906, + "tokens_seen": 1393160192 + }, + { + "epoch": 4.06, + "learning_rate": 0.00029188565697091276, + "loss": 2.6477, + "theoretical_loss": 3.53921132234603, + "tokens_seen": 1393225728 + }, + { + "epoch": 4.06, + "learning_rate": 0.00029187562688064194, + "loss": 2.6761, + "theoretical_loss": 3.5391964355208323, + "tokens_seen": 1393291264 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002918655967903711, + "loss": 2.6871, + "theoretical_loss": 3.5391815495919, + "tokens_seen": 1393356800 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002918555667001003, + "loss": 2.7094, + "theoretical_loss": 3.539166664559138, + "tokens_seen": 1393422336 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002918455366098295, + "loss": 2.7393, + "theoretical_loss": 3.5391517804224506, + "tokens_seen": 1393487872 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002918355065195587, + "loss": 2.6149, + "theoretical_loss": 3.5391368971817405, + "tokens_seen": 1393553408 + }, + { + "epoch": 4.06, + "learning_rate": 0.00029182547642928785, + "loss": 2.6424, + "theoretical_loss": 3.539122014836912, + "tokens_seen": 1393618944 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002918154463390171, + "loss": 2.7817, + "theoretical_loss": 3.539107133387869, + "tokens_seen": 1393684480 + }, + { + "epoch": 4.06, + "learning_rate": 0.00029180541624874626, + "loss": 2.7797, + "theoretical_loss": 3.539092252834517, + "tokens_seen": 1393750016 + }, + { + "epoch": 4.06, + "learning_rate": 0.00029179538615847544, + "loss": 2.4839, + "theoretical_loss": 3.5390773731767577, + "tokens_seen": 1393815552 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002917853560682046, + "loss": 2.8268, + "theoretical_loss": 3.5390624944144964, + "tokens_seen": 1393881088 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002917753259779338, + "loss": 2.6579, + "theoretical_loss": 3.5390476165476366, + "tokens_seen": 1393946624 + }, + { + "epoch": 4.06, + "learning_rate": 0.000291765295887663, + "loss": 2.5451, + "theoretical_loss": 3.5390327395760828, + "tokens_seen": 1394012160 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002917552657973922, + "loss": 2.6735, + "theoretical_loss": 3.5390178634997382, + "tokens_seen": 1394077696 + }, + { + "epoch": 4.06, + "learning_rate": 0.00029174523570712135, + "loss": 2.6657, + "theoretical_loss": 3.539002988318508, + "tokens_seen": 1394143232 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002917352056168506, + "loss": 2.6527, + "theoretical_loss": 3.5389881140322954, + "tokens_seen": 1394208768 + }, + { + "epoch": 4.06, + "objective/train/docs_used": 1559602, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4316999912261963, + "objective/train/theoretical_loss": 3.5389732406410044, + "objective/train/tokens_used": 1414734304, + "theoretical_loss": 3.5389732406410044, + "tokens_seen": 1394274304 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002917251755265797, + "loss": 2.7982, + "theoretical_loss": 3.5389732406410044, + "tokens_seen": 1394274304 + }, + { + "epoch": 4.06, + "learning_rate": 0.00029171514543630895, + "loss": 2.7828, + "theoretical_loss": 3.53895836814454, + "tokens_seen": 1394339840 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002917051153460381, + "loss": 2.6216, + "theoretical_loss": 3.5389434965428057, + "tokens_seen": 1394405376 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002916950852557673, + "loss": 2.6401, + "theoretical_loss": 3.5389286258357053, + "tokens_seen": 1394470912 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002916850551654965, + "loss": 2.5921, + "theoretical_loss": 3.5389137560231436, + "tokens_seen": 1394536448 + }, + { + "epoch": 4.06, + "learning_rate": 0.00029167502507522567, + "loss": 2.5946, + "theoretical_loss": 3.5388988871050246, + "tokens_seen": 1394601984 + }, + { + "epoch": 4.06, + "learning_rate": 0.00029166499498495485, + "loss": 2.7326, + "theoretical_loss": 3.538884019081252, + "tokens_seen": 1394667520 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002916549648946841, + "loss": 2.6635, + "theoretical_loss": 3.5388691519517304, + "tokens_seen": 1394733056 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002916449348044132, + "loss": 2.6518, + "theoretical_loss": 3.538854285716364, + "tokens_seen": 1394798592 + }, + { + "epoch": 4.06, + "learning_rate": 0.00029163490471414245, + "loss": 2.7532, + "theoretical_loss": 3.5388394203750573, + "tokens_seen": 1394864128 + }, + { + "epoch": 4.06, + "learning_rate": 0.00029162487462387163, + "loss": 2.6184, + "theoretical_loss": 3.5388245559277136, + "tokens_seen": 1394929664 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002916148445336008, + "loss": 2.7091, + "theoretical_loss": 3.5388096923742385, + "tokens_seen": 1394995200 + }, + { + "epoch": 4.06, + "learning_rate": 0.00029160481444333, + "loss": 2.4223, + "theoretical_loss": 3.5387948297145346, + "tokens_seen": 1395060736 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002915947843530592, + "loss": 2.5134, + "theoretical_loss": 3.5387799679485075, + "tokens_seen": 1395126272 + }, + { + "epoch": 4.06, + "learning_rate": 0.00029158475426278835, + "loss": 2.5809, + "theoretical_loss": 3.5387651070760606, + "tokens_seen": 1395191808 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002915747241725176, + "loss": 2.7848, + "theoretical_loss": 3.538750247097099, + "tokens_seen": 1395257344 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002915646940822467, + "loss": 2.6923, + "theoretical_loss": 3.538735388011527, + "tokens_seen": 1395322880 + }, + { + "epoch": 4.06, + "learning_rate": 0.00029155466399197595, + "loss": 2.5315, + "theoretical_loss": 3.5387205298192477, + "tokens_seen": 1395388416 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002915446339017051, + "loss": 2.7831, + "theoretical_loss": 3.538705672520167, + "tokens_seen": 1395453952 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002915346038114343, + "loss": 2.5866, + "theoretical_loss": 3.5386908161141886, + "tokens_seen": 1395519488 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002915245737211635, + "loss": 2.5707, + "theoretical_loss": 3.538675960601217, + "tokens_seen": 1395585024 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002915145436308927, + "loss": 2.5683, + "theoretical_loss": 3.5386611059811557, + "tokens_seen": 1395650560 + }, + { + "epoch": 4.06, + "learning_rate": 0.00029150451354062186, + "loss": 2.8669, + "theoretical_loss": 3.53864625225391, + "tokens_seen": 1395716096 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002914944834503511, + "loss": 2.4638, + "theoretical_loss": 3.5386313994193843, + "tokens_seen": 1395781632 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002914844533600802, + "loss": 2.4921, + "theoretical_loss": 3.538616547477483, + "tokens_seen": 1395847168 + }, + { + "epoch": 4.06, + "objective/train/docs_used": 1560364, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.397831916809082, + "objective/train/theoretical_loss": 3.5386016964281106, + "objective/train/tokens_used": 1416372704, + "theoretical_loss": 3.5386016964281106, + "tokens_seen": 1395912704 + }, + { + "epoch": 4.06, + "learning_rate": 0.00029147442326980946, + "loss": 2.7208, + "theoretical_loss": 3.5386016964281106, + "tokens_seen": 1395912704 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002914643931795386, + "loss": 2.6569, + "theoretical_loss": 3.538586846271172, + "tokens_seen": 1395978240 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002914543630892678, + "loss": 2.8526, + "theoretical_loss": 3.5385719970065703, + "tokens_seen": 1396043776 + }, + { + "epoch": 4.06, + "learning_rate": 0.000291444332998997, + "loss": 2.7612, + "theoretical_loss": 3.5385571486342107, + "tokens_seen": 1396109312 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002914343029087262, + "loss": 2.7004, + "theoretical_loss": 3.5385423011539983, + "tokens_seen": 1396174848 + }, + { + "epoch": 4.06, + "learning_rate": 0.00029142427281845536, + "loss": 2.6992, + "theoretical_loss": 3.5385274545658367, + "tokens_seen": 1396240384 + }, + { + "epoch": 4.06, + "learning_rate": 0.00029141424272818454, + "loss": 2.6948, + "theoretical_loss": 3.5385126088696315, + "tokens_seen": 1396305920 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002914042126379137, + "loss": 2.4636, + "theoretical_loss": 3.538497764065286, + "tokens_seen": 1396371456 + }, + { + "epoch": 4.06, + "learning_rate": 0.00029139418254764296, + "loss": 2.7728, + "theoretical_loss": 3.5384829201527057, + "tokens_seen": 1396436992 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002913841524573721, + "loss": 2.6798, + "theoretical_loss": 3.538468077131795, + "tokens_seen": 1396502528 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002913741223671013, + "loss": 2.813, + "theoretical_loss": 3.538453235002458, + "tokens_seen": 1396568064 + }, + { + "epoch": 4.06, + "learning_rate": 0.00029136409227683045, + "loss": 2.5104, + "theoretical_loss": 3.5384383937646, + "tokens_seen": 1396633600 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002913540621865597, + "loss": 2.4528, + "theoretical_loss": 3.538423553418125, + "tokens_seen": 1396699136 + }, + { + "epoch": 4.06, + "learning_rate": 0.00029134403209628886, + "loss": 2.546, + "theoretical_loss": 3.538408713962938, + "tokens_seen": 1396764672 + }, + { + "epoch": 4.06, + "learning_rate": 0.00029133400200601805, + "loss": 2.4749, + "theoretical_loss": 3.5383938753989437, + "tokens_seen": 1396830208 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002913239719157472, + "loss": 2.5486, + "theoretical_loss": 3.538379037726047, + "tokens_seen": 1396895744 + }, + { + "epoch": 4.06, + "learning_rate": 0.00029131394182547646, + "loss": 2.7272, + "theoretical_loss": 3.5383642009441516, + "tokens_seen": 1396961280 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002913039117352056, + "loss": 2.775, + "theoretical_loss": 3.5383493650531634, + "tokens_seen": 1397026816 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002912938816449348, + "loss": 2.7766, + "theoretical_loss": 3.5383345300529863, + "tokens_seen": 1397092352 + }, + { + "epoch": 4.06, + "learning_rate": 0.000291283851554664, + "loss": 2.5114, + "theoretical_loss": 3.5383196959435255, + "tokens_seen": 1397157888 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002912738214643932, + "loss": 2.717, + "theoretical_loss": 3.5383048627246856, + "tokens_seen": 1397223424 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002912637913741224, + "loss": 2.6179, + "theoretical_loss": 3.5382900303963707, + "tokens_seen": 1397288960 + }, + { + "epoch": 4.06, + "learning_rate": 0.00029125376128385155, + "loss": 2.6235, + "theoretical_loss": 3.5382751989584866, + "tokens_seen": 1397354496 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002912437311935808, + "loss": 2.628, + "theoretical_loss": 3.538260368410938, + "tokens_seen": 1397420032 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002912337011033099, + "loss": 2.5759, + "theoretical_loss": 3.538245538753629, + "tokens_seen": 1397485568 + }, + { + "epoch": 4.06, + "objective/train/docs_used": 1561743, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9683220386505127, + "objective/train/theoretical_loss": 3.5382307099864647, + "objective/train/tokens_used": 1418011104, + "theoretical_loss": 3.5382307099864647, + "tokens_seen": 1397551104 + }, + { + "epoch": 4.06, + "learning_rate": 0.00029122367101303915, + "loss": 2.7218, + "theoretical_loss": 3.5382307099864647, + "tokens_seen": 1397551104 + }, + { + "epoch": 4.06, + "learning_rate": 0.00029121364092276833, + "loss": 2.5486, + "theoretical_loss": 3.53821588210935, + "tokens_seen": 1397616640 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002912036108324975, + "loss": 2.5438, + "theoretical_loss": 3.5382010551221903, + "tokens_seen": 1397682176 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002911935807422267, + "loss": 2.6942, + "theoretical_loss": 3.5381862290248893, + "tokens_seen": 1397747712 + }, + { + "epoch": 4.06, + "learning_rate": 0.00029118355065195587, + "loss": 2.6858, + "theoretical_loss": 3.5381714038173526, + "tokens_seen": 1397813248 + }, + { + "epoch": 4.06, + "learning_rate": 0.00029117352056168505, + "loss": 2.7571, + "theoretical_loss": 3.538156579499485, + "tokens_seen": 1397878784 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002911634904714143, + "loss": 2.7165, + "theoretical_loss": 3.538141756071192, + "tokens_seen": 1397944320 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002911534603811434, + "loss": 2.6838, + "theoretical_loss": 3.538126933532377, + "tokens_seen": 1398009856 + }, + { + "epoch": 4.06, + "learning_rate": 0.00029114343029087265, + "loss": 2.4973, + "theoretical_loss": 3.5381121118829464, + "tokens_seen": 1398075392 + }, + { + "epoch": 4.06, + "learning_rate": 0.00029113340020060183, + "loss": 2.541, + "theoretical_loss": 3.538097291122804, + "tokens_seen": 1398140928 + }, + { + "epoch": 4.06, + "learning_rate": 0.000291123370110331, + "loss": 2.6344, + "theoretical_loss": 3.538082471251856, + "tokens_seen": 1398206464 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002911133400200602, + "loss": 2.5838, + "theoretical_loss": 3.5380676522700067, + "tokens_seen": 1398272000 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002911033099297894, + "loss": 2.4584, + "theoretical_loss": 3.5380528341771607, + "tokens_seen": 1398337536 + }, + { + "epoch": 4.06, + "learning_rate": 0.00029109327983951855, + "loss": 2.5209, + "theoretical_loss": 3.5380380169732235, + "tokens_seen": 1398403072 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002910832497492478, + "loss": 2.7778, + "theoretical_loss": 3.5380232006581003, + "tokens_seen": 1398468608 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002910732196589769, + "loss": 2.719, + "theoretical_loss": 3.538008385231696, + "tokens_seen": 1398534144 + }, + { + "epoch": 4.06, + "learning_rate": 0.00029106318956870615, + "loss": 2.6171, + "theoretical_loss": 3.537993570693915, + "tokens_seen": 1398599680 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002910531594784353, + "loss": 2.5908, + "theoretical_loss": 3.5379787570446632, + "tokens_seen": 1398665216 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002910431293881645, + "loss": 2.6341, + "theoretical_loss": 3.5379639442838453, + "tokens_seen": 1398730752 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002910330992978937, + "loss": 2.7537, + "theoretical_loss": 3.5379491324113665, + "tokens_seen": 1398796288 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002910230692076229, + "loss": 2.8437, + "theoretical_loss": 3.537934321427132, + "tokens_seen": 1398861824 + }, + { + "epoch": 4.06, + "learning_rate": 0.00029101303911735206, + "loss": 2.6106, + "theoretical_loss": 3.537919511331047, + "tokens_seen": 1398927360 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002910030090270813, + "loss": 2.6991, + "theoretical_loss": 3.537904702123016, + "tokens_seen": 1398992896 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002909929789368104, + "loss": 2.5194, + "theoretical_loss": 3.537889893802945, + "tokens_seen": 1399058432 + }, + { + "epoch": 4.06, + "learning_rate": 0.00029098294884653966, + "loss": 2.6954, + "theoretical_loss": 3.537875086370738, + "tokens_seen": 1399123968 + }, + { + "epoch": 4.06, + "objective/train/docs_used": 1562396, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.784968137741089, + "objective/train/theoretical_loss": 3.537860279826302, + "objective/train/tokens_used": 1419649504, + "theoretical_loss": 3.537860279826302, + "tokens_seen": 1399189504 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002909729187562688, + "loss": 2.6959, + "theoretical_loss": 3.537860279826302, + "tokens_seen": 1399189504 + }, + { + "epoch": 4.06, + "learning_rate": 0.000290962888665998, + "loss": 2.7707, + "theoretical_loss": 3.53784547416954, + "tokens_seen": 1399255040 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002909528585757272, + "loss": 2.6713, + "theoretical_loss": 3.5378306694003596, + "tokens_seen": 1399320576 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002909428284854564, + "loss": 2.5541, + "theoretical_loss": 3.537815865518664, + "tokens_seen": 1399386112 + }, + { + "epoch": 4.06, + "learning_rate": 0.00029093279839518556, + "loss": 2.7361, + "theoretical_loss": 3.537801062524359, + "tokens_seen": 1399451648 + }, + { + "epoch": 4.06, + "learning_rate": 0.00029092276830491474, + "loss": 2.5931, + "theoretical_loss": 3.5377862604173504, + "tokens_seen": 1399517184 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002909127382146439, + "loss": 2.6053, + "theoretical_loss": 3.5377714591975433, + "tokens_seen": 1399582720 + }, + { + "epoch": 4.06, + "learning_rate": 0.00029090270812437316, + "loss": 2.6946, + "theoretical_loss": 3.5377566588648426, + "tokens_seen": 1399648256 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002908926780341023, + "loss": 2.6109, + "theoretical_loss": 3.537741859419154, + "tokens_seen": 1399713792 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002908826479438315, + "loss": 2.5046, + "theoretical_loss": 3.537727060860383, + "tokens_seen": 1399779328 + }, + { + "epoch": 4.06, + "learning_rate": 0.00029087261785356065, + "loss": 2.7201, + "theoretical_loss": 3.5377122631884337, + "tokens_seen": 1399844864 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002908625877632899, + "loss": 2.5966, + "theoretical_loss": 3.5376974664032126, + "tokens_seen": 1399910400 + }, + { + "epoch": 4.06, + "learning_rate": 0.00029085255767301906, + "loss": 2.5142, + "theoretical_loss": 3.5376826705046254, + "tokens_seen": 1399975936 + }, + { + "epoch": 4.06, + "learning_rate": 0.00029084252758274825, + "loss": 2.3679, + "theoretical_loss": 3.537667875492576, + "tokens_seen": 1400041472 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002908324974924774, + "loss": 2.6371, + "theoretical_loss": 3.537653081366971, + "tokens_seen": 1400107008 + }, + { + "epoch": 4.06, + "learning_rate": 0.00029082246740220666, + "loss": 2.6135, + "theoretical_loss": 3.5376382881277153, + "tokens_seen": 1400172544 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002908124373119358, + "loss": 2.5081, + "theoretical_loss": 3.5376234957747146, + "tokens_seen": 1400238080 + }, + { + "epoch": 4.06, + "learning_rate": 0.000290802407221665, + "loss": 2.6574, + "theoretical_loss": 3.537608704307874, + "tokens_seen": 1400303616 + }, + { + "epoch": 4.06, + "learning_rate": 0.00029079237713139415, + "loss": 2.4363, + "theoretical_loss": 3.537593913727099, + "tokens_seen": 1400369152 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002907823470411234, + "loss": 2.6265, + "theoretical_loss": 3.5375791240322956, + "tokens_seen": 1400434688 + }, + { + "epoch": 4.06, + "learning_rate": 0.00029077231695085257, + "loss": 2.8114, + "theoretical_loss": 3.5375643352233683, + "tokens_seen": 1400500224 + }, + { + "epoch": 4.06, + "learning_rate": 0.00029076228686058175, + "loss": 2.444, + "theoretical_loss": 3.537549547300223, + "tokens_seen": 1400565760 + }, + { + "epoch": 4.06, + "learning_rate": 0.00029075225677031093, + "loss": 2.6148, + "theoretical_loss": 3.537534760262766, + "tokens_seen": 1400631296 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002907422266800401, + "loss": 2.5593, + "theoretical_loss": 3.537519974110902, + "tokens_seen": 1400696832 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002907321965897693, + "loss": 2.5544, + "theoretical_loss": 3.537505188844536, + "tokens_seen": 1400762368 + }, + { + "epoch": 4.06, + "objective/train/docs_used": 1563023, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.582470655441284, + "objective/train/theoretical_loss": 3.5374904044635747, + "objective/train/tokens_used": 1421287904, + "theoretical_loss": 3.5374904044635747, + "tokens_seen": 1400827904 + }, + { + "epoch": 4.06, + "learning_rate": 0.00029072216649949853, + "loss": 2.6132, + "theoretical_loss": 3.5374904044635747, + "tokens_seen": 1400827904 + }, + { + "epoch": 4.06, + "learning_rate": 0.00029071213640922765, + "loss": 2.778, + "theoretical_loss": 3.537475620967923, + "tokens_seen": 1400893440 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002907021063189569, + "loss": 2.6992, + "theoretical_loss": 3.5374608383574873, + "tokens_seen": 1400958976 + }, + { + "epoch": 4.06, + "learning_rate": 0.000290692076228686, + "loss": 2.7268, + "theoretical_loss": 3.537446056632172, + "tokens_seen": 1401024512 + }, + { + "epoch": 4.06, + "learning_rate": 0.00029068204613841525, + "loss": 2.3954, + "theoretical_loss": 3.5374312757918833, + "tokens_seen": 1401090048 + }, + { + "epoch": 4.06, + "learning_rate": 0.00029067201604814443, + "loss": 2.7221, + "theoretical_loss": 3.537416495836527, + "tokens_seen": 1401155584 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002906619859578736, + "loss": 2.4826, + "theoretical_loss": 3.5374017167660075, + "tokens_seen": 1401221120 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002906519558676028, + "loss": 2.5533, + "theoretical_loss": 3.537386938580233, + "tokens_seen": 1401286656 + }, + { + "epoch": 4.06, + "learning_rate": 0.00029064192577733203, + "loss": 2.7717, + "theoretical_loss": 3.537372161279106, + "tokens_seen": 1401352192 + }, + { + "epoch": 4.06, + "learning_rate": 0.00029063189568706116, + "loss": 2.566, + "theoretical_loss": 3.537357384862535, + "tokens_seen": 1401417728 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002906218655967904, + "loss": 2.6829, + "theoretical_loss": 3.5373426093304237, + "tokens_seen": 1401483264 + }, + { + "epoch": 4.06, + "learning_rate": 0.0002906118355065195, + "loss": 2.7533, + "theoretical_loss": 3.537327834682679, + "tokens_seen": 1401548800 + }, + { + "epoch": 4.07, + "learning_rate": 0.00029060180541624876, + "loss": 2.3787, + "theoretical_loss": 3.537313060919206, + "tokens_seen": 1401614336 + }, + { + "epoch": 4.07, + "learning_rate": 0.00029059177532597794, + "loss": 2.3038, + "theoretical_loss": 3.5372982880399104, + "tokens_seen": 1401679872 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002905817452357071, + "loss": 2.4445, + "theoretical_loss": 3.537283516044698, + "tokens_seen": 1401745408 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002905717151454363, + "loss": 2.7568, + "theoretical_loss": 3.5372687449334754, + "tokens_seen": 1401810944 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002905616850551655, + "loss": 2.6059, + "theoretical_loss": 3.5372539747061476, + "tokens_seen": 1401876480 + }, + { + "epoch": 4.07, + "learning_rate": 0.00029055165496489466, + "loss": 2.5998, + "theoretical_loss": 3.53723920536262, + "tokens_seen": 1401942016 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002905416248746239, + "loss": 2.6002, + "theoretical_loss": 3.5372244369027994, + "tokens_seen": 1402007552 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002905315947843531, + "loss": 2.7421, + "theoretical_loss": 3.537209669326591, + "tokens_seen": 1402073088 + }, + { + "epoch": 4.07, + "learning_rate": 0.00029052156469408226, + "loss": 2.6018, + "theoretical_loss": 3.5371949026339005, + "tokens_seen": 1402138624 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002905115346038115, + "loss": 2.7309, + "theoretical_loss": 3.5371801368246345, + "tokens_seen": 1402204160 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002905015045135406, + "loss": 2.7116, + "theoretical_loss": 3.5371653718986975, + "tokens_seen": 1402269696 + }, + { + "epoch": 4.07, + "learning_rate": 0.00029049147442326986, + "loss": 2.7406, + "theoretical_loss": 3.537150607855997, + "tokens_seen": 1402335232 + }, + { + "epoch": 4.07, + "learning_rate": 0.000290481444332999, + "loss": 2.6909, + "theoretical_loss": 3.537135844696438, + "tokens_seen": 1402400768 + }, + { + "epoch": 4.07, + "objective/train/docs_used": 1564219, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.3950443267822266, + "objective/train/theoretical_loss": 3.5371210824199264, + "objective/train/tokens_used": 1422926304, + "theoretical_loss": 3.5371210824199264, + "tokens_seen": 1402466304 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002904714142427282, + "loss": 2.5619, + "theoretical_loss": 3.5371210824199264, + "tokens_seen": 1402466304 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002904613841524574, + "loss": 2.7336, + "theoretical_loss": 3.5371063210263682, + "tokens_seen": 1402531840 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002904513540621866, + "loss": 2.7499, + "theoretical_loss": 3.5370915605156696, + "tokens_seen": 1402597376 + }, + { + "epoch": 4.07, + "learning_rate": 0.00029044132397191576, + "loss": 2.5718, + "theoretical_loss": 3.537076800887736, + "tokens_seen": 1402662912 + }, + { + "epoch": 4.07, + "learning_rate": 0.00029043129388164494, + "loss": 2.5744, + "theoretical_loss": 3.537062042142474, + "tokens_seen": 1402728448 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002904212637913741, + "loss": 2.651, + "theoretical_loss": 3.5370472842797893, + "tokens_seen": 1402793984 + }, + { + "epoch": 4.07, + "learning_rate": 0.00029041123370110336, + "loss": 2.6257, + "theoretical_loss": 3.537032527299588, + "tokens_seen": 1402859520 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002904012036108325, + "loss": 2.9672, + "theoretical_loss": 3.537017771201776, + "tokens_seen": 1402925056 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002903911735205617, + "loss": 2.7034, + "theoretical_loss": 3.537003015986259, + "tokens_seen": 1402990592 + }, + { + "epoch": 4.07, + "learning_rate": 0.00029038114343029085, + "loss": 2.8006, + "theoretical_loss": 3.536988261652943, + "tokens_seen": 1403056128 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002903711133400201, + "loss": 2.7806, + "theoretical_loss": 3.536973508201735, + "tokens_seen": 1403121664 + }, + { + "epoch": 4.07, + "learning_rate": 0.00029036108324974926, + "loss": 2.7626, + "theoretical_loss": 3.5369587556325404, + "tokens_seen": 1403187200 + }, + { + "epoch": 4.07, + "learning_rate": 0.00029035105315947845, + "loss": 2.884, + "theoretical_loss": 3.536944003945265, + "tokens_seen": 1403252736 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002903410230692076, + "loss": 2.6588, + "theoretical_loss": 3.5369292531398155, + "tokens_seen": 1403318272 + }, + { + "epoch": 4.07, + "learning_rate": 0.00029033099297893686, + "loss": 2.5644, + "theoretical_loss": 3.5369145032160976, + "tokens_seen": 1403383808 + }, + { + "epoch": 4.07, + "learning_rate": 0.000290320962888666, + "loss": 2.5827, + "theoretical_loss": 3.536899754174018, + "tokens_seen": 1403449344 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002903109327983952, + "loss": 2.5899, + "theoretical_loss": 3.536885006013482, + "tokens_seen": 1403514880 + }, + { + "epoch": 4.07, + "learning_rate": 0.00029030090270812435, + "loss": 2.7309, + "theoretical_loss": 3.536870258734396, + "tokens_seen": 1403580416 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002902908726178536, + "loss": 2.5328, + "theoretical_loss": 3.5368555123366665, + "tokens_seen": 1403645952 + }, + { + "epoch": 4.07, + "learning_rate": 0.00029028084252758277, + "loss": 2.8273, + "theoretical_loss": 3.5368407668202, + "tokens_seen": 1403711488 + }, + { + "epoch": 4.07, + "learning_rate": 0.00029027081243731195, + "loss": 2.8104, + "theoretical_loss": 3.5368260221849015, + "tokens_seen": 1403777024 + }, + { + "epoch": 4.07, + "learning_rate": 0.00029026078234704113, + "loss": 2.7349, + "theoretical_loss": 3.536811278430678, + "tokens_seen": 1403842560 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002902507522567703, + "loss": 2.7155, + "theoretical_loss": 3.536796535557436, + "tokens_seen": 1403908096 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002902407221664995, + "loss": 2.5073, + "theoretical_loss": 3.536781793565081, + "tokens_seen": 1403973632 + }, + { + "epoch": 4.07, + "learning_rate": 0.00029023069207622873, + "loss": 2.5849, + "theoretical_loss": 3.53676705245352, + "tokens_seen": 1404039168 + }, + { + "epoch": 4.07, + "objective/train/docs_used": 1565469, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9763519763946533, + "objective/train/theoretical_loss": 3.5367523122226583, + "objective/train/tokens_used": 1424564704, + "theoretical_loss": 3.5367523122226583, + "tokens_seen": 1404104704 + }, + { + "epoch": 4.07, + "learning_rate": 0.00029022066198595785, + "loss": 2.7733, + "theoretical_loss": 3.5367523122226583, + "tokens_seen": 1404104704 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002902106318956871, + "loss": 2.7111, + "theoretical_loss": 3.5367375728724033, + "tokens_seen": 1404170240 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002902006018054162, + "loss": 2.6443, + "theoretical_loss": 3.5367228344026604, + "tokens_seen": 1404235776 + }, + { + "epoch": 4.07, + "learning_rate": 0.00029019057171514545, + "loss": 2.7371, + "theoretical_loss": 3.5367080968133364, + "tokens_seen": 1404301312 + }, + { + "epoch": 4.07, + "learning_rate": 0.00029018054162487463, + "loss": 2.7286, + "theoretical_loss": 3.5366933601043375, + "tokens_seen": 1404366848 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002901705115346038, + "loss": 2.522, + "theoretical_loss": 3.53667862427557, + "tokens_seen": 1404432384 + }, + { + "epoch": 4.07, + "learning_rate": 0.000290160481444333, + "loss": 2.7689, + "theoretical_loss": 3.5366638893269404, + "tokens_seen": 1404497920 + }, + { + "epoch": 4.07, + "learning_rate": 0.00029015045135406223, + "loss": 2.6427, + "theoretical_loss": 3.5366491552583548, + "tokens_seen": 1404563456 + }, + { + "epoch": 4.07, + "learning_rate": 0.00029014042126379136, + "loss": 2.4538, + "theoretical_loss": 3.53663442206972, + "tokens_seen": 1404628992 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002901303911735206, + "loss": 2.6588, + "theoretical_loss": 3.5366196897609417, + "tokens_seen": 1404694528 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002901203610832497, + "loss": 2.7638, + "theoretical_loss": 3.536604958331927, + "tokens_seen": 1404760064 + }, + { + "epoch": 4.07, + "learning_rate": 0.00029011033099297896, + "loss": 2.5228, + "theoretical_loss": 3.5365902277825816, + "tokens_seen": 1404825600 + }, + { + "epoch": 4.07, + "learning_rate": 0.00029010030090270814, + "loss": 2.5804, + "theoretical_loss": 3.5365754981128132, + "tokens_seen": 1404891136 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002900902708124373, + "loss": 2.6405, + "theoretical_loss": 3.5365607693225267, + "tokens_seen": 1404956672 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002900802407221665, + "loss": 2.4659, + "theoretical_loss": 3.5365460414116296, + "tokens_seen": 1405022208 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002900702106318957, + "loss": 2.5059, + "theoretical_loss": 3.536531314380028, + "tokens_seen": 1405087744 + }, + { + "epoch": 4.07, + "learning_rate": 0.00029006018054162486, + "loss": 2.4888, + "theoretical_loss": 3.5365165882276286, + "tokens_seen": 1405153280 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002900501504513541, + "loss": 2.6572, + "theoretical_loss": 3.5365018629543377, + "tokens_seen": 1405218816 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002900401203610832, + "loss": 2.7108, + "theoretical_loss": 3.5364871385600622, + "tokens_seen": 1405284352 + }, + { + "epoch": 4.07, + "learning_rate": 0.00029003009027081246, + "loss": 2.6045, + "theoretical_loss": 3.5364724150447078, + "tokens_seen": 1405349888 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002900200601805416, + "loss": 2.7128, + "theoretical_loss": 3.5364576924081823, + "tokens_seen": 1405415424 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002900100300902708, + "loss": 2.6778, + "theoretical_loss": 3.5364429706503913, + "tokens_seen": 1405480960 + }, + { + "epoch": 4.07, + "learning_rate": 0.00029, + "loss": 2.6373, + "theoretical_loss": 3.5364282497712414, + "tokens_seen": 1405546496 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002899899699097292, + "loss": 2.5146, + "theoretical_loss": 3.5364135297706394, + "tokens_seen": 1405612032 + }, + { + "epoch": 4.07, + "learning_rate": 0.00028997993981945836, + "loss": 2.2929, + "theoretical_loss": 3.5363988106484925, + "tokens_seen": 1405677568 + }, + { + "epoch": 4.07, + "objective/train/docs_used": 1565832, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.80586838722229, + "objective/train/theoretical_loss": 3.5363840924047065, + "objective/train/tokens_used": 1426203104, + "theoretical_loss": 3.5363840924047065, + "tokens_seen": 1405743104 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002899699097291876, + "loss": 2.7417, + "theoretical_loss": 3.5363840924047065, + "tokens_seen": 1405743104 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002899598796389167, + "loss": 2.5818, + "theoretical_loss": 3.5363693750391882, + "tokens_seen": 1405808640 + }, + { + "epoch": 4.07, + "learning_rate": 0.00028994984954864596, + "loss": 2.6419, + "theoretical_loss": 3.5363546585518444, + "tokens_seen": 1405874176 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002899398194583751, + "loss": 2.5231, + "theoretical_loss": 3.5363399429425817, + "tokens_seen": 1405939712 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002899297893681043, + "loss": 2.6878, + "theoretical_loss": 3.5363252282113073, + "tokens_seen": 1406005248 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002899197592778335, + "loss": 2.837, + "theoretical_loss": 3.536310514357927, + "tokens_seen": 1406070784 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002899097291875627, + "loss": 2.7435, + "theoretical_loss": 3.536295801382348, + "tokens_seen": 1406136320 + }, + { + "epoch": 4.07, + "learning_rate": 0.00028989969909729187, + "loss": 2.6983, + "theoretical_loss": 3.5362810892844774, + "tokens_seen": 1406201856 + }, + { + "epoch": 4.07, + "learning_rate": 0.00028988966900702105, + "loss": 2.5337, + "theoretical_loss": 3.5362663780642207, + "tokens_seen": 1406267392 + }, + { + "epoch": 4.07, + "learning_rate": 0.00028987963891675023, + "loss": 2.5224, + "theoretical_loss": 3.536251667721486, + "tokens_seen": 1406332928 + }, + { + "epoch": 4.07, + "learning_rate": 0.00028986960882647946, + "loss": 2.5878, + "theoretical_loss": 3.536236958256179, + "tokens_seen": 1406398464 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002898595787362086, + "loss": 2.6069, + "theoretical_loss": 3.5362222496682074, + "tokens_seen": 1406464000 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002898495486459378, + "loss": 2.5348, + "theoretical_loss": 3.536207541957477, + "tokens_seen": 1406529536 + }, + { + "epoch": 4.07, + "learning_rate": 0.000289839518555667, + "loss": 2.7274, + "theoretical_loss": 3.536192835123896, + "tokens_seen": 1406595072 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002898294884653962, + "loss": 2.6032, + "theoretical_loss": 3.53617812916737, + "tokens_seen": 1406660608 + }, + { + "epoch": 4.07, + "learning_rate": 0.00028981945837512537, + "loss": 2.5445, + "theoretical_loss": 3.536163424087807, + "tokens_seen": 1406726144 + }, + { + "epoch": 4.07, + "learning_rate": 0.00028980942828485455, + "loss": 2.7627, + "theoretical_loss": 3.5361487198851123, + "tokens_seen": 1406791680 + }, + { + "epoch": 4.07, + "learning_rate": 0.00028979939819458373, + "loss": 2.6414, + "theoretical_loss": 3.5361340165591937, + "tokens_seen": 1406857216 + }, + { + "epoch": 4.07, + "learning_rate": 0.00028978936810431297, + "loss": 2.8794, + "theoretical_loss": 3.536119314109958, + "tokens_seen": 1406922752 + }, + { + "epoch": 4.07, + "learning_rate": 0.00028977933801404215, + "loss": 2.6667, + "theoretical_loss": 3.5361046125373123, + "tokens_seen": 1406988288 + }, + { + "epoch": 4.07, + "learning_rate": 0.00028976930792377133, + "loss": 2.5148, + "theoretical_loss": 3.536089911841163, + "tokens_seen": 1407053824 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002897592778335005, + "loss": 2.489, + "theoretical_loss": 3.5360752120214167, + "tokens_seen": 1407119360 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002897492477432297, + "loss": 2.8873, + "theoretical_loss": 3.536060513077982, + "tokens_seen": 1407184896 + }, + { + "epoch": 4.07, + "learning_rate": 0.00028973921765295893, + "loss": 2.7641, + "theoretical_loss": 3.5360458150107643, + "tokens_seen": 1407250432 + }, + { + "epoch": 4.07, + "learning_rate": 0.00028972918756268805, + "loss": 2.5708, + "theoretical_loss": 3.536031117819671, + "tokens_seen": 1407315968 + }, + { + "epoch": 4.07, + "objective/train/docs_used": 1567098, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.710190773010254, + "objective/train/theoretical_loss": 3.536016421504609, + "objective/train/tokens_used": 1427841504, + "theoretical_loss": 3.536016421504609, + "tokens_seen": 1407381504 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002897191574724173, + "loss": 2.6155, + "theoretical_loss": 3.536016421504609, + "tokens_seen": 1407381504 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002897091273821464, + "loss": 2.5796, + "theoretical_loss": 3.5360017260654857, + "tokens_seen": 1407447040 + }, + { + "epoch": 4.07, + "learning_rate": 0.00028969909729187565, + "loss": 2.5956, + "theoretical_loss": 3.5359870315022075, + "tokens_seen": 1407512576 + }, + { + "epoch": 4.07, + "learning_rate": 0.00028968906720160483, + "loss": 2.8037, + "theoretical_loss": 3.535972337814682, + "tokens_seen": 1407578112 + }, + { + "epoch": 4.07, + "learning_rate": 0.000289679037111334, + "loss": 2.4828, + "theoretical_loss": 3.5359576450028163, + "tokens_seen": 1407643648 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002896690070210632, + "loss": 2.8295, + "theoretical_loss": 3.5359429530665167, + "tokens_seen": 1407709184 + }, + { + "epoch": 4.07, + "learning_rate": 0.00028965897693079243, + "loss": 2.5858, + "theoretical_loss": 3.5359282620056907, + "tokens_seen": 1407774720 + }, + { + "epoch": 4.07, + "learning_rate": 0.00028964894684052156, + "loss": 2.9195, + "theoretical_loss": 3.5359135718202452, + "tokens_seen": 1407840256 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002896389167502508, + "loss": 2.532, + "theoretical_loss": 3.5358988825100877, + "tokens_seen": 1407905792 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002896288866599799, + "loss": 2.8376, + "theoretical_loss": 3.535884194075125, + "tokens_seen": 1407971328 + }, + { + "epoch": 4.07, + "learning_rate": 0.00028961885656970916, + "loss": 2.7427, + "theoretical_loss": 3.5358695065152643, + "tokens_seen": 1408036864 + }, + { + "epoch": 4.07, + "learning_rate": 0.00028960882647943834, + "loss": 2.674, + "theoretical_loss": 3.535854819830413, + "tokens_seen": 1408102400 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002895987963891675, + "loss": 2.6753, + "theoretical_loss": 3.535840134020478, + "tokens_seen": 1408167936 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002895887662988967, + "loss": 2.6783, + "theoretical_loss": 3.5358254490853662, + "tokens_seen": 1408233472 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002895787362086259, + "loss": 2.6066, + "theoretical_loss": 3.5358107650249853, + "tokens_seen": 1408299008 + }, + { + "epoch": 4.07, + "learning_rate": 0.00028956870611835506, + "loss": 2.7882, + "theoretical_loss": 3.535796081839242, + "tokens_seen": 1408364544 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002895586760280843, + "loss": 2.7076, + "theoretical_loss": 3.5357813995280436, + "tokens_seen": 1408430080 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002895486459378134, + "loss": 2.5654, + "theoretical_loss": 3.5357667180912973, + "tokens_seen": 1408495616 + }, + { + "epoch": 4.07, + "learning_rate": 0.00028953861584754266, + "loss": 2.5214, + "theoretical_loss": 3.535752037528911, + "tokens_seen": 1408561152 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002895285857572718, + "loss": 2.8717, + "theoretical_loss": 3.535737357840791, + "tokens_seen": 1408626688 + }, + { + "epoch": 4.07, + "learning_rate": 0.000289518555667001, + "loss": 2.5479, + "theoretical_loss": 3.5357226790268452, + "tokens_seen": 1408692224 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002895085255767302, + "loss": 2.418, + "theoretical_loss": 3.535708001086981, + "tokens_seen": 1408757760 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002894984954864594, + "loss": 2.6083, + "theoretical_loss": 3.5356933240211053, + "tokens_seen": 1408823296 + }, + { + "epoch": 4.07, + "learning_rate": 0.00028948846539618856, + "loss": 2.6263, + "theoretical_loss": 3.5356786478291253, + "tokens_seen": 1408888832 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002894784353059178, + "loss": 2.653, + "theoretical_loss": 3.535663972510948, + "tokens_seen": 1408954368 + }, + { + "epoch": 4.07, + "objective/train/docs_used": 1567854, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5623223781585693, + "objective/train/theoretical_loss": 3.535649298066482, + "objective/train/tokens_used": 1429479904, + "theoretical_loss": 3.535649298066482, + "tokens_seen": 1409019904 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002894684052156469, + "loss": 2.599, + "theoretical_loss": 3.535649298066482, + "tokens_seen": 1409019904 + }, + { + "epoch": 4.07, + "learning_rate": 0.00028945837512537616, + "loss": 2.7912, + "theoretical_loss": 3.5356346244956334, + "tokens_seen": 1409085440 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002894483450351053, + "loss": 2.6228, + "theoretical_loss": 3.53561995179831, + "tokens_seen": 1409150976 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002894383149448345, + "loss": 2.5917, + "theoretical_loss": 3.53560527997442, + "tokens_seen": 1409216512 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002894282848545637, + "loss": 2.544, + "theoretical_loss": 3.535590609023869, + "tokens_seen": 1409282048 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002894182547642929, + "loss": 2.4938, + "theoretical_loss": 3.535575938946566, + "tokens_seen": 1409347584 + }, + { + "epoch": 4.07, + "learning_rate": 0.00028940822467402207, + "loss": 2.6501, + "theoretical_loss": 3.5355612697424177, + "tokens_seen": 1409413120 + }, + { + "epoch": 4.07, + "learning_rate": 0.00028939819458375125, + "loss": 2.5533, + "theoretical_loss": 3.5355466014113315, + "tokens_seen": 1409478656 + }, + { + "epoch": 4.07, + "learning_rate": 0.00028938816449348043, + "loss": 2.7157, + "theoretical_loss": 3.535531933953215, + "tokens_seen": 1409544192 + }, + { + "epoch": 4.07, + "learning_rate": 0.00028937813440320966, + "loss": 2.5321, + "theoretical_loss": 3.5355172673679762, + "tokens_seen": 1409609728 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002893681043129388, + "loss": 2.5582, + "theoretical_loss": 3.5355026016555215, + "tokens_seen": 1409675264 + }, + { + "epoch": 4.07, + "learning_rate": 0.000289358074222668, + "loss": 2.6674, + "theoretical_loss": 3.535487936815759, + "tokens_seen": 1409740800 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002893480441323972, + "loss": 2.6364, + "theoretical_loss": 3.5354732728485967, + "tokens_seen": 1409806336 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002893380140421264, + "loss": 2.3849, + "theoretical_loss": 3.535458609753941, + "tokens_seen": 1409871872 + }, + { + "epoch": 4.07, + "learning_rate": 0.00028932798395185557, + "loss": 2.6955, + "theoretical_loss": 3.5354439475317, + "tokens_seen": 1409937408 + }, + { + "epoch": 4.07, + "learning_rate": 0.00028931795386158475, + "loss": 2.6259, + "theoretical_loss": 3.5354292861817815, + "tokens_seen": 1410002944 + }, + { + "epoch": 4.07, + "learning_rate": 0.00028930792377131393, + "loss": 2.6021, + "theoretical_loss": 3.5354146257040924, + "tokens_seen": 1410068480 + }, + { + "epoch": 4.07, + "learning_rate": 0.00028929789368104317, + "loss": 2.7806, + "theoretical_loss": 3.535399966098541, + "tokens_seen": 1410134016 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002892878635907723, + "loss": 2.5702, + "theoretical_loss": 3.5353853073650345, + "tokens_seen": 1410199552 + }, + { + "epoch": 4.07, + "learning_rate": 0.00028927783350050153, + "loss": 2.6023, + "theoretical_loss": 3.5353706495034807, + "tokens_seen": 1410265088 + }, + { + "epoch": 4.07, + "learning_rate": 0.00028926780341023066, + "loss": 2.4183, + "theoretical_loss": 3.5353559925137867, + "tokens_seen": 1410330624 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002892577733199599, + "loss": 2.4087, + "theoretical_loss": 3.5353413363958612, + "tokens_seen": 1410396160 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002892477432296891, + "loss": 2.7805, + "theoretical_loss": 3.535326681149611, + "tokens_seen": 1410461696 + }, + { + "epoch": 4.07, + "learning_rate": 0.00028923771313941825, + "loss": 2.7112, + "theoretical_loss": 3.5353120267749434, + "tokens_seen": 1410527232 + }, + { + "epoch": 4.07, + "learning_rate": 0.00028922768304914744, + "loss": 2.6916, + "theoretical_loss": 3.5352973732717667, + "tokens_seen": 1410592768 + }, + { + "epoch": 4.07, + "objective/train/docs_used": 1569260, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.391869068145752, + "objective/train/theoretical_loss": 3.5352827206399886, + "objective/train/tokens_used": 1431118304, + "theoretical_loss": 3.5352827206399886, + "tokens_seen": 1410658304 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002892176529588766, + "loss": 2.3331, + "theoretical_loss": 3.5352827206399886, + "tokens_seen": 1410658304 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002892076228686058, + "loss": 2.6157, + "theoretical_loss": 3.5352680688795166, + "tokens_seen": 1410723840 + }, + { + "epoch": 4.07, + "learning_rate": 0.00028919759277833503, + "loss": 2.5459, + "theoretical_loss": 3.535253417990259, + "tokens_seen": 1410789376 + }, + { + "epoch": 4.07, + "learning_rate": 0.00028918756268806416, + "loss": 2.6511, + "theoretical_loss": 3.5352387679721224, + "tokens_seen": 1410854912 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002891775325977934, + "loss": 2.5784, + "theoretical_loss": 3.535224118825015, + "tokens_seen": 1410920448 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002891675025075226, + "loss": 2.603, + "theoretical_loss": 3.5352094705488453, + "tokens_seen": 1410985984 + }, + { + "epoch": 4.07, + "learning_rate": 0.00028915747241725176, + "loss": 2.7896, + "theoretical_loss": 3.53519482314352, + "tokens_seen": 1411051520 + }, + { + "epoch": 4.07, + "learning_rate": 0.00028914744232698094, + "loss": 2.6002, + "theoretical_loss": 3.5351801766089483, + "tokens_seen": 1411117056 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002891374122367101, + "loss": 2.6282, + "theoretical_loss": 3.5351655309450365, + "tokens_seen": 1411182592 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002891273821464393, + "loss": 2.3283, + "theoretical_loss": 3.535150886151693, + "tokens_seen": 1411248128 + }, + { + "epoch": 4.07, + "learning_rate": 0.00028911735205616854, + "loss": 2.6072, + "theoretical_loss": 3.5351362422288255, + "tokens_seen": 1411313664 + }, + { + "epoch": 4.07, + "learning_rate": 0.00028910732196589766, + "loss": 2.5544, + "theoretical_loss": 3.5351215991763416, + "tokens_seen": 1411379200 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002890972918756269, + "loss": 2.6091, + "theoretical_loss": 3.5351069569941505, + "tokens_seen": 1411444736 + }, + { + "epoch": 4.07, + "learning_rate": 0.000289087261785356, + "loss": 2.6773, + "theoretical_loss": 3.5350923156821583, + "tokens_seen": 1411510272 + }, + { + "epoch": 4.07, + "learning_rate": 0.00028907723169508526, + "loss": 2.5854, + "theoretical_loss": 3.5350776752402737, + "tokens_seen": 1411575808 + }, + { + "epoch": 4.07, + "learning_rate": 0.00028906720160481444, + "loss": 2.7125, + "theoretical_loss": 3.535063035668405, + "tokens_seen": 1411641344 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002890571715145436, + "loss": 2.5343, + "theoretical_loss": 3.5350483969664594, + "tokens_seen": 1411706880 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002890471414242728, + "loss": 2.5468, + "theoretical_loss": 3.5350337591343455, + "tokens_seen": 1411772416 + }, + { + "epoch": 4.07, + "learning_rate": 0.000289037111334002, + "loss": 2.6072, + "theoretical_loss": 3.5350191221719705, + "tokens_seen": 1411837952 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002890270812437312, + "loss": 2.7006, + "theoretical_loss": 3.5350044860792424, + "tokens_seen": 1411903488 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002890170511534604, + "loss": 2.5986, + "theoretical_loss": 3.53498985085607, + "tokens_seen": 1411969024 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002890070210631896, + "loss": 2.6427, + "theoretical_loss": 3.5349752165023602, + "tokens_seen": 1412034560 + }, + { + "epoch": 4.07, + "learning_rate": 0.00028899699097291876, + "loss": 2.6691, + "theoretical_loss": 3.534960583018022, + "tokens_seen": 1412100096 + }, + { + "epoch": 4.07, + "learning_rate": 0.000288986960882648, + "loss": 2.7271, + "theoretical_loss": 3.534945950402963, + "tokens_seen": 1412165632 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002889769307923771, + "loss": 2.7876, + "theoretical_loss": 3.534931318657091, + "tokens_seen": 1412231168 + }, + { + "epoch": 4.07, + "objective/train/docs_used": 1569919, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.615917921066284, + "objective/train/theoretical_loss": 3.5349166877803144, + "objective/train/tokens_used": 1432756704, + "theoretical_loss": 3.5349166877803144, + "tokens_seen": 1412296704 + }, + { + "epoch": 4.07, + "learning_rate": 0.00028896690070210636, + "loss": 2.5592, + "theoretical_loss": 3.5349166877803144, + "tokens_seen": 1412296704 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002889568706118355, + "loss": 2.5736, + "theoretical_loss": 3.534902057772541, + "tokens_seen": 1412362240 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002889468405215647, + "loss": 2.8109, + "theoretical_loss": 3.5348874286336787, + "tokens_seen": 1412427776 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002889368104312939, + "loss": 2.6942, + "theoretical_loss": 3.534872800363636, + "tokens_seen": 1412493312 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002889267803410231, + "loss": 2.7169, + "theoretical_loss": 3.5348581729623207, + "tokens_seen": 1412558848 + }, + { + "epoch": 4.07, + "learning_rate": 0.00028891675025075227, + "loss": 2.6482, + "theoretical_loss": 3.534843546429641, + "tokens_seen": 1412624384 + }, + { + "epoch": 4.07, + "learning_rate": 0.00028890672016048145, + "loss": 2.917, + "theoretical_loss": 3.534828920765505, + "tokens_seen": 1412689920 + }, + { + "epoch": 4.07, + "learning_rate": 0.00028889669007021063, + "loss": 2.4384, + "theoretical_loss": 3.534814295969821, + "tokens_seen": 1412755456 + }, + { + "epoch": 4.07, + "learning_rate": 0.00028888665997993986, + "loss": 2.8092, + "theoretical_loss": 3.5347996720424972, + "tokens_seen": 1412820992 + }, + { + "epoch": 4.07, + "learning_rate": 0.000288876629889669, + "loss": 2.6139, + "theoretical_loss": 3.5347850489834416, + "tokens_seen": 1412886528 + }, + { + "epoch": 4.07, + "learning_rate": 0.00028886659979939823, + "loss": 2.747, + "theoretical_loss": 3.534770426792562, + "tokens_seen": 1412952064 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002888565697091274, + "loss": 2.7182, + "theoretical_loss": 3.534755805469767, + "tokens_seen": 1413017600 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002888465396188566, + "loss": 2.7166, + "theoretical_loss": 3.534741185014965, + "tokens_seen": 1413083136 + }, + { + "epoch": 4.07, + "learning_rate": 0.00028883650952858577, + "loss": 2.515, + "theoretical_loss": 3.5347265654280635, + "tokens_seen": 1413148672 + }, + { + "epoch": 4.07, + "learning_rate": 0.00028882647943831495, + "loss": 2.7623, + "theoretical_loss": 3.534711946708972, + "tokens_seen": 1413214208 + }, + { + "epoch": 4.07, + "learning_rate": 0.00028881644934804413, + "loss": 2.6792, + "theoretical_loss": 3.5346973288575976, + "tokens_seen": 1413279744 + }, + { + "epoch": 4.07, + "learning_rate": 0.00028880641925777337, + "loss": 2.7728, + "theoretical_loss": 3.534682711873849, + "tokens_seen": 1413345280 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002887963891675025, + "loss": 2.6048, + "theoretical_loss": 3.5346680957576337, + "tokens_seen": 1413410816 + }, + { + "epoch": 4.07, + "learning_rate": 0.00028878635907723173, + "loss": 2.7749, + "theoretical_loss": 3.5346534805088616, + "tokens_seen": 1413476352 + }, + { + "epoch": 4.07, + "learning_rate": 0.00028877632898696086, + "loss": 2.5928, + "theoretical_loss": 3.5346388661274393, + "tokens_seen": 1413541888 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002887662988966901, + "loss": 2.4303, + "theoretical_loss": 3.5346242526132765, + "tokens_seen": 1413607424 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002887562688064193, + "loss": 2.727, + "theoretical_loss": 3.534609639966281, + "tokens_seen": 1413672960 + }, + { + "epoch": 4.07, + "learning_rate": 0.00028874623871614845, + "loss": 2.5918, + "theoretical_loss": 3.5345950281863607, + "tokens_seen": 1413738496 + }, + { + "epoch": 4.07, + "learning_rate": 0.00028873620862587764, + "loss": 2.6209, + "theoretical_loss": 3.5345804172734248, + "tokens_seen": 1413804032 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002887261785356068, + "loss": 2.5483, + "theoretical_loss": 3.5345658072273807, + "tokens_seen": 1413869568 + }, + { + "epoch": 4.07, + "objective/train/docs_used": 1570961, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.574770927429199, + "objective/train/theoretical_loss": 3.5345511980481374, + "objective/train/tokens_used": 1434395104, + "theoretical_loss": 3.5345511980481374, + "tokens_seen": 1413935104 + }, + { + "epoch": 4.07, + "learning_rate": 0.000288716148445336, + "loss": 2.646, + "theoretical_loss": 3.5345511980481374, + "tokens_seen": 1413935104 + }, + { + "epoch": 4.07, + "learning_rate": 0.00028870611835506523, + "loss": 2.4373, + "theoretical_loss": 3.5345365897356036, + "tokens_seen": 1414000640 + }, + { + "epoch": 4.07, + "learning_rate": 0.00028869608826479436, + "loss": 2.6903, + "theoretical_loss": 3.5345219822896867, + "tokens_seen": 1414066176 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002886860581745236, + "loss": 2.2652, + "theoretical_loss": 3.5345073757102963, + "tokens_seen": 1414131712 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002886760280842528, + "loss": 2.5962, + "theoretical_loss": 3.53449276999734, + "tokens_seen": 1414197248 + }, + { + "epoch": 4.07, + "learning_rate": 0.00028866599799398196, + "loss": 2.4221, + "theoretical_loss": 3.5344781651507264, + "tokens_seen": 1414262784 + }, + { + "epoch": 4.07, + "learning_rate": 0.00028865596790371114, + "loss": 2.4368, + "theoretical_loss": 3.5344635611703645, + "tokens_seen": 1414328320 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002886459378134403, + "loss": 2.5616, + "theoretical_loss": 3.5344489580561627, + "tokens_seen": 1414393856 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002886359077231695, + "loss": 2.5009, + "theoretical_loss": 3.5344343558080284, + "tokens_seen": 1414459392 + }, + { + "epoch": 4.07, + "learning_rate": 0.00028862587763289874, + "loss": 2.7095, + "theoretical_loss": 3.534419754425871, + "tokens_seen": 1414524928 + }, + { + "epoch": 4.07, + "learning_rate": 0.00028861584754262786, + "loss": 2.5479, + "theoretical_loss": 3.5344051539095993, + "tokens_seen": 1414590464 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002886058174523571, + "loss": 2.8498, + "theoretical_loss": 3.5343905542591214, + "tokens_seen": 1414656000 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002885957873620862, + "loss": 2.5168, + "theoretical_loss": 3.5343759554743457, + "tokens_seen": 1414721536 + }, + { + "epoch": 4.07, + "learning_rate": 0.00028858575727181546, + "loss": 2.5852, + "theoretical_loss": 3.5343613575551815, + "tokens_seen": 1414787072 + }, + { + "epoch": 4.07, + "learning_rate": 0.00028857572718154464, + "loss": 2.692, + "theoretical_loss": 3.5343467605015366, + "tokens_seen": 1414852608 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002885656970912738, + "loss": 2.4516, + "theoretical_loss": 3.53433216431332, + "tokens_seen": 1414918144 + }, + { + "epoch": 4.07, + "learning_rate": 0.000288555667001003, + "loss": 2.6683, + "theoretical_loss": 3.53431756899044, + "tokens_seen": 1414983680 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002885456369107322, + "loss": 2.6535, + "theoretical_loss": 3.534302974532805, + "tokens_seen": 1415049216 + }, + { + "epoch": 4.07, + "learning_rate": 0.00028853560682046137, + "loss": 2.8792, + "theoretical_loss": 3.5342883809403247, + "tokens_seen": 1415114752 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002885255767301906, + "loss": 2.798, + "theoretical_loss": 3.534273788212907, + "tokens_seen": 1415180288 + }, + { + "epoch": 4.07, + "learning_rate": 0.00028851554663991973, + "loss": 2.4001, + "theoretical_loss": 3.5342591963504604, + "tokens_seen": 1415245824 + }, + { + "epoch": 4.07, + "learning_rate": 0.00028850551654964896, + "loss": 2.5954, + "theoretical_loss": 3.534244605352894, + "tokens_seen": 1415311360 + }, + { + "epoch": 4.07, + "learning_rate": 0.00028849548645937815, + "loss": 2.586, + "theoretical_loss": 3.5342300152201167, + "tokens_seen": 1415376896 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002884854563691073, + "loss": 2.7573, + "theoretical_loss": 3.534215425952036, + "tokens_seen": 1415442432 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002884754262788365, + "loss": 2.7199, + "theoretical_loss": 3.5342008375485623, + "tokens_seen": 1415507968 + }, + { + "epoch": 4.07, + "objective/train/docs_used": 1571590, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.518846035003662, + "objective/train/theoretical_loss": 3.534186250009603, + "objective/train/tokens_used": 1436033504, + "theoretical_loss": 3.534186250009603, + "tokens_seen": 1415573504 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002884653961885657, + "loss": 2.686, + "theoretical_loss": 3.534186250009603, + "tokens_seen": 1415573504 + }, + { + "epoch": 4.07, + "learning_rate": 0.00028845536609829487, + "loss": 2.5762, + "theoretical_loss": 3.5341716633350675, + "tokens_seen": 1415639040 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002884453360080241, + "loss": 2.6943, + "theoretical_loss": 3.5341570775248643, + "tokens_seen": 1415704576 + }, + { + "epoch": 4.07, + "learning_rate": 0.00028843530591775323, + "loss": 2.5015, + "theoretical_loss": 3.5341424925789022, + "tokens_seen": 1415770112 + }, + { + "epoch": 4.07, + "learning_rate": 0.00028842527582748247, + "loss": 2.6678, + "theoretical_loss": 3.5341279084970902, + "tokens_seen": 1415835648 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002884152457372116, + "loss": 2.7635, + "theoretical_loss": 3.5341133252793373, + "tokens_seen": 1415901184 + }, + { + "epoch": 4.07, + "learning_rate": 0.00028840521564694083, + "loss": 2.7631, + "theoretical_loss": 3.5340987429255515, + "tokens_seen": 1415966720 + }, + { + "epoch": 4.07, + "learning_rate": 0.00028839518555667, + "loss": 2.6395, + "theoretical_loss": 3.534084161435642, + "tokens_seen": 1416032256 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002883851554663992, + "loss": 2.5964, + "theoretical_loss": 3.5340695808095184, + "tokens_seen": 1416097792 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002883751253761284, + "loss": 2.6531, + "theoretical_loss": 3.5340550010470886, + "tokens_seen": 1416163328 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002883650952858576, + "loss": 2.6817, + "theoretical_loss": 3.5340404221482618, + "tokens_seen": 1416228864 + }, + { + "epoch": 4.07, + "learning_rate": 0.00028835506519558674, + "loss": 2.4982, + "theoretical_loss": 3.534025844112947, + "tokens_seen": 1416294400 + }, + { + "epoch": 4.07, + "learning_rate": 0.00028834503510531597, + "loss": 2.6608, + "theoretical_loss": 3.534011266941053, + "tokens_seen": 1416359936 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002883350050150451, + "loss": 2.5268, + "theoretical_loss": 3.5339966906324882, + "tokens_seen": 1416425472 + }, + { + "epoch": 4.07, + "learning_rate": 0.00028832497492477433, + "loss": 2.6619, + "theoretical_loss": 3.533982115187162, + "tokens_seen": 1416491008 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002883149448345035, + "loss": 2.6603, + "theoretical_loss": 3.5339675406049844, + "tokens_seen": 1416556544 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002883049147442327, + "loss": 2.6605, + "theoretical_loss": 3.5339529668858622, + "tokens_seen": 1416622080 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002882948846539619, + "loss": 2.7202, + "theoretical_loss": 3.533938394029706, + "tokens_seen": 1416687616 + }, + { + "epoch": 4.07, + "learning_rate": 0.00028828485456369106, + "loss": 2.4932, + "theoretical_loss": 3.5339238220364244, + "tokens_seen": 1416753152 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002882748244734203, + "loss": 2.659, + "theoretical_loss": 3.533909250905926, + "tokens_seen": 1416818688 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002882647943831495, + "loss": 2.6021, + "theoretical_loss": 3.53389468063812, + "tokens_seen": 1416884224 + }, + { + "epoch": 4.07, + "learning_rate": 0.00028825476429287866, + "loss": 2.7558, + "theoretical_loss": 3.533880111232916, + "tokens_seen": 1416949760 + }, + { + "epoch": 4.07, + "learning_rate": 0.00028824473420260784, + "loss": 2.7942, + "theoretical_loss": 3.533865542690222, + "tokens_seen": 1417015296 + }, + { + "epoch": 4.07, + "learning_rate": 0.000288234704112337, + "loss": 2.7587, + "theoretical_loss": 3.5338509750099485, + "tokens_seen": 1417080832 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002882246740220662, + "loss": 2.6476, + "theoretical_loss": 3.533836408192003, + "tokens_seen": 1417146368 + }, + { + "epoch": 4.07, + "objective/train/docs_used": 1572254, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7383265495300293, + "objective/train/theoretical_loss": 3.533821842236295, + "objective/train/tokens_used": 1437671904, + "theoretical_loss": 3.533821842236295, + "tokens_seen": 1417211904 + }, + { + "epoch": 4.07, + "learning_rate": 0.00028821464393179543, + "loss": 2.904, + "theoretical_loss": 3.533821842236295, + "tokens_seen": 1417211904 + }, + { + "epoch": 4.07, + "learning_rate": 0.00028820461384152456, + "loss": 2.7954, + "theoretical_loss": 3.533807277142734, + "tokens_seen": 1417277440 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002881945837512538, + "loss": 2.466, + "theoretical_loss": 3.533792712911229, + "tokens_seen": 1417342976 + }, + { + "epoch": 4.07, + "learning_rate": 0.000288184553660983, + "loss": 2.723, + "theoretical_loss": 3.533778149541689, + "tokens_seen": 1417408512 + }, + { + "epoch": 4.07, + "learning_rate": 0.00028817452357071216, + "loss": 2.7506, + "theoretical_loss": 3.5337635870340236, + "tokens_seen": 1417474048 + }, + { + "epoch": 4.07, + "learning_rate": 0.00028816449348044134, + "loss": 2.5223, + "theoretical_loss": 3.5337490253881416, + "tokens_seen": 1417539584 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002881544633901705, + "loss": 2.5958, + "theoretical_loss": 3.5337344646039517, + "tokens_seen": 1417605120 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002881444332998997, + "loss": 2.6868, + "theoretical_loss": 3.533719904681363, + "tokens_seen": 1417670656 + }, + { + "epoch": 4.07, + "learning_rate": 0.00028813440320962894, + "loss": 2.5505, + "theoretical_loss": 3.5337053456202865, + "tokens_seen": 1417736192 + }, + { + "epoch": 4.07, + "learning_rate": 0.00028812437311935806, + "loss": 2.5364, + "theoretical_loss": 3.533690787420629, + "tokens_seen": 1417801728 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002881143430290873, + "loss": 2.5286, + "theoretical_loss": 3.5336762300823015, + "tokens_seen": 1417867264 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002881043129388164, + "loss": 2.84, + "theoretical_loss": 3.533661673605212, + "tokens_seen": 1417932800 + }, + { + "epoch": 4.07, + "learning_rate": 0.00028809428284854566, + "loss": 2.6731, + "theoretical_loss": 3.5336471179892706, + "tokens_seen": 1417998336 + }, + { + "epoch": 4.07, + "learning_rate": 0.00028808425275827484, + "loss": 2.6108, + "theoretical_loss": 3.533632563234386, + "tokens_seen": 1418063872 + }, + { + "epoch": 4.07, + "learning_rate": 0.000288074222668004, + "loss": 2.8358, + "theoretical_loss": 3.5336180093404677, + "tokens_seen": 1418129408 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002880641925777332, + "loss": 2.8047, + "theoretical_loss": 3.533603456307425, + "tokens_seen": 1418194944 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002880541624874624, + "loss": 2.6638, + "theoretical_loss": 3.5335889041351676, + "tokens_seen": 1418260480 + }, + { + "epoch": 4.07, + "learning_rate": 0.00028804413239719157, + "loss": 2.7179, + "theoretical_loss": 3.533574352823604, + "tokens_seen": 1418326016 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002880341023069208, + "loss": 2.6641, + "theoretical_loss": 3.533559802372644, + "tokens_seen": 1418391552 + }, + { + "epoch": 4.07, + "learning_rate": 0.00028802407221664993, + "loss": 2.6614, + "theoretical_loss": 3.533545252782197, + "tokens_seen": 1418457088 + }, + { + "epoch": 4.07, + "learning_rate": 0.00028801404212637916, + "loss": 2.8433, + "theoretical_loss": 3.5335307040521724, + "tokens_seen": 1418522624 + }, + { + "epoch": 4.07, + "learning_rate": 0.00028800401203610835, + "loss": 2.282, + "theoretical_loss": 3.533516156182479, + "tokens_seen": 1418588160 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002879939819458375, + "loss": 2.5477, + "theoretical_loss": 3.533501609173027, + "tokens_seen": 1418653696 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002879839518555667, + "loss": 2.6269, + "theoretical_loss": 3.533487063023725, + "tokens_seen": 1418719232 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002879739217652959, + "loss": 2.5457, + "theoretical_loss": 3.5334725177344835, + "tokens_seen": 1418784768 + }, + { + "epoch": 4.07, + "objective/train/docs_used": 1573335, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.203737497329712, + "objective/train/theoretical_loss": 3.53345797330521, + "objective/train/tokens_used": 1439310304, + "theoretical_loss": 3.53345797330521, + "tokens_seen": 1418850304 + }, + { + "epoch": 4.07, + "learning_rate": 0.00028796389167502507, + "loss": 2.6313, + "theoretical_loss": 3.53345797330521, + "tokens_seen": 1418850304 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002879538615847543, + "loss": 2.4921, + "theoretical_loss": 3.533443429735816, + "tokens_seen": 1418915840 + }, + { + "epoch": 4.07, + "learning_rate": 0.00028794383149448343, + "loss": 2.6774, + "theoretical_loss": 3.53342888702621, + "tokens_seen": 1418981376 + }, + { + "epoch": 4.07, + "learning_rate": 0.00028793380140421267, + "loss": 2.7072, + "theoretical_loss": 3.5334143451763014, + "tokens_seen": 1419046912 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002879237713139418, + "loss": 2.6758, + "theoretical_loss": 3.533399804186, + "tokens_seen": 1419112448 + }, + { + "epoch": 4.07, + "learning_rate": 0.00028791374122367103, + "loss": 2.6, + "theoretical_loss": 3.5333852640552155, + "tokens_seen": 1419177984 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002879037111334002, + "loss": 2.538, + "theoretical_loss": 3.5333707247838566, + "tokens_seen": 1419243520 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002878936810431294, + "loss": 2.5949, + "theoretical_loss": 3.533356186371833, + "tokens_seen": 1419309056 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002878836509528586, + "loss": 2.7255, + "theoretical_loss": 3.5333416488190554, + "tokens_seen": 1419374592 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002878736208625878, + "loss": 2.6536, + "theoretical_loss": 3.5333271121254315, + "tokens_seen": 1419440128 + }, + { + "epoch": 4.07, + "learning_rate": 0.00028786359077231694, + "loss": 2.7465, + "theoretical_loss": 3.5333125762908724, + "tokens_seen": 1419505664 + }, + { + "epoch": 4.07, + "learning_rate": 0.00028785356068204617, + "loss": 2.581, + "theoretical_loss": 3.5332980413152866, + "tokens_seen": 1419571200 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002878435305917753, + "loss": 2.615, + "theoretical_loss": 3.5332835071985844, + "tokens_seen": 1419636736 + }, + { + "epoch": 4.07, + "learning_rate": 0.00028783350050150453, + "loss": 2.7341, + "theoretical_loss": 3.5332689739406753, + "tokens_seen": 1419702272 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002878234704112337, + "loss": 2.5138, + "theoretical_loss": 3.5332544415414686, + "tokens_seen": 1419767808 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002878134403209629, + "loss": 2.6365, + "theoretical_loss": 3.5332399100008747, + "tokens_seen": 1419833344 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002878034102306921, + "loss": 2.8449, + "theoretical_loss": 3.533225379318802, + "tokens_seen": 1419898880 + }, + { + "epoch": 4.07, + "learning_rate": 0.00028779338014042126, + "loss": 2.6908, + "theoretical_loss": 3.533210849495161, + "tokens_seen": 1419964416 + }, + { + "epoch": 4.07, + "learning_rate": 0.00028778335005015044, + "loss": 2.3565, + "theoretical_loss": 3.5331963205298615, + "tokens_seen": 1420029952 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002877733199598797, + "loss": 2.7702, + "theoretical_loss": 3.5331817924228126, + "tokens_seen": 1420095488 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002877632898696088, + "loss": 2.5993, + "theoretical_loss": 3.5331672651739243, + "tokens_seen": 1420161024 + }, + { + "epoch": 4.07, + "learning_rate": 0.00028775325977933804, + "loss": 2.6276, + "theoretical_loss": 3.533152738783106, + "tokens_seen": 1420226560 + }, + { + "epoch": 4.07, + "learning_rate": 0.00028774322968906716, + "loss": 2.9462, + "theoretical_loss": 3.5331382132502682, + "tokens_seen": 1420292096 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002877331995987964, + "loss": 2.6313, + "theoretical_loss": 3.5331236885753197, + "tokens_seen": 1420357632 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002877231695085256, + "loss": 2.7853, + "theoretical_loss": 3.533109164758171, + "tokens_seen": 1420423168 + }, + { + "epoch": 4.07, + "objective/train/docs_used": 1574634, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9888744354248047, + "objective/train/theoretical_loss": 3.5330946417987312, + "objective/train/tokens_used": 1440948704, + "theoretical_loss": 3.5330946417987312, + "tokens_seen": 1420488704 + }, + { + "epoch": 4.07, + "learning_rate": 0.00028771313941825476, + "loss": 2.6085, + "theoretical_loss": 3.5330946417987312, + "tokens_seen": 1420488704 + }, + { + "epoch": 4.07, + "learning_rate": 0.00028770310932798394, + "loss": 2.5191, + "theoretical_loss": 3.533080119696911, + "tokens_seen": 1420554240 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002876930792377132, + "loss": 2.7856, + "theoretical_loss": 3.5330655984526196, + "tokens_seen": 1420619776 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002876830491474423, + "loss": 2.586, + "theoretical_loss": 3.5330510780657662, + "tokens_seen": 1420685312 + }, + { + "epoch": 4.07, + "learning_rate": 0.00028767301905717154, + "loss": 2.7563, + "theoretical_loss": 3.5330365585362618, + "tokens_seen": 1420750848 + }, + { + "epoch": 4.07, + "learning_rate": 0.00028766298896690067, + "loss": 2.8421, + "theoretical_loss": 3.5330220398640155, + "tokens_seen": 1420816384 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002876529588766299, + "loss": 2.6803, + "theoretical_loss": 3.5330075220489374, + "tokens_seen": 1420881920 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002876429287863591, + "loss": 2.5614, + "theoretical_loss": 3.5329930050909373, + "tokens_seen": 1420947456 + }, + { + "epoch": 4.07, + "learning_rate": 0.00028763289869608826, + "loss": 2.654, + "theoretical_loss": 3.532978488989925, + "tokens_seen": 1421012992 + }, + { + "epoch": 4.07, + "learning_rate": 0.00028762286860581745, + "loss": 2.5684, + "theoretical_loss": 3.5329639737458107, + "tokens_seen": 1421078528 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002876128385155466, + "loss": 2.6544, + "theoretical_loss": 3.5329494593585036, + "tokens_seen": 1421144064 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002876028084252758, + "loss": 2.7579, + "theoretical_loss": 3.5329349458279147, + "tokens_seen": 1421209600 + }, + { + "epoch": 4.07, + "learning_rate": 0.00028759277833500504, + "loss": 2.7276, + "theoretical_loss": 3.5329204331539534, + "tokens_seen": 1421275136 + }, + { + "epoch": 4.07, + "learning_rate": 0.00028758274824473417, + "loss": 2.6584, + "theoretical_loss": 3.5329059213365293, + "tokens_seen": 1421340672 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002875727181544634, + "loss": 2.7329, + "theoretical_loss": 3.5328914103755524, + "tokens_seen": 1421406208 + }, + { + "epoch": 4.07, + "learning_rate": 0.00028756268806419253, + "loss": 2.8148, + "theoretical_loss": 3.5328769002709333, + "tokens_seen": 1421471744 + }, + { + "epoch": 4.07, + "learning_rate": 0.00028755265797392177, + "loss": 2.7957, + "theoretical_loss": 3.532862391022581, + "tokens_seen": 1421537280 + }, + { + "epoch": 4.07, + "learning_rate": 0.00028754262788365095, + "loss": 2.9081, + "theoretical_loss": 3.5328478826304064, + "tokens_seen": 1421602816 + }, + { + "epoch": 4.07, + "learning_rate": 0.00028753259779338013, + "loss": 2.7681, + "theoretical_loss": 3.5328333750943193, + "tokens_seen": 1421668352 + }, + { + "epoch": 4.07, + "learning_rate": 0.00028752256770310936, + "loss": 2.7856, + "theoretical_loss": 3.532818868414229, + "tokens_seen": 1421733888 + }, + { + "epoch": 4.07, + "learning_rate": 0.00028751253761283855, + "loss": 2.744, + "theoretical_loss": 3.5328043625900474, + "tokens_seen": 1421799424 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002875025075225677, + "loss": 2.7367, + "theoretical_loss": 3.5327898576216823, + "tokens_seen": 1421864960 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002874924774322969, + "loss": 2.4577, + "theoretical_loss": 3.532775353509045, + "tokens_seen": 1421930496 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002874824473420261, + "loss": 2.7095, + "theoretical_loss": 3.5327608502520453, + "tokens_seen": 1421996032 + }, + { + "epoch": 4.07, + "learning_rate": 0.00028747241725175527, + "loss": 2.8389, + "theoretical_loss": 3.5327463478505936, + "tokens_seen": 1422061568 + }, + { + "epoch": 4.07, + "objective/train/docs_used": 1575381, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.637848377227783, + "objective/train/theoretical_loss": 3.5327318463046, + "objective/train/tokens_used": 1442587104, + "theoretical_loss": 3.5327318463046, + "tokens_seen": 1422127104 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002874623871614845, + "loss": 2.8563, + "theoretical_loss": 3.5327318463046, + "tokens_seen": 1422127104 + }, + { + "epoch": 4.07, + "learning_rate": 0.00028745235707121363, + "loss": 2.8276, + "theoretical_loss": 3.5327173456139733, + "tokens_seen": 1422192640 + }, + { + "epoch": 4.07, + "learning_rate": 0.00028744232698094287, + "loss": 2.7305, + "theoretical_loss": 3.5327028457786254, + "tokens_seen": 1422258176 + }, + { + "epoch": 4.07, + "learning_rate": 0.000287432296890672, + "loss": 2.5748, + "theoretical_loss": 3.5326883467984658, + "tokens_seen": 1422323712 + }, + { + "epoch": 4.07, + "learning_rate": 0.00028742226680040123, + "loss": 2.6212, + "theoretical_loss": 3.5326738486734044, + "tokens_seen": 1422389248 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002874122367101304, + "loss": 2.5636, + "theoretical_loss": 3.5326593514033515, + "tokens_seen": 1422454784 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002874022066198596, + "loss": 2.6832, + "theoretical_loss": 3.532644854988218, + "tokens_seen": 1422520320 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002873921765295888, + "loss": 2.6347, + "theoretical_loss": 3.532630359427913, + "tokens_seen": 1422585856 + }, + { + "epoch": 4.07, + "learning_rate": 0.000287382146439318, + "loss": 2.5502, + "theoretical_loss": 3.532615864722347, + "tokens_seen": 1422651392 + }, + { + "epoch": 4.07, + "learning_rate": 0.00028737211634904714, + "loss": 2.4627, + "theoretical_loss": 3.532601370871431, + "tokens_seen": 1422716928 + }, + { + "epoch": 4.07, + "learning_rate": 0.00028736208625877637, + "loss": 2.6478, + "theoretical_loss": 3.532586877875074, + "tokens_seen": 1422782464 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002873520561685055, + "loss": 2.6382, + "theoretical_loss": 3.532572385733187, + "tokens_seen": 1422848000 + }, + { + "epoch": 4.07, + "learning_rate": 0.00028734202607823473, + "loss": 2.736, + "theoretical_loss": 3.532557894445681, + "tokens_seen": 1422913536 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002873319959879639, + "loss": 2.6704, + "theoretical_loss": 3.532543404012465, + "tokens_seen": 1422979072 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002873219658976931, + "loss": 2.7003, + "theoretical_loss": 3.532528914433449, + "tokens_seen": 1423044608 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002873119358074223, + "loss": 2.6309, + "theoretical_loss": 3.532514425708545, + "tokens_seen": 1423110144 + }, + { + "epoch": 4.07, + "learning_rate": 0.00028730190571715146, + "loss": 2.5853, + "theoretical_loss": 3.5324999378376623, + "tokens_seen": 1423175680 + }, + { + "epoch": 4.07, + "learning_rate": 0.00028729187562688064, + "loss": 2.7507, + "theoretical_loss": 3.532485450820711, + "tokens_seen": 1423241216 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002872818455366099, + "loss": 2.6948, + "theoretical_loss": 3.532470964657602, + "tokens_seen": 1423306752 + }, + { + "epoch": 4.07, + "learning_rate": 0.000287271815446339, + "loss": 2.5424, + "theoretical_loss": 3.5324564793482454, + "tokens_seen": 1423372288 + }, + { + "epoch": 4.07, + "learning_rate": 0.00028726178535606824, + "loss": 2.5131, + "theoretical_loss": 3.5324419948925514, + "tokens_seen": 1423437824 + }, + { + "epoch": 4.07, + "learning_rate": 0.00028725175526579736, + "loss": 2.636, + "theoretical_loss": 3.5324275112904306, + "tokens_seen": 1423503360 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002872417251755266, + "loss": 2.7088, + "theoretical_loss": 3.532413028541794, + "tokens_seen": 1423568896 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002872316950852558, + "loss": 2.6311, + "theoretical_loss": 3.532398546646551, + "tokens_seen": 1423634432 + }, + { + "epoch": 4.07, + "learning_rate": 0.00028722166499498496, + "loss": 2.9132, + "theoretical_loss": 3.532384065604613, + "tokens_seen": 1423699968 + }, + { + "epoch": 4.07, + "objective/train/docs_used": 1576738, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.3536036014556885, + "objective/train/theoretical_loss": 3.532369585415889, + "objective/train/tokens_used": 1444225504, + "theoretical_loss": 3.532369585415889, + "tokens_seen": 1423765504 + }, + { + "epoch": 4.07, + "learning_rate": 0.00028721163490471414, + "loss": 2.5959, + "theoretical_loss": 3.532369585415889, + "tokens_seen": 1423765504 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002872016048144434, + "loss": 2.4618, + "theoretical_loss": 3.532355106080291, + "tokens_seen": 1423831040 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002871915747241725, + "loss": 2.8228, + "theoretical_loss": 3.532340627597729, + "tokens_seen": 1423896576 + }, + { + "epoch": 4.07, + "learning_rate": 0.00028718154463390174, + "loss": 2.927, + "theoretical_loss": 3.532326149968113, + "tokens_seen": 1423962112 + }, + { + "epoch": 4.07, + "learning_rate": 0.00028717151454363087, + "loss": 2.5715, + "theoretical_loss": 3.5323116731913538, + "tokens_seen": 1424027648 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002871614844533601, + "loss": 2.7448, + "theoretical_loss": 3.5322971972673622, + "tokens_seen": 1424093184 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002871514543630893, + "loss": 2.6791, + "theoretical_loss": 3.532282722196048, + "tokens_seen": 1424158720 + }, + { + "epoch": 4.07, + "learning_rate": 0.00028714142427281846, + "loss": 2.5363, + "theoretical_loss": 3.5322682479773224, + "tokens_seen": 1424224256 + }, + { + "epoch": 4.07, + "learning_rate": 0.00028713139418254765, + "loss": 2.5642, + "theoretical_loss": 3.532253774611096, + "tokens_seen": 1424289792 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002871213640922768, + "loss": 2.6663, + "theoretical_loss": 3.5322393020972793, + "tokens_seen": 1424355328 + }, + { + "epoch": 4.07, + "learning_rate": 0.000287111334002006, + "loss": 2.8504, + "theoretical_loss": 3.5322248304357826, + "tokens_seen": 1424420864 + }, + { + "epoch": 4.07, + "learning_rate": 0.00028710130391173524, + "loss": 2.8866, + "theoretical_loss": 3.532210359626516, + "tokens_seen": 1424486400 + }, + { + "epoch": 4.07, + "learning_rate": 0.00028709127382146437, + "loss": 2.8101, + "theoretical_loss": 3.5321958896693917, + "tokens_seen": 1424551936 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002870812437311936, + "loss": 2.6613, + "theoretical_loss": 3.532181420564319, + "tokens_seen": 1424617472 + }, + { + "epoch": 4.07, + "learning_rate": 0.00028707121364092273, + "loss": 2.6923, + "theoretical_loss": 3.5321669523112087, + "tokens_seen": 1424683008 + }, + { + "epoch": 4.07, + "learning_rate": 0.00028706118355065197, + "loss": 2.7143, + "theoretical_loss": 3.5321524849099717, + "tokens_seen": 1424748544 + }, + { + "epoch": 4.07, + "learning_rate": 0.00028705115346038115, + "loss": 2.7368, + "theoretical_loss": 3.532138018360518, + "tokens_seen": 1424814080 + }, + { + "epoch": 4.07, + "learning_rate": 0.00028704112337011033, + "loss": 2.709, + "theoretical_loss": 3.53212355266276, + "tokens_seen": 1424879616 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002870310932798395, + "loss": 2.624, + "theoretical_loss": 3.5321090878166066, + "tokens_seen": 1424945152 + }, + { + "epoch": 4.07, + "learning_rate": 0.00028702106318956875, + "loss": 2.7142, + "theoretical_loss": 3.532094623821969, + "tokens_seen": 1425010688 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002870110330992979, + "loss": 2.9406, + "theoretical_loss": 3.5320801606787584, + "tokens_seen": 1425076224 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002870010030090271, + "loss": 2.578, + "theoretical_loss": 3.5320656983868854, + "tokens_seen": 1425141760 + }, + { + "epoch": 4.07, + "learning_rate": 0.00028699097291875624, + "loss": 2.6162, + "theoretical_loss": 3.53205123694626, + "tokens_seen": 1425207296 + }, + { + "epoch": 4.07, + "learning_rate": 0.00028698094282848547, + "loss": 2.6059, + "theoretical_loss": 3.5320367763567937, + "tokens_seen": 1425272832 + }, + { + "epoch": 4.07, + "learning_rate": 0.00028697091273821465, + "loss": 2.5249, + "theoretical_loss": 3.532022316618397, + "tokens_seen": 1425338368 + }, + { + "epoch": 4.07, + "objective/train/docs_used": 1577330, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.346597671508789, + "objective/train/theoretical_loss": 3.5320078577309806, + "objective/train/tokens_used": 1445863904, + "theoretical_loss": 3.5320078577309806, + "tokens_seen": 1425403904 + }, + { + "epoch": 4.07, + "learning_rate": 0.00028696088264794383, + "loss": 2.605, + "theoretical_loss": 3.5320078577309806, + "tokens_seen": 1425403904 + }, + { + "epoch": 4.07, + "learning_rate": 0.000286950852557673, + "loss": 2.644, + "theoretical_loss": 3.531993399694456, + "tokens_seen": 1425469440 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002869408224674022, + "loss": 2.7256, + "theoretical_loss": 3.531978942508733, + "tokens_seen": 1425534976 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002869307923771314, + "loss": 2.6636, + "theoretical_loss": 3.531964486173723, + "tokens_seen": 1425600512 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002869207622868606, + "loss": 2.9186, + "theoretical_loss": 3.5319500306893366, + "tokens_seen": 1425666048 + }, + { + "epoch": 4.07, + "learning_rate": 0.00028691073219658974, + "loss": 2.8108, + "theoretical_loss": 3.531935576055485, + "tokens_seen": 1425731584 + }, + { + "epoch": 4.07, + "learning_rate": 0.000286900702106319, + "loss": 2.8785, + "theoretical_loss": 3.5319211222720788, + "tokens_seen": 1425797120 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002868906720160481, + "loss": 2.8474, + "theoretical_loss": 3.531906669339029, + "tokens_seen": 1425862656 + }, + { + "epoch": 4.07, + "learning_rate": 0.00028688064192577734, + "loss": 2.6304, + "theoretical_loss": 3.531892217256246, + "tokens_seen": 1425928192 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002868706118355065, + "loss": 2.753, + "theoretical_loss": 3.531877766023641, + "tokens_seen": 1425993728 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002868605817452357, + "loss": 2.8028, + "theoretical_loss": 3.531863315641125, + "tokens_seen": 1426059264 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002868505516549649, + "loss": 2.5905, + "theoretical_loss": 3.53184886610861, + "tokens_seen": 1426124800 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002868405215646941, + "loss": 3.0052, + "theoretical_loss": 3.531834417426005, + "tokens_seen": 1426190336 + }, + { + "epoch": 4.07, + "learning_rate": 0.00028683049147442324, + "loss": 2.3945, + "theoretical_loss": 3.5318199695932218, + "tokens_seen": 1426255872 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002868204613841525, + "loss": 2.5216, + "theoretical_loss": 3.5318055226101714, + "tokens_seen": 1426321408 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002868104312938816, + "loss": 2.8529, + "theoretical_loss": 3.5317910764767655, + "tokens_seen": 1426386944 + }, + { + "epoch": 4.07, + "learning_rate": 0.00028680040120361084, + "loss": 2.6891, + "theoretical_loss": 3.5317766311929133, + "tokens_seen": 1426452480 + }, + { + "epoch": 4.07, + "learning_rate": 0.00028679037111334, + "loss": 2.5532, + "theoretical_loss": 3.5317621867585274, + "tokens_seen": 1426518016 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002867803410230692, + "loss": 2.6447, + "theoretical_loss": 3.5317477431735185, + "tokens_seen": 1426583552 + }, + { + "epoch": 4.07, + "learning_rate": 0.00028677031093279844, + "loss": 2.5323, + "theoretical_loss": 3.5317333004377973, + "tokens_seen": 1426649088 + }, + { + "epoch": 4.07, + "learning_rate": 0.00028676028084252756, + "loss": 2.6454, + "theoretical_loss": 3.531718858551275, + "tokens_seen": 1426714624 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002867502507522568, + "loss": 2.8039, + "theoretical_loss": 3.531704417513863, + "tokens_seen": 1426780160 + }, + { + "epoch": 4.07, + "learning_rate": 0.000286740220661986, + "loss": 2.3451, + "theoretical_loss": 3.531689977325472, + "tokens_seen": 1426845696 + }, + { + "epoch": 4.07, + "learning_rate": 0.00028673019057171516, + "loss": 2.9817, + "theoretical_loss": 3.531675537986013, + "tokens_seen": 1426911232 + }, + { + "epoch": 4.07, + "learning_rate": 0.00028672016048144434, + "loss": 2.749, + "theoretical_loss": 3.5316610994953974, + "tokens_seen": 1426976768 + }, + { + "epoch": 4.07, + "objective/train/docs_used": 1578940, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.657034158706665, + "objective/train/theoretical_loss": 3.531646661853536, + "objective/train/tokens_used": 1447502304, + "theoretical_loss": 3.531646661853536, + "tokens_seen": 1427042304 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002867101303911736, + "loss": 2.8357, + "theoretical_loss": 3.531646661853536, + "tokens_seen": 1427042304 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002867001003009027, + "loss": 2.7306, + "theoretical_loss": 3.5316322250603402, + "tokens_seen": 1427107840 + }, + { + "epoch": 4.07, + "learning_rate": 0.00028669007021063194, + "loss": 2.5669, + "theoretical_loss": 3.531617789115721, + "tokens_seen": 1427173376 + }, + { + "epoch": 4.07, + "learning_rate": 0.00028668004012036107, + "loss": 2.8655, + "theoretical_loss": 3.5316033540195892, + "tokens_seen": 1427238912 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002866700100300903, + "loss": 2.6255, + "theoretical_loss": 3.531588919771857, + "tokens_seen": 1427304448 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002866599799398195, + "loss": 2.7666, + "theoretical_loss": 3.5315744863724348, + "tokens_seen": 1427369984 + }, + { + "epoch": 4.07, + "learning_rate": 0.00028664994984954866, + "loss": 2.7119, + "theoretical_loss": 3.531560053821234, + "tokens_seen": 1427435520 + }, + { + "epoch": 4.07, + "learning_rate": 0.00028663991975927785, + "loss": 2.5825, + "theoretical_loss": 3.531545622118166, + "tokens_seen": 1427501056 + }, + { + "epoch": 4.07, + "learning_rate": 0.000286629889669007, + "loss": 2.3998, + "theoretical_loss": 3.5315311912631415, + "tokens_seen": 1427566592 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002866198595787362, + "loss": 2.5511, + "theoretical_loss": 3.531516761256072, + "tokens_seen": 1427632128 + }, + { + "epoch": 4.07, + "learning_rate": 0.00028660982948846544, + "loss": 2.8859, + "theoretical_loss": 3.531502332096869, + "tokens_seen": 1427697664 + }, + { + "epoch": 4.07, + "learning_rate": 0.00028659979939819457, + "loss": 2.5624, + "theoretical_loss": 3.531487903785443, + "tokens_seen": 1427763200 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002865897693079238, + "loss": 2.8676, + "theoretical_loss": 3.5314734763217066, + "tokens_seen": 1427828736 + }, + { + "epoch": 4.07, + "learning_rate": 0.00028657973921765293, + "loss": 2.8014, + "theoretical_loss": 3.5314590497055702, + "tokens_seen": 1427894272 + }, + { + "epoch": 4.07, + "learning_rate": 0.00028656970912738217, + "loss": 2.6093, + "theoretical_loss": 3.531444623936945, + "tokens_seen": 1427959808 + }, + { + "epoch": 4.07, + "learning_rate": 0.00028655967903711135, + "loss": 2.5359, + "theoretical_loss": 3.5314301990157424, + "tokens_seen": 1428025344 + }, + { + "epoch": 4.07, + "learning_rate": 0.00028654964894684053, + "loss": 2.6843, + "theoretical_loss": 3.531415774941874, + "tokens_seen": 1428090880 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002865396188565697, + "loss": 2.7487, + "theoretical_loss": 3.5314013517152514, + "tokens_seen": 1428156416 + }, + { + "epoch": 4.07, + "learning_rate": 0.00028652958876629895, + "loss": 2.4328, + "theoretical_loss": 3.531386929335785, + "tokens_seen": 1428221952 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002865195586760281, + "loss": 2.7507, + "theoretical_loss": 3.531372507803387, + "tokens_seen": 1428287488 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002865095285857573, + "loss": 2.7066, + "theoretical_loss": 3.531358087117969, + "tokens_seen": 1428353024 + }, + { + "epoch": 4.07, + "learning_rate": 0.00028649949849548644, + "loss": 2.5695, + "theoretical_loss": 3.531343667279441, + "tokens_seen": 1428418560 + }, + { + "epoch": 4.07, + "learning_rate": 0.00028648946840521567, + "loss": 2.377, + "theoretical_loss": 3.531329248287716, + "tokens_seen": 1428484096 + }, + { + "epoch": 4.07, + "learning_rate": 0.00028647943831494485, + "loss": 2.8074, + "theoretical_loss": 3.531314830142705, + "tokens_seen": 1428549632 + }, + { + "epoch": 4.07, + "learning_rate": 0.00028646940822467403, + "loss": 2.5772, + "theoretical_loss": 3.5313004128443195, + "tokens_seen": 1428615168 + }, + { + "epoch": 4.07, + "objective/train/docs_used": 1579734, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7796366214752197, + "objective/train/theoretical_loss": 3.53128599639247, + "objective/train/tokens_used": 1449140704, + "theoretical_loss": 3.53128599639247, + "tokens_seen": 1428680704 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002864593781344032, + "loss": 2.912, + "theoretical_loss": 3.53128599639247, + "tokens_seen": 1428680704 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002864493480441324, + "loss": 2.6375, + "theoretical_loss": 3.5312715807870685, + "tokens_seen": 1428746240 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002864393179538616, + "loss": 2.8322, + "theoretical_loss": 3.531257166028027, + "tokens_seen": 1428811776 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002864292878635908, + "loss": 2.5146, + "theoretical_loss": 3.5312427521152565, + "tokens_seen": 1428877312 + }, + { + "epoch": 4.07, + "learning_rate": 0.00028641925777331994, + "loss": 2.7135, + "theoretical_loss": 3.5312283390486687, + "tokens_seen": 1428942848 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002864092276830492, + "loss": 2.7155, + "theoretical_loss": 3.5312139268281753, + "tokens_seen": 1429008384 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002863991975927783, + "loss": 2.5024, + "theoretical_loss": 3.531199515453687, + "tokens_seen": 1429073920 + }, + { + "epoch": 4.07, + "learning_rate": 0.00028638916750250754, + "loss": 2.736, + "theoretical_loss": 3.5311851049251164, + "tokens_seen": 1429139456 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002863791374122367, + "loss": 2.7296, + "theoretical_loss": 3.5311706952423743, + "tokens_seen": 1429204992 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002863691073219659, + "loss": 2.68, + "theoretical_loss": 3.5311562864053725, + "tokens_seen": 1429270528 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002863590772316951, + "loss": 2.5365, + "theoretical_loss": 3.5311418784140227, + "tokens_seen": 1429336064 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002863490471414243, + "loss": 2.7918, + "theoretical_loss": 3.5311274712682366, + "tokens_seen": 1429401600 + }, + { + "epoch": 4.07, + "learning_rate": 0.00028633901705115344, + "loss": 2.5645, + "theoretical_loss": 3.531113064967925, + "tokens_seen": 1429467136 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002863289869608827, + "loss": 2.7413, + "theoretical_loss": 3.5310986595130007, + "tokens_seen": 1429532672 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002863189568706118, + "loss": 2.554, + "theoretical_loss": 3.5310842549033747, + "tokens_seen": 1429598208 + }, + { + "epoch": 4.07, + "learning_rate": 0.00028630892678034104, + "loss": 2.643, + "theoretical_loss": 3.5310698511389593, + "tokens_seen": 1429663744 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002862988966900702, + "loss": 2.5413, + "theoretical_loss": 3.531055448219665, + "tokens_seen": 1429729280 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002862888665997994, + "loss": 2.6201, + "theoretical_loss": 3.5310410461454045, + "tokens_seen": 1429794816 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002862788365095286, + "loss": 2.6887, + "theoretical_loss": 3.5310266449160888, + "tokens_seen": 1429860352 + }, + { + "epoch": 4.07, + "learning_rate": 0.00028626880641925776, + "loss": 2.6714, + "theoretical_loss": 3.53101224453163, + "tokens_seen": 1429925888 + }, + { + "epoch": 4.07, + "learning_rate": 0.00028625877632898694, + "loss": 2.5562, + "theoretical_loss": 3.5309978449919397, + "tokens_seen": 1429991424 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002862487462387162, + "loss": 2.7693, + "theoretical_loss": 3.5309834462969296, + "tokens_seen": 1430056960 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002862387161484453, + "loss": 2.8589, + "theoretical_loss": 3.5309690484465115, + "tokens_seen": 1430122496 + }, + { + "epoch": 4.07, + "learning_rate": 0.00028622868605817454, + "loss": 2.706, + "theoretical_loss": 3.530954651440597, + "tokens_seen": 1430188032 + }, + { + "epoch": 4.07, + "learning_rate": 0.00028621865596790367, + "loss": 2.8354, + "theoretical_loss": 3.5309402552790985, + "tokens_seen": 1430253568 + }, + { + "epoch": 4.07, + "objective/train/docs_used": 1580976, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8712832927703857, + "objective/train/theoretical_loss": 3.530925859961927, + "objective/train/tokens_used": 1450779104, + "theoretical_loss": 3.530925859961927, + "tokens_seen": 1430319104 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002862086258776329, + "loss": 2.7676, + "theoretical_loss": 3.530925859961927, + "tokens_seen": 1430319104 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002861985957873621, + "loss": 2.5262, + "theoretical_loss": 3.5309114654889946, + "tokens_seen": 1430384640 + }, + { + "epoch": 4.07, + "learning_rate": 0.00028618856569709127, + "loss": 2.659, + "theoretical_loss": 3.530897071860213, + "tokens_seen": 1430450176 + }, + { + "epoch": 4.07, + "learning_rate": 0.00028617853560682045, + "loss": 2.7417, + "theoretical_loss": 3.530882679075494, + "tokens_seen": 1430515712 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002861685055165497, + "loss": 2.7481, + "theoretical_loss": 3.53086828713475, + "tokens_seen": 1430581248 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002861584754262788, + "loss": 2.532, + "theoretical_loss": 3.5308538960378923, + "tokens_seen": 1430646784 + }, + { + "epoch": 4.07, + "learning_rate": 0.00028614844533600805, + "loss": 2.6953, + "theoretical_loss": 3.530839505784833, + "tokens_seen": 1430712320 + }, + { + "epoch": 4.07, + "learning_rate": 0.00028613841524573717, + "loss": 2.6791, + "theoretical_loss": 3.5308251163754836, + "tokens_seen": 1430777856 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002861283851554664, + "loss": 2.5014, + "theoretical_loss": 3.5308107278097562, + "tokens_seen": 1430843392 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002861183550651956, + "loss": 2.7834, + "theoretical_loss": 3.530796340087563, + "tokens_seen": 1430908928 + }, + { + "epoch": 4.07, + "learning_rate": 0.00028610832497492477, + "loss": 2.782, + "theoretical_loss": 3.530781953208816, + "tokens_seen": 1430974464 + }, + { + "epoch": 4.07, + "learning_rate": 0.00028609829488465395, + "loss": 2.7377, + "theoretical_loss": 3.5307675671734264, + "tokens_seen": 1431040000 + }, + { + "epoch": 4.07, + "learning_rate": 0.00028608826479438313, + "loss": 2.4967, + "theoretical_loss": 3.5307531819813063, + "tokens_seen": 1431105536 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002860782347041123, + "loss": 2.6931, + "theoretical_loss": 3.5307387976323685, + "tokens_seen": 1431171072 + }, + { + "epoch": 4.07, + "learning_rate": 0.00028606820461384155, + "loss": 2.6167, + "theoretical_loss": 3.5307244141265235, + "tokens_seen": 1431236608 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002860581745235707, + "loss": 2.6194, + "theoretical_loss": 3.530710031463685, + "tokens_seen": 1431302144 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002860481444332999, + "loss": 2.6382, + "theoretical_loss": 3.5306956496437643, + "tokens_seen": 1431367680 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002860381143430291, + "loss": 2.6777, + "theoretical_loss": 3.5306812686666724, + "tokens_seen": 1431433216 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002860280842527583, + "loss": 2.7881, + "theoretical_loss": 3.5306668885323225, + "tokens_seen": 1431498752 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002860180541624875, + "loss": 2.5622, + "theoretical_loss": 3.5306525092406265, + "tokens_seen": 1431564288 + }, + { + "epoch": 4.07, + "learning_rate": 0.00028600802407221664, + "loss": 2.7507, + "theoretical_loss": 3.5306381307914965, + "tokens_seen": 1431629824 + }, + { + "epoch": 4.07, + "learning_rate": 0.00028599799398194587, + "loss": 2.7967, + "theoretical_loss": 3.530623753184844, + "tokens_seen": 1431695360 + }, + { + "epoch": 4.07, + "learning_rate": 0.00028598796389167505, + "loss": 2.6421, + "theoretical_loss": 3.530609376420582, + "tokens_seen": 1431760896 + }, + { + "epoch": 4.07, + "learning_rate": 0.00028597793380140423, + "loss": 2.7102, + "theoretical_loss": 3.530595000498621, + "tokens_seen": 1431826432 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002859679037111334, + "loss": 2.5797, + "theoretical_loss": 3.5305806254188745, + "tokens_seen": 1431891968 + }, + { + "epoch": 4.07, + "objective/train/docs_used": 1581614, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.419712543487549, + "objective/train/theoretical_loss": 3.5305662511812548, + "objective/train/tokens_used": 1452417504, + "theoretical_loss": 3.5305662511812548, + "tokens_seen": 1431957504 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002859578736208626, + "loss": 2.4994, + "theoretical_loss": 3.5305662511812548, + "tokens_seen": 1431957504 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002859478435305918, + "loss": 2.7283, + "theoretical_loss": 3.5305518777856726, + "tokens_seen": 1432023040 + }, + { + "epoch": 4.07, + "learning_rate": 0.000285937813440321, + "loss": 2.7827, + "theoretical_loss": 3.530537505232041, + "tokens_seen": 1432088576 + }, + { + "epoch": 4.07, + "learning_rate": 0.00028592778335005014, + "loss": 2.7673, + "theoretical_loss": 3.5305231335202727, + "tokens_seen": 1432154112 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002859177532597794, + "loss": 2.849, + "theoretical_loss": 3.5305087626502787, + "tokens_seen": 1432219648 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002859077231695085, + "loss": 2.8395, + "theoretical_loss": 3.530494392621972, + "tokens_seen": 1432285184 + }, + { + "epoch": 4.07, + "learning_rate": 0.00028589769307923774, + "loss": 2.6592, + "theoretical_loss": 3.5304800234352642, + "tokens_seen": 1432350720 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002858876629889669, + "loss": 2.6934, + "theoretical_loss": 3.530465655090068, + "tokens_seen": 1432416256 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002858776328986961, + "loss": 2.6282, + "theoretical_loss": 3.530451287586295, + "tokens_seen": 1432481792 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002858676028084253, + "loss": 2.6504, + "theoretical_loss": 3.530436920923858, + "tokens_seen": 1432547328 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002858575727181545, + "loss": 2.4934, + "theoretical_loss": 3.5304225551026693, + "tokens_seen": 1432612864 + }, + { + "epoch": 4.07, + "learning_rate": 0.00028584754262788364, + "loss": 2.621, + "theoretical_loss": 3.5304081901226407, + "tokens_seen": 1432678400 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002858375125376129, + "loss": 2.5828, + "theoretical_loss": 3.530393825983685, + "tokens_seen": 1432743936 + }, + { + "epoch": 4.07, + "learning_rate": 0.000285827482447342, + "loss": 2.852, + "theoretical_loss": 3.5303794626857146, + "tokens_seen": 1432809472 + }, + { + "epoch": 4.07, + "learning_rate": 0.00028581745235707124, + "loss": 2.5021, + "theoretical_loss": 3.5303651002286407, + "tokens_seen": 1432875008 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002858074222668004, + "loss": 2.7758, + "theoretical_loss": 3.530350738612377, + "tokens_seen": 1432940544 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002857973921765296, + "loss": 2.4004, + "theoretical_loss": 3.5303363778368344, + "tokens_seen": 1433006080 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002857873620862588, + "loss": 2.4702, + "theoretical_loss": 3.530322017901926, + "tokens_seen": 1433071616 + }, + { + "epoch": 4.07, + "learning_rate": 0.00028577733199598796, + "loss": 2.6284, + "theoretical_loss": 3.5303076588075646, + "tokens_seen": 1433137152 + }, + { + "epoch": 4.07, + "learning_rate": 0.00028576730190571714, + "loss": 2.9295, + "theoretical_loss": 3.5302933005536623, + "tokens_seen": 1433202688 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002857572718154464, + "loss": 2.5359, + "theoretical_loss": 3.530278943140131, + "tokens_seen": 1433268224 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002857472417251755, + "loss": 2.5897, + "theoretical_loss": 3.530264586566883, + "tokens_seen": 1433333760 + }, + { + "epoch": 4.07, + "learning_rate": 0.00028573721163490474, + "loss": 2.6742, + "theoretical_loss": 3.5302502308338317, + "tokens_seen": 1433399296 + }, + { + "epoch": 4.07, + "learning_rate": 0.00028572718154463387, + "loss": 2.7224, + "theoretical_loss": 3.5302358759408885, + "tokens_seen": 1433464832 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002857171514543631, + "loss": 2.5141, + "theoretical_loss": 3.5302215218879667, + "tokens_seen": 1433530368 + }, + { + "epoch": 4.07, + "objective/train/docs_used": 1582179, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.372399091720581, + "objective/train/theoretical_loss": 3.530207168674978, + "objective/train/tokens_used": 1454055904, + "theoretical_loss": 3.530207168674978, + "tokens_seen": 1433595904 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002857071213640923, + "loss": 2.5721, + "theoretical_loss": 3.530207168674978, + "tokens_seen": 1433595904 + }, + { + "epoch": 4.07, + "learning_rate": 0.00028569709127382147, + "loss": 2.4279, + "theoretical_loss": 3.5301928163018346, + "tokens_seen": 1433661440 + }, + { + "epoch": 4.07, + "learning_rate": 0.00028568706118355065, + "loss": 2.593, + "theoretical_loss": 3.5301784647684498, + "tokens_seen": 1433726976 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002856770310932799, + "loss": 2.7227, + "theoretical_loss": 3.5301641140747364, + "tokens_seen": 1433792512 + }, + { + "epoch": 4.07, + "learning_rate": 0.000285667001003009, + "loss": 2.9401, + "theoretical_loss": 3.5301497642206057, + "tokens_seen": 1433858048 + }, + { + "epoch": 4.07, + "learning_rate": 0.00028565697091273825, + "loss": 2.4208, + "theoretical_loss": 3.530135415205971, + "tokens_seen": 1433923584 + }, + { + "epoch": 4.07, + "learning_rate": 0.00028564694082246737, + "loss": 2.3774, + "theoretical_loss": 3.530121067030745, + "tokens_seen": 1433989120 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002856369107321966, + "loss": 2.7481, + "theoretical_loss": 3.530106719694839, + "tokens_seen": 1434054656 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002856268806419258, + "loss": 2.5669, + "theoretical_loss": 3.5300923731981664, + "tokens_seen": 1434120192 + }, + { + "epoch": 4.07, + "learning_rate": 0.00028561685055165497, + "loss": 2.6281, + "theoretical_loss": 3.5300780275406405, + "tokens_seen": 1434185728 + }, + { + "epoch": 4.07, + "learning_rate": 0.00028560682046138415, + "loss": 2.5687, + "theoretical_loss": 3.5300636827221727, + "tokens_seen": 1434251264 + }, + { + "epoch": 4.07, + "learning_rate": 0.00028559679037111333, + "loss": 2.5335, + "theoretical_loss": 3.530049338742676, + "tokens_seen": 1434316800 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002855867602808425, + "loss": 2.568, + "theoretical_loss": 3.5300349956020636, + "tokens_seen": 1434382336 + }, + { + "epoch": 4.07, + "learning_rate": 0.00028557673019057175, + "loss": 2.7643, + "theoretical_loss": 3.530020653300247, + "tokens_seen": 1434447872 + }, + { + "epoch": 4.07, + "learning_rate": 0.0002855667001003009, + "loss": 2.5505, + "theoretical_loss": 3.5300063118371394, + "tokens_seen": 1434513408 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002855566700100301, + "loss": 2.7626, + "theoretical_loss": 3.5299919712126533, + "tokens_seen": 1434578944 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002855466399197593, + "loss": 2.6743, + "theoretical_loss": 3.5299776314267017, + "tokens_seen": 1434644480 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002855366098294885, + "loss": 2.7768, + "theoretical_loss": 3.5299632924791973, + "tokens_seen": 1434710016 + }, + { + "epoch": 4.08, + "learning_rate": 0.00028552657973921765, + "loss": 2.8286, + "theoretical_loss": 3.5299489543700524, + "tokens_seen": 1434775552 + }, + { + "epoch": 4.08, + "learning_rate": 0.00028551654964894684, + "loss": 2.5388, + "theoretical_loss": 3.529934617099179, + "tokens_seen": 1434841088 + }, + { + "epoch": 4.08, + "learning_rate": 0.000285506519558676, + "loss": 2.6986, + "theoretical_loss": 3.5299202806664915, + "tokens_seen": 1434906624 + }, + { + "epoch": 4.08, + "learning_rate": 0.00028549648946840525, + "loss": 2.6987, + "theoretical_loss": 3.5299059450719015, + "tokens_seen": 1434972160 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002854864593781344, + "loss": 2.9928, + "theoretical_loss": 3.529891610315322, + "tokens_seen": 1435037696 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002854764292878636, + "loss": 2.8229, + "theoretical_loss": 3.5298772763966655, + "tokens_seen": 1435103232 + }, + { + "epoch": 4.08, + "learning_rate": 0.00028546639919759274, + "loss": 2.7209, + "theoretical_loss": 3.5298629433158455, + "tokens_seen": 1435168768 + }, + { + "epoch": 4.08, + "objective/train/docs_used": 1583288, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.874298572540283, + "objective/train/theoretical_loss": 3.5298486110727736, + "objective/train/tokens_used": 1455694304, + "theoretical_loss": 3.5298486110727736, + "tokens_seen": 1435234304 + }, + { + "epoch": 4.08, + "learning_rate": 0.000285456369107322, + "loss": 2.8018, + "theoretical_loss": 3.5298486110727736, + "tokens_seen": 1435234304 + }, + { + "epoch": 4.08, + "learning_rate": 0.00028544633901705116, + "loss": 2.578, + "theoretical_loss": 3.529834279667363, + "tokens_seen": 1435299840 + }, + { + "epoch": 4.08, + "learning_rate": 0.00028543630892678034, + "loss": 2.7333, + "theoretical_loss": 3.5298199490995277, + "tokens_seen": 1435365376 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002854262788365095, + "loss": 2.7832, + "theoretical_loss": 3.529805619369179, + "tokens_seen": 1435430912 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002854162487462387, + "loss": 2.7618, + "theoretical_loss": 3.52979129047623, + "tokens_seen": 1435496448 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002854062186559679, + "loss": 2.5082, + "theoretical_loss": 3.5297769624205944, + "tokens_seen": 1435561984 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002853961885656971, + "loss": 2.6455, + "theoretical_loss": 3.529762635202184, + "tokens_seen": 1435627520 + }, + { + "epoch": 4.08, + "learning_rate": 0.00028538615847542624, + "loss": 2.5146, + "theoretical_loss": 3.5297483088209125, + "tokens_seen": 1435693056 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002853761283851555, + "loss": 2.7314, + "theoretical_loss": 3.529733983276692, + "tokens_seen": 1435758592 + }, + { + "epoch": 4.08, + "learning_rate": 0.00028536609829488466, + "loss": 2.7992, + "theoretical_loss": 3.529719658569436, + "tokens_seen": 1435824128 + }, + { + "epoch": 4.08, + "learning_rate": 0.00028535606820461384, + "loss": 2.7624, + "theoretical_loss": 3.529705334699057, + "tokens_seen": 1435889664 + }, + { + "epoch": 4.08, + "learning_rate": 0.000285346038114343, + "loss": 2.5357, + "theoretical_loss": 3.5296910116654683, + "tokens_seen": 1435955200 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002853360080240722, + "loss": 2.587, + "theoretical_loss": 3.529676689468582, + "tokens_seen": 1436020736 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002853259779338014, + "loss": 2.6797, + "theoretical_loss": 3.5296623681083124, + "tokens_seen": 1436086272 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002853159478435306, + "loss": 2.6747, + "theoretical_loss": 3.5296480475845713, + "tokens_seen": 1436151808 + }, + { + "epoch": 4.08, + "learning_rate": 0.00028530591775325975, + "loss": 2.9066, + "theoretical_loss": 3.5296337278972723, + "tokens_seen": 1436217344 + }, + { + "epoch": 4.08, + "learning_rate": 0.000285295887662989, + "loss": 2.9173, + "theoretical_loss": 3.5296194090463278, + "tokens_seen": 1436282880 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002852858575727181, + "loss": 2.7209, + "theoretical_loss": 3.5296050910316517, + "tokens_seen": 1436348416 + }, + { + "epoch": 4.08, + "learning_rate": 0.00028527582748244735, + "loss": 2.6102, + "theoretical_loss": 3.5295907738531564, + "tokens_seen": 1436413952 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002852657973921766, + "loss": 2.6939, + "theoretical_loss": 3.529576457510754, + "tokens_seen": 1436479488 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002852557673019057, + "loss": 2.8535, + "theoretical_loss": 3.5295621420043597, + "tokens_seen": 1436545024 + }, + { + "epoch": 4.08, + "learning_rate": 0.00028524573721163494, + "loss": 2.7347, + "theoretical_loss": 3.529547827333885, + "tokens_seen": 1436610560 + }, + { + "epoch": 4.08, + "learning_rate": 0.00028523570712136407, + "loss": 2.7182, + "theoretical_loss": 3.5295335134992434, + "tokens_seen": 1436676096 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002852256770310933, + "loss": 2.6752, + "theoretical_loss": 3.5295192005003475, + "tokens_seen": 1436741632 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002852156469408225, + "loss": 2.9172, + "theoretical_loss": 3.529504888337111, + "tokens_seen": 1436807168 + }, + { + "epoch": 4.08, + "objective/train/docs_used": 1583910, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5404577255249023, + "objective/train/theoretical_loss": 3.529490577009447, + "objective/train/tokens_used": 1457332704, + "theoretical_loss": 3.529490577009447, + "tokens_seen": 1436872704 + }, + { + "epoch": 4.08, + "learning_rate": 0.00028520561685055167, + "loss": 2.4372, + "theoretical_loss": 3.529490577009447, + "tokens_seen": 1436872704 + }, + { + "epoch": 4.08, + "learning_rate": 0.00028519558676028085, + "loss": 2.4868, + "theoretical_loss": 3.529476266517268, + "tokens_seen": 1436938240 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002851855566700101, + "loss": 2.4531, + "theoretical_loss": 3.5294619568604877, + "tokens_seen": 1437003776 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002851755265797392, + "loss": 2.7788, + "theoretical_loss": 3.529447648039019, + "tokens_seen": 1437069312 + }, + { + "epoch": 4.08, + "learning_rate": 0.00028516549648946845, + "loss": 2.7561, + "theoretical_loss": 3.5294333400527753, + "tokens_seen": 1437134848 + }, + { + "epoch": 4.08, + "learning_rate": 0.00028515546639919757, + "loss": 2.8554, + "theoretical_loss": 3.529419032901669, + "tokens_seen": 1437200384 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002851454363089268, + "loss": 2.7275, + "theoretical_loss": 3.5294047265856143, + "tokens_seen": 1437265920 + }, + { + "epoch": 4.08, + "learning_rate": 0.000285135406218656, + "loss": 2.5726, + "theoretical_loss": 3.529390421104524, + "tokens_seen": 1437331456 + }, + { + "epoch": 4.08, + "learning_rate": 0.00028512537612838517, + "loss": 2.8757, + "theoretical_loss": 3.5293761164583106, + "tokens_seen": 1437396992 + }, + { + "epoch": 4.08, + "learning_rate": 0.00028511534603811435, + "loss": 2.6895, + "theoretical_loss": 3.5293618126468886, + "tokens_seen": 1437462528 + }, + { + "epoch": 4.08, + "learning_rate": 0.00028510531594784353, + "loss": 2.6364, + "theoretical_loss": 3.52934750967017, + "tokens_seen": 1437528064 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002850952858575727, + "loss": 2.6436, + "theoretical_loss": 3.529333207528069, + "tokens_seen": 1437593600 + }, + { + "epoch": 4.08, + "learning_rate": 0.00028508525576730195, + "loss": 2.4625, + "theoretical_loss": 3.5293189062204986, + "tokens_seen": 1437659136 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002850752256770311, + "loss": 2.7444, + "theoretical_loss": 3.529304605747371, + "tokens_seen": 1437724672 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002850651955867603, + "loss": 2.6947, + "theoretical_loss": 3.529290306108601, + "tokens_seen": 1437790208 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002850551654964895, + "loss": 2.6257, + "theoretical_loss": 3.5292760073041016, + "tokens_seen": 1437855744 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002850451354062187, + "loss": 2.544, + "theoretical_loss": 3.5292617093337855, + "tokens_seen": 1437921280 + }, + { + "epoch": 4.08, + "learning_rate": 0.00028503510531594785, + "loss": 2.9978, + "theoretical_loss": 3.5292474121975665, + "tokens_seen": 1437986816 + }, + { + "epoch": 4.08, + "learning_rate": 0.00028502507522567704, + "loss": 2.4569, + "theoretical_loss": 3.5292331158953574, + "tokens_seen": 1438052352 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002850150451354062, + "loss": 2.5917, + "theoretical_loss": 3.5292188204270722, + "tokens_seen": 1438117888 + }, + { + "epoch": 4.08, + "learning_rate": 0.00028500501504513545, + "loss": 2.7325, + "theoretical_loss": 3.5292045257926237, + "tokens_seen": 1438183424 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002849949849548646, + "loss": 2.8818, + "theoretical_loss": 3.5291902319919255, + "tokens_seen": 1438248960 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002849849548645938, + "loss": 2.8042, + "theoretical_loss": 3.529175939024891, + "tokens_seen": 1438314496 + }, + { + "epoch": 4.08, + "learning_rate": 0.00028497492477432294, + "loss": 2.7066, + "theoretical_loss": 3.529161646891434, + "tokens_seen": 1438380032 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002849648946840522, + "loss": 2.6888, + "theoretical_loss": 3.529147355591467, + "tokens_seen": 1438445568 + }, + { + "epoch": 4.08, + "objective/train/docs_used": 1585264, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9627788066864014, + "objective/train/theoretical_loss": 3.5291330651249044, + "objective/train/tokens_used": 1458971104, + "theoretical_loss": 3.5291330651249044, + "tokens_seen": 1438511104 + }, + { + "epoch": 4.08, + "learning_rate": 0.00028495486459378136, + "loss": 2.6728, + "theoretical_loss": 3.5291330651249044, + "tokens_seen": 1438511104 + }, + { + "epoch": 4.08, + "learning_rate": 0.00028494483450351054, + "loss": 2.6711, + "theoretical_loss": 3.529118775491659, + "tokens_seen": 1438576640 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002849348044132397, + "loss": 2.7203, + "theoretical_loss": 3.5291044866916437, + "tokens_seen": 1438642176 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002849247743229689, + "loss": 2.835, + "theoretical_loss": 3.5290901987247736, + "tokens_seen": 1438707712 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002849147442326981, + "loss": 2.7568, + "theoretical_loss": 3.5290759115909607, + "tokens_seen": 1438773248 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002849047141424273, + "loss": 2.7452, + "theoretical_loss": 3.5290616252901192, + "tokens_seen": 1438838784 + }, + { + "epoch": 4.08, + "learning_rate": 0.00028489468405215644, + "loss": 2.5437, + "theoretical_loss": 3.5290473398221627, + "tokens_seen": 1438904320 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002848846539618857, + "loss": 2.9406, + "theoretical_loss": 3.529033055187004, + "tokens_seen": 1438969856 + }, + { + "epoch": 4.08, + "learning_rate": 0.00028487462387161486, + "loss": 2.7922, + "theoretical_loss": 3.529018771384557, + "tokens_seen": 1439035392 + }, + { + "epoch": 4.08, + "learning_rate": 0.00028486459378134404, + "loss": 2.6864, + "theoretical_loss": 3.529004488414736, + "tokens_seen": 1439100928 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002848545636910732, + "loss": 2.873, + "theoretical_loss": 3.528990206277453, + "tokens_seen": 1439166464 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002848445336008024, + "loss": 2.725, + "theoretical_loss": 3.528975924972623, + "tokens_seen": 1439232000 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002848345035105316, + "loss": 2.6527, + "theoretical_loss": 3.528961644500159, + "tokens_seen": 1439297536 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002848244734202608, + "loss": 2.5969, + "theoretical_loss": 3.528947364859974, + "tokens_seen": 1439363072 + }, + { + "epoch": 4.08, + "learning_rate": 0.00028481444332998995, + "loss": 2.6177, + "theoretical_loss": 3.5289330860519823, + "tokens_seen": 1439428608 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002848044132397192, + "loss": 2.7198, + "theoretical_loss": 3.5289188080760976, + "tokens_seen": 1439494144 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002847943831494483, + "loss": 2.4022, + "theoretical_loss": 3.5289045309322336, + "tokens_seen": 1439559680 + }, + { + "epoch": 4.08, + "learning_rate": 0.00028478435305917755, + "loss": 2.7359, + "theoretical_loss": 3.528890254620303, + "tokens_seen": 1439625216 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002847743229689067, + "loss": 2.6777, + "theoretical_loss": 3.5288759791402207, + "tokens_seen": 1439690752 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002847642928786359, + "loss": 2.518, + "theoretical_loss": 3.5288617044918995, + "tokens_seen": 1439756288 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002847542627883651, + "loss": 2.7267, + "theoretical_loss": 3.5288474306752535, + "tokens_seen": 1439821824 + }, + { + "epoch": 4.08, + "learning_rate": 0.00028474423269809427, + "loss": 2.708, + "theoretical_loss": 3.5288331576901966, + "tokens_seen": 1439887360 + }, + { + "epoch": 4.08, + "learning_rate": 0.00028473420260782345, + "loss": 2.6584, + "theoretical_loss": 3.5288188855366416, + "tokens_seen": 1439952896 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002847241725175527, + "loss": 2.5356, + "theoretical_loss": 3.5288046142145033, + "tokens_seen": 1440018432 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002847141424272818, + "loss": 2.7895, + "theoretical_loss": 3.5287903437236947, + "tokens_seen": 1440083968 + }, + { + "epoch": 4.08, + "objective/train/docs_used": 1585991, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4762494564056396, + "objective/train/theoretical_loss": 3.5287760740641296, + "objective/train/tokens_used": 1460609504, + "theoretical_loss": 3.5287760740641296, + "tokens_seen": 1440149504 + }, + { + "epoch": 4.08, + "learning_rate": 0.00028470411233701105, + "loss": 2.4325, + "theoretical_loss": 3.5287760740641296, + "tokens_seen": 1440149504 + }, + { + "epoch": 4.08, + "learning_rate": 0.00028469408224674023, + "loss": 2.9182, + "theoretical_loss": 3.528761805235722, + "tokens_seen": 1440215040 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002846840521564694, + "loss": 2.6841, + "theoretical_loss": 3.5287475372383854, + "tokens_seen": 1440280576 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002846740220661986, + "loss": 2.6416, + "theoretical_loss": 3.528733270072034, + "tokens_seen": 1440346112 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002846639919759278, + "loss": 2.7276, + "theoretical_loss": 3.5287190037365814, + "tokens_seen": 1440411648 + }, + { + "epoch": 4.08, + "learning_rate": 0.00028465396188565695, + "loss": 2.7666, + "theoretical_loss": 3.528704738231941, + "tokens_seen": 1440477184 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002846439317953862, + "loss": 2.4176, + "theoretical_loss": 3.5286904735580276, + "tokens_seen": 1440542720 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002846339017051153, + "loss": 2.6206, + "theoretical_loss": 3.528676209714754, + "tokens_seen": 1440608256 + }, + { + "epoch": 4.08, + "learning_rate": 0.00028462387161484455, + "loss": 2.676, + "theoretical_loss": 3.5286619467020346, + "tokens_seen": 1440673792 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002846138415245737, + "loss": 2.9246, + "theoretical_loss": 3.5286476845197834, + "tokens_seen": 1440739328 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002846038114343029, + "loss": 2.565, + "theoretical_loss": 3.5286334231679133, + "tokens_seen": 1440804864 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002845937813440321, + "loss": 2.5963, + "theoretical_loss": 3.5286191626463395, + "tokens_seen": 1440870400 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002845837512537613, + "loss": 2.309, + "theoretical_loss": 3.5286049029549753, + "tokens_seen": 1440935936 + }, + { + "epoch": 4.08, + "learning_rate": 0.00028457372116349046, + "loss": 2.7321, + "theoretical_loss": 3.5285906440937342, + "tokens_seen": 1441001472 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002845636910732197, + "loss": 2.9154, + "theoretical_loss": 3.528576386062531, + "tokens_seen": 1441067008 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002845536609829488, + "loss": 2.791, + "theoretical_loss": 3.528562128861279, + "tokens_seen": 1441132544 + }, + { + "epoch": 4.08, + "learning_rate": 0.00028454363089267805, + "loss": 2.5997, + "theoretical_loss": 3.5285478724898924, + "tokens_seen": 1441198080 + }, + { + "epoch": 4.08, + "learning_rate": 0.00028453360080240724, + "loss": 2.6299, + "theoretical_loss": 3.528533616948285, + "tokens_seen": 1441263616 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002845235707121364, + "loss": 2.5887, + "theoretical_loss": 3.5285193622363704, + "tokens_seen": 1441329152 + }, + { + "epoch": 4.08, + "learning_rate": 0.00028451354062186565, + "loss": 2.6146, + "theoretical_loss": 3.5285051083540635, + "tokens_seen": 1441394688 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002845035105315948, + "loss": 2.6975, + "theoretical_loss": 3.528490855301278, + "tokens_seen": 1441460224 + }, + { + "epoch": 4.08, + "learning_rate": 0.000284493480441324, + "loss": 2.5712, + "theoretical_loss": 3.5284766030779275, + "tokens_seen": 1441525760 + }, + { + "epoch": 4.08, + "learning_rate": 0.00028448345035105314, + "loss": 2.7834, + "theoretical_loss": 3.528462351683926, + "tokens_seen": 1441591296 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002844734202607824, + "loss": 2.5477, + "theoretical_loss": 3.5284481011191886, + "tokens_seen": 1441656832 + }, + { + "epoch": 4.08, + "learning_rate": 0.00028446339017051156, + "loss": 2.693, + "theoretical_loss": 3.528433851383628, + "tokens_seen": 1441722368 + }, + { + "epoch": 4.08, + "objective/train/docs_used": 1587458, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6036527156829834, + "objective/train/theoretical_loss": 3.5284196024771592, + "objective/train/tokens_used": 1462247904, + "theoretical_loss": 3.5284196024771592, + "tokens_seen": 1441787904 + }, + { + "epoch": 4.08, + "learning_rate": 0.00028445336008024074, + "loss": 2.7933, + "theoretical_loss": 3.5284196024771592, + "tokens_seen": 1441787904 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002844433299899699, + "loss": 2.87, + "theoretical_loss": 3.5284053543996956, + "tokens_seen": 1441853440 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002844332998996991, + "loss": 2.5991, + "theoretical_loss": 3.528391107151152, + "tokens_seen": 1441918976 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002844232698094283, + "loss": 2.6916, + "theoretical_loss": 3.5283768607314414, + "tokens_seen": 1441984512 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002844132397191575, + "loss": 2.7467, + "theoretical_loss": 3.5283626151404794, + "tokens_seen": 1442050048 + }, + { + "epoch": 4.08, + "learning_rate": 0.00028440320962888664, + "loss": 2.6921, + "theoretical_loss": 3.528348370378179, + "tokens_seen": 1442115584 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002843931795386159, + "loss": 2.8085, + "theoretical_loss": 3.528334126444455, + "tokens_seen": 1442181120 + }, + { + "epoch": 4.08, + "learning_rate": 0.00028438314944834506, + "loss": 2.7245, + "theoretical_loss": 3.528319883339221, + "tokens_seen": 1442246656 + }, + { + "epoch": 4.08, + "learning_rate": 0.00028437311935807424, + "loss": 2.7377, + "theoretical_loss": 3.528305641062391, + "tokens_seen": 1442312192 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002843630892678034, + "loss": 2.6306, + "theoretical_loss": 3.5282913996138805, + "tokens_seen": 1442377728 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002843530591775326, + "loss": 2.6701, + "theoretical_loss": 3.5282771589936024, + "tokens_seen": 1442443264 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002843430290872618, + "loss": 2.6255, + "theoretical_loss": 3.528262919201471, + "tokens_seen": 1442508800 + }, + { + "epoch": 4.08, + "learning_rate": 0.000284332998996991, + "loss": 2.5292, + "theoretical_loss": 3.528248680237401, + "tokens_seen": 1442574336 + }, + { + "epoch": 4.08, + "learning_rate": 0.00028432296890672015, + "loss": 2.5836, + "theoretical_loss": 3.528234442101307, + "tokens_seen": 1442639872 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002843129388164494, + "loss": 2.5171, + "theoretical_loss": 3.5282202047931017, + "tokens_seen": 1442705408 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002843029087261785, + "loss": 2.6853, + "theoretical_loss": 3.5282059683127014, + "tokens_seen": 1442770944 + }, + { + "epoch": 4.08, + "learning_rate": 0.00028429287863590775, + "loss": 2.6028, + "theoretical_loss": 3.528191732660019, + "tokens_seen": 1442836480 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002842828485456369, + "loss": 2.6122, + "theoretical_loss": 3.528177497834969, + "tokens_seen": 1442902016 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002842728184553661, + "loss": 2.7187, + "theoretical_loss": 3.528163263837466, + "tokens_seen": 1442967552 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002842627883650953, + "loss": 2.6089, + "theoretical_loss": 3.5281490306674237, + "tokens_seen": 1443033088 + }, + { + "epoch": 4.08, + "learning_rate": 0.00028425275827482447, + "loss": 2.5862, + "theoretical_loss": 3.528134798324757, + "tokens_seen": 1443098624 + }, + { + "epoch": 4.08, + "learning_rate": 0.00028424272818455365, + "loss": 2.5758, + "theoretical_loss": 3.5281205668093802, + "tokens_seen": 1443164160 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002842326980942829, + "loss": 2.8418, + "theoretical_loss": 3.5281063361212075, + "tokens_seen": 1443229696 + }, + { + "epoch": 4.08, + "learning_rate": 0.000284222668004012, + "loss": 2.6395, + "theoretical_loss": 3.5280921062601536, + "tokens_seen": 1443295232 + }, + { + "epoch": 4.08, + "learning_rate": 0.00028421263791374125, + "loss": 2.7947, + "theoretical_loss": 3.5280778772261314, + "tokens_seen": 1443360768 + }, + { + "epoch": 4.08, + "objective/train/docs_used": 1588273, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.190006971359253, + "objective/train/theoretical_loss": 3.5280636490190576, + "objective/train/tokens_used": 1463886304, + "theoretical_loss": 3.5280636490190576, + "tokens_seen": 1443426304 + }, + { + "epoch": 4.08, + "learning_rate": 0.00028420260782347043, + "loss": 2.4971, + "theoretical_loss": 3.5280636490190576, + "tokens_seen": 1443426304 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002841925777331996, + "loss": 2.6572, + "theoretical_loss": 3.528049421638845, + "tokens_seen": 1443491840 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002841825476429288, + "loss": 2.7693, + "theoretical_loss": 3.5280351950854087, + "tokens_seen": 1443557376 + }, + { + "epoch": 4.08, + "learning_rate": 0.000284172517552658, + "loss": 2.6632, + "theoretical_loss": 3.5280209693586624, + "tokens_seen": 1443622912 + }, + { + "epoch": 4.08, + "learning_rate": 0.00028416248746238715, + "loss": 2.5126, + "theoretical_loss": 3.528006744458522, + "tokens_seen": 1443688448 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002841524573721164, + "loss": 2.466, + "theoretical_loss": 3.5279925203849, + "tokens_seen": 1443753984 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002841424272818455, + "loss": 2.7694, + "theoretical_loss": 3.527978297137712, + "tokens_seen": 1443819520 + }, + { + "epoch": 4.08, + "learning_rate": 0.00028413239719157475, + "loss": 2.7568, + "theoretical_loss": 3.5279640747168726, + "tokens_seen": 1443885056 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002841223671013039, + "loss": 2.8102, + "theoretical_loss": 3.5279498531222955, + "tokens_seen": 1443950592 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002841123370110331, + "loss": 2.8483, + "theoretical_loss": 3.5279356323538957, + "tokens_seen": 1444016128 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002841023069207623, + "loss": 2.6838, + "theoretical_loss": 3.527921412411588, + "tokens_seen": 1444081664 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002840922768304915, + "loss": 2.4802, + "theoretical_loss": 3.527907193295287, + "tokens_seen": 1444147200 + }, + { + "epoch": 4.08, + "learning_rate": 0.00028408224674022066, + "loss": 2.6184, + "theoretical_loss": 3.5278929750049057, + "tokens_seen": 1444212736 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002840722166499499, + "loss": 2.8073, + "theoretical_loss": 3.527878757540361, + "tokens_seen": 1444278272 + }, + { + "epoch": 4.08, + "learning_rate": 0.000284062186559679, + "loss": 2.8705, + "theoretical_loss": 3.5278645409015654, + "tokens_seen": 1444343808 + }, + { + "epoch": 4.08, + "learning_rate": 0.00028405215646940825, + "loss": 2.7107, + "theoretical_loss": 3.527850325088435, + "tokens_seen": 1444409344 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002840421263791374, + "loss": 2.6718, + "theoretical_loss": 3.5278361101008833, + "tokens_seen": 1444474880 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002840320962888666, + "loss": 2.7017, + "theoretical_loss": 3.5278218959388257, + "tokens_seen": 1444540416 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002840220661985958, + "loss": 2.7548, + "theoretical_loss": 3.527807682602176, + "tokens_seen": 1444605952 + }, + { + "epoch": 4.08, + "learning_rate": 0.000284012036108325, + "loss": 2.4647, + "theoretical_loss": 3.5277934700908498, + "tokens_seen": 1444671488 + }, + { + "epoch": 4.08, + "learning_rate": 0.00028400200601805416, + "loss": 2.52, + "theoretical_loss": 3.5277792584047605, + "tokens_seen": 1444737024 + }, + { + "epoch": 4.08, + "learning_rate": 0.00028399197592778334, + "loss": 2.7247, + "theoretical_loss": 3.5277650475438236, + "tokens_seen": 1444802560 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002839819458375125, + "loss": 2.6323, + "theoretical_loss": 3.5277508375079543, + "tokens_seen": 1444868096 + }, + { + "epoch": 4.08, + "learning_rate": 0.00028397191574724176, + "loss": 2.763, + "theoretical_loss": 3.527736628297066, + "tokens_seen": 1444933632 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002839618856569709, + "loss": 2.557, + "theoretical_loss": 3.5277224199110746, + "tokens_seen": 1444999168 + }, + { + "epoch": 4.08, + "objective/train/docs_used": 1589516, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.031261682510376, + "objective/train/theoretical_loss": 3.5277082123498937, + "objective/train/tokens_used": 1465524704, + "theoretical_loss": 3.5277082123498937, + "tokens_seen": 1445064704 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002839518555667001, + "loss": 2.8223, + "theoretical_loss": 3.5277082123498937, + "tokens_seen": 1445064704 + }, + { + "epoch": 4.08, + "learning_rate": 0.00028394182547642925, + "loss": 2.7678, + "theoretical_loss": 3.5276940056134385, + "tokens_seen": 1445130240 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002839317953861585, + "loss": 2.5075, + "theoretical_loss": 3.527679799701624, + "tokens_seen": 1445195776 + }, + { + "epoch": 4.08, + "learning_rate": 0.00028392176529588766, + "loss": 2.5564, + "theoretical_loss": 3.5276655946143642, + "tokens_seen": 1445261312 + }, + { + "epoch": 4.08, + "learning_rate": 0.00028391173520561684, + "loss": 2.7625, + "theoretical_loss": 3.5276513903515747, + "tokens_seen": 1445326848 + }, + { + "epoch": 4.08, + "learning_rate": 0.000283901705115346, + "loss": 2.6866, + "theoretical_loss": 3.5276371869131697, + "tokens_seen": 1445392384 + }, + { + "epoch": 4.08, + "learning_rate": 0.00028389167502507526, + "loss": 2.5275, + "theoretical_loss": 3.5276229842990645, + "tokens_seen": 1445457920 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002838816449348044, + "loss": 2.4997, + "theoretical_loss": 3.5276087825091738, + "tokens_seen": 1445523456 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002838716148445336, + "loss": 2.8864, + "theoretical_loss": 3.5275945815434113, + "tokens_seen": 1445588992 + }, + { + "epoch": 4.08, + "learning_rate": 0.00028386158475426275, + "loss": 2.5567, + "theoretical_loss": 3.5275803814016937, + "tokens_seen": 1445654528 + }, + { + "epoch": 4.08, + "learning_rate": 0.000283851554663992, + "loss": 2.5588, + "theoretical_loss": 3.527566182083934, + "tokens_seen": 1445720064 + }, + { + "epoch": 4.08, + "learning_rate": 0.00028384152457372117, + "loss": 2.6456, + "theoretical_loss": 3.5275519835900484, + "tokens_seen": 1445785600 + }, + { + "epoch": 4.08, + "learning_rate": 0.00028383149448345035, + "loss": 2.6197, + "theoretical_loss": 3.527537785919951, + "tokens_seen": 1445851136 + }, + { + "epoch": 4.08, + "learning_rate": 0.00028382146439317953, + "loss": 2.7781, + "theoretical_loss": 3.527523589073557, + "tokens_seen": 1445916672 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002838114343029087, + "loss": 2.4368, + "theoretical_loss": 3.527509393050781, + "tokens_seen": 1445982208 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002838014042126379, + "loss": 2.6069, + "theoretical_loss": 3.527495197851538, + "tokens_seen": 1446047744 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002837913741223671, + "loss": 2.5526, + "theoretical_loss": 3.527481003475743, + "tokens_seen": 1446113280 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002837813440320963, + "loss": 2.7014, + "theoretical_loss": 3.527466809923311, + "tokens_seen": 1446178816 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002837713139418255, + "loss": 2.5824, + "theoretical_loss": 3.5274526171941574, + "tokens_seen": 1446244352 + }, + { + "epoch": 4.08, + "learning_rate": 0.00028376128385155467, + "loss": 2.5125, + "theoretical_loss": 3.527438425288196, + "tokens_seen": 1446309888 + }, + { + "epoch": 4.08, + "learning_rate": 0.00028375125376128385, + "loss": 2.6314, + "theoretical_loss": 3.527424234205342, + "tokens_seen": 1446375424 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002837412236710131, + "loss": 2.3199, + "theoretical_loss": 3.527410043945511, + "tokens_seen": 1446440960 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002837311935807422, + "loss": 2.8419, + "theoretical_loss": 3.527395854508618, + "tokens_seen": 1446506496 + }, + { + "epoch": 4.08, + "learning_rate": 0.00028372116349047145, + "loss": 2.8332, + "theoretical_loss": 3.527381665894577, + "tokens_seen": 1446572032 + }, + { + "epoch": 4.08, + "learning_rate": 0.00028371113340020063, + "loss": 2.5837, + "theoretical_loss": 3.5273674781033044, + "tokens_seen": 1446637568 + }, + { + "epoch": 4.08, + "objective/train/docs_used": 1590115, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.46427321434021, + "objective/train/theoretical_loss": 3.5273532911347143, + "objective/train/tokens_used": 1467163104, + "theoretical_loss": 3.5273532911347143, + "tokens_seen": 1446703104 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002837011033099298, + "loss": 2.628, + "theoretical_loss": 3.5273532911347143, + "tokens_seen": 1446703104 + }, + { + "epoch": 4.08, + "learning_rate": 0.000283691073219659, + "loss": 2.4291, + "theoretical_loss": 3.5273391049887213, + "tokens_seen": 1446768640 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002836810431293882, + "loss": 2.6043, + "theoretical_loss": 3.527324919665242, + "tokens_seen": 1446834176 + }, + { + "epoch": 4.08, + "learning_rate": 0.00028367101303911735, + "loss": 2.4816, + "theoretical_loss": 3.52731073516419, + "tokens_seen": 1446899712 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002836609829488466, + "loss": 2.534, + "theoretical_loss": 3.527296551485481, + "tokens_seen": 1446965248 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002836509528585757, + "loss": 2.5662, + "theoretical_loss": 3.5272823686290296, + "tokens_seen": 1447030784 + }, + { + "epoch": 4.08, + "learning_rate": 0.00028364092276830495, + "loss": 2.6041, + "theoretical_loss": 3.5272681865947515, + "tokens_seen": 1447096320 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002836308926780341, + "loss": 2.6108, + "theoretical_loss": 3.527254005382562, + "tokens_seen": 1447161856 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002836208625877633, + "loss": 2.2632, + "theoretical_loss": 3.5272398249923755, + "tokens_seen": 1447227392 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002836108324974925, + "loss": 2.6465, + "theoretical_loss": 3.5272256454241075, + "tokens_seen": 1447292928 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002836008024072217, + "loss": 2.7067, + "theoretical_loss": 3.527211466677673, + "tokens_seen": 1447358464 + }, + { + "epoch": 4.08, + "learning_rate": 0.00028359077231695086, + "loss": 2.3787, + "theoretical_loss": 3.527197288752987, + "tokens_seen": 1447424000 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002835807422266801, + "loss": 2.5942, + "theoretical_loss": 3.527183111649965, + "tokens_seen": 1447489536 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002835707121364092, + "loss": 2.8501, + "theoretical_loss": 3.5271689353685223, + "tokens_seen": 1447555072 + }, + { + "epoch": 4.08, + "learning_rate": 0.00028356068204613846, + "loss": 2.8219, + "theoretical_loss": 3.5271547599085737, + "tokens_seen": 1447620608 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002835506519558676, + "loss": 2.7338, + "theoretical_loss": 3.527140585270035, + "tokens_seen": 1447686144 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002835406218655968, + "loss": 2.8349, + "theoretical_loss": 3.5271264114528202, + "tokens_seen": 1447751680 + }, + { + "epoch": 4.08, + "learning_rate": 0.000283530591775326, + "loss": 2.5583, + "theoretical_loss": 3.527112238456846, + "tokens_seen": 1447817216 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002835205616850552, + "loss": 2.5847, + "theoretical_loss": 3.527098066282027, + "tokens_seen": 1447882752 + }, + { + "epoch": 4.08, + "learning_rate": 0.00028351053159478436, + "loss": 2.5197, + "theoretical_loss": 3.5270838949282783, + "tokens_seen": 1447948288 + }, + { + "epoch": 4.08, + "learning_rate": 0.00028350050150451354, + "loss": 2.5117, + "theoretical_loss": 3.5270697243955156, + "tokens_seen": 1448013824 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002834904714142427, + "loss": 2.6788, + "theoretical_loss": 3.5270555546836535, + "tokens_seen": 1448079360 + }, + { + "epoch": 4.08, + "learning_rate": 0.00028348044132397196, + "loss": 2.8404, + "theoretical_loss": 3.527041385792608, + "tokens_seen": 1448144896 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002834704112337011, + "loss": 2.6377, + "theoretical_loss": 3.527027217722294, + "tokens_seen": 1448210432 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002834603811434303, + "loss": 2.5585, + "theoretical_loss": 3.527013050472627, + "tokens_seen": 1448275968 + }, + { + "epoch": 4.08, + "objective/train/docs_used": 1591064, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.98321795463562, + "objective/train/theoretical_loss": 3.5269988840435222, + "objective/train/tokens_used": 1468801504, + "theoretical_loss": 3.5269988840435222, + "tokens_seen": 1448341504 + }, + { + "epoch": 4.08, + "learning_rate": 0.00028345035105315945, + "loss": 2.9032, + "theoretical_loss": 3.5269988840435222, + "tokens_seen": 1448341504 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002834403209628887, + "loss": 2.719, + "theoretical_loss": 3.5269847184348952, + "tokens_seen": 1448407040 + }, + { + "epoch": 4.08, + "learning_rate": 0.00028343029087261786, + "loss": 2.6729, + "theoretical_loss": 3.526970553646661, + "tokens_seen": 1448472576 + }, + { + "epoch": 4.08, + "learning_rate": 0.00028342026078234705, + "loss": 2.483, + "theoretical_loss": 3.526956389678735, + "tokens_seen": 1448538112 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002834102306920762, + "loss": 2.5919, + "theoretical_loss": 3.526942226531033, + "tokens_seen": 1448603648 + }, + { + "epoch": 4.08, + "learning_rate": 0.00028340020060180546, + "loss": 2.6517, + "theoretical_loss": 3.52692806420347, + "tokens_seen": 1448669184 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002833901705115346, + "loss": 2.5691, + "theoretical_loss": 3.526913902695962, + "tokens_seen": 1448734720 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002833801404212638, + "loss": 2.924, + "theoretical_loss": 3.526899742008423, + "tokens_seen": 1448800256 + }, + { + "epoch": 4.08, + "learning_rate": 0.00028337011033099295, + "loss": 2.8697, + "theoretical_loss": 3.5268855821407703, + "tokens_seen": 1448865792 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002833600802407222, + "loss": 2.475, + "theoretical_loss": 3.526871423092918, + "tokens_seen": 1448931328 + }, + { + "epoch": 4.08, + "learning_rate": 0.00028335005015045137, + "loss": 2.416, + "theoretical_loss": 3.5268572648647822, + "tokens_seen": 1448996864 + }, + { + "epoch": 4.08, + "learning_rate": 0.00028334002006018055, + "loss": 2.7226, + "theoretical_loss": 3.526843107456278, + "tokens_seen": 1449062400 + }, + { + "epoch": 4.08, + "learning_rate": 0.00028332998996990973, + "loss": 2.537, + "theoretical_loss": 3.526828950867321, + "tokens_seen": 1449127936 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002833199598796389, + "loss": 2.5673, + "theoretical_loss": 3.526814795097827, + "tokens_seen": 1449193472 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002833099297893681, + "loss": 2.6473, + "theoretical_loss": 3.5268006401477114, + "tokens_seen": 1449259008 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002832998996990973, + "loss": 2.5081, + "theoretical_loss": 3.5267864860168894, + "tokens_seen": 1449324544 + }, + { + "epoch": 4.08, + "learning_rate": 0.00028328986960882645, + "loss": 2.6657, + "theoretical_loss": 3.526772332705277, + "tokens_seen": 1449390080 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002832798395185557, + "loss": 2.6562, + "theoretical_loss": 3.5267581802127888, + "tokens_seen": 1449455616 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002832698094282848, + "loss": 2.634, + "theoretical_loss": 3.5267440285393414, + "tokens_seen": 1449521152 + }, + { + "epoch": 4.08, + "learning_rate": 0.00028325977933801405, + "loss": 2.645, + "theoretical_loss": 3.52672987768485, + "tokens_seen": 1449586688 + }, + { + "epoch": 4.08, + "learning_rate": 0.00028324974924774323, + "loss": 2.623, + "theoretical_loss": 3.5267157276492305, + "tokens_seen": 1449652224 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002832397191574724, + "loss": 2.714, + "theoretical_loss": 3.526701578432398, + "tokens_seen": 1449717760 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002832296890672016, + "loss": 2.6843, + "theoretical_loss": 3.5266874300342685, + "tokens_seen": 1449783296 + }, + { + "epoch": 4.08, + "learning_rate": 0.00028321965897693083, + "loss": 2.6982, + "theoretical_loss": 3.526673282454757, + "tokens_seen": 1449848832 + }, + { + "epoch": 4.08, + "learning_rate": 0.00028320962888665996, + "loss": 2.6372, + "theoretical_loss": 3.5266591356937793, + "tokens_seen": 1449914368 + }, + { + "epoch": 4.08, + "objective/train/docs_used": 1591697, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.471153736114502, + "objective/train/theoretical_loss": 3.5266449897512517, + "objective/train/tokens_used": 1470439904, + "theoretical_loss": 3.5266449897512517, + "tokens_seen": 1449979904 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002831995987963892, + "loss": 2.5672, + "theoretical_loss": 3.5266449897512517, + "tokens_seen": 1449979904 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002831895687061183, + "loss": 2.8299, + "theoretical_loss": 3.52663084462709, + "tokens_seen": 1450045440 + }, + { + "epoch": 4.08, + "learning_rate": 0.00028317953861584755, + "loss": 2.8545, + "theoretical_loss": 3.5266167003212088, + "tokens_seen": 1450110976 + }, + { + "epoch": 4.08, + "learning_rate": 0.00028316950852557674, + "loss": 2.716, + "theoretical_loss": 3.5266025568335246, + "tokens_seen": 1450176512 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002831594784353059, + "loss": 2.8401, + "theoretical_loss": 3.5265884141639523, + "tokens_seen": 1450242048 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002831494483450351, + "loss": 2.4752, + "theoretical_loss": 3.5265742723124087, + "tokens_seen": 1450307584 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002831394182547643, + "loss": 2.6708, + "theoretical_loss": 3.5265601312788086, + "tokens_seen": 1450373120 + }, + { + "epoch": 4.08, + "learning_rate": 0.00028312938816449346, + "loss": 2.5739, + "theoretical_loss": 3.5265459910630685, + "tokens_seen": 1450438656 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002831193580742227, + "loss": 2.3183, + "theoretical_loss": 3.526531851665103, + "tokens_seen": 1450504192 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002831093279839518, + "loss": 2.6056, + "theoretical_loss": 3.5265177130848295, + "tokens_seen": 1450569728 + }, + { + "epoch": 4.08, + "learning_rate": 0.00028309929789368106, + "loss": 2.3986, + "theoretical_loss": 3.5265035753221623, + "tokens_seen": 1450635264 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002830892678034102, + "loss": 2.254, + "theoretical_loss": 3.526489438377018, + "tokens_seen": 1450700800 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002830792377131394, + "loss": 2.3774, + "theoretical_loss": 3.5264753022493123, + "tokens_seen": 1450766336 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002830692076228686, + "loss": 2.6581, + "theoretical_loss": 3.5264611669389603, + "tokens_seen": 1450831872 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002830591775325978, + "loss": 2.7144, + "theoretical_loss": 3.5264470324458785, + "tokens_seen": 1450897408 + }, + { + "epoch": 4.08, + "learning_rate": 0.00028304914744232696, + "loss": 2.722, + "theoretical_loss": 3.5264328987699827, + "tokens_seen": 1450962944 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002830391173520562, + "loss": 2.7777, + "theoretical_loss": 3.526418765911189, + "tokens_seen": 1451028480 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002830290872617854, + "loss": 2.5871, + "theoretical_loss": 3.526404633869413, + "tokens_seen": 1451094016 + }, + { + "epoch": 4.08, + "learning_rate": 0.00028301905717151456, + "loss": 2.676, + "theoretical_loss": 3.5263905026445705, + "tokens_seen": 1451159552 + }, + { + "epoch": 4.08, + "learning_rate": 0.00028300902708124374, + "loss": 2.4169, + "theoretical_loss": 3.5263763722365766, + "tokens_seen": 1451225088 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002829989969909729, + "loss": 2.6727, + "theoretical_loss": 3.526362242645349, + "tokens_seen": 1451290624 + }, + { + "epoch": 4.08, + "learning_rate": 0.00028298896690070216, + "loss": 2.5742, + "theoretical_loss": 3.526348113870802, + "tokens_seen": 1451356160 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002829789368104313, + "loss": 2.6444, + "theoretical_loss": 3.526333985912852, + "tokens_seen": 1451421696 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002829689067201605, + "loss": 2.5713, + "theoretical_loss": 3.5263198587714157, + "tokens_seen": 1451487232 + }, + { + "epoch": 4.08, + "learning_rate": 0.00028295887662988965, + "loss": 2.6759, + "theoretical_loss": 3.5263057324464078, + "tokens_seen": 1451552768 + }, + { + "epoch": 4.08, + "objective/train/docs_used": 1592735, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.422314405441284, + "objective/train/theoretical_loss": 3.526291606937745, + "objective/train/tokens_used": 1472078304, + "theoretical_loss": 3.526291606937745, + "tokens_seen": 1451618304 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002829488465396189, + "loss": 2.6869, + "theoretical_loss": 3.526291606937745, + "tokens_seen": 1451618304 + }, + { + "epoch": 4.08, + "learning_rate": 0.00028293881644934806, + "loss": 2.4801, + "theoretical_loss": 3.5262774822453435, + "tokens_seen": 1451683840 + }, + { + "epoch": 4.08, + "learning_rate": 0.00028292878635907725, + "loss": 2.5524, + "theoretical_loss": 3.5262633583691185, + "tokens_seen": 1451749376 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002829187562688064, + "loss": 2.6287, + "theoretical_loss": 3.5262492353089865, + "tokens_seen": 1451814912 + }, + { + "epoch": 4.08, + "learning_rate": 0.00028290872617853566, + "loss": 2.9081, + "theoretical_loss": 3.5262351130648635, + "tokens_seen": 1451880448 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002828986960882648, + "loss": 2.4654, + "theoretical_loss": 3.5262209916366656, + "tokens_seen": 1451945984 + }, + { + "epoch": 4.08, + "learning_rate": 0.000282888665997994, + "loss": 2.6926, + "theoretical_loss": 3.5262068710243084, + "tokens_seen": 1452011520 + }, + { + "epoch": 4.08, + "learning_rate": 0.00028287863590772315, + "loss": 2.866, + "theoretical_loss": 3.5261927512277085, + "tokens_seen": 1452077056 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002828686058174524, + "loss": 2.7792, + "theoretical_loss": 3.5261786322467814, + "tokens_seen": 1452142592 + }, + { + "epoch": 4.08, + "learning_rate": 0.00028285857572718157, + "loss": 2.631, + "theoretical_loss": 3.5261645140814437, + "tokens_seen": 1452208128 + }, + { + "epoch": 4.08, + "learning_rate": 0.00028284854563691075, + "loss": 2.7531, + "theoretical_loss": 3.526150396731611, + "tokens_seen": 1452273664 + }, + { + "epoch": 4.08, + "learning_rate": 0.00028283851554663993, + "loss": 2.6413, + "theoretical_loss": 3.5261362801972, + "tokens_seen": 1452339200 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002828284854563691, + "loss": 2.46, + "theoretical_loss": 3.526122164478126, + "tokens_seen": 1452404736 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002828184553660983, + "loss": 2.7587, + "theoretical_loss": 3.526108049574306, + "tokens_seen": 1452470272 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002828084252758275, + "loss": 2.7158, + "theoretical_loss": 3.5260939354856555, + "tokens_seen": 1452535808 + }, + { + "epoch": 4.08, + "learning_rate": 0.00028279839518555665, + "loss": 2.7085, + "theoretical_loss": 3.5260798222120906, + "tokens_seen": 1452601344 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002827883650952859, + "loss": 2.5908, + "theoretical_loss": 3.526065709753528, + "tokens_seen": 1452666880 + }, + { + "epoch": 4.08, + "learning_rate": 0.000282778335005015, + "loss": 2.9158, + "theoretical_loss": 3.5260515981098832, + "tokens_seen": 1452732416 + }, + { + "epoch": 4.08, + "learning_rate": 0.00028276830491474425, + "loss": 2.6908, + "theoretical_loss": 3.5260374872810734, + "tokens_seen": 1452797952 + }, + { + "epoch": 4.08, + "learning_rate": 0.00028275827482447343, + "loss": 2.6313, + "theoretical_loss": 3.5260233772670135, + "tokens_seen": 1452863488 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002827482447342026, + "loss": 2.5825, + "theoretical_loss": 3.5260092680676207, + "tokens_seen": 1452929024 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002827382146439318, + "loss": 2.6314, + "theoretical_loss": 3.525995159682811, + "tokens_seen": 1452994560 + }, + { + "epoch": 4.08, + "learning_rate": 0.00028272818455366103, + "loss": 2.7025, + "theoretical_loss": 3.525981052112501, + "tokens_seen": 1453060096 + }, + { + "epoch": 4.08, + "learning_rate": 0.00028271815446339016, + "loss": 2.6858, + "theoretical_loss": 3.5259669453566054, + "tokens_seen": 1453125632 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002827081243731194, + "loss": 2.5014, + "theoretical_loss": 3.525952839415042, + "tokens_seen": 1453191168 + }, + { + "epoch": 4.08, + "objective/train/docs_used": 1593470, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.433061122894287, + "objective/train/theoretical_loss": 3.5259387342877266, + "objective/train/tokens_used": 1473716704, + "theoretical_loss": 3.5259387342877266, + "tokens_seen": 1453256704 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002826980942828485, + "loss": 2.6465, + "theoretical_loss": 3.5259387342877266, + "tokens_seen": 1453256704 + }, + { + "epoch": 4.08, + "learning_rate": 0.00028268806419257775, + "loss": 2.4855, + "theoretical_loss": 3.5259246299745755, + "tokens_seen": 1453322240 + }, + { + "epoch": 4.08, + "learning_rate": 0.00028267803410230694, + "loss": 2.6825, + "theoretical_loss": 3.525910526475505, + "tokens_seen": 1453387776 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002826680040120361, + "loss": 2.7234, + "theoretical_loss": 3.5258964237904316, + "tokens_seen": 1453453312 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002826579739217653, + "loss": 2.2644, + "theoretical_loss": 3.525882321919271, + "tokens_seen": 1453518848 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002826479438314945, + "loss": 2.8498, + "theoretical_loss": 3.5258682208619403, + "tokens_seen": 1453584384 + }, + { + "epoch": 4.08, + "learning_rate": 0.00028263791374122366, + "loss": 2.4475, + "theoretical_loss": 3.525854120618355, + "tokens_seen": 1453649920 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002826278836509529, + "loss": 2.6162, + "theoretical_loss": 3.525840021188432, + "tokens_seen": 1453715456 + }, + { + "epoch": 4.08, + "learning_rate": 0.000282617853560682, + "loss": 2.4738, + "theoretical_loss": 3.525825922572088, + "tokens_seen": 1453780992 + }, + { + "epoch": 4.08, + "learning_rate": 0.00028260782347041126, + "loss": 2.4866, + "theoretical_loss": 3.5258118247692387, + "tokens_seen": 1453846528 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002825977933801404, + "loss": 2.4781, + "theoretical_loss": 3.5257977277798007, + "tokens_seen": 1453912064 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002825877632898696, + "loss": 2.5941, + "theoretical_loss": 3.5257836316036903, + "tokens_seen": 1453977600 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002825777331995988, + "loss": 2.4183, + "theoretical_loss": 3.5257695362408246, + "tokens_seen": 1454043136 + }, + { + "epoch": 4.08, + "learning_rate": 0.000282567703109328, + "loss": 2.6699, + "theoretical_loss": 3.5257554416911194, + "tokens_seen": 1454108672 + }, + { + "epoch": 4.08, + "learning_rate": 0.00028255767301905716, + "loss": 2.4834, + "theoretical_loss": 3.525741347954491, + "tokens_seen": 1454174208 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002825476429287864, + "loss": 2.6883, + "theoretical_loss": 3.5257272550308563, + "tokens_seen": 1454239744 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002825376128385155, + "loss": 2.677, + "theoretical_loss": 3.5257131629201313, + "tokens_seen": 1454305280 + }, + { + "epoch": 4.08, + "learning_rate": 0.00028252758274824476, + "loss": 2.5145, + "theoretical_loss": 3.5256990716222334, + "tokens_seen": 1454370816 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002825175526579739, + "loss": 2.429, + "theoretical_loss": 3.5256849811370783, + "tokens_seen": 1454436352 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002825075225677031, + "loss": 2.6504, + "theoretical_loss": 3.5256708914645825, + "tokens_seen": 1454501888 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002824974924774323, + "loss": 2.5089, + "theoretical_loss": 3.5256568026046624, + "tokens_seen": 1454567424 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002824874623871615, + "loss": 2.8249, + "theoretical_loss": 3.525642714557235, + "tokens_seen": 1454632960 + }, + { + "epoch": 4.08, + "learning_rate": 0.00028247743229689067, + "loss": 2.4911, + "theoretical_loss": 3.525628627322217, + "tokens_seen": 1454698496 + }, + { + "epoch": 4.08, + "learning_rate": 0.00028246740220661985, + "loss": 2.5383, + "theoretical_loss": 3.5256145408995243, + "tokens_seen": 1454764032 + }, + { + "epoch": 4.08, + "learning_rate": 0.00028245737211634903, + "loss": 2.5971, + "theoretical_loss": 3.525600455289074, + "tokens_seen": 1454829568 + }, + { + "epoch": 4.08, + "objective/train/docs_used": 1594935, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5903680324554443, + "objective/train/theoretical_loss": 3.5255863704907826, + "objective/train/tokens_used": 1475355104, + "theoretical_loss": 3.5255863704907826, + "tokens_seen": 1454895104 + }, + { + "epoch": 4.08, + "learning_rate": 0.00028244734202607826, + "loss": 2.7843, + "theoretical_loss": 3.5255863704907826, + "tokens_seen": 1454895104 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002824373119358074, + "loss": 2.6224, + "theoretical_loss": 3.525572286504566, + "tokens_seen": 1454960640 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002824272818455366, + "loss": 2.5467, + "theoretical_loss": 3.525558203330342, + "tokens_seen": 1455026176 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002824172517552658, + "loss": 2.4688, + "theoretical_loss": 3.525544120968026, + "tokens_seen": 1455091712 + }, + { + "epoch": 4.08, + "learning_rate": 0.000282407221664995, + "loss": 2.5837, + "theoretical_loss": 3.5255300394175357, + "tokens_seen": 1455157248 + }, + { + "epoch": 4.08, + "learning_rate": 0.00028239719157472417, + "loss": 2.6213, + "theoretical_loss": 3.5255159586787865, + "tokens_seen": 1455222784 + }, + { + "epoch": 4.08, + "learning_rate": 0.00028238716148445335, + "loss": 2.6744, + "theoretical_loss": 3.5255018787516965, + "tokens_seen": 1455288320 + }, + { + "epoch": 4.08, + "learning_rate": 0.00028237713139418253, + "loss": 2.8926, + "theoretical_loss": 3.5254877996361813, + "tokens_seen": 1455353856 + }, + { + "epoch": 4.08, + "learning_rate": 0.00028236710130391177, + "loss": 2.6875, + "theoretical_loss": 3.5254737213321583, + "tokens_seen": 1455419392 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002823570712136409, + "loss": 2.7662, + "theoretical_loss": 3.525459643839543, + "tokens_seen": 1455484928 + }, + { + "epoch": 4.08, + "learning_rate": 0.00028234704112337013, + "loss": 2.8077, + "theoretical_loss": 3.525445567158254, + "tokens_seen": 1455550464 + }, + { + "epoch": 4.08, + "learning_rate": 0.00028233701103309926, + "loss": 2.5677, + "theoretical_loss": 3.5254314912882063, + "tokens_seen": 1455616000 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002823269809428285, + "loss": 2.7994, + "theoretical_loss": 3.5254174162293177, + "tokens_seen": 1455681536 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002823169508525577, + "loss": 2.6619, + "theoretical_loss": 3.5254033419815043, + "tokens_seen": 1455747072 + }, + { + "epoch": 4.08, + "learning_rate": 0.00028230692076228685, + "loss": 2.8364, + "theoretical_loss": 3.525389268544683, + "tokens_seen": 1455812608 + }, + { + "epoch": 4.08, + "learning_rate": 0.00028229689067201604, + "loss": 2.9088, + "theoretical_loss": 3.5253751959187705, + "tokens_seen": 1455878144 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002822868605817452, + "loss": 2.4792, + "theoretical_loss": 3.525361124103684, + "tokens_seen": 1455943680 + }, + { + "epoch": 4.08, + "learning_rate": 0.00028227683049147445, + "loss": 2.486, + "theoretical_loss": 3.5253470530993396, + "tokens_seen": 1456009216 + }, + { + "epoch": 4.08, + "learning_rate": 0.00028226680040120363, + "loss": 2.7525, + "theoretical_loss": 3.5253329829056543, + "tokens_seen": 1456074752 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002822567703109328, + "loss": 2.7045, + "theoretical_loss": 3.525318913522546, + "tokens_seen": 1456140288 + }, + { + "epoch": 4.08, + "learning_rate": 0.000282246740220662, + "loss": 2.6482, + "theoretical_loss": 3.52530484494993, + "tokens_seen": 1456205824 + }, + { + "epoch": 4.08, + "learning_rate": 0.00028223671013039123, + "loss": 2.884, + "theoretical_loss": 3.525290777187724, + "tokens_seen": 1456271360 + }, + { + "epoch": 4.08, + "learning_rate": 0.00028222668004012036, + "loss": 2.6727, + "theoretical_loss": 3.525276710235844, + "tokens_seen": 1456336896 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002822166499498496, + "loss": 2.4325, + "theoretical_loss": 3.5252626440942088, + "tokens_seen": 1456402432 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002822066198595787, + "loss": 2.681, + "theoretical_loss": 3.525248578762733, + "tokens_seen": 1456467968 + }, + { + "epoch": 4.08, + "objective/train/docs_used": 1595425, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.1949830055236816, + "objective/train/theoretical_loss": 3.525234514241334, + "objective/train/tokens_used": 1476993504, + "theoretical_loss": 3.525234514241334, + "tokens_seen": 1456533504 + }, + { + "epoch": 4.08, + "learning_rate": 0.00028219658976930795, + "loss": 2.3501, + "theoretical_loss": 3.525234514241334, + "tokens_seen": 1456533504 + }, + { + "epoch": 4.08, + "learning_rate": 0.00028218655967903714, + "loss": 2.496, + "theoretical_loss": 3.52522045052993, + "tokens_seen": 1456599040 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002821765295887663, + "loss": 2.7187, + "theoretical_loss": 3.5252063876284367, + "tokens_seen": 1456664576 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002821664994984955, + "loss": 2.5332, + "theoretical_loss": 3.5251923255367714, + "tokens_seen": 1456730112 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002821564694082247, + "loss": 2.4664, + "theoretical_loss": 3.5251782642548513, + "tokens_seen": 1456795648 + }, + { + "epoch": 4.08, + "learning_rate": 0.00028214643931795386, + "loss": 2.754, + "theoretical_loss": 3.5251642037825928, + "tokens_seen": 1456861184 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002821364092276831, + "loss": 2.6015, + "theoretical_loss": 3.525150144119913, + "tokens_seen": 1456926720 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002821263791374122, + "loss": 2.8958, + "theoretical_loss": 3.5251360852667286, + "tokens_seen": 1456992256 + }, + { + "epoch": 4.08, + "learning_rate": 0.00028211634904714146, + "loss": 2.6523, + "theoretical_loss": 3.5251220272229578, + "tokens_seen": 1457057792 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002821063189568706, + "loss": 2.6786, + "theoretical_loss": 3.5251079699885164, + "tokens_seen": 1457123328 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002820962888665998, + "loss": 2.7648, + "theoretical_loss": 3.525093913563322, + "tokens_seen": 1457188864 + }, + { + "epoch": 4.08, + "learning_rate": 0.000282086258776329, + "loss": 2.8179, + "theoretical_loss": 3.525079857947291, + "tokens_seen": 1457254400 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002820762286860582, + "loss": 2.5025, + "theoretical_loss": 3.5250658031403406, + "tokens_seen": 1457319936 + }, + { + "epoch": 4.08, + "learning_rate": 0.00028206619859578736, + "loss": 2.604, + "theoretical_loss": 3.5250517491423885, + "tokens_seen": 1457385472 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002820561685055166, + "loss": 2.5271, + "theoretical_loss": 3.5250376959533516, + "tokens_seen": 1457451008 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002820461384152457, + "loss": 2.5158, + "theoretical_loss": 3.5250236435731463, + "tokens_seen": 1457516544 + }, + { + "epoch": 4.08, + "learning_rate": 0.00028203610832497496, + "loss": 2.5655, + "theoretical_loss": 3.52500959200169, + "tokens_seen": 1457582080 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002820260782347041, + "loss": 2.622, + "theoretical_loss": 3.5249955412389005, + "tokens_seen": 1457647616 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002820160481444333, + "loss": 2.4582, + "theoretical_loss": 3.5249814912846933, + "tokens_seen": 1457713152 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002820060180541625, + "loss": 2.8293, + "theoretical_loss": 3.5249674421389874, + "tokens_seen": 1457778688 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002819959879638917, + "loss": 2.8033, + "theoretical_loss": 3.524953393801699, + "tokens_seen": 1457844224 + }, + { + "epoch": 4.08, + "learning_rate": 0.00028198595787362087, + "loss": 2.7273, + "theoretical_loss": 3.5249393462727445, + "tokens_seen": 1457909760 + }, + { + "epoch": 4.08, + "learning_rate": 0.00028197592778335005, + "loss": 2.7569, + "theoretical_loss": 3.5249252995520424, + "tokens_seen": 1457975296 + }, + { + "epoch": 4.08, + "learning_rate": 0.00028196589769307923, + "loss": 2.706, + "theoretical_loss": 3.5249112536395093, + "tokens_seen": 1458040832 + }, + { + "epoch": 4.08, + "learning_rate": 0.00028195586760280846, + "loss": 2.5933, + "theoretical_loss": 3.5248972085350623, + "tokens_seen": 1458106368 + }, + { + "epoch": 4.08, + "objective/train/docs_used": 1596012, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.374768018722534, + "objective/train/theoretical_loss": 3.5248831642386182, + "objective/train/tokens_used": 1478631904, + "theoretical_loss": 3.5248831642386182, + "tokens_seen": 1458171904 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002819458375125376, + "loss": 2.6555, + "theoretical_loss": 3.5248831642386182, + "tokens_seen": 1458171904 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002819358074222668, + "loss": 2.6308, + "theoretical_loss": 3.5248691207500955, + "tokens_seen": 1458237440 + }, + { + "epoch": 4.08, + "learning_rate": 0.000281925777331996, + "loss": 2.3286, + "theoretical_loss": 3.52485507806941, + "tokens_seen": 1458302976 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002819157472417252, + "loss": 2.4707, + "theoretical_loss": 3.52484103619648, + "tokens_seen": 1458368512 + }, + { + "epoch": 4.08, + "learning_rate": 0.00028190571715145437, + "loss": 2.646, + "theoretical_loss": 3.524826995131222, + "tokens_seen": 1458434048 + }, + { + "epoch": 4.08, + "learning_rate": 0.00028189568706118355, + "loss": 2.6066, + "theoretical_loss": 3.5248129548735534, + "tokens_seen": 1458499584 + }, + { + "epoch": 4.08, + "learning_rate": 0.00028188565697091273, + "loss": 2.616, + "theoretical_loss": 3.524798915423392, + "tokens_seen": 1458565120 + }, + { + "epoch": 4.08, + "learning_rate": 0.00028187562688064197, + "loss": 2.5804, + "theoretical_loss": 3.5247848767806538, + "tokens_seen": 1458630656 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002818655967903711, + "loss": 2.7479, + "theoretical_loss": 3.524770838945258, + "tokens_seen": 1458696192 + }, + { + "epoch": 4.08, + "learning_rate": 0.00028185556670010033, + "loss": 2.4494, + "theoretical_loss": 3.5247568019171203, + "tokens_seen": 1458761728 + }, + { + "epoch": 4.08, + "learning_rate": 0.00028184553660982946, + "loss": 2.7117, + "theoretical_loss": 3.5247427656961587, + "tokens_seen": 1458827264 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002818355065195587, + "loss": 2.6518, + "theoretical_loss": 3.5247287302822903, + "tokens_seen": 1458892800 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002818254764292879, + "loss": 2.5601, + "theoretical_loss": 3.5247146956754323, + "tokens_seen": 1458958336 + }, + { + "epoch": 4.08, + "learning_rate": 0.00028181544633901705, + "loss": 2.1958, + "theoretical_loss": 3.524700661875503, + "tokens_seen": 1459023872 + }, + { + "epoch": 4.08, + "learning_rate": 0.00028180541624874624, + "loss": 2.5015, + "theoretical_loss": 3.524686628882418, + "tokens_seen": 1459089408 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002817953861584754, + "loss": 2.5364, + "theoretical_loss": 3.524672596696097, + "tokens_seen": 1459154944 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002817853560682046, + "loss": 2.6715, + "theoretical_loss": 3.524658565316455, + "tokens_seen": 1459220480 + }, + { + "epoch": 4.08, + "learning_rate": 0.00028177532597793383, + "loss": 2.4276, + "theoretical_loss": 3.524644534743411, + "tokens_seen": 1459286016 + }, + { + "epoch": 4.08, + "learning_rate": 0.00028176529588766296, + "loss": 2.3955, + "theoretical_loss": 3.524630504976882, + "tokens_seen": 1459351552 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002817552657973922, + "loss": 2.4853, + "theoretical_loss": 3.524616476016785, + "tokens_seen": 1459417088 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002817452357071214, + "loss": 2.2738, + "theoretical_loss": 3.524602447863038, + "tokens_seen": 1459482624 + }, + { + "epoch": 4.08, + "learning_rate": 0.00028173520561685056, + "loss": 2.4406, + "theoretical_loss": 3.524588420515559, + "tokens_seen": 1459548160 + }, + { + "epoch": 4.08, + "learning_rate": 0.00028172517552657974, + "loss": 2.3551, + "theoretical_loss": 3.5245743939742633, + "tokens_seen": 1459613696 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002817151454363089, + "loss": 2.4789, + "theoretical_loss": 3.5245603682390705, + "tokens_seen": 1459679232 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002817051153460381, + "loss": 2.8512, + "theoretical_loss": 3.524546343309897, + "tokens_seen": 1459744768 + }, + { + "epoch": 4.08, + "objective/train/docs_used": 1597246, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.754894256591797, + "objective/train/theoretical_loss": 3.524532319186661, + "objective/train/tokens_used": 1480270304, + "theoretical_loss": 3.524532319186661, + "tokens_seen": 1459810304 + }, + { + "epoch": 4.08, + "learning_rate": 0.00028169508525576734, + "loss": 2.6619, + "theoretical_loss": 3.524532319186661, + "tokens_seen": 1459810304 + }, + { + "epoch": 4.08, + "learning_rate": 0.00028168505516549646, + "loss": 2.5364, + "theoretical_loss": 3.52451829586928, + "tokens_seen": 1459875840 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002816750250752257, + "loss": 2.5277, + "theoretical_loss": 3.52450427335767, + "tokens_seen": 1459941376 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002816649949849548, + "loss": 2.5471, + "theoretical_loss": 3.5244902516517507, + "tokens_seen": 1460006912 + }, + { + "epoch": 4.08, + "learning_rate": 0.00028165496489468406, + "loss": 2.6017, + "theoretical_loss": 3.5244762307514383, + "tokens_seen": 1460072448 + }, + { + "epoch": 4.08, + "learning_rate": 0.00028164493480441324, + "loss": 2.6548, + "theoretical_loss": 3.524462210656651, + "tokens_seen": 1460137984 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002816349047141424, + "loss": 2.3979, + "theoretical_loss": 3.5244481913673056, + "tokens_seen": 1460203520 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002816248746238716, + "loss": 2.5545, + "theoretical_loss": 3.5244341728833204, + "tokens_seen": 1460269056 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002816148445336008, + "loss": 2.6222, + "theoretical_loss": 3.524420155204613, + "tokens_seen": 1460334592 + }, + { + "epoch": 4.08, + "learning_rate": 0.00028160481444332997, + "loss": 2.2757, + "theoretical_loss": 3.5244061383311003, + "tokens_seen": 1460400128 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002815947843530592, + "loss": 2.5474, + "theoretical_loss": 3.5243921222627006, + "tokens_seen": 1460465664 + }, + { + "epoch": 4.08, + "learning_rate": 0.00028158475426278833, + "loss": 2.491, + "theoretical_loss": 3.524378106999331, + "tokens_seen": 1460531200 + }, + { + "epoch": 4.08, + "learning_rate": 0.00028157472417251756, + "loss": 2.7228, + "theoretical_loss": 3.5243640925409103, + "tokens_seen": 1460596736 + }, + { + "epoch": 4.08, + "learning_rate": 0.00028156469408224674, + "loss": 2.6113, + "theoretical_loss": 3.5243500788873545, + "tokens_seen": 1460662272 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002815546639919759, + "loss": 2.8053, + "theoretical_loss": 3.5243360660385825, + "tokens_seen": 1460727808 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002815446339017051, + "loss": 2.6721, + "theoretical_loss": 3.5243220539945113, + "tokens_seen": 1460793344 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002815346038114343, + "loss": 2.3533, + "theoretical_loss": 3.5243080427550586, + "tokens_seen": 1460858880 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002815245737211635, + "loss": 2.5673, + "theoretical_loss": 3.524294032320143, + "tokens_seen": 1460924416 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002815145436308927, + "loss": 2.4373, + "theoretical_loss": 3.5242800226896813, + "tokens_seen": 1460989952 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002815045135406219, + "loss": 2.5728, + "theoretical_loss": 3.524266013863591, + "tokens_seen": 1461055488 + }, + { + "epoch": 4.08, + "learning_rate": 0.00028149448345035107, + "loss": 2.708, + "theoretical_loss": 3.5242520058417908, + "tokens_seen": 1461121024 + }, + { + "epoch": 4.08, + "learning_rate": 0.00028148445336008025, + "loss": 2.3159, + "theoretical_loss": 3.524237998624198, + "tokens_seen": 1461186560 + }, + { + "epoch": 4.08, + "learning_rate": 0.00028147442326980943, + "loss": 2.5539, + "theoretical_loss": 3.5242239922107306, + "tokens_seen": 1461252096 + }, + { + "epoch": 4.08, + "learning_rate": 0.00028146439317953866, + "loss": 2.1944, + "theoretical_loss": 3.5242099866013055, + "tokens_seen": 1461317632 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002814543630892678, + "loss": 2.387, + "theoretical_loss": 3.5241959817958413, + "tokens_seen": 1461383168 + }, + { + "epoch": 4.08, + "objective/train/docs_used": 1598035, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9441568851470947, + "objective/train/theoretical_loss": 3.524181977794256, + "objective/train/tokens_used": 1481908704, + "theoretical_loss": 3.524181977794256, + "tokens_seen": 1461448704 + }, + { + "epoch": 4.08, + "learning_rate": 0.000281444332998997, + "loss": 2.757, + "theoretical_loss": 3.524181977794256, + "tokens_seen": 1461448704 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002814343029087262, + "loss": 2.4984, + "theoretical_loss": 3.5241679745964665, + "tokens_seen": 1461514240 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002814242728184554, + "loss": 2.6137, + "theoretical_loss": 3.524153972202391, + "tokens_seen": 1461579776 + }, + { + "epoch": 4.08, + "learning_rate": 0.00028141424272818457, + "loss": 2.5324, + "theoretical_loss": 3.524139970611948, + "tokens_seen": 1461645312 + }, + { + "epoch": 4.08, + "learning_rate": 0.00028140421263791375, + "loss": 2.6002, + "theoretical_loss": 3.5241259698250547, + "tokens_seen": 1461710848 + }, + { + "epoch": 4.08, + "learning_rate": 0.00028139418254764293, + "loss": 2.7401, + "theoretical_loss": 3.5241119698416288, + "tokens_seen": 1461776384 + }, + { + "epoch": 4.08, + "learning_rate": 0.00028138415245737217, + "loss": 2.7361, + "theoretical_loss": 3.5240979706615887, + "tokens_seen": 1461841920 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002813741223671013, + "loss": 2.6632, + "theoretical_loss": 3.5240839722848523, + "tokens_seen": 1461907456 + }, + { + "epoch": 4.08, + "learning_rate": 0.00028136409227683053, + "loss": 2.4013, + "theoretical_loss": 3.5240699747113364, + "tokens_seen": 1461972992 + }, + { + "epoch": 4.08, + "learning_rate": 0.00028135406218655966, + "loss": 2.5163, + "theoretical_loss": 3.524055977940961, + "tokens_seen": 1462038528 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002813440320962889, + "loss": 2.5318, + "theoretical_loss": 3.5240419819736415, + "tokens_seen": 1462104064 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002813340020060181, + "loss": 2.6787, + "theoretical_loss": 3.5240279868092976, + "tokens_seen": 1462169600 + }, + { + "epoch": 4.08, + "learning_rate": 0.00028132397191574725, + "loss": 2.6692, + "theoretical_loss": 3.524013992447847, + "tokens_seen": 1462235136 + }, + { + "epoch": 4.08, + "learning_rate": 0.00028131394182547644, + "loss": 2.4938, + "theoretical_loss": 3.5239999988892077, + "tokens_seen": 1462300672 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002813039117352056, + "loss": 2.5738, + "theoretical_loss": 3.5239860061332964, + "tokens_seen": 1462366208 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002812938816449348, + "loss": 2.5126, + "theoretical_loss": 3.523972014180033, + "tokens_seen": 1462431744 + }, + { + "epoch": 4.08, + "learning_rate": 0.00028128385155466403, + "loss": 2.4717, + "theoretical_loss": 3.5239580230293335, + "tokens_seen": 1462497280 + }, + { + "epoch": 4.08, + "learning_rate": 0.00028127382146439316, + "loss": 2.6733, + "theoretical_loss": 3.5239440326811176, + "tokens_seen": 1462562816 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002812637913741224, + "loss": 2.6682, + "theoretical_loss": 3.5239300431353024, + "tokens_seen": 1462628352 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002812537612838516, + "loss": 2.5902, + "theoretical_loss": 3.5239160543918064, + "tokens_seen": 1462693888 + }, + { + "epoch": 4.08, + "learning_rate": 0.00028124373119358076, + "loss": 2.415, + "theoretical_loss": 3.523902066450548, + "tokens_seen": 1462759424 + }, + { + "epoch": 4.08, + "learning_rate": 0.00028123370110330994, + "loss": 2.5721, + "theoretical_loss": 3.523888079311444, + "tokens_seen": 1462824960 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002812236710130391, + "loss": 2.4526, + "theoretical_loss": 3.5238740929744132, + "tokens_seen": 1462890496 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002812136409227683, + "loss": 2.51, + "theoretical_loss": 3.523860107439374, + "tokens_seen": 1462956032 + }, + { + "epoch": 4.08, + "learning_rate": 0.00028120361083249754, + "loss": 2.5591, + "theoretical_loss": 3.523846122706243, + "tokens_seen": 1463021568 + }, + { + "epoch": 4.08, + "objective/train/docs_used": 1603030, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7301270961761475, + "objective/train/theoretical_loss": 3.5238321387749405, + "objective/train/tokens_used": 1483547104, + "theoretical_loss": 3.5238321387749405, + "tokens_seen": 1463087104 + }, + { + "epoch": 4.08, + "learning_rate": 0.00028119358074222666, + "loss": 2.5086, + "theoretical_loss": 3.5238321387749405, + "tokens_seen": 1463087104 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002811835506519559, + "loss": 2.6898, + "theoretical_loss": 3.523818155645383, + "tokens_seen": 1463152640 + }, + { + "epoch": 4.08, + "learning_rate": 0.000281173520561685, + "loss": 2.3801, + "theoretical_loss": 3.5238041733174894, + "tokens_seen": 1463218176 + }, + { + "epoch": 4.08, + "learning_rate": 0.00028116349047141426, + "loss": 2.4418, + "theoretical_loss": 3.523790191791178, + "tokens_seen": 1463283712 + }, + { + "epoch": 4.08, + "learning_rate": 0.00028115346038114344, + "loss": 2.4984, + "theoretical_loss": 3.523776211066366, + "tokens_seen": 1463349248 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002811434302908726, + "loss": 2.51, + "theoretical_loss": 3.5237622311429724, + "tokens_seen": 1463414784 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002811334002006018, + "loss": 2.6372, + "theoretical_loss": 3.5237482520209147, + "tokens_seen": 1463480320 + }, + { + "epoch": 4.08, + "learning_rate": 0.000281123370110331, + "loss": 2.7331, + "theoretical_loss": 3.523734273700112, + "tokens_seen": 1463545856 + }, + { + "epoch": 4.08, + "learning_rate": 0.00028111334002006017, + "loss": 2.3877, + "theoretical_loss": 3.5237202961804814, + "tokens_seen": 1463611392 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002811033099297894, + "loss": 2.3393, + "theoretical_loss": 3.523706319461942, + "tokens_seen": 1463676928 + }, + { + "epoch": 4.08, + "learning_rate": 0.00028109327983951853, + "loss": 2.6241, + "theoretical_loss": 3.523692343544412, + "tokens_seen": 1463742464 + }, + { + "epoch": 4.08, + "learning_rate": 0.00028108324974924776, + "loss": 2.5906, + "theoretical_loss": 3.5236783684278086, + "tokens_seen": 1463808000 + }, + { + "epoch": 4.08, + "learning_rate": 0.00028107321965897695, + "loss": 2.1827, + "theoretical_loss": 3.5236643941120516, + "tokens_seen": 1463873536 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002810631895687061, + "loss": 2.8091, + "theoretical_loss": 3.5236504205970576, + "tokens_seen": 1463939072 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002810531594784353, + "loss": 2.621, + "theoretical_loss": 3.523636447882746, + "tokens_seen": 1464004608 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002810431293881645, + "loss": 2.3639, + "theoretical_loss": 3.5236224759690353, + "tokens_seen": 1464070144 + }, + { + "epoch": 4.08, + "learning_rate": 0.00028103309929789367, + "loss": 2.708, + "theoretical_loss": 3.5236085048558428, + "tokens_seen": 1464135680 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002810230692076229, + "loss": 2.3876, + "theoretical_loss": 3.5235945345430872, + "tokens_seen": 1464201216 + }, + { + "epoch": 4.08, + "learning_rate": 0.00028101303911735203, + "loss": 2.7517, + "theoretical_loss": 3.523580565030687, + "tokens_seen": 1464266752 + }, + { + "epoch": 4.08, + "learning_rate": 0.00028100300902708127, + "loss": 2.5516, + "theoretical_loss": 3.5235665963185605, + "tokens_seen": 1464332288 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002809929789368104, + "loss": 2.5998, + "theoretical_loss": 3.523552628406626, + "tokens_seen": 1464397824 + }, + { + "epoch": 4.08, + "learning_rate": 0.00028098294884653963, + "loss": 2.6177, + "theoretical_loss": 3.5235386612948014, + "tokens_seen": 1464463360 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002809729187562688, + "loss": 2.5494, + "theoretical_loss": 3.5235246949830064, + "tokens_seen": 1464528896 + }, + { + "epoch": 4.08, + "learning_rate": 0.000280962888665998, + "loss": 2.74, + "theoretical_loss": 3.5235107294711576, + "tokens_seen": 1464594432 + }, + { + "epoch": 4.08, + "learning_rate": 0.00028095285857572717, + "loss": 2.7292, + "theoretical_loss": 3.523496764759175, + "tokens_seen": 1464659968 + }, + { + "epoch": 4.08, + "objective/train/docs_used": 1603030, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.211653470993042, + "objective/train/theoretical_loss": 3.523482800846976, + "objective/train/tokens_used": 1483791840, + "theoretical_loss": 3.523482800846976, + "tokens_seen": 1464725504 + }, + { + "epoch": 4.08, + "learning_rate": 0.00028094282848545635, + "loss": 2.2415, + "theoretical_loss": 3.523482800846976, + "tokens_seen": 1464725504 + }, + { + "epoch": 4.08, + "learning_rate": 0.00028093279839518553, + "loss": 2.4344, + "theoretical_loss": 3.5234688377344785, + "tokens_seen": 1464791040 + }, + { + "epoch": 4.08, + "learning_rate": 0.00028092276830491477, + "loss": 2.6155, + "theoretical_loss": 3.5234548754216024, + "tokens_seen": 1464856576 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002809127382146439, + "loss": 2.6634, + "theoretical_loss": 3.5234409139082654, + "tokens_seen": 1464922112 + }, + { + "epoch": 4.08, + "learning_rate": 0.00028090270812437313, + "loss": 2.5183, + "theoretical_loss": 3.5234269531943863, + "tokens_seen": 1464987648 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002808926780341023, + "loss": 2.6951, + "theoretical_loss": 3.5234129932798828, + "tokens_seen": 1465053184 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002808826479438315, + "loss": 2.7359, + "theoretical_loss": 3.5233990341646737, + "tokens_seen": 1465118720 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002808726178535607, + "loss": 2.5248, + "theoretical_loss": 3.5233850758486778, + "tokens_seen": 1465184256 + }, + { + "epoch": 4.08, + "learning_rate": 0.00028086258776328986, + "loss": 2.6991, + "theoretical_loss": 3.5233711183318137, + "tokens_seen": 1465249792 + }, + { + "epoch": 4.08, + "learning_rate": 0.00028085255767301904, + "loss": 2.4007, + "theoretical_loss": 3.523357161613999, + "tokens_seen": 1465315328 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002808425275827483, + "loss": 2.5959, + "theoretical_loss": 3.523343205695153, + "tokens_seen": 1465380864 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002808324974924774, + "loss": 2.7473, + "theoretical_loss": 3.5233292505751943, + "tokens_seen": 1465446400 + }, + { + "epoch": 4.08, + "learning_rate": 0.00028082246740220664, + "loss": 2.5896, + "theoretical_loss": 3.523315296254041, + "tokens_seen": 1465511936 + }, + { + "epoch": 4.08, + "learning_rate": 0.00028081243731193576, + "loss": 2.8018, + "theoretical_loss": 3.523301342731612, + "tokens_seen": 1465577472 + }, + { + "epoch": 4.08, + "learning_rate": 0.000280802407221665, + "loss": 2.252, + "theoretical_loss": 3.523287390007826, + "tokens_seen": 1465643008 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002807923771313942, + "loss": 2.6345, + "theoretical_loss": 3.523273438082601, + "tokens_seen": 1465708544 + }, + { + "epoch": 4.08, + "learning_rate": 0.00028078234704112336, + "loss": 2.6385, + "theoretical_loss": 3.5232594869558556, + "tokens_seen": 1465774080 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002807723169508526, + "loss": 2.603, + "theoretical_loss": 3.5232455366275093, + "tokens_seen": 1465839616 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002807622868605818, + "loss": 2.5463, + "theoretical_loss": 3.52323158709748, + "tokens_seen": 1465905152 + }, + { + "epoch": 4.08, + "learning_rate": 0.00028075225677031096, + "loss": 2.6577, + "theoretical_loss": 3.5232176383656864, + "tokens_seen": 1465970688 + }, + { + "epoch": 4.08, + "learning_rate": 0.00028074222668004014, + "loss": 2.6319, + "theoretical_loss": 3.523203690432047, + "tokens_seen": 1466036224 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002807321965897693, + "loss": 2.1931, + "theoretical_loss": 3.5231897432964807, + "tokens_seen": 1466101760 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002807221664994985, + "loss": 2.6428, + "theoretical_loss": 3.523175796958906, + "tokens_seen": 1466167296 + }, + { + "epoch": 4.08, + "learning_rate": 0.00028071213640922774, + "loss": 2.5598, + "theoretical_loss": 3.5231618514192427, + "tokens_seen": 1466232832 + }, + { + "epoch": 4.08, + "learning_rate": 0.00028070210631895686, + "loss": 2.7383, + "theoretical_loss": 3.5231479066774076, + "tokens_seen": 1466298368 + }, + { + "epoch": 4.08, + "objective/train/docs_used": 1603030, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4567415714263916, + "objective/train/theoretical_loss": 3.5231339627333202, + "objective/train/tokens_used": 1483791840, + "theoretical_loss": 3.5231339627333202, + "tokens_seen": 1466363904 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002806920762286861, + "loss": 2.4262, + "theoretical_loss": 3.5231339627333202, + "tokens_seen": 1466363904 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002806820461384152, + "loss": 2.5823, + "theoretical_loss": 3.5231200195868997, + "tokens_seen": 1466429440 + }, + { + "epoch": 4.08, + "learning_rate": 0.00028067201604814446, + "loss": 2.5947, + "theoretical_loss": 3.523106077238064, + "tokens_seen": 1466494976 + }, + { + "epoch": 4.08, + "learning_rate": 0.00028066198595787364, + "loss": 2.4495, + "theoretical_loss": 3.523092135686732, + "tokens_seen": 1466560512 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002806519558676028, + "loss": 2.3593, + "theoretical_loss": 3.5230781949328236, + "tokens_seen": 1466626048 + }, + { + "epoch": 4.08, + "learning_rate": 0.000280641925777332, + "loss": 2.6452, + "theoretical_loss": 3.523064254976256, + "tokens_seen": 1466691584 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002806318956870612, + "loss": 2.5566, + "theoretical_loss": 3.523050315816949, + "tokens_seen": 1466757120 + }, + { + "epoch": 4.08, + "learning_rate": 0.00028062186559679037, + "loss": 2.4694, + "theoretical_loss": 3.5230363774548206, + "tokens_seen": 1466822656 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002806118355065196, + "loss": 2.4054, + "theoretical_loss": 3.5230224398897905, + "tokens_seen": 1466888192 + }, + { + "epoch": 4.08, + "learning_rate": 0.00028060180541624873, + "loss": 2.8066, + "theoretical_loss": 3.5230085031217766, + "tokens_seen": 1466953728 + }, + { + "epoch": 4.08, + "learning_rate": 0.00028059177532597796, + "loss": 2.6842, + "theoretical_loss": 3.5229945671506986, + "tokens_seen": 1467019264 + }, + { + "epoch": 4.08, + "learning_rate": 0.00028058174523570715, + "loss": 2.6991, + "theoretical_loss": 3.5229806319764743, + "tokens_seen": 1467084800 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002805717151454363, + "loss": 2.5365, + "theoretical_loss": 3.5229666975990233, + "tokens_seen": 1467150336 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002805616850551655, + "loss": 2.4685, + "theoretical_loss": 3.5229527640182647, + "tokens_seen": 1467215872 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002805516549648947, + "loss": 2.8453, + "theoretical_loss": 3.522938831234116, + "tokens_seen": 1467281408 + }, + { + "epoch": 4.08, + "learning_rate": 0.00028054162487462387, + "loss": 2.7806, + "theoretical_loss": 3.522924899246498, + "tokens_seen": 1467346944 + }, + { + "epoch": 4.08, + "learning_rate": 0.0002805315947843531, + "loss": 2.5334, + "theoretical_loss": 3.5229109680553283, + "tokens_seen": 1467412480 + }, + { + "epoch": 4.08, + "learning_rate": 0.00028052156469408223, + "loss": 2.609, + "theoretical_loss": 3.5228970376605258, + "tokens_seen": 1467478016 + }, + { + "epoch": 4.08, + "learning_rate": 0.00028051153460381147, + "loss": 2.3968, + "theoretical_loss": 3.52288310806201, + "tokens_seen": 1467543552 + }, + { + "epoch": 4.09, + "learning_rate": 0.0002805015045135406, + "loss": 2.8006, + "theoretical_loss": 3.5228691792596996, + "tokens_seen": 1467609088 + }, + { + "epoch": 4.09, + "learning_rate": 0.00028049147442326983, + "loss": 2.5682, + "theoretical_loss": 3.5228552512535134, + "tokens_seen": 1467674624 + }, + { + "epoch": 4.09, + "learning_rate": 0.000280481444332999, + "loss": 2.7127, + "theoretical_loss": 3.5228413240433705, + "tokens_seen": 1467740160 + }, + { + "epoch": 4.09, + "learning_rate": 0.0002804714142427282, + "loss": 2.2954, + "theoretical_loss": 3.5228273976291895, + "tokens_seen": 1467805696 + }, + { + "epoch": 4.09, + "learning_rate": 0.00028046138415245737, + "loss": 2.8381, + "theoretical_loss": 3.52281347201089, + "tokens_seen": 1467871232 + }, + { + "epoch": 4.09, + "learning_rate": 0.00028045135406218655, + "loss": 2.4445, + "theoretical_loss": 3.5227995471883906, + "tokens_seen": 1467936768 + }, + { + "epoch": 4.09, + "objective/train/docs_used": 1603030, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5689315795898438, + "objective/train/theoretical_loss": 3.52278562316161, + "objective/train/tokens_used": 1483791840, + "theoretical_loss": 3.52278562316161, + "tokens_seen": 1468002304 + }, + { + "epoch": 4.09, + "learning_rate": 0.00028044132397191574, + "loss": 2.6411, + "theoretical_loss": 3.52278562316161, + "tokens_seen": 1468002304 + }, + { + "epoch": 4.09, + "learning_rate": 0.00028043129388164497, + "loss": 2.5643, + "theoretical_loss": 3.522771699930468, + "tokens_seen": 1468067840 + }, + { + "epoch": 4.09, + "learning_rate": 0.0002804212637913741, + "loss": 2.7135, + "theoretical_loss": 3.522757777494883, + "tokens_seen": 1468133376 + }, + { + "epoch": 4.09, + "learning_rate": 0.00028041123370110333, + "loss": 2.7565, + "theoretical_loss": 3.522743855854774, + "tokens_seen": 1468198912 + }, + { + "epoch": 4.09, + "learning_rate": 0.0002804012036108325, + "loss": 2.6198, + "theoretical_loss": 3.5227299350100605, + "tokens_seen": 1468264448 + }, + { + "epoch": 4.09, + "learning_rate": 0.0002803911735205617, + "loss": 2.6975, + "theoretical_loss": 3.522716014960661, + "tokens_seen": 1468329984 + }, + { + "epoch": 4.09, + "learning_rate": 0.0002803811434302909, + "loss": 2.4973, + "theoretical_loss": 3.522702095706496, + "tokens_seen": 1468395520 + }, + { + "epoch": 4.09, + "learning_rate": 0.00028037111334002006, + "loss": 2.5945, + "theoretical_loss": 3.522688177247482, + "tokens_seen": 1468461056 + }, + { + "epoch": 4.09, + "learning_rate": 0.00028036108324974924, + "loss": 2.6434, + "theoretical_loss": 3.5226742595835407, + "tokens_seen": 1468526592 + }, + { + "epoch": 4.09, + "learning_rate": 0.0002803510531594785, + "loss": 2.4662, + "theoretical_loss": 3.5226603427145893, + "tokens_seen": 1468592128 + }, + { + "epoch": 4.09, + "learning_rate": 0.0002803410230692076, + "loss": 2.6721, + "theoretical_loss": 3.5226464266405477, + "tokens_seen": 1468657664 + }, + { + "epoch": 4.09, + "learning_rate": 0.00028033099297893684, + "loss": 2.4547, + "theoretical_loss": 3.5226325113613353, + "tokens_seen": 1468723200 + }, + { + "epoch": 4.09, + "learning_rate": 0.00028032096288866596, + "loss": 2.518, + "theoretical_loss": 3.522618596876871, + "tokens_seen": 1468788736 + }, + { + "epoch": 4.09, + "learning_rate": 0.0002803109327983952, + "loss": 2.4428, + "theoretical_loss": 3.522604683187074, + "tokens_seen": 1468854272 + }, + { + "epoch": 4.09, + "learning_rate": 0.0002803009027081244, + "loss": 2.8113, + "theoretical_loss": 3.5225907702918633, + "tokens_seen": 1468919808 + }, + { + "epoch": 4.09, + "learning_rate": 0.00028029087261785356, + "loss": 2.336, + "theoretical_loss": 3.5225768581911576, + "tokens_seen": 1468985344 + }, + { + "epoch": 4.09, + "learning_rate": 0.00028028084252758274, + "loss": 2.7046, + "theoretical_loss": 3.522562946884877, + "tokens_seen": 1469050880 + }, + { + "epoch": 4.09, + "learning_rate": 0.000280270812437312, + "loss": 2.4133, + "theoretical_loss": 3.5225490363729404, + "tokens_seen": 1469116416 + }, + { + "epoch": 4.09, + "learning_rate": 0.0002802607823470411, + "loss": 2.4705, + "theoretical_loss": 3.522535126655267, + "tokens_seen": 1469181952 + }, + { + "epoch": 4.09, + "learning_rate": 0.00028025075225677034, + "loss": 2.4143, + "theoretical_loss": 3.522521217731776, + "tokens_seen": 1469247488 + }, + { + "epoch": 4.09, + "learning_rate": 0.00028024072216649947, + "loss": 2.7378, + "theoretical_loss": 3.5225073096023864, + "tokens_seen": 1469313024 + }, + { + "epoch": 4.09, + "learning_rate": 0.0002802306920762287, + "loss": 2.5421, + "theoretical_loss": 3.5224934022670173, + "tokens_seen": 1469378560 + }, + { + "epoch": 4.09, + "learning_rate": 0.0002802206619859579, + "loss": 2.4663, + "theoretical_loss": 3.5224794957255887, + "tokens_seen": 1469444096 + }, + { + "epoch": 4.09, + "learning_rate": 0.00028021063189568706, + "loss": 2.6527, + "theoretical_loss": 3.5224655899780197, + "tokens_seen": 1469509632 + }, + { + "epoch": 4.09, + "learning_rate": 0.00028020060180541624, + "loss": 2.5792, + "theoretical_loss": 3.5224516850242287, + "tokens_seen": 1469575168 + }, + { + "epoch": 4.09, + "objective/train/docs_used": 1603030, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.003769636154175, + "objective/train/theoretical_loss": 3.522437780864136, + "objective/train/tokens_used": 1483791840, + "theoretical_loss": 3.522437780864136, + "tokens_seen": 1469640704 + }, + { + "epoch": 4.09, + "learning_rate": 0.0002801905717151454, + "loss": 2.6305, + "theoretical_loss": 3.522437780864136, + "tokens_seen": 1469640704 + }, + { + "epoch": 4.09, + "learning_rate": 0.0002801805416248746, + "loss": 2.6325, + "theoretical_loss": 3.5224238774976606, + "tokens_seen": 1469706240 + }, + { + "epoch": 4.09, + "learning_rate": 0.00028017051153460384, + "loss": 2.5469, + "theoretical_loss": 3.5224099749247215, + "tokens_seen": 1469771776 + }, + { + "epoch": 4.09, + "learning_rate": 0.00028016048144433297, + "loss": 2.7021, + "theoretical_loss": 3.5223960731452384, + "tokens_seen": 1469837312 + }, + { + "epoch": 4.09, + "learning_rate": 0.0002801504513540622, + "loss": 2.4954, + "theoretical_loss": 3.522382172159131, + "tokens_seen": 1469902848 + }, + { + "epoch": 4.09, + "learning_rate": 0.00028014042126379133, + "loss": 2.6064, + "theoretical_loss": 3.522368271966317, + "tokens_seen": 1469968384 + }, + { + "epoch": 4.09, + "learning_rate": 0.00028013039117352057, + "loss": 2.6808, + "theoretical_loss": 3.522354372566718, + "tokens_seen": 1470033920 + }, + { + "epoch": 4.09, + "learning_rate": 0.00028012036108324975, + "loss": 2.8744, + "theoretical_loss": 3.522340473960252, + "tokens_seen": 1470099456 + }, + { + "epoch": 4.09, + "learning_rate": 0.00028011033099297893, + "loss": 2.5823, + "theoretical_loss": 3.5223265761468388, + "tokens_seen": 1470164992 + }, + { + "epoch": 4.09, + "learning_rate": 0.0002801003009027081, + "loss": 2.361, + "theoretical_loss": 3.5223126791263977, + "tokens_seen": 1470230528 + }, + { + "epoch": 4.09, + "learning_rate": 0.00028009027081243735, + "loss": 2.6317, + "theoretical_loss": 3.522298782898848, + "tokens_seen": 1470296064 + }, + { + "epoch": 4.09, + "learning_rate": 0.00028008024072216647, + "loss": 2.5516, + "theoretical_loss": 3.5222848874641093, + "tokens_seen": 1470361600 + }, + { + "epoch": 4.09, + "learning_rate": 0.0002800702106318957, + "loss": 2.6678, + "theoretical_loss": 3.5222709928221008, + "tokens_seen": 1470427136 + }, + { + "epoch": 4.09, + "learning_rate": 0.00028006018054162483, + "loss": 2.4722, + "theoretical_loss": 3.522257098972742, + "tokens_seen": 1470492672 + }, + { + "epoch": 4.09, + "learning_rate": 0.00028005015045135407, + "loss": 2.779, + "theoretical_loss": 3.522243205915953, + "tokens_seen": 1470558208 + }, + { + "epoch": 4.09, + "learning_rate": 0.00028004012036108325, + "loss": 2.6158, + "theoretical_loss": 3.5222293136516525, + "tokens_seen": 1470623744 + }, + { + "epoch": 4.09, + "learning_rate": 0.00028003009027081243, + "loss": 2.41, + "theoretical_loss": 3.52221542217976, + "tokens_seen": 1470689280 + }, + { + "epoch": 4.09, + "learning_rate": 0.00028002006018054167, + "loss": 2.4115, + "theoretical_loss": 3.5222015315001958, + "tokens_seen": 1470754816 + }, + { + "epoch": 4.09, + "learning_rate": 0.0002800100300902708, + "loss": 2.6044, + "theoretical_loss": 3.522187641612878, + "tokens_seen": 1470820352 + }, + { + "epoch": 4.09, + "learning_rate": 0.00028000000000000003, + "loss": 2.3933, + "theoretical_loss": 3.5221737525177277, + "tokens_seen": 1470885888 + }, + { + "epoch": 4.09, + "learning_rate": 0.0002799899699097292, + "loss": 2.5452, + "theoretical_loss": 3.5221598642146636, + "tokens_seen": 1470951424 + }, + { + "epoch": 4.09, + "learning_rate": 0.0002799799398194584, + "loss": 2.4104, + "theoretical_loss": 3.522145976703605, + "tokens_seen": 1471016960 + }, + { + "epoch": 4.09, + "learning_rate": 0.0002799699097291876, + "loss": 2.4964, + "theoretical_loss": 3.522132089984472, + "tokens_seen": 1471082496 + }, + { + "epoch": 4.09, + "learning_rate": 0.00027995987963891675, + "loss": 2.5139, + "theoretical_loss": 3.522118204057184, + "tokens_seen": 1471148032 + }, + { + "epoch": 4.09, + "learning_rate": 0.00027994984954864594, + "loss": 2.5464, + "theoretical_loss": 3.5221043189216603, + "tokens_seen": 1471213568 + }, + { + "epoch": 4.09, + "objective/train/docs_used": 1603030, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.143112897872925, + "objective/train/theoretical_loss": 3.5220904345778203, + "objective/train/tokens_used": 1483791840, + "theoretical_loss": 3.5220904345778203, + "tokens_seen": 1471279104 + }, + { + "epoch": 4.09, + "learning_rate": 0.00027993981945837517, + "loss": 2.7808, + "theoretical_loss": 3.5220904345778203, + "tokens_seen": 1471279104 + }, + { + "epoch": 4.09, + "learning_rate": 0.0002799297893681043, + "loss": 2.6289, + "theoretical_loss": 3.5220765510255845, + "tokens_seen": 1471344640 + }, + { + "epoch": 4.09, + "learning_rate": 0.00027991975927783353, + "loss": 2.6422, + "theoretical_loss": 3.522062668264872, + "tokens_seen": 1471410176 + }, + { + "epoch": 4.09, + "learning_rate": 0.0002799097291875627, + "loss": 2.5013, + "theoretical_loss": 3.5220487862956027, + "tokens_seen": 1471475712 + }, + { + "epoch": 4.09, + "learning_rate": 0.0002798996990972919, + "loss": 2.5409, + "theoretical_loss": 3.522034905117695, + "tokens_seen": 1471541248 + }, + { + "epoch": 4.09, + "learning_rate": 0.0002798896690070211, + "loss": 2.5736, + "theoretical_loss": 3.52202102473107, + "tokens_seen": 1471606784 + }, + { + "epoch": 4.09, + "learning_rate": 0.00027987963891675026, + "loss": 2.7528, + "theoretical_loss": 3.5220071451356474, + "tokens_seen": 1471672320 + }, + { + "epoch": 4.09, + "learning_rate": 0.00027986960882647944, + "loss": 2.2941, + "theoretical_loss": 3.5219932663313456, + "tokens_seen": 1471737856 + }, + { + "epoch": 4.09, + "learning_rate": 0.0002798595787362087, + "loss": 2.6457, + "theoretical_loss": 3.5219793883180857, + "tokens_seen": 1471803392 + }, + { + "epoch": 4.09, + "learning_rate": 0.0002798495486459378, + "loss": 2.5519, + "theoretical_loss": 3.5219655110957864, + "tokens_seen": 1471868928 + }, + { + "epoch": 4.09, + "learning_rate": 0.00027983951855566704, + "loss": 2.6568, + "theoretical_loss": 3.521951634664368, + "tokens_seen": 1471934464 + }, + { + "epoch": 4.09, + "learning_rate": 0.00027982948846539616, + "loss": 2.4972, + "theoretical_loss": 3.521937759023749, + "tokens_seen": 1472000000 + }, + { + "epoch": 4.09, + "learning_rate": 0.0002798194583751254, + "loss": 2.4951, + "theoretical_loss": 3.5219238841738507, + "tokens_seen": 1472065536 + }, + { + "epoch": 4.09, + "learning_rate": 0.0002798094282848546, + "loss": 2.6323, + "theoretical_loss": 3.521910010114593, + "tokens_seen": 1472131072 + }, + { + "epoch": 4.09, + "learning_rate": 0.00027979939819458376, + "loss": 2.7124, + "theoretical_loss": 3.5218961368458936, + "tokens_seen": 1472196608 + }, + { + "epoch": 4.09, + "learning_rate": 0.00027978936810431294, + "loss": 2.4036, + "theoretical_loss": 3.521882264367674, + "tokens_seen": 1472262144 + }, + { + "epoch": 4.09, + "learning_rate": 0.0002797793380140422, + "loss": 2.3475, + "theoretical_loss": 3.5218683926798535, + "tokens_seen": 1472327680 + }, + { + "epoch": 4.09, + "learning_rate": 0.0002797693079237713, + "loss": 2.6737, + "theoretical_loss": 3.521854521782352, + "tokens_seen": 1472393216 + }, + { + "epoch": 4.09, + "learning_rate": 0.00027975927783350054, + "loss": 2.61, + "theoretical_loss": 3.5218406516750886, + "tokens_seen": 1472458752 + }, + { + "epoch": 4.09, + "learning_rate": 0.00027974924774322967, + "loss": 2.4702, + "theoretical_loss": 3.5218267823579845, + "tokens_seen": 1472524288 + }, + { + "epoch": 4.09, + "learning_rate": 0.0002797392176529589, + "loss": 2.5781, + "theoretical_loss": 3.521812913830958, + "tokens_seen": 1472589824 + }, + { + "epoch": 4.09, + "learning_rate": 0.0002797291875626881, + "loss": 2.704, + "theoretical_loss": 3.52179904609393, + "tokens_seen": 1472655360 + }, + { + "epoch": 4.09, + "learning_rate": 0.00027971915747241726, + "loss": 2.5015, + "theoretical_loss": 3.52178517914682, + "tokens_seen": 1472720896 + }, + { + "epoch": 4.09, + "learning_rate": 0.00027970912738214644, + "loss": 2.4233, + "theoretical_loss": 3.521771312989548, + "tokens_seen": 1472786432 + }, + { + "epoch": 4.09, + "learning_rate": 0.0002796990972918756, + "loss": 2.7242, + "theoretical_loss": 3.5217574476220337, + "tokens_seen": 1472851968 + }, + { + "epoch": 4.09, + "objective/train/docs_used": 1603030, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5638537406921387, + "objective/train/theoretical_loss": 3.5217435830441968, + "objective/train/tokens_used": 1483791840, + "theoretical_loss": 3.5217435830441968, + "tokens_seen": 1472917504 + }, + { + "epoch": 4.09, + "learning_rate": 0.0002796890672016048, + "loss": 2.5489, + "theoretical_loss": 3.5217435830441968, + "tokens_seen": 1472917504 + }, + { + "epoch": 4.09, + "learning_rate": 0.00027967903711133404, + "loss": 2.5476, + "theoretical_loss": 3.521729719255957, + "tokens_seen": 1472983040 + }, + { + "epoch": 4.09, + "learning_rate": 0.00027966900702106317, + "loss": 2.3642, + "theoretical_loss": 3.5217158562572353, + "tokens_seen": 1473048576 + }, + { + "epoch": 4.09, + "learning_rate": 0.0002796589769307924, + "loss": 2.4547, + "theoretical_loss": 3.521701994047951, + "tokens_seen": 1473114112 + }, + { + "epoch": 4.09, + "learning_rate": 0.00027964894684052153, + "loss": 2.5487, + "theoretical_loss": 3.5216881326280234, + "tokens_seen": 1473179648 + }, + { + "epoch": 4.09, + "learning_rate": 0.00027963891675025077, + "loss": 2.819, + "theoretical_loss": 3.521674271997373, + "tokens_seen": 1473245184 + }, + { + "epoch": 4.09, + "learning_rate": 0.00027962888665997995, + "loss": 2.6273, + "theoretical_loss": 3.52166041215592, + "tokens_seen": 1473310720 + }, + { + "epoch": 4.09, + "learning_rate": 0.00027961885656970913, + "loss": 2.664, + "theoretical_loss": 3.521646553103584, + "tokens_seen": 1473376256 + }, + { + "epoch": 4.09, + "learning_rate": 0.0002796088264794383, + "loss": 2.4287, + "theoretical_loss": 3.521632694840285, + "tokens_seen": 1473441792 + }, + { + "epoch": 4.09, + "learning_rate": 0.00027959879638916755, + "loss": 2.3851, + "theoretical_loss": 3.521618837365943, + "tokens_seen": 1473507328 + }, + { + "epoch": 4.09, + "learning_rate": 0.00027958876629889667, + "loss": 2.4027, + "theoretical_loss": 3.521604980680478, + "tokens_seen": 1473572864 + }, + { + "epoch": 4.09, + "learning_rate": 0.0002795787362086259, + "loss": 2.6463, + "theoretical_loss": 3.5215911247838103, + "tokens_seen": 1473638400 + }, + { + "epoch": 4.09, + "learning_rate": 0.00027956870611835503, + "loss": 2.3324, + "theoretical_loss": 3.5215772696758596, + "tokens_seen": 1473703936 + }, + { + "epoch": 4.09, + "learning_rate": 0.00027955867602808427, + "loss": 2.2111, + "theoretical_loss": 3.521563415356546, + "tokens_seen": 1473769472 + }, + { + "epoch": 4.09, + "learning_rate": 0.00027954864593781345, + "loss": 2.3678, + "theoretical_loss": 3.5215495618257897, + "tokens_seen": 1473835008 + }, + { + "epoch": 4.09, + "learning_rate": 0.00027953861584754263, + "loss": 2.5052, + "theoretical_loss": 3.52153570908351, + "tokens_seen": 1473900544 + }, + { + "epoch": 4.09, + "learning_rate": 0.0002795285857572718, + "loss": 2.6186, + "theoretical_loss": 3.5215218571296285, + "tokens_seen": 1473966080 + }, + { + "epoch": 4.09, + "learning_rate": 0.000279518555667001, + "loss": 2.4767, + "theoretical_loss": 3.5215080059640638, + "tokens_seen": 1474031616 + }, + { + "epoch": 4.09, + "learning_rate": 0.0002795085255767302, + "loss": 2.3363, + "theoretical_loss": 3.5214941555867365, + "tokens_seen": 1474097152 + }, + { + "epoch": 4.09, + "learning_rate": 0.0002794984954864594, + "loss": 2.7143, + "theoretical_loss": 3.5214803059975672, + "tokens_seen": 1474162688 + }, + { + "epoch": 4.09, + "learning_rate": 0.00027948846539618854, + "loss": 2.4726, + "theoretical_loss": 3.521466457196475, + "tokens_seen": 1474228224 + }, + { + "epoch": 4.09, + "learning_rate": 0.0002794784353059178, + "loss": 2.7101, + "theoretical_loss": 3.521452609183381, + "tokens_seen": 1474293760 + }, + { + "epoch": 4.09, + "learning_rate": 0.0002794684052156469, + "loss": 2.31, + "theoretical_loss": 3.521438761958205, + "tokens_seen": 1474359296 + }, + { + "epoch": 4.09, + "learning_rate": 0.00027945837512537614, + "loss": 2.6026, + "theoretical_loss": 3.521424915520867, + "tokens_seen": 1474424832 + }, + { + "epoch": 4.09, + "learning_rate": 0.0002794483450351053, + "loss": 2.5403, + "theoretical_loss": 3.521411069871287, + "tokens_seen": 1474490368 + }, + { + "debugging/Self-BLEU-5": 0.40022323577174246, + "debugging/distinct-1-grams": 0.792449235368197, + "debugging/distinct-2-grams": 0.964039917418843, + "debugging/entropy-1-grams": 5.830051148996791, + "debugging/entropy-2-grams": 6.574183943619101, + "debugging/length": 506.7, + "debugging/num_segments": 10, + "debugging/score": 0.007128608811555618, + "debugging/score_std": 0.008951280671415335, + "epoch": 4.09, + "objective/train/docs_used": 1603030, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.455482244491577, + "objective/train/theoretical_loss": 3.5213972250093857, + "objective/train/tokens_used": 1483791840, + "theoretical_loss": 3.5213972250093857, + "tokens_seen": 1474555904 + }, + { + "epoch": 4.09, + "learning_rate": 0.0002794383149448345, + "loss": 2.5027, + "theoretical_loss": 3.5213972250093857, + "tokens_seen": 1474555904 + }, + { + "epoch": 4.09, + "learning_rate": 0.0002794282848545637, + "loss": 2.409, + "theoretical_loss": 3.5213833809350827, + "tokens_seen": 1474621440 + }, + { + "epoch": 4.09, + "learning_rate": 0.0002794182547642929, + "loss": 2.7487, + "theoretical_loss": 3.5213695376482983, + "tokens_seen": 1474686976 + }, + { + "epoch": 4.09, + "learning_rate": 0.00027940822467402204, + "loss": 2.3847, + "theoretical_loss": 3.5213556951489533, + "tokens_seen": 1474752512 + }, + { + "epoch": 4.09, + "learning_rate": 0.0002793981945837513, + "loss": 2.4294, + "theoretical_loss": 3.521341853436968, + "tokens_seen": 1474818048 + }, + { + "epoch": 4.09, + "learning_rate": 0.0002793881644934804, + "loss": 2.7022, + "theoretical_loss": 3.521328012512261, + "tokens_seen": 1474883584 + }, + { + "epoch": 4.09, + "learning_rate": 0.00027937813440320964, + "loss": 2.4663, + "theoretical_loss": 3.5213141723747543, + "tokens_seen": 1474949120 + }, + { + "epoch": 4.09, + "learning_rate": 0.0002793681043129388, + "loss": 2.4916, + "theoretical_loss": 3.5213003330243673, + "tokens_seen": 1475014656 + }, + { + "epoch": 4.09, + "learning_rate": 0.000279358074222668, + "loss": 2.5263, + "theoretical_loss": 3.521286494461021, + "tokens_seen": 1475080192 + }, + { + "epoch": 4.09, + "learning_rate": 0.0002793480441323972, + "loss": 2.5372, + "theoretical_loss": 3.521272656684635, + "tokens_seen": 1475145728 + }, + { + "epoch": 4.09, + "learning_rate": 0.00027933801404212636, + "loss": 2.4316, + "theoretical_loss": 3.5212588196951295, + "tokens_seen": 1475211264 + }, + { + "epoch": 4.09, + "learning_rate": 0.00027932798395185554, + "loss": 2.3126, + "theoretical_loss": 3.5212449834924255, + "tokens_seen": 1475276800 + }, + { + "epoch": 4.09, + "learning_rate": 0.0002793179538615848, + "loss": 2.3813, + "theoretical_loss": 3.5212311480764424, + "tokens_seen": 1475342336 + }, + { + "epoch": 4.09, + "learning_rate": 0.0002793079237713139, + "loss": 2.8092, + "theoretical_loss": 3.521217313447101, + "tokens_seen": 1475407872 + }, + { + "epoch": 4.09, + "learning_rate": 0.00027929789368104314, + "loss": 2.5193, + "theoretical_loss": 3.5212034796043223, + "tokens_seen": 1475473408 + }, + { + "epoch": 4.09, + "learning_rate": 0.00027928786359077227, + "loss": 2.5998, + "theoretical_loss": 3.5211896465480255, + "tokens_seen": 1475538944 + }, + { + "epoch": 4.09, + "learning_rate": 0.0002792778335005015, + "loss": 2.4218, + "theoretical_loss": 3.5211758142781315, + "tokens_seen": 1475604480 + }, + { + "epoch": 4.09, + "learning_rate": 0.00027926780341023074, + "loss": 2.5907, + "theoretical_loss": 3.5211619827945606, + "tokens_seen": 1475670016 + }, + { + "epoch": 4.09, + "learning_rate": 0.00027925777331995987, + "loss": 2.3143, + "theoretical_loss": 3.5211481520972336, + "tokens_seen": 1475735552 + }, + { + "epoch": 4.09, + "learning_rate": 0.0002792477432296891, + "loss": 2.8273, + "theoretical_loss": 3.52113432218607, + "tokens_seen": 1475801088 + }, + { + "epoch": 4.09, + "learning_rate": 0.0002792377131394183, + "loss": 2.5817, + "theoretical_loss": 3.521120493060991, + "tokens_seen": 1475866624 + }, + { + "epoch": 4.09, + "learning_rate": 0.00027922768304914746, + "loss": 2.3576, + "theoretical_loss": 3.5211066647219162, + "tokens_seen": 1475932160 + }, + { + "epoch": 4.09, + "learning_rate": 0.00027921765295887664, + "loss": 2.3519, + "theoretical_loss": 3.521092837168767, + "tokens_seen": 1475997696 + }, + { + "epoch": 4.09, + "learning_rate": 0.0002792076228686058, + "loss": 2.62, + "theoretical_loss": 3.521079010401463, + "tokens_seen": 1476063232 + }, + { + "epoch": 4.09, + "learning_rate": 0.000279197592778335, + "loss": 2.4679, + "theoretical_loss": 3.5210651844199257, + "tokens_seen": 1476128768 + }, + { + "epoch": 4.09, + "objective/train/docs_used": 1603030, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.234206199645996, + "objective/train/theoretical_loss": 3.5210513592240744, + "objective/train/tokens_used": 1483791840, + "theoretical_loss": 3.5210513592240744, + "tokens_seen": 1476194304 + }, + { + "epoch": 4.09, + "learning_rate": 0.00027918756268806424, + "loss": 2.6296, + "theoretical_loss": 3.5210513592240744, + "tokens_seen": 1476194304 + }, + { + "epoch": 4.09, + "learning_rate": 0.00027917753259779337, + "loss": 2.5854, + "theoretical_loss": 3.5210375348138303, + "tokens_seen": 1476259840 + }, + { + "epoch": 4.09, + "learning_rate": 0.0002791675025075226, + "loss": 2.4903, + "theoretical_loss": 3.5210237111891134, + "tokens_seen": 1476325376 + }, + { + "epoch": 4.09, + "learning_rate": 0.00027915747241725173, + "loss": 2.7127, + "theoretical_loss": 3.5210098883498446, + "tokens_seen": 1476390912 + }, + { + "epoch": 4.09, + "learning_rate": 0.00027914744232698097, + "loss": 2.8816, + "theoretical_loss": 3.520996066295944, + "tokens_seen": 1476456448 + }, + { + "epoch": 4.09, + "learning_rate": 0.00027913741223671015, + "loss": 2.6881, + "theoretical_loss": 3.520982245027333, + "tokens_seen": 1476521984 + }, + { + "epoch": 4.09, + "learning_rate": 0.00027912738214643933, + "loss": 2.446, + "theoretical_loss": 3.5209684245439314, + "tokens_seen": 1476587520 + }, + { + "epoch": 4.09, + "learning_rate": 0.0002791173520561685, + "loss": 2.7502, + "theoretical_loss": 3.5209546048456595, + "tokens_seen": 1476653056 + }, + { + "epoch": 4.09, + "learning_rate": 0.00027910732196589775, + "loss": 2.6565, + "theoretical_loss": 3.520940785932438, + "tokens_seen": 1476718592 + }, + { + "epoch": 4.09, + "learning_rate": 0.00027909729187562687, + "loss": 2.3348, + "theoretical_loss": 3.520926967804188, + "tokens_seen": 1476784128 + }, + { + "epoch": 4.09, + "learning_rate": 0.0002790872617853561, + "loss": 2.6025, + "theoretical_loss": 3.52091315046083, + "tokens_seen": 1476849664 + }, + { + "epoch": 4.09, + "learning_rate": 0.00027907723169508523, + "loss": 2.6091, + "theoretical_loss": 3.5208993339022845, + "tokens_seen": 1476915200 + }, + { + "epoch": 4.09, + "learning_rate": 0.00027906720160481447, + "loss": 2.4213, + "theoretical_loss": 3.5208855181284715, + "tokens_seen": 1476980736 + }, + { + "epoch": 4.09, + "learning_rate": 0.00027905717151454365, + "loss": 2.6044, + "theoretical_loss": 3.5208717031393117, + "tokens_seen": 1477046272 + }, + { + "epoch": 4.09, + "learning_rate": 0.00027904714142427283, + "loss": 2.4268, + "theoretical_loss": 3.520857888934727, + "tokens_seen": 1477111808 + }, + { + "epoch": 4.09, + "learning_rate": 0.000279037111334002, + "loss": 2.4377, + "theoretical_loss": 3.5208440755146366, + "tokens_seen": 1477177344 + }, + { + "epoch": 4.09, + "learning_rate": 0.0002790270812437312, + "loss": 2.4946, + "theoretical_loss": 3.5208302628789614, + "tokens_seen": 1477242880 + }, + { + "epoch": 4.09, + "learning_rate": 0.0002790170511534604, + "loss": 2.3929, + "theoretical_loss": 3.520816451027623, + "tokens_seen": 1477308416 + }, + { + "epoch": 4.09, + "learning_rate": 0.0002790070210631896, + "loss": 2.605, + "theoretical_loss": 3.520802639960541, + "tokens_seen": 1477373952 + }, + { + "epoch": 4.09, + "learning_rate": 0.00027899699097291874, + "loss": 2.2421, + "theoretical_loss": 3.5207888296776364, + "tokens_seen": 1477439488 + }, + { + "epoch": 4.09, + "learning_rate": 0.000278986960882648, + "loss": 2.5311, + "theoretical_loss": 3.52077502017883, + "tokens_seen": 1477505024 + }, + { + "epoch": 4.09, + "learning_rate": 0.0002789769307923771, + "loss": 2.6109, + "theoretical_loss": 3.520761211464043, + "tokens_seen": 1477570560 + }, + { + "epoch": 4.09, + "learning_rate": 0.00027896690070210634, + "loss": 2.516, + "theoretical_loss": 3.520747403533195, + "tokens_seen": 1477636096 + }, + { + "epoch": 4.09, + "learning_rate": 0.0002789568706118355, + "loss": 2.4437, + "theoretical_loss": 3.5207335963862074, + "tokens_seen": 1477701632 + }, + { + "epoch": 4.09, + "learning_rate": 0.0002789468405215647, + "loss": 2.8298, + "theoretical_loss": 3.5207197900230014, + "tokens_seen": 1477767168 + }, + { + "epoch": 4.09, + "objective/train/docs_used": 1603030, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.0272233486175537, + "objective/train/theoretical_loss": 3.5207059844434965, + "objective/train/tokens_used": 1483791840, + "theoretical_loss": 3.5207059844434965, + "tokens_seen": 1477832704 + }, + { + "epoch": 4.09, + "learning_rate": 0.0002789368104312939, + "loss": 2.803, + "theoretical_loss": 3.5207059844434965, + "tokens_seen": 1477832704 + }, + { + "epoch": 4.09, + "learning_rate": 0.0002789267803410231, + "loss": 2.6148, + "theoretical_loss": 3.5206921796476145, + "tokens_seen": 1477898240 + }, + { + "epoch": 4.09, + "learning_rate": 0.00027891675025075224, + "loss": 2.6943, + "theoretical_loss": 3.520678375635276, + "tokens_seen": 1477963776 + }, + { + "epoch": 4.09, + "learning_rate": 0.0002789067201604815, + "loss": 2.4597, + "theoretical_loss": 3.5206645724064014, + "tokens_seen": 1478029312 + }, + { + "epoch": 4.09, + "learning_rate": 0.0002788966900702106, + "loss": 2.5187, + "theoretical_loss": 3.5206507699609118, + "tokens_seen": 1478094848 + }, + { + "epoch": 4.09, + "learning_rate": 0.00027888665997993984, + "loss": 2.762, + "theoretical_loss": 3.520636968298728, + "tokens_seen": 1478160384 + }, + { + "epoch": 4.09, + "learning_rate": 0.000278876629889669, + "loss": 2.8494, + "theoretical_loss": 3.5206231674197705, + "tokens_seen": 1478225920 + }, + { + "epoch": 4.09, + "learning_rate": 0.0002788665997993982, + "loss": 2.6576, + "theoretical_loss": 3.520609367323961, + "tokens_seen": 1478291456 + }, + { + "epoch": 4.09, + "learning_rate": 0.0002788565697091274, + "loss": 2.6144, + "theoretical_loss": 3.520595568011219, + "tokens_seen": 1478356992 + }, + { + "epoch": 4.09, + "learning_rate": 0.00027884653961885656, + "loss": 2.4924, + "theoretical_loss": 3.520581769481467, + "tokens_seen": 1478422528 + }, + { + "epoch": 4.09, + "learning_rate": 0.00027883650952858574, + "loss": 2.482, + "theoretical_loss": 3.520567971734624, + "tokens_seen": 1478488064 + }, + { + "epoch": 4.09, + "learning_rate": 0.000278826479438315, + "loss": 2.7568, + "theoretical_loss": 3.5205541747706124, + "tokens_seen": 1478553600 + }, + { + "epoch": 4.09, + "learning_rate": 0.0002788164493480441, + "loss": 2.5784, + "theoretical_loss": 3.5205403785893523, + "tokens_seen": 1478619136 + }, + { + "epoch": 4.09, + "learning_rate": 0.00027880641925777334, + "loss": 2.6064, + "theoretical_loss": 3.5205265831907653, + "tokens_seen": 1478684672 + }, + { + "epoch": 4.09, + "learning_rate": 0.00027879638916750247, + "loss": 2.2802, + "theoretical_loss": 3.520512788574772, + "tokens_seen": 1478750208 + }, + { + "epoch": 4.09, + "learning_rate": 0.0002787863590772317, + "loss": 2.5806, + "theoretical_loss": 3.5204989947412924, + "tokens_seen": 1478815744 + }, + { + "epoch": 4.09, + "learning_rate": 0.0002787763289869609, + "loss": 2.5576, + "theoretical_loss": 3.5204852016902484, + "tokens_seen": 1478881280 + }, + { + "epoch": 4.09, + "learning_rate": 0.00027876629889669007, + "loss": 2.5985, + "theoretical_loss": 3.520471409421561, + "tokens_seen": 1478946816 + }, + { + "epoch": 4.09, + "learning_rate": 0.00027875626880641925, + "loss": 2.5239, + "theoretical_loss": 3.5204576179351506, + "tokens_seen": 1479012352 + }, + { + "epoch": 4.09, + "learning_rate": 0.0002787462387161485, + "loss": 2.4756, + "theoretical_loss": 3.520443827230939, + "tokens_seen": 1479077888 + }, + { + "epoch": 4.09, + "learning_rate": 0.0002787362086258776, + "loss": 2.3017, + "theoretical_loss": 3.520430037308846, + "tokens_seen": 1479143424 + }, + { + "epoch": 4.09, + "learning_rate": 0.00027872617853560685, + "loss": 2.4728, + "theoretical_loss": 3.520416248168794, + "tokens_seen": 1479208960 + }, + { + "epoch": 4.09, + "learning_rate": 0.00027871614844533597, + "loss": 2.5853, + "theoretical_loss": 3.5204024598107027, + "tokens_seen": 1479274496 + }, + { + "epoch": 4.09, + "learning_rate": 0.0002787061183550652, + "loss": 2.6454, + "theoretical_loss": 3.5203886722344935, + "tokens_seen": 1479340032 + }, + { + "epoch": 4.09, + "learning_rate": 0.0002786960882647944, + "loss": 2.5003, + "theoretical_loss": 3.5203748854400883, + "tokens_seen": 1479405568 + }, + { + "epoch": 4.09, + "objective/train/docs_used": 1603030, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.675086259841919, + "objective/train/theoretical_loss": 3.520361099427407, + "objective/train/tokens_used": 1483791840, + "theoretical_loss": 3.520361099427407, + "tokens_seen": 1479471104 + }, + { + "epoch": 4.09, + "learning_rate": 0.00027868605817452357, + "loss": 2.5011, + "theoretical_loss": 3.520361099427407, + "tokens_seen": 1479471104 + }, + { + "epoch": 4.09, + "learning_rate": 0.00027867602808425275, + "loss": 2.553, + "theoretical_loss": 3.5203473141963713, + "tokens_seen": 1479536640 + }, + { + "epoch": 4.09, + "learning_rate": 0.00027866599799398193, + "loss": 2.6432, + "theoretical_loss": 3.5203335297469014, + "tokens_seen": 1479602176 + }, + { + "epoch": 4.09, + "learning_rate": 0.0002786559679037111, + "loss": 2.724, + "theoretical_loss": 3.52031974607892, + "tokens_seen": 1479667712 + }, + { + "epoch": 4.09, + "learning_rate": 0.00027864593781344035, + "loss": 2.5352, + "theoretical_loss": 3.520305963192347, + "tokens_seen": 1479733248 + }, + { + "epoch": 4.09, + "learning_rate": 0.0002786359077231695, + "loss": 2.5688, + "theoretical_loss": 3.520292181087103, + "tokens_seen": 1479798784 + }, + { + "epoch": 4.09, + "learning_rate": 0.0002786258776328987, + "loss": 2.5477, + "theoretical_loss": 3.520278399763111, + "tokens_seen": 1479864320 + }, + { + "epoch": 4.09, + "learning_rate": 0.0002786158475426279, + "loss": 2.4729, + "theoretical_loss": 3.5202646192202898, + "tokens_seen": 1479929856 + }, + { + "epoch": 4.09, + "learning_rate": 0.00027860581745235707, + "loss": 2.7487, + "theoretical_loss": 3.520250839458562, + "tokens_seen": 1479995392 + }, + { + "epoch": 4.09, + "learning_rate": 0.00027859578736208625, + "loss": 2.4694, + "theoretical_loss": 3.520237060477849, + "tokens_seen": 1480060928 + }, + { + "epoch": 4.09, + "learning_rate": 0.00027858575727181544, + "loss": 2.6762, + "theoretical_loss": 3.520223282278071, + "tokens_seen": 1480126464 + }, + { + "epoch": 4.09, + "learning_rate": 0.0002785757271815446, + "loss": 2.7235, + "theoretical_loss": 3.520209504859149, + "tokens_seen": 1480192000 + }, + { + "epoch": 4.09, + "learning_rate": 0.00027856569709127385, + "loss": 2.3901, + "theoretical_loss": 3.5201957282210055, + "tokens_seen": 1480257536 + }, + { + "epoch": 4.09, + "learning_rate": 0.000278555667001003, + "loss": 2.3777, + "theoretical_loss": 3.5201819523635605, + "tokens_seen": 1480323072 + }, + { + "epoch": 4.09, + "learning_rate": 0.0002785456369107322, + "loss": 2.2904, + "theoretical_loss": 3.5201681772867355, + "tokens_seen": 1480388608 + }, + { + "epoch": 4.09, + "learning_rate": 0.00027853560682046134, + "loss": 2.3948, + "theoretical_loss": 3.5201544029904523, + "tokens_seen": 1480454144 + }, + { + "epoch": 4.09, + "learning_rate": 0.0002785255767301906, + "loss": 2.6037, + "theoretical_loss": 3.520140629474631, + "tokens_seen": 1480519680 + }, + { + "epoch": 4.09, + "learning_rate": 0.0002785155466399198, + "loss": 2.7134, + "theoretical_loss": 3.5201268567391937, + "tokens_seen": 1480585216 + }, + { + "epoch": 4.09, + "learning_rate": 0.00027850551654964894, + "loss": 2.8329, + "theoretical_loss": 3.520113084784062, + "tokens_seen": 1480650752 + }, + { + "epoch": 4.09, + "learning_rate": 0.0002784954864593782, + "loss": 2.4178, + "theoretical_loss": 3.5200993136091556, + "tokens_seen": 1480716288 + }, + { + "epoch": 4.09, + "learning_rate": 0.0002784854563691073, + "loss": 2.4676, + "theoretical_loss": 3.520085543214397, + "tokens_seen": 1480781824 + }, + { + "epoch": 4.09, + "learning_rate": 0.00027847542627883654, + "loss": 2.1964, + "theoretical_loss": 3.5200717735997076, + "tokens_seen": 1480847360 + }, + { + "epoch": 4.09, + "learning_rate": 0.0002784653961885657, + "loss": 2.7747, + "theoretical_loss": 3.5200580047650076, + "tokens_seen": 1480912896 + }, + { + "epoch": 4.09, + "learning_rate": 0.0002784553660982949, + "loss": 2.5658, + "theoretical_loss": 3.52004423671022, + "tokens_seen": 1480978432 + }, + { + "epoch": 4.09, + "learning_rate": 0.0002784453360080241, + "loss": 2.6829, + "theoretical_loss": 3.520030469435264, + "tokens_seen": 1481043968 + }, + { + "epoch": 4.09, + "objective/train/docs_used": 1603030, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.576517105102539, + "objective/train/theoretical_loss": 3.520016702940063, + "objective/train/tokens_used": 1483791840, + "theoretical_loss": 3.520016702940063, + "tokens_seen": 1481109504 + }, + { + "epoch": 4.09, + "learning_rate": 0.0002784353059177533, + "loss": 2.6277, + "theoretical_loss": 3.520016702940063, + "tokens_seen": 1481109504 + }, + { + "epoch": 4.09, + "learning_rate": 0.00027842527582748244, + "loss": 2.4534, + "theoretical_loss": 3.520002937224537, + "tokens_seen": 1481175040 + }, + { + "epoch": 4.09, + "learning_rate": 0.0002784152457372117, + "loss": 2.5497, + "theoretical_loss": 3.519989172288608, + "tokens_seen": 1481240576 + }, + { + "epoch": 4.09, + "learning_rate": 0.0002784052156469408, + "loss": 2.5595, + "theoretical_loss": 3.519975408132197, + "tokens_seen": 1481306112 + }, + { + "epoch": 4.09, + "learning_rate": 0.00027839518555667004, + "loss": 2.4755, + "theoretical_loss": 3.519961644755225, + "tokens_seen": 1481371648 + }, + { + "epoch": 4.09, + "learning_rate": 0.0002783851554663992, + "loss": 2.5273, + "theoretical_loss": 3.5199478821576142, + "tokens_seen": 1481437184 + }, + { + "epoch": 4.09, + "learning_rate": 0.0002783751253761284, + "loss": 2.6981, + "theoretical_loss": 3.5199341203392853, + "tokens_seen": 1481502720 + }, + { + "epoch": 4.09, + "learning_rate": 0.0002783650952858576, + "loss": 2.6444, + "theoretical_loss": 3.5199203593001602, + "tokens_seen": 1481568256 + }, + { + "epoch": 4.09, + "learning_rate": 0.00027835506519558676, + "loss": 2.3927, + "theoretical_loss": 3.5199065990401603, + "tokens_seen": 1481633792 + }, + { + "epoch": 4.09, + "learning_rate": 0.00027834503510531594, + "loss": 2.4405, + "theoretical_loss": 3.5198928395592075, + "tokens_seen": 1481699328 + }, + { + "epoch": 4.09, + "learning_rate": 0.0002783350050150452, + "loss": 2.536, + "theoretical_loss": 3.519879080857222, + "tokens_seen": 1481764864 + }, + { + "epoch": 4.09, + "learning_rate": 0.0002783249749247743, + "loss": 2.4936, + "theoretical_loss": 3.5198653229341255, + "tokens_seen": 1481830400 + }, + { + "epoch": 4.09, + "learning_rate": 0.00027831494483450354, + "loss": 2.2565, + "theoretical_loss": 3.51985156578984, + "tokens_seen": 1481895936 + }, + { + "epoch": 4.09, + "learning_rate": 0.00027830491474423267, + "loss": 2.3814, + "theoretical_loss": 3.5198378094242875, + "tokens_seen": 1481961472 + }, + { + "epoch": 4.09, + "learning_rate": 0.0002782948846539619, + "loss": 2.4476, + "theoretical_loss": 3.5198240538373877, + "tokens_seen": 1482027008 + }, + { + "epoch": 4.09, + "learning_rate": 0.0002782848545636911, + "loss": 2.4944, + "theoretical_loss": 3.5198102990290643, + "tokens_seen": 1482092544 + }, + { + "epoch": 4.09, + "learning_rate": 0.00027827482447342027, + "loss": 2.4673, + "theoretical_loss": 3.519796544999237, + "tokens_seen": 1482158080 + }, + { + "epoch": 4.09, + "learning_rate": 0.00027826479438314945, + "loss": 2.3833, + "theoretical_loss": 3.5197827917478284, + "tokens_seen": 1482223616 + }, + { + "epoch": 4.09, + "learning_rate": 0.0002782547642928787, + "loss": 2.6754, + "theoretical_loss": 3.5197690392747596, + "tokens_seen": 1482289152 + }, + { + "epoch": 4.09, + "learning_rate": 0.0002782447342026078, + "loss": 2.2025, + "theoretical_loss": 3.519755287579952, + "tokens_seen": 1482354688 + }, + { + "epoch": 4.09, + "learning_rate": 0.00027823470411233705, + "loss": 2.522, + "theoretical_loss": 3.519741536663327, + "tokens_seen": 1482420224 + }, + { + "epoch": 4.09, + "learning_rate": 0.00027822467402206617, + "loss": 2.5283, + "theoretical_loss": 3.519727786524807, + "tokens_seen": 1482485760 + }, + { + "epoch": 4.09, + "learning_rate": 0.0002782146439317954, + "loss": 2.3373, + "theoretical_loss": 3.519714037164313, + "tokens_seen": 1482551296 + }, + { + "epoch": 4.09, + "learning_rate": 0.0002782046138415246, + "loss": 2.4523, + "theoretical_loss": 3.5197002885817668, + "tokens_seen": 1482616832 + }, + { + "epoch": 4.09, + "learning_rate": 0.00027819458375125377, + "loss": 2.8188, + "theoretical_loss": 3.5196865407770894, + "tokens_seen": 1482682368 + }, + { + "epoch": 4.09, + "objective/train/docs_used": 1603030, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.83261775970459, + "objective/train/theoretical_loss": 3.5196727937502033, + "objective/train/tokens_used": 1483791840, + "theoretical_loss": 3.5196727937502033, + "tokens_seen": 1482747904 + }, + { + "epoch": 4.09, + "learning_rate": 0.00027818455366098295, + "loss": 2.7207, + "theoretical_loss": 3.5196727937502033, + "tokens_seen": 1482747904 + }, + { + "epoch": 4.09, + "learning_rate": 0.00027817452357071213, + "loss": 2.5233, + "theoretical_loss": 3.5196590475010296, + "tokens_seen": 1482813440 + }, + { + "epoch": 4.09, + "learning_rate": 0.0002781644934804413, + "loss": 2.5227, + "theoretical_loss": 3.51964530202949, + "tokens_seen": 1482878976 + }, + { + "epoch": 4.09, + "learning_rate": 0.00027815446339017055, + "loss": 2.5866, + "theoretical_loss": 3.519631557335506, + "tokens_seen": 1482944512 + }, + { + "epoch": 4.09, + "learning_rate": 0.0002781444332998997, + "loss": 2.5737, + "theoretical_loss": 3.519617813419, + "tokens_seen": 1483010048 + }, + { + "epoch": 4.09, + "learning_rate": 0.0002781344032096289, + "loss": 2.4936, + "theoretical_loss": 3.5196040702798923, + "tokens_seen": 1483075584 + }, + { + "epoch": 4.09, + "learning_rate": 0.0002781243731193581, + "loss": 2.74, + "theoretical_loss": 3.519590327918106, + "tokens_seen": 1483141120 + }, + { + "epoch": 4.09, + "learning_rate": 0.0002781143430290873, + "loss": 2.5662, + "theoretical_loss": 3.519576586333562, + "tokens_seen": 1483206656 + }, + { + "epoch": 4.09, + "learning_rate": 0.00027810431293881645, + "loss": 2.6936, + "theoretical_loss": 3.519562845526182, + "tokens_seen": 1483272192 + }, + { + "epoch": 4.09, + "learning_rate": 0.00027809428284854564, + "loss": 2.5848, + "theoretical_loss": 3.519549105495888, + "tokens_seen": 1483337728 + }, + { + "epoch": 4.09, + "learning_rate": 0.0002780842527582748, + "loss": 2.735, + "theoretical_loss": 3.5195353662426014, + "tokens_seen": 1483403264 + }, + { + "epoch": 4.09, + "learning_rate": 0.00027807422266800405, + "loss": 2.4056, + "theoretical_loss": 3.5195216277662444, + "tokens_seen": 1483468800 + }, + { + "epoch": 4.09, + "learning_rate": 0.0002780641925777332, + "loss": 2.3115, + "theoretical_loss": 3.5195078900667385, + "tokens_seen": 1483534336 + }, + { + "epoch": 4.09, + "learning_rate": 0.0002780541624874624, + "loss": 2.5639, + "theoretical_loss": 3.5194941531440054, + "tokens_seen": 1483599872 + }, + { + "epoch": 4.09, + "learning_rate": 0.00027804413239719154, + "loss": 2.5111, + "theoretical_loss": 3.5194804169979665, + "tokens_seen": 1483665408 + }, + { + "epoch": 4.09, + "learning_rate": 0.0002780341023069208, + "loss": 2.6556, + "theoretical_loss": 3.5194666816285447, + "tokens_seen": 1483730944 + }, + { + "epoch": 4.09, + "learning_rate": 0.00027802407221664996, + "loss": 2.8688, + "theoretical_loss": 3.519452947035661, + "tokens_seen": 1483796480 + }, + { + "epoch": 5.0, + "learning_rate": 0.00027801404212637914, + "loss": 3.5425, + "theoretical_loss": 3.5194359944683216, + "tokens_seen": 1483877376 + }, + { + "epoch": 5.0, + "learning_rate": 0.0002780040120361083, + "loss": 2.8675, + "theoretical_loss": 3.5194222616102326, + "tokens_seen": 1483942912 + }, + { + "epoch": 5.0, + "learning_rate": 0.0002779939819458375, + "loss": 2.6917, + "theoretical_loss": 3.5194085295284285, + "tokens_seen": 1484008448 + }, + { + "epoch": 5.0, + "learning_rate": 0.0002779839518555667, + "loss": 2.8906, + "theoretical_loss": 3.519394798222832, + "tokens_seen": 1484073984 + }, + { + "epoch": 5.0, + "learning_rate": 0.0002779739217652959, + "loss": 2.6199, + "theoretical_loss": 3.519381067693365, + "tokens_seen": 1484139520 + }, + { + "epoch": 5.0, + "learning_rate": 0.00027796389167502504, + "loss": 2.701, + "theoretical_loss": 3.519367337939948, + "tokens_seen": 1484205056 + }, + { + "epoch": 5.0, + "learning_rate": 0.0002779538615847543, + "loss": 2.5548, + "theoretical_loss": 3.519353608962504, + "tokens_seen": 1484270592 + }, + { + "epoch": 5.0, + "learning_rate": 0.00027794383149448346, + "loss": 2.709, + "theoretical_loss": 3.5193398807609544, + "tokens_seen": 1484336128 + }, + { + "epoch": 5.0, + "objective/train/docs_used": 1667839, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.698960781097412, + "objective/train/theoretical_loss": 3.5193295851189266, + "objective/train/tokens_used": 1504845280, + "theoretical_loss": 3.5193295851189266, + "tokens_seen": 1484385280 + }, + { + "epoch": 5.0, + "learning_rate": 0.00027793380140421264, + "loss": 2.6594, + "theoretical_loss": 3.519326153335222, + "tokens_seen": 1484401664 + }, + { + "epoch": 5.0, + "learning_rate": 0.0002779237713139418, + "loss": 2.679, + "theoretical_loss": 3.5193124266852274, + "tokens_seen": 1484467200 + }, + { + "epoch": 5.0, + "learning_rate": 0.000277913741223671, + "loss": 2.6059, + "theoretical_loss": 3.519298700810893, + "tokens_seen": 1484532736 + }, + { + "epoch": 5.0, + "learning_rate": 0.0002779037111334002, + "loss": 2.5911, + "theoretical_loss": 3.519284975712141, + "tokens_seen": 1484598272 + }, + { + "epoch": 5.0, + "learning_rate": 0.0002778936810431294, + "loss": 2.542, + "theoretical_loss": 3.5192712513888935, + "tokens_seen": 1484663808 + }, + { + "epoch": 5.0, + "learning_rate": 0.00027788365095285855, + "loss": 2.634, + "theoretical_loss": 3.5192575278410723, + "tokens_seen": 1484729344 + }, + { + "epoch": 5.0, + "learning_rate": 0.0002778736208625878, + "loss": 2.5404, + "theoretical_loss": 3.519243805068599, + "tokens_seen": 1484794880 + }, + { + "epoch": 5.0, + "learning_rate": 0.0002778635907723169, + "loss": 2.4077, + "theoretical_loss": 3.5192300830713954, + "tokens_seen": 1484860416 + }, + { + "epoch": 5.0, + "learning_rate": 0.00027785356068204614, + "loss": 2.8491, + "theoretical_loss": 3.5192163618493844, + "tokens_seen": 1484925952 + }, + { + "epoch": 5.0, + "learning_rate": 0.0002778435305917753, + "loss": 2.6214, + "theoretical_loss": 3.5192026414024875, + "tokens_seen": 1484991488 + }, + { + "epoch": 5.0, + "learning_rate": 0.0002778335005015045, + "loss": 2.8468, + "theoretical_loss": 3.5191889217306267, + "tokens_seen": 1485057024 + }, + { + "epoch": 5.0, + "learning_rate": 0.0002778234704112337, + "loss": 2.8204, + "theoretical_loss": 3.5191752028337238, + "tokens_seen": 1485122560 + }, + { + "epoch": 5.0, + "learning_rate": 0.00027781344032096287, + "loss": 2.645, + "theoretical_loss": 3.5191614847117005, + "tokens_seen": 1485188096 + }, + { + "epoch": 5.0, + "learning_rate": 0.00027780341023069205, + "loss": 2.8198, + "theoretical_loss": 3.51914776736448, + "tokens_seen": 1485253632 + }, + { + "epoch": 5.0, + "learning_rate": 0.0002777933801404213, + "loss": 2.6427, + "theoretical_loss": 3.519134050791984, + "tokens_seen": 1485319168 + }, + { + "epoch": 5.0, + "learning_rate": 0.0002777833500501504, + "loss": 2.6776, + "theoretical_loss": 3.519120334994134, + "tokens_seen": 1485384704 + }, + { + "epoch": 5.0, + "learning_rate": 0.00027777331995987965, + "loss": 2.641, + "theoretical_loss": 3.5191066199708523, + "tokens_seen": 1485450240 + }, + { + "epoch": 5.0, + "learning_rate": 0.0002777632898696089, + "loss": 2.4597, + "theoretical_loss": 3.5190929057220615, + "tokens_seen": 1485515776 + }, + { + "epoch": 5.0, + "learning_rate": 0.000277753259779338, + "loss": 2.6771, + "theoretical_loss": 3.5190791922476823, + "tokens_seen": 1485581312 + }, + { + "epoch": 5.0, + "learning_rate": 0.00027774322968906725, + "loss": 2.6651, + "theoretical_loss": 3.5190654795476384, + "tokens_seen": 1485646848 + }, + { + "epoch": 5.0, + "learning_rate": 0.00027773319959879637, + "loss": 2.4468, + "theoretical_loss": 3.5190517676218516, + "tokens_seen": 1485712384 + }, + { + "epoch": 5.0, + "learning_rate": 0.0002777231695085256, + "loss": 2.6605, + "theoretical_loss": 3.5190380564702433, + "tokens_seen": 1485777920 + }, + { + "epoch": 5.0, + "learning_rate": 0.0002777131394182548, + "loss": 2.7455, + "theoretical_loss": 3.519024346092736, + "tokens_seen": 1485843456 + }, + { + "epoch": 5.0, + "learning_rate": 0.00027770310932798397, + "loss": 2.6425, + "theoretical_loss": 3.519010636489252, + "tokens_seen": 1485908992 + }, + { + "epoch": 5.0, + "learning_rate": 0.00027769307923771315, + "loss": 2.6573, + "theoretical_loss": 3.518996927659714, + "tokens_seen": 1485974528 + }, + { + "epoch": 5.0, + "objective/train/docs_used": 1672845, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.550705909729004, + "objective/train/theoretical_loss": 3.5189866465454154, + "objective/train/tokens_used": 1506483680, + "theoretical_loss": 3.5189866465454154, + "tokens_seen": 1486023680 + }, + { + "epoch": 5.0, + "learning_rate": 0.00027768304914744233, + "loss": 2.6352, + "theoretical_loss": 3.518983219604043, + "tokens_seen": 1486040064 + }, + { + "epoch": 5.0, + "learning_rate": 0.0002776730190571715, + "loss": 2.702, + "theoretical_loss": 3.518969512322162, + "tokens_seen": 1486105600 + }, + { + "epoch": 5.0, + "learning_rate": 0.00027766298896690075, + "loss": 2.739, + "theoretical_loss": 3.518955805813993, + "tokens_seen": 1486171136 + }, + { + "epoch": 5.0, + "learning_rate": 0.0002776529588766299, + "loss": 2.5802, + "theoretical_loss": 3.518942100079458, + "tokens_seen": 1486236672 + }, + { + "epoch": 5.0, + "learning_rate": 0.0002776429287863591, + "loss": 2.5728, + "theoretical_loss": 3.5189283951184795, + "tokens_seen": 1486302208 + }, + { + "epoch": 5.0, + "learning_rate": 0.0002776328986960883, + "loss": 2.5793, + "theoretical_loss": 3.5189146909309796, + "tokens_seen": 1486367744 + }, + { + "epoch": 5.0, + "learning_rate": 0.0002776228686058175, + "loss": 2.6295, + "theoretical_loss": 3.5189009875168806, + "tokens_seen": 1486433280 + }, + { + "epoch": 5.0, + "learning_rate": 0.00027761283851554665, + "loss": 2.6874, + "theoretical_loss": 3.518887284876105, + "tokens_seen": 1486498816 + }, + { + "epoch": 5.0, + "learning_rate": 0.00027760280842527584, + "loss": 2.6939, + "theoretical_loss": 3.518873583008574, + "tokens_seen": 1486564352 + }, + { + "epoch": 5.0, + "learning_rate": 0.000277592778335005, + "loss": 2.7765, + "theoretical_loss": 3.518859881914212, + "tokens_seen": 1486629888 + }, + { + "epoch": 5.0, + "learning_rate": 0.00027758274824473425, + "loss": 2.6764, + "theoretical_loss": 3.518846181592939, + "tokens_seen": 1486695424 + }, + { + "epoch": 5.0, + "learning_rate": 0.0002775727181544634, + "loss": 2.7328, + "theoretical_loss": 3.518832482044678, + "tokens_seen": 1486760960 + }, + { + "epoch": 5.0, + "learning_rate": 0.0002775626880641926, + "loss": 2.4887, + "theoretical_loss": 3.518818783269352, + "tokens_seen": 1486826496 + }, + { + "epoch": 5.0, + "learning_rate": 0.00027755265797392174, + "loss": 2.6795, + "theoretical_loss": 3.5188050852668833, + "tokens_seen": 1486892032 + }, + { + "epoch": 5.0, + "learning_rate": 0.000277542627883651, + "loss": 2.791, + "theoretical_loss": 3.5187913880371937, + "tokens_seen": 1486957568 + }, + { + "epoch": 5.0, + "learning_rate": 0.00027753259779338016, + "loss": 2.6841, + "theoretical_loss": 3.5187776915802056, + "tokens_seen": 1487023104 + }, + { + "epoch": 5.0, + "learning_rate": 0.00027752256770310934, + "loss": 2.5651, + "theoretical_loss": 3.518763995895841, + "tokens_seen": 1487088640 + }, + { + "epoch": 5.0, + "learning_rate": 0.0002775125376128385, + "loss": 2.7094, + "theoretical_loss": 3.5187503009840233, + "tokens_seen": 1487154176 + }, + { + "epoch": 5.0, + "learning_rate": 0.0002775025075225677, + "loss": 2.6081, + "theoretical_loss": 3.5187366068446746, + "tokens_seen": 1487219712 + }, + { + "epoch": 5.0, + "learning_rate": 0.0002774924774322969, + "loss": 2.62, + "theoretical_loss": 3.518722913477716, + "tokens_seen": 1487285248 + }, + { + "epoch": 5.0, + "learning_rate": 0.0002774824473420261, + "loss": 2.5542, + "theoretical_loss": 3.518709220883072, + "tokens_seen": 1487350784 + }, + { + "epoch": 5.0, + "learning_rate": 0.00027747241725175524, + "loss": 2.6896, + "theoretical_loss": 3.518695529060663, + "tokens_seen": 1487416320 + }, + { + "epoch": 5.0, + "learning_rate": 0.0002774623871614845, + "loss": 2.788, + "theoretical_loss": 3.518681838010413, + "tokens_seen": 1487481856 + }, + { + "epoch": 5.0, + "learning_rate": 0.00027745235707121366, + "loss": 2.6174, + "theoretical_loss": 3.5186681477322437, + "tokens_seen": 1487547392 + }, + { + "epoch": 5.0, + "learning_rate": 0.00027744232698094284, + "loss": 2.8734, + "theoretical_loss": 3.5186544582260773, + "tokens_seen": 1487612928 + }, + { + "epoch": 5.0, + "objective/train/docs_used": 1677864, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8672678470611572, + "objective/train/theoretical_loss": 3.518644191603033, + "objective/train/tokens_used": 1508122080, + "theoretical_loss": 3.518644191603033, + "tokens_seen": 1487662080 + }, + { + "epoch": 5.0, + "learning_rate": 0.000277432296890672, + "loss": 2.636, + "theoretical_loss": 3.518640769491837, + "tokens_seen": 1487678464 + }, + { + "epoch": 5.0, + "learning_rate": 0.0002774222668004012, + "loss": 2.7245, + "theoretical_loss": 3.5186270815294445, + "tokens_seen": 1487744000 + }, + { + "epoch": 5.0, + "learning_rate": 0.0002774122367101304, + "loss": 2.6811, + "theoretical_loss": 3.5186133943388227, + "tokens_seen": 1487809536 + }, + { + "epoch": 5.0, + "learning_rate": 0.0002774022066198596, + "loss": 2.6277, + "theoretical_loss": 3.5185997079198943, + "tokens_seen": 1487875072 + }, + { + "epoch": 5.0, + "learning_rate": 0.00027739217652958875, + "loss": 2.4533, + "theoretical_loss": 3.518586022272581, + "tokens_seen": 1487940608 + }, + { + "epoch": 5.0, + "learning_rate": 0.000277382146439318, + "loss": 2.6915, + "theoretical_loss": 3.5185723373968063, + "tokens_seen": 1488006144 + }, + { + "epoch": 5.0, + "learning_rate": 0.0002773721163490471, + "loss": 2.6598, + "theoretical_loss": 3.5185586532924917, + "tokens_seen": 1488071680 + }, + { + "epoch": 5.0, + "learning_rate": 0.00027736208625877634, + "loss": 2.5223, + "theoretical_loss": 3.518544969959561, + "tokens_seen": 1488137216 + }, + { + "epoch": 5.0, + "learning_rate": 0.0002773520561685055, + "loss": 2.6451, + "theoretical_loss": 3.518531287397936, + "tokens_seen": 1488202752 + }, + { + "epoch": 5.0, + "learning_rate": 0.0002773420260782347, + "loss": 2.6403, + "theoretical_loss": 3.5185176056075393, + "tokens_seen": 1488268288 + }, + { + "epoch": 5.0, + "learning_rate": 0.0002773319959879639, + "loss": 2.7058, + "theoretical_loss": 3.518503924588293, + "tokens_seen": 1488333824 + }, + { + "epoch": 5.0, + "learning_rate": 0.00027732196589769307, + "loss": 2.7885, + "theoretical_loss": 3.5184902443401205, + "tokens_seen": 1488399360 + }, + { + "epoch": 5.0, + "learning_rate": 0.00027731193580742225, + "loss": 2.7033, + "theoretical_loss": 3.518476564862944, + "tokens_seen": 1488464896 + }, + { + "epoch": 5.0, + "learning_rate": 0.0002773019057171515, + "loss": 2.6601, + "theoretical_loss": 3.5184628861566867, + "tokens_seen": 1488530432 + }, + { + "epoch": 5.0, + "learning_rate": 0.0002772918756268806, + "loss": 2.7296, + "theoretical_loss": 3.51844920822127, + "tokens_seen": 1488595968 + }, + { + "epoch": 5.0, + "learning_rate": 0.00027728184553660985, + "loss": 2.7617, + "theoretical_loss": 3.5184355310566175, + "tokens_seen": 1488661504 + }, + { + "epoch": 5.0, + "learning_rate": 0.00027727181544633903, + "loss": 2.736, + "theoretical_loss": 3.518421854662652, + "tokens_seen": 1488727040 + }, + { + "epoch": 5.0, + "learning_rate": 0.0002772617853560682, + "loss": 2.7205, + "theoretical_loss": 3.5184081790392945, + "tokens_seen": 1488792576 + }, + { + "epoch": 5.0, + "learning_rate": 0.0002772517552657974, + "loss": 2.8174, + "theoretical_loss": 3.51839450418647, + "tokens_seen": 1488858112 + }, + { + "epoch": 5.0, + "learning_rate": 0.00027724172517552657, + "loss": 2.6738, + "theoretical_loss": 3.5183808301040997, + "tokens_seen": 1488923648 + }, + { + "epoch": 5.0, + "learning_rate": 0.00027723169508525575, + "loss": 2.6079, + "theoretical_loss": 3.518367156792106, + "tokens_seen": 1488989184 + }, + { + "epoch": 5.0, + "learning_rate": 0.000277221664994985, + "loss": 2.7517, + "theoretical_loss": 3.518353484250413, + "tokens_seen": 1489054720 + }, + { + "epoch": 5.0, + "learning_rate": 0.0002772116349047141, + "loss": 2.6961, + "theoretical_loss": 3.518339812478942, + "tokens_seen": 1489120256 + }, + { + "epoch": 5.0, + "learning_rate": 0.00027720160481444335, + "loss": 2.6361, + "theoretical_loss": 3.5183261414776172, + "tokens_seen": 1489185792 + }, + { + "epoch": 5.0, + "learning_rate": 0.0002771915747241725, + "loss": 2.7121, + "theoretical_loss": 3.51831247124636, + "tokens_seen": 1489251328 + }, + { + "epoch": 5.0, + "objective/train/docs_used": 1682798, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.633384943008423, + "objective/train/theoretical_loss": 3.5183022190782274, + "objective/train/tokens_used": 1509760480, + "theoretical_loss": 3.5183022190782274, + "tokens_seen": 1489300480 + }, + { + "epoch": 5.0, + "learning_rate": 0.0002771815446339017, + "loss": 2.6501, + "theoretical_loss": 3.518298801785093, + "tokens_seen": 1489316864 + }, + { + "epoch": 5.0, + "learning_rate": 0.0002771715145436309, + "loss": 2.6853, + "theoretical_loss": 3.51828513309374, + "tokens_seen": 1489382400 + }, + { + "epoch": 5.0, + "learning_rate": 0.0002771614844533601, + "loss": 2.7524, + "theoretical_loss": 3.5182714651722238, + "tokens_seen": 1489447936 + }, + { + "epoch": 5.0, + "learning_rate": 0.00027715145436308926, + "loss": 2.6166, + "theoretical_loss": 3.5182577980204663, + "tokens_seen": 1489513472 + }, + { + "epoch": 5.0, + "learning_rate": 0.0002771414242728185, + "loss": 2.7881, + "theoretical_loss": 3.5182441316383906, + "tokens_seen": 1489579008 + }, + { + "epoch": 5.0, + "learning_rate": 0.0002771313941825476, + "loss": 2.5965, + "theoretical_loss": 3.5182304660259196, + "tokens_seen": 1489644544 + }, + { + "epoch": 5.0, + "learning_rate": 0.00027712136409227685, + "loss": 2.7613, + "theoretical_loss": 3.518216801182976, + "tokens_seen": 1489710080 + }, + { + "epoch": 5.0, + "learning_rate": 0.000277111334002006, + "loss": 2.791, + "theoretical_loss": 3.5182031371094826, + "tokens_seen": 1489775616 + }, + { + "epoch": 5.0, + "learning_rate": 0.0002771013039117352, + "loss": 2.5291, + "theoretical_loss": 3.5181894738053625, + "tokens_seen": 1489841152 + }, + { + "epoch": 5.0, + "learning_rate": 0.0002770912738214644, + "loss": 2.6356, + "theoretical_loss": 3.518175811270538, + "tokens_seen": 1489906688 + }, + { + "epoch": 5.0, + "learning_rate": 0.0002770812437311936, + "loss": 2.7813, + "theoretical_loss": 3.518162149504933, + "tokens_seen": 1489972224 + }, + { + "epoch": 5.0, + "learning_rate": 0.00027707121364092276, + "loss": 2.685, + "theoretical_loss": 3.5181484885084693, + "tokens_seen": 1490037760 + }, + { + "epoch": 5.0, + "learning_rate": 0.00027706118355065194, + "loss": 2.5715, + "theoretical_loss": 3.5181348282810703, + "tokens_seen": 1490103296 + }, + { + "epoch": 5.0, + "learning_rate": 0.0002770511534603811, + "loss": 2.4107, + "theoretical_loss": 3.518121168822658, + "tokens_seen": 1490168832 + }, + { + "epoch": 5.0, + "learning_rate": 0.00027704112337011036, + "loss": 2.5857, + "theoretical_loss": 3.5181075101331567, + "tokens_seen": 1490234368 + }, + { + "epoch": 5.0, + "learning_rate": 0.0002770310932798395, + "loss": 2.7753, + "theoretical_loss": 3.5180938522124885, + "tokens_seen": 1490299904 + }, + { + "epoch": 5.0, + "learning_rate": 0.0002770210631895687, + "loss": 2.6862, + "theoretical_loss": 3.518080195060576, + "tokens_seen": 1490365440 + }, + { + "epoch": 5.0, + "learning_rate": 0.0002770110330992979, + "loss": 2.7125, + "theoretical_loss": 3.518066538677343, + "tokens_seen": 1490430976 + }, + { + "epoch": 5.0, + "learning_rate": 0.0002770010030090271, + "loss": 2.6126, + "theoretical_loss": 3.518052883062712, + "tokens_seen": 1490496512 + }, + { + "epoch": 5.0, + "learning_rate": 0.0002769909729187563, + "loss": 2.5852, + "theoretical_loss": 3.5180392282166055, + "tokens_seen": 1490562048 + }, + { + "epoch": 5.0, + "learning_rate": 0.00027698094282848544, + "loss": 2.5653, + "theoretical_loss": 3.5180255741389477, + "tokens_seen": 1490627584 + }, + { + "epoch": 5.0, + "learning_rate": 0.0002769709127382147, + "loss": 2.7681, + "theoretical_loss": 3.51801192082966, + "tokens_seen": 1490693120 + }, + { + "epoch": 5.0, + "learning_rate": 0.00027696088264794386, + "loss": 2.6062, + "theoretical_loss": 3.5179982682886664, + "tokens_seen": 1490758656 + }, + { + "epoch": 5.0, + "learning_rate": 0.00027695085255767304, + "loss": 2.831, + "theoretical_loss": 3.5179846165158897, + "tokens_seen": 1490824192 + }, + { + "epoch": 5.0, + "learning_rate": 0.0002769408224674022, + "loss": 2.7401, + "theoretical_loss": 3.5179709655112528, + "tokens_seen": 1490889728 + }, + { + "epoch": 5.0, + "objective/train/docs_used": 1687806, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.763873815536499, + "objective/train/theoretical_loss": 3.517960727761821, + "objective/train/tokens_used": 1511398880, + "theoretical_loss": 3.517960727761821, + "tokens_seen": 1490938880 + }, + { + "epoch": 5.0, + "learning_rate": 0.0002769307923771314, + "loss": 2.7617, + "theoretical_loss": 3.5179573152746793, + "tokens_seen": 1490955264 + }, + { + "epoch": 5.0, + "learning_rate": 0.0002769207622868606, + "loss": 2.5945, + "theoretical_loss": 3.517943665806091, + "tokens_seen": 1491020800 + }, + { + "epoch": 5.0, + "learning_rate": 0.0002769107321965898, + "loss": 2.783, + "theoretical_loss": 3.517930017105412, + "tokens_seen": 1491086336 + }, + { + "epoch": 5.0, + "learning_rate": 0.00027690070210631895, + "loss": 2.6221, + "theoretical_loss": 3.5179163691725646, + "tokens_seen": 1491151872 + }, + { + "epoch": 5.0, + "learning_rate": 0.0002768906720160482, + "loss": 2.5724, + "theoretical_loss": 3.517902722007473, + "tokens_seen": 1491217408 + }, + { + "epoch": 5.0, + "learning_rate": 0.0002768806419257773, + "loss": 2.7295, + "theoretical_loss": 3.517889075610059, + "tokens_seen": 1491282944 + }, + { + "epoch": 5.0, + "learning_rate": 0.00027687061183550655, + "loss": 2.7662, + "theoretical_loss": 3.5178754299802457, + "tokens_seen": 1491348480 + }, + { + "epoch": 5.0, + "learning_rate": 0.0002768605817452357, + "loss": 2.7277, + "theoretical_loss": 3.517861785117957, + "tokens_seen": 1491414016 + }, + { + "epoch": 5.0, + "learning_rate": 0.0002768505516549649, + "loss": 2.7318, + "theoretical_loss": 3.5178481410231166, + "tokens_seen": 1491479552 + }, + { + "epoch": 5.0, + "learning_rate": 0.0002768405215646941, + "loss": 2.7809, + "theoretical_loss": 3.517834497695646, + "tokens_seen": 1491545088 + }, + { + "epoch": 5.0, + "learning_rate": 0.00027683049147442327, + "loss": 2.6275, + "theoretical_loss": 3.517820855135469, + "tokens_seen": 1491610624 + }, + { + "epoch": 5.0, + "learning_rate": 0.00027682046138415245, + "loss": 2.5591, + "theoretical_loss": 3.5178072133425093, + "tokens_seen": 1491676160 + }, + { + "epoch": 5.0, + "learning_rate": 0.0002768104312938817, + "loss": 2.908, + "theoretical_loss": 3.517793572316689, + "tokens_seen": 1491741696 + }, + { + "epoch": 5.0, + "learning_rate": 0.0002768004012036108, + "loss": 2.5793, + "theoretical_loss": 3.517779932057932, + "tokens_seen": 1491807232 + }, + { + "epoch": 5.0, + "learning_rate": 0.00027679037111334005, + "loss": 2.6962, + "theoretical_loss": 3.5177662925661615, + "tokens_seen": 1491872768 + }, + { + "epoch": 5.0, + "learning_rate": 0.00027678034102306923, + "loss": 2.4658, + "theoretical_loss": 3.5177526538413, + "tokens_seen": 1491938304 + }, + { + "epoch": 5.0, + "learning_rate": 0.0002767703109327984, + "loss": 2.5116, + "theoretical_loss": 3.5177390158832713, + "tokens_seen": 1492003840 + }, + { + "epoch": 5.0, + "learning_rate": 0.0002767602808425276, + "loss": 2.6111, + "theoretical_loss": 3.517725378691998, + "tokens_seen": 1492069376 + }, + { + "epoch": 5.0, + "learning_rate": 0.00027675025075225677, + "loss": 2.6248, + "theoretical_loss": 3.5177117422674047, + "tokens_seen": 1492134912 + }, + { + "epoch": 5.0, + "learning_rate": 0.00027674022066198595, + "loss": 2.7078, + "theoretical_loss": 3.517698106609413, + "tokens_seen": 1492200448 + }, + { + "epoch": 5.0, + "learning_rate": 0.0002767301905717152, + "loss": 2.6051, + "theoretical_loss": 3.5176844717179474, + "tokens_seen": 1492265984 + }, + { + "epoch": 5.0, + "learning_rate": 0.0002767201604814443, + "loss": 2.6811, + "theoretical_loss": 3.51767083759293, + "tokens_seen": 1492331520 + }, + { + "epoch": 5.0, + "learning_rate": 0.00027671013039117355, + "loss": 2.5908, + "theoretical_loss": 3.5176572042342853, + "tokens_seen": 1492397056 + }, + { + "epoch": 5.0, + "learning_rate": 0.0002767001003009027, + "loss": 2.817, + "theoretical_loss": 3.517643571641935, + "tokens_seen": 1492462592 + }, + { + "epoch": 5.0, + "learning_rate": 0.0002766900702106319, + "loss": 2.8072, + "theoretical_loss": 3.517629939815804, + "tokens_seen": 1492528128 + }, + { + "epoch": 5.0, + "objective/train/docs_used": 1692905, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.619441032409668, + "objective/train/theoretical_loss": 3.5176197164489906, + "objective/train/tokens_used": 1513037280, + "theoretical_loss": 3.5176197164489906, + "tokens_seen": 1492577280 + }, + { + "epoch": 5.0, + "learning_rate": 0.0002766800401203611, + "loss": 2.6709, + "theoretical_loss": 3.517616308755815, + "tokens_seen": 1492593664 + }, + { + "epoch": 5.0, + "learning_rate": 0.0002766700100300903, + "loss": 2.7228, + "theoretical_loss": 3.5176026784618912, + "tokens_seen": 1492659200 + }, + { + "epoch": 5.0, + "learning_rate": 0.00027665997993981946, + "loss": 2.8684, + "theoretical_loss": 3.5175890489339556, + "tokens_seen": 1492724736 + }, + { + "epoch": 5.0, + "learning_rate": 0.0002766499498495487, + "loss": 2.6712, + "theoretical_loss": 3.517575420171932, + "tokens_seen": 1492790272 + }, + { + "epoch": 5.0, + "learning_rate": 0.0002766399197592778, + "loss": 2.5039, + "theoretical_loss": 3.5175617921757434, + "tokens_seen": 1492855808 + }, + { + "epoch": 5.0, + "learning_rate": 0.00027662988966900705, + "loss": 2.7792, + "theoretical_loss": 3.517548164945313, + "tokens_seen": 1492921344 + }, + { + "epoch": 5.0, + "learning_rate": 0.0002766198595787362, + "loss": 2.8762, + "theoretical_loss": 3.517534538480565, + "tokens_seen": 1492986880 + }, + { + "epoch": 5.0, + "learning_rate": 0.0002766098294884654, + "loss": 2.7011, + "theoretical_loss": 3.517520912781422, + "tokens_seen": 1493052416 + }, + { + "epoch": 5.0, + "learning_rate": 0.0002765997993981946, + "loss": 2.6816, + "theoretical_loss": 3.517507287847808, + "tokens_seen": 1493117952 + }, + { + "epoch": 5.0, + "learning_rate": 0.0002765897693079238, + "loss": 2.6194, + "theoretical_loss": 3.517493663679646, + "tokens_seen": 1493183488 + }, + { + "epoch": 5.0, + "learning_rate": 0.00027657973921765296, + "loss": 2.918, + "theoretical_loss": 3.517480040276859, + "tokens_seen": 1493249024 + }, + { + "epoch": 5.0, + "learning_rate": 0.00027656970912738214, + "loss": 2.5906, + "theoretical_loss": 3.517466417639371, + "tokens_seen": 1493314560 + }, + { + "epoch": 5.0, + "learning_rate": 0.0002765596790371113, + "loss": 2.7555, + "theoretical_loss": 3.517452795767105, + "tokens_seen": 1493380096 + }, + { + "epoch": 5.0, + "learning_rate": 0.00027654964894684056, + "loss": 2.8005, + "theoretical_loss": 3.5174391746599856, + "tokens_seen": 1493445632 + }, + { + "epoch": 5.0, + "learning_rate": 0.0002765396188565697, + "loss": 2.5952, + "theoretical_loss": 3.517425554317935, + "tokens_seen": 1493511168 + }, + { + "epoch": 5.0, + "learning_rate": 0.0002765295887662989, + "loss": 2.6816, + "theoretical_loss": 3.5174119347408768, + "tokens_seen": 1493576704 + }, + { + "epoch": 5.0, + "learning_rate": 0.00027651955867602805, + "loss": 2.6175, + "theoretical_loss": 3.5173983159287348, + "tokens_seen": 1493642240 + }, + { + "epoch": 5.0, + "learning_rate": 0.0002765095285857573, + "loss": 2.6861, + "theoretical_loss": 3.5173846978814316, + "tokens_seen": 1493707776 + }, + { + "epoch": 5.0, + "learning_rate": 0.00027649949849548646, + "loss": 2.6073, + "theoretical_loss": 3.5173710805988927, + "tokens_seen": 1493773312 + }, + { + "epoch": 5.0, + "learning_rate": 0.00027648946840521564, + "loss": 2.5486, + "theoretical_loss": 3.51735746408104, + "tokens_seen": 1493838848 + }, + { + "epoch": 5.0, + "learning_rate": 0.0002764794383149448, + "loss": 2.7331, + "theoretical_loss": 3.517343848327797, + "tokens_seen": 1493904384 + }, + { + "epoch": 5.0, + "learning_rate": 0.00027646940822467406, + "loss": 2.6075, + "theoretical_loss": 3.517330233339088, + "tokens_seen": 1493969920 + }, + { + "epoch": 5.0, + "learning_rate": 0.0002764593781344032, + "loss": 2.7378, + "theoretical_loss": 3.517316619114836, + "tokens_seen": 1494035456 + }, + { + "epoch": 5.0, + "learning_rate": 0.0002764493480441324, + "loss": 2.7399, + "theoretical_loss": 3.517303005654964, + "tokens_seen": 1494100992 + }, + { + "epoch": 5.0, + "learning_rate": 0.00027643931795386155, + "loss": 2.465, + "theoretical_loss": 3.517289392959397, + "tokens_seen": 1494166528 + }, + { + "epoch": 5.0, + "objective/train/docs_used": 1697937, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4400620460510254, + "objective/train/theoretical_loss": 3.5172791839392508, + "objective/train/tokens_used": 1514675680, + "theoretical_loss": 3.5172791839392508, + "tokens_seen": 1494215680 + }, + { + "epoch": 5.0, + "learning_rate": 0.0002764292878635908, + "loss": 2.6827, + "theoretical_loss": 3.517275781028058, + "tokens_seen": 1494232064 + }, + { + "epoch": 5.0, + "learning_rate": 0.00027641925777331997, + "loss": 2.6565, + "theoretical_loss": 3.51726216986087, + "tokens_seen": 1494297600 + }, + { + "epoch": 5.0, + "learning_rate": 0.00027640922768304915, + "loss": 2.4964, + "theoretical_loss": 3.517248559457757, + "tokens_seen": 1494363136 + }, + { + "epoch": 5.0, + "learning_rate": 0.00027639919759277833, + "loss": 2.8277, + "theoretical_loss": 3.5172349498186426, + "tokens_seen": 1494428672 + }, + { + "epoch": 5.0, + "learning_rate": 0.0002763891675025075, + "loss": 2.545, + "theoretical_loss": 3.5172213409434505, + "tokens_seen": 1494494208 + }, + { + "epoch": 5.0, + "learning_rate": 0.0002763791374122367, + "loss": 2.6756, + "theoretical_loss": 3.517207732832104, + "tokens_seen": 1494559744 + }, + { + "epoch": 5.0, + "learning_rate": 0.0002763691073219659, + "loss": 2.6207, + "theoretical_loss": 3.5171941254845267, + "tokens_seen": 1494625280 + }, + { + "epoch": 5.0, + "learning_rate": 0.00027635907723169505, + "loss": 2.6007, + "theoretical_loss": 3.517180518900643, + "tokens_seen": 1494690816 + }, + { + "epoch": 5.0, + "learning_rate": 0.0002763490471414243, + "loss": 2.6077, + "theoretical_loss": 3.517166913080376, + "tokens_seen": 1494756352 + }, + { + "epoch": 5.0, + "learning_rate": 0.0002763390170511534, + "loss": 2.7704, + "theoretical_loss": 3.5171533080236497, + "tokens_seen": 1494821888 + }, + { + "epoch": 5.0, + "learning_rate": 0.00027632898696088265, + "loss": 2.7405, + "theoretical_loss": 3.517139703730387, + "tokens_seen": 1494887424 + }, + { + "epoch": 5.0, + "learning_rate": 0.00027631895687061183, + "loss": 2.606, + "theoretical_loss": 3.5171261002005125, + "tokens_seen": 1494952960 + }, + { + "epoch": 5.0, + "learning_rate": 0.000276308926780341, + "loss": 2.5175, + "theoretical_loss": 3.517112497433949, + "tokens_seen": 1495018496 + }, + { + "epoch": 5.0, + "learning_rate": 0.0002762988966900702, + "loss": 2.661, + "theoretical_loss": 3.517098895430621, + "tokens_seen": 1495084032 + }, + { + "epoch": 5.0, + "learning_rate": 0.00027628886659979943, + "loss": 2.6579, + "theoretical_loss": 3.5170852941904522, + "tokens_seen": 1495149568 + }, + { + "epoch": 5.0, + "learning_rate": 0.00027627883650952856, + "loss": 2.5546, + "theoretical_loss": 3.5170716937133655, + "tokens_seen": 1495215104 + }, + { + "epoch": 5.0, + "learning_rate": 0.0002762688064192578, + "loss": 2.6988, + "theoretical_loss": 3.5170580939992853, + "tokens_seen": 1495280640 + }, + { + "epoch": 5.0, + "learning_rate": 0.00027625877632898697, + "loss": 2.7503, + "theoretical_loss": 3.5170444950481357, + "tokens_seen": 1495346176 + }, + { + "epoch": 5.0, + "learning_rate": 0.00027624874623871615, + "loss": 2.5672, + "theoretical_loss": 3.51703089685984, + "tokens_seen": 1495411712 + }, + { + "epoch": 5.0, + "learning_rate": 0.0002762387161484454, + "loss": 2.8523, + "theoretical_loss": 3.517017299434322, + "tokens_seen": 1495477248 + }, + { + "epoch": 5.0, + "learning_rate": 0.0002762286860581745, + "loss": 2.598, + "theoretical_loss": 3.517003702771505, + "tokens_seen": 1495542784 + }, + { + "epoch": 5.0, + "learning_rate": 0.00027621865596790375, + "loss": 2.5702, + "theoretical_loss": 3.5169901068713134, + "tokens_seen": 1495608320 + }, + { + "epoch": 5.0, + "learning_rate": 0.0002762086258776329, + "loss": 2.7542, + "theoretical_loss": 3.516976511733671, + "tokens_seen": 1495673856 + }, + { + "epoch": 5.0, + "learning_rate": 0.0002761985957873621, + "loss": 2.7126, + "theoretical_loss": 3.516962917358502, + "tokens_seen": 1495739392 + }, + { + "epoch": 5.0, + "learning_rate": 0.0002761885656970913, + "loss": 2.6053, + "theoretical_loss": 3.516949323745729, + "tokens_seen": 1495804928 + }, + { + "epoch": 5.0, + "objective/train/docs_used": 1700816, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9013986587524414, + "objective/train/theoretical_loss": 3.5169391290364267, + "objective/train/tokens_used": 1516314080, + "theoretical_loss": 3.5169391290364267, + "tokens_seen": 1495854080 + }, + { + "epoch": 5.0, + "learning_rate": 0.0002761785356068205, + "loss": 2.8365, + "theoretical_loss": 3.516935730895277, + "tokens_seen": 1495870464 + }, + { + "epoch": 5.0, + "learning_rate": 0.00027616850551654966, + "loss": 2.5379, + "theoretical_loss": 3.51692213880707, + "tokens_seen": 1495936000 + }, + { + "epoch": 5.0, + "learning_rate": 0.0002761584754262789, + "loss": 2.7247, + "theoretical_loss": 3.51690854748103, + "tokens_seen": 1496001536 + }, + { + "epoch": 5.0, + "learning_rate": 0.000276148445336008, + "loss": 2.5445, + "theoretical_loss": 3.516894956917083, + "tokens_seen": 1496067072 + }, + { + "epoch": 5.0, + "learning_rate": 0.00027613841524573725, + "loss": 2.6178, + "theoretical_loss": 3.516881367115152, + "tokens_seen": 1496132608 + }, + { + "epoch": 5.0, + "learning_rate": 0.0002761283851554664, + "loss": 2.7579, + "theoretical_loss": 3.5168677780751616, + "tokens_seen": 1496198144 + }, + { + "epoch": 5.0, + "learning_rate": 0.0002761183550651956, + "loss": 2.7081, + "theoretical_loss": 3.5168541897970345, + "tokens_seen": 1496263680 + }, + { + "epoch": 5.0, + "learning_rate": 0.0002761083249749248, + "loss": 2.5498, + "theoretical_loss": 3.516840602280695, + "tokens_seen": 1496329216 + }, + { + "epoch": 5.0, + "learning_rate": 0.000276098294884654, + "loss": 2.5387, + "theoretical_loss": 3.5168270155260677, + "tokens_seen": 1496394752 + }, + { + "epoch": 5.0, + "learning_rate": 0.00027608826479438316, + "loss": 2.6254, + "theoretical_loss": 3.5168134295330757, + "tokens_seen": 1496460288 + }, + { + "epoch": 5.0, + "learning_rate": 0.00027607823470411234, + "loss": 2.6312, + "theoretical_loss": 3.5167998443016435, + "tokens_seen": 1496525824 + }, + { + "epoch": 5.0, + "learning_rate": 0.0002760682046138415, + "loss": 2.6842, + "theoretical_loss": 3.516786259831695, + "tokens_seen": 1496591360 + }, + { + "epoch": 5.0, + "learning_rate": 0.00027605817452357076, + "loss": 2.6796, + "theoretical_loss": 3.516772676123154, + "tokens_seen": 1496656896 + }, + { + "epoch": 5.0, + "learning_rate": 0.0002760481444332999, + "loss": 2.6596, + "theoretical_loss": 3.5167590931759447, + "tokens_seen": 1496722432 + }, + { + "epoch": 5.0, + "learning_rate": 0.0002760381143430291, + "loss": 2.7718, + "theoretical_loss": 3.5167455109899906, + "tokens_seen": 1496787968 + }, + { + "epoch": 5.0, + "learning_rate": 0.00027602808425275825, + "loss": 2.7429, + "theoretical_loss": 3.516731929565216, + "tokens_seen": 1496853504 + }, + { + "epoch": 5.0, + "learning_rate": 0.0002760180541624875, + "loss": 2.7515, + "theoretical_loss": 3.5167183489015454, + "tokens_seen": 1496919040 + }, + { + "epoch": 5.0, + "learning_rate": 0.00027600802407221666, + "loss": 2.7387, + "theoretical_loss": 3.5167047689989026, + "tokens_seen": 1496984576 + }, + { + "epoch": 5.0, + "learning_rate": 0.00027599799398194584, + "loss": 2.6507, + "theoretical_loss": 3.5166911898572106, + "tokens_seen": 1497050112 + }, + { + "epoch": 5.0, + "learning_rate": 0.000275987963891675, + "loss": 2.7042, + "theoretical_loss": 3.5166776114763953, + "tokens_seen": 1497115648 + }, + { + "epoch": 5.0, + "learning_rate": 0.00027597793380140426, + "loss": 2.6221, + "theoretical_loss": 3.516664033856379, + "tokens_seen": 1497181184 + }, + { + "epoch": 5.0, + "learning_rate": 0.0002759679037111334, + "loss": 2.7729, + "theoretical_loss": 3.5166504569970867, + "tokens_seen": 1497246720 + }, + { + "epoch": 5.0, + "learning_rate": 0.0002759578736208626, + "loss": 2.7716, + "theoretical_loss": 3.5166368808984423, + "tokens_seen": 1497312256 + }, + { + "epoch": 5.0, + "learning_rate": 0.00027594784353059175, + "loss": 2.6233, + "theoretical_loss": 3.51662330556037, + "tokens_seen": 1497377792 + }, + { + "epoch": 5.0, + "learning_rate": 0.000275937813440321, + "loss": 2.721, + "theoretical_loss": 3.516609730982794, + "tokens_seen": 1497443328 + }, + { + "epoch": 5.0, + "objective/train/docs_used": 1707842, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.416391372680664, + "objective/train/theoretical_loss": 3.5165995505486416, + "objective/train/tokens_used": 1517952480, + "theoretical_loss": 3.5165995505486416, + "tokens_seen": 1497492480 + }, + { + "epoch": 5.0, + "learning_rate": 0.00027592778335005017, + "loss": 2.5181, + "theoretical_loss": 3.516596157165638, + "tokens_seen": 1497508864 + }, + { + "epoch": 5.0, + "learning_rate": 0.00027591775325977935, + "loss": 2.5366, + "theoretical_loss": 3.5165825841088267, + "tokens_seen": 1497574400 + }, + { + "epoch": 5.0, + "learning_rate": 0.00027590772316950853, + "loss": 2.6365, + "theoretical_loss": 3.5165690118122837, + "tokens_seen": 1497639936 + }, + { + "epoch": 5.0, + "learning_rate": 0.0002758976930792377, + "loss": 2.67, + "theoretical_loss": 3.5165554402759334, + "tokens_seen": 1497705472 + }, + { + "epoch": 5.0, + "learning_rate": 0.0002758876629889669, + "loss": 2.5694, + "theoretical_loss": 3.5165418694996995, + "tokens_seen": 1497771008 + }, + { + "epoch": 5.0, + "learning_rate": 0.0002758776328986961, + "loss": 2.6292, + "theoretical_loss": 3.5165282994835065, + "tokens_seen": 1497836544 + }, + { + "epoch": 5.0, + "learning_rate": 0.00027586760280842525, + "loss": 2.4334, + "theoretical_loss": 3.5165147302272795, + "tokens_seen": 1497902080 + }, + { + "epoch": 5.0, + "learning_rate": 0.0002758575727181545, + "loss": 2.6236, + "theoretical_loss": 3.5165011617309414, + "tokens_seen": 1497967616 + }, + { + "epoch": 5.0, + "learning_rate": 0.0002758475426278836, + "loss": 2.4739, + "theoretical_loss": 3.516487593994417, + "tokens_seen": 1498033152 + }, + { + "epoch": 5.0, + "learning_rate": 0.00027583751253761285, + "loss": 2.5641, + "theoretical_loss": 3.51647402701763, + "tokens_seen": 1498098688 + }, + { + "epoch": 5.0, + "learning_rate": 0.00027582748244734203, + "loss": 2.6901, + "theoretical_loss": 3.516460460800505, + "tokens_seen": 1498164224 + }, + { + "epoch": 5.0, + "learning_rate": 0.0002758174523570712, + "loss": 2.6487, + "theoretical_loss": 3.5164468953429666, + "tokens_seen": 1498229760 + }, + { + "epoch": 5.0, + "learning_rate": 0.0002758074222668004, + "loss": 2.5901, + "theoretical_loss": 3.5164333306449382, + "tokens_seen": 1498295296 + }, + { + "epoch": 5.0, + "learning_rate": 0.00027579739217652963, + "loss": 2.5624, + "theoretical_loss": 3.5164197667063455, + "tokens_seen": 1498360832 + }, + { + "epoch": 5.0, + "learning_rate": 0.00027578736208625876, + "loss": 2.7026, + "theoretical_loss": 3.5164062035271106, + "tokens_seen": 1498426368 + }, + { + "epoch": 5.0, + "learning_rate": 0.000275777331995988, + "loss": 2.7093, + "theoretical_loss": 3.51639264110716, + "tokens_seen": 1498491904 + }, + { + "epoch": 5.0, + "learning_rate": 0.0002757673019057171, + "loss": 2.5889, + "theoretical_loss": 3.5163790794464163, + "tokens_seen": 1498557440 + }, + { + "epoch": 5.0, + "learning_rate": 0.00027575727181544635, + "loss": 2.5409, + "theoretical_loss": 3.516365518544805, + "tokens_seen": 1498622976 + }, + { + "epoch": 5.0, + "learning_rate": 0.00027574724172517554, + "loss": 2.592, + "theoretical_loss": 3.5163519584022493, + "tokens_seen": 1498688512 + }, + { + "epoch": 5.0, + "learning_rate": 0.0002757372116349047, + "loss": 2.6117, + "theoretical_loss": 3.5163383990186743, + "tokens_seen": 1498754048 + }, + { + "epoch": 5.0, + "learning_rate": 0.0002757271815446339, + "loss": 2.6196, + "theoretical_loss": 3.516324840394004, + "tokens_seen": 1498819584 + }, + { + "epoch": 5.0, + "learning_rate": 0.0002757171514543631, + "loss": 2.6298, + "theoretical_loss": 3.5163112825281635, + "tokens_seen": 1498885120 + }, + { + "epoch": 5.0, + "learning_rate": 0.00027570712136409226, + "loss": 2.6609, + "theoretical_loss": 3.516297725421076, + "tokens_seen": 1498950656 + }, + { + "epoch": 5.0, + "learning_rate": 0.0002756970912738215, + "loss": 2.7129, + "theoretical_loss": 3.5162841690726667, + "tokens_seen": 1499016192 + }, + { + "epoch": 5.0, + "learning_rate": 0.0002756870611835506, + "loss": 2.6601, + "theoretical_loss": 3.5162706134828596, + "tokens_seen": 1499081728 + }, + { + "epoch": 5.0, + "objective/train/docs_used": 1710663, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6549386978149414, + "objective/train/theoretical_loss": 3.5162604472882912, + "objective/train/tokens_used": 1519590880, + "theoretical_loss": 3.5162604472882912, + "tokens_seen": 1499130880 + }, + { + "epoch": 5.0, + "learning_rate": 0.00027567703109327986, + "loss": 2.6568, + "theoretical_loss": 3.516257058651579, + "tokens_seen": 1499147264 + }, + { + "epoch": 5.0, + "learning_rate": 0.000275667001003009, + "loss": 2.7675, + "theoretical_loss": 3.5162435045787497, + "tokens_seen": 1499212800 + }, + { + "epoch": 5.0, + "learning_rate": 0.0002756569709127382, + "loss": 2.6526, + "theoretical_loss": 3.5162299512642954, + "tokens_seen": 1499278336 + }, + { + "epoch": 5.0, + "learning_rate": 0.0002756469408224674, + "loss": 2.6224, + "theoretical_loss": 3.516216398708141, + "tokens_seen": 1499343872 + }, + { + "epoch": 5.0, + "learning_rate": 0.0002756369107321966, + "loss": 2.682, + "theoretical_loss": 3.516202846910212, + "tokens_seen": 1499409408 + }, + { + "epoch": 5.0, + "learning_rate": 0.00027562688064192576, + "loss": 2.7775, + "theoretical_loss": 3.516189295870431, + "tokens_seen": 1499474944 + }, + { + "epoch": 5.0, + "learning_rate": 0.000275616850551655, + "loss": 2.6704, + "theoretical_loss": 3.5161757455887233, + "tokens_seen": 1499540480 + }, + { + "epoch": 5.0, + "learning_rate": 0.0002756068204613841, + "loss": 2.8034, + "theoretical_loss": 3.5161621960650136, + "tokens_seen": 1499606016 + }, + { + "epoch": 5.0, + "learning_rate": 0.00027559679037111336, + "loss": 2.5323, + "theoretical_loss": 3.516148647299225, + "tokens_seen": 1499671552 + }, + { + "epoch": 5.0, + "learning_rate": 0.0002755867602808425, + "loss": 2.7179, + "theoretical_loss": 3.5161350992912843, + "tokens_seen": 1499737088 + }, + { + "epoch": 5.0, + "learning_rate": 0.0002755767301905717, + "loss": 2.5213, + "theoretical_loss": 3.516121552041114, + "tokens_seen": 1499802624 + }, + { + "epoch": 5.0, + "learning_rate": 0.0002755667001003009, + "loss": 2.5912, + "theoretical_loss": 3.51610800554864, + "tokens_seen": 1499868160 + }, + { + "epoch": 5.0, + "learning_rate": 0.0002755566700100301, + "loss": 2.6746, + "theoretical_loss": 3.5160944598137855, + "tokens_seen": 1499933696 + }, + { + "epoch": 5.0, + "learning_rate": 0.00027554663991975927, + "loss": 2.6959, + "theoretical_loss": 3.5160809148364764, + "tokens_seen": 1499999232 + }, + { + "epoch": 5.0, + "learning_rate": 0.00027553660982948845, + "loss": 2.5214, + "theoretical_loss": 3.5160673706166357, + "tokens_seen": 1500064768 + }, + { + "epoch": 5.0, + "learning_rate": 0.00027552657973921763, + "loss": 2.7854, + "theoretical_loss": 3.5160538271541895, + "tokens_seen": 1500130304 + }, + { + "epoch": 5.0, + "learning_rate": 0.00027551654964894686, + "loss": 2.6988, + "theoretical_loss": 3.516040284449061, + "tokens_seen": 1500195840 + }, + { + "epoch": 5.0, + "learning_rate": 0.00027550651955867604, + "loss": 2.6114, + "theoretical_loss": 3.516026742501176, + "tokens_seen": 1500261376 + }, + { + "epoch": 5.01, + "learning_rate": 0.0002754964894684052, + "loss": 2.5612, + "theoretical_loss": 3.516013201310458, + "tokens_seen": 1500326912 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027548645937813446, + "loss": 2.5483, + "theoretical_loss": 3.515999660876833, + "tokens_seen": 1500392448 + }, + { + "epoch": 5.01, + "learning_rate": 0.0002754764292878636, + "loss": 2.7099, + "theoretical_loss": 3.515986121200224, + "tokens_seen": 1500457984 + }, + { + "epoch": 5.01, + "learning_rate": 0.0002754663991975928, + "loss": 2.6436, + "theoretical_loss": 3.5159725822805568, + "tokens_seen": 1500523520 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027545636910732195, + "loss": 2.7573, + "theoretical_loss": 3.5159590441177544, + "tokens_seen": 1500589056 + }, + { + "epoch": 5.01, + "learning_rate": 0.0002754463390170512, + "loss": 2.7131, + "theoretical_loss": 3.515945506711744, + "tokens_seen": 1500654592 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027543630892678037, + "loss": 2.5178, + "theoretical_loss": 3.5159319700624483, + "tokens_seen": 1500720128 + }, + { + "epoch": 5.01, + "objective/train/docs_used": 1715522, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.640793800354004, + "objective/train/theoretical_loss": 3.515921818072025, + "objective/train/tokens_used": 1521229280, + "theoretical_loss": 3.515921818072025, + "tokens_seen": 1500769280 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027542627883650955, + "loss": 2.7359, + "theoretical_loss": 3.5159184341697918, + "tokens_seen": 1500785664 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027541624874623873, + "loss": 2.7044, + "theoretical_loss": 3.5159048990337007, + "tokens_seen": 1500851200 + }, + { + "epoch": 5.01, + "learning_rate": 0.0002754062186559679, + "loss": 2.5357, + "theoretical_loss": 3.515891364654099, + "tokens_seen": 1500916736 + }, + { + "epoch": 5.01, + "learning_rate": 0.0002753961885656971, + "loss": 2.7109, + "theoretical_loss": 3.5158778310309104, + "tokens_seen": 1500982272 + }, + { + "epoch": 5.01, + "learning_rate": 0.0002753861584754263, + "loss": 2.6688, + "theoretical_loss": 3.5158642981640607, + "tokens_seen": 1501047808 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027537612838515545, + "loss": 2.5025, + "theoretical_loss": 3.5158507660534744, + "tokens_seen": 1501113344 + }, + { + "epoch": 5.01, + "learning_rate": 0.0002753660982948847, + "loss": 2.4948, + "theoretical_loss": 3.5158372346990765, + "tokens_seen": 1501178880 + }, + { + "epoch": 5.01, + "learning_rate": 0.0002753560682046138, + "loss": 2.6206, + "theoretical_loss": 3.515823704100791, + "tokens_seen": 1501244416 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027534603811434305, + "loss": 2.5264, + "theoretical_loss": 3.5158101742585437, + "tokens_seen": 1501309952 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027533600802407223, + "loss": 2.7406, + "theoretical_loss": 3.5157966451722578, + "tokens_seen": 1501375488 + }, + { + "epoch": 5.01, + "learning_rate": 0.0002753259779338014, + "loss": 2.6902, + "theoretical_loss": 3.515783116841859, + "tokens_seen": 1501441024 + }, + { + "epoch": 5.01, + "learning_rate": 0.0002753159478435306, + "loss": 2.6181, + "theoretical_loss": 3.515769589267273, + "tokens_seen": 1501506560 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027530591775325983, + "loss": 2.6366, + "theoretical_loss": 3.515756062448423, + "tokens_seen": 1501572096 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027529588766298896, + "loss": 2.6837, + "theoretical_loss": 3.5157425363852344, + "tokens_seen": 1501637632 + }, + { + "epoch": 5.01, + "learning_rate": 0.0002752858575727182, + "loss": 2.7563, + "theoretical_loss": 3.515729011077632, + "tokens_seen": 1501703168 + }, + { + "epoch": 5.01, + "learning_rate": 0.0002752758274824473, + "loss": 2.6408, + "theoretical_loss": 3.515715486525541, + "tokens_seen": 1501768704 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027526579739217655, + "loss": 2.487, + "theoretical_loss": 3.515701962728885, + "tokens_seen": 1501834240 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027525576730190574, + "loss": 2.7103, + "theoretical_loss": 3.5156884396875907, + "tokens_seen": 1501899776 + }, + { + "epoch": 5.01, + "learning_rate": 0.0002752457372116349, + "loss": 2.5855, + "theoretical_loss": 3.5156749174015816, + "tokens_seen": 1501965312 + }, + { + "epoch": 5.01, + "learning_rate": 0.0002752357071213641, + "loss": 2.6207, + "theoretical_loss": 3.515661395870783, + "tokens_seen": 1502030848 + }, + { + "epoch": 5.01, + "learning_rate": 0.0002752256770310933, + "loss": 2.7456, + "theoretical_loss": 3.5156478750951194, + "tokens_seen": 1502096384 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027521564694082246, + "loss": 2.5779, + "theoretical_loss": 3.515634355074517, + "tokens_seen": 1502161920 + }, + { + "epoch": 5.01, + "learning_rate": 0.0002752056168505517, + "loss": 2.5976, + "theoretical_loss": 3.5156208358088987, + "tokens_seen": 1502227456 + }, + { + "epoch": 5.01, + "learning_rate": 0.0002751955867602808, + "loss": 2.5067, + "theoretical_loss": 3.5156073172981905, + "tokens_seen": 1502292992 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027518555667001006, + "loss": 2.6524, + "theoretical_loss": 3.5155937995423177, + "tokens_seen": 1502358528 + }, + { + "epoch": 5.01, + "objective/train/docs_used": 1720743, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.869342803955078, + "objective/train/theoretical_loss": 3.515583661720728, + "objective/train/tokens_used": 1522867680, + "theoretical_loss": 3.515583661720728, + "tokens_seen": 1502407680 + }, + { + "epoch": 5.01, + "learning_rate": 0.0002751755265797392, + "loss": 2.6944, + "theoretical_loss": 3.5155802825412046, + "tokens_seen": 1502424064 + }, + { + "epoch": 5.01, + "learning_rate": 0.0002751654964894684, + "loss": 2.6483, + "theoretical_loss": 3.515566766294776, + "tokens_seen": 1502489600 + }, + { + "epoch": 5.01, + "learning_rate": 0.0002751554663991976, + "loss": 2.5766, + "theoretical_loss": 3.515553250802957, + "tokens_seen": 1502555136 + }, + { + "epoch": 5.01, + "learning_rate": 0.0002751454363089268, + "loss": 2.8034, + "theoretical_loss": 3.5155397360656733, + "tokens_seen": 1502620672 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027513540621865596, + "loss": 2.5673, + "theoretical_loss": 3.5155262220828485, + "tokens_seen": 1502686208 + }, + { + "epoch": 5.01, + "learning_rate": 0.0002751253761283852, + "loss": 2.6488, + "theoretical_loss": 3.515512708854409, + "tokens_seen": 1502751744 + }, + { + "epoch": 5.01, + "learning_rate": 0.0002751153460381143, + "loss": 2.5111, + "theoretical_loss": 3.5154991963802793, + "tokens_seen": 1502817280 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027510531594784356, + "loss": 2.7109, + "theoretical_loss": 3.5154856846603835, + "tokens_seen": 1502882816 + }, + { + "epoch": 5.01, + "learning_rate": 0.0002750952858575727, + "loss": 2.5817, + "theoretical_loss": 3.515472173694648, + "tokens_seen": 1502948352 + }, + { + "epoch": 5.01, + "learning_rate": 0.0002750852557673019, + "loss": 2.5877, + "theoretical_loss": 3.5154586634829967, + "tokens_seen": 1503013888 + }, + { + "epoch": 5.01, + "learning_rate": 0.0002750752256770311, + "loss": 2.6702, + "theoretical_loss": 3.5154451540253553, + "tokens_seen": 1503079424 + }, + { + "epoch": 5.01, + "learning_rate": 0.0002750651955867603, + "loss": 2.7882, + "theoretical_loss": 3.515431645321649, + "tokens_seen": 1503144960 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027505516549648947, + "loss": 2.5627, + "theoretical_loss": 3.5154181373718023, + "tokens_seen": 1503210496 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027504513540621865, + "loss": 2.7719, + "theoretical_loss": 3.5154046301757402, + "tokens_seen": 1503276032 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027503510531594783, + "loss": 2.8695, + "theoretical_loss": 3.515391123733388, + "tokens_seen": 1503341568 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027502507522567706, + "loss": 2.4746, + "theoretical_loss": 3.5153776180446714, + "tokens_seen": 1503407104 + }, + { + "epoch": 5.01, + "learning_rate": 0.0002750150451354062, + "loss": 2.664, + "theoretical_loss": 3.5153641131095146, + "tokens_seen": 1503472640 + }, + { + "epoch": 5.01, + "learning_rate": 0.0002750050150451354, + "loss": 2.6918, + "theoretical_loss": 3.515350608927843, + "tokens_seen": 1503538176 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027499498495486455, + "loss": 2.8506, + "theoretical_loss": 3.515337105499582, + "tokens_seen": 1503603712 + }, + { + "epoch": 5.01, + "learning_rate": 0.0002749849548645938, + "loss": 2.6836, + "theoretical_loss": 3.5153236028246564, + "tokens_seen": 1503669248 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027497492477432297, + "loss": 2.752, + "theoretical_loss": 3.5153101009029912, + "tokens_seen": 1503734784 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027496489468405215, + "loss": 2.6325, + "theoretical_loss": 3.515296599734512, + "tokens_seen": 1503800320 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027495486459378133, + "loss": 2.7755, + "theoretical_loss": 3.515283099319144, + "tokens_seen": 1503865856 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027494483450351057, + "loss": 2.7143, + "theoretical_loss": 3.5152695996568117, + "tokens_seen": 1503931392 + }, + { + "epoch": 5.01, + "learning_rate": 0.0002749348044132397, + "loss": 2.6559, + "theoretical_loss": 3.515256100747441, + "tokens_seen": 1503996928 + }, + { + "epoch": 5.01, + "objective/train/docs_used": 1725747, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6323916912078857, + "objective/train/theoretical_loss": 3.515245977059499, + "objective/train/tokens_used": 1524506080, + "theoretical_loss": 3.515245977059499, + "tokens_seen": 1504046080 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027492477432296893, + "loss": 2.7248, + "theoretical_loss": 3.515242602590957, + "tokens_seen": 1504062464 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027491474423269806, + "loss": 2.5724, + "theoretical_loss": 3.5152291051872844, + "tokens_seen": 1504128000 + }, + { + "epoch": 5.01, + "learning_rate": 0.0002749047141424273, + "loss": 2.6799, + "theoretical_loss": 3.515215608536349, + "tokens_seen": 1504193536 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027489468405215647, + "loss": 2.515, + "theoretical_loss": 3.5152021126380752, + "tokens_seen": 1504259072 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027488465396188565, + "loss": 2.4422, + "theoretical_loss": 3.5151886174923894, + "tokens_seen": 1504324608 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027487462387161483, + "loss": 2.5653, + "theoretical_loss": 3.5151751230992163, + "tokens_seen": 1504390144 + }, + { + "epoch": 5.01, + "learning_rate": 0.000274864593781344, + "loss": 2.668, + "theoretical_loss": 3.515161629458481, + "tokens_seen": 1504455680 + }, + { + "epoch": 5.01, + "learning_rate": 0.0002748545636910732, + "loss": 2.6074, + "theoretical_loss": 3.5151481365701085, + "tokens_seen": 1504521216 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027484453360080243, + "loss": 2.7049, + "theoretical_loss": 3.515134644434025, + "tokens_seen": 1504586752 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027483450351053156, + "loss": 2.7778, + "theoretical_loss": 3.5151211530501545, + "tokens_seen": 1504652288 + }, + { + "epoch": 5.01, + "learning_rate": 0.0002748244734202608, + "loss": 2.4842, + "theoretical_loss": 3.5151076624184237, + "tokens_seen": 1504717824 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027481444332999, + "loss": 2.7311, + "theoretical_loss": 3.515094172538757, + "tokens_seen": 1504783360 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027480441323971916, + "loss": 2.7872, + "theoretical_loss": 3.515080683411081, + "tokens_seen": 1504848896 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027479438314944834, + "loss": 2.5982, + "theoretical_loss": 3.5150671950353187, + "tokens_seen": 1504914432 + }, + { + "epoch": 5.01, + "learning_rate": 0.0002747843530591775, + "loss": 2.6789, + "theoretical_loss": 3.515053707411397, + "tokens_seen": 1504979968 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027477432296890675, + "loss": 2.6325, + "theoretical_loss": 3.5150402205392415, + "tokens_seen": 1505045504 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027476429287863594, + "loss": 2.5831, + "theoretical_loss": 3.515026734418777, + "tokens_seen": 1505111040 + }, + { + "epoch": 5.01, + "learning_rate": 0.0002747542627883651, + "loss": 2.6323, + "theoretical_loss": 3.515013249049929, + "tokens_seen": 1505176576 + }, + { + "epoch": 5.01, + "learning_rate": 0.0002747442326980943, + "loss": 2.5551, + "theoretical_loss": 3.5149997644326225, + "tokens_seen": 1505242112 + }, + { + "epoch": 5.01, + "learning_rate": 0.0002747342026078235, + "loss": 2.6337, + "theoretical_loss": 3.5149862805667835, + "tokens_seen": 1505307648 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027472417251755266, + "loss": 2.5774, + "theoretical_loss": 3.5149727974523373, + "tokens_seen": 1505373184 + }, + { + "epoch": 5.01, + "learning_rate": 0.0002747141424272819, + "loss": 2.5681, + "theoretical_loss": 3.514959315089209, + "tokens_seen": 1505438720 + }, + { + "epoch": 5.01, + "learning_rate": 0.000274704112337011, + "loss": 2.6553, + "theoretical_loss": 3.5149458334773245, + "tokens_seen": 1505504256 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027469408224674026, + "loss": 2.5902, + "theoretical_loss": 3.5149323526166087, + "tokens_seen": 1505569792 + }, + { + "epoch": 5.01, + "learning_rate": 0.0002746840521564694, + "loss": 2.5925, + "theoretical_loss": 3.5149188725069873, + "tokens_seen": 1505635328 + }, + { + "epoch": 5.01, + "objective/train/docs_used": 1730984, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6221024990081787, + "objective/train/theoretical_loss": 3.514908762917632, + "objective/train/tokens_used": 1526144480, + "theoretical_loss": 3.514908762917632, + "tokens_seen": 1505684480 + }, + { + "epoch": 5.01, + "learning_rate": 0.0002746740220661986, + "loss": 2.5532, + "theoretical_loss": 3.5149053931483856, + "tokens_seen": 1505700864 + }, + { + "epoch": 5.01, + "learning_rate": 0.0002746639919759278, + "loss": 2.5331, + "theoretical_loss": 3.51489191454073, + "tokens_seen": 1505766400 + }, + { + "epoch": 5.01, + "learning_rate": 0.000274653961885657, + "loss": 2.6332, + "theoretical_loss": 3.5148784366839445, + "tokens_seen": 1505831936 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027464393179538616, + "loss": 2.7405, + "theoretical_loss": 3.514864959577955, + "tokens_seen": 1505897472 + }, + { + "epoch": 5.01, + "learning_rate": 0.0002746339017051154, + "loss": 2.7854, + "theoretical_loss": 3.514851483222688, + "tokens_seen": 1505963008 + }, + { + "epoch": 5.01, + "learning_rate": 0.0002746238716148445, + "loss": 2.6165, + "theoretical_loss": 3.514838007618068, + "tokens_seen": 1506028544 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027461384152457376, + "loss": 2.7394, + "theoretical_loss": 3.5148245327640213, + "tokens_seen": 1506094080 + }, + { + "epoch": 5.01, + "learning_rate": 0.0002746038114343029, + "loss": 2.6271, + "theoretical_loss": 3.514811058660473, + "tokens_seen": 1506159616 + }, + { + "epoch": 5.01, + "learning_rate": 0.0002745937813440321, + "loss": 2.6569, + "theoretical_loss": 3.514797585307348, + "tokens_seen": 1506225152 + }, + { + "epoch": 5.01, + "learning_rate": 0.0002745837512537613, + "loss": 2.7429, + "theoretical_loss": 3.514784112704573, + "tokens_seen": 1506290688 + }, + { + "epoch": 5.01, + "learning_rate": 0.0002745737211634905, + "loss": 2.7054, + "theoretical_loss": 3.514770640852073, + "tokens_seen": 1506356224 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027456369107321967, + "loss": 2.7931, + "theoretical_loss": 3.5147571697497737, + "tokens_seen": 1506421760 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027455366098294885, + "loss": 2.6221, + "theoretical_loss": 3.5147436993976005, + "tokens_seen": 1506487296 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027454363089267803, + "loss": 2.5807, + "theoretical_loss": 3.5147302297954797, + "tokens_seen": 1506552832 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027453360080240726, + "loss": 2.5029, + "theoretical_loss": 3.514716760943336, + "tokens_seen": 1506618368 + }, + { + "epoch": 5.01, + "learning_rate": 0.0002745235707121364, + "loss": 2.5789, + "theoretical_loss": 3.5147032928410953, + "tokens_seen": 1506683904 + }, + { + "epoch": 5.01, + "learning_rate": 0.0002745135406218656, + "loss": 2.564, + "theoretical_loss": 3.514689825488683, + "tokens_seen": 1506749440 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027450351053159475, + "loss": 2.6247, + "theoretical_loss": 3.5146763588860255, + "tokens_seen": 1506814976 + }, + { + "epoch": 5.01, + "learning_rate": 0.000274493480441324, + "loss": 2.6367, + "theoretical_loss": 3.514662893033048, + "tokens_seen": 1506880512 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027448345035105317, + "loss": 2.6014, + "theoretical_loss": 3.514649427929676, + "tokens_seen": 1506946048 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027447342026078235, + "loss": 2.6573, + "theoretical_loss": 3.5146359635758353, + "tokens_seen": 1507011584 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027446339017051153, + "loss": 2.7415, + "theoretical_loss": 3.514622499971452, + "tokens_seen": 1507077120 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027445336008024077, + "loss": 2.7173, + "theoretical_loss": 3.5146090371164505, + "tokens_seen": 1507142656 + }, + { + "epoch": 5.01, + "learning_rate": 0.0002744433299899699, + "loss": 2.6439, + "theoretical_loss": 3.5145955750107585, + "tokens_seen": 1507208192 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027443329989969913, + "loss": 2.534, + "theoretical_loss": 3.5145821136543, + "tokens_seen": 1507273728 + }, + { + "epoch": 5.01, + "objective/train/docs_used": 1733768, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.598609209060669, + "objective/train/theoretical_loss": 3.5145720181285967, + "objective/train/tokens_used": 1527782880, + "theoretical_loss": 3.5145720181285967, + "tokens_seen": 1507322880 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027442326980942826, + "loss": 2.6034, + "theoretical_loss": 3.514568653047002, + "tokens_seen": 1507339264 + }, + { + "epoch": 5.01, + "learning_rate": 0.0002744132397191575, + "loss": 2.5776, + "theoretical_loss": 3.514555193188789, + "tokens_seen": 1507404800 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027440320962888667, + "loss": 2.5075, + "theoretical_loss": 3.5145417340795877, + "tokens_seen": 1507470336 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027439317953861585, + "loss": 2.6137, + "theoretical_loss": 3.5145282757193232, + "tokens_seen": 1507535872 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027438314944834503, + "loss": 2.6294, + "theoretical_loss": 3.514514818107922, + "tokens_seen": 1507601408 + }, + { + "epoch": 5.01, + "learning_rate": 0.0002743731193580742, + "loss": 2.5912, + "theoretical_loss": 3.5145013612453093, + "tokens_seen": 1507666944 + }, + { + "epoch": 5.01, + "learning_rate": 0.0002743630892678034, + "loss": 2.4158, + "theoretical_loss": 3.514487905131411, + "tokens_seen": 1507732480 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027435305917753263, + "loss": 2.4401, + "theoretical_loss": 3.514474449766153, + "tokens_seen": 1507798016 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027434302908726176, + "loss": 2.6716, + "theoretical_loss": 3.514460995149461, + "tokens_seen": 1507863552 + }, + { + "epoch": 5.01, + "learning_rate": 0.000274332998996991, + "loss": 2.51, + "theoretical_loss": 3.5144475412812612, + "tokens_seen": 1507929088 + }, + { + "epoch": 5.01, + "learning_rate": 0.0002743229689067202, + "loss": 2.5596, + "theoretical_loss": 3.514434088161479, + "tokens_seen": 1507994624 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027431293881644936, + "loss": 2.5836, + "theoretical_loss": 3.5144206357900405, + "tokens_seen": 1508060160 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027430290872617854, + "loss": 2.562, + "theoretical_loss": 3.5144071841668714, + "tokens_seen": 1508125696 + }, + { + "epoch": 5.01, + "learning_rate": 0.0002742928786359077, + "loss": 2.6443, + "theoretical_loss": 3.514393733291897, + "tokens_seen": 1508191232 + }, + { + "epoch": 5.01, + "learning_rate": 0.0002742828485456369, + "loss": 2.6106, + "theoretical_loss": 3.5143802831650444, + "tokens_seen": 1508256768 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027427281845536614, + "loss": 2.4577, + "theoretical_loss": 3.5143668337862386, + "tokens_seen": 1508322304 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027426278836509526, + "loss": 2.6082, + "theoretical_loss": 3.514353385155406, + "tokens_seen": 1508387840 + }, + { + "epoch": 5.01, + "learning_rate": 0.0002742527582748245, + "loss": 2.6183, + "theoretical_loss": 3.514339937272472, + "tokens_seen": 1508453376 + }, + { + "epoch": 5.01, + "learning_rate": 0.0002742427281845536, + "loss": 2.7128, + "theoretical_loss": 3.514326490137363, + "tokens_seen": 1508518912 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027423269809428286, + "loss": 2.7367, + "theoretical_loss": 3.514313043750005, + "tokens_seen": 1508584448 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027422266800401204, + "loss": 2.6525, + "theoretical_loss": 3.514299598110323, + "tokens_seen": 1508649984 + }, + { + "epoch": 5.01, + "learning_rate": 0.0002742126379137412, + "loss": 2.6169, + "theoretical_loss": 3.5142861532182437, + "tokens_seen": 1508715520 + }, + { + "epoch": 5.01, + "learning_rate": 0.0002742026078234704, + "loss": 2.7413, + "theoretical_loss": 3.514272709073693, + "tokens_seen": 1508781056 + }, + { + "epoch": 5.01, + "learning_rate": 0.0002741925777331996, + "loss": 2.5877, + "theoretical_loss": 3.514259265676597, + "tokens_seen": 1508846592 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027418254764292877, + "loss": 2.5007, + "theoretical_loss": 3.5142458230268816, + "tokens_seen": 1508912128 + }, + { + "epoch": 5.01, + "objective/train/docs_used": 1734888, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6094367504119873, + "objective/train/theoretical_loss": 3.514235741530019, + "objective/train/tokens_used": 1529421280, + "theoretical_loss": 3.514235741530019, + "tokens_seen": 1508961280 + }, + { + "epoch": 5.01, + "learning_rate": 0.000274172517552658, + "loss": 2.504, + "theoretical_loss": 3.5142323811244722, + "tokens_seen": 1508977664 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027416248746238713, + "loss": 2.7591, + "theoretical_loss": 3.514218939969296, + "tokens_seen": 1509043200 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027415245737211636, + "loss": 2.4556, + "theoretical_loss": 3.514205499561278, + "tokens_seen": 1509108736 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027414242728184554, + "loss": 2.7308, + "theoretical_loss": 3.5141920599003447, + "tokens_seen": 1509174272 + }, + { + "epoch": 5.01, + "learning_rate": 0.0002741323971915747, + "loss": 2.7473, + "theoretical_loss": 3.5141786209864216, + "tokens_seen": 1509239808 + }, + { + "epoch": 5.01, + "learning_rate": 0.0002741223671013039, + "loss": 2.4884, + "theoretical_loss": 3.5141651828194354, + "tokens_seen": 1509305344 + }, + { + "epoch": 5.01, + "learning_rate": 0.0002741123370110331, + "loss": 2.702, + "theoretical_loss": 3.514151745399312, + "tokens_seen": 1509370880 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027410230692076227, + "loss": 2.7508, + "theoretical_loss": 3.514138308725977, + "tokens_seen": 1509436416 + }, + { + "epoch": 5.01, + "learning_rate": 0.0002740922768304915, + "loss": 2.5619, + "theoretical_loss": 3.514124872799357, + "tokens_seen": 1509501952 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027408224674022063, + "loss": 2.8769, + "theoretical_loss": 3.514111437619378, + "tokens_seen": 1509567488 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027407221664994987, + "loss": 2.7444, + "theoretical_loss": 3.514098003185966, + "tokens_seen": 1509633024 + }, + { + "epoch": 5.01, + "learning_rate": 0.000274062186559679, + "loss": 2.481, + "theoretical_loss": 3.5140845694990466, + "tokens_seen": 1509698560 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027405215646940823, + "loss": 2.6672, + "theoretical_loss": 3.5140711365585466, + "tokens_seen": 1509764096 + }, + { + "epoch": 5.01, + "learning_rate": 0.0002740421263791374, + "loss": 2.6577, + "theoretical_loss": 3.5140577043643924, + "tokens_seen": 1509829632 + }, + { + "epoch": 5.01, + "learning_rate": 0.0002740320962888666, + "loss": 2.5711, + "theoretical_loss": 3.514044272916509, + "tokens_seen": 1509895168 + }, + { + "epoch": 5.01, + "learning_rate": 0.0002740220661985958, + "loss": 2.5343, + "theoretical_loss": 3.514030842214824, + "tokens_seen": 1509960704 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027401203610832495, + "loss": 2.4917, + "theoretical_loss": 3.5140174122592627, + "tokens_seen": 1510026240 + }, + { + "epoch": 5.01, + "learning_rate": 0.0002740020060180542, + "loss": 2.7321, + "theoretical_loss": 3.5140039830497507, + "tokens_seen": 1510091776 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027399197592778337, + "loss": 2.5898, + "theoretical_loss": 3.513990554586216, + "tokens_seen": 1510157312 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027398194583751255, + "loss": 2.7621, + "theoretical_loss": 3.513977126868583, + "tokens_seen": 1510222848 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027397191574724173, + "loss": 2.4087, + "theoretical_loss": 3.5139636998967783, + "tokens_seen": 1510288384 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027396188565697097, + "loss": 2.6989, + "theoretical_loss": 3.513950273670728, + "tokens_seen": 1510353920 + }, + { + "epoch": 5.01, + "learning_rate": 0.0002739518555667001, + "loss": 2.6959, + "theoretical_loss": 3.51393684819036, + "tokens_seen": 1510419456 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027394182547642933, + "loss": 2.5562, + "theoretical_loss": 3.513923423455598, + "tokens_seen": 1510484992 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027393179538615846, + "loss": 2.4551, + "theoretical_loss": 3.51390999946637, + "tokens_seen": 1510550528 + }, + { + "epoch": 5.01, + "objective/train/docs_used": 1735676, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.696056842803955, + "objective/train/theoretical_loss": 3.513899931963661, + "objective/train/tokens_used": 1531059680, + "theoretical_loss": 3.513899931963661, + "tokens_seen": 1510599680 + }, + { + "epoch": 5.01, + "learning_rate": 0.0002739217652958877, + "loss": 2.5213, + "theoretical_loss": 3.5138965762226015, + "tokens_seen": 1510616064 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027391173520561687, + "loss": 2.6409, + "theoretical_loss": 3.513883153724219, + "tokens_seen": 1510681600 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027390170511534605, + "loss": 2.7535, + "theoretical_loss": 3.513869731971149, + "tokens_seen": 1510747136 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027389167502507524, + "loss": 2.6657, + "theoretical_loss": 3.5138563109633174, + "tokens_seen": 1510812672 + }, + { + "epoch": 5.01, + "learning_rate": 0.0002738816449348044, + "loss": 2.759, + "theoretical_loss": 3.5138428907006505, + "tokens_seen": 1510878208 + }, + { + "epoch": 5.01, + "learning_rate": 0.0002738716148445336, + "loss": 2.4819, + "theoretical_loss": 3.513829471183075, + "tokens_seen": 1510943744 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027386158475426283, + "loss": 2.7243, + "theoretical_loss": 3.513816052410517, + "tokens_seen": 1511009280 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027385155466399196, + "loss": 2.7477, + "theoretical_loss": 3.5138026343829027, + "tokens_seen": 1511074816 + }, + { + "epoch": 5.01, + "learning_rate": 0.0002738415245737212, + "loss": 2.6833, + "theoretical_loss": 3.5137892171001583, + "tokens_seen": 1511140352 + }, + { + "epoch": 5.01, + "learning_rate": 0.0002738314944834504, + "loss": 2.567, + "theoretical_loss": 3.5137758005622106, + "tokens_seen": 1511205888 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027382146439317956, + "loss": 2.689, + "theoretical_loss": 3.513762384768986, + "tokens_seen": 1511271424 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027381143430290874, + "loss": 2.7233, + "theoretical_loss": 3.5137489697204103, + "tokens_seen": 1511336960 + }, + { + "epoch": 5.01, + "learning_rate": 0.0002738014042126379, + "loss": 2.6076, + "theoretical_loss": 3.5137355554164103, + "tokens_seen": 1511402496 + }, + { + "epoch": 5.01, + "learning_rate": 0.0002737913741223671, + "loss": 2.8242, + "theoretical_loss": 3.513722141856912, + "tokens_seen": 1511468032 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027378134403209634, + "loss": 2.7138, + "theoretical_loss": 3.5137087290418423, + "tokens_seen": 1511533568 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027377131394182546, + "loss": 2.6239, + "theoretical_loss": 3.5136953169711274, + "tokens_seen": 1511599104 + }, + { + "epoch": 5.01, + "learning_rate": 0.0002737612838515547, + "loss": 2.5643, + "theoretical_loss": 3.5136819056446935, + "tokens_seen": 1511664640 + }, + { + "epoch": 5.01, + "learning_rate": 0.0002737512537612838, + "loss": 2.8376, + "theoretical_loss": 3.5136684950624675, + "tokens_seen": 1511730176 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027374122367101306, + "loss": 2.4982, + "theoretical_loss": 3.5136550852243755, + "tokens_seen": 1511795712 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027373119358074224, + "loss": 2.5081, + "theoretical_loss": 3.5136416761303444, + "tokens_seen": 1511861248 + }, + { + "epoch": 5.01, + "learning_rate": 0.0002737211634904714, + "loss": 2.5411, + "theoretical_loss": 3.5136282677803, + "tokens_seen": 1511926784 + }, + { + "epoch": 5.01, + "learning_rate": 0.0002737111334002006, + "loss": 2.689, + "theoretical_loss": 3.513614860174169, + "tokens_seen": 1511992320 + }, + { + "epoch": 5.01, + "learning_rate": 0.0002737011033099298, + "loss": 2.6541, + "theoretical_loss": 3.513601453311878, + "tokens_seen": 1512057856 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027369107321965897, + "loss": 2.6243, + "theoretical_loss": 3.5135880471933536, + "tokens_seen": 1512123392 + }, + { + "epoch": 5.01, + "learning_rate": 0.0002736810431293882, + "loss": 2.6046, + "theoretical_loss": 3.513574641818522, + "tokens_seen": 1512188928 + }, + { + "epoch": 5.01, + "objective/train/docs_used": 1736838, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.042192220687866, + "objective/train/theoretical_loss": 3.5135645882754023, + "objective/train/tokens_used": 1532698080, + "theoretical_loss": 3.5135645882754023, + "tokens_seen": 1512238080 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027367101303911733, + "loss": 2.7713, + "theoretical_loss": 3.51356123718731, + "tokens_seen": 1512254464 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027366098294884656, + "loss": 2.7605, + "theoretical_loss": 3.513547833299644, + "tokens_seen": 1512320000 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027365095285857574, + "loss": 2.6, + "theoretical_loss": 3.5135344301554503, + "tokens_seen": 1512385536 + }, + { + "epoch": 5.01, + "learning_rate": 0.0002736409227683049, + "loss": 2.5412, + "theoretical_loss": 3.5135210277546562, + "tokens_seen": 1512451072 + }, + { + "epoch": 5.01, + "learning_rate": 0.0002736308926780341, + "loss": 2.7291, + "theoretical_loss": 3.513507626097187, + "tokens_seen": 1512516608 + }, + { + "epoch": 5.01, + "learning_rate": 0.0002736208625877633, + "loss": 2.4899, + "theoretical_loss": 3.5134942251829706, + "tokens_seen": 1512582144 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027361083249749247, + "loss": 2.5796, + "theoretical_loss": 3.513480825011933, + "tokens_seen": 1512647680 + }, + { + "epoch": 5.01, + "learning_rate": 0.0002736008024072217, + "loss": 2.5228, + "theoretical_loss": 3.513467425584001, + "tokens_seen": 1512713216 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027359077231695083, + "loss": 2.7755, + "theoretical_loss": 3.5134540268991006, + "tokens_seen": 1512778752 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027358074222668007, + "loss": 2.5736, + "theoretical_loss": 3.5134406289571585, + "tokens_seen": 1512844288 + }, + { + "epoch": 5.01, + "learning_rate": 0.0002735707121364092, + "loss": 2.7156, + "theoretical_loss": 3.5134272317581026, + "tokens_seen": 1512909824 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027356068204613843, + "loss": 2.702, + "theoretical_loss": 3.513413835301858, + "tokens_seen": 1512975360 + }, + { + "epoch": 5.01, + "learning_rate": 0.0002735506519558676, + "loss": 2.7033, + "theoretical_loss": 3.513400439588352, + "tokens_seen": 1513040896 + }, + { + "epoch": 5.01, + "learning_rate": 0.0002735406218655968, + "loss": 2.6876, + "theoretical_loss": 3.513387044617511, + "tokens_seen": 1513106432 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027353059177532597, + "loss": 2.7163, + "theoretical_loss": 3.5133736503892616, + "tokens_seen": 1513171968 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027352056168505515, + "loss": 2.7879, + "theoretical_loss": 3.513360256903531, + "tokens_seen": 1513237504 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027351053159478433, + "loss": 2.6548, + "theoretical_loss": 3.5133468641602454, + "tokens_seen": 1513303040 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027350050150451357, + "loss": 2.7286, + "theoretical_loss": 3.513333472159332, + "tokens_seen": 1513368576 + }, + { + "epoch": 5.01, + "learning_rate": 0.0002734904714142427, + "loss": 2.7595, + "theoretical_loss": 3.5133200809007166, + "tokens_seen": 1513434112 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027348044132397193, + "loss": 2.5187, + "theoretical_loss": 3.513306690384327, + "tokens_seen": 1513499648 + }, + { + "epoch": 5.01, + "learning_rate": 0.0002734704112337011, + "loss": 2.6689, + "theoretical_loss": 3.5132933006100893, + "tokens_seen": 1513565184 + }, + { + "epoch": 5.01, + "learning_rate": 0.0002734603811434303, + "loss": 2.5274, + "theoretical_loss": 3.5132799115779303, + "tokens_seen": 1513630720 + }, + { + "epoch": 5.01, + "learning_rate": 0.0002734503510531595, + "loss": 2.8102, + "theoretical_loss": 3.5132665232877764, + "tokens_seen": 1513696256 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027344032096288866, + "loss": 2.6089, + "theoretical_loss": 3.5132531357395553, + "tokens_seen": 1513761792 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027343029087261784, + "loss": 2.6788, + "theoretical_loss": 3.5132397489331932, + "tokens_seen": 1513827328 + }, + { + "epoch": 5.01, + "objective/train/docs_used": 1737452, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.892695426940918, + "objective/train/theoretical_loss": 3.5132297093152225, + "objective/train/tokens_used": 1534336480, + "theoretical_loss": 3.5132297093152225, + "tokens_seen": 1513876480 + }, + { + "epoch": 5.01, + "learning_rate": 0.0002734202607823471, + "loss": 2.5524, + "theoretical_loss": 3.5132263628686164, + "tokens_seen": 1513892864 + }, + { + "epoch": 5.01, + "learning_rate": 0.0002734102306920762, + "loss": 2.61, + "theoretical_loss": 3.513212977545753, + "tokens_seen": 1513958400 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027340020060180544, + "loss": 2.6958, + "theoretical_loss": 3.5131995929645283, + "tokens_seen": 1514023936 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027339017051153456, + "loss": 2.6809, + "theoretical_loss": 3.5131862091248696, + "tokens_seen": 1514089472 + }, + { + "epoch": 5.01, + "learning_rate": 0.0002733801404212638, + "loss": 2.7049, + "theoretical_loss": 3.5131728260267048, + "tokens_seen": 1514155008 + }, + { + "epoch": 5.01, + "learning_rate": 0.000273370110330993, + "loss": 2.6297, + "theoretical_loss": 3.513159443669959, + "tokens_seen": 1514220544 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027336008024072216, + "loss": 2.7736, + "theoretical_loss": 3.51314606205456, + "tokens_seen": 1514286080 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027335005015045134, + "loss": 2.6571, + "theoretical_loss": 3.5131326811804344, + "tokens_seen": 1514351616 + }, + { + "epoch": 5.01, + "learning_rate": 0.0002733400200601806, + "loss": 2.559, + "theoretical_loss": 3.5131193010475097, + "tokens_seen": 1514417152 + }, + { + "epoch": 5.01, + "learning_rate": 0.0002733299899699097, + "loss": 2.8363, + "theoretical_loss": 3.513105921655712, + "tokens_seen": 1514482688 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027331995987963894, + "loss": 2.7014, + "theoretical_loss": 3.5130925430049684, + "tokens_seen": 1514548224 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027330992978936807, + "loss": 2.586, + "theoretical_loss": 3.513079165095206, + "tokens_seen": 1514613760 + }, + { + "epoch": 5.01, + "learning_rate": 0.0002732998996990973, + "loss": 2.4239, + "theoretical_loss": 3.513065787926351, + "tokens_seen": 1514679296 + }, + { + "epoch": 5.01, + "learning_rate": 0.0002732898696088265, + "loss": 2.4609, + "theoretical_loss": 3.5130524114983315, + "tokens_seen": 1514744832 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027327983951855566, + "loss": 2.5722, + "theoretical_loss": 3.5130390358110732, + "tokens_seen": 1514810368 + }, + { + "epoch": 5.01, + "learning_rate": 0.0002732698094282849, + "loss": 2.5675, + "theoretical_loss": 3.513025660864504, + "tokens_seen": 1514875904 + }, + { + "epoch": 5.01, + "learning_rate": 0.000273259779338014, + "loss": 2.4711, + "theoretical_loss": 3.51301228665855, + "tokens_seen": 1514941440 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027324974924774326, + "loss": 2.494, + "theoretical_loss": 3.5129989131931394, + "tokens_seen": 1515006976 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027323971915747244, + "loss": 2.4075, + "theoretical_loss": 3.5129855404681978, + "tokens_seen": 1515072512 + }, + { + "epoch": 5.01, + "learning_rate": 0.0002732296890672016, + "loss": 2.7047, + "theoretical_loss": 3.5129721684836523, + "tokens_seen": 1515138048 + }, + { + "epoch": 5.01, + "learning_rate": 0.0002732196589769308, + "loss": 2.5743, + "theoretical_loss": 3.5129587972394307, + "tokens_seen": 1515203584 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027320962888666, + "loss": 2.5521, + "theoretical_loss": 3.5129454267354596, + "tokens_seen": 1515269120 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027319959879638917, + "loss": 2.7014, + "theoretical_loss": 3.512932056971666, + "tokens_seen": 1515334656 + }, + { + "epoch": 5.01, + "learning_rate": 0.0002731895687061184, + "loss": 2.6108, + "theoretical_loss": 3.5129186879479772, + "tokens_seen": 1515400192 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027317953861584753, + "loss": 2.618, + "theoretical_loss": 3.51290531966432, + "tokens_seen": 1515465728 + }, + { + "epoch": 5.01, + "objective/train/docs_used": 1739078, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.715820789337158, + "objective/train/theoretical_loss": 3.5128952939371785, + "objective/train/tokens_used": 1535974880, + "theoretical_loss": 3.5128952939371785, + "tokens_seen": 1515514880 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027316950852557676, + "loss": 2.6964, + "theoretical_loss": 3.5128919521206208, + "tokens_seen": 1515531264 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027315947843530594, + "loss": 2.8865, + "theoretical_loss": 3.512878585316807, + "tokens_seen": 1515596800 + }, + { + "epoch": 5.01, + "learning_rate": 0.0002731494483450351, + "loss": 2.5913, + "theoretical_loss": 3.5128652192528067, + "tokens_seen": 1515662336 + }, + { + "epoch": 5.01, + "learning_rate": 0.0002731394182547643, + "loss": 2.4339, + "theoretical_loss": 3.512851853928546, + "tokens_seen": 1515727872 + }, + { + "epoch": 5.01, + "learning_rate": 0.0002731293881644935, + "loss": 2.5569, + "theoretical_loss": 3.5128384893439524, + "tokens_seen": 1515793408 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027311935807422267, + "loss": 2.6283, + "theoretical_loss": 3.512825125498952, + "tokens_seen": 1515858944 + }, + { + "epoch": 5.01, + "learning_rate": 0.0002731093279839519, + "loss": 2.6183, + "theoretical_loss": 3.512811762393473, + "tokens_seen": 1515924480 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027309929789368103, + "loss": 2.5518, + "theoretical_loss": 3.512798400027442, + "tokens_seen": 1515990016 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027308926780341027, + "loss": 2.6761, + "theoretical_loss": 3.5127850384007866, + "tokens_seen": 1516055552 + }, + { + "epoch": 5.01, + "learning_rate": 0.0002730792377131394, + "loss": 2.5924, + "theoretical_loss": 3.5127716775134328, + "tokens_seen": 1516121088 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027306920762286863, + "loss": 2.7086, + "theoretical_loss": 3.5127583173653094, + "tokens_seen": 1516186624 + }, + { + "epoch": 5.01, + "learning_rate": 0.0002730591775325978, + "loss": 2.7092, + "theoretical_loss": 3.5127449579563423, + "tokens_seen": 1516252160 + }, + { + "epoch": 5.01, + "learning_rate": 0.000273049147442327, + "loss": 2.6952, + "theoretical_loss": 3.5127315992864587, + "tokens_seen": 1516317696 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027303911735205617, + "loss": 2.5468, + "theoretical_loss": 3.5127182413555866, + "tokens_seen": 1516383232 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027302908726178535, + "loss": 2.581, + "theoretical_loss": 3.5127048841636523, + "tokens_seen": 1516448768 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027301905717151453, + "loss": 2.3842, + "theoretical_loss": 3.512691527710584, + "tokens_seen": 1516514304 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027300902708124377, + "loss": 2.5223, + "theoretical_loss": 3.512678171996307, + "tokens_seen": 1516579840 + }, + { + "epoch": 5.01, + "learning_rate": 0.0002729989969909729, + "loss": 2.5641, + "theoretical_loss": 3.5126648170207515, + "tokens_seen": 1516645376 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027298896690070213, + "loss": 2.8428, + "theoretical_loss": 3.512651462783842, + "tokens_seen": 1516710912 + }, + { + "epoch": 5.01, + "learning_rate": 0.0002729789368104313, + "loss": 2.4741, + "theoretical_loss": 3.512638109285507, + "tokens_seen": 1516776448 + }, + { + "epoch": 5.01, + "learning_rate": 0.0002729689067201605, + "loss": 2.6423, + "theoretical_loss": 3.512624756525673, + "tokens_seen": 1516841984 + }, + { + "epoch": 5.01, + "learning_rate": 0.0002729588766298897, + "loss": 2.5588, + "theoretical_loss": 3.512611404504268, + "tokens_seen": 1516907520 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027294884653961886, + "loss": 2.7271, + "theoretical_loss": 3.512598053221219, + "tokens_seen": 1516973056 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027293881644934804, + "loss": 2.5997, + "theoretical_loss": 3.5125847026764534, + "tokens_seen": 1517038592 + }, + { + "epoch": 5.01, + "learning_rate": 0.0002729287863590773, + "loss": 2.5554, + "theoretical_loss": 3.5125713528698985, + "tokens_seen": 1517104128 + }, + { + "epoch": 5.01, + "objective/train/docs_used": 1739851, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.781825542449951, + "objective/train/theoretical_loss": 3.5125613409993894, + "objective/train/tokens_used": 1537613280, + "theoretical_loss": 3.5125613409993894, + "tokens_seen": 1517153280 + }, + { + "epoch": 5.01, + "learning_rate": 0.0002729187562688064, + "loss": 2.5735, + "theoretical_loss": 3.512558003801481, + "tokens_seen": 1517169664 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027290872617853564, + "loss": 2.8424, + "theoretical_loss": 3.512544655471129, + "tokens_seen": 1517235200 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027289869608826476, + "loss": 2.7819, + "theoretical_loss": 3.5125313078787697, + "tokens_seen": 1517300736 + }, + { + "epoch": 5.01, + "learning_rate": 0.000272888665997994, + "loss": 2.8765, + "theoretical_loss": 3.51251796102433, + "tokens_seen": 1517366272 + }, + { + "epoch": 5.01, + "learning_rate": 0.0002728786359077232, + "loss": 2.6052, + "theoretical_loss": 3.512504614907737, + "tokens_seen": 1517431808 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027286860581745236, + "loss": 2.6188, + "theoretical_loss": 3.512491269528919, + "tokens_seen": 1517497344 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027285857572718154, + "loss": 2.6225, + "theoretical_loss": 3.5124779248878024, + "tokens_seen": 1517562880 + }, + { + "epoch": 5.01, + "learning_rate": 0.0002728485456369108, + "loss": 2.6682, + "theoretical_loss": 3.5124645809843154, + "tokens_seen": 1517628416 + }, + { + "epoch": 5.01, + "learning_rate": 0.0002728385155466399, + "loss": 2.7252, + "theoretical_loss": 3.512451237818385, + "tokens_seen": 1517693952 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027282848545636914, + "loss": 2.5136, + "theoretical_loss": 3.512437895389938, + "tokens_seen": 1517759488 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027281845536609827, + "loss": 2.5185, + "theoretical_loss": 3.512424553698903, + "tokens_seen": 1517825024 + }, + { + "epoch": 5.01, + "learning_rate": 0.0002728084252758275, + "loss": 2.5597, + "theoretical_loss": 3.5124112127452065, + "tokens_seen": 1517890560 + }, + { + "epoch": 5.01, + "learning_rate": 0.0002727983951855567, + "loss": 2.5773, + "theoretical_loss": 3.5123978725287763, + "tokens_seen": 1517956096 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027278836509528586, + "loss": 2.4945, + "theoretical_loss": 3.5123845330495396, + "tokens_seen": 1518021632 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027277833500501504, + "loss": 2.591, + "theoretical_loss": 3.5123711943074234, + "tokens_seen": 1518087168 + }, + { + "epoch": 5.01, + "learning_rate": 0.0002727683049147442, + "loss": 2.6178, + "theoretical_loss": 3.5123578563023568, + "tokens_seen": 1518152704 + }, + { + "epoch": 5.01, + "learning_rate": 0.0002727582748244734, + "loss": 2.6907, + "theoretical_loss": 3.5123445190342655, + "tokens_seen": 1518218240 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027274824473420264, + "loss": 2.376, + "theoretical_loss": 3.5123311825030776, + "tokens_seen": 1518283776 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027273821464393177, + "loss": 2.5858, + "theoretical_loss": 3.512317846708721, + "tokens_seen": 1518349312 + }, + { + "epoch": 5.01, + "learning_rate": 0.000272728184553661, + "loss": 2.5271, + "theoretical_loss": 3.5123045116511227, + "tokens_seen": 1518414848 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027271815446339013, + "loss": 2.6194, + "theoretical_loss": 3.51229117733021, + "tokens_seen": 1518480384 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027270812437311937, + "loss": 2.5939, + "theoretical_loss": 3.5122778437459106, + "tokens_seen": 1518545920 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027269809428284855, + "loss": 2.6146, + "theoretical_loss": 3.5122645108981523, + "tokens_seen": 1518611456 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027268806419257773, + "loss": 2.6375, + "theoretical_loss": 3.5122511787868627, + "tokens_seen": 1518676992 + }, + { + "epoch": 5.01, + "learning_rate": 0.0002726780341023069, + "loss": 2.6602, + "theoretical_loss": 3.512237847411969, + "tokens_seen": 1518742528 + }, + { + "epoch": 5.01, + "objective/train/docs_used": 1741412, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7110002040863037, + "objective/train/theoretical_loss": 3.512227849364015, + "objective/train/tokens_used": 1539251680, + "theoretical_loss": 3.512227849364015, + "tokens_seen": 1518791680 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027266800401203614, + "loss": 2.5932, + "theoretical_loss": 3.512224516773399, + "tokens_seen": 1518808064 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027265797392176527, + "loss": 2.7002, + "theoretical_loss": 3.51221118687108, + "tokens_seen": 1518873600 + }, + { + "epoch": 5.01, + "learning_rate": 0.0002726479438314945, + "loss": 2.6501, + "theoretical_loss": 3.5121978577049395, + "tokens_seen": 1518939136 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027263791374122363, + "loss": 2.6066, + "theoretical_loss": 3.512184529274905, + "tokens_seen": 1519004672 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027262788365095287, + "loss": 2.6338, + "theoretical_loss": 3.5121712015809052, + "tokens_seen": 1519070208 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027261785356068205, + "loss": 2.7052, + "theoretical_loss": 3.5121578746228668, + "tokens_seen": 1519135744 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027260782347041123, + "loss": 2.6389, + "theoretical_loss": 3.512144548400717, + "tokens_seen": 1519201280 + }, + { + "epoch": 5.01, + "learning_rate": 0.0002725977933801404, + "loss": 2.7032, + "theoretical_loss": 3.512131222914384, + "tokens_seen": 1519266816 + }, + { + "epoch": 5.01, + "learning_rate": 0.0002725877632898696, + "loss": 2.6396, + "theoretical_loss": 3.512117898163795, + "tokens_seen": 1519332352 + }, + { + "epoch": 5.01, + "learning_rate": 0.0002725777331995988, + "loss": 2.6893, + "theoretical_loss": 3.5121045741488786, + "tokens_seen": 1519397888 + }, + { + "epoch": 5.01, + "learning_rate": 0.000272567703109328, + "loss": 2.7178, + "theoretical_loss": 3.5120912508695614, + "tokens_seen": 1519463424 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027255767301905714, + "loss": 2.8325, + "theoretical_loss": 3.5120779283257715, + "tokens_seen": 1519528960 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027254764292878637, + "loss": 2.5731, + "theoretical_loss": 3.512064606517437, + "tokens_seen": 1519594496 + }, + { + "epoch": 5.01, + "learning_rate": 0.0002725376128385155, + "loss": 2.5434, + "theoretical_loss": 3.5120512854444845, + "tokens_seen": 1519660032 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027252758274824473, + "loss": 2.5804, + "theoretical_loss": 3.512037965106843, + "tokens_seen": 1519725568 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027251755265797397, + "loss": 2.5707, + "theoretical_loss": 3.5120246455044395, + "tokens_seen": 1519791104 + }, + { + "epoch": 5.01, + "learning_rate": 0.0002725075225677031, + "loss": 2.6185, + "theoretical_loss": 3.512011326637201, + "tokens_seen": 1519856640 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027249749247743233, + "loss": 2.5685, + "theoretical_loss": 3.5119980085050573, + "tokens_seen": 1519922176 + }, + { + "epoch": 5.01, + "learning_rate": 0.0002724874623871615, + "loss": 2.6181, + "theoretical_loss": 3.5119846911079335, + "tokens_seen": 1519987712 + }, + { + "epoch": 5.01, + "learning_rate": 0.0002724774322968907, + "loss": 2.5499, + "theoretical_loss": 3.5119713744457597, + "tokens_seen": 1520053248 + }, + { + "epoch": 5.01, + "learning_rate": 0.0002724674022066199, + "loss": 2.4947, + "theoretical_loss": 3.511958058518462, + "tokens_seen": 1520118784 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027245737211634906, + "loss": 2.6411, + "theoretical_loss": 3.511944743325969, + "tokens_seen": 1520184320 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027244734202607824, + "loss": 2.5907, + "theoretical_loss": 3.511931428868208, + "tokens_seen": 1520249856 + }, + { + "epoch": 5.01, + "learning_rate": 0.0002724373119358075, + "loss": 2.5607, + "theoretical_loss": 3.5119181151451078, + "tokens_seen": 1520315392 + }, + { + "epoch": 5.01, + "learning_rate": 0.0002724272818455366, + "loss": 2.4661, + "theoretical_loss": 3.511904802156595, + "tokens_seen": 1520380928 + }, + { + "epoch": 5.01, + "objective/train/docs_used": 1741916, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7754528522491455, + "objective/train/theoretical_loss": 3.51189481789724, + "objective/train/tokens_used": 1540890080, + "theoretical_loss": 3.51189481789724, + "tokens_seen": 1520430080 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027241725175526584, + "loss": 2.504, + "theoretical_loss": 3.511891489902598, + "tokens_seen": 1520446464 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027240722166499496, + "loss": 2.7549, + "theoretical_loss": 3.511878178383044, + "tokens_seen": 1520512000 + }, + { + "epoch": 5.01, + "learning_rate": 0.0002723971915747242, + "loss": 2.7554, + "theoretical_loss": 3.511864867597862, + "tokens_seen": 1520577536 + }, + { + "epoch": 5.01, + "learning_rate": 0.0002723871614844534, + "loss": 2.7106, + "theoretical_loss": 3.5118515575469784, + "tokens_seen": 1520643072 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027237713139418256, + "loss": 2.6407, + "theoretical_loss": 3.5118382482303225, + "tokens_seen": 1520708608 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027236710130391174, + "loss": 2.5378, + "theoretical_loss": 3.511824939647821, + "tokens_seen": 1520774144 + }, + { + "epoch": 5.01, + "learning_rate": 0.000272357071213641, + "loss": 2.7233, + "theoretical_loss": 3.5118116317994024, + "tokens_seen": 1520839680 + }, + { + "epoch": 5.01, + "learning_rate": 0.0002723470411233701, + "loss": 2.9172, + "theoretical_loss": 3.5117983246849946, + "tokens_seen": 1520905216 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027233701103309934, + "loss": 2.4646, + "theoretical_loss": 3.5117850183045247, + "tokens_seen": 1520970752 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027232698094282847, + "loss": 2.6007, + "theoretical_loss": 3.511771712657922, + "tokens_seen": 1521036288 + }, + { + "epoch": 5.01, + "learning_rate": 0.0002723169508525577, + "loss": 2.4467, + "theoretical_loss": 3.5117584077451127, + "tokens_seen": 1521101824 + }, + { + "epoch": 5.01, + "learning_rate": 0.0002723069207622869, + "loss": 2.5762, + "theoretical_loss": 3.5117451035660263, + "tokens_seen": 1521167360 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027229689067201606, + "loss": 2.7585, + "theoretical_loss": 3.51173180012059, + "tokens_seen": 1521232896 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027228686058174524, + "loss": 2.8096, + "theoretical_loss": 3.5117184974087317, + "tokens_seen": 1521298432 + }, + { + "epoch": 5.01, + "learning_rate": 0.0002722768304914744, + "loss": 2.6325, + "theoretical_loss": 3.5117051954303795, + "tokens_seen": 1521363968 + }, + { + "epoch": 5.01, + "learning_rate": 0.0002722668004012036, + "loss": 2.6678, + "theoretical_loss": 3.511691894185461, + "tokens_seen": 1521429504 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027225677031093284, + "loss": 2.7784, + "theoretical_loss": 3.5116785936739046, + "tokens_seen": 1521495040 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027224674022066197, + "loss": 2.8446, + "theoretical_loss": 3.5116652938956383, + "tokens_seen": 1521560576 + }, + { + "epoch": 5.01, + "learning_rate": 0.0002722367101303912, + "loss": 2.9045, + "theoretical_loss": 3.51165199485059, + "tokens_seen": 1521626112 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027222668004012033, + "loss": 2.5861, + "theoretical_loss": 3.511638696538687, + "tokens_seen": 1521691648 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027221664994984957, + "loss": 2.6447, + "theoretical_loss": 3.5116253989598585, + "tokens_seen": 1521757184 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027220661985957875, + "loss": 2.5343, + "theoretical_loss": 3.5116121021140323, + "tokens_seen": 1521822720 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027219658976930793, + "loss": 2.7088, + "theoretical_loss": 3.5115988060011354, + "tokens_seen": 1521888256 + }, + { + "epoch": 5.01, + "learning_rate": 0.0002721865596790371, + "loss": 2.4863, + "theoretical_loss": 3.5115855106210967, + "tokens_seen": 1521953792 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027217652958876635, + "loss": 2.5216, + "theoretical_loss": 3.5115722159738443, + "tokens_seen": 1522019328 + }, + { + "epoch": 5.01, + "objective/train/docs_used": 1743369, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8488197326660156, + "objective/train/theoretical_loss": 3.5115622454692526, + "objective/train/tokens_used": 1542528480, + "theoretical_loss": 3.5115622454692526, + "tokens_seen": 1522068480 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027216649949849547, + "loss": 2.6903, + "theoretical_loss": 3.511558922059306, + "tokens_seen": 1522084864 + }, + { + "epoch": 5.01, + "learning_rate": 0.0002721564694082247, + "loss": 2.5928, + "theoretical_loss": 3.51154562887741, + "tokens_seen": 1522150400 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027214643931795383, + "loss": 2.532, + "theoretical_loss": 3.5115323364280844, + "tokens_seen": 1522215936 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027213640922768307, + "loss": 2.502, + "theoretical_loss": 3.5115190447112568, + "tokens_seen": 1522281472 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027212637913741225, + "loss": 2.5598, + "theoretical_loss": 3.511505753726856, + "tokens_seen": 1522347008 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027211634904714143, + "loss": 2.5905, + "theoretical_loss": 3.51149246347481, + "tokens_seen": 1522412544 + }, + { + "epoch": 5.01, + "learning_rate": 0.0002721063189568706, + "loss": 2.7652, + "theoretical_loss": 3.5114791739550464, + "tokens_seen": 1522478080 + }, + { + "epoch": 5.01, + "learning_rate": 0.0002720962888665998, + "loss": 2.6179, + "theoretical_loss": 3.511465885167494, + "tokens_seen": 1522543616 + }, + { + "epoch": 5.01, + "learning_rate": 0.000272086258776329, + "loss": 2.6407, + "theoretical_loss": 3.51145259711208, + "tokens_seen": 1522609152 + }, + { + "epoch": 5.01, + "learning_rate": 0.0002720762286860582, + "loss": 2.5974, + "theoretical_loss": 3.511439309788734, + "tokens_seen": 1522674688 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027206619859578734, + "loss": 2.6676, + "theoretical_loss": 3.511426023197383, + "tokens_seen": 1522740224 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027205616850551657, + "loss": 2.4657, + "theoretical_loss": 3.511412737337955, + "tokens_seen": 1522805760 + }, + { + "epoch": 5.01, + "learning_rate": 0.0002720461384152457, + "loss": 2.5989, + "theoretical_loss": 3.5113994522103793, + "tokens_seen": 1522871296 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027203610832497493, + "loss": 2.4463, + "theoretical_loss": 3.5113861678145835, + "tokens_seen": 1522936832 + }, + { + "epoch": 5.01, + "learning_rate": 0.0002720260782347041, + "loss": 2.5057, + "theoretical_loss": 3.511372884150495, + "tokens_seen": 1523002368 + }, + { + "epoch": 5.01, + "learning_rate": 0.0002720160481444333, + "loss": 2.6699, + "theoretical_loss": 3.511359601218044, + "tokens_seen": 1523067904 + }, + { + "epoch": 5.01, + "learning_rate": 0.0002720060180541625, + "loss": 2.723, + "theoretical_loss": 3.5113463190171568, + "tokens_seen": 1523133440 + }, + { + "epoch": 5.01, + "learning_rate": 0.0002719959879638917, + "loss": 2.7831, + "theoretical_loss": 3.5113330375477623, + "tokens_seen": 1523198976 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027198595787362084, + "loss": 2.7578, + "theoretical_loss": 3.5113197568097894, + "tokens_seen": 1523264512 + }, + { + "epoch": 5.01, + "learning_rate": 0.0002719759277833501, + "loss": 2.4327, + "theoretical_loss": 3.511306476803165, + "tokens_seen": 1523330048 + }, + { + "epoch": 5.01, + "learning_rate": 0.0002719658976930792, + "loss": 2.6328, + "theoretical_loss": 3.5112931975278183, + "tokens_seen": 1523395584 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027195586760280844, + "loss": 2.3377, + "theoretical_loss": 3.5112799189836776, + "tokens_seen": 1523461120 + }, + { + "epoch": 5.01, + "learning_rate": 0.0002719458375125376, + "loss": 2.5108, + "theoretical_loss": 3.511266641170671, + "tokens_seen": 1523526656 + }, + { + "epoch": 5.01, + "learning_rate": 0.0002719358074222668, + "loss": 2.4412, + "theoretical_loss": 3.511253364088727, + "tokens_seen": 1523592192 + }, + { + "epoch": 5.01, + "learning_rate": 0.000271925777331996, + "loss": 2.7561, + "theoretical_loss": 3.5112400877377734, + "tokens_seen": 1523657728 + }, + { + "epoch": 5.01, + "objective/train/docs_used": 1744115, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8299615383148193, + "objective/train/theoretical_loss": 3.5112301309542273, + "objective/train/tokens_used": 1544166880, + "theoretical_loss": 3.5112301309542273, + "tokens_seen": 1523706880 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027191574724172516, + "loss": 2.7278, + "theoretical_loss": 3.5112268121177386, + "tokens_seen": 1523723264 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027190571715145434, + "loss": 2.9239, + "theoretical_loss": 3.511213537228551, + "tokens_seen": 1523788800 + }, + { + "epoch": 5.01, + "learning_rate": 0.0002718956870611836, + "loss": 2.5451, + "theoretical_loss": 3.5112002630701395, + "tokens_seen": 1523854336 + }, + { + "epoch": 5.01, + "learning_rate": 0.0002718856569709127, + "loss": 2.5095, + "theoretical_loss": 3.5111869896424324, + "tokens_seen": 1523919872 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027187562688064194, + "loss": 2.7109, + "theoretical_loss": 3.5111737169453567, + "tokens_seen": 1523985408 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027186559679037107, + "loss": 2.753, + "theoretical_loss": 3.5111604449788425, + "tokens_seen": 1524050944 + }, + { + "epoch": 5.01, + "learning_rate": 0.0002718555667001003, + "loss": 2.6246, + "theoretical_loss": 3.511147173742817, + "tokens_seen": 1524116480 + }, + { + "epoch": 5.01, + "learning_rate": 0.0002718455366098295, + "loss": 2.4407, + "theoretical_loss": 3.511133903237209, + "tokens_seen": 1524182016 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027183550651955867, + "loss": 2.674, + "theoretical_loss": 3.511120633461947, + "tokens_seen": 1524247552 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027182547642928785, + "loss": 2.5761, + "theoretical_loss": 3.5111073644169593, + "tokens_seen": 1524313088 + }, + { + "epoch": 5.01, + "learning_rate": 0.0002718154463390171, + "loss": 2.6593, + "theoretical_loss": 3.511094096102174, + "tokens_seen": 1524378624 + }, + { + "epoch": 5.01, + "learning_rate": 0.0002718054162487462, + "loss": 2.5198, + "theoretical_loss": 3.51108082851752, + "tokens_seen": 1524444160 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027179538615847544, + "loss": 2.6014, + "theoretical_loss": 3.511067561662926, + "tokens_seen": 1524509696 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027178535606820457, + "loss": 2.7626, + "theoretical_loss": 3.5110542955383197, + "tokens_seen": 1524575232 + }, + { + "epoch": 5.01, + "learning_rate": 0.0002717753259779338, + "loss": 2.3878, + "theoretical_loss": 3.51104103014363, + "tokens_seen": 1524640768 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027176529588766304, + "loss": 2.4231, + "theoretical_loss": 3.5110277654787847, + "tokens_seen": 1524706304 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027175526579739217, + "loss": 2.6618, + "theoretical_loss": 3.5110145015437135, + "tokens_seen": 1524771840 + }, + { + "epoch": 5.01, + "learning_rate": 0.0002717452357071214, + "loss": 2.7225, + "theoretical_loss": 3.511001238338344, + "tokens_seen": 1524837376 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027173520561685053, + "loss": 2.6813, + "theoretical_loss": 3.5109879758626046, + "tokens_seen": 1524902912 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027172517552657977, + "loss": 2.6589, + "theoretical_loss": 3.5109747141164243, + "tokens_seen": 1524968448 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027171514543630895, + "loss": 2.7244, + "theoretical_loss": 3.5109614530997315, + "tokens_seen": 1525033984 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027170511534603813, + "loss": 2.5411, + "theoretical_loss": 3.510948192812455, + "tokens_seen": 1525099520 + }, + { + "epoch": 5.01, + "learning_rate": 0.0002716950852557673, + "loss": 2.6788, + "theoretical_loss": 3.5109349332545223, + "tokens_seen": 1525165056 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027168505516549655, + "loss": 2.607, + "theoretical_loss": 3.510921674425863, + "tokens_seen": 1525230592 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027167502507522567, + "loss": 2.4647, + "theoretical_loss": 3.510908416326405, + "tokens_seen": 1525296128 + }, + { + "epoch": 5.01, + "objective/train/docs_used": 1744854, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9327380657196045, + "objective/train/theoretical_loss": 3.5108984732303075, + "objective/train/tokens_used": 1545805280, + "theoretical_loss": 3.5108984732303075, + "tokens_seen": 1525345280 + }, + { + "epoch": 5.01, + "learning_rate": 0.0002716649949849549, + "loss": 2.6041, + "theoretical_loss": 3.5108951589560773, + "tokens_seen": 1525361664 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027165496489468403, + "loss": 2.6702, + "theoretical_loss": 3.5108819023148086, + "tokens_seen": 1525427200 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027164493480441327, + "loss": 2.725, + "theoretical_loss": 3.5108686464025274, + "tokens_seen": 1525492736 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027163490471414245, + "loss": 2.749, + "theoretical_loss": 3.5108553912191613, + "tokens_seen": 1525558272 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027162487462387163, + "loss": 2.7232, + "theoretical_loss": 3.5108421367646403, + "tokens_seen": 1525623808 + }, + { + "epoch": 5.01, + "learning_rate": 0.0002716148445336008, + "loss": 2.596, + "theoretical_loss": 3.5108288830388923, + "tokens_seen": 1525689344 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027160481444333, + "loss": 2.5888, + "theoretical_loss": 3.5108156300418463, + "tokens_seen": 1525754880 + }, + { + "epoch": 5.01, + "learning_rate": 0.0002715947843530592, + "loss": 2.5965, + "theoretical_loss": 3.5108023777734303, + "tokens_seen": 1525820416 + }, + { + "epoch": 5.01, + "learning_rate": 0.0002715847542627884, + "loss": 2.5714, + "theoretical_loss": 3.5107891262335738, + "tokens_seen": 1525885952 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027157472417251754, + "loss": 2.7426, + "theoretical_loss": 3.5107758754222047, + "tokens_seen": 1525951488 + }, + { + "epoch": 5.01, + "learning_rate": 0.0002715646940822468, + "loss": 2.4285, + "theoretical_loss": 3.510762625339252, + "tokens_seen": 1526017024 + }, + { + "epoch": 5.01, + "learning_rate": 0.0002715546639919759, + "loss": 2.4279, + "theoretical_loss": 3.5107493759846444, + "tokens_seen": 1526082560 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027154463390170514, + "loss": 2.433, + "theoretical_loss": 3.5107361273583106, + "tokens_seen": 1526148096 + }, + { + "epoch": 5.01, + "learning_rate": 0.0002715346038114343, + "loss": 2.8518, + "theoretical_loss": 3.5107228794601797, + "tokens_seen": 1526213632 + }, + { + "epoch": 5.01, + "learning_rate": 0.0002715245737211635, + "loss": 2.5255, + "theoretical_loss": 3.510709632290179, + "tokens_seen": 1526279168 + }, + { + "epoch": 5.01, + "learning_rate": 0.0002715145436308927, + "loss": 2.759, + "theoretical_loss": 3.510696385848239, + "tokens_seen": 1526344704 + }, + { + "epoch": 5.01, + "learning_rate": 0.0002715045135406219, + "loss": 2.4789, + "theoretical_loss": 3.5106831401342875, + "tokens_seen": 1526410240 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027149448345035104, + "loss": 2.3398, + "theoretical_loss": 3.510669895148253, + "tokens_seen": 1526475776 + }, + { + "epoch": 5.01, + "learning_rate": 0.0002714844533600803, + "loss": 2.5167, + "theoretical_loss": 3.5106566508900645, + "tokens_seen": 1526541312 + }, + { + "epoch": 5.01, + "learning_rate": 0.0002714744232698094, + "loss": 2.5327, + "theoretical_loss": 3.510643407359651, + "tokens_seen": 1526606848 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027146439317953864, + "loss": 2.558, + "theoretical_loss": 3.5106301645569413, + "tokens_seen": 1526672384 + }, + { + "epoch": 5.01, + "learning_rate": 0.0002714543630892678, + "loss": 2.6568, + "theoretical_loss": 3.510616922481864, + "tokens_seen": 1526737920 + }, + { + "epoch": 5.01, + "learning_rate": 0.000271444332998997, + "loss": 2.6549, + "theoretical_loss": 3.5106036811343477, + "tokens_seen": 1526803456 + }, + { + "epoch": 5.01, + "learning_rate": 0.0002714343029087262, + "loss": 2.7285, + "theoretical_loss": 3.5105904405143216, + "tokens_seen": 1526868992 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027142427281845536, + "loss": 2.4567, + "theoretical_loss": 3.5105772006217144, + "tokens_seen": 1526934528 + }, + { + "epoch": 5.01, + "objective/train/docs_used": 1746475, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7063658237457275, + "objective/train/theoretical_loss": 3.5105672711795846, + "objective/train/tokens_used": 1547443680, + "theoretical_loss": 3.5105672711795846, + "tokens_seen": 1526983680 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027141424272818454, + "loss": 2.7586, + "theoretical_loss": 3.5105639614564543, + "tokens_seen": 1527000064 + }, + { + "epoch": 5.01, + "learning_rate": 0.0002714042126379138, + "loss": 2.5778, + "theoretical_loss": 3.510550723018471, + "tokens_seen": 1527065600 + }, + { + "epoch": 5.01, + "learning_rate": 0.0002713941825476429, + "loss": 2.6325, + "theoretical_loss": 3.510537485307693, + "tokens_seen": 1527131136 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027138415245737214, + "loss": 2.6391, + "theoretical_loss": 3.5105242483240486, + "tokens_seen": 1527196672 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027137412236710127, + "loss": 2.6166, + "theoretical_loss": 3.510511012067468, + "tokens_seen": 1527262208 + }, + { + "epoch": 5.01, + "learning_rate": 0.0002713640922768305, + "loss": 2.8506, + "theoretical_loss": 3.510497776537879, + "tokens_seen": 1527327744 + }, + { + "epoch": 5.01, + "learning_rate": 0.0002713540621865597, + "loss": 2.5764, + "theoretical_loss": 3.5104845417352104, + "tokens_seen": 1527393280 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027134403209628887, + "loss": 2.6689, + "theoretical_loss": 3.510471307659392, + "tokens_seen": 1527458816 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027133400200601805, + "loss": 2.7366, + "theoretical_loss": 3.5104580743103515, + "tokens_seen": 1527524352 + }, + { + "epoch": 5.01, + "learning_rate": 0.0002713239719157473, + "loss": 2.6548, + "theoretical_loss": 3.5104448416880185, + "tokens_seen": 1527589888 + }, + { + "epoch": 5.01, + "learning_rate": 0.0002713139418254764, + "loss": 2.5055, + "theoretical_loss": 3.510431609792322, + "tokens_seen": 1527655424 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027130391173520564, + "loss": 2.6919, + "theoretical_loss": 3.510418378623191, + "tokens_seen": 1527720960 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027129388164493477, + "loss": 2.437, + "theoretical_loss": 3.510405148180554, + "tokens_seen": 1527786496 + }, + { + "epoch": 5.01, + "learning_rate": 0.000271283851554664, + "loss": 2.5164, + "theoretical_loss": 3.5103919184643404, + "tokens_seen": 1527852032 + }, + { + "epoch": 5.01, + "learning_rate": 0.0002712738214643932, + "loss": 2.5583, + "theoretical_loss": 3.5103786894744786, + "tokens_seen": 1527917568 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027126379137412237, + "loss": 2.5135, + "theoretical_loss": 3.510365461210898, + "tokens_seen": 1527983104 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027125376128385155, + "loss": 2.7525, + "theoretical_loss": 3.5103522336735278, + "tokens_seen": 1528048640 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027124373119358073, + "loss": 2.6378, + "theoretical_loss": 3.510339006862296, + "tokens_seen": 1528114176 + }, + { + "epoch": 5.01, + "learning_rate": 0.0002712337011033099, + "loss": 2.7063, + "theoretical_loss": 3.5103257807771326, + "tokens_seen": 1528179712 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027122367101303915, + "loss": 2.4968, + "theoretical_loss": 3.5103125554179666, + "tokens_seen": 1528245248 + }, + { + "epoch": 5.01, + "learning_rate": 0.0002712136409227683, + "loss": 2.6722, + "theoretical_loss": 3.5102993307847266, + "tokens_seen": 1528310784 + }, + { + "epoch": 5.01, + "learning_rate": 0.0002712036108324975, + "loss": 2.5445, + "theoretical_loss": 3.510286106877341, + "tokens_seen": 1528376320 + }, + { + "epoch": 5.01, + "learning_rate": 0.0002711935807422267, + "loss": 2.4946, + "theoretical_loss": 3.51027288369574, + "tokens_seen": 1528441856 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027118355065195587, + "loss": 2.6322, + "theoretical_loss": 3.5102596612398522, + "tokens_seen": 1528507392 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027117352056168505, + "loss": 2.7352, + "theoretical_loss": 3.510246439509607, + "tokens_seen": 1528572928 + }, + { + "epoch": 5.01, + "objective/train/docs_used": 1747177, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.3864831924438477, + "objective/train/theoretical_loss": 3.5102365236880826, + "objective/train/tokens_used": 1549082080, + "theoretical_loss": 3.5102365236880826, + "tokens_seen": 1528622080 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027116349047141423, + "loss": 2.586, + "theoretical_loss": 3.5102332185049328, + "tokens_seen": 1528638464 + }, + { + "epoch": 5.01, + "learning_rate": 0.0002711534603811434, + "loss": 2.5084, + "theoretical_loss": 3.510219998225759, + "tokens_seen": 1528704000 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027114343029087265, + "loss": 2.6257, + "theoretical_loss": 3.5102067786720146, + "tokens_seen": 1528769536 + }, + { + "epoch": 5.01, + "learning_rate": 0.0002711334002006018, + "loss": 2.5572, + "theoretical_loss": 3.510193559843629, + "tokens_seen": 1528835072 + }, + { + "epoch": 5.01, + "learning_rate": 0.000271123370110331, + "loss": 2.5593, + "theoretical_loss": 3.510180341740531, + "tokens_seen": 1528900608 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027111334002006014, + "loss": 2.6829, + "theoretical_loss": 3.51016712436265, + "tokens_seen": 1528966144 + }, + { + "epoch": 5.01, + "learning_rate": 0.0002711033099297894, + "loss": 2.7308, + "theoretical_loss": 3.510153907709914, + "tokens_seen": 1529031680 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027109327983951856, + "loss": 2.76, + "theoretical_loss": 3.5101406917822544, + "tokens_seen": 1529097216 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027108324974924774, + "loss": 2.4291, + "theoretical_loss": 3.5101274765795987, + "tokens_seen": 1529162752 + }, + { + "epoch": 5.01, + "learning_rate": 0.0002710732196589769, + "loss": 2.5715, + "theoretical_loss": 3.510114262101876, + "tokens_seen": 1529228288 + }, + { + "epoch": 5.01, + "learning_rate": 0.0002710631895687061, + "loss": 2.822, + "theoretical_loss": 3.510101048349016, + "tokens_seen": 1529293824 + }, + { + "epoch": 5.01, + "learning_rate": 0.0002710531594784353, + "loss": 2.5231, + "theoretical_loss": 3.510087835320948, + "tokens_seen": 1529359360 + }, + { + "epoch": 5.01, + "learning_rate": 0.0002710431293881645, + "loss": 2.6624, + "theoretical_loss": 3.5100746230176005, + "tokens_seen": 1529424896 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027103309929789364, + "loss": 2.5751, + "theoretical_loss": 3.5100614114389033, + "tokens_seen": 1529490432 + }, + { + "epoch": 5.01, + "learning_rate": 0.0002710230692076229, + "loss": 2.7269, + "theoretical_loss": 3.5100482005847855, + "tokens_seen": 1529555968 + }, + { + "epoch": 5.01, + "learning_rate": 0.0002710130391173521, + "loss": 2.7465, + "theoretical_loss": 3.5100349904551766, + "tokens_seen": 1529621504 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027100300902708124, + "loss": 2.4731, + "theoretical_loss": 3.5100217810500047, + "tokens_seen": 1529687040 + }, + { + "epoch": 5.01, + "learning_rate": 0.0002709929789368105, + "loss": 2.578, + "theoretical_loss": 3.5100085723692, + "tokens_seen": 1529752576 + }, + { + "epoch": 5.01, + "learning_rate": 0.0002709829488465396, + "loss": 2.4907, + "theoretical_loss": 3.509995364412692, + "tokens_seen": 1529818112 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027097291875626884, + "loss": 2.592, + "theoretical_loss": 3.509982157180409, + "tokens_seen": 1529883648 + }, + { + "epoch": 5.01, + "learning_rate": 0.000270962888665998, + "loss": 2.54, + "theoretical_loss": 3.5099689506722815, + "tokens_seen": 1529949184 + }, + { + "epoch": 5.01, + "learning_rate": 0.0002709528585757272, + "loss": 2.6529, + "theoretical_loss": 3.5099557448882375, + "tokens_seen": 1530014720 + }, + { + "epoch": 5.01, + "learning_rate": 0.0002709428284854564, + "loss": 2.7304, + "theoretical_loss": 3.509942539828207, + "tokens_seen": 1530080256 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027093279839518556, + "loss": 2.5196, + "theoretical_loss": 3.509929335492119, + "tokens_seen": 1530145792 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027092276830491474, + "loss": 2.6669, + "theoretical_loss": 3.5099161318799035, + "tokens_seen": 1530211328 + }, + { + "epoch": 5.01, + "objective/train/docs_used": 1748466, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8011348247528076, + "objective/train/theoretical_loss": 3.5099062296457397, + "objective/train/tokens_used": 1550720480, + "theoretical_loss": 3.5099062296457397, + "tokens_seen": 1530260480 + }, + { + "epoch": 5.01, + "learning_rate": 0.000270912738214644, + "loss": 2.5529, + "theoretical_loss": 3.5099029289914885, + "tokens_seen": 1530276864 + }, + { + "epoch": 5.01, + "learning_rate": 0.0002709027081243731, + "loss": 2.4209, + "theoretical_loss": 3.509889726826805, + "tokens_seen": 1530342400 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027089267803410234, + "loss": 2.7938, + "theoretical_loss": 3.509876525385781, + "tokens_seen": 1530407936 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027088264794383147, + "loss": 2.5396, + "theoretical_loss": 3.5098633246683457, + "tokens_seen": 1530473472 + }, + { + "epoch": 5.01, + "learning_rate": 0.0002708726178535607, + "loss": 2.5344, + "theoretical_loss": 3.50985012467443, + "tokens_seen": 1530539008 + }, + { + "epoch": 5.01, + "learning_rate": 0.0002708625877632899, + "loss": 2.6562, + "theoretical_loss": 3.509836925403962, + "tokens_seen": 1530604544 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027085255767301907, + "loss": 2.6576, + "theoretical_loss": 3.509823726856871, + "tokens_seen": 1530670080 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027084252758274825, + "loss": 2.6448, + "theoretical_loss": 3.5098105290330874, + "tokens_seen": 1530735616 + }, + { + "epoch": 5.01, + "learning_rate": 0.0002708324974924775, + "loss": 2.7434, + "theoretical_loss": 3.509797331932539, + "tokens_seen": 1530801152 + }, + { + "epoch": 5.01, + "learning_rate": 0.0002708224674022066, + "loss": 2.6026, + "theoretical_loss": 3.509784135555157, + "tokens_seen": 1530866688 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027081243731193584, + "loss": 2.6652, + "theoretical_loss": 3.5097709399008696, + "tokens_seen": 1530932224 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027080240722166497, + "loss": 2.4217, + "theoretical_loss": 3.5097577449696074, + "tokens_seen": 1530997760 + }, + { + "epoch": 5.01, + "learning_rate": 0.0002707923771313942, + "loss": 2.6055, + "theoretical_loss": 3.5097445507612983, + "tokens_seen": 1531063296 + }, + { + "epoch": 5.01, + "learning_rate": 0.0002707823470411234, + "loss": 2.6107, + "theoretical_loss": 3.5097313572758724, + "tokens_seen": 1531128832 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027077231695085257, + "loss": 2.6486, + "theoretical_loss": 3.50971816451326, + "tokens_seen": 1531194368 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027076228686058175, + "loss": 2.6052, + "theoretical_loss": 3.509704972473389, + "tokens_seen": 1531259904 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027075225677031093, + "loss": 2.7182, + "theoretical_loss": 3.5096917811561905, + "tokens_seen": 1531325440 + }, + { + "epoch": 5.01, + "learning_rate": 0.0002707422266800401, + "loss": 2.6964, + "theoretical_loss": 3.5096785905615926, + "tokens_seen": 1531390976 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027073219658976935, + "loss": 2.5845, + "theoretical_loss": 3.509665400689525, + "tokens_seen": 1531456512 + }, + { + "epoch": 5.01, + "learning_rate": 0.0002707221664994985, + "loss": 2.6935, + "theoretical_loss": 3.5096522115399185, + "tokens_seen": 1531522048 + }, + { + "epoch": 5.01, + "learning_rate": 0.0002707121364092277, + "loss": 2.6935, + "theoretical_loss": 3.509639023112701, + "tokens_seen": 1531587584 + }, + { + "epoch": 5.01, + "learning_rate": 0.0002707021063189569, + "loss": 2.4785, + "theoretical_loss": 3.5096258354078036, + "tokens_seen": 1531653120 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027069207622868607, + "loss": 2.4478, + "theoretical_loss": 3.509612648425154, + "tokens_seen": 1531718656 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027068204613841525, + "loss": 2.5772, + "theoretical_loss": 3.509599462164683, + "tokens_seen": 1531784192 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027067201604814443, + "loss": 2.6304, + "theoretical_loss": 3.5095862766263206, + "tokens_seen": 1531849728 + }, + { + "epoch": 5.01, + "objective/train/docs_used": 1749169, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.604491949081421, + "objective/train/theoretical_loss": 3.509576387946389, + "objective/train/tokens_used": 1552358880, + "theoretical_loss": 3.509576387946389, + "tokens_seen": 1531898880 + }, + { + "epoch": 5.01, + "learning_rate": 0.0002706619859578736, + "loss": 2.7063, + "theoretical_loss": 3.5095730918099948, + "tokens_seen": 1531915264 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027065195586760285, + "loss": 2.7013, + "theoretical_loss": 3.5095599077156363, + "tokens_seen": 1531980800 + }, + { + "epoch": 5.01, + "learning_rate": 0.000270641925777332, + "loss": 2.6335, + "theoretical_loss": 3.509546724343174, + "tokens_seen": 1532046336 + }, + { + "epoch": 5.01, + "learning_rate": 0.0002706318956870612, + "loss": 2.4774, + "theoretical_loss": 3.5095335416925386, + "tokens_seen": 1532111872 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027062186559679034, + "loss": 2.5814, + "theoretical_loss": 3.5095203597636586, + "tokens_seen": 1532177408 + }, + { + "epoch": 5.01, + "learning_rate": 0.0002706118355065196, + "loss": 2.6218, + "theoretical_loss": 3.509507178556464, + "tokens_seen": 1532242944 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027060180541624876, + "loss": 2.5315, + "theoretical_loss": 3.5094939980708846, + "tokens_seen": 1532308480 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027059177532597794, + "loss": 2.7335, + "theoretical_loss": 3.5094808183068498, + "tokens_seen": 1532374016 + }, + { + "epoch": 5.01, + "learning_rate": 0.0002705817452357071, + "loss": 2.503, + "theoretical_loss": 3.509467639264289, + "tokens_seen": 1532439552 + }, + { + "epoch": 5.01, + "learning_rate": 0.0002705717151454363, + "loss": 2.6337, + "theoretical_loss": 3.5094544609431324, + "tokens_seen": 1532505088 + }, + { + "epoch": 5.01, + "learning_rate": 0.0002705616850551655, + "loss": 2.5151, + "theoretical_loss": 3.5094412833433095, + "tokens_seen": 1532570624 + }, + { + "epoch": 5.01, + "learning_rate": 0.0002705516549648947, + "loss": 2.4761, + "theoretical_loss": 3.5094281064647497, + "tokens_seen": 1532636160 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027054162487462384, + "loss": 2.6397, + "theoretical_loss": 3.5094149303073827, + "tokens_seen": 1532701696 + }, + { + "epoch": 5.01, + "learning_rate": 0.0002705315947843531, + "loss": 2.796, + "theoretical_loss": 3.509401754871139, + "tokens_seen": 1532767232 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027052156469408226, + "loss": 2.5567, + "theoretical_loss": 3.5093885801559472, + "tokens_seen": 1532832768 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027051153460381144, + "loss": 2.4442, + "theoretical_loss": 3.5093754061617375, + "tokens_seen": 1532898304 + }, + { + "epoch": 5.01, + "learning_rate": 0.0002705015045135406, + "loss": 2.3743, + "theoretical_loss": 3.50936223288844, + "tokens_seen": 1532963840 + }, + { + "epoch": 5.01, + "learning_rate": 0.0002704914744232698, + "loss": 2.8448, + "theoretical_loss": 3.5093490603359836, + "tokens_seen": 1533029376 + }, + { + "epoch": 5.01, + "learning_rate": 0.000270481444332999, + "loss": 2.6492, + "theoretical_loss": 3.5093358885042987, + "tokens_seen": 1533094912 + }, + { + "epoch": 5.01, + "learning_rate": 0.0002704714142427282, + "loss": 2.6417, + "theoretical_loss": 3.5093227173933146, + "tokens_seen": 1533160448 + }, + { + "epoch": 5.01, + "learning_rate": 0.00027046138415245735, + "loss": 2.6674, + "theoretical_loss": 3.5093095470029616, + "tokens_seen": 1533225984 + }, + { + "epoch": 5.01, + "learning_rate": 0.0002704513540621866, + "loss": 2.7446, + "theoretical_loss": 3.509296377333169, + "tokens_seen": 1533291520 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002704413239719157, + "loss": 2.5301, + "theoretical_loss": 3.509283208383867, + "tokens_seen": 1533357056 + }, + { + "epoch": 5.02, + "learning_rate": 0.00027043129388164494, + "loss": 2.2968, + "theoretical_loss": 3.509270040154985, + "tokens_seen": 1533422592 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002704212637913741, + "loss": 2.4793, + "theoretical_loss": 3.5092568726464526, + "tokens_seen": 1533488128 + }, + { + "epoch": 5.02, + "objective/train/docs_used": 1750709, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6057729721069336, + "objective/train/theoretical_loss": 3.5092469974877414, + "objective/train/tokens_used": 1553997280, + "theoretical_loss": 3.5092469974877414, + "tokens_seen": 1533537280 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002704112337011033, + "loss": 2.634, + "theoretical_loss": 3.509243705858201, + "tokens_seen": 1533553664 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002704012036108325, + "loss": 2.6113, + "theoretical_loss": 3.509230539790158, + "tokens_seen": 1533619200 + }, + { + "epoch": 5.02, + "learning_rate": 0.00027039117352056167, + "loss": 2.5485, + "theoretical_loss": 3.5092173744422546, + "tokens_seen": 1533684736 + }, + { + "epoch": 5.02, + "learning_rate": 0.00027038114343029085, + "loss": 2.6423, + "theoretical_loss": 3.5092042098144205, + "tokens_seen": 1533750272 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002703711133400201, + "loss": 2.7032, + "theoretical_loss": 3.5091910459065856, + "tokens_seen": 1533815808 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002703610832497492, + "loss": 2.5698, + "theoretical_loss": 3.5091778827186797, + "tokens_seen": 1533881344 + }, + { + "epoch": 5.02, + "learning_rate": 0.00027035105315947845, + "loss": 2.6178, + "theoretical_loss": 3.5091647202506326, + "tokens_seen": 1533946880 + }, + { + "epoch": 5.02, + "learning_rate": 0.00027034102306920763, + "loss": 2.5169, + "theoretical_loss": 3.5091515585023743, + "tokens_seen": 1534012416 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002703309929789368, + "loss": 2.6137, + "theoretical_loss": 3.5091383974738344, + "tokens_seen": 1534077952 + }, + { + "epoch": 5.02, + "learning_rate": 0.000270320962888666, + "loss": 2.7768, + "theoretical_loss": 3.509125237164943, + "tokens_seen": 1534143488 + }, + { + "epoch": 5.02, + "learning_rate": 0.00027031093279839517, + "loss": 2.5983, + "theoretical_loss": 3.5091120775756304, + "tokens_seen": 1534209024 + }, + { + "epoch": 5.02, + "learning_rate": 0.00027030090270812435, + "loss": 2.6226, + "theoretical_loss": 3.5090989187058255, + "tokens_seen": 1534274560 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002702908726178536, + "loss": 2.7032, + "theoretical_loss": 3.5090857605554593, + "tokens_seen": 1534340096 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002702808425275827, + "loss": 2.5462, + "theoretical_loss": 3.5090726031244612, + "tokens_seen": 1534405632 + }, + { + "epoch": 5.02, + "learning_rate": 0.00027027081243731195, + "loss": 2.5581, + "theoretical_loss": 3.509059446412761, + "tokens_seen": 1534471168 + }, + { + "epoch": 5.02, + "learning_rate": 0.00027026078234704113, + "loss": 2.6566, + "theoretical_loss": 3.5090462904202893, + "tokens_seen": 1534536704 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002702507522567703, + "loss": 2.4762, + "theoretical_loss": 3.5090331351469755, + "tokens_seen": 1534602240 + }, + { + "epoch": 5.02, + "learning_rate": 0.00027024072216649955, + "loss": 2.6938, + "theoretical_loss": 3.50901998059275, + "tokens_seen": 1534667776 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002702306920762287, + "loss": 2.3536, + "theoretical_loss": 3.5090068267575423, + "tokens_seen": 1534733312 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002702206619859579, + "loss": 2.7465, + "theoretical_loss": 3.5089936736412826, + "tokens_seen": 1534798848 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002702106318956871, + "loss": 2.3321, + "theoretical_loss": 3.508980521243901, + "tokens_seen": 1534864384 + }, + { + "epoch": 5.02, + "learning_rate": 0.00027020060180541627, + "loss": 2.6597, + "theoretical_loss": 3.508967369565328, + "tokens_seen": 1534929920 + }, + { + "epoch": 5.02, + "learning_rate": 0.00027019057171514545, + "loss": 2.4596, + "theoretical_loss": 3.5089542186054925, + "tokens_seen": 1534995456 + }, + { + "epoch": 5.02, + "learning_rate": 0.00027018054162487463, + "loss": 2.597, + "theoretical_loss": 3.508941068364325, + "tokens_seen": 1535060992 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002701705115346038, + "loss": 2.7156, + "theoretical_loss": 3.5089279188417564, + "tokens_seen": 1535126528 + }, + { + "epoch": 5.02, + "objective/train/docs_used": 1751481, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.573998212814331, + "objective/train/theoretical_loss": 3.5089180571713676, + "objective/train/tokens_used": 1555635680, + "theoretical_loss": 3.5089180571713676, + "tokens_seen": 1535175680 + }, + { + "epoch": 5.02, + "learning_rate": 0.00027016048144433305, + "loss": 2.7034, + "theoretical_loss": 3.5089147700377152, + "tokens_seen": 1535192064 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002701504513540622, + "loss": 2.6163, + "theoretical_loss": 3.508901621952133, + "tokens_seen": 1535257600 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002701404212637914, + "loss": 2.6168, + "theoretical_loss": 3.508888474584939, + "tokens_seen": 1535323136 + }, + { + "epoch": 5.02, + "learning_rate": 0.00027013039117352054, + "loss": 2.6139, + "theoretical_loss": 3.508875327936063, + "tokens_seen": 1535388672 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002701203610832498, + "loss": 2.7076, + "theoretical_loss": 3.508862182005436, + "tokens_seen": 1535454208 + }, + { + "epoch": 5.02, + "learning_rate": 0.00027011033099297896, + "loss": 2.5746, + "theoretical_loss": 3.508849036792988, + "tokens_seen": 1535519744 + }, + { + "epoch": 5.02, + "learning_rate": 0.00027010030090270814, + "loss": 2.8796, + "theoretical_loss": 3.5088358922986482, + "tokens_seen": 1535585280 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002700902708124373, + "loss": 2.5963, + "theoretical_loss": 3.5088227485223475, + "tokens_seen": 1535650816 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002700802407221665, + "loss": 2.4503, + "theoretical_loss": 3.508809605464016, + "tokens_seen": 1535716352 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002700702106318957, + "loss": 2.7169, + "theoretical_loss": 3.5087964631235833, + "tokens_seen": 1535781888 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002700601805416249, + "loss": 2.8176, + "theoretical_loss": 3.5087833215009807, + "tokens_seen": 1535847424 + }, + { + "epoch": 5.02, + "learning_rate": 0.00027005015045135404, + "loss": 2.5217, + "theoretical_loss": 3.508770180596137, + "tokens_seen": 1535912960 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002700401203610833, + "loss": 2.5807, + "theoretical_loss": 3.508757040408983, + "tokens_seen": 1535978496 + }, + { + "epoch": 5.02, + "learning_rate": 0.00027003009027081246, + "loss": 2.8575, + "theoretical_loss": 3.508743900939449, + "tokens_seen": 1536044032 + }, + { + "epoch": 5.02, + "learning_rate": 0.00027002006018054164, + "loss": 2.7623, + "theoretical_loss": 3.508730762187465, + "tokens_seen": 1536109568 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002700100300902708, + "loss": 2.7278, + "theoretical_loss": 3.508717624152961, + "tokens_seen": 1536175104 + }, + { + "epoch": 5.02, + "learning_rate": 0.00027, + "loss": 2.656, + "theoretical_loss": 3.5087044868358683, + "tokens_seen": 1536240640 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002699899699097292, + "loss": 2.6187, + "theoretical_loss": 3.5086913502361154, + "tokens_seen": 1536306176 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002699799398194584, + "loss": 2.4959, + "theoretical_loss": 3.508678214353634, + "tokens_seen": 1536371712 + }, + { + "epoch": 5.02, + "learning_rate": 0.00026996990972918755, + "loss": 2.5832, + "theoretical_loss": 3.508665079188354, + "tokens_seen": 1536437248 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002699598796389168, + "loss": 2.5339, + "theoretical_loss": 3.5086519447402047, + "tokens_seen": 1536502784 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002699498495486459, + "loss": 2.4782, + "theoretical_loss": 3.508638811009118, + "tokens_seen": 1536568320 + }, + { + "epoch": 5.02, + "learning_rate": 0.00026993981945837514, + "loss": 2.7155, + "theoretical_loss": 3.5086256779950222, + "tokens_seen": 1536633856 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002699297893681043, + "loss": 2.7312, + "theoretical_loss": 3.5086125456978494, + "tokens_seen": 1536699392 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002699197592778335, + "loss": 2.7479, + "theoretical_loss": 3.508599414117529, + "tokens_seen": 1536764928 + }, + { + "epoch": 5.02, + "objective/train/docs_used": 1752901, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.562196969985962, + "objective/train/theoretical_loss": 3.508589565902681, + "objective/train/tokens_used": 1557274080, + "theoretical_loss": 3.508589565902681, + "tokens_seen": 1536814080 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002699097291875627, + "loss": 2.6855, + "theoretical_loss": 3.508586283253991, + "tokens_seen": 1536830464 + }, + { + "epoch": 5.02, + "learning_rate": 0.00026989969909729187, + "loss": 2.5581, + "theoretical_loss": 3.508573153107167, + "tokens_seen": 1536896000 + }, + { + "epoch": 5.02, + "learning_rate": 0.00026988966900702105, + "loss": 2.6018, + "theoretical_loss": 3.508560023676986, + "tokens_seen": 1536961536 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002698796389167503, + "loss": 2.5101, + "theoretical_loss": 3.508546894963379, + "tokens_seen": 1537027072 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002698696088264794, + "loss": 2.5575, + "theoretical_loss": 3.5085337669662753, + "tokens_seen": 1537092608 + }, + { + "epoch": 5.02, + "learning_rate": 0.00026985957873620865, + "loss": 2.6392, + "theoretical_loss": 3.5085206396856075, + "tokens_seen": 1537158144 + }, + { + "epoch": 5.02, + "learning_rate": 0.00026984954864593783, + "loss": 2.6987, + "theoretical_loss": 3.5085075131213035, + "tokens_seen": 1537223680 + }, + { + "epoch": 5.02, + "learning_rate": 0.000269839518555667, + "loss": 2.6348, + "theoretical_loss": 3.508494387273295, + "tokens_seen": 1537289216 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002698294884653962, + "loss": 2.6593, + "theoretical_loss": 3.508481262141512, + "tokens_seen": 1537354752 + }, + { + "epoch": 5.02, + "learning_rate": 0.00026981945837512537, + "loss": 2.7099, + "theoretical_loss": 3.508468137725885, + "tokens_seen": 1537420288 + }, + { + "epoch": 5.02, + "learning_rate": 0.00026980942828485455, + "loss": 2.5047, + "theoretical_loss": 3.508455014026345, + "tokens_seen": 1537485824 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002697993981945838, + "loss": 2.6675, + "theoretical_loss": 3.5084418910428212, + "tokens_seen": 1537551360 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002697893681043129, + "loss": 2.6193, + "theoretical_loss": 3.5084287687752447, + "tokens_seen": 1537616896 + }, + { + "epoch": 5.02, + "learning_rate": 0.00026977933801404215, + "loss": 2.7267, + "theoretical_loss": 3.5084156472235457, + "tokens_seen": 1537682432 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002697693079237713, + "loss": 2.6067, + "theoretical_loss": 3.508402526387655, + "tokens_seen": 1537747968 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002697592778335005, + "loss": 2.54, + "theoretical_loss": 3.5083894062675025, + "tokens_seen": 1537813504 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002697492477432297, + "loss": 2.7133, + "theoretical_loss": 3.5083762868630197, + "tokens_seen": 1537879040 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002697392176529589, + "loss": 2.3966, + "theoretical_loss": 3.508363168174136, + "tokens_seen": 1537944576 + }, + { + "epoch": 5.02, + "learning_rate": 0.00026972918756268806, + "loss": 2.5644, + "theoretical_loss": 3.508350050200782, + "tokens_seen": 1538010112 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002697191574724173, + "loss": 2.6717, + "theoretical_loss": 3.508336932942888, + "tokens_seen": 1538075648 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002697091273821464, + "loss": 2.7196, + "theoretical_loss": 3.5083238164003863, + "tokens_seen": 1538141184 + }, + { + "epoch": 5.02, + "learning_rate": 0.00026969909729187565, + "loss": 2.482, + "theoretical_loss": 3.5083107005732046, + "tokens_seen": 1538206720 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002696890672016048, + "loss": 2.6231, + "theoretical_loss": 3.5082975854612757, + "tokens_seen": 1538272256 + }, + { + "epoch": 5.02, + "learning_rate": 0.000269679037111334, + "loss": 2.6503, + "theoretical_loss": 3.508284471064529, + "tokens_seen": 1538337792 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002696690070210632, + "loss": 2.5515, + "theoretical_loss": 3.5082713573828954, + "tokens_seen": 1538403328 + }, + { + "epoch": 5.02, + "objective/train/docs_used": 1753471, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.3333654403686523, + "objective/train/theoretical_loss": 3.508261522590921, + "objective/train/tokens_used": 1558912480, + "theoretical_loss": 3.508261522590921, + "tokens_seen": 1538452480 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002696589769307924, + "loss": 2.6811, + "theoretical_loss": 3.508258244416305, + "tokens_seen": 1538468864 + }, + { + "epoch": 5.02, + "learning_rate": 0.00026964894684052156, + "loss": 2.8215, + "theoretical_loss": 3.508245132164689, + "tokens_seen": 1538534400 + }, + { + "epoch": 5.02, + "learning_rate": 0.00026963891675025074, + "loss": 2.5912, + "theoretical_loss": 3.5082320206279776, + "tokens_seen": 1538599936 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002696288866599799, + "loss": 2.5781, + "theoretical_loss": 3.508218909806101, + "tokens_seen": 1538665472 + }, + { + "epoch": 5.02, + "learning_rate": 0.00026961885656970916, + "loss": 2.7099, + "theoretical_loss": 3.508205799698991, + "tokens_seen": 1538731008 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002696088264794383, + "loss": 2.8744, + "theoretical_loss": 3.508192690306577, + "tokens_seen": 1538796544 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002695987963891675, + "loss": 2.599, + "theoretical_loss": 3.5081795816287897, + "tokens_seen": 1538862080 + }, + { + "epoch": 5.02, + "learning_rate": 0.00026958876629889665, + "loss": 2.6964, + "theoretical_loss": 3.5081664736655602, + "tokens_seen": 1538927616 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002695787362086259, + "loss": 2.7133, + "theoretical_loss": 3.5081533664168196, + "tokens_seen": 1538993152 + }, + { + "epoch": 5.02, + "learning_rate": 0.00026956870611835506, + "loss": 2.7044, + "theoretical_loss": 3.508140259882497, + "tokens_seen": 1539058688 + }, + { + "epoch": 5.02, + "learning_rate": 0.00026955867602808424, + "loss": 2.4334, + "theoretical_loss": 3.5081271540625245, + "tokens_seen": 1539124224 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002695486459378134, + "loss": 2.7716, + "theoretical_loss": 3.5081140489568314, + "tokens_seen": 1539189760 + }, + { + "epoch": 5.02, + "learning_rate": 0.00026953861584754266, + "loss": 2.6196, + "theoretical_loss": 3.50810094456535, + "tokens_seen": 1539255296 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002695285857572718, + "loss": 2.683, + "theoretical_loss": 3.5080878408880096, + "tokens_seen": 1539320832 + }, + { + "epoch": 5.02, + "learning_rate": 0.000269518555667001, + "loss": 2.5391, + "theoretical_loss": 3.5080747379247414, + "tokens_seen": 1539386368 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002695085255767302, + "loss": 2.3732, + "theoretical_loss": 3.5080616356754764, + "tokens_seen": 1539451904 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002694984954864594, + "loss": 2.4932, + "theoretical_loss": 3.5080485341401446, + "tokens_seen": 1539517440 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002694884653961886, + "loss": 2.5774, + "theoretical_loss": 3.5080354333186774, + "tokens_seen": 1539582976 + }, + { + "epoch": 5.02, + "learning_rate": 0.00026947843530591775, + "loss": 2.5516, + "theoretical_loss": 3.508022333211005, + "tokens_seen": 1539648512 + }, + { + "epoch": 5.02, + "learning_rate": 0.000269468405215647, + "loss": 2.6029, + "theoretical_loss": 3.5080092338170585, + "tokens_seen": 1539714048 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002694583751253761, + "loss": 2.5132, + "theoretical_loss": 3.507996135136768, + "tokens_seen": 1539779584 + }, + { + "epoch": 5.02, + "learning_rate": 0.00026944834503510534, + "loss": 2.6247, + "theoretical_loss": 3.507983037170065, + "tokens_seen": 1539845120 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002694383149448345, + "loss": 2.4717, + "theoretical_loss": 3.50796993991688, + "tokens_seen": 1539910656 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002694282848545637, + "loss": 2.6097, + "theoretical_loss": 3.5079568433771438, + "tokens_seen": 1539976192 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002694182547642929, + "loss": 2.463, + "theoretical_loss": 3.5079437475507866, + "tokens_seen": 1540041728 + }, + { + "epoch": 5.02, + "objective/train/docs_used": 1754850, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4801652431488037, + "objective/train/theoretical_loss": 3.507933926149133, + "objective/train/tokens_used": 1560550880, + "theoretical_loss": 3.507933926149133, + "tokens_seen": 1540090880 + }, + { + "epoch": 5.02, + "learning_rate": 0.00026940822467402207, + "loss": 2.5182, + "theoretical_loss": 3.5079306524377403, + "tokens_seen": 1540107264 + }, + { + "epoch": 5.02, + "learning_rate": 0.00026939819458375125, + "loss": 2.5174, + "theoretical_loss": 3.507917558037935, + "tokens_seen": 1540172800 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002693881644934805, + "loss": 2.5887, + "theoretical_loss": 3.5079044643513013, + "tokens_seen": 1540238336 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002693781344032096, + "loss": 2.5088, + "theoretical_loss": 3.507891371377771, + "tokens_seen": 1540303872 + }, + { + "epoch": 5.02, + "learning_rate": 0.00026936810431293885, + "loss": 2.6441, + "theoretical_loss": 3.507878279117273, + "tokens_seen": 1540369408 + }, + { + "epoch": 5.02, + "learning_rate": 0.00026935807422266803, + "loss": 2.6603, + "theoretical_loss": 3.507865187569741, + "tokens_seen": 1540434944 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002693480441323972, + "loss": 2.4486, + "theoretical_loss": 3.507852096735103, + "tokens_seen": 1540500480 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002693380140421264, + "loss": 2.4665, + "theoretical_loss": 3.507839006613291, + "tokens_seen": 1540566016 + }, + { + "epoch": 5.02, + "learning_rate": 0.00026932798395185557, + "loss": 2.7573, + "theoretical_loss": 3.5078259172042365, + "tokens_seen": 1540631552 + }, + { + "epoch": 5.02, + "learning_rate": 0.00026931795386158475, + "loss": 2.4961, + "theoretical_loss": 3.50781282850787, + "tokens_seen": 1540697088 + }, + { + "epoch": 5.02, + "learning_rate": 0.000269307923771314, + "loss": 2.6341, + "theoretical_loss": 3.5077997405241215, + "tokens_seen": 1540762624 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002692978936810431, + "loss": 2.4809, + "theoretical_loss": 3.507786653252923, + "tokens_seen": 1540828160 + }, + { + "epoch": 5.02, + "learning_rate": 0.00026928786359077235, + "loss": 2.6053, + "theoretical_loss": 3.5077735666942047, + "tokens_seen": 1540893696 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002692778335005015, + "loss": 2.6354, + "theoretical_loss": 3.507760480847898, + "tokens_seen": 1540959232 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002692678034102307, + "loss": 2.5585, + "theoretical_loss": 3.507747395713934, + "tokens_seen": 1541024768 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002692577733199599, + "loss": 2.4733, + "theoretical_loss": 3.507734311292243, + "tokens_seen": 1541090304 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002692477432296891, + "loss": 2.6158, + "theoretical_loss": 3.507721227582756, + "tokens_seen": 1541155840 + }, + { + "epoch": 5.02, + "learning_rate": 0.00026923771313941826, + "loss": 2.6618, + "theoretical_loss": 3.507708144585404, + "tokens_seen": 1541221376 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002692276830491475, + "loss": 2.8516, + "theoretical_loss": 3.507695062300118, + "tokens_seen": 1541286912 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002692176529588766, + "loss": 2.5671, + "theoretical_loss": 3.5076819807268294, + "tokens_seen": 1541352448 + }, + { + "epoch": 5.02, + "learning_rate": 0.00026920762286860585, + "loss": 2.5019, + "theoretical_loss": 3.507668899865469, + "tokens_seen": 1541417984 + }, + { + "epoch": 5.02, + "learning_rate": 0.000269197592778335, + "loss": 2.6017, + "theoretical_loss": 3.5076558197159673, + "tokens_seen": 1541483520 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002691875626880642, + "loss": 2.7025, + "theoretical_loss": 3.507642740278256, + "tokens_seen": 1541549056 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002691775325977934, + "loss": 2.6308, + "theoretical_loss": 3.5076296615522655, + "tokens_seen": 1541614592 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002691675025075226, + "loss": 2.4931, + "theoretical_loss": 3.507616583537927, + "tokens_seen": 1541680128 + }, + { + "epoch": 5.02, + "objective/train/docs_used": 1755645, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.596935987472534, + "objective/train/theoretical_loss": 3.5076067754941533, + "objective/train/tokens_used": 1562189280, + "theoretical_loss": 3.5076067754941533, + "tokens_seen": 1541729280 + }, + { + "epoch": 5.02, + "learning_rate": 0.00026915747241725176, + "loss": 2.7508, + "theoretical_loss": 3.5076035062351716, + "tokens_seen": 1541745664 + }, + { + "epoch": 5.02, + "learning_rate": 0.00026914744232698094, + "loss": 2.6642, + "theoretical_loss": 3.5075904296439306, + "tokens_seen": 1541811200 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002691374122367101, + "loss": 2.749, + "theoretical_loss": 3.507577353764135, + "tokens_seen": 1541876736 + }, + { + "epoch": 5.02, + "learning_rate": 0.00026912738214643936, + "loss": 2.6091, + "theoretical_loss": 3.507564278595715, + "tokens_seen": 1541942272 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002691173520561685, + "loss": 2.549, + "theoretical_loss": 3.5075512041386023, + "tokens_seen": 1542007808 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002691073219658977, + "loss": 2.5448, + "theoretical_loss": 3.507538130392728, + "tokens_seen": 1542073344 + }, + { + "epoch": 5.02, + "learning_rate": 0.00026909729187562685, + "loss": 2.4752, + "theoretical_loss": 3.5075250573580234, + "tokens_seen": 1542138880 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002690872617853561, + "loss": 2.6303, + "theoretical_loss": 3.5075119850344194, + "tokens_seen": 1542204416 + }, + { + "epoch": 5.02, + "learning_rate": 0.00026907723169508526, + "loss": 2.5021, + "theoretical_loss": 3.507498913421847, + "tokens_seen": 1542269952 + }, + { + "epoch": 5.02, + "learning_rate": 0.00026906720160481444, + "loss": 2.6581, + "theoretical_loss": 3.5074858425202375, + "tokens_seen": 1542335488 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002690571715145436, + "loss": 2.6053, + "theoretical_loss": 3.5074727723295216, + "tokens_seen": 1542401024 + }, + { + "epoch": 5.02, + "learning_rate": 0.00026904714142427286, + "loss": 2.5677, + "theoretical_loss": 3.507459702849631, + "tokens_seen": 1542466560 + }, + { + "epoch": 5.02, + "learning_rate": 0.000269037111334002, + "loss": 2.7589, + "theoretical_loss": 3.5074466340804964, + "tokens_seen": 1542532096 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002690270812437312, + "loss": 2.6401, + "theoretical_loss": 3.507433566022049, + "tokens_seen": 1542597632 + }, + { + "epoch": 5.02, + "learning_rate": 0.00026901705115346035, + "loss": 2.4273, + "theoretical_loss": 3.5074204986742203, + "tokens_seen": 1542663168 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002690070210631896, + "loss": 2.7618, + "theoretical_loss": 3.507407432036941, + "tokens_seen": 1542728704 + }, + { + "epoch": 5.02, + "learning_rate": 0.00026899699097291877, + "loss": 2.7275, + "theoretical_loss": 3.5073943661101428, + "tokens_seen": 1542794240 + }, + { + "epoch": 5.02, + "learning_rate": 0.00026898696088264795, + "loss": 2.6308, + "theoretical_loss": 3.5073813008937567, + "tokens_seen": 1542859776 + }, + { + "epoch": 5.02, + "learning_rate": 0.00026897693079237713, + "loss": 2.3812, + "theoretical_loss": 3.5073682363877134, + "tokens_seen": 1542925312 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002689669007021063, + "loss": 2.7549, + "theoretical_loss": 3.507355172591945, + "tokens_seen": 1542990848 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002689568706118355, + "loss": 2.3385, + "theoretical_loss": 3.507342109506382, + "tokens_seen": 1543056384 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002689468405215647, + "loss": 2.6484, + "theoretical_loss": 3.5073290471309564, + "tokens_seen": 1543121920 + }, + { + "epoch": 5.02, + "learning_rate": 0.00026893681043129385, + "loss": 2.4296, + "theoretical_loss": 3.507315985465598, + "tokens_seen": 1543187456 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002689267803410231, + "loss": 2.732, + "theoretical_loss": 3.5073029245102396, + "tokens_seen": 1543252992 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002689167502507522, + "loss": 2.429, + "theoretical_loss": 3.507289864264812, + "tokens_seen": 1543318528 + }, + { + "epoch": 5.02, + "objective/train/docs_used": 1757071, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.122141122817993, + "objective/train/theoretical_loss": 3.5072800695465913, + "objective/train/tokens_used": 1563827680, + "theoretical_loss": 3.5072800695465913, + "tokens_seen": 1543367680 + }, + { + "epoch": 5.02, + "learning_rate": 0.00026890672016048145, + "loss": 2.5328, + "theoretical_loss": 3.507276804729246, + "tokens_seen": 1543384064 + }, + { + "epoch": 5.02, + "learning_rate": 0.00026889669007021063, + "loss": 2.6986, + "theoretical_loss": 3.5072637459034732, + "tokens_seen": 1543449600 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002688866599799398, + "loss": 2.3406, + "theoretical_loss": 3.5072506877874243, + "tokens_seen": 1543515136 + }, + { + "epoch": 5.02, + "learning_rate": 0.000268876629889669, + "loss": 2.4533, + "theoretical_loss": 3.5072376303810318, + "tokens_seen": 1543580672 + }, + { + "epoch": 5.02, + "learning_rate": 0.00026886659979939823, + "loss": 2.6335, + "theoretical_loss": 3.5072245736842262, + "tokens_seen": 1543646208 + }, + { + "epoch": 5.02, + "learning_rate": 0.00026885656970912736, + "loss": 2.7103, + "theoretical_loss": 3.5072115176969394, + "tokens_seen": 1543711744 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002688465396188566, + "loss": 2.771, + "theoretical_loss": 3.507198462419102, + "tokens_seen": 1543777280 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002688365095285857, + "loss": 2.6098, + "theoretical_loss": 3.5071854078506455, + "tokens_seen": 1543842816 + }, + { + "epoch": 5.02, + "learning_rate": 0.00026882647943831495, + "loss": 2.6712, + "theoretical_loss": 3.507172353991501, + "tokens_seen": 1543908352 + }, + { + "epoch": 5.02, + "learning_rate": 0.00026881644934804413, + "loss": 2.5959, + "theoretical_loss": 3.507159300841601, + "tokens_seen": 1543973888 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002688064192577733, + "loss": 2.8207, + "theoretical_loss": 3.5071462484008755, + "tokens_seen": 1544039424 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002687963891675025, + "loss": 2.3538, + "theoretical_loss": 3.507133196669257, + "tokens_seen": 1544104960 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002687863590772317, + "loss": 2.6198, + "theoretical_loss": 3.507120145646676, + "tokens_seen": 1544170496 + }, + { + "epoch": 5.02, + "learning_rate": 0.00026877632898696086, + "loss": 2.6675, + "theoretical_loss": 3.5071070953330645, + "tokens_seen": 1544236032 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002687662988966901, + "loss": 2.5703, + "theoretical_loss": 3.507094045728353, + "tokens_seen": 1544301568 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002687562688064193, + "loss": 2.8102, + "theoretical_loss": 3.5070809968324745, + "tokens_seen": 1544367104 + }, + { + "epoch": 5.02, + "learning_rate": 0.00026874623871614846, + "loss": 2.4871, + "theoretical_loss": 3.507067948645359, + "tokens_seen": 1544432640 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002687362086258777, + "loss": 2.674, + "theoretical_loss": 3.5070549011669385, + "tokens_seen": 1544498176 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002687261785356068, + "loss": 2.6122, + "theoretical_loss": 3.507041854397144, + "tokens_seen": 1544563712 + }, + { + "epoch": 5.02, + "learning_rate": 0.00026871614844533605, + "loss": 2.4205, + "theoretical_loss": 3.5070288083359076, + "tokens_seen": 1544629248 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002687061183550652, + "loss": 2.5357, + "theoretical_loss": 3.50701576298316, + "tokens_seen": 1544694784 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002686960882647944, + "loss": 2.6311, + "theoretical_loss": 3.507002718338834, + "tokens_seen": 1544760320 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002686860581745236, + "loss": 2.7446, + "theoretical_loss": 3.5069896744028597, + "tokens_seen": 1544825856 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002686760280842528, + "loss": 2.7055, + "theoretical_loss": 3.506976631175169, + "tokens_seen": 1544891392 + }, + { + "epoch": 5.02, + "learning_rate": 0.00026866599799398196, + "loss": 2.8652, + "theoretical_loss": 3.506963588655694, + "tokens_seen": 1544956928 + }, + { + "epoch": 5.02, + "objective/train/docs_used": 1757632, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.365391969680786, + "objective/train/theoretical_loss": 3.506953807230812, + "objective/train/tokens_used": 1565466080, + "theoretical_loss": 3.506953807230812, + "tokens_seen": 1545006080 + }, + { + "epoch": 5.02, + "learning_rate": 0.00026865596790371114, + "loss": 2.5397, + "theoretical_loss": 3.506950546844365, + "tokens_seen": 1545022464 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002686459378134403, + "loss": 2.7858, + "theoretical_loss": 3.5069375057411145, + "tokens_seen": 1545088000 + }, + { + "epoch": 5.02, + "learning_rate": 0.00026863590772316956, + "loss": 2.6465, + "theoretical_loss": 3.5069244653458735, + "tokens_seen": 1545153536 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002686258776328987, + "loss": 2.4181, + "theoretical_loss": 3.506911425658574, + "tokens_seen": 1545219072 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002686158475426279, + "loss": 2.6752, + "theoretical_loss": 3.5068983866791474, + "tokens_seen": 1545284608 + }, + { + "epoch": 5.02, + "learning_rate": 0.00026860581745235705, + "loss": 2.8851, + "theoretical_loss": 3.506885348407525, + "tokens_seen": 1545350144 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002685957873620863, + "loss": 2.6663, + "theoretical_loss": 3.5068723108436384, + "tokens_seen": 1545415680 + }, + { + "epoch": 5.02, + "learning_rate": 0.00026858575727181546, + "loss": 2.6977, + "theoretical_loss": 3.50685927398742, + "tokens_seen": 1545481216 + }, + { + "epoch": 5.02, + "learning_rate": 0.00026857572718154464, + "loss": 2.7299, + "theoretical_loss": 3.5068462378387997, + "tokens_seen": 1545546752 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002685656970912738, + "loss": 2.7172, + "theoretical_loss": 3.5068332023977105, + "tokens_seen": 1545612288 + }, + { + "epoch": 5.02, + "learning_rate": 0.00026855566700100306, + "loss": 2.5429, + "theoretical_loss": 3.5068201676640838, + "tokens_seen": 1545677824 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002685456369107322, + "loss": 2.7161, + "theoretical_loss": 3.506807133637851, + "tokens_seen": 1545743360 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002685356068204614, + "loss": 2.6667, + "theoretical_loss": 3.506794100318943, + "tokens_seen": 1545808896 + }, + { + "epoch": 5.02, + "learning_rate": 0.00026852557673019055, + "loss": 2.8401, + "theoretical_loss": 3.506781067707293, + "tokens_seen": 1545874432 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002685155466399198, + "loss": 2.5245, + "theoretical_loss": 3.506768035802831, + "tokens_seen": 1545939968 + }, + { + "epoch": 5.02, + "learning_rate": 0.00026850551654964897, + "loss": 2.7198, + "theoretical_loss": 3.5067550046054903, + "tokens_seen": 1546005504 + }, + { + "epoch": 5.02, + "learning_rate": 0.00026849548645937815, + "loss": 2.5475, + "theoretical_loss": 3.506741974115201, + "tokens_seen": 1546071040 + }, + { + "epoch": 5.02, + "learning_rate": 0.00026848545636910733, + "loss": 2.363, + "theoretical_loss": 3.5067289443318956, + "tokens_seen": 1546136576 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002684754262788365, + "loss": 2.755, + "theoretical_loss": 3.506715915255506, + "tokens_seen": 1546202112 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002684653961885657, + "loss": 2.4602, + "theoretical_loss": 3.5067028868859627, + "tokens_seen": 1546267648 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002684553660982949, + "loss": 2.4851, + "theoretical_loss": 3.506689859223199, + "tokens_seen": 1546333184 + }, + { + "epoch": 5.02, + "learning_rate": 0.00026844533600802405, + "loss": 2.4482, + "theoretical_loss": 3.5066768322671455, + "tokens_seen": 1546398720 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002684353059177533, + "loss": 2.4997, + "theoretical_loss": 3.506663806017734, + "tokens_seen": 1546464256 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002684252758274824, + "loss": 2.5183, + "theoretical_loss": 3.5066507804748968, + "tokens_seen": 1546529792 + }, + { + "epoch": 5.02, + "learning_rate": 0.00026841524573721165, + "loss": 2.7061, + "theoretical_loss": 3.506637755638565, + "tokens_seen": 1546595328 + }, + { + "epoch": 5.02, + "objective/train/docs_used": 1758939, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.418158769607544, + "objective/train/theoretical_loss": 3.5066279874749195, + "objective/train/tokens_used": 1567104480, + "theoretical_loss": 3.5066279874749195, + "tokens_seen": 1546644480 + }, + { + "epoch": 5.02, + "learning_rate": 0.00026840521564694083, + "loss": 2.5213, + "theoretical_loss": 3.506624731508671, + "tokens_seen": 1546660864 + }, + { + "epoch": 5.02, + "learning_rate": 0.00026839518555667, + "loss": 2.4823, + "theoretical_loss": 3.5066117080851456, + "tokens_seen": 1546726400 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002683851554663992, + "loss": 2.4184, + "theoretical_loss": 3.506598685367922, + "tokens_seen": 1546791936 + }, + { + "epoch": 5.02, + "learning_rate": 0.00026837512537612843, + "loss": 2.6234, + "theoretical_loss": 3.5065856633569306, + "tokens_seen": 1546857472 + }, + { + "epoch": 5.02, + "learning_rate": 0.00026836509528585756, + "loss": 2.7849, + "theoretical_loss": 3.5065726420521033, + "tokens_seen": 1546923008 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002683550651955868, + "loss": 2.7529, + "theoretical_loss": 3.506559621453373, + "tokens_seen": 1546988544 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002683450351053159, + "loss": 2.6108, + "theoretical_loss": 3.5065466015606708, + "tokens_seen": 1547054080 + }, + { + "epoch": 5.02, + "learning_rate": 0.00026833500501504515, + "loss": 2.7479, + "theoretical_loss": 3.506533582373928, + "tokens_seen": 1547119616 + }, + { + "epoch": 5.02, + "learning_rate": 0.00026832497492477433, + "loss": 2.6387, + "theoretical_loss": 3.506520563893077, + "tokens_seen": 1547185152 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002683149448345035, + "loss": 2.6088, + "theoretical_loss": 3.5065075461180495, + "tokens_seen": 1547250688 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002683049147442327, + "loss": 2.7851, + "theoretical_loss": 3.506494529048778, + "tokens_seen": 1547316224 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002682948846539619, + "loss": 2.5244, + "theoretical_loss": 3.506481512685193, + "tokens_seen": 1547381760 + }, + { + "epoch": 5.02, + "learning_rate": 0.00026828485456369106, + "loss": 2.6853, + "theoretical_loss": 3.5064684970272277, + "tokens_seen": 1547447296 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002682748244734203, + "loss": 2.7179, + "theoretical_loss": 3.506455482074813, + "tokens_seen": 1547512832 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002682647943831494, + "loss": 2.4258, + "theoretical_loss": 3.506442467827881, + "tokens_seen": 1547578368 + }, + { + "epoch": 5.02, + "learning_rate": 0.00026825476429287866, + "loss": 2.7019, + "theoretical_loss": 3.5064294542863643, + "tokens_seen": 1547643904 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002682447342026078, + "loss": 2.7355, + "theoretical_loss": 3.506416441450194, + "tokens_seen": 1547709440 + }, + { + "epoch": 5.02, + "learning_rate": 0.000268234704112337, + "loss": 2.5778, + "theoretical_loss": 3.506403429319302, + "tokens_seen": 1547774976 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002682246740220662, + "loss": 2.4957, + "theoretical_loss": 3.5063904178936207, + "tokens_seen": 1547840512 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002682146439317954, + "loss": 2.7598, + "theoretical_loss": 3.5063774071730816, + "tokens_seen": 1547906048 + }, + { + "epoch": 5.02, + "learning_rate": 0.00026820461384152456, + "loss": 2.859, + "theoretical_loss": 3.5063643971576166, + "tokens_seen": 1547971584 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002681945837512538, + "loss": 2.73, + "theoretical_loss": 3.5063513878471584, + "tokens_seen": 1548037120 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002681845536609829, + "loss": 2.6316, + "theoretical_loss": 3.506338379241638, + "tokens_seen": 1548102656 + }, + { + "epoch": 5.02, + "learning_rate": 0.00026817452357071216, + "loss": 2.4397, + "theoretical_loss": 3.5063253713409877, + "tokens_seen": 1548168192 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002681644934804413, + "loss": 2.6663, + "theoretical_loss": 3.5063123641451397, + "tokens_seen": 1548233728 + }, + { + "epoch": 5.02, + "objective/train/docs_used": 1759677, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.676605463027954, + "objective/train/theoretical_loss": 3.5063026092107394, + "objective/train/tokens_used": 1568742880, + "theoretical_loss": 3.5063026092107394, + "tokens_seen": 1548282880 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002681544633901705, + "loss": 2.6747, + "theoretical_loss": 3.506299357654026, + "tokens_seen": 1548299264 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002681444332998997, + "loss": 2.5841, + "theoretical_loss": 3.506286351867578, + "tokens_seen": 1548364800 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002681344032096289, + "loss": 2.6078, + "theoretical_loss": 3.5062733467857283, + "tokens_seen": 1548430336 + }, + { + "epoch": 5.02, + "learning_rate": 0.00026812437311935807, + "loss": 2.7631, + "theoretical_loss": 3.5062603424084084, + "tokens_seen": 1548495872 + }, + { + "epoch": 5.02, + "learning_rate": 0.00026811434302908725, + "loss": 2.5097, + "theoretical_loss": 3.506247338735551, + "tokens_seen": 1548561408 + }, + { + "epoch": 5.02, + "learning_rate": 0.00026810431293881643, + "loss": 2.6734, + "theoretical_loss": 3.506234335767088, + "tokens_seen": 1548626944 + }, + { + "epoch": 5.02, + "learning_rate": 0.00026809428284854566, + "loss": 2.2408, + "theoretical_loss": 3.5062213335029506, + "tokens_seen": 1548692480 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002680842527582748, + "loss": 2.6834, + "theoretical_loss": 3.5062083319430717, + "tokens_seen": 1548758016 + }, + { + "epoch": 5.02, + "learning_rate": 0.000268074222668004, + "loss": 2.9695, + "theoretical_loss": 3.5061953310873832, + "tokens_seen": 1548823552 + }, + { + "epoch": 5.02, + "learning_rate": 0.00026806419257773315, + "loss": 2.544, + "theoretical_loss": 3.5061823309358164, + "tokens_seen": 1548889088 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002680541624874624, + "loss": 2.6153, + "theoretical_loss": 3.506169331488305, + "tokens_seen": 1548954624 + }, + { + "epoch": 5.02, + "learning_rate": 0.00026804413239719157, + "loss": 2.7174, + "theoretical_loss": 3.506156332744779, + "tokens_seen": 1549020160 + }, + { + "epoch": 5.02, + "learning_rate": 0.00026803410230692075, + "loss": 2.7422, + "theoretical_loss": 3.5061433347051727, + "tokens_seen": 1549085696 + }, + { + "epoch": 5.02, + "learning_rate": 0.00026802407221664993, + "loss": 2.7367, + "theoretical_loss": 3.5061303373694166, + "tokens_seen": 1549151232 + }, + { + "epoch": 5.02, + "learning_rate": 0.00026801404212637917, + "loss": 2.5945, + "theoretical_loss": 3.5061173407374433, + "tokens_seen": 1549216768 + }, + { + "epoch": 5.02, + "learning_rate": 0.00026800401203610835, + "loss": 2.4936, + "theoretical_loss": 3.5061043448091853, + "tokens_seen": 1549282304 + }, + { + "epoch": 5.02, + "learning_rate": 0.00026799398194583753, + "loss": 2.7071, + "theoretical_loss": 3.5060913495845742, + "tokens_seen": 1549347840 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002679839518555667, + "loss": 2.6455, + "theoretical_loss": 3.506078355063542, + "tokens_seen": 1549413376 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002679739217652959, + "loss": 2.599, + "theoretical_loss": 3.506065361246021, + "tokens_seen": 1549478912 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002679638916750251, + "loss": 2.7328, + "theoretical_loss": 3.5060523681319444, + "tokens_seen": 1549544448 + }, + { + "epoch": 5.02, + "learning_rate": 0.00026795386158475425, + "loss": 2.6709, + "theoretical_loss": 3.506039375721243, + "tokens_seen": 1549609984 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002679438314944835, + "loss": 2.8018, + "theoretical_loss": 3.5060263840138495, + "tokens_seen": 1549675520 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002679338014042126, + "loss": 2.733, + "theoretical_loss": 3.506013393009696, + "tokens_seen": 1549741056 + }, + { + "epoch": 5.02, + "learning_rate": 0.00026792377131394185, + "loss": 2.5778, + "theoretical_loss": 3.506000402708715, + "tokens_seen": 1549806592 + }, + { + "epoch": 5.02, + "learning_rate": 0.00026791374122367103, + "loss": 2.4503, + "theoretical_loss": 3.505987413110838, + "tokens_seen": 1549872128 + }, + { + "epoch": 5.02, + "objective/train/docs_used": 1761272, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8500523567199707, + "objective/train/theoretical_loss": 3.5059776713738025, + "objective/train/tokens_used": 1570381280, + "theoretical_loss": 3.5059776713738025, + "tokens_seen": 1549921280 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002679037111334002, + "loss": 2.5811, + "theoretical_loss": 3.505974424215998, + "tokens_seen": 1549937664 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002678936810431294, + "loss": 2.7321, + "theoretical_loss": 3.505961436024127, + "tokens_seen": 1550003200 + }, + { + "epoch": 5.02, + "learning_rate": 0.00026788365095285863, + "loss": 2.4762, + "theoretical_loss": 3.505948448535157, + "tokens_seen": 1550068736 + }, + { + "epoch": 5.02, + "learning_rate": 0.00026787362086258776, + "loss": 2.72, + "theoretical_loss": 3.5059354617490204, + "tokens_seen": 1550134272 + }, + { + "epoch": 5.02, + "learning_rate": 0.000267863590772317, + "loss": 2.6949, + "theoretical_loss": 3.5059224756656495, + "tokens_seen": 1550199808 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002678535606820461, + "loss": 2.7632, + "theoretical_loss": 3.505909490284976, + "tokens_seen": 1550265344 + }, + { + "epoch": 5.02, + "learning_rate": 0.00026784353059177535, + "loss": 2.6267, + "theoretical_loss": 3.5058965056069336, + "tokens_seen": 1550330880 + }, + { + "epoch": 5.02, + "learning_rate": 0.00026783350050150453, + "loss": 2.6218, + "theoretical_loss": 3.5058835216314534, + "tokens_seen": 1550396416 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002678234704112337, + "loss": 2.7226, + "theoretical_loss": 3.5058705383584674, + "tokens_seen": 1550461952 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002678134403209629, + "loss": 2.5492, + "theoretical_loss": 3.505857555787909, + "tokens_seen": 1550527488 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002678034102306921, + "loss": 2.5225, + "theoretical_loss": 3.5058445739197097, + "tokens_seen": 1550593024 + }, + { + "epoch": 5.02, + "learning_rate": 0.00026779338014042126, + "loss": 2.6679, + "theoretical_loss": 3.505831592753802, + "tokens_seen": 1550658560 + }, + { + "epoch": 5.02, + "learning_rate": 0.00026779338014042126, + "loss": 2.7302, + "theoretical_loss": 3.5058186122901187, + "tokens_seen": 1550724096 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002677833500501505, + "loss": 2.5314, + "theoretical_loss": 3.5058056325285913, + "tokens_seen": 1550789632 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002677733199598796, + "loss": 2.5831, + "theoretical_loss": 3.5057926534691526, + "tokens_seen": 1550855168 + }, + { + "epoch": 5.02, + "learning_rate": 0.00026776328986960886, + "loss": 2.685, + "theoretical_loss": 3.505779675111735, + "tokens_seen": 1550920704 + }, + { + "epoch": 5.02, + "learning_rate": 0.000267753259779338, + "loss": 2.6247, + "theoretical_loss": 3.505766697456271, + "tokens_seen": 1550986240 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002677432296890672, + "loss": 2.6779, + "theoretical_loss": 3.505753720502693, + "tokens_seen": 1551051776 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002677331995987964, + "loss": 2.5999, + "theoretical_loss": 3.5057407442509327, + "tokens_seen": 1551117312 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002677231695085256, + "loss": 2.4158, + "theoretical_loss": 3.5057277687009236, + "tokens_seen": 1551182848 + }, + { + "epoch": 5.02, + "learning_rate": 0.00026771313941825476, + "loss": 2.4457, + "theoretical_loss": 3.5057147938525968, + "tokens_seen": 1551248384 + }, + { + "epoch": 5.02, + "learning_rate": 0.000267703109327984, + "loss": 2.3523, + "theoretical_loss": 3.5057018197058856, + "tokens_seen": 1551313920 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002676930792377131, + "loss": 2.6527, + "theoretical_loss": 3.505688846260722, + "tokens_seen": 1551379456 + }, + { + "epoch": 5.02, + "learning_rate": 0.00026768304914744236, + "loss": 2.7381, + "theoretical_loss": 3.5056758735170392, + "tokens_seen": 1551444992 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002676730190571715, + "loss": 2.6779, + "theoretical_loss": 3.5056629014747687, + "tokens_seen": 1551510528 + }, + { + "epoch": 5.02, + "objective/train/docs_used": 1761875, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.585963010787964, + "objective/train/theoretical_loss": 3.5056531729033273, + "objective/train/tokens_used": 1572019680, + "theoretical_loss": 3.5056531729033273, + "tokens_seen": 1551559680 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002676629889669007, + "loss": 2.6818, + "theoretical_loss": 3.5056499301338433, + "tokens_seen": 1551576064 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002676529588766299, + "loss": 2.6806, + "theoretical_loss": 3.5056369594941956, + "tokens_seen": 1551641600 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002676429287863591, + "loss": 2.4555, + "theoretical_loss": 3.505623989555758, + "tokens_seen": 1551707136 + }, + { + "epoch": 5.02, + "learning_rate": 0.00026763289869608827, + "loss": 2.4361, + "theoretical_loss": 3.505611020318463, + "tokens_seen": 1551772672 + }, + { + "epoch": 5.02, + "learning_rate": 0.00026762286860581745, + "loss": 2.6085, + "theoretical_loss": 3.505598051782243, + "tokens_seen": 1551838208 + }, + { + "epoch": 5.02, + "learning_rate": 0.00026761283851554663, + "loss": 2.5242, + "theoretical_loss": 3.5055850839470306, + "tokens_seen": 1551903744 + }, + { + "epoch": 5.02, + "learning_rate": 0.00026760280842527586, + "loss": 2.6377, + "theoretical_loss": 3.5055721168127576, + "tokens_seen": 1551969280 + }, + { + "epoch": 5.02, + "learning_rate": 0.000267592778335005, + "loss": 2.6599, + "theoretical_loss": 3.5055591503793577, + "tokens_seen": 1552034816 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002675827482447342, + "loss": 2.6179, + "theoretical_loss": 3.505546184646763, + "tokens_seen": 1552100352 + }, + { + "epoch": 5.02, + "learning_rate": 0.00026757271815446335, + "loss": 2.6306, + "theoretical_loss": 3.5055332196149056, + "tokens_seen": 1552165888 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002675626880641926, + "loss": 2.6899, + "theoretical_loss": 3.505520255283719, + "tokens_seen": 1552231424 + }, + { + "epoch": 5.02, + "learning_rate": 0.00026755265797392177, + "loss": 2.6717, + "theoretical_loss": 3.5055072916531342, + "tokens_seen": 1552296960 + }, + { + "epoch": 5.02, + "learning_rate": 0.00026754262788365095, + "loss": 2.8465, + "theoretical_loss": 3.505494328723085, + "tokens_seen": 1552362496 + }, + { + "epoch": 5.02, + "learning_rate": 0.00026753259779338013, + "loss": 2.7475, + "theoretical_loss": 3.5054813664935036, + "tokens_seen": 1552428032 + }, + { + "epoch": 5.02, + "learning_rate": 0.00026752256770310937, + "loss": 2.6695, + "theoretical_loss": 3.505468404964323, + "tokens_seen": 1552493568 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002675125376128385, + "loss": 2.5977, + "theoretical_loss": 3.5054554441354755, + "tokens_seen": 1552559104 + }, + { + "epoch": 5.02, + "learning_rate": 0.00026750250752256773, + "loss": 2.4163, + "theoretical_loss": 3.5054424840068927, + "tokens_seen": 1552624640 + }, + { + "epoch": 5.02, + "learning_rate": 0.00026749247743229686, + "loss": 2.7349, + "theoretical_loss": 3.505429524578509, + "tokens_seen": 1552690176 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002674824473420261, + "loss": 2.5672, + "theoretical_loss": 3.505416565850256, + "tokens_seen": 1552755712 + }, + { + "epoch": 5.02, + "learning_rate": 0.00026747241725175527, + "loss": 2.6153, + "theoretical_loss": 3.505403607822066, + "tokens_seen": 1552821248 + }, + { + "epoch": 5.02, + "learning_rate": 0.00026746238716148445, + "loss": 2.814, + "theoretical_loss": 3.5053906504938723, + "tokens_seen": 1552886784 + }, + { + "epoch": 5.02, + "learning_rate": 0.00026745235707121363, + "loss": 2.6414, + "theoretical_loss": 3.505377693865608, + "tokens_seen": 1552952320 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002674423269809428, + "loss": 2.7226, + "theoretical_loss": 3.505364737937204, + "tokens_seen": 1553017856 + }, + { + "epoch": 5.02, + "learning_rate": 0.000267432296890672, + "loss": 2.7976, + "theoretical_loss": 3.505351782708595, + "tokens_seen": 1553083392 + }, + { + "epoch": 5.02, + "learning_rate": 0.00026742226680040123, + "loss": 2.9329, + "theoretical_loss": 3.5053388281797124, + "tokens_seen": 1553148928 + }, + { + "epoch": 5.02, + "objective/train/docs_used": 1763200, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.717268466949463, + "objective/train/theoretical_loss": 3.5053291127422055, + "objective/train/tokens_used": 1573658080, + "theoretical_loss": 3.5053291127422055, + "tokens_seen": 1553198080 + }, + { + "epoch": 5.02, + "learning_rate": 0.00026741223671013036, + "loss": 2.7241, + "theoretical_loss": 3.505325874350489, + "tokens_seen": 1553214464 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002674022066198596, + "loss": 2.6349, + "theoretical_loss": 3.505312921220858, + "tokens_seen": 1553280000 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002673921765295888, + "loss": 2.6627, + "theoretical_loss": 3.5052999687907516, + "tokens_seen": 1553345536 + }, + { + "epoch": 5.02, + "learning_rate": 0.00026738214643931796, + "loss": 2.6706, + "theoretical_loss": 3.505287017060103, + "tokens_seen": 1553411072 + }, + { + "epoch": 5.02, + "learning_rate": 0.00026737211634904714, + "loss": 2.7722, + "theoretical_loss": 3.5052740660288446, + "tokens_seen": 1553476608 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002673620862587763, + "loss": 2.7066, + "theoretical_loss": 3.5052611156969093, + "tokens_seen": 1553542144 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002673520561685055, + "loss": 2.5324, + "theoretical_loss": 3.50524816606423, + "tokens_seen": 1553607680 + }, + { + "epoch": 5.02, + "learning_rate": 0.00026734202607823474, + "loss": 2.6319, + "theoretical_loss": 3.505235217130738, + "tokens_seen": 1553673216 + }, + { + "epoch": 5.02, + "learning_rate": 0.00026733199598796386, + "loss": 2.7366, + "theoretical_loss": 3.5052222688963686, + "tokens_seen": 1553738752 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002673219658976931, + "loss": 2.8268, + "theoretical_loss": 3.5052093213610522, + "tokens_seen": 1553804288 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002673119358074222, + "loss": 2.4775, + "theoretical_loss": 3.5051963745247234, + "tokens_seen": 1553869824 + }, + { + "epoch": 5.02, + "learning_rate": 0.00026730190571715146, + "loss": 2.6987, + "theoretical_loss": 3.505183428387314, + "tokens_seen": 1553935360 + }, + { + "epoch": 5.02, + "learning_rate": 0.00026729187562688064, + "loss": 2.7447, + "theoretical_loss": 3.5051704829487567, + "tokens_seen": 1554000896 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002672818455366098, + "loss": 2.7616, + "theoretical_loss": 3.5051575382089846, + "tokens_seen": 1554066432 + }, + { + "epoch": 5.02, + "learning_rate": 0.000267271815446339, + "loss": 2.8065, + "theoretical_loss": 3.5051445941679304, + "tokens_seen": 1554131968 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002672617853560682, + "loss": 2.7473, + "theoretical_loss": 3.5051316508255272, + "tokens_seen": 1554197504 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002672517552657974, + "loss": 2.6068, + "theoretical_loss": 3.5051187081817075, + "tokens_seen": 1554263040 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002672417251755266, + "loss": 2.6967, + "theoretical_loss": 3.5051057662364045, + "tokens_seen": 1554328576 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002672316950852558, + "loss": 2.6442, + "theoretical_loss": 3.5050928249895508, + "tokens_seen": 1554394112 + }, + { + "epoch": 5.02, + "learning_rate": 0.00026722166499498496, + "loss": 2.6032, + "theoretical_loss": 3.5050798844410793, + "tokens_seen": 1554459648 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002672116349047142, + "loss": 2.603, + "theoretical_loss": 3.5050669445909226, + "tokens_seen": 1554525184 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002672016048144433, + "loss": 2.5763, + "theoretical_loss": 3.505054005439014, + "tokens_seen": 1554590720 + }, + { + "epoch": 5.02, + "learning_rate": 0.00026719157472417256, + "loss": 2.8347, + "theoretical_loss": 3.505041066985286, + "tokens_seen": 1554656256 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002671815446339017, + "loss": 2.6721, + "theoretical_loss": 3.5050281292296726, + "tokens_seen": 1554721792 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002671715145436309, + "loss": 2.6613, + "theoretical_loss": 3.505015192172105, + "tokens_seen": 1554787328 + }, + { + "epoch": 5.02, + "objective/train/docs_used": 1763875, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4189612865448, + "objective/train/theoretical_loss": 3.5050054898369822, + "objective/train/tokens_used": 1575296480, + "theoretical_loss": 3.5050054898369822, + "tokens_seen": 1554836480 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002671614844533601, + "loss": 2.6215, + "theoretical_loss": 3.505002255812517, + "tokens_seen": 1554852864 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002671514543630893, + "loss": 2.4481, + "theoretical_loss": 3.504989320150842, + "tokens_seen": 1554918400 + }, + { + "epoch": 5.02, + "learning_rate": 0.00026714142427281847, + "loss": 2.583, + "theoretical_loss": 3.504976385187012, + "tokens_seen": 1554983936 + }, + { + "epoch": 5.02, + "learning_rate": 0.00026713139418254765, + "loss": 2.4658, + "theoretical_loss": 3.5049634509209606, + "tokens_seen": 1555049472 + }, + { + "epoch": 5.02, + "learning_rate": 0.00026712136409227683, + "loss": 2.6812, + "theoretical_loss": 3.5049505173526203, + "tokens_seen": 1555115008 + }, + { + "epoch": 5.02, + "learning_rate": 0.00026711133400200606, + "loss": 2.7396, + "theoretical_loss": 3.504937584481924, + "tokens_seen": 1555180544 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002671013039117352, + "loss": 2.8398, + "theoretical_loss": 3.5049246523088056, + "tokens_seen": 1555246080 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002670912738214644, + "loss": 2.6925, + "theoretical_loss": 3.5049117208331975, + "tokens_seen": 1555311616 + }, + { + "epoch": 5.02, + "learning_rate": 0.00026708124373119355, + "loss": 2.7556, + "theoretical_loss": 3.504898790055032, + "tokens_seen": 1555377152 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002670712136409228, + "loss": 2.7088, + "theoretical_loss": 3.504885859974243, + "tokens_seen": 1555442688 + }, + { + "epoch": 5.02, + "learning_rate": 0.00026706118355065197, + "loss": 2.6861, + "theoretical_loss": 3.5048729305907633, + "tokens_seen": 1555508224 + }, + { + "epoch": 5.02, + "learning_rate": 0.00026705115346038115, + "loss": 2.5036, + "theoretical_loss": 3.5048600019045257, + "tokens_seen": 1555573760 + }, + { + "epoch": 5.02, + "learning_rate": 0.00026704112337011033, + "loss": 2.6393, + "theoretical_loss": 3.5048470739154634, + "tokens_seen": 1555639296 + }, + { + "epoch": 5.02, + "learning_rate": 0.00026703109327983957, + "loss": 2.4422, + "theoretical_loss": 3.5048341466235096, + "tokens_seen": 1555704832 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002670210631895687, + "loss": 2.7981, + "theoretical_loss": 3.504821220028597, + "tokens_seen": 1555770368 + }, + { + "epoch": 5.02, + "learning_rate": 0.00026701103309929793, + "loss": 2.5328, + "theoretical_loss": 3.504808294130659, + "tokens_seen": 1555835904 + }, + { + "epoch": 5.02, + "learning_rate": 0.00026700100300902706, + "loss": 2.7095, + "theoretical_loss": 3.504795368929628, + "tokens_seen": 1555901440 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002669909729187563, + "loss": 2.6313, + "theoretical_loss": 3.504782444425438, + "tokens_seen": 1555966976 + }, + { + "epoch": 5.02, + "learning_rate": 0.00026698094282848547, + "loss": 2.54, + "theoretical_loss": 3.504769520618021, + "tokens_seen": 1556032512 + }, + { + "epoch": 5.02, + "learning_rate": 0.00026697091273821465, + "loss": 2.7396, + "theoretical_loss": 3.5047565975073116, + "tokens_seen": 1556098048 + }, + { + "epoch": 5.02, + "learning_rate": 0.00026696088264794383, + "loss": 2.5316, + "theoretical_loss": 3.5047436750932417, + "tokens_seen": 1556163584 + }, + { + "epoch": 5.02, + "learning_rate": 0.000266950852557673, + "loss": 2.624, + "theoretical_loss": 3.504730753375745, + "tokens_seen": 1556229120 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002669408224674022, + "loss": 2.7467, + "theoretical_loss": 3.504717832354754, + "tokens_seen": 1556294656 + }, + { + "epoch": 5.02, + "learning_rate": 0.00026693079237713143, + "loss": 2.5787, + "theoretical_loss": 3.504704912030202, + "tokens_seen": 1556360192 + }, + { + "epoch": 5.02, + "learning_rate": 0.00026692076228686056, + "loss": 2.6299, + "theoretical_loss": 3.504691992402023, + "tokens_seen": 1556425728 + }, + { + "epoch": 5.02, + "objective/train/docs_used": 1765110, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9499735832214355, + "objective/train/theoretical_loss": 3.5046823031378422, + "objective/train/tokens_used": 1576934880, + "theoretical_loss": 3.5046823031378422, + "tokens_seen": 1556474880 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002669107321965898, + "loss": 2.9113, + "theoretical_loss": 3.5046790734701485, + "tokens_seen": 1556491264 + }, + { + "epoch": 5.02, + "learning_rate": 0.000266900702106319, + "loss": 2.4546, + "theoretical_loss": 3.5046661552345135, + "tokens_seen": 1556556800 + }, + { + "epoch": 5.02, + "learning_rate": 0.00026689067201604816, + "loss": 2.8173, + "theoretical_loss": 3.5046532376950497, + "tokens_seen": 1556622336 + }, + { + "epoch": 5.02, + "learning_rate": 0.00026688064192577734, + "loss": 2.6191, + "theoretical_loss": 3.504640320851691, + "tokens_seen": 1556687872 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002668706118355065, + "loss": 2.6093, + "theoretical_loss": 3.504627404704371, + "tokens_seen": 1556753408 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002668605817452357, + "loss": 2.7305, + "theoretical_loss": 3.504614489253022, + "tokens_seen": 1556818944 + }, + { + "epoch": 5.02, + "learning_rate": 0.00026685055165496494, + "loss": 2.5957, + "theoretical_loss": 3.5046015744975776, + "tokens_seen": 1556884480 + }, + { + "epoch": 5.02, + "learning_rate": 0.00026684052156469406, + "loss": 2.744, + "theoretical_loss": 3.504588660437971, + "tokens_seen": 1556950016 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002668304914744233, + "loss": 2.8159, + "theoretical_loss": 3.504575747074135, + "tokens_seen": 1557015552 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002668204613841524, + "loss": 2.5389, + "theoretical_loss": 3.504562834406004, + "tokens_seen": 1557081088 + }, + { + "epoch": 5.02, + "learning_rate": 0.00026681043129388166, + "loss": 2.4884, + "theoretical_loss": 3.50454992243351, + "tokens_seen": 1557146624 + }, + { + "epoch": 5.02, + "learning_rate": 0.00026680040120361084, + "loss": 2.7264, + "theoretical_loss": 3.5045370111565863, + "tokens_seen": 1557212160 + }, + { + "epoch": 5.02, + "learning_rate": 0.00026679037111334, + "loss": 2.6224, + "theoretical_loss": 3.504524100575167, + "tokens_seen": 1557277696 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002667803410230692, + "loss": 2.5575, + "theoretical_loss": 3.504511190689185, + "tokens_seen": 1557343232 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002667703109327984, + "loss": 2.5781, + "theoretical_loss": 3.5044982814985737, + "tokens_seen": 1557408768 + }, + { + "epoch": 5.02, + "learning_rate": 0.00026676028084252757, + "loss": 2.6302, + "theoretical_loss": 3.5044853730032655, + "tokens_seen": 1557474304 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002667502507522568, + "loss": 2.5893, + "theoretical_loss": 3.504472465203195, + "tokens_seen": 1557539840 + }, + { + "epoch": 5.02, + "learning_rate": 0.00026674022066198593, + "loss": 2.5045, + "theoretical_loss": 3.5044595580982945, + "tokens_seen": 1557605376 + }, + { + "epoch": 5.02, + "learning_rate": 0.00026673019057171516, + "loss": 2.556, + "theoretical_loss": 3.504446651688498, + "tokens_seen": 1557670912 + }, + { + "epoch": 5.02, + "learning_rate": 0.00026672016048144434, + "loss": 2.8052, + "theoretical_loss": 3.5044337459737385, + "tokens_seen": 1557736448 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002667101303911735, + "loss": 2.6902, + "theoretical_loss": 3.5044208409539492, + "tokens_seen": 1557801984 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002667001003009027, + "loss": 2.7238, + "theoretical_loss": 3.5044079366290637, + "tokens_seen": 1557867520 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002666900702106319, + "loss": 2.6548, + "theoretical_loss": 3.5043950329990152, + "tokens_seen": 1557933056 + }, + { + "epoch": 5.02, + "learning_rate": 0.00026668004012036107, + "loss": 2.6523, + "theoretical_loss": 3.504382130063737, + "tokens_seen": 1557998592 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002666700100300903, + "loss": 2.5946, + "theoretical_loss": 3.504369227823163, + "tokens_seen": 1558064128 + }, + { + "epoch": 5.02, + "objective/train/docs_used": 1766105, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.555666446685791, + "objective/train/theoretical_loss": 3.5043595515985917, + "objective/train/tokens_used": 1578573280, + "theoretical_loss": 3.5043595515985917, + "tokens_seen": 1558113280 + }, + { + "epoch": 5.02, + "learning_rate": 0.00026665997993981943, + "loss": 2.6693, + "theoretical_loss": 3.504356326277226, + "tokens_seen": 1558129664 + }, + { + "epoch": 5.02, + "learning_rate": 0.00026664994984954867, + "loss": 2.6442, + "theoretical_loss": 3.5043434254258594, + "tokens_seen": 1558195200 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002666399197592778, + "loss": 2.6077, + "theoretical_loss": 3.504330525268997, + "tokens_seen": 1558260736 + }, + { + "epoch": 5.02, + "learning_rate": 0.00026662988966900703, + "loss": 2.8557, + "theoretical_loss": 3.504317625806572, + "tokens_seen": 1558326272 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002666198595787362, + "loss": 2.6912, + "theoretical_loss": 3.5043047270385177, + "tokens_seen": 1558391808 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002666098294884654, + "loss": 2.5286, + "theoretical_loss": 3.5042918289647673, + "tokens_seen": 1558457344 + }, + { + "epoch": 5.02, + "learning_rate": 0.00026659979939819457, + "loss": 2.6179, + "theoretical_loss": 3.5042789315852545, + "tokens_seen": 1558522880 + }, + { + "epoch": 5.02, + "learning_rate": 0.00026658976930792375, + "loss": 2.7925, + "theoretical_loss": 3.504266034899913, + "tokens_seen": 1558588416 + }, + { + "epoch": 5.02, + "learning_rate": 0.00026657973921765293, + "loss": 2.6147, + "theoretical_loss": 3.504253138908676, + "tokens_seen": 1558653952 + }, + { + "epoch": 5.02, + "learning_rate": 0.00026656970912738217, + "loss": 2.64, + "theoretical_loss": 3.504240243611477, + "tokens_seen": 1558719488 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002665596790371113, + "loss": 2.7022, + "theoretical_loss": 3.504227349008249, + "tokens_seen": 1558785024 + }, + { + "epoch": 5.02, + "learning_rate": 0.00026654964894684053, + "loss": 2.422, + "theoretical_loss": 3.5042144550989267, + "tokens_seen": 1558850560 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002665396188565697, + "loss": 2.7155, + "theoretical_loss": 3.5042015618834426, + "tokens_seen": 1558916096 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002665295887662989, + "loss": 2.6079, + "theoretical_loss": 3.50418866936173, + "tokens_seen": 1558981632 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002665195586760281, + "loss": 2.6836, + "theoretical_loss": 3.5041757775337237, + "tokens_seen": 1559047168 + }, + { + "epoch": 5.02, + "learning_rate": 0.00026650952858575726, + "loss": 2.4562, + "theoretical_loss": 3.5041628863993557, + "tokens_seen": 1559112704 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002664994984954865, + "loss": 2.7825, + "theoretical_loss": 3.50414999595856, + "tokens_seen": 1559178240 + }, + { + "epoch": 5.02, + "learning_rate": 0.00026648946840521567, + "loss": 2.8305, + "theoretical_loss": 3.5041371062112705, + "tokens_seen": 1559243776 + }, + { + "epoch": 5.02, + "learning_rate": 0.00026647943831494485, + "loss": 2.5331, + "theoretical_loss": 3.504124217157421, + "tokens_seen": 1559309312 + }, + { + "epoch": 5.02, + "learning_rate": 0.00026646940822467403, + "loss": 2.619, + "theoretical_loss": 3.5041113287969434, + "tokens_seen": 1559374848 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002664593781344032, + "loss": 2.7494, + "theoretical_loss": 3.5040984411297735, + "tokens_seen": 1559440384 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002664493480441324, + "loss": 2.7947, + "theoretical_loss": 3.5040855541558438, + "tokens_seen": 1559505920 + }, + { + "epoch": 5.02, + "learning_rate": 0.00026643931795386163, + "loss": 2.5748, + "theoretical_loss": 3.5040726678750875, + "tokens_seen": 1559571456 + }, + { + "epoch": 5.02, + "learning_rate": 0.00026642928786359076, + "loss": 2.9177, + "theoretical_loss": 3.5040597822874386, + "tokens_seen": 1559636992 + }, + { + "epoch": 5.02, + "learning_rate": 0.00026641925777332, + "loss": 2.75, + "theoretical_loss": 3.504046897392831, + "tokens_seen": 1559702528 + }, + { + "epoch": 5.02, + "objective/train/docs_used": 1766788, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9355313777923584, + "objective/train/theoretical_loss": 3.5040372341766433, + "objective/train/tokens_used": 1580211680, + "theoretical_loss": 3.5040372341766433, + "tokens_seen": 1559751680 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002664092276830492, + "loss": 2.9388, + "theoretical_loss": 3.5040340131911973, + "tokens_seen": 1559768064 + }, + { + "epoch": 5.02, + "learning_rate": 0.00026639919759277836, + "loss": 2.8814, + "theoretical_loss": 3.5040211296824726, + "tokens_seen": 1559833600 + }, + { + "epoch": 5.02, + "learning_rate": 0.00026638916750250754, + "loss": 2.5502, + "theoretical_loss": 3.5040082468665896, + "tokens_seen": 1559899136 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002663791374122367, + "loss": 2.7294, + "theoretical_loss": 3.503995364743482, + "tokens_seen": 1559964672 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002663691073219659, + "loss": 2.781, + "theoretical_loss": 3.5039824833130835, + "tokens_seen": 1560030208 + }, + { + "epoch": 5.02, + "learning_rate": 0.00026635907723169514, + "loss": 2.7887, + "theoretical_loss": 3.503969602575328, + "tokens_seen": 1560095744 + }, + { + "epoch": 5.02, + "learning_rate": 0.00026634904714142426, + "loss": 2.626, + "theoretical_loss": 3.5039567225301482, + "tokens_seen": 1560161280 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002663390170511535, + "loss": 2.7191, + "theoretical_loss": 3.5039438431774794, + "tokens_seen": 1560226816 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002663289869608826, + "loss": 2.5363, + "theoretical_loss": 3.5039309645172536, + "tokens_seen": 1560292352 + }, + { + "epoch": 5.02, + "learning_rate": 0.00026631895687061186, + "loss": 2.8227, + "theoretical_loss": 3.5039180865494055, + "tokens_seen": 1560357888 + }, + { + "epoch": 5.02, + "learning_rate": 0.00026630892678034104, + "loss": 2.8357, + "theoretical_loss": 3.503905209273869, + "tokens_seen": 1560423424 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002662988966900702, + "loss": 2.599, + "theoretical_loss": 3.503892332690577, + "tokens_seen": 1560488960 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002662888665997994, + "loss": 2.6265, + "theoretical_loss": 3.5038794567994636, + "tokens_seen": 1560554496 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002662788365095286, + "loss": 2.6774, + "theoretical_loss": 3.503866581600463, + "tokens_seen": 1560620032 + }, + { + "epoch": 5.02, + "learning_rate": 0.00026626880641925777, + "loss": 2.6937, + "theoretical_loss": 3.503853707093508, + "tokens_seen": 1560685568 + }, + { + "epoch": 5.02, + "learning_rate": 0.000266258776328987, + "loss": 2.4141, + "theoretical_loss": 3.5038408332785327, + "tokens_seen": 1560751104 + }, + { + "epoch": 5.02, + "learning_rate": 0.00026624874623871613, + "loss": 2.4629, + "theoretical_loss": 3.5038279601554714, + "tokens_seen": 1560816640 + }, + { + "epoch": 5.02, + "learning_rate": 0.00026623871614844536, + "loss": 2.7788, + "theoretical_loss": 3.503815087724257, + "tokens_seen": 1560882176 + }, + { + "epoch": 5.02, + "learning_rate": 0.00026622868605817454, + "loss": 2.4715, + "theoretical_loss": 3.503802215984824, + "tokens_seen": 1560947712 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002662186559679037, + "loss": 2.6418, + "theoretical_loss": 3.503789344937106, + "tokens_seen": 1561013248 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002662086258776329, + "loss": 2.796, + "theoretical_loss": 3.503776474581036, + "tokens_seen": 1561078784 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002661985957873621, + "loss": 2.5447, + "theoretical_loss": 3.503763604916549, + "tokens_seen": 1561144320 + }, + { + "epoch": 5.02, + "learning_rate": 0.00026618856569709127, + "loss": 2.6333, + "theoretical_loss": 3.503750735943578, + "tokens_seen": 1561209856 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002661785356068205, + "loss": 2.657, + "theoretical_loss": 3.5037378676620574, + "tokens_seen": 1561275392 + }, + { + "epoch": 5.02, + "learning_rate": 0.00026616850551654963, + "loss": 2.8328, + "theoretical_loss": 3.5037250000719204, + "tokens_seen": 1561340928 + }, + { + "epoch": 5.02, + "objective/train/docs_used": 1767472, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.634784460067749, + "objective/train/theoretical_loss": 3.5037153498329987, + "objective/train/tokens_used": 1581850080, + "theoretical_loss": 3.5037153498329987, + "tokens_seen": 1561390080 + }, + { + "epoch": 5.02, + "learning_rate": 0.00026615847542627887, + "loss": 2.4677, + "theoretical_loss": 3.5037121331731016, + "tokens_seen": 1561406464 + }, + { + "epoch": 5.02, + "learning_rate": 0.000266148445336008, + "loss": 2.7243, + "theoretical_loss": 3.5036992669655342, + "tokens_seen": 1561472000 + }, + { + "epoch": 5.02, + "learning_rate": 0.00026613841524573723, + "loss": 2.814, + "theoretical_loss": 3.503686401449152, + "tokens_seen": 1561537536 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002661283851554664, + "loss": 2.7473, + "theoretical_loss": 3.5036735366238894, + "tokens_seen": 1561603072 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002661183550651956, + "loss": 2.5054, + "theoretical_loss": 3.50366067248968, + "tokens_seen": 1561668608 + }, + { + "epoch": 5.02, + "learning_rate": 0.00026610832497492477, + "loss": 2.3481, + "theoretical_loss": 3.5036478090464573, + "tokens_seen": 1561734144 + }, + { + "epoch": 5.02, + "learning_rate": 0.00026609829488465395, + "loss": 2.5554, + "theoretical_loss": 3.503634946294156, + "tokens_seen": 1561799680 + }, + { + "epoch": 5.02, + "learning_rate": 0.00026608826479438313, + "loss": 2.5268, + "theoretical_loss": 3.503622084232709, + "tokens_seen": 1561865216 + }, + { + "epoch": 5.02, + "learning_rate": 0.00026607823470411237, + "loss": 2.5213, + "theoretical_loss": 3.5036092228620515, + "tokens_seen": 1561930752 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002660682046138415, + "loss": 2.7919, + "theoretical_loss": 3.5035963621821162, + "tokens_seen": 1561996288 + }, + { + "epoch": 5.02, + "learning_rate": 0.00026605817452357073, + "loss": 2.657, + "theoretical_loss": 3.5035835021928374, + "tokens_seen": 1562061824 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002660481444332999, + "loss": 2.4765, + "theoretical_loss": 3.5035706428941493, + "tokens_seen": 1562127360 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002660381143430291, + "loss": 2.5715, + "theoretical_loss": 3.5035577842859857, + "tokens_seen": 1562192896 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002660280842527583, + "loss": 2.3602, + "theoretical_loss": 3.5035449263682805, + "tokens_seen": 1562258432 + }, + { + "epoch": 5.02, + "learning_rate": 0.00026601805416248746, + "loss": 2.741, + "theoretical_loss": 3.503532069140968, + "tokens_seen": 1562323968 + }, + { + "epoch": 5.02, + "learning_rate": 0.00026600802407221664, + "loss": 2.6975, + "theoretical_loss": 3.5035192126039814, + "tokens_seen": 1562389504 + }, + { + "epoch": 5.02, + "learning_rate": 0.00026599799398194587, + "loss": 2.6357, + "theoretical_loss": 3.5035063567572555, + "tokens_seen": 1562455040 + }, + { + "epoch": 5.02, + "learning_rate": 0.000265987963891675, + "loss": 2.7784, + "theoretical_loss": 3.5034935016007243, + "tokens_seen": 1562520576 + }, + { + "epoch": 5.02, + "learning_rate": 0.00026597793380140423, + "loss": 2.7628, + "theoretical_loss": 3.503480647134321, + "tokens_seen": 1562586112 + }, + { + "epoch": 5.02, + "learning_rate": 0.00026596790371113336, + "loss": 2.7268, + "theoretical_loss": 3.50346779335798, + "tokens_seen": 1562651648 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002659578736208626, + "loss": 2.5596, + "theoretical_loss": 3.5034549402716353, + "tokens_seen": 1562717184 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002659478435305918, + "loss": 2.7433, + "theoretical_loss": 3.5034420878752206, + "tokens_seen": 1562782720 + }, + { + "epoch": 5.02, + "learning_rate": 0.00026593781344032096, + "loss": 2.7509, + "theoretical_loss": 3.503429236168671, + "tokens_seen": 1562848256 + }, + { + "epoch": 5.02, + "learning_rate": 0.00026592778335005014, + "loss": 2.7708, + "theoretical_loss": 3.5034163851519198, + "tokens_seen": 1562913792 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002659177532597794, + "loss": 2.6534, + "theoretical_loss": 3.5034035348249013, + "tokens_seen": 1562979328 + }, + { + "epoch": 5.02, + "objective/train/docs_used": 1768239, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8608767986297607, + "objective/train/theoretical_loss": 3.5033938975322343, + "objective/train/tokens_used": 1583488480, + "theoretical_loss": 3.5033938975322343, + "tokens_seen": 1563028480 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002659077231695085, + "loss": 2.7638, + "theoretical_loss": 3.503390685187549, + "tokens_seen": 1563044864 + }, + { + "epoch": 5.02, + "learning_rate": 0.00026589769307923774, + "loss": 2.6592, + "theoretical_loss": 3.503377836239798, + "tokens_seen": 1563110400 + }, + { + "epoch": 5.02, + "learning_rate": 0.00026588766298896686, + "loss": 2.7249, + "theoretical_loss": 3.5033649879815814, + "tokens_seen": 1563175936 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002658776328986961, + "loss": 2.6386, + "theoretical_loss": 3.503352140412833, + "tokens_seen": 1563241472 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002658676028084253, + "loss": 2.5453, + "theoretical_loss": 3.503339293533488, + "tokens_seen": 1563307008 + }, + { + "epoch": 5.02, + "learning_rate": 0.00026585757271815446, + "loss": 2.7011, + "theoretical_loss": 3.5033264473434804, + "tokens_seen": 1563372544 + }, + { + "epoch": 5.02, + "learning_rate": 0.00026584754262788364, + "loss": 2.805, + "theoretical_loss": 3.503313601842744, + "tokens_seen": 1563438080 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002658375125376128, + "loss": 2.708, + "theoretical_loss": 3.5033007570312122, + "tokens_seen": 1563503616 + }, + { + "epoch": 5.02, + "learning_rate": 0.000265827482447342, + "loss": 2.8063, + "theoretical_loss": 3.50328791290882, + "tokens_seen": 1563569152 + }, + { + "epoch": 5.02, + "learning_rate": 0.00026581745235707124, + "loss": 2.6561, + "theoretical_loss": 3.5032750694755017, + "tokens_seen": 1563634688 + }, + { + "epoch": 5.02, + "learning_rate": 0.00026580742226680037, + "loss": 2.6712, + "theoretical_loss": 3.5032622267311915, + "tokens_seen": 1563700224 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002657973921765296, + "loss": 2.639, + "theoretical_loss": 3.503249384675822, + "tokens_seen": 1563765760 + }, + { + "epoch": 5.02, + "learning_rate": 0.00026578736208625873, + "loss": 2.7569, + "theoretical_loss": 3.5032365433093293, + "tokens_seen": 1563831296 + }, + { + "epoch": 5.02, + "learning_rate": 0.00026577733199598797, + "loss": 2.5356, + "theoretical_loss": 3.5032237026316473, + "tokens_seen": 1563896832 + }, + { + "epoch": 5.02, + "learning_rate": 0.00026576730190571715, + "loss": 2.5125, + "theoretical_loss": 3.503210862642709, + "tokens_seen": 1563962368 + }, + { + "epoch": 5.02, + "learning_rate": 0.00026575727181544633, + "loss": 2.7183, + "theoretical_loss": 3.503198023342449, + "tokens_seen": 1564027904 + }, + { + "epoch": 5.02, + "learning_rate": 0.00026574724172517556, + "loss": 2.4693, + "theoretical_loss": 3.5031851847308024, + "tokens_seen": 1564093440 + }, + { + "epoch": 5.02, + "learning_rate": 0.00026573721163490474, + "loss": 2.8912, + "theoretical_loss": 3.503172346807703, + "tokens_seen": 1564158976 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002657271815446339, + "loss": 2.3473, + "theoretical_loss": 3.503159509573085, + "tokens_seen": 1564224512 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002657171514543631, + "loss": 2.6239, + "theoretical_loss": 3.5031466730268823, + "tokens_seen": 1564290048 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002657071213640923, + "loss": 2.5645, + "theoretical_loss": 3.5031338371690293, + "tokens_seen": 1564355584 + }, + { + "epoch": 5.02, + "learning_rate": 0.00026569709127382147, + "loss": 2.6451, + "theoretical_loss": 3.5031210019994603, + "tokens_seen": 1564421120 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002656870611835507, + "loss": 2.6027, + "theoretical_loss": 3.5031081675181093, + "tokens_seen": 1564486656 + }, + { + "epoch": 5.02, + "learning_rate": 0.00026567703109327983, + "loss": 2.6622, + "theoretical_loss": 3.503095333724911, + "tokens_seen": 1564552192 + }, + { + "epoch": 5.02, + "learning_rate": 0.00026566700100300907, + "loss": 2.5073, + "theoretical_loss": 3.5030825006198, + "tokens_seen": 1564617728 + }, + { + "epoch": 5.02, + "objective/train/docs_used": 1769768, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.596804141998291, + "objective/train/theoretical_loss": 3.5030728762424834, + "objective/train/tokens_used": 1585126880, + "theoretical_loss": 3.5030728762424834, + "tokens_seen": 1564666880 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002656569709127382, + "loss": 2.5982, + "theoretical_loss": 3.503069668202709, + "tokens_seen": 1564683264 + }, + { + "epoch": 5.02, + "learning_rate": 0.00026564694082246743, + "loss": 2.6338, + "theoretical_loss": 3.5030568364735744, + "tokens_seen": 1564748800 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002656369107321966, + "loss": 2.7805, + "theoretical_loss": 3.5030440054323293, + "tokens_seen": 1564814336 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002656268806419258, + "loss": 2.7623, + "theoretical_loss": 3.503031175078908, + "tokens_seen": 1564879872 + }, + { + "epoch": 5.02, + "learning_rate": 0.00026561685055165497, + "loss": 2.4328, + "theoretical_loss": 3.503018345413245, + "tokens_seen": 1564945408 + }, + { + "epoch": 5.02, + "learning_rate": 0.00026560682046138415, + "loss": 2.615, + "theoretical_loss": 3.503005516435275, + "tokens_seen": 1565010944 + }, + { + "epoch": 5.02, + "learning_rate": 0.00026559679037111333, + "loss": 2.6627, + "theoretical_loss": 3.502992688144932, + "tokens_seen": 1565076480 + }, + { + "epoch": 5.02, + "learning_rate": 0.00026558676028084257, + "loss": 2.5018, + "theoretical_loss": 3.5029798605421503, + "tokens_seen": 1565142016 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002655767301905717, + "loss": 2.3945, + "theoretical_loss": 3.5029670336268643, + "tokens_seen": 1565207552 + }, + { + "epoch": 5.02, + "learning_rate": 0.00026556670010030093, + "loss": 2.6775, + "theoretical_loss": 3.5029542073990085, + "tokens_seen": 1565273088 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002655566700100301, + "loss": 2.7513, + "theoretical_loss": 3.502941381858517, + "tokens_seen": 1565338624 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002655466399197593, + "loss": 2.4477, + "theoretical_loss": 3.502928557005325, + "tokens_seen": 1565404160 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002655366098294885, + "loss": 2.8699, + "theoretical_loss": 3.502915732839366, + "tokens_seen": 1565469696 + }, + { + "epoch": 5.02, + "learning_rate": 0.00026552657973921766, + "loss": 2.6521, + "theoretical_loss": 3.502902909360574, + "tokens_seen": 1565535232 + }, + { + "epoch": 5.02, + "learning_rate": 0.00026551654964894684, + "loss": 2.6694, + "theoretical_loss": 3.502890086568885, + "tokens_seen": 1565600768 + }, + { + "epoch": 5.02, + "learning_rate": 0.00026550651955867607, + "loss": 2.5778, + "theoretical_loss": 3.5028772644642316, + "tokens_seen": 1565666304 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002654964894684052, + "loss": 2.7916, + "theoretical_loss": 3.5028644430465503, + "tokens_seen": 1565731840 + }, + { + "epoch": 5.02, + "learning_rate": 0.00026548645937813443, + "loss": 2.5472, + "theoretical_loss": 3.5028516223157737, + "tokens_seen": 1565797376 + }, + { + "epoch": 5.02, + "learning_rate": 0.00026547642928786356, + "loss": 2.6669, + "theoretical_loss": 3.502838802271837, + "tokens_seen": 1565862912 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002654663991975928, + "loss": 2.6802, + "theoretical_loss": 3.5028259829146746, + "tokens_seen": 1565928448 + }, + { + "epoch": 5.02, + "learning_rate": 0.000265456369107322, + "loss": 2.548, + "theoretical_loss": 3.502813164244221, + "tokens_seen": 1565993984 + }, + { + "epoch": 5.02, + "learning_rate": 0.00026544633901705116, + "loss": 2.7317, + "theoretical_loss": 3.5028003462604107, + "tokens_seen": 1566059520 + }, + { + "epoch": 5.02, + "learning_rate": 0.00026543630892678034, + "loss": 2.7592, + "theoretical_loss": 3.5027875289631782, + "tokens_seen": 1566125056 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002654262788365096, + "loss": 2.2943, + "theoretical_loss": 3.502774712352458, + "tokens_seen": 1566190592 + }, + { + "epoch": 5.02, + "learning_rate": 0.0002654162487462387, + "loss": 2.5666, + "theoretical_loss": 3.5027618964281846, + "tokens_seen": 1566256128 + }, + { + "epoch": 5.02, + "objective/train/docs_used": 1770273, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4401655197143555, + "objective/train/theoretical_loss": 3.5027522849354202, + "objective/train/tokens_used": 1586765280, + "theoretical_loss": 3.5027522849354202, + "tokens_seen": 1566305280 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026540621865596794, + "loss": 2.5743, + "theoretical_loss": 3.5027490811902924, + "tokens_seen": 1566321664 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026539618856569707, + "loss": 2.7278, + "theoretical_loss": 3.5027362666387156, + "tokens_seen": 1566387200 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002653861584754263, + "loss": 2.7307, + "theoretical_loss": 3.5027234527733895, + "tokens_seen": 1566452736 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002653761283851555, + "loss": 2.7276, + "theoretical_loss": 3.502710639594248, + "tokens_seen": 1566518272 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026536609829488466, + "loss": 2.4546, + "theoretical_loss": 3.502697827101226, + "tokens_seen": 1566583808 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026535606820461384, + "loss": 2.4774, + "theoretical_loss": 3.5026850152942584, + "tokens_seen": 1566649344 + }, + { + "epoch": 5.03, + "learning_rate": 0.000265346038114343, + "loss": 2.6686, + "theoretical_loss": 3.5026722041732787, + "tokens_seen": 1566714880 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002653360080240722, + "loss": 2.5979, + "theoretical_loss": 3.502659393738223, + "tokens_seen": 1566780416 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026532597793380144, + "loss": 2.6913, + "theoretical_loss": 3.5026465839890237, + "tokens_seen": 1566845952 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026531594784353057, + "loss": 2.5873, + "theoretical_loss": 3.5026337749256182, + "tokens_seen": 1566911488 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002653059177532598, + "loss": 2.4014, + "theoretical_loss": 3.5026209665479384, + "tokens_seen": 1566977024 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026529588766298893, + "loss": 2.7904, + "theoretical_loss": 3.502608158855921, + "tokens_seen": 1567042560 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026528585757271817, + "loss": 2.5611, + "theoretical_loss": 3.5025953518494988, + "tokens_seen": 1567108096 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026527582748244735, + "loss": 2.5719, + "theoretical_loss": 3.502582545528608, + "tokens_seen": 1567173632 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026526579739217653, + "loss": 2.5116, + "theoretical_loss": 3.5025697398931825, + "tokens_seen": 1567239168 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002652557673019057, + "loss": 2.4293, + "theoretical_loss": 3.502556934943157, + "tokens_seen": 1567304704 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026524573721163494, + "loss": 2.7416, + "theoretical_loss": 3.502544130678466, + "tokens_seen": 1567370240 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026523570712136407, + "loss": 2.5996, + "theoretical_loss": 3.5025313270990446, + "tokens_seen": 1567435776 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002652256770310933, + "loss": 2.5542, + "theoretical_loss": 3.5025185242048273, + "tokens_seen": 1567501312 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026521564694082243, + "loss": 2.6636, + "theoretical_loss": 3.5025057219957487, + "tokens_seen": 1567566848 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026520561685055167, + "loss": 2.7647, + "theoretical_loss": 3.5024929204717434, + "tokens_seen": 1567632384 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026519558676028085, + "loss": 2.7057, + "theoretical_loss": 3.502480119632746, + "tokens_seen": 1567697920 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026518555667001003, + "loss": 2.5071, + "theoretical_loss": 3.5024673194786917, + "tokens_seen": 1567763456 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002651755265797392, + "loss": 2.8567, + "theoretical_loss": 3.5024545200095156, + "tokens_seen": 1567828992 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002651654964894684, + "loss": 2.7552, + "theoretical_loss": 3.502441721225151, + "tokens_seen": 1567894528 + }, + { + "epoch": 5.03, + "objective/train/docs_used": 1771462, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.3107831478118896, + "objective/train/theoretical_loss": 3.5024321225862467, + "objective/train/tokens_used": 1588403680, + "theoretical_loss": 3.5024321225862467, + "tokens_seen": 1567943680 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002651554663991976, + "loss": 2.8068, + "theoretical_loss": 3.502428923125534, + "tokens_seen": 1567960064 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002651454363089268, + "loss": 2.6769, + "theoretical_loss": 3.5024161257105986, + "tokens_seen": 1568025600 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026513540621865594, + "loss": 2.4889, + "theoretical_loss": 3.5024033289802796, + "tokens_seen": 1568091136 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026512537612838517, + "loss": 2.7356, + "theoretical_loss": 3.5023905329345117, + "tokens_seen": 1568156672 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002651153460381143, + "loss": 2.5249, + "theoretical_loss": 3.50237773757323, + "tokens_seen": 1568222208 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026510531594784353, + "loss": 2.4725, + "theoretical_loss": 3.502364942896369, + "tokens_seen": 1568287744 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002650952858575727, + "loss": 2.6477, + "theoretical_loss": 3.5023521489038645, + "tokens_seen": 1568353280 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002650852557673019, + "loss": 2.5498, + "theoretical_loss": 3.5023393555956495, + "tokens_seen": 1568418816 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002650752256770311, + "loss": 2.7988, + "theoretical_loss": 3.50232656297166, + "tokens_seen": 1568484352 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002650651955867603, + "loss": 2.7056, + "theoretical_loss": 3.5023137710318304, + "tokens_seen": 1568549888 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026505516549648944, + "loss": 2.723, + "theoretical_loss": 3.5023009797760962, + "tokens_seen": 1568615424 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002650451354062187, + "loss": 2.8434, + "theoretical_loss": 3.5022881892043918, + "tokens_seen": 1568680960 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002650351053159478, + "loss": 2.6988, + "theoretical_loss": 3.5022753993166518, + "tokens_seen": 1568746496 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026502507522567704, + "loss": 2.5269, + "theoretical_loss": 3.502262610112811, + "tokens_seen": 1568812032 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002650150451354062, + "loss": 2.7634, + "theoretical_loss": 3.502249821592805, + "tokens_seen": 1568877568 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002650050150451354, + "loss": 2.457, + "theoretical_loss": 3.502237033756568, + "tokens_seen": 1568943104 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026499498495486464, + "loss": 2.5449, + "theoretical_loss": 3.5022242466040345, + "tokens_seen": 1569008640 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026498495486459376, + "loss": 2.5428, + "theoretical_loss": 3.5022114601351406, + "tokens_seen": 1569074176 + }, + { + "epoch": 5.03, + "learning_rate": 0.000264974924774323, + "loss": 2.5365, + "theoretical_loss": 3.5021986743498204, + "tokens_seen": 1569139712 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002649648946840522, + "loss": 2.7373, + "theoretical_loss": 3.502185889248009, + "tokens_seen": 1569205248 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026495486459378136, + "loss": 2.6758, + "theoretical_loss": 3.502173104829641, + "tokens_seen": 1569270784 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026494483450351054, + "loss": 2.9497, + "theoretical_loss": 3.5021603210946517, + "tokens_seen": 1569336320 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002649348044132398, + "loss": 2.7386, + "theoretical_loss": 3.5021475380429763, + "tokens_seen": 1569401856 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002649247743229689, + "loss": 2.3083, + "theoretical_loss": 3.502134755674549, + "tokens_seen": 1569467392 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026491474423269814, + "loss": 2.6491, + "theoretical_loss": 3.502121973989305, + "tokens_seen": 1569532928 + }, + { + "epoch": 5.03, + "objective/train/docs_used": 1772261, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.672778844833374, + "objective/train/theoretical_loss": 3.5021123881736727, + "objective/train/tokens_used": 1590042080, + "theoretical_loss": 3.5021123881736727, + "tokens_seen": 1569582080 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026490471414242727, + "loss": 2.5048, + "theoretical_loss": 3.50210919298718, + "tokens_seen": 1569598464 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002648946840521565, + "loss": 2.5785, + "theoretical_loss": 3.502096412668108, + "tokens_seen": 1569664000 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002648846539618857, + "loss": 2.6309, + "theoretical_loss": 3.5020836330320244, + "tokens_seen": 1569729536 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026487462387161486, + "loss": 2.6183, + "theoretical_loss": 3.5020708540788643, + "tokens_seen": 1569795072 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026486459378134404, + "loss": 2.7797, + "theoretical_loss": 3.502058075808562, + "tokens_seen": 1569860608 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002648545636910732, + "loss": 2.5406, + "theoretical_loss": 3.5020452982210535, + "tokens_seen": 1569926144 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002648445336008024, + "loss": 2.641, + "theoretical_loss": 3.5020325213162735, + "tokens_seen": 1569991680 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026483450351053164, + "loss": 2.7582, + "theoretical_loss": 3.5020197450941564, + "tokens_seen": 1570057216 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026482447342026077, + "loss": 2.8391, + "theoretical_loss": 3.502006969554638, + "tokens_seen": 1570122752 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026481444332999, + "loss": 2.5771, + "theoretical_loss": 3.501994194697653, + "tokens_seen": 1570188288 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026480441323971913, + "loss": 2.791, + "theoretical_loss": 3.5019814205231365, + "tokens_seen": 1570253824 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026479438314944837, + "loss": 2.7583, + "theoretical_loss": 3.501968647031023, + "tokens_seen": 1570319360 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026478435305917755, + "loss": 2.6849, + "theoretical_loss": 3.5019558742212493, + "tokens_seen": 1570384896 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026477432296890673, + "loss": 2.4884, + "theoretical_loss": 3.5019431020937484, + "tokens_seen": 1570450432 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002647642928786359, + "loss": 2.7081, + "theoretical_loss": 3.501930330648457, + "tokens_seen": 1570515968 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026475426278836514, + "loss": 2.5406, + "theoretical_loss": 3.501917559885309, + "tokens_seen": 1570581504 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026474423269809427, + "loss": 2.7217, + "theoretical_loss": 3.50190478980424, + "tokens_seen": 1570647040 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002647342026078235, + "loss": 2.8213, + "theoretical_loss": 3.501892020405185, + "tokens_seen": 1570712576 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026472417251755263, + "loss": 2.8029, + "theoretical_loss": 3.5018792516880795, + "tokens_seen": 1570778112 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026471414242728187, + "loss": 2.7009, + "theoretical_loss": 3.501866483652858, + "tokens_seen": 1570843648 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026470411233701105, + "loss": 2.5567, + "theoretical_loss": 3.5018537162994567, + "tokens_seen": 1570909184 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026469408224674023, + "loss": 2.6985, + "theoretical_loss": 3.5018409496278093, + "tokens_seen": 1570974720 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002646840521564694, + "loss": 2.6412, + "theoretical_loss": 3.501828183637852, + "tokens_seen": 1571040256 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002646740220661986, + "loss": 2.6996, + "theoretical_loss": 3.50181541832952, + "tokens_seen": 1571105792 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002646639919759278, + "loss": 2.4996, + "theoretical_loss": 3.5018026537027476, + "tokens_seen": 1571171328 + }, + { + "epoch": 5.03, + "objective/train/docs_used": 1773327, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5414953231811523, + "objective/train/theoretical_loss": 3.5017930806799034, + "objective/train/tokens_used": 1591680480, + "theoretical_loss": 3.5017930806799034, + "tokens_seen": 1571220480 + }, + { + "epoch": 5.03, + "learning_rate": 0.000264653961885657, + "loss": 2.4725, + "theoretical_loss": 3.501789889757471, + "tokens_seen": 1571236864 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026464393179538614, + "loss": 2.4687, + "theoretical_loss": 3.5017771264936246, + "tokens_seen": 1571302400 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026463390170511537, + "loss": 2.6159, + "theoretical_loss": 3.5017643639111435, + "tokens_seen": 1571367936 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002646238716148445, + "loss": 2.6949, + "theoretical_loss": 3.501751602009964, + "tokens_seen": 1571433472 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026461384152457373, + "loss": 2.6275, + "theoretical_loss": 3.5017388407900207, + "tokens_seen": 1571499008 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002646038114343029, + "loss": 2.6047, + "theoretical_loss": 3.5017260802512484, + "tokens_seen": 1571564544 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002645937813440321, + "loss": 2.6284, + "theoretical_loss": 3.501713320393583, + "tokens_seen": 1571630080 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002645837512537613, + "loss": 2.8202, + "theoretical_loss": 3.5017005612169596, + "tokens_seen": 1571695616 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002645737211634905, + "loss": 2.5874, + "theoretical_loss": 3.501687802721313, + "tokens_seen": 1571761152 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026456369107321964, + "loss": 2.7119, + "theoretical_loss": 3.5016750449065785, + "tokens_seen": 1571826688 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002645536609829489, + "loss": 2.5645, + "theoretical_loss": 3.501662287772692, + "tokens_seen": 1571892224 + }, + { + "epoch": 5.03, + "learning_rate": 0.000264543630892678, + "loss": 2.6312, + "theoretical_loss": 3.5016495313195883, + "tokens_seen": 1571957760 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026453360080240724, + "loss": 2.5627, + "theoretical_loss": 3.501636775547203, + "tokens_seen": 1572023296 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002645235707121364, + "loss": 2.8078, + "theoretical_loss": 3.5016240204554716, + "tokens_seen": 1572088832 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002645135406218656, + "loss": 2.4745, + "theoretical_loss": 3.5016112660443284, + "tokens_seen": 1572154368 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002645035105315948, + "loss": 2.4327, + "theoretical_loss": 3.501598512313709, + "tokens_seen": 1572219904 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026449348044132396, + "loss": 2.6913, + "theoretical_loss": 3.50158575926355, + "tokens_seen": 1572285440 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026448345035105314, + "loss": 2.6209, + "theoretical_loss": 3.5015730068937856, + "tokens_seen": 1572350976 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002644734202607824, + "loss": 2.6321, + "theoretical_loss": 3.501560255204351, + "tokens_seen": 1572416512 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002644633901705115, + "loss": 2.8599, + "theoretical_loss": 3.501547504195182, + "tokens_seen": 1572482048 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026445336008024074, + "loss": 2.852, + "theoretical_loss": 3.5015347538662143, + "tokens_seen": 1572547584 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026444332998996987, + "loss": 2.7477, + "theoretical_loss": 3.501522004217382, + "tokens_seen": 1572613120 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002644332998996991, + "loss": 2.7597, + "theoretical_loss": 3.5015092552486218, + "tokens_seen": 1572678656 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002644232698094283, + "loss": 2.8664, + "theoretical_loss": 3.5014965069598687, + "tokens_seen": 1572744192 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026441323971915747, + "loss": 2.6851, + "theoretical_loss": 3.501483759351058, + "tokens_seen": 1572809728 + }, + { + "epoch": 5.03, + "objective/train/docs_used": 1773816, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 1.9573558568954468, + "objective/train/theoretical_loss": 3.501474199090623, + "objective/train/tokens_used": 1593318880, + "theoretical_loss": 3.501474199090623, + "tokens_seen": 1572858880 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026440320962888665, + "loss": 2.6914, + "theoretical_loss": 3.501471012422125, + "tokens_seen": 1572875264 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002643931795386159, + "loss": 2.5659, + "theoretical_loss": 3.501458266173005, + "tokens_seen": 1572940800 + }, + { + "epoch": 5.03, + "learning_rate": 0.000264383149448345, + "loss": 2.8077, + "theoretical_loss": 3.501445520603634, + "tokens_seen": 1573006336 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026437311935807424, + "loss": 2.7236, + "theoretical_loss": 3.5014327757139467, + "tokens_seen": 1573071872 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026436308926780337, + "loss": 2.6091, + "theoretical_loss": 3.501420031503879, + "tokens_seen": 1573137408 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002643530591775326, + "loss": 2.5839, + "theoretical_loss": 3.5014072879733664, + "tokens_seen": 1573202944 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002643430290872618, + "loss": 2.5391, + "theoretical_loss": 3.501394545122344, + "tokens_seen": 1573268480 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026433299899699097, + "loss": 2.8725, + "theoretical_loss": 3.501381802950748, + "tokens_seen": 1573334016 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026432296890672015, + "loss": 2.803, + "theoretical_loss": 3.501369061458513, + "tokens_seen": 1573399552 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026431293881644933, + "loss": 2.682, + "theoretical_loss": 3.5013563206455744, + "tokens_seen": 1573465088 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002643029087261785, + "loss": 2.5005, + "theoretical_loss": 3.501343580511869, + "tokens_seen": 1573530624 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026429287863590775, + "loss": 2.9047, + "theoretical_loss": 3.5013308410573307, + "tokens_seen": 1573596160 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002642828485456369, + "loss": 2.7758, + "theoretical_loss": 3.501318102281896, + "tokens_seen": 1573661696 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002642728184553661, + "loss": 2.6322, + "theoretical_loss": 3.5013053641855008, + "tokens_seen": 1573727232 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026426278836509534, + "loss": 2.8375, + "theoretical_loss": 3.501292626768079, + "tokens_seen": 1573792768 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026425275827482447, + "loss": 2.714, + "theoretical_loss": 3.5012798900295676, + "tokens_seen": 1573858304 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002642427281845537, + "loss": 2.743, + "theoretical_loss": 3.501267153969902, + "tokens_seen": 1573923840 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026423269809428283, + "loss": 2.6854, + "theoretical_loss": 3.5012544185890166, + "tokens_seen": 1573989376 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026422266800401207, + "loss": 2.5074, + "theoretical_loss": 3.5012416838868483, + "tokens_seen": 1574054912 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026421263791374125, + "loss": 2.6326, + "theoretical_loss": 3.501228949863332, + "tokens_seen": 1574120448 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026420260782347043, + "loss": 2.7372, + "theoretical_loss": 3.5012162165184035, + "tokens_seen": 1574185984 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002641925777331996, + "loss": 2.724, + "theoretical_loss": 3.5012034838519988, + "tokens_seen": 1574251520 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002641825476429288, + "loss": 2.444, + "theoretical_loss": 3.5011907518640526, + "tokens_seen": 1574317056 + }, + { + "epoch": 5.03, + "learning_rate": 0.000264172517552658, + "loss": 2.8405, + "theoretical_loss": 3.5011780205545007, + "tokens_seen": 1574382592 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002641624874623872, + "loss": 2.5441, + "theoretical_loss": 3.5011652899232795, + "tokens_seen": 1574448128 + }, + { + "epoch": 5.03, + "objective/train/docs_used": 1774770, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.815545082092285, + "objective/train/theoretical_loss": 3.501155742394979, + "objective/train/tokens_used": 1594957280, + "theoretical_loss": 3.501155742394979, + "tokens_seen": 1574497280 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026415245737211634, + "loss": 2.7969, + "theoretical_loss": 3.5011525599703237, + "tokens_seen": 1574513664 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026414242728184557, + "loss": 2.6354, + "theoretical_loss": 3.5011398306955694, + "tokens_seen": 1574579200 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002641323971915747, + "loss": 2.7739, + "theoretical_loss": 3.5011271020989527, + "tokens_seen": 1574644736 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026412236710130393, + "loss": 2.8288, + "theoretical_loss": 3.501114374180408, + "tokens_seen": 1574710272 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002641123370110331, + "loss": 2.5933, + "theoretical_loss": 3.5011016469398717, + "tokens_seen": 1574775808 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002641023069207623, + "loss": 2.6361, + "theoretical_loss": 3.5010889203772804, + "tokens_seen": 1574841344 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002640922768304915, + "loss": 2.5667, + "theoretical_loss": 3.5010761944925677, + "tokens_seen": 1574906880 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002640822467402207, + "loss": 2.7329, + "theoretical_loss": 3.501063469285671, + "tokens_seen": 1574972416 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026407221664994984, + "loss": 2.7016, + "theoretical_loss": 3.501050744756525, + "tokens_seen": 1575037952 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002640621865596791, + "loss": 2.7043, + "theoretical_loss": 3.501038020905066, + "tokens_seen": 1575103488 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002640521564694082, + "loss": 2.8215, + "theoretical_loss": 3.5010252977312297, + "tokens_seen": 1575169024 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026404212637913744, + "loss": 2.5045, + "theoretical_loss": 3.5010125752349515, + "tokens_seen": 1575234560 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002640320962888666, + "loss": 2.7202, + "theoretical_loss": 3.5009998534161673, + "tokens_seen": 1575300096 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002640220661985958, + "loss": 2.7227, + "theoretical_loss": 3.5009871322748127, + "tokens_seen": 1575365632 + }, + { + "epoch": 5.03, + "learning_rate": 0.000264012036108325, + "loss": 2.6589, + "theoretical_loss": 3.500974411810824, + "tokens_seen": 1575431168 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026400200601805416, + "loss": 2.7051, + "theoretical_loss": 3.500961692024136, + "tokens_seen": 1575496704 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026399197592778334, + "loss": 2.717, + "theoretical_loss": 3.500948972914685, + "tokens_seen": 1575562240 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002639819458375126, + "loss": 2.7733, + "theoretical_loss": 3.500936254482407, + "tokens_seen": 1575627776 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002639719157472417, + "loss": 2.8105, + "theoretical_loss": 3.500923536727237, + "tokens_seen": 1575693312 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026396188565697094, + "loss": 2.8266, + "theoretical_loss": 3.5009108196491123, + "tokens_seen": 1575758848 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026395185556670007, + "loss": 2.8239, + "theoretical_loss": 3.500898103247967, + "tokens_seen": 1575824384 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002639418254764293, + "loss": 2.6971, + "theoretical_loss": 3.500885387523738, + "tokens_seen": 1575889920 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002639317953861585, + "loss": 2.7925, + "theoretical_loss": 3.500872672476361, + "tokens_seen": 1575955456 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026392176529588767, + "loss": 2.7641, + "theoretical_loss": 3.500859958105771, + "tokens_seen": 1576020992 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026391173520561685, + "loss": 2.9513, + "theoretical_loss": 3.5008472444119048, + "tokens_seen": 1576086528 + }, + { + "epoch": 5.03, + "objective/train/docs_used": 1774770, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.41874098777771, + "objective/train/theoretical_loss": 3.500837709585566, + "objective/train/tokens_used": 1596595680, + "theoretical_loss": 3.500837709585566, + "tokens_seen": 1576135680 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002639017051153461, + "loss": 2.8032, + "theoretical_loss": 3.5008345313946974, + "tokens_seen": 1576152064 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002638916750250752, + "loss": 2.5515, + "theoretical_loss": 3.5008218190540856, + "tokens_seen": 1576217600 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026388164493480444, + "loss": 2.8856, + "theoretical_loss": 3.5008091073900047, + "tokens_seen": 1576283136 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026387161484453357, + "loss": 2.8323, + "theoretical_loss": 3.5007963964023903, + "tokens_seen": 1576348672 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002638615847542628, + "loss": 2.6877, + "theoretical_loss": 3.500783686091179, + "tokens_seen": 1576414208 + }, + { + "epoch": 5.03, + "learning_rate": 0.000263851554663992, + "loss": 2.5179, + "theoretical_loss": 3.500770976456306, + "tokens_seen": 1576479744 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026384152457372117, + "loss": 2.9558, + "theoretical_loss": 3.5007582674977082, + "tokens_seen": 1576545280 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026383149448345035, + "loss": 2.8958, + "theoretical_loss": 3.5007455592153205, + "tokens_seen": 1576610816 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026382146439317953, + "loss": 2.7204, + "theoretical_loss": 3.500732851609079, + "tokens_seen": 1576676352 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002638114343029087, + "loss": 2.9708, + "theoretical_loss": 3.50072014467892, + "tokens_seen": 1576741888 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026380140421263795, + "loss": 2.8209, + "theoretical_loss": 3.5007074384247785, + "tokens_seen": 1576807424 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002637913741223671, + "loss": 3.039, + "theoretical_loss": 3.5006947328465916, + "tokens_seen": 1576872960 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002637813440320963, + "loss": 2.7014, + "theoretical_loss": 3.500682027944295, + "tokens_seen": 1576938496 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002637713139418255, + "loss": 2.9119, + "theoretical_loss": 3.5006693237178244, + "tokens_seen": 1577004032 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026376128385155467, + "loss": 2.7541, + "theoretical_loss": 3.5006566201671157, + "tokens_seen": 1577069568 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026375125376128385, + "loss": 3.0195, + "theoretical_loss": 3.5006439172921047, + "tokens_seen": 1577135104 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026374122367101303, + "loss": 2.8242, + "theoretical_loss": 3.500631215092728, + "tokens_seen": 1577200640 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002637311935807422, + "loss": 2.9552, + "theoretical_loss": 3.5006185135689214, + "tokens_seen": 1577266176 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026372116349047145, + "loss": 2.8054, + "theoretical_loss": 3.5006058127206208, + "tokens_seen": 1577331712 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002637111334002006, + "loss": 2.9058, + "theoretical_loss": 3.5005931125477616, + "tokens_seen": 1577397248 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002637011033099298, + "loss": 2.7677, + "theoretical_loss": 3.5005804130502804, + "tokens_seen": 1577462784 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026369107321965894, + "loss": 2.8287, + "theoretical_loss": 3.500567714228114, + "tokens_seen": 1577528320 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002636810431293882, + "loss": 2.7696, + "theoretical_loss": 3.500555016081197, + "tokens_seen": 1577593856 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026367101303911736, + "loss": 2.9985, + "theoretical_loss": 3.5005423186094666, + "tokens_seen": 1577659392 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026366098294884654, + "loss": 2.96, + "theoretical_loss": 3.5005296218128583, + "tokens_seen": 1577724928 + }, + { + "epoch": 5.03, + "objective/train/docs_used": 1775526, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7164270877838135, + "objective/train/theoretical_loss": 3.500520099658412, + "objective/train/tokens_used": 1598234080, + "theoretical_loss": 3.500520099658412, + "tokens_seen": 1577774080 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002636509528585757, + "loss": 2.664, + "theoretical_loss": 3.500516925691308, + "tokens_seen": 1577790464 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002636409227683049, + "loss": 2.6874, + "theoretical_loss": 3.5005042302447515, + "tokens_seen": 1577856000 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002636308926780341, + "loss": 2.6584, + "theoretical_loss": 3.500491535473126, + "tokens_seen": 1577921536 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002636208625877633, + "loss": 2.9151, + "theoretical_loss": 3.500478841376367, + "tokens_seen": 1577987072 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026361083249749244, + "loss": 3.0229, + "theoretical_loss": 3.50046614795441, + "tokens_seen": 1578052608 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002636008024072217, + "loss": 2.8641, + "theoretical_loss": 3.500453455207192, + "tokens_seen": 1578118144 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026359077231695086, + "loss": 2.692, + "theoretical_loss": 3.500440763134649, + "tokens_seen": 1578183680 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026358074222668004, + "loss": 2.9396, + "theoretical_loss": 3.5004280717367164, + "tokens_seen": 1578249216 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002635707121364092, + "loss": 2.8203, + "theoretical_loss": 3.5004153810133314, + "tokens_seen": 1578314752 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002635606820461384, + "loss": 2.5741, + "theoretical_loss": 3.5004026909644295, + "tokens_seen": 1578380288 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002635506519558676, + "loss": 2.9397, + "theoretical_loss": 3.5003900015899467, + "tokens_seen": 1578445824 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002635406218655968, + "loss": 2.8295, + "theoretical_loss": 3.5003773128898192, + "tokens_seen": 1578511360 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026353059177532595, + "loss": 2.7902, + "theoretical_loss": 3.500364624863984, + "tokens_seen": 1578576896 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002635205616850552, + "loss": 2.9844, + "theoretical_loss": 3.5003519375123764, + "tokens_seen": 1578642432 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026351053159478436, + "loss": 2.8751, + "theoretical_loss": 3.5003392508349327, + "tokens_seen": 1578707968 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026350050150451354, + "loss": 2.7126, + "theoretical_loss": 3.500326564831589, + "tokens_seen": 1578773504 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002634904714142428, + "loss": 2.8797, + "theoretical_loss": 3.500313879502282, + "tokens_seen": 1578839040 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002634804413239719, + "loss": 2.9553, + "theoretical_loss": 3.500301194846947, + "tokens_seen": 1578904576 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026347041123370114, + "loss": 2.9701, + "theoretical_loss": 3.5002885108655217, + "tokens_seen": 1578970112 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026346038114343027, + "loss": 2.9054, + "theoretical_loss": 3.5002758275579406, + "tokens_seen": 1579035648 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002634503510531595, + "loss": 2.67, + "theoretical_loss": 3.5002631449241415, + "tokens_seen": 1579101184 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002634403209628887, + "loss": 2.7871, + "theoretical_loss": 3.5002504629640594, + "tokens_seen": 1579166720 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026343029087261787, + "loss": 2.5263, + "theoretical_loss": 3.5002377816776318, + "tokens_seen": 1579232256 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026342026078234705, + "loss": 2.8631, + "theoretical_loss": 3.500225101064794, + "tokens_seen": 1579297792 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002634102306920763, + "loss": 2.9627, + "theoretical_loss": 3.5002124211254824, + "tokens_seen": 1579363328 + }, + { + "epoch": 5.03, + "objective/train/docs_used": 1776340, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.49752140045166, + "objective/train/theoretical_loss": 3.500202911612962, + "objective/train/tokens_used": 1599872480, + "theoretical_loss": 3.500202911612962, + "tokens_seen": 1579412480 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002634002006018054, + "loss": 2.5422, + "theoretical_loss": 3.500199741859633, + "tokens_seen": 1579428864 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026339017051153464, + "loss": 2.7841, + "theoretical_loss": 3.500187063267183, + "tokens_seen": 1579494400 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026338014042126377, + "loss": 2.7422, + "theoretical_loss": 3.5001743853480685, + "tokens_seen": 1579559936 + }, + { + "epoch": 5.03, + "learning_rate": 0.000263370110330993, + "loss": 2.7585, + "theoretical_loss": 3.500161708102225, + "tokens_seen": 1579625472 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002633600802407222, + "loss": 2.6565, + "theoretical_loss": 3.5001490315295896, + "tokens_seen": 1579691008 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026335005015045137, + "loss": 3.0777, + "theoretical_loss": 3.500136355630098, + "tokens_seen": 1579756544 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026334002006018055, + "loss": 2.8242, + "theoretical_loss": 3.5001236804036875, + "tokens_seen": 1579822080 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026332998996990973, + "loss": 2.898, + "theoretical_loss": 3.500111005850293, + "tokens_seen": 1579887616 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002633199598796389, + "loss": 2.5527, + "theoretical_loss": 3.500098331969852, + "tokens_seen": 1579953152 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026330992978936815, + "loss": 2.9106, + "theoretical_loss": 3.5000856587623006, + "tokens_seen": 1580018688 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002632998996990973, + "loss": 2.6539, + "theoretical_loss": 3.5000729862275755, + "tokens_seen": 1580084224 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002632898696088265, + "loss": 2.8476, + "theoretical_loss": 3.500060314365612, + "tokens_seen": 1580149760 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002632798395185557, + "loss": 2.7165, + "theoretical_loss": 3.500047643176347, + "tokens_seen": 1580215296 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026326980942828487, + "loss": 2.874, + "theoretical_loss": 3.5000349726597175, + "tokens_seen": 1580280832 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026325977933801405, + "loss": 2.8665, + "theoretical_loss": 3.5000223028156596, + "tokens_seen": 1580346368 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026324974924774323, + "loss": 3.0691, + "theoretical_loss": 3.5000096336441096, + "tokens_seen": 1580411904 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002632397191574724, + "loss": 2.854, + "theoretical_loss": 3.4999969651450034, + "tokens_seen": 1580477440 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026322968906720165, + "loss": 2.6651, + "theoretical_loss": 3.499984297318278, + "tokens_seen": 1580542976 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002632196589769308, + "loss": 2.9626, + "theoretical_loss": 3.49997163016387, + "tokens_seen": 1580608512 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026320962888666, + "loss": 2.6678, + "theoretical_loss": 3.499958963681715, + "tokens_seen": 1580674048 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026319959879638914, + "loss": 2.7437, + "theoretical_loss": 3.499946297871751, + "tokens_seen": 1580739584 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002631895687061184, + "loss": 2.8165, + "theoretical_loss": 3.499933632733913, + "tokens_seen": 1580805120 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026317953861584756, + "loss": 2.8247, + "theoretical_loss": 3.499920968268138, + "tokens_seen": 1580870656 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026316950852557674, + "loss": 2.7621, + "theoretical_loss": 3.499908304474362, + "tokens_seen": 1580936192 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002631594784353059, + "loss": 2.7327, + "theoretical_loss": 3.4998956413525226, + "tokens_seen": 1581001728 + }, + { + "epoch": 5.03, + "objective/train/docs_used": 1777599, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8258018493652344, + "objective/train/theoretical_loss": 3.499886144452063, + "objective/train/tokens_used": 1601510880, + "theoretical_loss": 3.499886144452063, + "tokens_seen": 1581050880 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002631494483450351, + "loss": 2.9461, + "theoretical_loss": 3.499882978902555, + "tokens_seen": 1581067264 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002631394182547643, + "loss": 2.7952, + "theoretical_loss": 3.499870317124397, + "tokens_seen": 1581132800 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002631293881644935, + "loss": 2.8345, + "theoretical_loss": 3.499857656017984, + "tokens_seen": 1581198336 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026311935807422264, + "loss": 2.544, + "theoretical_loss": 3.499844995583253, + "tokens_seen": 1581263872 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002631093279839519, + "loss": 2.7987, + "theoretical_loss": 3.499832335820141, + "tokens_seen": 1581329408 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026309929789368106, + "loss": 2.6151, + "theoretical_loss": 3.4998196767285834, + "tokens_seen": 1581394944 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026308926780341024, + "loss": 2.4125, + "theoretical_loss": 3.499807018308518, + "tokens_seen": 1581460480 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002630792377131394, + "loss": 2.914, + "theoretical_loss": 3.4997943605598802, + "tokens_seen": 1581526016 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002630692076228686, + "loss": 2.8157, + "theoretical_loss": 3.499781703482607, + "tokens_seen": 1581591552 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002630591775325978, + "loss": 3.023, + "theoretical_loss": 3.4997690470766356, + "tokens_seen": 1581657088 + }, + { + "epoch": 5.03, + "learning_rate": 0.000263049147442327, + "loss": 2.708, + "theoretical_loss": 3.4997563913419016, + "tokens_seen": 1581722624 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026303911735205615, + "loss": 2.8942, + "theoretical_loss": 3.4997437362783423, + "tokens_seen": 1581788160 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002630290872617854, + "loss": 2.7689, + "theoretical_loss": 3.499731081885894, + "tokens_seen": 1581853696 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002630190571715145, + "loss": 2.7761, + "theoretical_loss": 3.4997184281644937, + "tokens_seen": 1581919232 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026300902708124374, + "loss": 3.1327, + "theoretical_loss": 3.4997057751140774, + "tokens_seen": 1581984768 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002629989969909729, + "loss": 2.7557, + "theoretical_loss": 3.499693122734582, + "tokens_seen": 1582050304 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002629889669007021, + "loss": 2.7901, + "theoretical_loss": 3.499680471025944, + "tokens_seen": 1582115840 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002629789368104313, + "loss": 2.7257, + "theoretical_loss": 3.4996678199881, + "tokens_seen": 1582181376 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026296890672016047, + "loss": 2.7572, + "theoretical_loss": 3.499655169620987, + "tokens_seen": 1582246912 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026295887662988965, + "loss": 2.7508, + "theoretical_loss": 3.4996425199245422, + "tokens_seen": 1582312448 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002629488465396189, + "loss": 2.8917, + "theoretical_loss": 3.4996298708987013, + "tokens_seen": 1582377984 + }, + { + "epoch": 5.03, + "learning_rate": 0.000262938816449348, + "loss": 3.0054, + "theoretical_loss": 3.499617222543401, + "tokens_seen": 1582443520 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026292878635907725, + "loss": 2.4528, + "theoretical_loss": 3.499604574858578, + "tokens_seen": 1582509056 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026291875626880643, + "loss": 2.8008, + "theoretical_loss": 3.499591927844169, + "tokens_seen": 1582574592 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002629087261785356, + "loss": 2.8668, + "theoretical_loss": 3.499579281500112, + "tokens_seen": 1582640128 + }, + { + "epoch": 5.03, + "objective/train/docs_used": 1778167, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9653608798980713, + "objective/train/theoretical_loss": 3.499569797181948, + "objective/train/tokens_used": 1603149280, + "theoretical_loss": 3.499569797181948, + "tokens_seen": 1582689280 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002628986960882648, + "loss": 2.8982, + "theoretical_loss": 3.4995666358263415, + "tokens_seen": 1582705664 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026288866599799397, + "loss": 2.8606, + "theoretical_loss": 3.499553990822796, + "tokens_seen": 1582771200 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026287863590772315, + "loss": 2.8867, + "theoretical_loss": 3.499541346489411, + "tokens_seen": 1582836736 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002628686058174524, + "loss": 2.7481, + "theoretical_loss": 3.499528702826125, + "tokens_seen": 1582902272 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002628585757271815, + "loss": 2.69, + "theoretical_loss": 3.4995160598328727, + "tokens_seen": 1582967808 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026284854563691075, + "loss": 2.939, + "theoretical_loss": 3.499503417509592, + "tokens_seen": 1583033344 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002628385155466399, + "loss": 2.9496, + "theoretical_loss": 3.499490775856219, + "tokens_seen": 1583098880 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002628284854563691, + "loss": 3.0367, + "theoretical_loss": 3.4994781348726915, + "tokens_seen": 1583164416 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002628184553660983, + "loss": 2.8122, + "theoretical_loss": 3.4994654945589447, + "tokens_seen": 1583229952 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002628084252758275, + "loss": 2.8815, + "theoretical_loss": 3.4994528549149173, + "tokens_seen": 1583295488 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026279839518555666, + "loss": 2.6773, + "theoretical_loss": 3.499440215940545, + "tokens_seen": 1583361024 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002627883650952859, + "loss": 2.7158, + "theoretical_loss": 3.499427577635764, + "tokens_seen": 1583426560 + }, + { + "epoch": 5.03, + "learning_rate": 0.000262778335005015, + "loss": 2.6024, + "theoretical_loss": 3.499414940000513, + "tokens_seen": 1583492096 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026276830491474425, + "loss": 2.7819, + "theoretical_loss": 3.4994023030347265, + "tokens_seen": 1583557632 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026275827482447343, + "loss": 2.7682, + "theoretical_loss": 3.499389666738343, + "tokens_seen": 1583623168 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002627482447342026, + "loss": 2.922, + "theoretical_loss": 3.499377031111299, + "tokens_seen": 1583688704 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026273821464393185, + "loss": 2.8512, + "theoretical_loss": 3.4993643961535312, + "tokens_seen": 1583754240 + }, + { + "epoch": 5.03, + "learning_rate": 0.000262728184553661, + "loss": 2.9017, + "theoretical_loss": 3.4993517618649763, + "tokens_seen": 1583819776 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002627181544633902, + "loss": 2.6698, + "theoretical_loss": 3.499339128245571, + "tokens_seen": 1583885312 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026270812437311934, + "loss": 2.8969, + "theoretical_loss": 3.4993264952952527, + "tokens_seen": 1583950848 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002626980942828486, + "loss": 2.7105, + "theoretical_loss": 3.4993138630139584, + "tokens_seen": 1584016384 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026268806419257776, + "loss": 2.7529, + "theoretical_loss": 3.4993012314016245, + "tokens_seen": 1584081920 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026267803410230694, + "loss": 2.9912, + "theoretical_loss": 3.499288600458188, + "tokens_seen": 1584147456 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002626680040120361, + "loss": 2.6269, + "theoretical_loss": 3.499275970183586, + "tokens_seen": 1584212992 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002626579739217653, + "loss": 2.8153, + "theoretical_loss": 3.499263340577755, + "tokens_seen": 1584278528 + }, + { + "epoch": 5.03, + "objective/train/docs_used": 1779618, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.2659285068511963, + "objective/train/theoretical_loss": 3.499253868812225, + "objective/train/tokens_used": 1604787680, + "theoretical_loss": 3.499253868812225, + "tokens_seen": 1584327680 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002626479438314945, + "loss": 2.7086, + "theoretical_loss": 3.4992507116406326, + "tokens_seen": 1584344064 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002626379137412237, + "loss": 2.8713, + "theoretical_loss": 3.499238083372155, + "tokens_seen": 1584409600 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026262788365095284, + "loss": 2.7341, + "theoretical_loss": 3.49922545577226, + "tokens_seen": 1584475136 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002626178535606821, + "loss": 2.5213, + "theoretical_loss": 3.4992128288408835, + "tokens_seen": 1584540672 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026260782347041126, + "loss": 3.0435, + "theoretical_loss": 3.4992002025779634, + "tokens_seen": 1584606208 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026259779338014044, + "loss": 2.5818, + "theoretical_loss": 3.499187576983436, + "tokens_seen": 1584671744 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002625877632898696, + "loss": 2.8307, + "theoretical_loss": 3.4991749520572384, + "tokens_seen": 1584737280 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002625777331995988, + "loss": 2.7998, + "theoretical_loss": 3.499162327799308, + "tokens_seen": 1584802816 + }, + { + "epoch": 5.03, + "learning_rate": 0.000262567703109328, + "loss": 2.754, + "theoretical_loss": 3.4991497042095814, + "tokens_seen": 1584868352 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002625576730190572, + "loss": 2.781, + "theoretical_loss": 3.499137081287996, + "tokens_seen": 1584933888 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026254764292878635, + "loss": 2.876, + "theoretical_loss": 3.4991244590344888, + "tokens_seen": 1584999424 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002625376128385156, + "loss": 2.8324, + "theoretical_loss": 3.4991118374489965, + "tokens_seen": 1585064960 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002625275827482447, + "loss": 2.6642, + "theoretical_loss": 3.4990992165314556, + "tokens_seen": 1585130496 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026251755265797394, + "loss": 2.7775, + "theoretical_loss": 3.499086596281804, + "tokens_seen": 1585196032 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002625075225677031, + "loss": 2.9426, + "theoretical_loss": 3.4990739766999788, + "tokens_seen": 1585261568 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002624974924774323, + "loss": 2.7238, + "theoretical_loss": 3.4990613577859166, + "tokens_seen": 1585327104 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002624874623871615, + "loss": 2.7153, + "theoretical_loss": 3.499048739539554, + "tokens_seen": 1585392640 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026247743229689067, + "loss": 2.6911, + "theoretical_loss": 3.4990361219608292, + "tokens_seen": 1585458176 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026246740220661985, + "loss": 2.6232, + "theoretical_loss": 3.4990235050496787, + "tokens_seen": 1585523712 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002624573721163491, + "loss": 2.8488, + "theoretical_loss": 3.49901088880604, + "tokens_seen": 1585589248 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002624473420260782, + "loss": 2.6884, + "theoretical_loss": 3.4989982732298497, + "tokens_seen": 1585654784 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026243731193580745, + "loss": 2.7215, + "theoretical_loss": 3.4989856583210446, + "tokens_seen": 1585720320 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026242728184553663, + "loss": 2.5312, + "theoretical_loss": 3.4989730440795626, + "tokens_seen": 1585785856 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002624172517552658, + "loss": 2.8339, + "theoretical_loss": 3.49896043050534, + "tokens_seen": 1585851392 + }, + { + "epoch": 5.03, + "learning_rate": 0.000262407221664995, + "loss": 2.8494, + "theoretical_loss": 3.498947817598315, + "tokens_seen": 1585916928 + }, + { + "epoch": 5.03, + "objective/train/docs_used": 1782479, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7037224769592285, + "objective/train/theoretical_loss": 3.4989383583558564, + "objective/train/tokens_used": 1606426080, + "theoretical_loss": 3.4989383583558564, + "tokens_seen": 1585966080 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026239719157472417, + "loss": 2.5762, + "theoretical_loss": 3.498935205358424, + "tokens_seen": 1585982464 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026238716148445335, + "loss": 2.5875, + "theoretical_loss": 3.4989225937856046, + "tokens_seen": 1586048000 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002623771313941826, + "loss": 2.8368, + "theoretical_loss": 3.498909982879793, + "tokens_seen": 1586113536 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002623671013039117, + "loss": 2.8362, + "theoretical_loss": 3.4988973726409274, + "tokens_seen": 1586179072 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026235707121364095, + "loss": 2.8293, + "theoretical_loss": 3.4988847630689444, + "tokens_seen": 1586244608 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002623470411233701, + "loss": 2.7497, + "theoretical_loss": 3.4988721541637817, + "tokens_seen": 1586310144 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002623370110330993, + "loss": 2.6262, + "theoretical_loss": 3.498859545925376, + "tokens_seen": 1586375680 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002623269809428285, + "loss": 2.7631, + "theoretical_loss": 3.4988469383536644, + "tokens_seen": 1586441216 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002623169508525577, + "loss": 2.7859, + "theoretical_loss": 3.498834331448585, + "tokens_seen": 1586506752 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026230692076228686, + "loss": 2.6273, + "theoretical_loss": 3.4988217252100737, + "tokens_seen": 1586572288 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002622968906720161, + "loss": 2.9191, + "theoretical_loss": 3.498809119638069, + "tokens_seen": 1586637824 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002622868605817452, + "loss": 2.5803, + "theoretical_loss": 3.4987965147325073, + "tokens_seen": 1586703360 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026227683049147445, + "loss": 2.6403, + "theoretical_loss": 3.4987839104933265, + "tokens_seen": 1586768896 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002622668004012036, + "loss": 2.7257, + "theoretical_loss": 3.498771306920463, + "tokens_seen": 1586834432 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002622567703109328, + "loss": 2.9422, + "theoretical_loss": 3.4987587040138544, + "tokens_seen": 1586899968 + }, + { + "epoch": 5.03, + "learning_rate": 0.000262246740220662, + "loss": 2.6503, + "theoretical_loss": 3.4987461017734383, + "tokens_seen": 1586965504 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002622367101303912, + "loss": 2.7249, + "theoretical_loss": 3.4987335001991515, + "tokens_seen": 1587031040 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026222668004012036, + "loss": 2.7806, + "theoretical_loss": 3.4987208992909316, + "tokens_seen": 1587096576 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026221664994984954, + "loss": 2.6487, + "theoretical_loss": 3.498708299048716, + "tokens_seen": 1587162112 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002622066198595787, + "loss": 2.6574, + "theoretical_loss": 3.4986956994724414, + "tokens_seen": 1587227648 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026219658976930796, + "loss": 2.7044, + "theoretical_loss": 3.498683100562046, + "tokens_seen": 1587293184 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002621865596790371, + "loss": 2.8528, + "theoretical_loss": 3.4986705023174665, + "tokens_seen": 1587358720 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002621765295887663, + "loss": 2.8764, + "theoretical_loss": 3.4986579047386406, + "tokens_seen": 1587424256 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026216649949849545, + "loss": 2.6997, + "theoretical_loss": 3.4986453078255053, + "tokens_seen": 1587489792 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002621564694082247, + "loss": 2.639, + "theoretical_loss": 3.498632711577998, + "tokens_seen": 1587555328 + }, + { + "epoch": 5.03, + "objective/train/docs_used": 1787756, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.56807279586792, + "objective/train/theoretical_loss": 3.498623264829148, + "objective/train/tokens_used": 1608064480, + "theoretical_loss": 3.498623264829148, + "tokens_seen": 1587604480 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026214643931795386, + "loss": 2.8254, + "theoretical_loss": 3.498620115996056, + "tokens_seen": 1587620864 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026213640922768304, + "loss": 2.7024, + "theoretical_loss": 3.4986075210796166, + "tokens_seen": 1587686400 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002621263791374122, + "loss": 2.7606, + "theoretical_loss": 3.498594926828618, + "tokens_seen": 1587751936 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026211634904714146, + "loss": 2.6963, + "theoretical_loss": 3.498582333242996, + "tokens_seen": 1587817472 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002621063189568706, + "loss": 2.6815, + "theoretical_loss": 3.498569740322689, + "tokens_seen": 1587883008 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002620962888665998, + "loss": 2.7735, + "theoretical_loss": 3.4985571480676354, + "tokens_seen": 1587948544 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026208625877632895, + "loss": 2.8927, + "theoretical_loss": 3.4985445564777704, + "tokens_seen": 1588014080 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002620762286860582, + "loss": 2.7365, + "theoretical_loss": 3.498531965553033, + "tokens_seen": 1588079616 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026206619859578737, + "loss": 2.8605, + "theoretical_loss": 3.49851937529336, + "tokens_seen": 1588145152 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026205616850551655, + "loss": 2.768, + "theoretical_loss": 3.498506785698689, + "tokens_seen": 1588210688 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026204613841524573, + "loss": 2.7571, + "theoretical_loss": 3.4984941967689576, + "tokens_seen": 1588276224 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002620361083249749, + "loss": 2.9175, + "theoretical_loss": 3.498481608504103, + "tokens_seen": 1588341760 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002620260782347041, + "loss": 2.8245, + "theoretical_loss": 3.4984690209040625, + "tokens_seen": 1588407296 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002620160481444333, + "loss": 2.7586, + "theoretical_loss": 3.498456433968774, + "tokens_seen": 1588472832 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002620060180541625, + "loss": 2.6923, + "theoretical_loss": 3.498443847698174, + "tokens_seen": 1588538368 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002619959879638917, + "loss": 2.4443, + "theoretical_loss": 3.4984312620922013, + "tokens_seen": 1588603904 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026198595787362087, + "loss": 2.8155, + "theoretical_loss": 3.498418677150793, + "tokens_seen": 1588669440 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026197592778335005, + "loss": 2.9109, + "theoretical_loss": 3.4984060928738856, + "tokens_seen": 1588734976 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002619658976930793, + "loss": 2.6826, + "theoretical_loss": 3.498393509261418, + "tokens_seen": 1588800512 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002619558676028084, + "loss": 2.8529, + "theoretical_loss": 3.4983809263133274, + "tokens_seen": 1588866048 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026194583751253765, + "loss": 2.6624, + "theoretical_loss": 3.4983683440295503, + "tokens_seen": 1588931584 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026193580742226683, + "loss": 2.7856, + "theoretical_loss": 3.4983557624100254, + "tokens_seen": 1588997120 + }, + { + "epoch": 5.03, + "learning_rate": 0.000261925777331996, + "loss": 2.7946, + "theoretical_loss": 3.4983431814546897, + "tokens_seen": 1589062656 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002619157472417252, + "loss": 2.487, + "theoretical_loss": 3.498330601163481, + "tokens_seen": 1589128192 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026190571715145437, + "loss": 2.6616, + "theoretical_loss": 3.4983180215363365, + "tokens_seen": 1589193728 + }, + { + "epoch": 5.03, + "objective/train/docs_used": 1792626, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8070321083068848, + "objective/train/theoretical_loss": 3.4983085872517328, + "objective/train/tokens_used": 1609702880, + "theoretical_loss": 3.4983085872517328, + "tokens_seen": 1589242880 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026189568706118355, + "loss": 2.7286, + "theoretical_loss": 3.498305442573194, + "tokens_seen": 1589259264 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002618856569709128, + "loss": 2.8236, + "theoretical_loss": 3.498292864273991, + "tokens_seen": 1589324800 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002618756268806419, + "loss": 2.7265, + "theoretical_loss": 3.498280286638665, + "tokens_seen": 1589390336 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026186559679037115, + "loss": 2.6342, + "theoretical_loss": 3.498267709667154, + "tokens_seen": 1589455872 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002618555667001003, + "loss": 2.5883, + "theoretical_loss": 3.498255133359395, + "tokens_seen": 1589521408 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002618455366098295, + "loss": 2.8205, + "theoretical_loss": 3.4982425577153267, + "tokens_seen": 1589586944 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002618355065195587, + "loss": 2.4463, + "theoretical_loss": 3.498229982734885, + "tokens_seen": 1589652480 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002618254764292879, + "loss": 2.73, + "theoretical_loss": 3.498217408418009, + "tokens_seen": 1589718016 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026181544633901706, + "loss": 2.6927, + "theoretical_loss": 3.4982048347646355, + "tokens_seen": 1589783552 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002618054162487463, + "loss": 2.8462, + "theoretical_loss": 3.4981922617747023, + "tokens_seen": 1589849088 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002617953861584754, + "loss": 2.648, + "theoretical_loss": 3.4981796894481474, + "tokens_seen": 1589914624 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026178535606820465, + "loss": 2.6182, + "theoretical_loss": 3.498167117784908, + "tokens_seen": 1589980160 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002617753259779338, + "loss": 2.7333, + "theoretical_loss": 3.498154546784922, + "tokens_seen": 1590045696 + }, + { + "epoch": 5.03, + "learning_rate": 0.000261765295887663, + "loss": 2.7277, + "theoretical_loss": 3.498141976448127, + "tokens_seen": 1590111232 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002617552657973922, + "loss": 2.7771, + "theoretical_loss": 3.4981294067744617, + "tokens_seen": 1590176768 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002617452357071214, + "loss": 2.7176, + "theoretical_loss": 3.498116837763862, + "tokens_seen": 1590242304 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026173520561685056, + "loss": 2.5995, + "theoretical_loss": 3.4981042694162663, + "tokens_seen": 1590307840 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026172517552657974, + "loss": 2.8918, + "theoretical_loss": 3.498091701731613, + "tokens_seen": 1590373376 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002617151454363089, + "loss": 2.9067, + "theoretical_loss": 3.4980791347098386, + "tokens_seen": 1590438912 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026170511534603816, + "loss": 2.637, + "theoretical_loss": 3.498066568350882, + "tokens_seen": 1590504448 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002616950852557673, + "loss": 2.973, + "theoretical_loss": 3.49805400265468, + "tokens_seen": 1590569984 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002616850551654965, + "loss": 2.7506, + "theoretical_loss": 3.498041437621171, + "tokens_seen": 1590635520 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026167502507522565, + "loss": 2.6337, + "theoretical_loss": 3.498028873250292, + "tokens_seen": 1590701056 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002616649949849549, + "loss": 2.7196, + "theoretical_loss": 3.498016309541982, + "tokens_seen": 1590766592 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026165496489468406, + "loss": 2.7316, + "theoretical_loss": 3.498003746496178, + "tokens_seen": 1590832128 + }, + { + "epoch": 5.03, + "objective/train/docs_used": 1797804, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7346248626708984, + "objective/train/theoretical_loss": 3.497994324646557, + "objective/train/tokens_used": 1611341280, + "theoretical_loss": 3.497994324646557, + "tokens_seen": 1590881280 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026164493480441324, + "loss": 2.7022, + "theoretical_loss": 3.4979911841128173, + "tokens_seen": 1590897664 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002616349047141424, + "loss": 2.4981, + "theoretical_loss": 3.4979786223918388, + "tokens_seen": 1590963200 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026162487462387166, + "loss": 2.7815, + "theoretical_loss": 3.4979660613331793, + "tokens_seen": 1591028736 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002616148445336008, + "loss": 2.7909, + "theoretical_loss": 3.4979535009367773, + "tokens_seen": 1591094272 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026160481444333, + "loss": 2.8896, + "theoretical_loss": 3.4979409412025695, + "tokens_seen": 1591159808 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026159478435305915, + "loss": 2.8332, + "theoretical_loss": 3.497928382130495, + "tokens_seen": 1591225344 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002615847542627884, + "loss": 2.6551, + "theoretical_loss": 3.4979158237204913, + "tokens_seen": 1591290880 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026157472417251757, + "loss": 2.8388, + "theoretical_loss": 3.4979032659724956, + "tokens_seen": 1591356416 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026156469408224675, + "loss": 2.8684, + "theoretical_loss": 3.4978907088864464, + "tokens_seen": 1591421952 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026155466399197593, + "loss": 2.8727, + "theoretical_loss": 3.4978781524622815, + "tokens_seen": 1591487488 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002615446339017051, + "loss": 2.7349, + "theoretical_loss": 3.4978655966999384, + "tokens_seen": 1591553024 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002615346038114343, + "loss": 2.8585, + "theoretical_loss": 3.497853041599355, + "tokens_seen": 1591618560 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002615245737211635, + "loss": 2.9453, + "theoretical_loss": 3.4978404871604702, + "tokens_seen": 1591684096 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026151454363089265, + "loss": 2.53, + "theoretical_loss": 3.4978279333832205, + "tokens_seen": 1591749632 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002615045135406219, + "loss": 2.663, + "theoretical_loss": 3.497815380267544, + "tokens_seen": 1591815168 + }, + { + "epoch": 5.03, + "learning_rate": 0.000261494483450351, + "loss": 2.6538, + "theoretical_loss": 3.4978028278133797, + "tokens_seen": 1591880704 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026148445336008025, + "loss": 2.9559, + "theoretical_loss": 3.497790276020664, + "tokens_seen": 1591946240 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026147442326980943, + "loss": 2.7134, + "theoretical_loss": 3.4977777248893354, + "tokens_seen": 1592011776 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002614643931795386, + "loss": 2.4606, + "theoretical_loss": 3.4977651744193325, + "tokens_seen": 1592077312 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002614543630892678, + "loss": 2.692, + "theoretical_loss": 3.4977526246105928, + "tokens_seen": 1592142848 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026144433299899703, + "loss": 2.8035, + "theoretical_loss": 3.497740075463054, + "tokens_seen": 1592208384 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026143430290872616, + "loss": 2.7924, + "theoretical_loss": 3.4977275269766537, + "tokens_seen": 1592273920 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002614242728184554, + "loss": 2.8615, + "theoretical_loss": 3.497714979151331, + "tokens_seen": 1592339456 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002614142427281845, + "loss": 2.8544, + "theoretical_loss": 3.4977024319870234, + "tokens_seen": 1592404992 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026140421263791375, + "loss": 2.6875, + "theoretical_loss": 3.497689885483668, + "tokens_seen": 1592470528 + }, + { + "epoch": 5.03, + "objective/train/docs_used": 1800873, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.813413143157959, + "objective/train/theoretical_loss": 3.4976804760398648, + "objective/train/tokens_used": 1612979680, + "theoretical_loss": 3.4976804760398648, + "tokens_seen": 1592519680 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026139418254764293, + "loss": 2.5509, + "theoretical_loss": 3.497677339641204, + "tokens_seen": 1592536064 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002613841524573721, + "loss": 2.6167, + "theoretical_loss": 3.4976647944595687, + "tokens_seen": 1592601600 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002613741223671013, + "loss": 2.8014, + "theoretical_loss": 3.4976522499387004, + "tokens_seen": 1592667136 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002613640922768305, + "loss": 2.6458, + "theoretical_loss": 3.4976397060785365, + "tokens_seen": 1592732672 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026135406218655966, + "loss": 2.7171, + "theoretical_loss": 3.497627162879016, + "tokens_seen": 1592798208 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002613440320962889, + "loss": 2.8442, + "theoretical_loss": 3.4976146203400766, + "tokens_seen": 1592863744 + }, + { + "epoch": 5.03, + "learning_rate": 0.000261334002006018, + "loss": 2.9746, + "theoretical_loss": 3.4976020784616555, + "tokens_seen": 1592929280 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026132397191574726, + "loss": 2.8905, + "theoretical_loss": 3.497589537243692, + "tokens_seen": 1592994816 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002613139418254764, + "loss": 2.7806, + "theoretical_loss": 3.4975769966861234, + "tokens_seen": 1593060352 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002613039117352056, + "loss": 2.7803, + "theoretical_loss": 3.497564456788888, + "tokens_seen": 1593125888 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002612938816449348, + "loss": 2.7665, + "theoretical_loss": 3.497551917551924, + "tokens_seen": 1593191424 + }, + { + "epoch": 5.03, + "learning_rate": 0.000261283851554664, + "loss": 2.6711, + "theoretical_loss": 3.4975393789751688, + "tokens_seen": 1593256960 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026127382146439316, + "loss": 2.6177, + "theoretical_loss": 3.497526841058561, + "tokens_seen": 1593322496 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002612637913741224, + "loss": 2.5415, + "theoretical_loss": 3.4975143038020384, + "tokens_seen": 1593388032 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002612537612838516, + "loss": 2.8254, + "theoretical_loss": 3.4975017672055397, + "tokens_seen": 1593453568 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026124373119358076, + "loss": 2.6499, + "theoretical_loss": 3.497489231269003, + "tokens_seen": 1593519104 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026123370110330994, + "loss": 2.8149, + "theoretical_loss": 3.4974766959923658, + "tokens_seen": 1593584640 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002612236710130391, + "loss": 2.6921, + "theoretical_loss": 3.497464161375566, + "tokens_seen": 1593650176 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026121364092276836, + "loss": 2.878, + "theoretical_loss": 3.4974516274185428, + "tokens_seen": 1593715712 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002612036108324975, + "loss": 2.6749, + "theoretical_loss": 3.497439094121234, + "tokens_seen": 1593781248 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002611935807422267, + "loss": 2.6689, + "theoretical_loss": 3.497426561483577, + "tokens_seen": 1593846784 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026118355065195585, + "loss": 2.6218, + "theoretical_loss": 3.497414029505511, + "tokens_seen": 1593912320 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002611735205616851, + "loss": 2.685, + "theoretical_loss": 3.4974014981869734, + "tokens_seen": 1593977856 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026116349047141426, + "loss": 2.7086, + "theoretical_loss": 3.497388967527902, + "tokens_seen": 1594043392 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026115346038114344, + "loss": 2.5245, + "theoretical_loss": 3.4973764375282363, + "tokens_seen": 1594108928 + }, + { + "epoch": 5.03, + "objective/train/docs_used": 1801349, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.83113694190979, + "objective/train/theoretical_loss": 3.4973670404611843, + "objective/train/tokens_used": 1614618080, + "theoretical_loss": 3.4973670404611843, + "tokens_seen": 1594158080 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002611434302908726, + "loss": 2.5249, + "theoretical_loss": 3.4973639081879138, + "tokens_seen": 1594174464 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026113340020060186, + "loss": 2.8712, + "theoretical_loss": 3.4973513795068727, + "tokens_seen": 1594240000 + }, + { + "epoch": 5.03, + "learning_rate": 0.000261123370110331, + "loss": 2.8316, + "theoretical_loss": 3.4973388514850514, + "tokens_seen": 1594305536 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002611133400200602, + "loss": 2.7948, + "theoretical_loss": 3.4973263241223873, + "tokens_seen": 1594371072 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026110330992978935, + "loss": 2.8875, + "theoretical_loss": 3.49731379741882, + "tokens_seen": 1594436608 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002610932798395186, + "loss": 2.7071, + "theoretical_loss": 3.4973012713742864, + "tokens_seen": 1594502144 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026108324974924777, + "loss": 2.6609, + "theoretical_loss": 3.4972887459887256, + "tokens_seen": 1594567680 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026107321965897695, + "loss": 2.9401, + "theoretical_loss": 3.4972762212620756, + "tokens_seen": 1594633216 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026106318956870613, + "loss": 2.6306, + "theoretical_loss": 3.4972636971942745, + "tokens_seen": 1594698752 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002610531594784353, + "loss": 2.9314, + "theoretical_loss": 3.497251173785261, + "tokens_seen": 1594764288 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002610431293881645, + "loss": 2.7628, + "theoretical_loss": 3.497238651034973, + "tokens_seen": 1594829824 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002610330992978937, + "loss": 2.6542, + "theoretical_loss": 3.497226128943349, + "tokens_seen": 1594895360 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026102306920762285, + "loss": 2.7527, + "theoretical_loss": 3.497213607510327, + "tokens_seen": 1594960896 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002610130391173521, + "loss": 2.9548, + "theoretical_loss": 3.4972010867358456, + "tokens_seen": 1595026432 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002610030090270812, + "loss": 2.7296, + "theoretical_loss": 3.4971885666198426, + "tokens_seen": 1595091968 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026099297893681045, + "loss": 2.8855, + "theoretical_loss": 3.497176047162257, + "tokens_seen": 1595157504 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026098294884653963, + "loss": 2.5206, + "theoretical_loss": 3.4971635283630267, + "tokens_seen": 1595223040 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002609729187562688, + "loss": 2.8073, + "theoretical_loss": 3.4971510102220904, + "tokens_seen": 1595288576 + }, + { + "epoch": 5.03, + "learning_rate": 0.000260962888665998, + "loss": 2.5685, + "theoretical_loss": 3.497138492739386, + "tokens_seen": 1595354112 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026095285857572723, + "loss": 2.7139, + "theoretical_loss": 3.4971259759148525, + "tokens_seen": 1595419648 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026094282848545636, + "loss": 2.6919, + "theoretical_loss": 3.4971134597484275, + "tokens_seen": 1595485184 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002609327983951856, + "loss": 2.7325, + "theoretical_loss": 3.4971009442400494, + "tokens_seen": 1595550720 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002609227683049147, + "loss": 2.6534, + "theoretical_loss": 3.497088429389657, + "tokens_seen": 1595616256 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026091273821464395, + "loss": 2.6995, + "theoretical_loss": 3.497075915197189, + "tokens_seen": 1595681792 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026090270812437313, + "loss": 2.8369, + "theoretical_loss": 3.497063401662583, + "tokens_seen": 1595747328 + }, + { + "epoch": 5.03, + "objective/train/docs_used": 1802636, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.037900686264038, + "objective/train/theoretical_loss": 3.4970540169433133, + "objective/train/tokens_used": 1616256480, + "theoretical_loss": 3.4970540169433133, + "tokens_seen": 1595796480 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002608926780341023, + "loss": 2.5637, + "theoretical_loss": 3.4970508887857776, + "tokens_seen": 1595812864 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002608826479438315, + "loss": 2.7449, + "theoretical_loss": 3.4970383765667115, + "tokens_seen": 1595878400 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002608726178535607, + "loss": 2.7603, + "theoretical_loss": 3.497025865005323, + "tokens_seen": 1595943936 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026086258776328986, + "loss": 2.7163, + "theoretical_loss": 3.4970133541015507, + "tokens_seen": 1596009472 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002608525576730191, + "loss": 2.7163, + "theoretical_loss": 3.497000843855332, + "tokens_seen": 1596075008 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002608425275827482, + "loss": 2.876, + "theoretical_loss": 3.4969883342666073, + "tokens_seen": 1596140544 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026083249749247746, + "loss": 2.9052, + "theoretical_loss": 3.496975825335313, + "tokens_seen": 1596206080 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002608224674022066, + "loss": 2.751, + "theoretical_loss": 3.496963317061389, + "tokens_seen": 1596271616 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002608124373119358, + "loss": 2.7262, + "theoretical_loss": 3.496950809444773, + "tokens_seen": 1596337152 + }, + { + "epoch": 5.03, + "learning_rate": 0.000260802407221665, + "loss": 2.8612, + "theoretical_loss": 3.496938302485404, + "tokens_seen": 1596402688 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002607923771313942, + "loss": 2.6211, + "theoretical_loss": 3.49692579618322, + "tokens_seen": 1596468224 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026078234704112336, + "loss": 2.7479, + "theoretical_loss": 3.49691329053816, + "tokens_seen": 1596533760 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002607723169508526, + "loss": 2.7243, + "theoretical_loss": 3.496900785550162, + "tokens_seen": 1596599296 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002607622868605817, + "loss": 2.7206, + "theoretical_loss": 3.4968882812191646, + "tokens_seen": 1596664832 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026075225677031096, + "loss": 2.637, + "theoretical_loss": 3.4968757775451063, + "tokens_seen": 1596730368 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002607422266800401, + "loss": 2.6786, + "theoretical_loss": 3.496863274527926, + "tokens_seen": 1596795904 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002607321965897693, + "loss": 2.7166, + "theoretical_loss": 3.4968507721675617, + "tokens_seen": 1596861440 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002607221664994985, + "loss": 2.801, + "theoretical_loss": 3.4968382704639525, + "tokens_seen": 1596926976 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002607121364092277, + "loss": 2.6867, + "theoretical_loss": 3.4968257694170366, + "tokens_seen": 1596992512 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026070210631895687, + "loss": 2.7219, + "theoretical_loss": 3.4968132690267524, + "tokens_seen": 1597058048 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026069207622868605, + "loss": 2.7463, + "theoretical_loss": 3.4968007692930394, + "tokens_seen": 1597123584 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026068204613841523, + "loss": 2.6852, + "theoretical_loss": 3.496788270215835, + "tokens_seen": 1597189120 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026067201604814446, + "loss": 2.6713, + "theoretical_loss": 3.496775771795078, + "tokens_seen": 1597254656 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002606619859578736, + "loss": 2.618, + "theoretical_loss": 3.496763274030708, + "tokens_seen": 1597320192 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002606519558676028, + "loss": 2.7221, + "theoretical_loss": 3.496750776922662, + "tokens_seen": 1597385728 + }, + { + "epoch": 5.03, + "objective/train/docs_used": 1803304, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7198164463043213, + "objective/train/theoretical_loss": 3.4967414045223038, + "objective/train/tokens_used": 1617894880, + "theoretical_loss": 3.4967414045223038, + "tokens_seen": 1597434880 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026064192577733195, + "loss": 2.8111, + "theoretical_loss": 3.49673828047088, + "tokens_seen": 1597451264 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002606318956870612, + "loss": 2.7815, + "theoretical_loss": 3.4967257846752995, + "tokens_seen": 1597516800 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026062186559679037, + "loss": 2.6373, + "theoretical_loss": 3.4967132895358604, + "tokens_seen": 1597582336 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026061183550651955, + "loss": 2.8427, + "theoretical_loss": 3.4967007950525, + "tokens_seen": 1597647872 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026060180541624873, + "loss": 2.7914, + "theoretical_loss": 3.4966883012251584, + "tokens_seen": 1597713408 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026059177532597797, + "loss": 2.6316, + "theoretical_loss": 3.496675808053773, + "tokens_seen": 1597778944 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002605817452357071, + "loss": 2.8277, + "theoretical_loss": 3.4966633155382825, + "tokens_seen": 1597844480 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026057171514543633, + "loss": 2.7669, + "theoretical_loss": 3.4966508236786265, + "tokens_seen": 1597910016 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026056168505516546, + "loss": 2.4671, + "theoretical_loss": 3.4966383324747428, + "tokens_seen": 1597975552 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002605516549648947, + "loss": 2.7547, + "theoretical_loss": 3.49662584192657, + "tokens_seen": 1598041088 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026054162487462387, + "loss": 2.4883, + "theoretical_loss": 3.496613352034048, + "tokens_seen": 1598106624 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026053159478435305, + "loss": 2.676, + "theoretical_loss": 3.496600862797114, + "tokens_seen": 1598172160 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026052156469408223, + "loss": 2.6167, + "theoretical_loss": 3.4965883742157082, + "tokens_seen": 1598237696 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002605115346038114, + "loss": 2.9133, + "theoretical_loss": 3.496575886289768, + "tokens_seen": 1598303232 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026050150451354065, + "loss": 2.7645, + "theoretical_loss": 3.4965633990192324, + "tokens_seen": 1598368768 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026049147442326983, + "loss": 2.6988, + "theoretical_loss": 3.496550912404041, + "tokens_seen": 1598434304 + }, + { + "epoch": 5.03, + "learning_rate": 0.000260481444332999, + "loss": 2.688, + "theoretical_loss": 3.4965384264441313, + "tokens_seen": 1598499840 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002604714142427282, + "loss": 2.9453, + "theoretical_loss": 3.496525941139443, + "tokens_seen": 1598565376 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026046138415245743, + "loss": 2.7754, + "theoretical_loss": 3.4965134564899145, + "tokens_seen": 1598630912 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026045135406218656, + "loss": 2.8114, + "theoretical_loss": 3.4965009724954843, + "tokens_seen": 1598696448 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002604413239719158, + "loss": 2.7888, + "theoretical_loss": 3.496488489156092, + "tokens_seen": 1598761984 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002604312938816449, + "loss": 3.0041, + "theoretical_loss": 3.496476006471675, + "tokens_seen": 1598827520 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026042126379137415, + "loss": 2.6466, + "theoretical_loss": 3.4964635244421736, + "tokens_seen": 1598893056 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026041123370110333, + "loss": 2.5899, + "theoretical_loss": 3.496451043067526, + "tokens_seen": 1598958592 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002604012036108325, + "loss": 2.84, + "theoretical_loss": 3.49643856234767, + "tokens_seen": 1599024128 + }, + { + "epoch": 5.03, + "objective/train/docs_used": 1804585, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.1171164512634277, + "objective/train/theoretical_loss": 3.4964292022374495, + "objective/train/tokens_used": 1619533280, + "theoretical_loss": 3.4964292022374495, + "tokens_seen": 1599073280 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002603911735205617, + "loss": 2.8474, + "theoretical_loss": 3.496426082282546, + "tokens_seen": 1599089664 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002603811434302909, + "loss": 2.5638, + "theoretical_loss": 3.4964136028720922, + "tokens_seen": 1599155200 + }, + { + "epoch": 5.03, + "learning_rate": 0.00026037111334002006, + "loss": 2.6904, + "theoretical_loss": 3.4964011241162476, + "tokens_seen": 1599220736 + }, + { + "epoch": 5.03, + "learning_rate": 0.0002603610832497493, + "loss": 2.6742, + "theoretical_loss": 3.4963886460149505, + "tokens_seen": 1599286272 + }, + { + "epoch": 5.04, + "learning_rate": 0.0002603510531594784, + "loss": 2.7654, + "theoretical_loss": 3.49637616856814, + "tokens_seen": 1599351808 + }, + { + "epoch": 5.04, + "learning_rate": 0.00026034102306920766, + "loss": 2.6412, + "theoretical_loss": 3.4963636917757555, + "tokens_seen": 1599417344 + }, + { + "epoch": 5.04, + "learning_rate": 0.0002603309929789368, + "loss": 2.8438, + "theoretical_loss": 3.4963512156377345, + "tokens_seen": 1599482880 + }, + { + "epoch": 5.04, + "learning_rate": 0.000260320962888666, + "loss": 2.7242, + "theoretical_loss": 3.4963387401540174, + "tokens_seen": 1599548416 + }, + { + "epoch": 5.04, + "learning_rate": 0.0002603109327983952, + "loss": 2.8938, + "theoretical_loss": 3.4963262653245426, + "tokens_seen": 1599613952 + }, + { + "epoch": 5.04, + "learning_rate": 0.0002603009027081244, + "loss": 2.7322, + "theoretical_loss": 3.496313791149248, + "tokens_seen": 1599679488 + }, + { + "epoch": 5.04, + "learning_rate": 0.00026029087261785356, + "loss": 2.489, + "theoretical_loss": 3.4963013176280744, + "tokens_seen": 1599745024 + }, + { + "epoch": 5.04, + "learning_rate": 0.0002602808425275828, + "loss": 2.7086, + "theoretical_loss": 3.496288844760959, + "tokens_seen": 1599810560 + }, + { + "epoch": 5.04, + "learning_rate": 0.0002602708124373119, + "loss": 2.6735, + "theoretical_loss": 3.4962763725478414, + "tokens_seen": 1599876096 + }, + { + "epoch": 5.04, + "learning_rate": 0.00026026078234704116, + "loss": 2.527, + "theoretical_loss": 3.4962639009886605, + "tokens_seen": 1599941632 + }, + { + "epoch": 5.04, + "learning_rate": 0.0002602507522567703, + "loss": 2.7704, + "theoretical_loss": 3.4962514300833556, + "tokens_seen": 1600007168 + }, + { + "epoch": 5.04, + "learning_rate": 0.0002602407221664995, + "loss": 2.7558, + "theoretical_loss": 3.4962389598318646, + "tokens_seen": 1600072704 + }, + { + "epoch": 5.04, + "learning_rate": 0.0002602306920762287, + "loss": 2.4681, + "theoretical_loss": 3.496226490234127, + "tokens_seen": 1600138240 + }, + { + "epoch": 5.04, + "learning_rate": 0.0002602206619859579, + "loss": 2.437, + "theoretical_loss": 3.4962140212900827, + "tokens_seen": 1600203776 + }, + { + "epoch": 5.04, + "learning_rate": 0.00026021063189568707, + "loss": 2.6458, + "theoretical_loss": 3.4962015529996693, + "tokens_seen": 1600269312 + }, + { + "epoch": 5.04, + "learning_rate": 0.00026020060180541625, + "loss": 2.7163, + "theoretical_loss": 3.4961890853628264, + "tokens_seen": 1600334848 + }, + { + "epoch": 5.04, + "learning_rate": 0.00026019057171514543, + "loss": 2.6421, + "theoretical_loss": 3.4961766183794922, + "tokens_seen": 1600400384 + }, + { + "epoch": 5.04, + "learning_rate": 0.00026018054162487466, + "loss": 2.5906, + "theoretical_loss": 3.496164152049607, + "tokens_seen": 1600465920 + }, + { + "epoch": 5.04, + "learning_rate": 0.0002601705115346038, + "loss": 2.7296, + "theoretical_loss": 3.496151686373109, + "tokens_seen": 1600531456 + }, + { + "epoch": 5.04, + "learning_rate": 0.000260160481444333, + "loss": 2.9529, + "theoretical_loss": 3.496139221349937, + "tokens_seen": 1600596992 + }, + { + "epoch": 5.04, + "learning_rate": 0.00026015045135406215, + "loss": 2.7484, + "theoretical_loss": 3.4961267569800305, + "tokens_seen": 1600662528 + }, + { + "epoch": 5.04, + "objective/train/docs_used": 1805307, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9074008464813232, + "objective/train/theoretical_loss": 3.4961174091312692, + "objective/train/tokens_used": 1621171680, + "theoretical_loss": 3.4961174091312692, + "tokens_seen": 1600711680 + }, + { + "epoch": 5.04, + "learning_rate": 0.0002601404212637914, + "loss": 2.8461, + "theoretical_loss": 3.4961142932633287, + "tokens_seen": 1600728064 + }, + { + "epoch": 5.04, + "learning_rate": 0.00026013039117352057, + "loss": 2.701, + "theoretical_loss": 3.4961018301997697, + "tokens_seen": 1600793600 + }, + { + "epoch": 5.04, + "learning_rate": 0.00026012036108324975, + "loss": 2.7185, + "theoretical_loss": 3.4960893677892937, + "tokens_seen": 1600859136 + }, + { + "epoch": 5.04, + "learning_rate": 0.00026011033099297893, + "loss": 2.4665, + "theoretical_loss": 3.496076906031839, + "tokens_seen": 1600924672 + }, + { + "epoch": 5.04, + "learning_rate": 0.00026010030090270817, + "loss": 2.6753, + "theoretical_loss": 3.4960644449273444, + "tokens_seen": 1600990208 + }, + { + "epoch": 5.04, + "learning_rate": 0.0002600902708124373, + "loss": 2.8114, + "theoretical_loss": 3.49605198447575, + "tokens_seen": 1601055744 + }, + { + "epoch": 5.04, + "learning_rate": 0.00026008024072216653, + "loss": 2.7058, + "theoretical_loss": 3.496039524676994, + "tokens_seen": 1601121280 + }, + { + "epoch": 5.04, + "learning_rate": 0.00026007021063189566, + "loss": 2.7671, + "theoretical_loss": 3.496027065531015, + "tokens_seen": 1601186816 + }, + { + "epoch": 5.04, + "learning_rate": 0.0002600601805416249, + "loss": 2.688, + "theoretical_loss": 3.496014607037754, + "tokens_seen": 1601252352 + }, + { + "epoch": 5.04, + "learning_rate": 0.00026005015045135407, + "loss": 2.7581, + "theoretical_loss": 3.496002149197148, + "tokens_seen": 1601317888 + }, + { + "epoch": 5.04, + "learning_rate": 0.00026004012036108325, + "loss": 2.8447, + "theoretical_loss": 3.4959896920091373, + "tokens_seen": 1601383424 + }, + { + "epoch": 5.04, + "learning_rate": 0.00026003009027081243, + "loss": 2.6624, + "theoretical_loss": 3.495977235473661, + "tokens_seen": 1601448960 + }, + { + "epoch": 5.04, + "learning_rate": 0.0002600200601805416, + "loss": 2.6325, + "theoretical_loss": 3.4959647795906577, + "tokens_seen": 1601514496 + }, + { + "epoch": 5.04, + "learning_rate": 0.0002600100300902708, + "loss": 2.5836, + "theoretical_loss": 3.4959523243600668, + "tokens_seen": 1601580032 + }, + { + "epoch": 5.04, + "learning_rate": 0.00026000000000000003, + "loss": 2.7603, + "theoretical_loss": 3.495939869781827, + "tokens_seen": 1601645568 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025998996990972916, + "loss": 2.6747, + "theoretical_loss": 3.495927415855878, + "tokens_seen": 1601711104 + }, + { + "epoch": 5.04, + "learning_rate": 0.0002599799398194584, + "loss": 2.5385, + "theoretical_loss": 3.495914962582159, + "tokens_seen": 1601776640 + }, + { + "epoch": 5.04, + "learning_rate": 0.0002599699097291876, + "loss": 2.6931, + "theoretical_loss": 3.495902509960609, + "tokens_seen": 1601842176 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025995987963891676, + "loss": 2.5442, + "theoretical_loss": 3.4958900579911667, + "tokens_seen": 1601907712 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025994984954864594, + "loss": 2.7225, + "theoretical_loss": 3.495877606673772, + "tokens_seen": 1601973248 + }, + { + "epoch": 5.04, + "learning_rate": 0.0002599398194583751, + "loss": 2.8537, + "theoretical_loss": 3.495865156008364, + "tokens_seen": 1602038784 + }, + { + "epoch": 5.04, + "learning_rate": 0.0002599297893681043, + "loss": 2.4171, + "theoretical_loss": 3.495852705994881, + "tokens_seen": 1602104320 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025991975927783353, + "loss": 2.6795, + "theoretical_loss": 3.4958402566332634, + "tokens_seen": 1602169856 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025990972918756266, + "loss": 2.7819, + "theoretical_loss": 3.4958278079234493, + "tokens_seen": 1602235392 + }, + { + "epoch": 5.04, + "learning_rate": 0.0002598996990972919, + "loss": 2.6898, + "theoretical_loss": 3.4958153598653787, + "tokens_seen": 1602300928 + }, + { + "epoch": 5.04, + "objective/train/docs_used": 1806486, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.2253284454345703, + "objective/train/theoretical_loss": 3.4958060242494957, + "objective/train/tokens_used": 1622810080, + "theoretical_loss": 3.4958060242494957, + "tokens_seen": 1602350080 + }, + { + "epoch": 5.04, + "learning_rate": 0.000259889669007021, + "loss": 2.7106, + "theoretical_loss": 3.495802912458991, + "tokens_seen": 1602366464 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025987963891675026, + "loss": 2.9012, + "theoretical_loss": 3.4957904657042245, + "tokens_seen": 1602432000 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025986960882647944, + "loss": 2.6426, + "theoretical_loss": 3.495778019601019, + "tokens_seen": 1602497536 + }, + { + "epoch": 5.04, + "learning_rate": 0.0002598595787362086, + "loss": 2.8334, + "theoretical_loss": 3.495765574149314, + "tokens_seen": 1602563072 + }, + { + "epoch": 5.04, + "learning_rate": 0.0002598495486459378, + "loss": 2.8165, + "theoretical_loss": 3.4957531293490485, + "tokens_seen": 1602628608 + }, + { + "epoch": 5.04, + "learning_rate": 0.000259839518555667, + "loss": 2.6539, + "theoretical_loss": 3.495740685200161, + "tokens_seen": 1602694144 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025982948846539616, + "loss": 2.5846, + "theoretical_loss": 3.4957282417025928, + "tokens_seen": 1602759680 + }, + { + "epoch": 5.04, + "learning_rate": 0.0002598194583751254, + "loss": 2.8119, + "theoretical_loss": 3.495715798856281, + "tokens_seen": 1602825216 + }, + { + "epoch": 5.04, + "learning_rate": 0.0002598094282848545, + "loss": 2.7783, + "theoretical_loss": 3.4957033566611657, + "tokens_seen": 1602890752 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025979939819458376, + "loss": 2.4276, + "theoretical_loss": 3.4956909151171867, + "tokens_seen": 1602956288 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025978936810431294, + "loss": 2.6884, + "theoretical_loss": 3.495678474224283, + "tokens_seen": 1603021824 + }, + { + "epoch": 5.04, + "learning_rate": 0.0002597793380140421, + "loss": 2.5727, + "theoretical_loss": 3.495666033982393, + "tokens_seen": 1603087360 + }, + { + "epoch": 5.04, + "learning_rate": 0.0002597693079237713, + "loss": 2.6541, + "theoretical_loss": 3.4956535943914573, + "tokens_seen": 1603152896 + }, + { + "epoch": 5.04, + "learning_rate": 0.0002597592778335005, + "loss": 2.813, + "theoretical_loss": 3.4956411554514144, + "tokens_seen": 1603218432 + }, + { + "epoch": 5.04, + "learning_rate": 0.0002597492477432297, + "loss": 2.6882, + "theoretical_loss": 3.4956287171622047, + "tokens_seen": 1603283968 + }, + { + "epoch": 5.04, + "learning_rate": 0.0002597392176529589, + "loss": 2.6274, + "theoretical_loss": 3.495616279523766, + "tokens_seen": 1603349504 + }, + { + "epoch": 5.04, + "learning_rate": 0.0002597291875626881, + "loss": 2.5218, + "theoretical_loss": 3.495603842536039, + "tokens_seen": 1603415040 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025971915747241727, + "loss": 2.8809, + "theoretical_loss": 3.4955914061989626, + "tokens_seen": 1603480576 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025970912738214645, + "loss": 2.6464, + "theoretical_loss": 3.495578970512476, + "tokens_seen": 1603546112 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025969909729187563, + "loss": 2.7924, + "theoretical_loss": 3.4955665354765184, + "tokens_seen": 1603611648 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025968906720160486, + "loss": 2.6967, + "theoretical_loss": 3.49555410109103, + "tokens_seen": 1603677184 + }, + { + "epoch": 5.04, + "learning_rate": 0.000259679037111334, + "loss": 2.6436, + "theoretical_loss": 3.4955416673559494, + "tokens_seen": 1603742720 + }, + { + "epoch": 5.04, + "learning_rate": 0.0002596690070210632, + "loss": 2.6897, + "theoretical_loss": 3.495529234271216, + "tokens_seen": 1603808256 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025965897693079235, + "loss": 2.7027, + "theoretical_loss": 3.4955168018367697, + "tokens_seen": 1603873792 + }, + { + "epoch": 5.04, + "learning_rate": 0.0002596489468405216, + "loss": 2.8437, + "theoretical_loss": 3.49550437005255, + "tokens_seen": 1603939328 + }, + { + "epoch": 5.04, + "objective/train/docs_used": 1807150, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.3969147205352783, + "objective/train/theoretical_loss": 3.495495046641059, + "objective/train/tokens_used": 1624448480, + "theoretical_loss": 3.495495046641059, + "tokens_seen": 1603988480 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025963891675025077, + "loss": 2.6088, + "theoretical_loss": 3.4954919389184953, + "tokens_seen": 1604004864 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025962888665997995, + "loss": 2.5807, + "theoretical_loss": 3.4954795084345465, + "tokens_seen": 1604070400 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025961885656970913, + "loss": 2.5547, + "theoretical_loss": 3.4954670786006417, + "tokens_seen": 1604135936 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025960882647943837, + "loss": 2.7026, + "theoretical_loss": 3.495454649416721, + "tokens_seen": 1604201472 + }, + { + "epoch": 5.04, + "learning_rate": 0.0002595987963891675, + "loss": 2.704, + "theoretical_loss": 3.4954422208827243, + "tokens_seen": 1604267008 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025958876629889673, + "loss": 2.7917, + "theoretical_loss": 3.4954297929985905, + "tokens_seen": 1604332544 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025957873620862586, + "loss": 2.8454, + "theoretical_loss": 3.4954173657642587, + "tokens_seen": 1604398080 + }, + { + "epoch": 5.04, + "learning_rate": 0.0002595687061183551, + "loss": 2.7291, + "theoretical_loss": 3.4954049391796693, + "tokens_seen": 1604463616 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025955867602808427, + "loss": 2.582, + "theoretical_loss": 3.495392513244761, + "tokens_seen": 1604529152 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025954864593781345, + "loss": 2.6793, + "theoretical_loss": 3.4953800879594743, + "tokens_seen": 1604594688 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025953861584754263, + "loss": 2.5043, + "theoretical_loss": 3.495367663323747, + "tokens_seen": 1604660224 + }, + { + "epoch": 5.04, + "learning_rate": 0.0002595285857572718, + "loss": 2.5954, + "theoretical_loss": 3.4953552393375205, + "tokens_seen": 1604725760 + }, + { + "epoch": 5.04, + "learning_rate": 0.000259518555667001, + "loss": 2.6704, + "theoretical_loss": 3.495342816000733, + "tokens_seen": 1604791296 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025950852557673023, + "loss": 2.5959, + "theoretical_loss": 3.4953303933133246, + "tokens_seen": 1604856832 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025949849548645936, + "loss": 2.6485, + "theoretical_loss": 3.495317971275235, + "tokens_seen": 1604922368 + }, + { + "epoch": 5.04, + "learning_rate": 0.0002594884653961886, + "loss": 2.6488, + "theoretical_loss": 3.495305549886403, + "tokens_seen": 1604987904 + }, + { + "epoch": 5.04, + "learning_rate": 0.0002594784353059178, + "loss": 2.664, + "theoretical_loss": 3.495293129146769, + "tokens_seen": 1605053440 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025946840521564696, + "loss": 2.5702, + "theoretical_loss": 3.4952807090562725, + "tokens_seen": 1605118976 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025945837512537614, + "loss": 2.8248, + "theoretical_loss": 3.4952682896148524, + "tokens_seen": 1605184512 + }, + { + "epoch": 5.04, + "learning_rate": 0.0002594483450351053, + "loss": 2.5666, + "theoretical_loss": 3.495255870822449, + "tokens_seen": 1605250048 + }, + { + "epoch": 5.04, + "learning_rate": 0.0002594383149448345, + "loss": 2.5176, + "theoretical_loss": 3.4952434526790013, + "tokens_seen": 1605315584 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025942828485456373, + "loss": 2.7384, + "theoretical_loss": 3.495231035184449, + "tokens_seen": 1605381120 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025941825476429286, + "loss": 2.4549, + "theoretical_loss": 3.4952186183387326, + "tokens_seen": 1605446656 + }, + { + "epoch": 5.04, + "learning_rate": 0.0002594082246740221, + "loss": 2.6885, + "theoretical_loss": 3.49520620214179, + "tokens_seen": 1605512192 + }, + { + "epoch": 5.04, + "learning_rate": 0.0002593981945837512, + "loss": 2.7188, + "theoretical_loss": 3.4951937865935623, + "tokens_seen": 1605577728 + }, + { + "epoch": 5.04, + "objective/train/docs_used": 1808698, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.928004503250122, + "objective/train/theoretical_loss": 3.495184475358074, + "objective/train/tokens_used": 1626086880, + "theoretical_loss": 3.495184475358074, + "tokens_seen": 1605626880 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025938816449348046, + "loss": 2.8332, + "theoretical_loss": 3.495181371693989, + "tokens_seen": 1605643264 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025937813440320964, + "loss": 2.6397, + "theoretical_loss": 3.4951689574430085, + "tokens_seen": 1605708800 + }, + { + "epoch": 5.04, + "learning_rate": 0.0002593681043129388, + "loss": 2.628, + "theoretical_loss": 3.4951565438405616, + "tokens_seen": 1605774336 + }, + { + "epoch": 5.04, + "learning_rate": 0.000259358074222668, + "loss": 2.8962, + "theoretical_loss": 3.4951441308865885, + "tokens_seen": 1605839872 + }, + { + "epoch": 5.04, + "learning_rate": 0.0002593480441323972, + "loss": 2.7003, + "theoretical_loss": 3.495131718581027, + "tokens_seen": 1605905408 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025933801404212636, + "loss": 2.779, + "theoretical_loss": 3.4951193069238182, + "tokens_seen": 1605970944 + }, + { + "epoch": 5.04, + "learning_rate": 0.0002593279839518556, + "loss": 2.77, + "theoretical_loss": 3.495106895914901, + "tokens_seen": 1606036480 + }, + { + "epoch": 5.04, + "learning_rate": 0.0002593179538615847, + "loss": 2.6084, + "theoretical_loss": 3.4950944855542163, + "tokens_seen": 1606102016 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025930792377131396, + "loss": 2.7552, + "theoretical_loss": 3.4950820758417023, + "tokens_seen": 1606167552 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025929789368104314, + "loss": 2.5836, + "theoretical_loss": 3.4950696667772996, + "tokens_seen": 1606233088 + }, + { + "epoch": 5.04, + "learning_rate": 0.0002592878635907723, + "loss": 2.63, + "theoretical_loss": 3.495057258360948, + "tokens_seen": 1606298624 + }, + { + "epoch": 5.04, + "learning_rate": 0.0002592778335005015, + "loss": 2.6871, + "theoretical_loss": 3.495044850592586, + "tokens_seen": 1606364160 + }, + { + "epoch": 5.04, + "learning_rate": 0.0002592678034102307, + "loss": 2.5204, + "theoretical_loss": 3.4950324434721547, + "tokens_seen": 1606429696 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025925777331995987, + "loss": 2.8965, + "theoretical_loss": 3.4950200369995934, + "tokens_seen": 1606495232 + }, + { + "epoch": 5.04, + "learning_rate": 0.0002592477432296891, + "loss": 2.6848, + "theoretical_loss": 3.4950076311748415, + "tokens_seen": 1606560768 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025923771313941823, + "loss": 2.5228, + "theoretical_loss": 3.4949952259978394, + "tokens_seen": 1606626304 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025922768304914747, + "loss": 2.6486, + "theoretical_loss": 3.4949828214685263, + "tokens_seen": 1606691840 + }, + { + "epoch": 5.04, + "learning_rate": 0.0002592176529588766, + "loss": 2.7272, + "theoretical_loss": 3.4949704175868423, + "tokens_seen": 1606757376 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025920762286860583, + "loss": 2.7516, + "theoretical_loss": 3.494958014352727, + "tokens_seen": 1606822912 + }, + { + "epoch": 5.04, + "learning_rate": 0.000259197592778335, + "loss": 2.7038, + "theoretical_loss": 3.4949456117661204, + "tokens_seen": 1606888448 + }, + { + "epoch": 5.04, + "learning_rate": 0.0002591875626880642, + "loss": 2.6016, + "theoretical_loss": 3.4949332098269617, + "tokens_seen": 1606953984 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025917753259779337, + "loss": 2.7287, + "theoretical_loss": 3.4949208085351913, + "tokens_seen": 1607019520 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025916750250752255, + "loss": 2.6791, + "theoretical_loss": 3.4949084078907493, + "tokens_seen": 1607085056 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025915747241725173, + "loss": 2.9492, + "theoretical_loss": 3.4948960078935745, + "tokens_seen": 1607150592 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025914744232698097, + "loss": 2.7316, + "theoretical_loss": 3.4948836085436072, + "tokens_seen": 1607216128 + }, + { + "epoch": 5.04, + "objective/train/docs_used": 1809362, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.568077564239502, + "objective/train/theoretical_loss": 3.4948743094558257, + "objective/train/tokens_used": 1627725280, + "theoretical_loss": 3.4948743094558257, + "tokens_seen": 1607265280 + }, + { + "epoch": 5.04, + "learning_rate": 0.0002591374122367101, + "loss": 2.7754, + "theoretical_loss": 3.4948712098407873, + "tokens_seen": 1607281664 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025912738214643933, + "loss": 2.8811, + "theoretical_loss": 3.494858811785055, + "tokens_seen": 1607347200 + }, + { + "epoch": 5.04, + "learning_rate": 0.0002591173520561685, + "loss": 2.8683, + "theoretical_loss": 3.494846414376349, + "tokens_seen": 1607412736 + }, + { + "epoch": 5.04, + "learning_rate": 0.0002591073219658977, + "loss": 2.8006, + "theoretical_loss": 3.494834017614611, + "tokens_seen": 1607478272 + }, + { + "epoch": 5.04, + "learning_rate": 0.0002590972918756269, + "loss": 2.6925, + "theoretical_loss": 3.494821621499779, + "tokens_seen": 1607543808 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025908726178535606, + "loss": 2.5741, + "theoretical_loss": 3.4948092260317942, + "tokens_seen": 1607609344 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025907723169508524, + "loss": 2.6539, + "theoretical_loss": 3.4947968312105955, + "tokens_seen": 1607674880 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025906720160481447, + "loss": 2.5719, + "theoretical_loss": 3.4947844370361234, + "tokens_seen": 1607740416 + }, + { + "epoch": 5.04, + "learning_rate": 0.0002590571715145436, + "loss": 2.818, + "theoretical_loss": 3.4947720435083176, + "tokens_seen": 1607805952 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025904714142427283, + "loss": 2.7345, + "theoretical_loss": 3.494759650627118, + "tokens_seen": 1607871488 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025903711133400196, + "loss": 2.6123, + "theoretical_loss": 3.494747258392465, + "tokens_seen": 1607937024 + }, + { + "epoch": 5.04, + "learning_rate": 0.0002590270812437312, + "loss": 2.5474, + "theoretical_loss": 3.4947348668042975, + "tokens_seen": 1608002560 + }, + { + "epoch": 5.04, + "learning_rate": 0.0002590170511534604, + "loss": 2.691, + "theoretical_loss": 3.494722475862556, + "tokens_seen": 1608068096 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025900702106318956, + "loss": 2.5017, + "theoretical_loss": 3.4947100855671804, + "tokens_seen": 1608133632 + }, + { + "epoch": 5.04, + "learning_rate": 0.0002589969909729188, + "loss": 2.5006, + "theoretical_loss": 3.494697695918111, + "tokens_seen": 1608199168 + }, + { + "epoch": 5.04, + "learning_rate": 0.000258986960882648, + "loss": 2.4911, + "theoretical_loss": 3.4946853069152874, + "tokens_seen": 1608264704 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025897693079237716, + "loss": 2.7563, + "theoretical_loss": 3.494672918558649, + "tokens_seen": 1608330240 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025896690070210634, + "loss": 2.8289, + "theoretical_loss": 3.494660530848137, + "tokens_seen": 1608395776 + }, + { + "epoch": 5.04, + "learning_rate": 0.0002589568706118355, + "loss": 2.7974, + "theoretical_loss": 3.4946481437836905, + "tokens_seen": 1608461312 + }, + { + "epoch": 5.04, + "learning_rate": 0.0002589468405215647, + "loss": 2.6105, + "theoretical_loss": 3.49463575736525, + "tokens_seen": 1608526848 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025893681043129393, + "loss": 2.6397, + "theoretical_loss": 3.4946233715927546, + "tokens_seen": 1608592384 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025892678034102306, + "loss": 2.7687, + "theoretical_loss": 3.494610986466145, + "tokens_seen": 1608657920 + }, + { + "epoch": 5.04, + "learning_rate": 0.0002589167502507523, + "loss": 2.7845, + "theoretical_loss": 3.494598601985362, + "tokens_seen": 1608723456 + }, + { + "epoch": 5.04, + "learning_rate": 0.0002589067201604814, + "loss": 2.6956, + "theoretical_loss": 3.4945862181503435, + "tokens_seen": 1608788992 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025889669007021066, + "loss": 2.8939, + "theoretical_loss": 3.4945738349610314, + "tokens_seen": 1608854528 + }, + { + "epoch": 5.04, + "objective/train/docs_used": 1810830, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7473294734954834, + "objective/train/theoretical_loss": 3.4945645479927556, + "objective/train/tokens_used": 1629363680, + "theoretical_loss": 3.4945645479927556, + "tokens_seen": 1608903680 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025888665997993984, + "loss": 2.6125, + "theoretical_loss": 3.494561452417365, + "tokens_seen": 1608920064 + }, + { + "epoch": 5.04, + "learning_rate": 0.000258876629889669, + "loss": 2.8013, + "theoretical_loss": 3.4945490705192843, + "tokens_seen": 1608985600 + }, + { + "epoch": 5.04, + "learning_rate": 0.0002588665997993982, + "loss": 2.5571, + "theoretical_loss": 3.4945366892667296, + "tokens_seen": 1609051136 + }, + { + "epoch": 5.04, + "learning_rate": 0.0002588565697091274, + "loss": 2.6408, + "theoretical_loss": 3.4945243086596407, + "tokens_seen": 1609116672 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025884653961885657, + "loss": 2.5487, + "theoretical_loss": 3.494511928697958, + "tokens_seen": 1609182208 + }, + { + "epoch": 5.04, + "learning_rate": 0.0002588365095285858, + "loss": 2.8844, + "theoretical_loss": 3.4944995493816213, + "tokens_seen": 1609247744 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025882647943831493, + "loss": 2.5929, + "theoretical_loss": 3.4944871707105705, + "tokens_seen": 1609313280 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025881644934804416, + "loss": 2.7201, + "theoretical_loss": 3.4944747926847457, + "tokens_seen": 1609378816 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025880641925777334, + "loss": 2.591, + "theoretical_loss": 3.4944624153040875, + "tokens_seen": 1609444352 + }, + { + "epoch": 5.04, + "learning_rate": 0.0002587963891675025, + "loss": 2.6937, + "theoretical_loss": 3.494450038568536, + "tokens_seen": 1609509888 + }, + { + "epoch": 5.04, + "learning_rate": 0.0002587863590772317, + "loss": 2.7347, + "theoretical_loss": 3.4944376624780307, + "tokens_seen": 1609575424 + }, + { + "epoch": 5.04, + "learning_rate": 0.0002587763289869609, + "loss": 2.5359, + "theoretical_loss": 3.494425287032512, + "tokens_seen": 1609640960 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025876629889669007, + "loss": 2.771, + "theoretical_loss": 3.4944129122319207, + "tokens_seen": 1609706496 + }, + { + "epoch": 5.04, + "learning_rate": 0.0002587562688064193, + "loss": 2.6761, + "theoretical_loss": 3.4944005380761958, + "tokens_seen": 1609772032 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025874623871614843, + "loss": 2.6478, + "theoretical_loss": 3.4943881645652777, + "tokens_seen": 1609837568 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025873620862587767, + "loss": 2.5769, + "theoretical_loss": 3.494375791699107, + "tokens_seen": 1609903104 + }, + { + "epoch": 5.04, + "learning_rate": 0.0002587261785356068, + "loss": 2.5977, + "theoretical_loss": 3.494363419477624, + "tokens_seen": 1609968640 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025871614844533603, + "loss": 2.528, + "theoretical_loss": 3.4943510479007687, + "tokens_seen": 1610034176 + }, + { + "epoch": 5.04, + "learning_rate": 0.0002587061183550652, + "loss": 2.6623, + "theoretical_loss": 3.4943386769684803, + "tokens_seen": 1610099712 + }, + { + "epoch": 5.04, + "learning_rate": 0.0002586960882647944, + "loss": 2.8623, + "theoretical_loss": 3.4943263066807004, + "tokens_seen": 1610165248 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025868605817452357, + "loss": 2.5631, + "theoretical_loss": 3.4943139370373686, + "tokens_seen": 1610230784 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025867602808425275, + "loss": 2.7814, + "theoretical_loss": 3.494301568038425, + "tokens_seen": 1610296320 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025866599799398193, + "loss": 2.2406, + "theoretical_loss": 3.4942891996838097, + "tokens_seen": 1610361856 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025865596790371117, + "loss": 2.6263, + "theoretical_loss": 3.494276831973463, + "tokens_seen": 1610427392 + }, + { + "epoch": 5.04, + "learning_rate": 0.0002586459378134403, + "loss": 2.7764, + "theoretical_loss": 3.494264464907326, + "tokens_seen": 1610492928 + }, + { + "epoch": 5.04, + "objective/train/docs_used": 1811590, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.983952760696411, + "objective/train/theoretical_loss": 3.494255190030449, + "objective/train/tokens_used": 1631002080, + "theoretical_loss": 3.494255190030449, + "tokens_seen": 1610542080 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025863590772316953, + "loss": 2.7173, + "theoretical_loss": 3.4942520984853376, + "tokens_seen": 1610558464 + }, + { + "epoch": 5.04, + "learning_rate": 0.0002586258776328987, + "loss": 2.7323, + "theoretical_loss": 3.494239732707438, + "tokens_seen": 1610624000 + }, + { + "epoch": 5.04, + "learning_rate": 0.0002586158475426279, + "loss": 2.8016, + "theoretical_loss": 3.4942273675735693, + "tokens_seen": 1610689536 + }, + { + "epoch": 5.04, + "learning_rate": 0.0002586058174523571, + "loss": 2.7218, + "theoretical_loss": 3.49421500308367, + "tokens_seen": 1610755072 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025859578736208626, + "loss": 2.6106, + "theoretical_loss": 3.4942026392376806, + "tokens_seen": 1610820608 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025858575727181544, + "loss": 2.6007, + "theoretical_loss": 3.4941902760355417, + "tokens_seen": 1610886144 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025857572718154467, + "loss": 2.7377, + "theoretical_loss": 3.494177913477194, + "tokens_seen": 1610951680 + }, + { + "epoch": 5.04, + "learning_rate": 0.0002585656970912738, + "loss": 2.5021, + "theoretical_loss": 3.4941655515625767, + "tokens_seen": 1611017216 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025855566700100303, + "loss": 2.7546, + "theoretical_loss": 3.494153190291631, + "tokens_seen": 1611082752 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025854563691073216, + "loss": 2.6762, + "theoretical_loss": 3.494140829664297, + "tokens_seen": 1611148288 + }, + { + "epoch": 5.04, + "learning_rate": 0.0002585356068204614, + "loss": 2.6792, + "theoretical_loss": 3.494128469680515, + "tokens_seen": 1611213824 + }, + { + "epoch": 5.04, + "learning_rate": 0.0002585255767301906, + "loss": 2.6197, + "theoretical_loss": 3.494116110340225, + "tokens_seen": 1611279360 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025851554663991976, + "loss": 2.4191, + "theoretical_loss": 3.4941037516433675, + "tokens_seen": 1611344896 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025850551654964894, + "loss": 2.7003, + "theoretical_loss": 3.4940913935898834, + "tokens_seen": 1611410432 + }, + { + "epoch": 5.04, + "learning_rate": 0.0002584954864593782, + "loss": 2.8108, + "theoretical_loss": 3.494079036179712, + "tokens_seen": 1611475968 + }, + { + "epoch": 5.04, + "learning_rate": 0.0002584854563691073, + "loss": 2.676, + "theoretical_loss": 3.4940666794127946, + "tokens_seen": 1611541504 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025847542627883654, + "loss": 2.7139, + "theoretical_loss": 3.494054323289071, + "tokens_seen": 1611607040 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025846539618856566, + "loss": 2.7514, + "theoretical_loss": 3.4940419678084815, + "tokens_seen": 1611672576 + }, + { + "epoch": 5.04, + "learning_rate": 0.0002584553660982949, + "loss": 2.6642, + "theoretical_loss": 3.4940296129709667, + "tokens_seen": 1611738112 + }, + { + "epoch": 5.04, + "learning_rate": 0.0002584453360080241, + "loss": 2.6856, + "theoretical_loss": 3.4940172587764673, + "tokens_seen": 1611803648 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025843530591775326, + "loss": 2.4908, + "theoretical_loss": 3.4940049052249234, + "tokens_seen": 1611869184 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025842527582748244, + "loss": 2.5908, + "theoretical_loss": 3.4939925523162754, + "tokens_seen": 1611934720 + }, + { + "epoch": 5.04, + "learning_rate": 0.0002584152457372116, + "loss": 2.4254, + "theoretical_loss": 3.4939802000504634, + "tokens_seen": 1612000256 + }, + { + "epoch": 5.04, + "learning_rate": 0.0002584052156469408, + "loss": 2.5605, + "theoretical_loss": 3.4939678484274284, + "tokens_seen": 1612065792 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025839518555667004, + "loss": 2.5772, + "theoretical_loss": 3.49395549744711, + "tokens_seen": 1612131328 + }, + { + "epoch": 5.04, + "objective/train/docs_used": 1813183, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.669978380203247, + "objective/train/theoretical_loss": 3.4939462346336185, + "objective/train/tokens_used": 1632640480, + "theoretical_loss": 3.4939462346336185, + "tokens_seen": 1612180480 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025838515546639917, + "loss": 2.6425, + "theoretical_loss": 3.4939431471094498, + "tokens_seen": 1612196864 + }, + { + "epoch": 5.04, + "learning_rate": 0.0002583751253761284, + "loss": 2.664, + "theoretical_loss": 3.4939307974143867, + "tokens_seen": 1612262400 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025836509528585753, + "loss": 2.7776, + "theoretical_loss": 3.493918448361863, + "tokens_seen": 1612327936 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025835506519558677, + "loss": 2.6187, + "theoretical_loss": 3.4939060999518174, + "tokens_seen": 1612393472 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025834503510531595, + "loss": 2.6924, + "theoretical_loss": 3.493893752184192, + "tokens_seen": 1612459008 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025833500501504513, + "loss": 2.572, + "theoretical_loss": 3.493881405058926, + "tokens_seen": 1612524544 + }, + { + "epoch": 5.04, + "learning_rate": 0.0002583249749247743, + "loss": 2.5499, + "theoretical_loss": 3.49386905857596, + "tokens_seen": 1612590080 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025831494483450354, + "loss": 2.7041, + "theoretical_loss": 3.4938567127352353, + "tokens_seen": 1612655616 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025830491474423267, + "loss": 2.5322, + "theoretical_loss": 3.4938443675366915, + "tokens_seen": 1612721152 + }, + { + "epoch": 5.04, + "learning_rate": 0.0002582948846539619, + "loss": 2.6045, + "theoretical_loss": 3.4938320229802695, + "tokens_seen": 1612786688 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025828485456369103, + "loss": 2.9335, + "theoretical_loss": 3.49381967906591, + "tokens_seen": 1612852224 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025827482447342027, + "loss": 2.556, + "theoretical_loss": 3.4938073357935533, + "tokens_seen": 1612917760 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025826479438314945, + "loss": 2.5735, + "theoretical_loss": 3.4937949931631396, + "tokens_seen": 1612983296 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025825476429287863, + "loss": 2.7174, + "theoretical_loss": 3.49378265117461, + "tokens_seen": 1613048832 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025824473420260787, + "loss": 2.7566, + "theoretical_loss": 3.493770309827905, + "tokens_seen": 1613114368 + }, + { + "epoch": 5.04, + "learning_rate": 0.000258234704112337, + "loss": 2.668, + "theoretical_loss": 3.493757969122965, + "tokens_seen": 1613179904 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025822467402206623, + "loss": 2.6568, + "theoretical_loss": 3.49374562905973, + "tokens_seen": 1613245440 + }, + { + "epoch": 5.04, + "learning_rate": 0.0002582146439317954, + "loss": 2.5037, + "theoretical_loss": 3.4937332896381417, + "tokens_seen": 1613310976 + }, + { + "epoch": 5.04, + "learning_rate": 0.0002582046138415246, + "loss": 2.7663, + "theoretical_loss": 3.4937209508581395, + "tokens_seen": 1613376512 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025819458375125377, + "loss": 2.7212, + "theoretical_loss": 3.493708612719665, + "tokens_seen": 1613442048 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025818455366098295, + "loss": 2.6807, + "theoretical_loss": 3.493696275222658, + "tokens_seen": 1613507584 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025817452357071213, + "loss": 2.5853, + "theoretical_loss": 3.4936839383670595, + "tokens_seen": 1613573120 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025816449348044137, + "loss": 2.5835, + "theoretical_loss": 3.4936716021528103, + "tokens_seen": 1613638656 + }, + { + "epoch": 5.04, + "learning_rate": 0.0002581544633901705, + "loss": 2.6914, + "theoretical_loss": 3.4936592665798507, + "tokens_seen": 1613704192 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025814443329989973, + "loss": 2.6472, + "theoretical_loss": 3.493646931648121, + "tokens_seen": 1613769728 + }, + { + "epoch": 5.04, + "objective/train/docs_used": 1814061, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6768484115600586, + "objective/train/theoretical_loss": 3.493637680870095, + "objective/train/tokens_used": 1634278880, + "theoretical_loss": 3.493637680870095, + "tokens_seen": 1613818880 + }, + { + "epoch": 5.04, + "learning_rate": 0.0002581344032096289, + "loss": 2.5848, + "theoretical_loss": 3.493634597357562, + "tokens_seen": 1613835264 + }, + { + "epoch": 5.04, + "learning_rate": 0.0002581243731193581, + "loss": 2.6934, + "theoretical_loss": 3.493622263708115, + "tokens_seen": 1613900800 + }, + { + "epoch": 5.04, + "learning_rate": 0.0002581143430290873, + "loss": 2.5939, + "theoretical_loss": 3.49360993069972, + "tokens_seen": 1613966336 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025810431293881646, + "loss": 2.3703, + "theoretical_loss": 3.493597598332318, + "tokens_seen": 1614031872 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025809428284854564, + "loss": 2.7793, + "theoretical_loss": 3.49358526660585, + "tokens_seen": 1614097408 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025808425275827487, + "loss": 2.7513, + "theoretical_loss": 3.4935729355202554, + "tokens_seen": 1614162944 + }, + { + "epoch": 5.04, + "learning_rate": 0.000258074222668004, + "loss": 2.7046, + "theoretical_loss": 3.4935606050754755, + "tokens_seen": 1614228480 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025806419257773323, + "loss": 2.6072, + "theoretical_loss": 3.4935482752714515, + "tokens_seen": 1614294016 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025805416248746236, + "loss": 2.6282, + "theoretical_loss": 3.4935359461081235, + "tokens_seen": 1614359552 + }, + { + "epoch": 5.04, + "learning_rate": 0.0002580441323971916, + "loss": 2.5873, + "theoretical_loss": 3.4935236175854323, + "tokens_seen": 1614425088 + }, + { + "epoch": 5.04, + "learning_rate": 0.0002580341023069208, + "loss": 2.7102, + "theoretical_loss": 3.493511289703319, + "tokens_seen": 1614490624 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025802407221664996, + "loss": 2.6728, + "theoretical_loss": 3.4934989624617243, + "tokens_seen": 1614556160 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025801404212637914, + "loss": 2.7576, + "theoretical_loss": 3.493486635860588, + "tokens_seen": 1614621696 + }, + { + "epoch": 5.04, + "learning_rate": 0.0002580040120361084, + "loss": 2.7063, + "theoretical_loss": 3.493474309899852, + "tokens_seen": 1614687232 + }, + { + "epoch": 5.04, + "learning_rate": 0.0002579939819458375, + "loss": 2.6767, + "theoretical_loss": 3.493461984579456, + "tokens_seen": 1614752768 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025798395185556674, + "loss": 2.5628, + "theoretical_loss": 3.493449659899342, + "tokens_seen": 1614818304 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025797392176529586, + "loss": 2.701, + "theoretical_loss": 3.4934373358594497, + "tokens_seen": 1614883840 + }, + { + "epoch": 5.04, + "learning_rate": 0.0002579638916750251, + "loss": 2.6278, + "theoretical_loss": 3.4934250124597197, + "tokens_seen": 1614949376 + }, + { + "epoch": 5.04, + "learning_rate": 0.0002579538615847543, + "loss": 2.565, + "theoretical_loss": 3.4934126897000937, + "tokens_seen": 1615014912 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025794383149448346, + "loss": 2.6736, + "theoretical_loss": 3.493400367580512, + "tokens_seen": 1615080448 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025793380140421264, + "loss": 2.7948, + "theoretical_loss": 3.4933880461009155, + "tokens_seen": 1615145984 + }, + { + "epoch": 5.04, + "learning_rate": 0.0002579237713139418, + "loss": 2.6424, + "theoretical_loss": 3.493375725261245, + "tokens_seen": 1615211520 + }, + { + "epoch": 5.04, + "learning_rate": 0.000257913741223671, + "loss": 2.597, + "theoretical_loss": 3.4933634050614413, + "tokens_seen": 1615277056 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025790371113340024, + "loss": 2.5277, + "theoretical_loss": 3.4933510855014447, + "tokens_seen": 1615342592 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025789368104312937, + "loss": 2.7266, + "theoretical_loss": 3.493338766581197, + "tokens_seen": 1615408128 + }, + { + "epoch": 5.04, + "objective/train/docs_used": 1814677, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.616262197494507, + "objective/train/theoretical_loss": 3.4933295278108103, + "objective/train/tokens_used": 1635917280, + "theoretical_loss": 3.4933295278108103, + "tokens_seen": 1615457280 + }, + { + "epoch": 5.04, + "learning_rate": 0.0002578836509528586, + "loss": 2.6068, + "theoretical_loss": 3.493326448300638, + "tokens_seen": 1615473664 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025787362086258773, + "loss": 2.5708, + "theoretical_loss": 3.493314130659709, + "tokens_seen": 1615539200 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025786359077231697, + "loss": 2.6691, + "theoretical_loss": 3.493301813658351, + "tokens_seen": 1615604736 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025785356068204615, + "loss": 2.6395, + "theoretical_loss": 3.493289497296505, + "tokens_seen": 1615670272 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025784353059177533, + "loss": 2.6006, + "theoretical_loss": 3.4932771815741113, + "tokens_seen": 1615735808 + }, + { + "epoch": 5.04, + "learning_rate": 0.0002578335005015045, + "loss": 2.5032, + "theoretical_loss": 3.493264866491111, + "tokens_seen": 1615801344 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025782347041123374, + "loss": 2.67, + "theoretical_loss": 3.493252552047445, + "tokens_seen": 1615866880 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025781344032096287, + "loss": 2.7613, + "theoretical_loss": 3.493240238243054, + "tokens_seen": 1615932416 + }, + { + "epoch": 5.04, + "learning_rate": 0.0002578034102306921, + "loss": 2.6414, + "theoretical_loss": 3.4932279250778793, + "tokens_seen": 1615997952 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025779338014042123, + "loss": 2.7801, + "theoretical_loss": 3.493215612551862, + "tokens_seen": 1616063488 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025778335005015047, + "loss": 2.6191, + "theoretical_loss": 3.4932033006649417, + "tokens_seen": 1616129024 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025777331995987965, + "loss": 2.4602, + "theoretical_loss": 3.4931909894170605, + "tokens_seen": 1616194560 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025776328986960883, + "loss": 2.7383, + "theoretical_loss": 3.4931786788081594, + "tokens_seen": 1616260096 + }, + { + "epoch": 5.04, + "learning_rate": 0.000257753259779338, + "loss": 2.7904, + "theoretical_loss": 3.493166368838178, + "tokens_seen": 1616325632 + }, + { + "epoch": 5.04, + "learning_rate": 0.0002577432296890672, + "loss": 2.7796, + "theoretical_loss": 3.4931540595070594, + "tokens_seen": 1616391168 + }, + { + "epoch": 5.04, + "learning_rate": 0.0002577331995987964, + "loss": 2.615, + "theoretical_loss": 3.4931417508147424, + "tokens_seen": 1616456704 + }, + { + "epoch": 5.04, + "learning_rate": 0.0002577231695085256, + "loss": 2.3553, + "theoretical_loss": 3.4931294427611697, + "tokens_seen": 1616522240 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025771313941825474, + "loss": 2.7692, + "theoretical_loss": 3.4931171353462807, + "tokens_seen": 1616587776 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025770310932798397, + "loss": 2.5657, + "theoretical_loss": 3.4931048285700177, + "tokens_seen": 1616653312 + }, + { + "epoch": 5.04, + "learning_rate": 0.0002576930792377131, + "loss": 2.8594, + "theoretical_loss": 3.4930925224323204, + "tokens_seen": 1616718848 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025768304914744233, + "loss": 2.7142, + "theoretical_loss": 3.493080216933131, + "tokens_seen": 1616784384 + }, + { + "epoch": 5.04, + "learning_rate": 0.0002576730190571715, + "loss": 2.5003, + "theoretical_loss": 3.493067912072389, + "tokens_seen": 1616849920 + }, + { + "epoch": 5.04, + "learning_rate": 0.0002576629889669007, + "loss": 2.7273, + "theoretical_loss": 3.493055607850037, + "tokens_seen": 1616915456 + }, + { + "epoch": 5.04, + "learning_rate": 0.0002576529588766299, + "loss": 2.7189, + "theoretical_loss": 3.4930433042660156, + "tokens_seen": 1616980992 + }, + { + "epoch": 5.04, + "learning_rate": 0.0002576429287863591, + "loss": 2.5459, + "theoretical_loss": 3.4930310013202654, + "tokens_seen": 1617046528 + }, + { + "epoch": 5.04, + "objective/train/docs_used": 1816051, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6610658168792725, + "objective/train/theoretical_loss": 3.493021774529783, + "objective/train/tokens_used": 1637555680, + "theoretical_loss": 3.493021774529783, + "tokens_seen": 1617095680 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025763289869608824, + "loss": 2.6226, + "theoretical_loss": 3.493018699012728, + "tokens_seen": 1617112064 + }, + { + "epoch": 5.04, + "learning_rate": 0.0002576228686058175, + "loss": 2.4937, + "theoretical_loss": 3.493006397343343, + "tokens_seen": 1617177600 + }, + { + "epoch": 5.04, + "learning_rate": 0.0002576128385155466, + "loss": 2.5907, + "theoretical_loss": 3.492994096312053, + "tokens_seen": 1617243136 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025760280842527584, + "loss": 2.4163, + "theoretical_loss": 3.4929817959187988, + "tokens_seen": 1617308672 + }, + { + "epoch": 5.04, + "learning_rate": 0.000257592778335005, + "loss": 2.4671, + "theoretical_loss": 3.4929694961635205, + "tokens_seen": 1617374208 + }, + { + "epoch": 5.04, + "learning_rate": 0.0002575827482447342, + "loss": 2.5927, + "theoretical_loss": 3.4929571970461604, + "tokens_seen": 1617439744 + }, + { + "epoch": 5.04, + "learning_rate": 0.0002575727181544634, + "loss": 2.5462, + "theoretical_loss": 3.492944898566659, + "tokens_seen": 1617505280 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025756268806419256, + "loss": 2.4217, + "theoretical_loss": 3.4929326007249566, + "tokens_seen": 1617570816 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025755265797392174, + "loss": 2.6032, + "theoretical_loss": 3.4929203035209957, + "tokens_seen": 1617636352 + }, + { + "epoch": 5.04, + "learning_rate": 0.000257542627883651, + "loss": 2.7347, + "theoretical_loss": 3.4929080069547167, + "tokens_seen": 1617701888 + }, + { + "epoch": 5.04, + "learning_rate": 0.0002575325977933801, + "loss": 2.5122, + "theoretical_loss": 3.4928957110260606, + "tokens_seen": 1617767424 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025752256770310934, + "loss": 2.7135, + "theoretical_loss": 3.4928834157349686, + "tokens_seen": 1617832960 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025751253761283847, + "loss": 2.6551, + "theoretical_loss": 3.4928711210813823, + "tokens_seen": 1617898496 + }, + { + "epoch": 5.04, + "learning_rate": 0.0002575025075225677, + "loss": 2.9374, + "theoretical_loss": 3.4928588270652416, + "tokens_seen": 1617964032 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025749247743229694, + "loss": 2.5136, + "theoretical_loss": 3.492846533686489, + "tokens_seen": 1618029568 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025748244734202606, + "loss": 2.5835, + "theoretical_loss": 3.492834240945065, + "tokens_seen": 1618095104 + }, + { + "epoch": 5.04, + "learning_rate": 0.0002574724172517553, + "loss": 2.5203, + "theoretical_loss": 3.492821948840911, + "tokens_seen": 1618160640 + }, + { + "epoch": 5.04, + "learning_rate": 0.0002574623871614845, + "loss": 2.6176, + "theoretical_loss": 3.4928096573739675, + "tokens_seen": 1618226176 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025745235707121366, + "loss": 2.6967, + "theoretical_loss": 3.4927973665441763, + "tokens_seen": 1618291712 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025744232698094284, + "loss": 2.3947, + "theoretical_loss": 3.4927850763514785, + "tokens_seen": 1618357248 + }, + { + "epoch": 5.04, + "learning_rate": 0.000257432296890672, + "loss": 2.8494, + "theoretical_loss": 3.4927727867958147, + "tokens_seen": 1618422784 + }, + { + "epoch": 5.04, + "learning_rate": 0.0002574222668004012, + "loss": 2.5314, + "theoretical_loss": 3.4927604978771276, + "tokens_seen": 1618488320 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025741223671013044, + "loss": 2.4884, + "theoretical_loss": 3.492748209595357, + "tokens_seen": 1618553856 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025740220661985957, + "loss": 2.6668, + "theoretical_loss": 3.4927359219504437, + "tokens_seen": 1618619392 + }, + { + "epoch": 5.04, + "learning_rate": 0.0002573921765295888, + "loss": 2.5161, + "theoretical_loss": 3.4927236349423305, + "tokens_seen": 1618684928 + }, + { + "epoch": 5.04, + "objective/train/docs_used": 1816712, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6617588996887207, + "objective/train/theoretical_loss": 3.492714420104109, + "objective/train/tokens_used": 1639194080, + "theoretical_loss": 3.492714420104109, + "tokens_seen": 1618734080 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025738214643931793, + "loss": 2.4688, + "theoretical_loss": 3.492711348570957, + "tokens_seen": 1618750464 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025737211634904717, + "loss": 2.6443, + "theoretical_loss": 3.492699062836266, + "tokens_seen": 1618816000 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025736208625877635, + "loss": 2.8617, + "theoretical_loss": 3.4926867777381974, + "tokens_seen": 1618881536 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025735205616850553, + "loss": 2.5455, + "theoretical_loss": 3.4926744932766933, + "tokens_seen": 1618947072 + }, + { + "epoch": 5.04, + "learning_rate": 0.0002573420260782347, + "loss": 2.6395, + "theoretical_loss": 3.4926622094516944, + "tokens_seen": 1619012608 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025733199598796394, + "loss": 2.5923, + "theoretical_loss": 3.4926499262631427, + "tokens_seen": 1619078144 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025732196589769307, + "loss": 2.6676, + "theoretical_loss": 3.4926376437109785, + "tokens_seen": 1619143680 + }, + { + "epoch": 5.04, + "learning_rate": 0.0002573119358074223, + "loss": 2.7692, + "theoretical_loss": 3.492625361795144, + "tokens_seen": 1619209216 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025730190571715143, + "loss": 2.8743, + "theoretical_loss": 3.4926130805155795, + "tokens_seen": 1619274752 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025729187562688067, + "loss": 2.8537, + "theoretical_loss": 3.492600799872227, + "tokens_seen": 1619340288 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025728184553660985, + "loss": 2.6658, + "theoretical_loss": 3.492588519865028, + "tokens_seen": 1619405824 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025727181544633903, + "loss": 2.6737, + "theoretical_loss": 3.4925762404939227, + "tokens_seen": 1619471360 + }, + { + "epoch": 5.04, + "learning_rate": 0.0002572617853560682, + "loss": 2.8957, + "theoretical_loss": 3.4925639617588535, + "tokens_seen": 1619536896 + }, + { + "epoch": 5.04, + "learning_rate": 0.0002572517552657974, + "loss": 2.6378, + "theoretical_loss": 3.4925516836597614, + "tokens_seen": 1619602432 + }, + { + "epoch": 5.04, + "learning_rate": 0.0002572417251755266, + "loss": 2.6016, + "theoretical_loss": 3.4925394061965873, + "tokens_seen": 1619667968 + }, + { + "epoch": 5.04, + "learning_rate": 0.0002572316950852558, + "loss": 2.5372, + "theoretical_loss": 3.4925271293692735, + "tokens_seen": 1619733504 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025722166499498494, + "loss": 2.428, + "theoretical_loss": 3.4925148531777603, + "tokens_seen": 1619799040 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025721163490471417, + "loss": 2.5885, + "theoretical_loss": 3.4925025776219893, + "tokens_seen": 1619864576 + }, + { + "epoch": 5.04, + "learning_rate": 0.0002572016048144433, + "loss": 2.6786, + "theoretical_loss": 3.4924903027019027, + "tokens_seen": 1619930112 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025719157472417253, + "loss": 2.5506, + "theoretical_loss": 3.492478028417441, + "tokens_seen": 1619995648 + }, + { + "epoch": 5.04, + "learning_rate": 0.0002571815446339017, + "loss": 2.4361, + "theoretical_loss": 3.4924657547685456, + "tokens_seen": 1620061184 + }, + { + "epoch": 5.04, + "learning_rate": 0.0002571715145436309, + "loss": 2.6449, + "theoretical_loss": 3.492453481755158, + "tokens_seen": 1620126720 + }, + { + "epoch": 5.04, + "learning_rate": 0.0002571614844533601, + "loss": 2.548, + "theoretical_loss": 3.49244120937722, + "tokens_seen": 1620192256 + }, + { + "epoch": 5.04, + "learning_rate": 0.0002571514543630893, + "loss": 2.6244, + "theoretical_loss": 3.4924289376346724, + "tokens_seen": 1620257792 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025714142427281844, + "loss": 2.7454, + "theoretical_loss": 3.4924166665274567, + "tokens_seen": 1620323328 + }, + { + "epoch": 5.04, + "objective/train/docs_used": 1817999, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.959540843963623, + "objective/train/theoretical_loss": 3.4924074636139464, + "objective/train/tokens_used": 1640832480, + "theoretical_loss": 3.4924074636139464, + "tokens_seen": 1620372480 + }, + { + "epoch": 5.04, + "learning_rate": 0.0002571313941825477, + "loss": 2.6683, + "theoretical_loss": 3.4924043960555147, + "tokens_seen": 1620388864 + }, + { + "epoch": 5.04, + "learning_rate": 0.0002571213640922768, + "loss": 2.671, + "theoretical_loss": 3.4923921262187876, + "tokens_seen": 1620454400 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025711133400200604, + "loss": 2.8518, + "theoretical_loss": 3.492379857017217, + "tokens_seen": 1620519936 + }, + { + "epoch": 5.04, + "learning_rate": 0.0002571013039117352, + "loss": 2.3591, + "theoretical_loss": 3.492367588450744, + "tokens_seen": 1620585472 + }, + { + "epoch": 5.04, + "learning_rate": 0.0002570912738214644, + "loss": 2.4134, + "theoretical_loss": 3.49235532051931, + "tokens_seen": 1620651008 + }, + { + "epoch": 5.04, + "learning_rate": 0.0002570812437311936, + "loss": 2.6232, + "theoretical_loss": 3.492343053222857, + "tokens_seen": 1620716544 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025707121364092276, + "loss": 2.6938, + "theoretical_loss": 3.4923307865613262, + "tokens_seen": 1620782080 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025706118355065194, + "loss": 2.4861, + "theoretical_loss": 3.492318520534658, + "tokens_seen": 1620847616 + }, + { + "epoch": 5.04, + "learning_rate": 0.0002570511534603812, + "loss": 2.7885, + "theoretical_loss": 3.492306255142796, + "tokens_seen": 1620913152 + }, + { + "epoch": 5.04, + "learning_rate": 0.0002570411233701103, + "loss": 2.6434, + "theoretical_loss": 3.4922939903856802, + "tokens_seen": 1620978688 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025703109327983954, + "loss": 2.5004, + "theoretical_loss": 3.4922817262632524, + "tokens_seen": 1621044224 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025702106318956867, + "loss": 2.7335, + "theoretical_loss": 3.4922694627754542, + "tokens_seen": 1621109760 + }, + { + "epoch": 5.04, + "learning_rate": 0.0002570110330992979, + "loss": 2.734, + "theoretical_loss": 3.4922571999222267, + "tokens_seen": 1621175296 + }, + { + "epoch": 5.04, + "learning_rate": 0.0002570010030090271, + "loss": 2.7062, + "theoretical_loss": 3.4922449377035125, + "tokens_seen": 1621240832 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025699097291875626, + "loss": 2.5759, + "theoretical_loss": 3.4922326761192517, + "tokens_seen": 1621306368 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025698094282848545, + "loss": 2.4875, + "theoretical_loss": 3.4922204151693865, + "tokens_seen": 1621371904 + }, + { + "epoch": 5.04, + "learning_rate": 0.0002569709127382147, + "loss": 2.8393, + "theoretical_loss": 3.492208154853859, + "tokens_seen": 1621437440 + }, + { + "epoch": 5.04, + "learning_rate": 0.0002569608826479438, + "loss": 2.6645, + "theoretical_loss": 3.4921958951726095, + "tokens_seen": 1621502976 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025695085255767304, + "loss": 2.8177, + "theoretical_loss": 3.4921836361255805, + "tokens_seen": 1621568512 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025694082246740217, + "loss": 2.7412, + "theoretical_loss": 3.4921713777127135, + "tokens_seen": 1621634048 + }, + { + "epoch": 5.04, + "learning_rate": 0.0002569307923771314, + "loss": 2.534, + "theoretical_loss": 3.4921591199339495, + "tokens_seen": 1621699584 + }, + { + "epoch": 5.04, + "learning_rate": 0.0002569207622868606, + "loss": 2.5425, + "theoretical_loss": 3.4921468627892307, + "tokens_seen": 1621765120 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025691073219658977, + "loss": 2.6161, + "theoretical_loss": 3.4921346062784986, + "tokens_seen": 1621830656 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025690070210631895, + "loss": 2.3787, + "theoretical_loss": 3.4921223504016945, + "tokens_seen": 1621896192 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025689067201604813, + "loss": 2.6759, + "theoretical_loss": 3.49211009515876, + "tokens_seen": 1621961728 + }, + { + "epoch": 5.04, + "objective/train/docs_used": 1818775, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.1677393913269043, + "objective/train/theoretical_loss": 3.492100904142501, + "objective/train/tokens_used": 1642470880, + "theoretical_loss": 3.492100904142501, + "tokens_seen": 1622010880 + }, + { + "epoch": 5.04, + "learning_rate": 0.0002568806419257773, + "loss": 2.469, + "theoretical_loss": 3.4920978405496363, + "tokens_seen": 1622027264 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025687061183550655, + "loss": 2.6083, + "theoretical_loss": 3.4920855865742664, + "tokens_seen": 1622092800 + }, + { + "epoch": 5.04, + "learning_rate": 0.0002568605817452357, + "loss": 2.5079, + "theoretical_loss": 3.4920733332325904, + "tokens_seen": 1622158336 + }, + { + "epoch": 5.04, + "learning_rate": 0.0002568505516549649, + "loss": 2.6879, + "theoretical_loss": 3.492061080524551, + "tokens_seen": 1622223872 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025684052156469404, + "loss": 2.5899, + "theoretical_loss": 3.4920488284500895, + "tokens_seen": 1622289408 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025683049147442327, + "loss": 2.4714, + "theoretical_loss": 3.4920365770091473, + "tokens_seen": 1622354944 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025682046138415245, + "loss": 2.527, + "theoretical_loss": 3.4920243262016664, + "tokens_seen": 1622420480 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025681043129388163, + "loss": 2.808, + "theoretical_loss": 3.4920120760275877, + "tokens_seen": 1622486016 + }, + { + "epoch": 5.04, + "learning_rate": 0.0002568004012036108, + "loss": 2.5658, + "theoretical_loss": 3.491999826486854, + "tokens_seen": 1622551552 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025679037111334005, + "loss": 2.596, + "theoretical_loss": 3.4919875775794065, + "tokens_seen": 1622617088 + }, + { + "epoch": 5.04, + "learning_rate": 0.0002567803410230692, + "loss": 2.5577, + "theoretical_loss": 3.4919753293051867, + "tokens_seen": 1622682624 + }, + { + "epoch": 5.04, + "learning_rate": 0.0002567703109327984, + "loss": 2.8376, + "theoretical_loss": 3.491963081664136, + "tokens_seen": 1622748160 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025676028084252754, + "loss": 2.6855, + "theoretical_loss": 3.4919508346561967, + "tokens_seen": 1622813696 + }, + { + "epoch": 5.04, + "learning_rate": 0.0002567502507522568, + "loss": 2.5951, + "theoretical_loss": 3.491938588281311, + "tokens_seen": 1622879232 + }, + { + "epoch": 5.04, + "learning_rate": 0.000256740220661986, + "loss": 2.5642, + "theoretical_loss": 3.491926342539419, + "tokens_seen": 1622944768 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025673019057171514, + "loss": 2.6147, + "theoretical_loss": 3.491914097430464, + "tokens_seen": 1623010304 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025672016048144437, + "loss": 2.5273, + "theoretical_loss": 3.4919018529543866, + "tokens_seen": 1623075840 + }, + { + "epoch": 5.04, + "learning_rate": 0.0002567101303911735, + "loss": 2.6065, + "theoretical_loss": 3.491889609111129, + "tokens_seen": 1623141376 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025670010030090273, + "loss": 2.5336, + "theoretical_loss": 3.491877365900633, + "tokens_seen": 1623206912 + }, + { + "epoch": 5.04, + "learning_rate": 0.0002566900702106319, + "loss": 2.6442, + "theoretical_loss": 3.4918651233228406, + "tokens_seen": 1623272448 + }, + { + "epoch": 5.04, + "learning_rate": 0.0002566800401203611, + "loss": 2.6089, + "theoretical_loss": 3.4918528813776932, + "tokens_seen": 1623337984 + }, + { + "epoch": 5.04, + "learning_rate": 0.0002566700100300903, + "loss": 2.3904, + "theoretical_loss": 3.4918406400651327, + "tokens_seen": 1623403520 + }, + { + "epoch": 5.04, + "learning_rate": 0.0002566599799398195, + "loss": 2.7396, + "theoretical_loss": 3.491828399385101, + "tokens_seen": 1623469056 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025664994984954864, + "loss": 2.7331, + "theoretical_loss": 3.491816159337539, + "tokens_seen": 1623534592 + }, + { + "epoch": 5.04, + "learning_rate": 0.0002566399197592779, + "loss": 2.762, + "theoretical_loss": 3.49180391992239, + "tokens_seen": 1623600128 + }, + { + "epoch": 5.04, + "objective/train/docs_used": 1820300, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7725954055786133, + "objective/train/theoretical_loss": 3.4917947407760135, + "objective/train/tokens_used": 1644109280, + "theoretical_loss": 3.4917947407760135, + "tokens_seen": 1623649280 + }, + { + "epoch": 5.04, + "learning_rate": 0.000256629889669007, + "loss": 2.5771, + "theoretical_loss": 3.4917916811395946, + "tokens_seen": 1623665664 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025661985957873624, + "loss": 2.6068, + "theoretical_loss": 3.491779442989095, + "tokens_seen": 1623731200 + }, + { + "epoch": 5.04, + "learning_rate": 0.0002566098294884654, + "loss": 2.5716, + "theoretical_loss": 3.491767205470833, + "tokens_seen": 1623796736 + }, + { + "epoch": 5.04, + "learning_rate": 0.0002565997993981946, + "loss": 2.5825, + "theoretical_loss": 3.4917549685847504, + "tokens_seen": 1623862272 + }, + { + "epoch": 5.04, + "learning_rate": 0.0002565897693079238, + "loss": 2.7518, + "theoretical_loss": 3.4917427323307892, + "tokens_seen": 1623927808 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025657973921765296, + "loss": 2.507, + "theoretical_loss": 3.4917304967088914, + "tokens_seen": 1623993344 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025656970912738214, + "loss": 2.7368, + "theoretical_loss": 3.491718261718998, + "tokens_seen": 1624058880 + }, + { + "epoch": 5.04, + "learning_rate": 0.0002565596790371114, + "loss": 2.7704, + "theoretical_loss": 3.491706027361052, + "tokens_seen": 1624124416 + }, + { + "epoch": 5.04, + "learning_rate": 0.0002565496489468405, + "loss": 2.5858, + "theoretical_loss": 3.4916937936349943, + "tokens_seen": 1624189952 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025653961885656974, + "loss": 2.8094, + "theoretical_loss": 3.4916815605407674, + "tokens_seen": 1624255488 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025652958876629887, + "loss": 2.7634, + "theoretical_loss": 3.4916693280783124, + "tokens_seen": 1624321024 + }, + { + "epoch": 5.04, + "learning_rate": 0.0002565195586760281, + "loss": 2.5955, + "theoretical_loss": 3.491657096247572, + "tokens_seen": 1624386560 + }, + { + "epoch": 5.04, + "learning_rate": 0.0002565095285857573, + "loss": 2.5747, + "theoretical_loss": 3.491644865048488, + "tokens_seen": 1624452096 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025649949849548647, + "loss": 2.7707, + "theoretical_loss": 3.491632634481002, + "tokens_seen": 1624517632 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025648946840521565, + "loss": 2.6173, + "theoretical_loss": 3.491620404545056, + "tokens_seen": 1624583168 + }, + { + "epoch": 5.04, + "learning_rate": 0.0002564794383149449, + "loss": 2.6951, + "theoretical_loss": 3.4916081752405916, + "tokens_seen": 1624648704 + }, + { + "epoch": 5.04, + "learning_rate": 0.000256469408224674, + "loss": 2.7188, + "theoretical_loss": 3.4915959465675517, + "tokens_seen": 1624714240 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025645937813440324, + "loss": 2.4266, + "theoretical_loss": 3.4915837185258765, + "tokens_seen": 1624779776 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025644934804413237, + "loss": 2.6004, + "theoretical_loss": 3.49157149111551, + "tokens_seen": 1624845312 + }, + { + "epoch": 5.04, + "learning_rate": 0.0002564393179538616, + "loss": 2.5292, + "theoretical_loss": 3.491559264336393, + "tokens_seen": 1624910848 + }, + { + "epoch": 5.04, + "learning_rate": 0.0002564292878635908, + "loss": 2.4791, + "theoretical_loss": 3.4915470381884672, + "tokens_seen": 1624976384 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025641925777331997, + "loss": 2.6726, + "theoretical_loss": 3.4915348126716754, + "tokens_seen": 1625041920 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025640922768304915, + "loss": 2.8152, + "theoretical_loss": 3.4915225877859593, + "tokens_seen": 1625107456 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025639919759277833, + "loss": 2.5362, + "theoretical_loss": 3.49151036353126, + "tokens_seen": 1625172992 + }, + { + "epoch": 5.04, + "learning_rate": 0.0002563891675025075, + "loss": 2.5838, + "theoretical_loss": 3.4914981399075207, + "tokens_seen": 1625238528 + }, + { + "epoch": 5.04, + "objective/train/docs_used": 1820966, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.0666143894195557, + "objective/train/theoretical_loss": 3.491488972603748, + "objective/train/tokens_used": 1645747680, + "theoretical_loss": 3.491488972603748, + "tokens_seen": 1625287680 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025637913741223675, + "loss": 2.6085, + "theoretical_loss": 3.491485916914683, + "tokens_seen": 1625304064 + }, + { + "epoch": 5.04, + "learning_rate": 0.0002563691073219659, + "loss": 2.7686, + "theoretical_loss": 3.4914736945526883, + "tokens_seen": 1625369600 + }, + { + "epoch": 5.04, + "learning_rate": 0.0002563590772316951, + "loss": 2.6789, + "theoretical_loss": 3.491461472821479, + "tokens_seen": 1625435136 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025634904714142424, + "loss": 2.4061, + "theoretical_loss": 3.491449251720998, + "tokens_seen": 1625500672 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025633901705115347, + "loss": 2.522, + "theoretical_loss": 3.491437031251186, + "tokens_seen": 1625566208 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025632898696088265, + "loss": 2.5905, + "theoretical_loss": 3.491424811411986, + "tokens_seen": 1625631744 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025631895687061183, + "loss": 2.5403, + "theoretical_loss": 3.491412592203339, + "tokens_seen": 1625697280 + }, + { + "epoch": 5.04, + "learning_rate": 0.000256308926780341, + "loss": 2.7574, + "theoretical_loss": 3.491400373625188, + "tokens_seen": 1625762816 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025629889669007025, + "loss": 2.646, + "theoretical_loss": 3.491388155677475, + "tokens_seen": 1625828352 + }, + { + "epoch": 5.04, + "learning_rate": 0.0002562888665997994, + "loss": 2.9001, + "theoretical_loss": 3.4913759383601413, + "tokens_seen": 1625893888 + }, + { + "epoch": 5.04, + "learning_rate": 0.0002562788365095286, + "loss": 2.7254, + "theoretical_loss": 3.4913637216731295, + "tokens_seen": 1625959424 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025626880641925774, + "loss": 2.7043, + "theoretical_loss": 3.4913515056163815, + "tokens_seen": 1626024960 + }, + { + "epoch": 5.04, + "learning_rate": 0.000256258776328987, + "loss": 2.6255, + "theoretical_loss": 3.49133929018984, + "tokens_seen": 1626090496 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025624874623871616, + "loss": 2.5757, + "theoretical_loss": 3.4913270753934462, + "tokens_seen": 1626156032 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025623871614844534, + "loss": 2.5782, + "theoretical_loss": 3.491314861227143, + "tokens_seen": 1626221568 + }, + { + "epoch": 5.04, + "learning_rate": 0.0002562286860581745, + "loss": 2.4161, + "theoretical_loss": 3.4913026476908717, + "tokens_seen": 1626287104 + }, + { + "epoch": 5.04, + "learning_rate": 0.0002562186559679037, + "loss": 2.7689, + "theoretical_loss": 3.491290434784575, + "tokens_seen": 1626352640 + }, + { + "epoch": 5.04, + "learning_rate": 0.0002562086258776329, + "loss": 2.7879, + "theoretical_loss": 3.4912782225081944, + "tokens_seen": 1626418176 + }, + { + "epoch": 5.04, + "learning_rate": 0.0002561985957873621, + "loss": 2.5714, + "theoretical_loss": 3.491266010861673, + "tokens_seen": 1626483712 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025618856569709124, + "loss": 2.4168, + "theoretical_loss": 3.4912537998449515, + "tokens_seen": 1626549248 + }, + { + "epoch": 5.04, + "learning_rate": 0.0002561785356068205, + "loss": 2.7801, + "theoretical_loss": 3.491241589457974, + "tokens_seen": 1626614784 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025616850551654966, + "loss": 2.4912, + "theoretical_loss": 3.4912293797006813, + "tokens_seen": 1626680320 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025615847542627884, + "loss": 2.6434, + "theoretical_loss": 3.491217170573016, + "tokens_seen": 1626745856 + }, + { + "epoch": 5.04, + "learning_rate": 0.000256148445336008, + "loss": 2.7688, + "theoretical_loss": 3.49120496207492, + "tokens_seen": 1626811392 + }, + { + "epoch": 5.04, + "learning_rate": 0.0002561384152457372, + "loss": 2.777, + "theoretical_loss": 3.4911927542063355, + "tokens_seen": 1626876928 + }, + { + "epoch": 5.04, + "objective/train/docs_used": 1822305, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.456148147583008, + "objective/train/theoretical_loss": 3.4911835987179787, + "objective/train/tokens_used": 1647386080, + "theoretical_loss": 3.4911835987179787, + "tokens_seen": 1626926080 + }, + { + "epoch": 5.04, + "learning_rate": 0.0002561283851554664, + "loss": 2.8612, + "theoretical_loss": 3.4911805469672044, + "tokens_seen": 1626942464 + }, + { + "epoch": 5.04, + "learning_rate": 0.0002561183550651956, + "loss": 2.6333, + "theoretical_loss": 3.49116834035747, + "tokens_seen": 1627008000 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025610832497492475, + "loss": 2.8309, + "theoretical_loss": 3.491156134377073, + "tokens_seen": 1627073536 + }, + { + "epoch": 5.04, + "learning_rate": 0.000256098294884654, + "loss": 2.5449, + "theoretical_loss": 3.4911439290259567, + "tokens_seen": 1627139072 + }, + { + "epoch": 5.04, + "learning_rate": 0.0002560882647943831, + "loss": 2.7455, + "theoretical_loss": 3.491131724304063, + "tokens_seen": 1627204608 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025607823470411234, + "loss": 2.7514, + "theoretical_loss": 3.4911195202113348, + "tokens_seen": 1627270144 + }, + { + "epoch": 5.04, + "learning_rate": 0.0002560682046138415, + "loss": 2.5495, + "theoretical_loss": 3.491107316747713, + "tokens_seen": 1627335680 + }, + { + "epoch": 5.04, + "learning_rate": 0.0002560581745235707, + "loss": 2.5578, + "theoretical_loss": 3.4910951139131408, + "tokens_seen": 1627401216 + }, + { + "epoch": 5.04, + "learning_rate": 0.0002560481444332999, + "loss": 2.5765, + "theoretical_loss": 3.4910829117075597, + "tokens_seen": 1627466752 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025603811434302907, + "loss": 2.5857, + "theoretical_loss": 3.4910707101309124, + "tokens_seen": 1627532288 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025602808425275825, + "loss": 2.7842, + "theoretical_loss": 3.491058509183141, + "tokens_seen": 1627597824 + }, + { + "epoch": 5.04, + "learning_rate": 0.0002560180541624875, + "loss": 2.6268, + "theoretical_loss": 3.4910463088641883, + "tokens_seen": 1627663360 + }, + { + "epoch": 5.04, + "learning_rate": 0.0002560080240722166, + "loss": 2.7106, + "theoretical_loss": 3.491034109173996, + "tokens_seen": 1627728896 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025599799398194585, + "loss": 2.7822, + "theoretical_loss": 3.491021910112507, + "tokens_seen": 1627794432 + }, + { + "epoch": 5.04, + "learning_rate": 0.0002559879638916751, + "loss": 2.4951, + "theoretical_loss": 3.491009711679663, + "tokens_seen": 1627859968 + }, + { + "epoch": 5.04, + "learning_rate": 0.0002559779338014042, + "loss": 2.7125, + "theoretical_loss": 3.490997513875406, + "tokens_seen": 1627925504 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025596790371113344, + "loss": 2.4878, + "theoretical_loss": 3.490985316699679, + "tokens_seen": 1627991040 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025595787362086257, + "loss": 2.5812, + "theoretical_loss": 3.4909731201524243, + "tokens_seen": 1628056576 + }, + { + "epoch": 5.04, + "learning_rate": 0.0002559478435305918, + "loss": 2.5068, + "theoretical_loss": 3.4909609242335833, + "tokens_seen": 1628122112 + }, + { + "epoch": 5.04, + "learning_rate": 0.000255937813440321, + "loss": 2.596, + "theoretical_loss": 3.4909487289430996, + "tokens_seen": 1628187648 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025592778335005017, + "loss": 2.5647, + "theoretical_loss": 3.490936534280915, + "tokens_seen": 1628253184 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025591775325977935, + "loss": 2.7733, + "theoretical_loss": 3.4909243402469716, + "tokens_seen": 1628318720 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025590772316950853, + "loss": 2.5484, + "theoretical_loss": 3.490912146841212, + "tokens_seen": 1628384256 + }, + { + "epoch": 5.04, + "learning_rate": 0.0002558976930792377, + "loss": 2.493, + "theoretical_loss": 3.490899954063578, + "tokens_seen": 1628449792 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025588766298896695, + "loss": 2.6368, + "theoretical_loss": 3.4908877619140135, + "tokens_seen": 1628515328 + }, + { + "epoch": 5.04, + "objective/train/docs_used": 1822948, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.1905274391174316, + "objective/train/theoretical_loss": 3.4908786182139746, + "objective/train/tokens_used": 1649024480, + "theoretical_loss": 3.4908786182139746, + "tokens_seen": 1628564480 + }, + { + "epoch": 5.04, + "learning_rate": 0.0002558776328986961, + "loss": 2.5279, + "theoretical_loss": 3.4908755703924594, + "tokens_seen": 1628580864 + }, + { + "epoch": 5.04, + "learning_rate": 0.0002558676028084253, + "loss": 2.841, + "theoretical_loss": 3.490863379498858, + "tokens_seen": 1628646400 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025585757271815444, + "loss": 2.432, + "theoretical_loss": 3.490851189233153, + "tokens_seen": 1628711936 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025584754262788367, + "loss": 2.8448, + "theoretical_loss": 3.4908389995952858, + "tokens_seen": 1628777472 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025583751253761285, + "loss": 2.4533, + "theoretical_loss": 3.4908268105851987, + "tokens_seen": 1628843008 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025582748244734203, + "loss": 2.639, + "theoretical_loss": 3.4908146222028344, + "tokens_seen": 1628908544 + }, + { + "epoch": 5.04, + "learning_rate": 0.0002558174523570712, + "loss": 2.7056, + "theoretical_loss": 3.4908024344481356, + "tokens_seen": 1628974080 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025580742226680045, + "loss": 2.5056, + "theoretical_loss": 3.490790247321045, + "tokens_seen": 1629039616 + }, + { + "epoch": 5.04, + "learning_rate": 0.0002557973921765296, + "loss": 2.5828, + "theoretical_loss": 3.490778060821504, + "tokens_seen": 1629105152 + }, + { + "epoch": 5.04, + "learning_rate": 0.0002557873620862588, + "loss": 2.7358, + "theoretical_loss": 3.4907658749494552, + "tokens_seen": 1629170688 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025577733199598794, + "loss": 2.3925, + "theoretical_loss": 3.490753689704842, + "tokens_seen": 1629236224 + }, + { + "epoch": 5.04, + "learning_rate": 0.0002557673019057172, + "loss": 2.5133, + "theoretical_loss": 3.490741505087606, + "tokens_seen": 1629301760 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025575727181544636, + "loss": 2.7957, + "theoretical_loss": 3.4907293210976897, + "tokens_seen": 1629367296 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025574724172517554, + "loss": 2.4953, + "theoretical_loss": 3.490717137735036, + "tokens_seen": 1629432832 + }, + { + "epoch": 5.04, + "learning_rate": 0.0002557372116349047, + "loss": 2.4671, + "theoretical_loss": 3.4907049549995874, + "tokens_seen": 1629498368 + }, + { + "epoch": 5.04, + "learning_rate": 0.0002557271815446339, + "loss": 2.3538, + "theoretical_loss": 3.490692772891286, + "tokens_seen": 1629563904 + }, + { + "epoch": 5.04, + "learning_rate": 0.0002557171514543631, + "loss": 2.63, + "theoretical_loss": 3.4906805914100745, + "tokens_seen": 1629629440 + }, + { + "epoch": 5.04, + "learning_rate": 0.0002557071213640923, + "loss": 2.5732, + "theoretical_loss": 3.4906684105558954, + "tokens_seen": 1629694976 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025569709127382144, + "loss": 2.575, + "theoretical_loss": 3.4906562303286908, + "tokens_seen": 1629760512 + }, + { + "epoch": 5.04, + "learning_rate": 0.0002556870611835507, + "loss": 2.9597, + "theoretical_loss": 3.4906440507284042, + "tokens_seen": 1629826048 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025567703109327986, + "loss": 2.5434, + "theoretical_loss": 3.490631871754977, + "tokens_seen": 1629891584 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025566700100300904, + "loss": 2.3853, + "theoretical_loss": 3.4906196934083527, + "tokens_seen": 1629957120 + }, + { + "epoch": 5.04, + "learning_rate": 0.0002556569709127382, + "loss": 2.6846, + "theoretical_loss": 3.490607515688473, + "tokens_seen": 1630022656 + }, + { + "epoch": 5.04, + "learning_rate": 0.0002556469408224674, + "loss": 2.4938, + "theoretical_loss": 3.490595338595281, + "tokens_seen": 1630088192 + }, + { + "epoch": 5.04, + "learning_rate": 0.0002556369107321966, + "loss": 2.77, + "theoretical_loss": 3.4905831621287193, + "tokens_seen": 1630153728 + }, + { + "epoch": 5.04, + "objective/train/docs_used": 1824579, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.721491575241089, + "objective/train/theoretical_loss": 3.4905740301899892, + "objective/train/tokens_used": 1650662880, + "theoretical_loss": 3.4905740301899892, + "tokens_seen": 1630202880 + }, + { + "epoch": 5.04, + "learning_rate": 0.0002556268806419258, + "loss": 2.7009, + "theoretical_loss": 3.4905709862887297, + "tokens_seen": 1630219264 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025561685055165495, + "loss": 2.9586, + "theoretical_loss": 3.490558811075256, + "tokens_seen": 1630284800 + }, + { + "epoch": 5.04, + "learning_rate": 0.0002556068204613842, + "loss": 2.7171, + "theoretical_loss": 3.49054663648824, + "tokens_seen": 1630350336 + }, + { + "epoch": 5.04, + "learning_rate": 0.0002555967903711133, + "loss": 2.6523, + "theoretical_loss": 3.4905344625276244, + "tokens_seen": 1630415872 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025558676028084254, + "loss": 2.5221, + "theoretical_loss": 3.4905222891933514, + "tokens_seen": 1630481408 + }, + { + "epoch": 5.04, + "learning_rate": 0.0002555767301905717, + "loss": 2.6267, + "theoretical_loss": 3.4905101164853645, + "tokens_seen": 1630546944 + }, + { + "epoch": 5.04, + "learning_rate": 0.0002555667001003009, + "loss": 2.6661, + "theoretical_loss": 3.4904979444036055, + "tokens_seen": 1630612480 + }, + { + "epoch": 5.04, + "learning_rate": 0.0002555566700100301, + "loss": 2.7051, + "theoretical_loss": 3.490485772948017, + "tokens_seen": 1630678016 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025554663991975927, + "loss": 2.7112, + "theoretical_loss": 3.4904736021185427, + "tokens_seen": 1630743552 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025553660982948845, + "loss": 2.7162, + "theoretical_loss": 3.490461431915124, + "tokens_seen": 1630809088 + }, + { + "epoch": 5.04, + "learning_rate": 0.0002555265797392177, + "loss": 2.6602, + "theoretical_loss": 3.4904492623377044, + "tokens_seen": 1630874624 + }, + { + "epoch": 5.04, + "learning_rate": 0.0002555165496489468, + "loss": 2.6757, + "theoretical_loss": 3.490437093386226, + "tokens_seen": 1630940160 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025550651955867605, + "loss": 2.5184, + "theoretical_loss": 3.4904249250606316, + "tokens_seen": 1631005696 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025549648946840523, + "loss": 2.65, + "theoretical_loss": 3.4904127573608634, + "tokens_seen": 1631071232 + }, + { + "epoch": 5.04, + "learning_rate": 0.0002554864593781344, + "loss": 2.6598, + "theoretical_loss": 3.490400590286865, + "tokens_seen": 1631136768 + }, + { + "epoch": 5.04, + "learning_rate": 0.0002554764292878636, + "loss": 2.4051, + "theoretical_loss": 3.4903884238385787, + "tokens_seen": 1631202304 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025546639919759277, + "loss": 2.7635, + "theoretical_loss": 3.4903762580159468, + "tokens_seen": 1631267840 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025545636910732195, + "loss": 2.6505, + "theoretical_loss": 3.4903640928189126, + "tokens_seen": 1631333376 + }, + { + "epoch": 5.04, + "learning_rate": 0.0002554463390170512, + "loss": 2.6143, + "theoretical_loss": 3.4903519282474185, + "tokens_seen": 1631398912 + }, + { + "epoch": 5.04, + "learning_rate": 0.0002554363089267803, + "loss": 2.8442, + "theoretical_loss": 3.490339764301407, + "tokens_seen": 1631464448 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025542627883650955, + "loss": 2.6405, + "theoretical_loss": 3.4903276009808213, + "tokens_seen": 1631529984 + }, + { + "epoch": 5.04, + "learning_rate": 0.0002554162487462387, + "loss": 2.4992, + "theoretical_loss": 3.4903154382856036, + "tokens_seen": 1631595520 + }, + { + "epoch": 5.04, + "learning_rate": 0.0002554062186559679, + "loss": 2.5556, + "theoretical_loss": 3.4903032762156974, + "tokens_seen": 1631661056 + }, + { + "epoch": 5.04, + "learning_rate": 0.0002553961885656971, + "loss": 2.648, + "theoretical_loss": 3.4902911147710443, + "tokens_seen": 1631726592 + }, + { + "epoch": 5.04, + "learning_rate": 0.0002553861584754263, + "loss": 2.6144, + "theoretical_loss": 3.490278953951588, + "tokens_seen": 1631792128 + }, + { + "epoch": 5.04, + "objective/train/docs_used": 1825363, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6945481300354004, + "objective/train/theoretical_loss": 3.490269833747246, + "objective/train/tokens_used": 1652301280, + "theoretical_loss": 3.490269833747246, + "tokens_seen": 1631841280 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025537612838515546, + "loss": 2.5007, + "theoretical_loss": 3.4902667937572707, + "tokens_seen": 1631857664 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025536609829488464, + "loss": 2.9637, + "theoretical_loss": 3.4902546341880356, + "tokens_seen": 1631923200 + }, + { + "epoch": 5.04, + "learning_rate": 0.0002553560682046138, + "loss": 2.6689, + "theoretical_loss": 3.490242475243825, + "tokens_seen": 1631988736 + }, + { + "epoch": 5.04, + "learning_rate": 0.00025534603811434305, + "loss": 2.5679, + "theoretical_loss": 3.490230316924582, + "tokens_seen": 1632054272 + }, + { + "epoch": 5.04, + "learning_rate": 0.0002553360080240722, + "loss": 2.5961, + "theoretical_loss": 3.490218159230249, + "tokens_seen": 1632119808 + }, + { + "epoch": 5.04, + "learning_rate": 0.0002553259779338014, + "loss": 2.5681, + "theoretical_loss": 3.49020600216077, + "tokens_seen": 1632185344 + }, + { + "epoch": 5.04, + "learning_rate": 0.0002553159478435306, + "loss": 2.5314, + "theoretical_loss": 3.490193845716086, + "tokens_seen": 1632250880 + }, + { + "epoch": 5.05, + "learning_rate": 0.0002553059177532598, + "loss": 2.8687, + "theoretical_loss": 3.490181689896141, + "tokens_seen": 1632316416 + }, + { + "epoch": 5.05, + "learning_rate": 0.00025529588766298896, + "loss": 2.5809, + "theoretical_loss": 3.4901695347008777, + "tokens_seen": 1632381952 + }, + { + "epoch": 5.05, + "learning_rate": 0.00025528585757271814, + "loss": 2.4463, + "theoretical_loss": 3.4901573801302384, + "tokens_seen": 1632447488 + }, + { + "epoch": 5.05, + "learning_rate": 0.0002552758274824473, + "loss": 2.5499, + "theoretical_loss": 3.4901452261841666, + "tokens_seen": 1632513024 + }, + { + "epoch": 5.05, + "learning_rate": 0.00025526579739217656, + "loss": 2.5645, + "theoretical_loss": 3.4901330728626045, + "tokens_seen": 1632578560 + }, + { + "epoch": 5.05, + "learning_rate": 0.0002552557673019057, + "loss": 2.341, + "theoretical_loss": 3.490120920165496, + "tokens_seen": 1632644096 + }, + { + "epoch": 5.05, + "learning_rate": 0.0002552457372116349, + "loss": 2.5379, + "theoretical_loss": 3.490108768092782, + "tokens_seen": 1632709632 + }, + { + "epoch": 5.05, + "learning_rate": 0.0002552357071213641, + "loss": 2.6763, + "theoretical_loss": 3.490096616644407, + "tokens_seen": 1632775168 + }, + { + "epoch": 5.05, + "learning_rate": 0.0002552256770310933, + "loss": 2.8082, + "theoretical_loss": 3.4900844658203134, + "tokens_seen": 1632840704 + }, + { + "epoch": 5.05, + "learning_rate": 0.0002552156469408225, + "loss": 2.6397, + "theoretical_loss": 3.4900723156204445, + "tokens_seen": 1632906240 + }, + { + "epoch": 5.05, + "learning_rate": 0.00025520561685055164, + "loss": 2.4946, + "theoretical_loss": 3.4900601660447426, + "tokens_seen": 1632971776 + }, + { + "epoch": 5.05, + "learning_rate": 0.0002551955867602809, + "loss": 2.6199, + "theoretical_loss": 3.4900480170931507, + "tokens_seen": 1633037312 + }, + { + "epoch": 5.05, + "learning_rate": 0.00025518555667001006, + "loss": 2.5152, + "theoretical_loss": 3.4900358687656112, + "tokens_seen": 1633102848 + }, + { + "epoch": 5.05, + "learning_rate": 0.00025517552657973924, + "loss": 2.6778, + "theoretical_loss": 3.4900237210620686, + "tokens_seen": 1633168384 + }, + { + "epoch": 5.05, + "learning_rate": 0.0002551654964894684, + "loss": 2.4758, + "theoretical_loss": 3.4900115739824638, + "tokens_seen": 1633233920 + }, + { + "epoch": 5.05, + "learning_rate": 0.0002551554663991976, + "loss": 2.4492, + "theoretical_loss": 3.489999427526741, + "tokens_seen": 1633299456 + }, + { + "epoch": 5.05, + "learning_rate": 0.0002551454363089268, + "loss": 2.8651, + "theoretical_loss": 3.4899872816948427, + "tokens_seen": 1633364992 + }, + { + "epoch": 5.05, + "learning_rate": 0.000255135406218656, + "loss": 2.7476, + "theoretical_loss": 3.4899751364867124, + "tokens_seen": 1633430528 + }, + { + "epoch": 5.05, + "objective/train/docs_used": 1826734, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7838215827941895, + "objective/train/theoretical_loss": 3.4899660279899276, + "objective/train/tokens_used": 1653939680, + "theoretical_loss": 3.4899660279899276, + "tokens_seen": 1633479680 + }, + { + "epoch": 5.05, + "learning_rate": 0.00025512537612838515, + "loss": 2.687, + "theoretical_loss": 3.489962991902292, + "tokens_seen": 1633496064 + }, + { + "epoch": 5.05, + "learning_rate": 0.0002551153460381144, + "loss": 2.6393, + "theoretical_loss": 3.4899508479415253, + "tokens_seen": 1633561600 + }, + { + "epoch": 5.05, + "learning_rate": 0.0002551053159478435, + "loss": 2.8947, + "theoretical_loss": 3.489938704604355, + "tokens_seen": 1633627136 + }, + { + "epoch": 5.05, + "learning_rate": 0.00025509528585757274, + "loss": 2.7382, + "theoretical_loss": 3.4899265618907243, + "tokens_seen": 1633692672 + }, + { + "epoch": 5.05, + "learning_rate": 0.0002550852557673019, + "loss": 2.7037, + "theoretical_loss": 3.4899144198005754, + "tokens_seen": 1633758208 + }, + { + "epoch": 5.05, + "learning_rate": 0.0002550752256770311, + "loss": 2.8429, + "theoretical_loss": 3.4899022783338522, + "tokens_seen": 1633823744 + }, + { + "epoch": 5.05, + "learning_rate": 0.0002550651955867603, + "loss": 2.5956, + "theoretical_loss": 3.489890137490497, + "tokens_seen": 1633889280 + }, + { + "epoch": 5.05, + "learning_rate": 0.00025505516549648947, + "loss": 2.5744, + "theoretical_loss": 3.4898779972704537, + "tokens_seen": 1633954816 + }, + { + "epoch": 5.05, + "learning_rate": 0.00025504513540621865, + "loss": 2.344, + "theoretical_loss": 3.489865857673664, + "tokens_seen": 1634020352 + }, + { + "epoch": 5.05, + "learning_rate": 0.0002550351053159479, + "loss": 2.459, + "theoretical_loss": 3.489853718700072, + "tokens_seen": 1634085888 + }, + { + "epoch": 5.05, + "learning_rate": 0.000255025075225677, + "loss": 2.4011, + "theoretical_loss": 3.48984158034962, + "tokens_seen": 1634151424 + }, + { + "epoch": 5.05, + "learning_rate": 0.00025501504513540625, + "loss": 2.6465, + "theoretical_loss": 3.489829442622251, + "tokens_seen": 1634216960 + }, + { + "epoch": 5.05, + "learning_rate": 0.00025500501504513543, + "loss": 2.6639, + "theoretical_loss": 3.489817305517909, + "tokens_seen": 1634282496 + }, + { + "epoch": 5.05, + "learning_rate": 0.0002549949849548646, + "loss": 2.5077, + "theoretical_loss": 3.4898051690365364, + "tokens_seen": 1634348032 + }, + { + "epoch": 5.05, + "learning_rate": 0.0002549849548645938, + "loss": 2.3277, + "theoretical_loss": 3.489793033178076, + "tokens_seen": 1634413568 + }, + { + "epoch": 5.05, + "learning_rate": 0.00025497492477432297, + "loss": 2.8964, + "theoretical_loss": 3.489780897942471, + "tokens_seen": 1634479104 + }, + { + "epoch": 5.05, + "learning_rate": 0.00025496489468405215, + "loss": 2.5993, + "theoretical_loss": 3.4897687633296646, + "tokens_seen": 1634544640 + }, + { + "epoch": 5.05, + "learning_rate": 0.0002549548645937814, + "loss": 2.5346, + "theoretical_loss": 3.4897566293396, + "tokens_seen": 1634610176 + }, + { + "epoch": 5.05, + "learning_rate": 0.0002549448345035105, + "loss": 2.6303, + "theoretical_loss": 3.4897444959722197, + "tokens_seen": 1634675712 + }, + { + "epoch": 5.05, + "learning_rate": 0.00025493480441323975, + "loss": 2.3833, + "theoretical_loss": 3.4897323632274677, + "tokens_seen": 1634741248 + }, + { + "epoch": 5.05, + "learning_rate": 0.0002549247743229689, + "loss": 2.7033, + "theoretical_loss": 3.489720231105286, + "tokens_seen": 1634806784 + }, + { + "epoch": 5.05, + "learning_rate": 0.0002549147442326981, + "loss": 2.5677, + "theoretical_loss": 3.489708099605619, + "tokens_seen": 1634872320 + }, + { + "epoch": 5.05, + "learning_rate": 0.0002549047141424273, + "loss": 2.3206, + "theoretical_loss": 3.489695968728409, + "tokens_seen": 1634937856 + }, + { + "epoch": 5.05, + "learning_rate": 0.0002548946840521565, + "loss": 2.5654, + "theoretical_loss": 3.4896838384735984, + "tokens_seen": 1635003392 + }, + { + "epoch": 5.05, + "learning_rate": 0.00025488465396188566, + "loss": 2.5729, + "theoretical_loss": 3.4896717088411315, + "tokens_seen": 1635068928 + }, + { + "epoch": 5.05, + "objective/train/docs_used": 1827286, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7116994857788086, + "objective/train/theoretical_loss": 3.4896626120251604, + "objective/train/tokens_used": 1655578080, + "theoretical_loss": 3.4896626120251604, + "tokens_seen": 1635118080 + }, + { + "epoch": 5.05, + "learning_rate": 0.00025487462387161484, + "loss": 2.5476, + "theoretical_loss": 3.4896595798309513, + "tokens_seen": 1635134464 + }, + { + "epoch": 5.05, + "learning_rate": 0.000254864593781344, + "loss": 2.3753, + "theoretical_loss": 3.4896474514430005, + "tokens_seen": 1635200000 + }, + { + "epoch": 5.05, + "learning_rate": 0.00025485456369107325, + "loss": 2.6086, + "theoretical_loss": 3.4896353236772226, + "tokens_seen": 1635265536 + }, + { + "epoch": 5.05, + "learning_rate": 0.0002548445336008024, + "loss": 2.5868, + "theoretical_loss": 3.48962319653356, + "tokens_seen": 1635331072 + }, + { + "epoch": 5.05, + "learning_rate": 0.0002548345035105316, + "loss": 2.7026, + "theoretical_loss": 3.489611070011957, + "tokens_seen": 1635396608 + }, + { + "epoch": 5.05, + "learning_rate": 0.0002548244734202608, + "loss": 2.662, + "theoretical_loss": 3.489598944112356, + "tokens_seen": 1635462144 + }, + { + "epoch": 5.05, + "learning_rate": 0.00025481444332999, + "loss": 2.7653, + "theoretical_loss": 3.4895868188347, + "tokens_seen": 1635527680 + }, + { + "epoch": 5.05, + "learning_rate": 0.00025480441323971916, + "loss": 2.5967, + "theoretical_loss": 3.489574694178933, + "tokens_seen": 1635593216 + }, + { + "epoch": 5.05, + "learning_rate": 0.00025479438314944834, + "loss": 2.6639, + "theoretical_loss": 3.4895625701449977, + "tokens_seen": 1635658752 + }, + { + "epoch": 5.05, + "learning_rate": 0.0002547843530591775, + "loss": 2.5723, + "theoretical_loss": 3.4895504467328373, + "tokens_seen": 1635724288 + }, + { + "epoch": 5.05, + "learning_rate": 0.00025477432296890676, + "loss": 2.4181, + "theoretical_loss": 3.489538323942395, + "tokens_seen": 1635789824 + }, + { + "epoch": 5.05, + "learning_rate": 0.0002547642928786359, + "loss": 2.5984, + "theoretical_loss": 3.489526201773614, + "tokens_seen": 1635855360 + }, + { + "epoch": 5.05, + "learning_rate": 0.0002547542627883651, + "loss": 2.7379, + "theoretical_loss": 3.4895140802264377, + "tokens_seen": 1635920896 + }, + { + "epoch": 5.05, + "learning_rate": 0.00025474423269809425, + "loss": 2.6937, + "theoretical_loss": 3.489501959300809, + "tokens_seen": 1635986432 + }, + { + "epoch": 5.05, + "learning_rate": 0.0002547342026078235, + "loss": 2.6661, + "theoretical_loss": 3.489489838996671, + "tokens_seen": 1636051968 + }, + { + "epoch": 5.05, + "learning_rate": 0.00025472417251755266, + "loss": 2.8623, + "theoretical_loss": 3.4894777193139683, + "tokens_seen": 1636117504 + }, + { + "epoch": 5.05, + "learning_rate": 0.00025471414242728184, + "loss": 2.8822, + "theoretical_loss": 3.4894656002526423, + "tokens_seen": 1636183040 + }, + { + "epoch": 5.05, + "learning_rate": 0.000254704112337011, + "loss": 2.4939, + "theoretical_loss": 3.489453481812637, + "tokens_seen": 1636248576 + }, + { + "epoch": 5.05, + "learning_rate": 0.00025469408224674026, + "loss": 2.9008, + "theoretical_loss": 3.4894413639938957, + "tokens_seen": 1636314112 + }, + { + "epoch": 5.05, + "learning_rate": 0.0002546840521564694, + "loss": 2.754, + "theoretical_loss": 3.4894292467963623, + "tokens_seen": 1636379648 + }, + { + "epoch": 5.05, + "learning_rate": 0.0002546740220661986, + "loss": 2.4276, + "theoretical_loss": 3.4894171302199792, + "tokens_seen": 1636445184 + }, + { + "epoch": 5.05, + "learning_rate": 0.00025466399197592775, + "loss": 2.67, + "theoretical_loss": 3.4894050142646895, + "tokens_seen": 1636510720 + }, + { + "epoch": 5.05, + "learning_rate": 0.000254653961885657, + "loss": 2.4721, + "theoretical_loss": 3.4893928989304372, + "tokens_seen": 1636576256 + }, + { + "epoch": 5.05, + "learning_rate": 0.00025464393179538616, + "loss": 2.9586, + "theoretical_loss": 3.4893807842171656, + "tokens_seen": 1636641792 + }, + { + "epoch": 5.05, + "learning_rate": 0.00025463390170511535, + "loss": 2.6099, + "theoretical_loss": 3.4893686701248177, + "tokens_seen": 1636707328 + }, + { + "epoch": 5.05, + "objective/train/docs_used": 1827288, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.952646493911743, + "objective/train/theoretical_loss": 3.4893595849630037, + "objective/train/tokens_used": 1657216480, + "theoretical_loss": 3.4893595849630037, + "tokens_seen": 1636756480 + }, + { + "epoch": 5.05, + "learning_rate": 0.00025462387161484453, + "loss": 2.7477, + "theoretical_loss": 3.4893565566533367, + "tokens_seen": 1636772864 + }, + { + "epoch": 5.05, + "learning_rate": 0.0002546138415245737, + "loss": 2.6074, + "theoretical_loss": 3.489344443802666, + "tokens_seen": 1636838400 + }, + { + "epoch": 5.05, + "learning_rate": 0.0002546038114343029, + "loss": 2.8813, + "theoretical_loss": 3.4893323315727494, + "tokens_seen": 1636903936 + }, + { + "epoch": 5.05, + "learning_rate": 0.0002545937813440321, + "loss": 2.5295, + "theoretical_loss": 3.4893202199635294, + "tokens_seen": 1636969472 + }, + { + "epoch": 5.05, + "learning_rate": 0.00025458375125376125, + "loss": 2.7456, + "theoretical_loss": 3.4893081089749503, + "tokens_seen": 1637035008 + }, + { + "epoch": 5.05, + "learning_rate": 0.0002545737211634905, + "loss": 2.6556, + "theoretical_loss": 3.4892959986069547, + "tokens_seen": 1637100544 + }, + { + "epoch": 5.05, + "learning_rate": 0.0002545636910732196, + "loss": 2.7824, + "theoretical_loss": 3.489283888859486, + "tokens_seen": 1637166080 + }, + { + "epoch": 5.05, + "learning_rate": 0.00025455366098294885, + "loss": 2.7233, + "theoretical_loss": 3.489271779732488, + "tokens_seen": 1637231616 + }, + { + "epoch": 5.05, + "learning_rate": 0.00025454363089267803, + "loss": 2.649, + "theoretical_loss": 3.489259671225904, + "tokens_seen": 1637297152 + }, + { + "epoch": 5.05, + "learning_rate": 0.0002545336008024072, + "loss": 2.5388, + "theoretical_loss": 3.489247563339677, + "tokens_seen": 1637362688 + }, + { + "epoch": 5.05, + "learning_rate": 0.0002545235707121364, + "loss": 2.5585, + "theoretical_loss": 3.48923545607375, + "tokens_seen": 1637428224 + }, + { + "epoch": 5.05, + "learning_rate": 0.00025451354062186563, + "loss": 2.5326, + "theoretical_loss": 3.489223349428068, + "tokens_seen": 1637493760 + }, + { + "epoch": 5.05, + "learning_rate": 0.0002545035105315948, + "loss": 2.5587, + "theoretical_loss": 3.4892112434025733, + "tokens_seen": 1637559296 + }, + { + "epoch": 5.05, + "learning_rate": 0.000254493480441324, + "loss": 2.8104, + "theoretical_loss": 3.4891991379972094, + "tokens_seen": 1637624832 + }, + { + "epoch": 5.05, + "learning_rate": 0.00025448345035105317, + "loss": 2.6764, + "theoretical_loss": 3.4891870332119193, + "tokens_seen": 1637690368 + }, + { + "epoch": 5.05, + "learning_rate": 0.00025447342026078235, + "loss": 2.5624, + "theoretical_loss": 3.489174929046647, + "tokens_seen": 1637755904 + }, + { + "epoch": 5.05, + "learning_rate": 0.0002544633901705116, + "loss": 2.5065, + "theoretical_loss": 3.489162825501336, + "tokens_seen": 1637821440 + }, + { + "epoch": 5.05, + "learning_rate": 0.0002544533600802407, + "loss": 2.4074, + "theoretical_loss": 3.4891507225759297, + "tokens_seen": 1637886976 + }, + { + "epoch": 5.05, + "learning_rate": 0.00025444332998996995, + "loss": 2.6044, + "theoretical_loss": 3.4891386202703716, + "tokens_seen": 1637952512 + }, + { + "epoch": 5.05, + "learning_rate": 0.0002544332998996991, + "loss": 2.516, + "theoretical_loss": 3.4891265185846043, + "tokens_seen": 1638018048 + }, + { + "epoch": 5.05, + "learning_rate": 0.0002544232698094283, + "loss": 2.4602, + "theoretical_loss": 3.4891144175185724, + "tokens_seen": 1638083584 + }, + { + "epoch": 5.05, + "learning_rate": 0.0002544132397191575, + "loss": 2.6896, + "theoretical_loss": 3.4891023170722186, + "tokens_seen": 1638149120 + }, + { + "epoch": 5.05, + "learning_rate": 0.0002544032096288867, + "loss": 2.7171, + "theoretical_loss": 3.4890902172454865, + "tokens_seen": 1638214656 + }, + { + "epoch": 5.05, + "learning_rate": 0.00025439317953861586, + "loss": 2.6357, + "theoretical_loss": 3.4890781180383206, + "tokens_seen": 1638280192 + }, + { + "epoch": 5.05, + "learning_rate": 0.00025438314944834504, + "loss": 2.5777, + "theoretical_loss": 3.4890660194506626, + "tokens_seen": 1638345728 + }, + { + "debugging/Self-BLEU-5": 0.28035211984762975, + "debugging/distinct-1-grams": 0.8180070273723525, + "debugging/distinct-2-grams": 0.9661205773003526, + "debugging/entropy-1-grams": 5.418256584988372, + "debugging/entropy-2-grams": 5.950145975632809, + "debugging/length": 475.1666666666667, + "debugging/num_segments": 6, + "debugging/score": 0.006501734263504407, + "debugging/score_std": 0.00880767366589508, + "epoch": 5.05, + "objective/train/docs_used": 1828052, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.91074538230896, + "objective/train/theoretical_loss": 3.4890569459164382, + "objective/train/tokens_used": 1658854880, + "theoretical_loss": 3.4890569459164382, + "tokens_seen": 1638394880 + }, + { + "epoch": 5.05, + "learning_rate": 0.0002543731193580742, + "loss": 2.7252, + "theoretical_loss": 3.4890539214824576, + "tokens_seen": 1638411264 + }, + { + "epoch": 5.05, + "learning_rate": 0.00025436308926780345, + "loss": 2.8064, + "theoretical_loss": 3.489041824133648, + "tokens_seen": 1638476800 + }, + { + "epoch": 5.05, + "learning_rate": 0.0002543530591775326, + "loss": 2.5308, + "theoretical_loss": 3.489029727404178, + "tokens_seen": 1638542336 + }, + { + "epoch": 5.05, + "learning_rate": 0.0002543430290872618, + "loss": 2.651, + "theoretical_loss": 3.489017631293991, + "tokens_seen": 1638607872 + }, + { + "epoch": 5.05, + "learning_rate": 0.000254332998996991, + "loss": 2.3064, + "theoretical_loss": 3.48900553580303, + "tokens_seen": 1638673408 + }, + { + "epoch": 5.05, + "learning_rate": 0.0002543229689067202, + "loss": 2.8929, + "theoretical_loss": 3.4889934409312398, + "tokens_seen": 1638738944 + }, + { + "epoch": 5.05, + "learning_rate": 0.00025431293881644936, + "loss": 2.516, + "theoretical_loss": 3.488981346678562, + "tokens_seen": 1638804480 + }, + { + "epoch": 5.05, + "learning_rate": 0.00025430290872617854, + "loss": 2.5252, + "theoretical_loss": 3.4889692530449423, + "tokens_seen": 1638870016 + }, + { + "epoch": 5.05, + "learning_rate": 0.0002542928786359077, + "loss": 2.4958, + "theoretical_loss": 3.4889571600303224, + "tokens_seen": 1638935552 + }, + { + "epoch": 5.05, + "learning_rate": 0.00025428284854563696, + "loss": 2.6536, + "theoretical_loss": 3.4889450676346474, + "tokens_seen": 1639001088 + }, + { + "epoch": 5.05, + "learning_rate": 0.0002542728184553661, + "loss": 2.7862, + "theoretical_loss": 3.4889329758578596, + "tokens_seen": 1639066624 + }, + { + "epoch": 5.05, + "learning_rate": 0.0002542627883650953, + "loss": 2.6614, + "theoretical_loss": 3.4889208846999034, + "tokens_seen": 1639132160 + }, + { + "epoch": 5.05, + "learning_rate": 0.00025425275827482445, + "loss": 2.631, + "theoretical_loss": 3.488908794160722, + "tokens_seen": 1639197696 + }, + { + "epoch": 5.05, + "learning_rate": 0.0002542427281845537, + "loss": 2.7136, + "theoretical_loss": 3.4888967042402594, + "tokens_seen": 1639263232 + }, + { + "epoch": 5.05, + "learning_rate": 0.00025423269809428286, + "loss": 2.5916, + "theoretical_loss": 3.4888846149384585, + "tokens_seen": 1639328768 + }, + { + "epoch": 5.05, + "learning_rate": 0.00025422266800401204, + "loss": 2.4989, + "theoretical_loss": 3.4888725262552636, + "tokens_seen": 1639394304 + }, + { + "epoch": 5.05, + "learning_rate": 0.0002542126379137412, + "loss": 2.7752, + "theoretical_loss": 3.488860438190618, + "tokens_seen": 1639459840 + }, + { + "epoch": 5.05, + "learning_rate": 0.00025420260782347046, + "loss": 2.7072, + "theoretical_loss": 3.4888483507444654, + "tokens_seen": 1639525376 + }, + { + "epoch": 5.05, + "learning_rate": 0.0002541925777331996, + "loss": 2.6852, + "theoretical_loss": 3.4888362639167494, + "tokens_seen": 1639590912 + }, + { + "epoch": 5.05, + "learning_rate": 0.0002541825476429288, + "loss": 2.4922, + "theoretical_loss": 3.488824177707414, + "tokens_seen": 1639656448 + }, + { + "epoch": 5.05, + "learning_rate": 0.00025417251755265795, + "loss": 2.611, + "theoretical_loss": 3.488812092116402, + "tokens_seen": 1639721984 + }, + { + "epoch": 5.05, + "learning_rate": 0.0002541624874623872, + "loss": 2.8802, + "theoretical_loss": 3.488800007143658, + "tokens_seen": 1639787520 + }, + { + "epoch": 5.05, + "learning_rate": 0.00025415245737211637, + "loss": 2.7579, + "theoretical_loss": 3.488787922789125, + "tokens_seen": 1639853056 + }, + { + "epoch": 5.05, + "learning_rate": 0.00025414242728184555, + "loss": 2.868, + "theoretical_loss": 3.488775839052747, + "tokens_seen": 1639918592 + }, + { + "epoch": 5.05, + "learning_rate": 0.00025413239719157473, + "loss": 2.8075, + "theoretical_loss": 3.4887637559344675, + "tokens_seen": 1639984128 + }, + { + "epoch": 5.05, + "objective/train/docs_used": 1828793, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.076061725616455, + "objective/train/theoretical_loss": 3.4887546940013507, + "objective/train/tokens_used": 1660493280, + "theoretical_loss": 3.4887546940013507, + "tokens_seen": 1640033280 + }, + { + "epoch": 5.05, + "learning_rate": 0.0002541223671013039, + "loss": 2.544, + "theoretical_loss": 3.48875167343423, + "tokens_seen": 1640049664 + }, + { + "epoch": 5.05, + "learning_rate": 0.0002541123370110331, + "loss": 2.5412, + "theoretical_loss": 3.4887395915519788, + "tokens_seen": 1640115200 + }, + { + "epoch": 5.05, + "learning_rate": 0.0002541023069207623, + "loss": 2.5361, + "theoretical_loss": 3.4887275102876574, + "tokens_seen": 1640180736 + }, + { + "epoch": 5.05, + "learning_rate": 0.00025409227683049145, + "loss": 2.9203, + "theoretical_loss": 3.488715429641209, + "tokens_seen": 1640246272 + }, + { + "epoch": 5.05, + "learning_rate": 0.0002540822467402207, + "loss": 2.5023, + "theoretical_loss": 3.488703349612578, + "tokens_seen": 1640311808 + }, + { + "epoch": 5.05, + "learning_rate": 0.0002540722166499498, + "loss": 2.7101, + "theoretical_loss": 3.4886912702017074, + "tokens_seen": 1640377344 + }, + { + "epoch": 5.05, + "learning_rate": 0.00025406218655967905, + "loss": 2.7177, + "theoretical_loss": 3.4886791914085418, + "tokens_seen": 1640442880 + }, + { + "epoch": 5.05, + "learning_rate": 0.00025405215646940823, + "loss": 2.6426, + "theoretical_loss": 3.488667113233024, + "tokens_seen": 1640508416 + }, + { + "epoch": 5.05, + "learning_rate": 0.0002540421263791374, + "loss": 2.8069, + "theoretical_loss": 3.488655035675098, + "tokens_seen": 1640573952 + }, + { + "epoch": 5.05, + "learning_rate": 0.0002540320962888666, + "loss": 2.714, + "theoretical_loss": 3.4886429587347085, + "tokens_seen": 1640639488 + }, + { + "epoch": 5.05, + "learning_rate": 0.00025402206619859583, + "loss": 2.6145, + "theoretical_loss": 3.4886308824117984, + "tokens_seen": 1640705024 + }, + { + "epoch": 5.05, + "learning_rate": 0.00025401203610832495, + "loss": 2.6574, + "theoretical_loss": 3.4886188067063113, + "tokens_seen": 1640770560 + }, + { + "epoch": 5.05, + "learning_rate": 0.0002540020060180542, + "loss": 2.4431, + "theoretical_loss": 3.4886067316181917, + "tokens_seen": 1640836096 + }, + { + "epoch": 5.05, + "learning_rate": 0.0002539919759277833, + "loss": 2.7695, + "theoretical_loss": 3.4885946571473827, + "tokens_seen": 1640901632 + }, + { + "epoch": 5.05, + "learning_rate": 0.00025398194583751255, + "loss": 2.7228, + "theoretical_loss": 3.488582583293828, + "tokens_seen": 1640967168 + }, + { + "epoch": 5.05, + "learning_rate": 0.00025397191574724173, + "loss": 2.6124, + "theoretical_loss": 3.488570510057472, + "tokens_seen": 1641032704 + }, + { + "epoch": 5.05, + "learning_rate": 0.0002539618856569709, + "loss": 2.6302, + "theoretical_loss": 3.4885584374382583, + "tokens_seen": 1641098240 + }, + { + "epoch": 5.05, + "learning_rate": 0.0002539518555667001, + "loss": 2.6826, + "theoretical_loss": 3.4885463654361306, + "tokens_seen": 1641163776 + }, + { + "epoch": 5.05, + "learning_rate": 0.0002539418254764293, + "loss": 2.5072, + "theoretical_loss": 3.488534294051033, + "tokens_seen": 1641229312 + }, + { + "epoch": 5.05, + "learning_rate": 0.00025393179538615846, + "loss": 2.5015, + "theoretical_loss": 3.4885222232829083, + "tokens_seen": 1641294848 + }, + { + "epoch": 5.05, + "learning_rate": 0.0002539217652958877, + "loss": 2.7234, + "theoretical_loss": 3.4885101531317018, + "tokens_seen": 1641360384 + }, + { + "epoch": 5.05, + "learning_rate": 0.0002539117352056168, + "loss": 2.8709, + "theoretical_loss": 3.4884980835973565, + "tokens_seen": 1641425920 + }, + { + "epoch": 5.05, + "learning_rate": 0.00025390170511534606, + "loss": 2.6931, + "theoretical_loss": 3.4884860146798164, + "tokens_seen": 1641491456 + }, + { + "epoch": 5.05, + "learning_rate": 0.0002538916750250752, + "loss": 2.4736, + "theoretical_loss": 3.488473946379025, + "tokens_seen": 1641556992 + }, + { + "epoch": 5.05, + "learning_rate": 0.0002538816449348044, + "loss": 2.6352, + "theoretical_loss": 3.488461878694927, + "tokens_seen": 1641622528 + }, + { + "epoch": 5.05, + "objective/train/docs_used": 1830032, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.0165646076202393, + "objective/train/theoretical_loss": 3.4884528283365244, + "objective/train/tokens_used": 1662131680, + "theoretical_loss": 3.4884528283365244, + "tokens_seen": 1641671680 + }, + { + "epoch": 5.05, + "learning_rate": 0.0002538716148445336, + "loss": 2.8231, + "theoretical_loss": 3.488449811627466, + "tokens_seen": 1641688064 + }, + { + "epoch": 5.05, + "learning_rate": 0.0002538615847542628, + "loss": 2.7286, + "theoretical_loss": 3.4884377451765847, + "tokens_seen": 1641753600 + }, + { + "epoch": 5.05, + "learning_rate": 0.00025385155466399196, + "loss": 2.6162, + "theoretical_loss": 3.488425679342229, + "tokens_seen": 1641819136 + }, + { + "epoch": 5.05, + "learning_rate": 0.0002538415245737212, + "loss": 2.6865, + "theoretical_loss": 3.488413614124341, + "tokens_seen": 1641884672 + }, + { + "epoch": 5.05, + "learning_rate": 0.0002538314944834503, + "loss": 2.6419, + "theoretical_loss": 3.488401549522866, + "tokens_seen": 1641950208 + }, + { + "epoch": 5.05, + "learning_rate": 0.00025382146439317956, + "loss": 2.6924, + "theoretical_loss": 3.4883894855377466, + "tokens_seen": 1642015744 + }, + { + "epoch": 5.05, + "learning_rate": 0.0002538114343029087, + "loss": 2.498, + "theoretical_loss": 3.488377422168928, + "tokens_seen": 1642081280 + }, + { + "epoch": 5.05, + "learning_rate": 0.0002538014042126379, + "loss": 2.7008, + "theoretical_loss": 3.4883653594163535, + "tokens_seen": 1642146816 + }, + { + "epoch": 5.05, + "learning_rate": 0.0002537913741223671, + "loss": 2.6494, + "theoretical_loss": 3.4883532972799665, + "tokens_seen": 1642212352 + }, + { + "epoch": 5.05, + "learning_rate": 0.0002537813440320963, + "loss": 2.5609, + "theoretical_loss": 3.4883412357597114, + "tokens_seen": 1642277888 + }, + { + "epoch": 5.05, + "learning_rate": 0.00025377131394182546, + "loss": 2.6344, + "theoretical_loss": 3.4883291748555325, + "tokens_seen": 1642343424 + }, + { + "epoch": 5.05, + "learning_rate": 0.00025376128385155465, + "loss": 2.6554, + "theoretical_loss": 3.4883171145673737, + "tokens_seen": 1642408960 + }, + { + "epoch": 5.05, + "learning_rate": 0.0002537512537612839, + "loss": 2.6871, + "theoretical_loss": 3.4883050548951777, + "tokens_seen": 1642474496 + }, + { + "epoch": 5.05, + "learning_rate": 0.00025374122367101306, + "loss": 2.554, + "theoretical_loss": 3.488292995838891, + "tokens_seen": 1642540032 + }, + { + "epoch": 5.05, + "learning_rate": 0.00025373119358074224, + "loss": 2.6946, + "theoretical_loss": 3.4882809373984545, + "tokens_seen": 1642605568 + }, + { + "epoch": 5.05, + "learning_rate": 0.0002537211634904714, + "loss": 2.7, + "theoretical_loss": 3.4882688795738144, + "tokens_seen": 1642671104 + }, + { + "epoch": 5.05, + "learning_rate": 0.00025371113340020066, + "loss": 2.8572, + "theoretical_loss": 3.488256822364914, + "tokens_seen": 1642736640 + }, + { + "epoch": 5.05, + "learning_rate": 0.0002537011033099298, + "loss": 2.7233, + "theoretical_loss": 3.4882447657716975, + "tokens_seen": 1642802176 + }, + { + "epoch": 5.05, + "learning_rate": 0.000253691073219659, + "loss": 2.5799, + "theoretical_loss": 3.4882327097941084, + "tokens_seen": 1642867712 + }, + { + "epoch": 5.05, + "learning_rate": 0.00025368104312938815, + "loss": 2.6587, + "theoretical_loss": 3.4882206544320913, + "tokens_seen": 1642933248 + }, + { + "epoch": 5.05, + "learning_rate": 0.0002536710130391174, + "loss": 2.5166, + "theoretical_loss": 3.4882085996855894, + "tokens_seen": 1642998784 + }, + { + "epoch": 5.05, + "learning_rate": 0.00025366098294884657, + "loss": 2.5442, + "theoretical_loss": 3.4881965455545476, + "tokens_seen": 1643064320 + }, + { + "epoch": 5.05, + "learning_rate": 0.00025365095285857575, + "loss": 2.7119, + "theoretical_loss": 3.4881844920389096, + "tokens_seen": 1643129856 + }, + { + "epoch": 5.05, + "learning_rate": 0.00025364092276830493, + "loss": 2.648, + "theoretical_loss": 3.4881724391386193, + "tokens_seen": 1643195392 + }, + { + "epoch": 5.05, + "learning_rate": 0.0002536308926780341, + "loss": 2.7163, + "theoretical_loss": 3.488160386853621, + "tokens_seen": 1643260928 + }, + { + "epoch": 5.05, + "objective/train/docs_used": 1830619, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8538637161254883, + "objective/train/theoretical_loss": 3.488151348043624, + "objective/train/tokens_used": 1663770080, + "theoretical_loss": 3.488151348043624, + "tokens_seen": 1643310080 + }, + { + "epoch": 5.05, + "learning_rate": 0.0002536208625877633, + "loss": 2.7363, + "theoretical_loss": 3.4881483351838583, + "tokens_seen": 1643326464 + }, + { + "epoch": 5.05, + "learning_rate": 0.0002536108324974925, + "loss": 2.5418, + "theoretical_loss": 3.488136284129276, + "tokens_seen": 1643392000 + }, + { + "epoch": 5.05, + "learning_rate": 0.00025360080240722165, + "loss": 2.5203, + "theoretical_loss": 3.4881242336898177, + "tokens_seen": 1643457536 + }, + { + "epoch": 5.05, + "learning_rate": 0.0002535907723169509, + "loss": 2.7345, + "theoretical_loss": 3.488112183865427, + "tokens_seen": 1643523072 + }, + { + "epoch": 5.05, + "learning_rate": 0.00025358074222668, + "loss": 2.5824, + "theoretical_loss": 3.488100134656049, + "tokens_seen": 1643588608 + }, + { + "epoch": 5.05, + "learning_rate": 0.00025357071213640925, + "loss": 2.6666, + "theoretical_loss": 3.4880880860616275, + "tokens_seen": 1643654144 + }, + { + "epoch": 5.05, + "learning_rate": 0.00025356068204613843, + "loss": 2.7623, + "theoretical_loss": 3.488076038082106, + "tokens_seen": 1643719680 + }, + { + "epoch": 5.05, + "learning_rate": 0.0002535506519558676, + "loss": 2.6642, + "theoretical_loss": 3.4880639907174285, + "tokens_seen": 1643785216 + }, + { + "epoch": 5.05, + "learning_rate": 0.0002535406218655968, + "loss": 2.5737, + "theoretical_loss": 3.48805194396754, + "tokens_seen": 1643850752 + }, + { + "epoch": 5.05, + "learning_rate": 0.00025353059177532603, + "loss": 2.5808, + "theoretical_loss": 3.488039897832385, + "tokens_seen": 1643916288 + }, + { + "epoch": 5.05, + "learning_rate": 0.00025352056168505516, + "loss": 2.6431, + "theoretical_loss": 3.488027852311906, + "tokens_seen": 1643981824 + }, + { + "epoch": 5.05, + "learning_rate": 0.0002535105315947844, + "loss": 2.4755, + "theoretical_loss": 3.488015807406048, + "tokens_seen": 1644047360 + }, + { + "epoch": 5.05, + "learning_rate": 0.0002535005015045135, + "loss": 2.6564, + "theoretical_loss": 3.4880037631147554, + "tokens_seen": 1644112896 + }, + { + "epoch": 5.05, + "learning_rate": 0.00025349047141424275, + "loss": 2.6924, + "theoretical_loss": 3.487991719437972, + "tokens_seen": 1644178432 + }, + { + "epoch": 5.05, + "learning_rate": 0.00025348044132397193, + "loss": 2.6345, + "theoretical_loss": 3.487979676375642, + "tokens_seen": 1644243968 + }, + { + "epoch": 5.05, + "learning_rate": 0.0002534704112337011, + "loss": 2.4965, + "theoretical_loss": 3.4879676339277097, + "tokens_seen": 1644309504 + }, + { + "epoch": 5.05, + "learning_rate": 0.0002534603811434303, + "loss": 2.6393, + "theoretical_loss": 3.487955592094119, + "tokens_seen": 1644375040 + }, + { + "epoch": 5.05, + "learning_rate": 0.0002534503510531595, + "loss": 2.8411, + "theoretical_loss": 3.487943550874814, + "tokens_seen": 1644440576 + }, + { + "epoch": 5.05, + "learning_rate": 0.00025344032096288866, + "loss": 2.7036, + "theoretical_loss": 3.487931510269739, + "tokens_seen": 1644506112 + }, + { + "epoch": 5.05, + "learning_rate": 0.0002534302908726179, + "loss": 2.6083, + "theoretical_loss": 3.4879194702788388, + "tokens_seen": 1644571648 + }, + { + "epoch": 5.05, + "learning_rate": 0.000253420260782347, + "loss": 2.8604, + "theoretical_loss": 3.487907430902057, + "tokens_seen": 1644637184 + }, + { + "epoch": 5.05, + "learning_rate": 0.00025341023069207626, + "loss": 2.5739, + "theoretical_loss": 3.487895392139338, + "tokens_seen": 1644702720 + }, + { + "epoch": 5.05, + "learning_rate": 0.0002534002006018054, + "loss": 2.6247, + "theoretical_loss": 3.4878833539906258, + "tokens_seen": 1644768256 + }, + { + "epoch": 5.05, + "learning_rate": 0.0002533901705115346, + "loss": 2.6973, + "theoretical_loss": 3.4878713164558643, + "tokens_seen": 1644833792 + }, + { + "epoch": 5.05, + "learning_rate": 0.0002533801404212638, + "loss": 2.6411, + "theoretical_loss": 3.487859279534999, + "tokens_seen": 1644899328 + }, + { + "epoch": 5.05, + "objective/train/docs_used": 1831835, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8763232231140137, + "objective/train/theoretical_loss": 3.487850252247185, + "objective/train/tokens_used": 1665408480, + "theoretical_loss": 3.487850252247185, + "tokens_seen": 1644948480 + }, + { + "epoch": 5.05, + "learning_rate": 0.000253370110330993, + "loss": 2.8589, + "theoretical_loss": 3.487847243227973, + "tokens_seen": 1644964864 + }, + { + "epoch": 5.05, + "learning_rate": 0.00025336008024072216, + "loss": 2.5824, + "theoretical_loss": 3.4878352075347308, + "tokens_seen": 1645030400 + }, + { + "epoch": 5.05, + "learning_rate": 0.0002533500501504514, + "loss": 2.7122, + "theoretical_loss": 3.487823172455217, + "tokens_seen": 1645095936 + }, + { + "epoch": 5.05, + "learning_rate": 0.0002533400200601805, + "loss": 2.7071, + "theoretical_loss": 3.487811137989375, + "tokens_seen": 1645161472 + }, + { + "epoch": 5.05, + "learning_rate": 0.00025332998996990976, + "loss": 2.5932, + "theoretical_loss": 3.4877991041371494, + "tokens_seen": 1645227008 + }, + { + "epoch": 5.05, + "learning_rate": 0.0002533199598796389, + "loss": 2.7684, + "theoretical_loss": 3.4877870708984853, + "tokens_seen": 1645292544 + }, + { + "epoch": 5.05, + "learning_rate": 0.0002533099297893681, + "loss": 2.6536, + "theoretical_loss": 3.4877750382733264, + "tokens_seen": 1645358080 + }, + { + "epoch": 5.05, + "learning_rate": 0.0002532998996990973, + "loss": 2.5601, + "theoretical_loss": 3.4877630062616167, + "tokens_seen": 1645423616 + }, + { + "epoch": 5.05, + "learning_rate": 0.0002532898696088265, + "loss": 2.557, + "theoretical_loss": 3.487750974863301, + "tokens_seen": 1645489152 + }, + { + "epoch": 5.05, + "learning_rate": 0.00025327983951855566, + "loss": 2.7465, + "theoretical_loss": 3.4877389440783233, + "tokens_seen": 1645554688 + }, + { + "epoch": 5.05, + "learning_rate": 0.00025326980942828485, + "loss": 2.2815, + "theoretical_loss": 3.4877269139066276, + "tokens_seen": 1645620224 + }, + { + "epoch": 5.05, + "learning_rate": 0.000253259779338014, + "loss": 2.5423, + "theoretical_loss": 3.4877148843481587, + "tokens_seen": 1645685760 + }, + { + "epoch": 5.05, + "learning_rate": 0.00025324974924774326, + "loss": 2.7766, + "theoretical_loss": 3.487702855402861, + "tokens_seen": 1645751296 + }, + { + "epoch": 5.05, + "learning_rate": 0.0002532397191574724, + "loss": 2.895, + "theoretical_loss": 3.4876908270706783, + "tokens_seen": 1645816832 + }, + { + "epoch": 5.05, + "learning_rate": 0.0002532296890672016, + "loss": 2.8808, + "theoretical_loss": 3.487678799351556, + "tokens_seen": 1645882368 + }, + { + "epoch": 5.05, + "learning_rate": 0.00025321965897693075, + "loss": 2.5823, + "theoretical_loss": 3.4876667722454373, + "tokens_seen": 1645947904 + }, + { + "epoch": 5.05, + "learning_rate": 0.00025320962888666, + "loss": 2.641, + "theoretical_loss": 3.487654745752267, + "tokens_seen": 1646013440 + }, + { + "epoch": 5.05, + "learning_rate": 0.00025319959879638917, + "loss": 2.5569, + "theoretical_loss": 3.487642719871989, + "tokens_seen": 1646078976 + }, + { + "epoch": 5.05, + "learning_rate": 0.00025318956870611835, + "loss": 2.7729, + "theoretical_loss": 3.4876306946045483, + "tokens_seen": 1646144512 + }, + { + "epoch": 5.05, + "learning_rate": 0.00025317953861584753, + "loss": 2.7548, + "theoretical_loss": 3.4876186699498897, + "tokens_seen": 1646210048 + }, + { + "epoch": 5.05, + "learning_rate": 0.00025316950852557677, + "loss": 2.6345, + "theoretical_loss": 3.487606645907956, + "tokens_seen": 1646275584 + }, + { + "epoch": 5.05, + "learning_rate": 0.0002531594784353059, + "loss": 2.5346, + "theoretical_loss": 3.487594622478693, + "tokens_seen": 1646341120 + }, + { + "epoch": 5.05, + "learning_rate": 0.00025314944834503513, + "loss": 2.7141, + "theoretical_loss": 3.487582599662045, + "tokens_seen": 1646406656 + }, + { + "epoch": 5.05, + "learning_rate": 0.00025313941825476425, + "loss": 2.7156, + "theoretical_loss": 3.4875705774579555, + "tokens_seen": 1646472192 + }, + { + "epoch": 5.05, + "learning_rate": 0.0002531293881644935, + "loss": 2.6339, + "theoretical_loss": 3.4875585558663698, + "tokens_seen": 1646537728 + }, + { + "epoch": 5.05, + "objective/train/docs_used": 1832689, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.549023389816284, + "objective/train/theoretical_loss": 3.4875495400746024, + "objective/train/tokens_used": 1667046880, + "theoretical_loss": 3.4875495400746024, + "tokens_seen": 1646586880 + }, + { + "epoch": 5.05, + "learning_rate": 0.00025311935807422267, + "loss": 2.5987, + "theoretical_loss": 3.4875465348872314, + "tokens_seen": 1646603264 + }, + { + "epoch": 5.05, + "learning_rate": 0.00025310932798395185, + "loss": 2.8123, + "theoretical_loss": 3.487534514520486, + "tokens_seen": 1646668800 + }, + { + "epoch": 5.05, + "learning_rate": 0.00025309929789368103, + "loss": 2.6352, + "theoretical_loss": 3.487522494766077, + "tokens_seen": 1646734336 + }, + { + "epoch": 5.05, + "learning_rate": 0.0002530892678034102, + "loss": 2.5625, + "theoretical_loss": 3.4875104756239486, + "tokens_seen": 1646799872 + }, + { + "epoch": 5.05, + "learning_rate": 0.0002530792377131394, + "loss": 2.5651, + "theoretical_loss": 3.4874984570940466, + "tokens_seen": 1646865408 + }, + { + "epoch": 5.05, + "learning_rate": 0.00025306920762286863, + "loss": 2.6599, + "theoretical_loss": 3.4874864391763145, + "tokens_seen": 1646930944 + }, + { + "epoch": 5.05, + "learning_rate": 0.00025305917753259776, + "loss": 2.5127, + "theoretical_loss": 3.4874744218706963, + "tokens_seen": 1646996480 + }, + { + "epoch": 5.05, + "learning_rate": 0.000253049147442327, + "loss": 2.4157, + "theoretical_loss": 3.487462405177138, + "tokens_seen": 1647062016 + }, + { + "epoch": 5.05, + "learning_rate": 0.0002530391173520562, + "loss": 2.5339, + "theoretical_loss": 3.4874503890955824, + "tokens_seen": 1647127552 + }, + { + "epoch": 5.05, + "learning_rate": 0.00025302908726178536, + "loss": 2.5977, + "theoretical_loss": 3.487438373625975, + "tokens_seen": 1647193088 + }, + { + "epoch": 5.05, + "learning_rate": 0.00025301905717151454, + "loss": 2.7327, + "theoretical_loss": 3.48742635876826, + "tokens_seen": 1647258624 + }, + { + "epoch": 5.05, + "learning_rate": 0.0002530090270812437, + "loss": 2.7865, + "theoretical_loss": 3.4874143445223824, + "tokens_seen": 1647324160 + }, + { + "epoch": 5.05, + "learning_rate": 0.00025299899699097295, + "loss": 2.6948, + "theoretical_loss": 3.4874023308882856, + "tokens_seen": 1647389696 + }, + { + "epoch": 5.05, + "learning_rate": 0.00025298896690070213, + "loss": 2.7051, + "theoretical_loss": 3.487390317865915, + "tokens_seen": 1647455232 + }, + { + "epoch": 5.05, + "learning_rate": 0.0002529789368104313, + "loss": 2.6647, + "theoretical_loss": 3.4873783054552145, + "tokens_seen": 1647520768 + }, + { + "epoch": 5.05, + "learning_rate": 0.0002529689067201605, + "loss": 2.564, + "theoretical_loss": 3.4873662936561294, + "tokens_seen": 1647586304 + }, + { + "epoch": 5.05, + "learning_rate": 0.0002529588766298897, + "loss": 2.7542, + "theoretical_loss": 3.487354282468604, + "tokens_seen": 1647651840 + }, + { + "epoch": 5.05, + "learning_rate": 0.00025294884653961886, + "loss": 2.3254, + "theoretical_loss": 3.487342271892582, + "tokens_seen": 1647717376 + }, + { + "epoch": 5.05, + "learning_rate": 0.0002529388164493481, + "loss": 2.5738, + "theoretical_loss": 3.487330261928009, + "tokens_seen": 1647782912 + }, + { + "epoch": 5.05, + "learning_rate": 0.0002529287863590772, + "loss": 2.679, + "theoretical_loss": 3.4873182525748296, + "tokens_seen": 1647848448 + }, + { + "epoch": 5.05, + "learning_rate": 0.00025291875626880646, + "loss": 2.4659, + "theoretical_loss": 3.4873062438329874, + "tokens_seen": 1647913984 + }, + { + "epoch": 5.05, + "learning_rate": 0.0002529087261785356, + "loss": 2.5778, + "theoretical_loss": 3.4872942357024277, + "tokens_seen": 1647979520 + }, + { + "epoch": 5.05, + "learning_rate": 0.0002528986960882648, + "loss": 2.4391, + "theoretical_loss": 3.4872822281830946, + "tokens_seen": 1648045056 + }, + { + "epoch": 5.05, + "learning_rate": 0.000252888665997994, + "loss": 2.6824, + "theoretical_loss": 3.487270221274933, + "tokens_seen": 1648110592 + }, + { + "epoch": 5.05, + "learning_rate": 0.0002528786359077232, + "loss": 2.7971, + "theoretical_loss": 3.4872582149778877, + "tokens_seen": 1648176128 + }, + { + "epoch": 5.05, + "objective/train/docs_used": 1834029, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5318849086761475, + "objective/train/theoretical_loss": 3.487249210656115, + "objective/train/tokens_used": 1668685280, + "theoretical_loss": 3.487249210656115, + "tokens_seen": 1648225280 + }, + { + "epoch": 5.05, + "learning_rate": 0.00025286860581745236, + "loss": 2.652, + "theoretical_loss": 3.4872462092919028, + "tokens_seen": 1648241664 + }, + { + "epoch": 5.05, + "learning_rate": 0.0002528585757271816, + "loss": 2.6296, + "theoretical_loss": 3.4872342042169233, + "tokens_seen": 1648307200 + }, + { + "epoch": 5.05, + "learning_rate": 0.0002528485456369107, + "loss": 2.4941, + "theoretical_loss": 3.4872221997528934, + "tokens_seen": 1648372736 + }, + { + "epoch": 5.05, + "learning_rate": 0.00025283851554663996, + "loss": 2.4465, + "theoretical_loss": 3.4872101958997583, + "tokens_seen": 1648438272 + }, + { + "epoch": 5.05, + "learning_rate": 0.0002528284854563691, + "loss": 2.3638, + "theoretical_loss": 3.4871981926574622, + "tokens_seen": 1648503808 + }, + { + "epoch": 5.05, + "learning_rate": 0.0002528184553660983, + "loss": 2.703, + "theoretical_loss": 3.48718619002595, + "tokens_seen": 1648569344 + }, + { + "epoch": 5.05, + "learning_rate": 0.0002528084252758275, + "loss": 2.6302, + "theoretical_loss": 3.4871741880051657, + "tokens_seen": 1648634880 + }, + { + "epoch": 5.05, + "learning_rate": 0.0002527983951855567, + "loss": 2.5522, + "theoretical_loss": 3.487162186595055, + "tokens_seen": 1648700416 + }, + { + "epoch": 5.05, + "learning_rate": 0.00025278836509528586, + "loss": 2.687, + "theoretical_loss": 3.487150185795562, + "tokens_seen": 1648765952 + }, + { + "epoch": 5.05, + "learning_rate": 0.00025277833500501505, + "loss": 2.5756, + "theoretical_loss": 3.487138185606631, + "tokens_seen": 1648831488 + }, + { + "epoch": 5.05, + "learning_rate": 0.0002527683049147442, + "loss": 2.7657, + "theoretical_loss": 3.487126186028207, + "tokens_seen": 1648897024 + }, + { + "epoch": 5.05, + "learning_rate": 0.00025275827482447346, + "loss": 2.6197, + "theoretical_loss": 3.487114187060235, + "tokens_seen": 1648962560 + }, + { + "epoch": 5.05, + "learning_rate": 0.0002527482447342026, + "loss": 2.3855, + "theoretical_loss": 3.487102188702659, + "tokens_seen": 1649028096 + }, + { + "epoch": 5.05, + "learning_rate": 0.0002527382146439318, + "loss": 2.6836, + "theoretical_loss": 3.487090190955424, + "tokens_seen": 1649093632 + }, + { + "epoch": 5.05, + "learning_rate": 0.00025272818455366095, + "loss": 2.6625, + "theoretical_loss": 3.4870781938184754, + "tokens_seen": 1649159168 + }, + { + "epoch": 5.05, + "learning_rate": 0.0002527181544633902, + "loss": 2.639, + "theoretical_loss": 3.4870661972917567, + "tokens_seen": 1649224704 + }, + { + "epoch": 5.05, + "learning_rate": 0.00025270812437311937, + "loss": 2.6155, + "theoretical_loss": 3.4870542013752135, + "tokens_seen": 1649290240 + }, + { + "epoch": 5.05, + "learning_rate": 0.00025269809428284855, + "loss": 2.753, + "theoretical_loss": 3.4870422060687902, + "tokens_seen": 1649355776 + }, + { + "epoch": 5.05, + "learning_rate": 0.00025268806419257773, + "loss": 2.4928, + "theoretical_loss": 3.4870302113724314, + "tokens_seen": 1649421312 + }, + { + "epoch": 5.05, + "learning_rate": 0.00025267803410230697, + "loss": 2.6719, + "theoretical_loss": 3.4870182172860824, + "tokens_seen": 1649486848 + }, + { + "epoch": 5.05, + "learning_rate": 0.0002526680040120361, + "loss": 2.5814, + "theoretical_loss": 3.4870062238096873, + "tokens_seen": 1649552384 + }, + { + "epoch": 5.05, + "learning_rate": 0.00025265797392176533, + "loss": 2.3619, + "theoretical_loss": 3.486994230943191, + "tokens_seen": 1649617920 + }, + { + "epoch": 5.05, + "learning_rate": 0.00025264794383149445, + "loss": 2.5583, + "theoretical_loss": 3.4869822386865383, + "tokens_seen": 1649683456 + }, + { + "epoch": 5.05, + "learning_rate": 0.0002526379137412237, + "loss": 2.5248, + "theoretical_loss": 3.4869702470396735, + "tokens_seen": 1649748992 + }, + { + "epoch": 5.05, + "learning_rate": 0.00025262788365095287, + "loss": 2.7687, + "theoretical_loss": 3.4869582560025423, + "tokens_seen": 1649814528 + }, + { + "epoch": 5.05, + "objective/train/docs_used": 1834679, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.2152130603790283, + "objective/train/theoretical_loss": 3.4869492631247985, + "objective/train/tokens_used": 1670323680, + "theoretical_loss": 3.4869492631247985, + "tokens_seen": 1649863680 + }, + { + "epoch": 5.05, + "learning_rate": 0.00025261785356068205, + "loss": 2.7612, + "theoretical_loss": 3.4869462655750896, + "tokens_seen": 1649880064 + }, + { + "epoch": 5.05, + "learning_rate": 0.00025260782347041123, + "loss": 2.6469, + "theoretical_loss": 3.486934275757259, + "tokens_seen": 1649945600 + }, + { + "epoch": 5.05, + "learning_rate": 0.0002525977933801404, + "loss": 2.5645, + "theoretical_loss": 3.486922286548996, + "tokens_seen": 1650011136 + }, + { + "epoch": 5.05, + "learning_rate": 0.0002525877632898696, + "loss": 2.5006, + "theoretical_loss": 3.4869102979502458, + "tokens_seen": 1650076672 + } + ], + "max_steps": 50354, + "num_train_epochs": 9223372036854775807, + "total_flos": 8.42093528481792e+17, + "trial_name": null, + "trial_params": null +}