| { | |
| "best_metric": null, | |
| "best_model_checkpoint": null, | |
| "epoch": 0.9056603773584906, | |
| "eval_steps": 500, | |
| "global_step": 120, | |
| "is_hyper_param_search": false, | |
| "is_local_process_zero": true, | |
| "is_world_process_zero": true, | |
| "log_history": [ | |
| { | |
| "epoch": 0.007547169811320755, | |
| "grad_norm": 4.023828029632568, | |
| "learning_rate": 1.8518518518518518e-07, | |
| "logps/chosen": -29.54261589050293, | |
| "logps/rejected": -34.2619514465332, | |
| "loss": 0.6928, | |
| "losses/dpo": 0.6891345977783203, | |
| "losses/sft": 1.0977673530578613, | |
| "losses/total": 0.6891345977783203, | |
| "ref_logps/chosen": -29.5439395904541, | |
| "ref_logps/rejected": -34.251976013183594, | |
| "rewards/accuracies": 0.515625, | |
| "rewards/chosen": 0.00013234722428023815, | |
| "rewards/margins": 0.0011300418991595507, | |
| "rewards/rejected": -0.0009976944420486689, | |
| "step": 1 | |
| }, | |
| { | |
| "epoch": 0.01509433962264151, | |
| "grad_norm": 3.774498701095581, | |
| "learning_rate": 3.7037037037037036e-07, | |
| "logps/chosen": -28.00726890563965, | |
| "logps/rejected": -34.31509017944336, | |
| "loss": 0.6933, | |
| "losses/dpo": 0.6932892799377441, | |
| "losses/sft": 1.1102490425109863, | |
| "losses/total": 0.6932892799377441, | |
| "ref_logps/chosen": -27.994796752929688, | |
| "ref_logps/rejected": -34.30337905883789, | |
| "rewards/accuracies": 0.4921875, | |
| "rewards/chosen": -0.0012474276591092348, | |
| "rewards/margins": -7.60925468057394e-05, | |
| "rewards/rejected": -0.0011713354615494609, | |
| "step": 2 | |
| }, | |
| { | |
| "epoch": 0.022641509433962263, | |
| "grad_norm": 4.361927032470703, | |
| "learning_rate": 5.555555555555555e-07, | |
| "logps/chosen": -32.22264099121094, | |
| "logps/rejected": -39.77468490600586, | |
| "loss": 0.6923, | |
| "losses/dpo": 0.6875869035720825, | |
| "losses/sft": 1.1771142482757568, | |
| "losses/total": 0.6875869035720825, | |
| "ref_logps/chosen": -32.23395919799805, | |
| "ref_logps/rejected": -39.76538848876953, | |
| "rewards/accuracies": 0.546875, | |
| "rewards/chosen": 0.0011316344607621431, | |
| "rewards/margins": 0.002061467617750168, | |
| "rewards/rejected": -0.0009298332734033465, | |
| "step": 3 | |
| }, | |
| { | |
| "epoch": 0.03018867924528302, | |
| "grad_norm": 4.030606746673584, | |
| "learning_rate": 7.407407407407407e-07, | |
| "logps/chosen": -29.95352554321289, | |
| "logps/rejected": -35.96381378173828, | |
| "loss": 0.6931, | |
| "losses/dpo": 0.6984821557998657, | |
| "losses/sft": 1.119036078453064, | |
| "losses/total": 0.6984821557998657, | |
| "ref_logps/chosen": -29.93365478515625, | |
| "ref_logps/rejected": -35.9401969909668, | |
| "rewards/accuracies": 0.53125, | |
| "rewards/chosen": -0.0019873534329235554, | |
| "rewards/margins": 0.00037475768476724625, | |
| "rewards/rejected": -0.002362110884860158, | |
| "step": 4 | |
| }, | |
| { | |
| "epoch": 0.03773584905660377, | |
| "grad_norm": 4.286405563354492, | |
| "learning_rate": 9.259259259259259e-07, | |
| "logps/chosen": -33.54985046386719, | |
| "logps/rejected": -38.25576400756836, | |
| "loss": 0.6932, | |
| "losses/dpo": 0.6964197158813477, | |
| "losses/sft": 0.9851586818695068, | |
| "losses/total": 0.6964197158813477, | |
| "ref_logps/chosen": -33.534912109375, | |
| "ref_logps/rejected": -38.2386474609375, | |
| "rewards/accuracies": 0.484375, | |
| "rewards/chosen": -0.0014938730746507645, | |
| "rewards/margins": 0.00021800631657242775, | |
| "rewards/rejected": -0.0017118793912231922, | |
| "step": 5 | |
| }, | |
| { | |
| "epoch": 0.045283018867924525, | |
| "grad_norm": 3.715064764022827, | |
| "learning_rate": 1.111111111111111e-06, | |
| "logps/chosen": -28.753326416015625, | |
| "logps/rejected": -33.94843292236328, | |
| "loss": 0.6956, | |
| "losses/dpo": 0.696850061416626, | |
| "losses/sft": 0.9852187633514404, | |
| "losses/total": 0.696850061416626, | |
| "ref_logps/chosen": -28.712608337402344, | |
| "ref_logps/rejected": -33.953330993652344, | |
| "rewards/accuracies": 0.4375, | |
| "rewards/chosen": -0.004071743693202734, | |
| "rewards/margins": -0.0045617930591106415, | |
| "rewards/rejected": 0.0004900495987385511, | |
| "step": 6 | |
| }, | |
| { | |
| "epoch": 0.052830188679245285, | |
| "grad_norm": 4.02261209487915, | |
| "learning_rate": 1.2962962962962962e-06, | |
| "logps/chosen": -29.45867347717285, | |
| "logps/rejected": -34.641387939453125, | |
| "loss": 0.6942, | |
| "losses/dpo": 0.6925374269485474, | |
| "losses/sft": 1.0857254266738892, | |
| "losses/total": 0.6925374269485474, | |
| "ref_logps/chosen": -29.440855026245117, | |
| "ref_logps/rejected": -34.63925552368164, | |
| "rewards/accuracies": 0.4765625, | |
| "rewards/chosen": -0.001781845698133111, | |
| "rewards/margins": -0.0015682985540479422, | |
| "rewards/rejected": -0.00021354644559323788, | |
| "step": 7 | |
| }, | |
| { | |
| "epoch": 0.06037735849056604, | |
| "grad_norm": 4.2059502601623535, | |
| "learning_rate": 1.4814814814814815e-06, | |
| "logps/chosen": -34.3134651184082, | |
| "logps/rejected": -35.359073638916016, | |
| "loss": 0.693, | |
| "losses/dpo": 0.6923079490661621, | |
| "losses/sft": 1.0715954303741455, | |
| "losses/total": 0.6923079490661621, | |
| "ref_logps/chosen": -34.33416748046875, | |
| "ref_logps/rejected": -35.37400817871094, | |
| "rewards/accuracies": 0.5234375, | |
| "rewards/chosen": 0.0020700739696621895, | |
| "rewards/margins": 0.000576640130020678, | |
| "rewards/rejected": 0.001493434072472155, | |
| "step": 8 | |
| }, | |
| { | |
| "epoch": 0.06792452830188679, | |
| "grad_norm": 5.0255889892578125, | |
| "learning_rate": 1.6666666666666667e-06, | |
| "logps/chosen": -30.589462280273438, | |
| "logps/rejected": -39.17620849609375, | |
| "loss": 0.6948, | |
| "losses/dpo": 0.6894321441650391, | |
| "losses/sft": 0.8859687447547913, | |
| "losses/total": 0.6894321441650391, | |
| "ref_logps/chosen": -30.609333038330078, | |
| "ref_logps/rejected": -39.22538757324219, | |
| "rewards/accuracies": 0.5390625, | |
| "rewards/chosen": 0.0019869431853294373, | |
| "rewards/margins": -0.002930472604930401, | |
| "rewards/rejected": 0.004917416721582413, | |
| "step": 9 | |
| }, | |
| { | |
| "epoch": 0.07547169811320754, | |
| "grad_norm": 4.278906345367432, | |
| "learning_rate": 1.8518518518518519e-06, | |
| "logps/chosen": -26.85662841796875, | |
| "logps/rejected": -34.258155822753906, | |
| "loss": 0.6886, | |
| "losses/dpo": 0.6904473304748535, | |
| "losses/sft": 1.016200304031372, | |
| "losses/total": 0.6904473304748535, | |
| "ref_logps/chosen": -27.009971618652344, | |
| "ref_logps/rejected": -34.316612243652344, | |
| "rewards/accuracies": 0.578125, | |
| "rewards/chosen": 0.015334523282945156, | |
| "rewards/margins": 0.009488685056567192, | |
| "rewards/rejected": 0.005845838226377964, | |
| "step": 10 | |
| }, | |
| { | |
| "epoch": 0.0830188679245283, | |
| "grad_norm": 3.700289487838745, | |
| "learning_rate": 2.037037037037037e-06, | |
| "logps/chosen": -29.495962142944336, | |
| "logps/rejected": -33.246185302734375, | |
| "loss": 0.6915, | |
| "losses/dpo": 0.6893143653869629, | |
| "losses/sft": 1.2568368911743164, | |
| "losses/total": 0.6893143653869629, | |
| "ref_logps/chosen": -29.658611297607422, | |
| "ref_logps/rejected": -33.370635986328125, | |
| "rewards/accuracies": 0.5390625, | |
| "rewards/chosen": 0.01626480370759964, | |
| "rewards/margins": 0.003819521516561508, | |
| "rewards/rejected": 0.012445282191038132, | |
| "step": 11 | |
| }, | |
| { | |
| "epoch": 0.09056603773584905, | |
| "grad_norm": 3.853537082672119, | |
| "learning_rate": 2.222222222222222e-06, | |
| "logps/chosen": -30.57427215576172, | |
| "logps/rejected": -34.356842041015625, | |
| "loss": 0.6943, | |
| "losses/dpo": 0.702874481678009, | |
| "losses/sft": 1.008819580078125, | |
| "losses/total": 0.702874481678009, | |
| "ref_logps/chosen": -30.768930435180664, | |
| "ref_logps/rejected": -34.566131591796875, | |
| "rewards/accuracies": 0.5234375, | |
| "rewards/chosen": 0.01946595311164856, | |
| "rewards/margins": -0.0014625652693212032, | |
| "rewards/rejected": 0.0209285207092762, | |
| "step": 12 | |
| }, | |
| { | |
| "epoch": 0.09811320754716982, | |
| "grad_norm": 4.208248138427734, | |
| "learning_rate": 2.4074074074074075e-06, | |
| "logps/chosen": -30.429744720458984, | |
| "logps/rejected": -39.09281539916992, | |
| "loss": 0.6895, | |
| "losses/dpo": 0.7104582786560059, | |
| "losses/sft": 1.0792236328125, | |
| "losses/total": 0.7104582786560059, | |
| "ref_logps/chosen": -30.72281265258789, | |
| "ref_logps/rejected": -39.29808044433594, | |
| "rewards/accuracies": 0.578125, | |
| "rewards/chosen": 0.029306560754776, | |
| "rewards/margins": 0.00877977255731821, | |
| "rewards/rejected": 0.020526789128780365, | |
| "step": 13 | |
| }, | |
| { | |
| "epoch": 0.10566037735849057, | |
| "grad_norm": 3.7299113273620605, | |
| "learning_rate": 2.5925925925925925e-06, | |
| "logps/chosen": -27.8311710357666, | |
| "logps/rejected": -33.181427001953125, | |
| "loss": 0.6862, | |
| "losses/dpo": 0.6882991790771484, | |
| "losses/sft": 1.0185338258743286, | |
| "losses/total": 0.6882991790771484, | |
| "ref_logps/chosen": -28.355030059814453, | |
| "ref_logps/rejected": -33.544822692871094, | |
| "rewards/accuracies": 0.5625, | |
| "rewards/chosen": 0.052385710179805756, | |
| "rewards/margins": 0.016046730801463127, | |
| "rewards/rejected": 0.03633897751569748, | |
| "step": 14 | |
| }, | |
| { | |
| "epoch": 0.11320754716981132, | |
| "grad_norm": 3.8657593727111816, | |
| "learning_rate": 2.7777777777777783e-06, | |
| "logps/chosen": -28.884387969970703, | |
| "logps/rejected": -34.36138916015625, | |
| "loss": 0.6913, | |
| "losses/dpo": 0.6939971446990967, | |
| "losses/sft": 1.0502238273620605, | |
| "losses/total": 0.6939971446990967, | |
| "ref_logps/chosen": -29.45929718017578, | |
| "ref_logps/rejected": -34.86326599121094, | |
| "rewards/accuracies": 0.5703125, | |
| "rewards/chosen": 0.05749073624610901, | |
| "rewards/margins": 0.0073026856407523155, | |
| "rewards/rejected": 0.05018804967403412, | |
| "step": 15 | |
| }, | |
| { | |
| "epoch": 0.12075471698113208, | |
| "grad_norm": 3.9700570106506348, | |
| "learning_rate": 2.962962962962963e-06, | |
| "logps/chosen": -30.124591827392578, | |
| "logps/rejected": -31.684345245361328, | |
| "loss": 0.6928, | |
| "losses/dpo": 0.6692728996276855, | |
| "losses/sft": 1.005508542060852, | |
| "losses/total": 0.6692728996276855, | |
| "ref_logps/chosen": -30.774919509887695, | |
| "ref_logps/rejected": -32.27117919921875, | |
| "rewards/accuracies": 0.515625, | |
| "rewards/chosen": 0.06503286957740784, | |
| "rewards/margins": 0.00634903647005558, | |
| "rewards/rejected": 0.058683834969997406, | |
| "step": 16 | |
| }, | |
| { | |
| "epoch": 0.12830188679245283, | |
| "grad_norm": 3.9416866302490234, | |
| "learning_rate": 3.1481481481481483e-06, | |
| "logps/chosen": -30.775650024414062, | |
| "logps/rejected": -34.58787155151367, | |
| "loss": 0.6834, | |
| "losses/dpo": 0.7104189395904541, | |
| "losses/sft": 0.9935987591743469, | |
| "losses/total": 0.7104189395904541, | |
| "ref_logps/chosen": -31.45960235595703, | |
| "ref_logps/rejected": -35.01028060913086, | |
| "rewards/accuracies": 0.5625, | |
| "rewards/chosen": 0.0683954730629921, | |
| "rewards/margins": 0.0261550135910511, | |
| "rewards/rejected": 0.042240455746650696, | |
| "step": 17 | |
| }, | |
| { | |
| "epoch": 0.13584905660377358, | |
| "grad_norm": 3.8401975631713867, | |
| "learning_rate": 3.3333333333333333e-06, | |
| "logps/chosen": -29.183441162109375, | |
| "logps/rejected": -33.91481018066406, | |
| "loss": 0.6689, | |
| "losses/dpo": 0.6731781363487244, | |
| "losses/sft": 1.0918514728546143, | |
| "losses/total": 0.6731781363487244, | |
| "ref_logps/chosen": -30.14259147644043, | |
| "ref_logps/rejected": -34.299861907958984, | |
| "rewards/accuracies": 0.6328125, | |
| "rewards/chosen": 0.09591513872146606, | |
| "rewards/margins": 0.057410385459661484, | |
| "rewards/rejected": 0.03850475698709488, | |
| "step": 18 | |
| }, | |
| { | |
| "epoch": 0.14339622641509434, | |
| "grad_norm": 3.793499231338501, | |
| "learning_rate": 3.5185185185185187e-06, | |
| "logps/chosen": -25.331146240234375, | |
| "logps/rejected": -34.978057861328125, | |
| "loss": 0.6692, | |
| "losses/dpo": 0.7569133639335632, | |
| "losses/sft": 0.9651777744293213, | |
| "losses/total": 0.7569133639335632, | |
| "ref_logps/chosen": -26.447914123535156, | |
| "ref_logps/rejected": -35.43275833129883, | |
| "rewards/accuracies": 0.65625, | |
| "rewards/chosen": 0.1116766482591629, | |
| "rewards/margins": 0.06620671600103378, | |
| "rewards/rejected": 0.04546992480754852, | |
| "step": 19 | |
| }, | |
| { | |
| "epoch": 0.1509433962264151, | |
| "grad_norm": 3.7824015617370605, | |
| "learning_rate": 3.7037037037037037e-06, | |
| "logps/chosen": -29.415006637573242, | |
| "logps/rejected": -34.36804962158203, | |
| "loss": 0.6666, | |
| "losses/dpo": 0.6621171832084656, | |
| "losses/sft": 0.7287149429321289, | |
| "losses/total": 0.6621171832084656, | |
| "ref_logps/chosen": -30.477230072021484, | |
| "ref_logps/rejected": -34.688987731933594, | |
| "rewards/accuracies": 0.6015625, | |
| "rewards/chosen": 0.10622246563434601, | |
| "rewards/margins": 0.0741288959980011, | |
| "rewards/rejected": 0.03209357336163521, | |
| "step": 20 | |
| }, | |
| { | |
| "epoch": 0.15849056603773584, | |
| "grad_norm": 4.047840595245361, | |
| "learning_rate": 3.88888888888889e-06, | |
| "logps/chosen": -29.250553131103516, | |
| "logps/rejected": -39.266815185546875, | |
| "loss": 0.6475, | |
| "losses/dpo": 0.647861123085022, | |
| "losses/sft": 0.9502246379852295, | |
| "losses/total": 0.647861123085022, | |
| "ref_logps/chosen": -30.495431900024414, | |
| "ref_logps/rejected": -39.307708740234375, | |
| "rewards/accuracies": 0.609375, | |
| "rewards/chosen": 0.12448810040950775, | |
| "rewards/margins": 0.12039919942617416, | |
| "rewards/rejected": 0.0040889037773013115, | |
| "step": 21 | |
| }, | |
| { | |
| "epoch": 0.1660377358490566, | |
| "grad_norm": 4.19422721862793, | |
| "learning_rate": 4.074074074074074e-06, | |
| "logps/chosen": -27.82355308532715, | |
| "logps/rejected": -35.92847442626953, | |
| "loss": 0.6735, | |
| "losses/dpo": 0.5619722604751587, | |
| "losses/sft": 0.8959212899208069, | |
| "losses/total": 0.5619722604751587, | |
| "ref_logps/chosen": -29.235668182373047, | |
| "ref_logps/rejected": -36.55792999267578, | |
| "rewards/accuracies": 0.5859375, | |
| "rewards/chosen": 0.14121143519878387, | |
| "rewards/margins": 0.07826600223779678, | |
| "rewards/rejected": 0.06294544041156769, | |
| "step": 22 | |
| }, | |
| { | |
| "epoch": 0.17358490566037735, | |
| "grad_norm": 3.807786464691162, | |
| "learning_rate": 4.2592592592592596e-06, | |
| "logps/chosen": -27.979816436767578, | |
| "logps/rejected": -38.823341369628906, | |
| "loss": 0.6627, | |
| "losses/dpo": 0.6460346579551697, | |
| "losses/sft": 1.1445313692092896, | |
| "losses/total": 0.6460346579551697, | |
| "ref_logps/chosen": -28.973108291625977, | |
| "ref_logps/rejected": -38.76581573486328, | |
| "rewards/accuracies": 0.5859375, | |
| "rewards/chosen": 0.09932918101549149, | |
| "rewards/margins": 0.10508126765489578, | |
| "rewards/rejected": -0.005752084776759148, | |
| "step": 23 | |
| }, | |
| { | |
| "epoch": 0.1811320754716981, | |
| "grad_norm": 4.215601921081543, | |
| "learning_rate": 4.444444444444444e-06, | |
| "logps/chosen": -30.01553726196289, | |
| "logps/rejected": -36.99456787109375, | |
| "loss": 0.6726, | |
| "losses/dpo": 0.7518600225448608, | |
| "losses/sft": 1.1231310367584229, | |
| "losses/total": 0.7518600225448608, | |
| "ref_logps/chosen": -30.876399993896484, | |
| "ref_logps/rejected": -37.021759033203125, | |
| "rewards/accuracies": 0.625, | |
| "rewards/chosen": 0.08608602732419968, | |
| "rewards/margins": 0.08336685597896576, | |
| "rewards/rejected": 0.0027191713452339172, | |
| "step": 24 | |
| }, | |
| { | |
| "epoch": 0.18867924528301888, | |
| "grad_norm": 3.991433620452881, | |
| "learning_rate": 4.62962962962963e-06, | |
| "logps/chosen": -29.20758056640625, | |
| "logps/rejected": -33.56412124633789, | |
| "loss": 0.6631, | |
| "losses/dpo": 0.5801557302474976, | |
| "losses/sft": 1.1197171211242676, | |
| "losses/total": 0.5801557302474976, | |
| "ref_logps/chosen": -30.49410629272461, | |
| "ref_logps/rejected": -33.72259521484375, | |
| "rewards/accuracies": 0.6171875, | |
| "rewards/chosen": 0.1286524534225464, | |
| "rewards/margins": 0.11280516535043716, | |
| "rewards/rejected": 0.01584728993475437, | |
| "step": 25 | |
| }, | |
| { | |
| "epoch": 0.19622641509433963, | |
| "grad_norm": 4.136443614959717, | |
| "learning_rate": 4.814814814814815e-06, | |
| "logps/chosen": -27.908275604248047, | |
| "logps/rejected": -36.784366607666016, | |
| "loss": 0.6475, | |
| "losses/dpo": 0.7332102060317993, | |
| "losses/sft": 0.7210355401039124, | |
| "losses/total": 0.7332102060317993, | |
| "ref_logps/chosen": -29.110748291015625, | |
| "ref_logps/rejected": -36.522212982177734, | |
| "rewards/accuracies": 0.6796875, | |
| "rewards/chosen": 0.12024737149477005, | |
| "rewards/margins": 0.14646272361278534, | |
| "rewards/rejected": -0.026215344667434692, | |
| "step": 26 | |
| }, | |
| { | |
| "epoch": 0.2037735849056604, | |
| "grad_norm": 3.6287527084350586, | |
| "learning_rate": 5e-06, | |
| "logps/chosen": -25.643430709838867, | |
| "logps/rejected": -33.46253204345703, | |
| "loss": 0.603, | |
| "losses/dpo": 0.6435042023658752, | |
| "losses/sft": 0.7656551599502563, | |
| "losses/total": 0.6435042023658752, | |
| "ref_logps/chosen": -27.53249740600586, | |
| "ref_logps/rejected": -32.8140869140625, | |
| "rewards/accuracies": 0.671875, | |
| "rewards/chosen": 0.18890666961669922, | |
| "rewards/margins": 0.25375133752822876, | |
| "rewards/rejected": -0.06484466791152954, | |
| "step": 27 | |
| }, | |
| { | |
| "epoch": 0.21132075471698114, | |
| "grad_norm": 5.5130720138549805, | |
| "learning_rate": 4.978902953586498e-06, | |
| "logps/chosen": -32.41269302368164, | |
| "logps/rejected": -36.27760314941406, | |
| "loss": 0.6718, | |
| "losses/dpo": 0.7667650580406189, | |
| "losses/sft": 1.180605411529541, | |
| "losses/total": 0.7667650580406189, | |
| "ref_logps/chosen": -32.947940826416016, | |
| "ref_logps/rejected": -35.710548400878906, | |
| "rewards/accuracies": 0.625, | |
| "rewards/chosen": 0.05352487415075302, | |
| "rewards/margins": 0.11023038625717163, | |
| "rewards/rejected": -0.05670551210641861, | |
| "step": 28 | |
| }, | |
| { | |
| "epoch": 0.2188679245283019, | |
| "grad_norm": 4.2082695960998535, | |
| "learning_rate": 4.957805907172996e-06, | |
| "logps/chosen": -30.38204574584961, | |
| "logps/rejected": -38.79253387451172, | |
| "loss": 0.6259, | |
| "losses/dpo": 0.6825762987136841, | |
| "losses/sft": 1.3010542392730713, | |
| "losses/total": 0.6825762987136841, | |
| "ref_logps/chosen": -31.19179916381836, | |
| "ref_logps/rejected": -37.440677642822266, | |
| "rewards/accuracies": 0.6640625, | |
| "rewards/chosen": 0.08097504824399948, | |
| "rewards/margins": 0.21616099774837494, | |
| "rewards/rejected": -0.13518595695495605, | |
| "step": 29 | |
| }, | |
| { | |
| "epoch": 0.22641509433962265, | |
| "grad_norm": 4.6551737785339355, | |
| "learning_rate": 4.936708860759495e-06, | |
| "logps/chosen": -30.135501861572266, | |
| "logps/rejected": -37.48721694946289, | |
| "loss": 0.6665, | |
| "losses/dpo": 0.5913557410240173, | |
| "losses/sft": 1.0625046491622925, | |
| "losses/total": 0.5913557410240173, | |
| "ref_logps/chosen": -30.550817489624023, | |
| "ref_logps/rejected": -36.66627502441406, | |
| "rewards/accuracies": 0.5859375, | |
| "rewards/chosen": 0.04153158515691757, | |
| "rewards/margins": 0.12362557649612427, | |
| "rewards/rejected": -0.08209399878978729, | |
| "step": 30 | |
| }, | |
| { | |
| "epoch": 0.2339622641509434, | |
| "grad_norm": 4.481552600860596, | |
| "learning_rate": 4.915611814345992e-06, | |
| "logps/chosen": -29.738265991210938, | |
| "logps/rejected": -40.146636962890625, | |
| "loss": 0.6341, | |
| "losses/dpo": 0.6154491305351257, | |
| "losses/sft": 0.7811817526817322, | |
| "losses/total": 0.6154491305351257, | |
| "ref_logps/chosen": -29.985429763793945, | |
| "ref_logps/rejected": -38.49150085449219, | |
| "rewards/accuracies": 0.625, | |
| "rewards/chosen": 0.02471642568707466, | |
| "rewards/margins": 0.19022998213768005, | |
| "rewards/rejected": -0.1655135601758957, | |
| "step": 31 | |
| }, | |
| { | |
| "epoch": 0.24150943396226415, | |
| "grad_norm": 4.287616729736328, | |
| "learning_rate": 4.89451476793249e-06, | |
| "logps/chosen": -28.966516494750977, | |
| "logps/rejected": -38.64032745361328, | |
| "loss": 0.6213, | |
| "losses/dpo": 0.5834950804710388, | |
| "losses/sft": 1.19804847240448, | |
| "losses/total": 0.5834950804710388, | |
| "ref_logps/chosen": -29.167491912841797, | |
| "ref_logps/rejected": -36.74009704589844, | |
| "rewards/accuracies": 0.671875, | |
| "rewards/chosen": 0.0200975239276886, | |
| "rewards/margins": 0.21012060344219208, | |
| "rewards/rejected": -0.19002306461334229, | |
| "step": 32 | |
| }, | |
| { | |
| "epoch": 0.2490566037735849, | |
| "grad_norm": 4.030296325683594, | |
| "learning_rate": 4.873417721518987e-06, | |
| "logps/chosen": -26.15782928466797, | |
| "logps/rejected": -33.3205451965332, | |
| "loss": 0.6113, | |
| "losses/dpo": 0.5751947164535522, | |
| "losses/sft": 0.7719302177429199, | |
| "losses/total": 0.5751947164535522, | |
| "ref_logps/chosen": -26.509279251098633, | |
| "ref_logps/rejected": -31.373973846435547, | |
| "rewards/accuracies": 0.71875, | |
| "rewards/chosen": 0.03514501452445984, | |
| "rewards/margins": 0.22980214655399323, | |
| "rewards/rejected": -0.1946571320295334, | |
| "step": 33 | |
| }, | |
| { | |
| "epoch": 0.25660377358490566, | |
| "grad_norm": 4.367461204528809, | |
| "learning_rate": 4.852320675105486e-06, | |
| "logps/chosen": -32.38875961303711, | |
| "logps/rejected": -40.95237350463867, | |
| "loss": 0.6236, | |
| "losses/dpo": 0.6270190477371216, | |
| "losses/sft": 1.1532888412475586, | |
| "losses/total": 0.6270190477371216, | |
| "ref_logps/chosen": -31.350940704345703, | |
| "ref_logps/rejected": -37.58573913574219, | |
| "rewards/accuracies": 0.71875, | |
| "rewards/chosen": -0.10378223657608032, | |
| "rewards/margins": 0.2328808605670929, | |
| "rewards/rejected": -0.3366630971431732, | |
| "step": 34 | |
| }, | |
| { | |
| "epoch": 0.2641509433962264, | |
| "grad_norm": 4.614798545837402, | |
| "learning_rate": 4.831223628691984e-06, | |
| "logps/chosen": -32.56526184082031, | |
| "logps/rejected": -40.50109100341797, | |
| "loss": 0.6059, | |
| "losses/dpo": 0.7241290211677551, | |
| "losses/sft": 1.3340615034103394, | |
| "losses/total": 0.7241290211677551, | |
| "ref_logps/chosen": -30.577272415161133, | |
| "ref_logps/rejected": -35.84385681152344, | |
| "rewards/accuracies": 0.6796875, | |
| "rewards/chosen": -0.19879914820194244, | |
| "rewards/margins": 0.26692429184913635, | |
| "rewards/rejected": -0.46572345495224, | |
| "step": 35 | |
| }, | |
| { | |
| "epoch": 0.27169811320754716, | |
| "grad_norm": 4.4597649574279785, | |
| "learning_rate": 4.8101265822784815e-06, | |
| "logps/chosen": -34.825111389160156, | |
| "logps/rejected": -44.15177917480469, | |
| "loss": 0.5946, | |
| "losses/dpo": 0.5129742622375488, | |
| "losses/sft": 0.856516420841217, | |
| "losses/total": 0.5129742622375488, | |
| "ref_logps/chosen": -32.00858688354492, | |
| "ref_logps/rejected": -38.221412658691406, | |
| "rewards/accuracies": 0.6953125, | |
| "rewards/chosen": -0.2816521227359772, | |
| "rewards/margins": 0.31138482689857483, | |
| "rewards/rejected": -0.593036949634552, | |
| "step": 36 | |
| }, | |
| { | |
| "epoch": 0.2792452830188679, | |
| "grad_norm": 4.562221527099609, | |
| "learning_rate": 4.789029535864979e-06, | |
| "logps/chosen": -33.27934646606445, | |
| "logps/rejected": -39.6660041809082, | |
| "loss": 0.598, | |
| "losses/dpo": 0.6043655276298523, | |
| "losses/sft": 1.0378785133361816, | |
| "losses/total": 0.6043655276298523, | |
| "ref_logps/chosen": -29.917388916015625, | |
| "ref_logps/rejected": -33.3465690612793, | |
| "rewards/accuracies": 0.6484375, | |
| "rewards/chosen": -0.33619576692581177, | |
| "rewards/margins": 0.29574763774871826, | |
| "rewards/rejected": -0.6319433450698853, | |
| "step": 37 | |
| }, | |
| { | |
| "epoch": 0.28679245283018867, | |
| "grad_norm": 5.42985725402832, | |
| "learning_rate": 4.767932489451477e-06, | |
| "logps/chosen": -36.16193389892578, | |
| "logps/rejected": -43.71050262451172, | |
| "loss": 0.6555, | |
| "losses/dpo": 0.6992093324661255, | |
| "losses/sft": 1.3419675827026367, | |
| "losses/total": 0.6992093324661255, | |
| "ref_logps/chosen": -31.39295196533203, | |
| "ref_logps/rejected": -36.14948654174805, | |
| "rewards/accuracies": 0.65625, | |
| "rewards/chosen": -0.47689831256866455, | |
| "rewards/margins": 0.2792032063007355, | |
| "rewards/rejected": -0.7561015486717224, | |
| "step": 38 | |
| }, | |
| { | |
| "epoch": 0.2943396226415094, | |
| "grad_norm": 4.9820051193237305, | |
| "learning_rate": 4.746835443037975e-06, | |
| "logps/chosen": -33.79109191894531, | |
| "logps/rejected": -42.09747314453125, | |
| "loss": 0.5883, | |
| "losses/dpo": 0.6338505148887634, | |
| "losses/sft": 1.2124994993209839, | |
| "losses/total": 0.6338505148887634, | |
| "ref_logps/chosen": -28.94110107421875, | |
| "ref_logps/rejected": -33.52173614501953, | |
| "rewards/accuracies": 0.7109375, | |
| "rewards/chosen": -0.4849993586540222, | |
| "rewards/margins": 0.3725742697715759, | |
| "rewards/rejected": -0.8575735688209534, | |
| "step": 39 | |
| }, | |
| { | |
| "epoch": 0.3018867924528302, | |
| "grad_norm": 5.098467826843262, | |
| "learning_rate": 4.725738396624473e-06, | |
| "logps/chosen": -34.21745681762695, | |
| "logps/rejected": -43.00538635253906, | |
| "loss": 0.6283, | |
| "losses/dpo": 0.7655435800552368, | |
| "losses/sft": 1.7303447723388672, | |
| "losses/total": 0.7655435800552368, | |
| "ref_logps/chosen": -27.329925537109375, | |
| "ref_logps/rejected": -33.21820068359375, | |
| "rewards/accuracies": 0.671875, | |
| "rewards/chosen": -0.6887531876564026, | |
| "rewards/margins": 0.2899653911590576, | |
| "rewards/rejected": -0.9787185788154602, | |
| "step": 40 | |
| }, | |
| { | |
| "epoch": 0.30943396226415093, | |
| "grad_norm": 4.568510055541992, | |
| "learning_rate": 4.7046413502109714e-06, | |
| "logps/chosen": -34.30303955078125, | |
| "logps/rejected": -45.21646499633789, | |
| "loss": 0.5376, | |
| "losses/dpo": 0.466990202665329, | |
| "losses/sft": 1.3411931991577148, | |
| "losses/total": 0.466990202665329, | |
| "ref_logps/chosen": -27.724037170410156, | |
| "ref_logps/rejected": -33.146297454833984, | |
| "rewards/accuracies": 0.7265625, | |
| "rewards/chosen": -0.657900333404541, | |
| "rewards/margins": 0.5491164326667786, | |
| "rewards/rejected": -1.2070167064666748, | |
| "step": 41 | |
| }, | |
| { | |
| "epoch": 0.3169811320754717, | |
| "grad_norm": 4.940774440765381, | |
| "learning_rate": 4.683544303797468e-06, | |
| "logps/chosen": -37.19551086425781, | |
| "logps/rejected": -48.22083282470703, | |
| "loss": 0.5488, | |
| "losses/dpo": 0.7535874843597412, | |
| "losses/sft": 1.5672276020050049, | |
| "losses/total": 0.7535874843597412, | |
| "ref_logps/chosen": -30.618778228759766, | |
| "ref_logps/rejected": -36.40138244628906, | |
| "rewards/accuracies": 0.7421875, | |
| "rewards/chosen": -0.6576732993125916, | |
| "rewards/margins": 0.5242711901664734, | |
| "rewards/rejected": -1.181944489479065, | |
| "step": 42 | |
| }, | |
| { | |
| "epoch": 0.32452830188679244, | |
| "grad_norm": 5.207009792327881, | |
| "learning_rate": 4.662447257383967e-06, | |
| "logps/chosen": -38.67348861694336, | |
| "logps/rejected": -46.259944915771484, | |
| "loss": 0.6086, | |
| "losses/dpo": 0.580439567565918, | |
| "losses/sft": 1.5248361825942993, | |
| "losses/total": 0.580439567565918, | |
| "ref_logps/chosen": -30.998300552368164, | |
| "ref_logps/rejected": -34.962703704833984, | |
| "rewards/accuracies": 0.671875, | |
| "rewards/chosen": -0.767518937587738, | |
| "rewards/margins": 0.3622052073478699, | |
| "rewards/rejected": -1.129724144935608, | |
| "step": 43 | |
| }, | |
| { | |
| "epoch": 0.3320754716981132, | |
| "grad_norm": 5.374455451965332, | |
| "learning_rate": 4.641350210970465e-06, | |
| "logps/chosen": -35.60981750488281, | |
| "logps/rejected": -45.90471649169922, | |
| "loss": 0.6258, | |
| "losses/dpo": 0.7469555139541626, | |
| "losses/sft": 1.5075256824493408, | |
| "losses/total": 0.7469555139541626, | |
| "ref_logps/chosen": -27.66849136352539, | |
| "ref_logps/rejected": -34.36457824707031, | |
| "rewards/accuracies": 0.6484375, | |
| "rewards/chosen": -0.79413241147995, | |
| "rewards/margins": 0.35988110303878784, | |
| "rewards/rejected": -1.1540133953094482, | |
| "step": 44 | |
| }, | |
| { | |
| "epoch": 0.33962264150943394, | |
| "grad_norm": 5.304266452789307, | |
| "learning_rate": 4.620253164556963e-06, | |
| "logps/chosen": -38.42974090576172, | |
| "logps/rejected": -45.68645477294922, | |
| "loss": 0.6133, | |
| "losses/dpo": 0.6757426261901855, | |
| "losses/sft": 1.243510127067566, | |
| "losses/total": 0.6757426261901855, | |
| "ref_logps/chosen": -29.328536987304688, | |
| "ref_logps/rejected": -32.262840270996094, | |
| "rewards/accuracies": 0.6796875, | |
| "rewards/chosen": -0.9101204872131348, | |
| "rewards/margins": 0.4322410821914673, | |
| "rewards/rejected": -1.342361569404602, | |
| "step": 45 | |
| }, | |
| { | |
| "epoch": 0.3471698113207547, | |
| "grad_norm": 5.428191184997559, | |
| "learning_rate": 4.5991561181434605e-06, | |
| "logps/chosen": -38.54790115356445, | |
| "logps/rejected": -49.04346466064453, | |
| "loss": 0.5944, | |
| "losses/dpo": 0.7710250020027161, | |
| "losses/sft": 1.5200599431991577, | |
| "losses/total": 0.7710250020027161, | |
| "ref_logps/chosen": -29.571321487426758, | |
| "ref_logps/rejected": -35.5488166809082, | |
| "rewards/accuracies": 0.671875, | |
| "rewards/chosen": -0.897658109664917, | |
| "rewards/margins": 0.45180660486221313, | |
| "rewards/rejected": -1.3494646549224854, | |
| "step": 46 | |
| }, | |
| { | |
| "epoch": 0.35471698113207545, | |
| "grad_norm": 5.276749610900879, | |
| "learning_rate": 4.578059071729958e-06, | |
| "logps/chosen": -42.278629302978516, | |
| "logps/rejected": -49.082420349121094, | |
| "loss": 0.5844, | |
| "losses/dpo": 0.5217583179473877, | |
| "losses/sft": 1.251055121421814, | |
| "losses/total": 0.5217583179473877, | |
| "ref_logps/chosen": -32.70330047607422, | |
| "ref_logps/rejected": -34.48522186279297, | |
| "rewards/accuracies": 0.7734375, | |
| "rewards/chosen": -0.9575330018997192, | |
| "rewards/margins": 0.5021871328353882, | |
| "rewards/rejected": -1.4597201347351074, | |
| "step": 47 | |
| }, | |
| { | |
| "epoch": 0.3622641509433962, | |
| "grad_norm": 4.85624361038208, | |
| "learning_rate": 4.556962025316456e-06, | |
| "logps/chosen": -37.618473052978516, | |
| "logps/rejected": -49.08882141113281, | |
| "loss": 0.5444, | |
| "losses/dpo": 0.4470449984073639, | |
| "losses/sft": 1.1268991231918335, | |
| "losses/total": 0.4470449984073639, | |
| "ref_logps/chosen": -29.47239112854004, | |
| "ref_logps/rejected": -34.867340087890625, | |
| "rewards/accuracies": 0.734375, | |
| "rewards/chosen": -0.8146083354949951, | |
| "rewards/margins": 0.6075396537780762, | |
| "rewards/rejected": -1.4221479892730713, | |
| "step": 48 | |
| }, | |
| { | |
| "epoch": 0.36981132075471695, | |
| "grad_norm": 5.045034408569336, | |
| "learning_rate": 4.535864978902954e-06, | |
| "logps/chosen": -33.628997802734375, | |
| "logps/rejected": -46.788063049316406, | |
| "loss": 0.5441, | |
| "losses/dpo": 0.4152263402938843, | |
| "losses/sft": 1.2738722562789917, | |
| "losses/total": 0.4152263402938843, | |
| "ref_logps/chosen": -25.57724952697754, | |
| "ref_logps/rejected": -33.02429962158203, | |
| "rewards/accuracies": 0.7265625, | |
| "rewards/chosen": -0.8051748275756836, | |
| "rewards/margins": 0.5712013840675354, | |
| "rewards/rejected": -1.3763761520385742, | |
| "step": 49 | |
| }, | |
| { | |
| "epoch": 0.37735849056603776, | |
| "grad_norm": 4.926025867462158, | |
| "learning_rate": 4.514767932489452e-06, | |
| "logps/chosen": -40.1754264831543, | |
| "logps/rejected": -53.34943389892578, | |
| "loss": 0.5357, | |
| "losses/dpo": 0.42820218205451965, | |
| "losses/sft": 1.4205973148345947, | |
| "losses/total": 0.42820218205451965, | |
| "ref_logps/chosen": -30.12688636779785, | |
| "ref_logps/rejected": -36.76472091674805, | |
| "rewards/accuracies": 0.7421875, | |
| "rewards/chosen": -1.0048537254333496, | |
| "rewards/margins": 0.6536170840263367, | |
| "rewards/rejected": -1.658470869064331, | |
| "step": 50 | |
| }, | |
| { | |
| "epoch": 0.3849056603773585, | |
| "grad_norm": 5.366219997406006, | |
| "learning_rate": 4.4936708860759495e-06, | |
| "logps/chosen": -41.36455535888672, | |
| "logps/rejected": -51.32182312011719, | |
| "loss": 0.5498, | |
| "losses/dpo": 0.5175436735153198, | |
| "losses/sft": 1.3962128162384033, | |
| "losses/total": 0.5175436735153198, | |
| "ref_logps/chosen": -31.545040130615234, | |
| "ref_logps/rejected": -34.96824264526367, | |
| "rewards/accuracies": 0.6875, | |
| "rewards/chosen": -0.9819516539573669, | |
| "rewards/margins": 0.6534063220024109, | |
| "rewards/rejected": -1.6353578567504883, | |
| "step": 51 | |
| }, | |
| { | |
| "epoch": 0.39245283018867927, | |
| "grad_norm": 5.386270523071289, | |
| "learning_rate": 4.472573839662447e-06, | |
| "logps/chosen": -37.03908920288086, | |
| "logps/rejected": -50.11444854736328, | |
| "loss": 0.5742, | |
| "losses/dpo": 0.8617876768112183, | |
| "losses/sft": 2.0467562675476074, | |
| "losses/total": 0.8617876768112183, | |
| "ref_logps/chosen": -27.370098114013672, | |
| "ref_logps/rejected": -34.820045471191406, | |
| "rewards/accuracies": 0.7109375, | |
| "rewards/chosen": -0.9668989181518555, | |
| "rewards/margins": 0.5625417232513428, | |
| "rewards/rejected": -1.5294406414031982, | |
| "step": 52 | |
| }, | |
| { | |
| "epoch": 0.4, | |
| "grad_norm": 4.991089820861816, | |
| "learning_rate": 4.451476793248945e-06, | |
| "logps/chosen": -36.202457427978516, | |
| "logps/rejected": -46.09682083129883, | |
| "loss": 0.5363, | |
| "losses/dpo": 0.4159512221813202, | |
| "losses/sft": 1.2818732261657715, | |
| "losses/total": 0.4159512221813202, | |
| "ref_logps/chosen": -27.4339542388916, | |
| "ref_logps/rejected": -30.41155433654785, | |
| "rewards/accuracies": 0.734375, | |
| "rewards/chosen": -0.8768501281738281, | |
| "rewards/margins": 0.6916766166687012, | |
| "rewards/rejected": -1.5685267448425293, | |
| "step": 53 | |
| }, | |
| { | |
| "epoch": 0.4075471698113208, | |
| "grad_norm": 5.318048000335693, | |
| "learning_rate": 4.430379746835443e-06, | |
| "logps/chosen": -40.78857421875, | |
| "logps/rejected": -48.64263153076172, | |
| "loss": 0.5552, | |
| "losses/dpo": 0.5250239372253418, | |
| "losses/sft": 1.3805458545684814, | |
| "losses/total": 0.5250239372253418, | |
| "ref_logps/chosen": -31.36919403076172, | |
| "ref_logps/rejected": -32.82971954345703, | |
| "rewards/accuracies": 0.6640625, | |
| "rewards/chosen": -0.9419378638267517, | |
| "rewards/margins": 0.6393535733222961, | |
| "rewards/rejected": -1.5812914371490479, | |
| "step": 54 | |
| }, | |
| { | |
| "epoch": 0.41509433962264153, | |
| "grad_norm": 5.705236911773682, | |
| "learning_rate": 4.409282700421942e-06, | |
| "logps/chosen": -39.51799011230469, | |
| "logps/rejected": -50.53668975830078, | |
| "loss": 0.5586, | |
| "losses/dpo": 0.7799692153930664, | |
| "losses/sft": 1.2691373825073242, | |
| "losses/total": 0.7799692153930664, | |
| "ref_logps/chosen": -29.568626403808594, | |
| "ref_logps/rejected": -34.02891540527344, | |
| "rewards/accuracies": 0.6953125, | |
| "rewards/chosen": -0.9949361681938171, | |
| "rewards/margins": 0.6558418273925781, | |
| "rewards/rejected": -1.6507779359817505, | |
| "step": 55 | |
| }, | |
| { | |
| "epoch": 0.4226415094339623, | |
| "grad_norm": 5.554488658905029, | |
| "learning_rate": 4.3881856540084394e-06, | |
| "logps/chosen": -38.21480941772461, | |
| "logps/rejected": -47.99286651611328, | |
| "loss": 0.5795, | |
| "losses/dpo": 0.5031176805496216, | |
| "losses/sft": 1.3220183849334717, | |
| "losses/total": 0.5031176805496216, | |
| "ref_logps/chosen": -29.2410888671875, | |
| "ref_logps/rejected": -33.15393829345703, | |
| "rewards/accuracies": 0.703125, | |
| "rewards/chosen": -0.8973721265792847, | |
| "rewards/margins": 0.5865209698677063, | |
| "rewards/rejected": -1.4838931560516357, | |
| "step": 56 | |
| }, | |
| { | |
| "epoch": 0.43018867924528303, | |
| "grad_norm": 5.692888259887695, | |
| "learning_rate": 4.367088607594937e-06, | |
| "logps/chosen": -41.21876907348633, | |
| "logps/rejected": -53.25712585449219, | |
| "loss": 0.5296, | |
| "losses/dpo": 0.4523683488368988, | |
| "losses/sft": 1.5277466773986816, | |
| "losses/total": 0.4523683488368988, | |
| "ref_logps/chosen": -31.35172462463379, | |
| "ref_logps/rejected": -35.32482147216797, | |
| "rewards/accuracies": 0.7421875, | |
| "rewards/chosen": -0.9867046475410461, | |
| "rewards/margins": 0.806525707244873, | |
| "rewards/rejected": -1.7932302951812744, | |
| "step": 57 | |
| }, | |
| { | |
| "epoch": 0.4377358490566038, | |
| "grad_norm": 5.991930961608887, | |
| "learning_rate": 4.345991561181435e-06, | |
| "logps/chosen": -39.15966033935547, | |
| "logps/rejected": -50.0914306640625, | |
| "loss": 0.5912, | |
| "losses/dpo": 0.8516952991485596, | |
| "losses/sft": 1.4551334381103516, | |
| "losses/total": 0.8516952991485596, | |
| "ref_logps/chosen": -28.90890884399414, | |
| "ref_logps/rejected": -32.83641052246094, | |
| "rewards/accuracies": 0.6953125, | |
| "rewards/chosen": -1.0250749588012695, | |
| "rewards/margins": 0.7004267573356628, | |
| "rewards/rejected": -1.7255017757415771, | |
| "step": 58 | |
| }, | |
| { | |
| "epoch": 0.44528301886792454, | |
| "grad_norm": 5.214711666107178, | |
| "learning_rate": 4.324894514767933e-06, | |
| "logps/chosen": -40.67414474487305, | |
| "logps/rejected": -57.66020584106445, | |
| "loss": 0.4895, | |
| "losses/dpo": 0.5767215490341187, | |
| "losses/sft": 1.3480836153030396, | |
| "losses/total": 0.5767215490341187, | |
| "ref_logps/chosen": -31.317989349365234, | |
| "ref_logps/rejected": -38.934444427490234, | |
| "rewards/accuracies": 0.7578125, | |
| "rewards/chosen": -0.9356154799461365, | |
| "rewards/margins": 0.93696129322052, | |
| "rewards/rejected": -1.8725767135620117, | |
| "step": 59 | |
| }, | |
| { | |
| "epoch": 0.4528301886792453, | |
| "grad_norm": 6.370638847351074, | |
| "learning_rate": 4.303797468354431e-06, | |
| "logps/chosen": -40.94059753417969, | |
| "logps/rejected": -51.065185546875, | |
| "loss": 0.6121, | |
| "losses/dpo": 0.4795827865600586, | |
| "losses/sft": 1.4322285652160645, | |
| "losses/total": 0.4795827865600586, | |
| "ref_logps/chosen": -29.871326446533203, | |
| "ref_logps/rejected": -34.67937469482422, | |
| "rewards/accuracies": 0.703125, | |
| "rewards/chosen": -1.1069271564483643, | |
| "rewards/margins": 0.5316535830497742, | |
| "rewards/rejected": -1.6385807991027832, | |
| "step": 60 | |
| }, | |
| { | |
| "epoch": 0.46037735849056605, | |
| "grad_norm": 6.262545585632324, | |
| "learning_rate": 4.2827004219409285e-06, | |
| "logps/chosen": -39.27662658691406, | |
| "logps/rejected": -55.63613510131836, | |
| "loss": 0.5328, | |
| "losses/dpo": 0.596257746219635, | |
| "losses/sft": 1.5779916048049927, | |
| "losses/total": 0.596257746219635, | |
| "ref_logps/chosen": -29.888713836669922, | |
| "ref_logps/rejected": -38.11284637451172, | |
| "rewards/accuracies": 0.7734375, | |
| "rewards/chosen": -0.9387915134429932, | |
| "rewards/margins": 0.8135374784469604, | |
| "rewards/rejected": -1.752328872680664, | |
| "step": 61 | |
| }, | |
| { | |
| "epoch": 0.4679245283018868, | |
| "grad_norm": 5.742541313171387, | |
| "learning_rate": 4.261603375527426e-06, | |
| "logps/chosen": -40.12831497192383, | |
| "logps/rejected": -48.92723846435547, | |
| "loss": 0.5512, | |
| "losses/dpo": 0.6246651411056519, | |
| "losses/sft": 1.4945807456970215, | |
| "losses/total": 0.6246651411056519, | |
| "ref_logps/chosen": -31.467824935913086, | |
| "ref_logps/rejected": -33.796714782714844, | |
| "rewards/accuracies": 0.7109375, | |
| "rewards/chosen": -0.86604905128479, | |
| "rewards/margins": 0.6470035314559937, | |
| "rewards/rejected": -1.5130527019500732, | |
| "step": 62 | |
| }, | |
| { | |
| "epoch": 0.47547169811320755, | |
| "grad_norm": 5.606804370880127, | |
| "learning_rate": 4.240506329113924e-06, | |
| "logps/chosen": -38.19676971435547, | |
| "logps/rejected": -51.48821258544922, | |
| "loss": 0.5277, | |
| "losses/dpo": 0.6507617235183716, | |
| "losses/sft": 1.7701728343963623, | |
| "losses/total": 0.6507617235183716, | |
| "ref_logps/chosen": -28.915279388427734, | |
| "ref_logps/rejected": -34.66787338256836, | |
| "rewards/accuracies": 0.7421875, | |
| "rewards/chosen": -0.9281493425369263, | |
| "rewards/margins": 0.7538847327232361, | |
| "rewards/rejected": -1.6820340156555176, | |
| "step": 63 | |
| }, | |
| { | |
| "epoch": 0.4830188679245283, | |
| "grad_norm": 5.0319647789001465, | |
| "learning_rate": 4.219409282700423e-06, | |
| "logps/chosen": -36.99481964111328, | |
| "logps/rejected": -49.86357116699219, | |
| "loss": 0.5064, | |
| "losses/dpo": 0.3545170724391937, | |
| "losses/sft": 1.4933311939239502, | |
| "losses/total": 0.3545170724391937, | |
| "ref_logps/chosen": -27.901212692260742, | |
| "ref_logps/rejected": -32.68770980834961, | |
| "rewards/accuracies": 0.7265625, | |
| "rewards/chosen": -0.9093605875968933, | |
| "rewards/margins": 0.8082252144813538, | |
| "rewards/rejected": -1.717585802078247, | |
| "step": 64 | |
| }, | |
| { | |
| "epoch": 0.49056603773584906, | |
| "grad_norm": 5.624882698059082, | |
| "learning_rate": 4.19831223628692e-06, | |
| "logps/chosen": -41.812522888183594, | |
| "logps/rejected": -50.7585334777832, | |
| "loss": 0.5876, | |
| "losses/dpo": 0.7280862331390381, | |
| "losses/sft": 1.730393648147583, | |
| "losses/total": 0.7280862331390381, | |
| "ref_logps/chosen": -32.30836868286133, | |
| "ref_logps/rejected": -35.13983917236328, | |
| "rewards/accuracies": 0.734375, | |
| "rewards/chosen": -0.9504156708717346, | |
| "rewards/margins": 0.6114538908004761, | |
| "rewards/rejected": -1.5618693828582764, | |
| "step": 65 | |
| }, | |
| { | |
| "epoch": 0.4981132075471698, | |
| "grad_norm": 5.693253517150879, | |
| "learning_rate": 4.177215189873418e-06, | |
| "logps/chosen": -41.0583381652832, | |
| "logps/rejected": -57.39263153076172, | |
| "loss": 0.4943, | |
| "losses/dpo": 0.4355122745037079, | |
| "losses/sft": 1.1784332990646362, | |
| "losses/total": 0.4355122745037079, | |
| "ref_logps/chosen": -31.086488723754883, | |
| "ref_logps/rejected": -38.95460891723633, | |
| "rewards/accuracies": 0.7734375, | |
| "rewards/chosen": -0.9971848726272583, | |
| "rewards/margins": 0.8466169834136963, | |
| "rewards/rejected": -1.8438018560409546, | |
| "step": 66 | |
| }, | |
| { | |
| "epoch": 0.5056603773584906, | |
| "grad_norm": 7.065197944641113, | |
| "learning_rate": 4.156118143459915e-06, | |
| "logps/chosen": -39.25422668457031, | |
| "logps/rejected": -50.87791442871094, | |
| "loss": 0.5339, | |
| "losses/dpo": 0.8317179083824158, | |
| "losses/sft": 1.3175883293151855, | |
| "losses/total": 0.8317179083824158, | |
| "ref_logps/chosen": -28.742538452148438, | |
| "ref_logps/rejected": -33.21300506591797, | |
| "rewards/accuracies": 0.71875, | |
| "rewards/chosen": -1.0511683225631714, | |
| "rewards/margins": 0.7153225541114807, | |
| "rewards/rejected": -1.7664909362792969, | |
| "step": 67 | |
| }, | |
| { | |
| "epoch": 0.5132075471698113, | |
| "grad_norm": 5.2859907150268555, | |
| "learning_rate": 4.135021097046414e-06, | |
| "logps/chosen": -38.8599853515625, | |
| "logps/rejected": -53.00445556640625, | |
| "loss": 0.4691, | |
| "losses/dpo": 0.5039323568344116, | |
| "losses/sft": 0.7760534286499023, | |
| "losses/total": 0.5039323568344116, | |
| "ref_logps/chosen": -30.572181701660156, | |
| "ref_logps/rejected": -35.3000602722168, | |
| "rewards/accuracies": 0.78125, | |
| "rewards/chosen": -0.828780472278595, | |
| "rewards/margins": 0.9416592121124268, | |
| "rewards/rejected": -1.770439863204956, | |
| "step": 68 | |
| }, | |
| { | |
| "epoch": 0.5207547169811321, | |
| "grad_norm": 5.869091033935547, | |
| "learning_rate": 4.113924050632912e-06, | |
| "logps/chosen": -39.38776397705078, | |
| "logps/rejected": -56.88161849975586, | |
| "loss": 0.5432, | |
| "losses/dpo": 0.28467729687690735, | |
| "losses/sft": 1.18190336227417, | |
| "losses/total": 0.28467729687690735, | |
| "ref_logps/chosen": -28.762392044067383, | |
| "ref_logps/rejected": -37.78593063354492, | |
| "rewards/accuracies": 0.7109375, | |
| "rewards/chosen": -1.0625369548797607, | |
| "rewards/margins": 0.8470318913459778, | |
| "rewards/rejected": -1.9095687866210938, | |
| "step": 69 | |
| }, | |
| { | |
| "epoch": 0.5283018867924528, | |
| "grad_norm": 5.994374752044678, | |
| "learning_rate": 4.09282700421941e-06, | |
| "logps/chosen": -39.55097579956055, | |
| "logps/rejected": -50.52848815917969, | |
| "loss": 0.5564, | |
| "losses/dpo": 0.6404341459274292, | |
| "losses/sft": 1.6006711721420288, | |
| "losses/total": 0.6404341459274292, | |
| "ref_logps/chosen": -29.427146911621094, | |
| "ref_logps/rejected": -32.81227111816406, | |
| "rewards/accuracies": 0.7109375, | |
| "rewards/chosen": -1.0123828649520874, | |
| "rewards/margins": 0.759239137172699, | |
| "rewards/rejected": -1.7716220617294312, | |
| "step": 70 | |
| }, | |
| { | |
| "epoch": 0.5358490566037736, | |
| "grad_norm": 6.391396999359131, | |
| "learning_rate": 4.0717299578059074e-06, | |
| "logps/chosen": -39.2120475769043, | |
| "logps/rejected": -53.28424072265625, | |
| "loss": 0.5533, | |
| "losses/dpo": 0.35934409499168396, | |
| "losses/sft": 1.446973443031311, | |
| "losses/total": 0.35934409499168396, | |
| "ref_logps/chosen": -28.753549575805664, | |
| "ref_logps/rejected": -34.92060470581055, | |
| "rewards/accuracies": 0.71875, | |
| "rewards/chosen": -1.0458496809005737, | |
| "rewards/margins": 0.790514349937439, | |
| "rewards/rejected": -1.8363640308380127, | |
| "step": 71 | |
| }, | |
| { | |
| "epoch": 0.5433962264150943, | |
| "grad_norm": 6.23837423324585, | |
| "learning_rate": 4.050632911392405e-06, | |
| "logps/chosen": -41.26081466674805, | |
| "logps/rejected": -57.397159576416016, | |
| "loss": 0.5566, | |
| "losses/dpo": 0.6022348403930664, | |
| "losses/sft": 1.4900131225585938, | |
| "losses/total": 0.6022348403930664, | |
| "ref_logps/chosen": -29.823455810546875, | |
| "ref_logps/rejected": -38.314422607421875, | |
| "rewards/accuracies": 0.6953125, | |
| "rewards/chosen": -1.1437358856201172, | |
| "rewards/margins": 0.7645378112792969, | |
| "rewards/rejected": -1.908273696899414, | |
| "step": 72 | |
| }, | |
| { | |
| "epoch": 0.5509433962264151, | |
| "grad_norm": 5.54686164855957, | |
| "learning_rate": 4.029535864978903e-06, | |
| "logps/chosen": -40.462684631347656, | |
| "logps/rejected": -56.720314025878906, | |
| "loss": 0.4834, | |
| "losses/dpo": 0.5470014810562134, | |
| "losses/sft": 1.4223295450210571, | |
| "losses/total": 0.5470014810562134, | |
| "ref_logps/chosen": -30.118234634399414, | |
| "ref_logps/rejected": -36.64280700683594, | |
| "rewards/accuracies": 0.75, | |
| "rewards/chosen": -1.0344449281692505, | |
| "rewards/margins": 0.9733060002326965, | |
| "rewards/rejected": -2.007750988006592, | |
| "step": 73 | |
| }, | |
| { | |
| "epoch": 0.5584905660377358, | |
| "grad_norm": 6.6403656005859375, | |
| "learning_rate": 4.008438818565401e-06, | |
| "logps/chosen": -43.17585754394531, | |
| "logps/rejected": -54.99962615966797, | |
| "loss": 0.6214, | |
| "losses/dpo": 0.8616761565208435, | |
| "losses/sft": 1.4494423866271973, | |
| "losses/total": 0.8616761565208435, | |
| "ref_logps/chosen": -30.735164642333984, | |
| "ref_logps/rejected": -36.314048767089844, | |
| "rewards/accuracies": 0.6875, | |
| "rewards/chosen": -1.2440688610076904, | |
| "rewards/margins": 0.6244890689849854, | |
| "rewards/rejected": -1.8685579299926758, | |
| "step": 74 | |
| }, | |
| { | |
| "epoch": 0.5660377358490566, | |
| "grad_norm": 5.264912128448486, | |
| "learning_rate": 3.9873417721518995e-06, | |
| "logps/chosen": -40.38694763183594, | |
| "logps/rejected": -57.54372024536133, | |
| "loss": 0.4474, | |
| "losses/dpo": 0.49264034628868103, | |
| "losses/sft": 1.4342858791351318, | |
| "losses/total": 0.49264034628868103, | |
| "ref_logps/chosen": -30.248275756835938, | |
| "ref_logps/rejected": -35.96062469482422, | |
| "rewards/accuracies": 0.8046875, | |
| "rewards/chosen": -1.0138667821884155, | |
| "rewards/margins": 1.1444426774978638, | |
| "rewards/rejected": -2.1583094596862793, | |
| "step": 75 | |
| }, | |
| { | |
| "epoch": 0.5735849056603773, | |
| "grad_norm": 5.621297836303711, | |
| "learning_rate": 3.9662447257383965e-06, | |
| "logps/chosen": -40.704254150390625, | |
| "logps/rejected": -54.67380905151367, | |
| "loss": 0.5208, | |
| "losses/dpo": 0.3526512086391449, | |
| "losses/sft": 1.205520749092102, | |
| "losses/total": 0.3526512086391449, | |
| "ref_logps/chosen": -30.080062866210938, | |
| "ref_logps/rejected": -35.63865661621094, | |
| "rewards/accuracies": 0.7265625, | |
| "rewards/chosen": -1.0624192953109741, | |
| "rewards/margins": 0.8410958647727966, | |
| "rewards/rejected": -1.903515100479126, | |
| "step": 76 | |
| }, | |
| { | |
| "epoch": 0.5811320754716981, | |
| "grad_norm": 5.440826416015625, | |
| "learning_rate": 3.945147679324895e-06, | |
| "logps/chosen": -38.52376174926758, | |
| "logps/rejected": -51.74415588378906, | |
| "loss": 0.5272, | |
| "losses/dpo": 0.43253982067108154, | |
| "losses/sft": 1.3709319829940796, | |
| "losses/total": 0.43253982067108154, | |
| "ref_logps/chosen": -29.032743453979492, | |
| "ref_logps/rejected": -34.911163330078125, | |
| "rewards/accuracies": 0.75, | |
| "rewards/chosen": -0.9491016268730164, | |
| "rewards/margins": 0.7341974377632141, | |
| "rewards/rejected": -1.68329918384552, | |
| "step": 77 | |
| }, | |
| { | |
| "epoch": 0.5886792452830188, | |
| "grad_norm": 4.8831377029418945, | |
| "learning_rate": 3.924050632911393e-06, | |
| "logps/chosen": -37.76980972290039, | |
| "logps/rejected": -57.21007537841797, | |
| "loss": 0.4277, | |
| "losses/dpo": 0.4043513536453247, | |
| "losses/sft": 1.341131329536438, | |
| "losses/total": 0.4043513536453247, | |
| "ref_logps/chosen": -29.5567684173584, | |
| "ref_logps/rejected": -37.932891845703125, | |
| "rewards/accuracies": 0.828125, | |
| "rewards/chosen": -0.8213039636611938, | |
| "rewards/margins": 1.1064141988754272, | |
| "rewards/rejected": -1.927718162536621, | |
| "step": 78 | |
| }, | |
| { | |
| "epoch": 0.5962264150943396, | |
| "grad_norm": 4.781276702880859, | |
| "learning_rate": 3.902953586497891e-06, | |
| "logps/chosen": -39.535728454589844, | |
| "logps/rejected": -55.912841796875, | |
| "loss": 0.4133, | |
| "losses/dpo": 0.22891421616077423, | |
| "losses/sft": 1.0599099397659302, | |
| "losses/total": 0.22891421616077423, | |
| "ref_logps/chosen": -30.658559799194336, | |
| "ref_logps/rejected": -35.42290115356445, | |
| "rewards/accuracies": 0.8359375, | |
| "rewards/chosen": -0.8877166509628296, | |
| "rewards/margins": 1.1612776517868042, | |
| "rewards/rejected": -2.048994302749634, | |
| "step": 79 | |
| }, | |
| { | |
| "epoch": 0.6037735849056604, | |
| "grad_norm": 5.942996501922607, | |
| "learning_rate": 3.8818565400843886e-06, | |
| "logps/chosen": -42.60219192504883, | |
| "logps/rejected": -59.31925582885742, | |
| "loss": 0.4876, | |
| "losses/dpo": 0.18772652745246887, | |
| "losses/sft": 1.6294547319412231, | |
| "losses/total": 0.18772652745246887, | |
| "ref_logps/chosen": -31.457292556762695, | |
| "ref_logps/rejected": -36.81336212158203, | |
| "rewards/accuracies": 0.7578125, | |
| "rewards/chosen": -1.114490032196045, | |
| "rewards/margins": 1.1360994577407837, | |
| "rewards/rejected": -2.250589370727539, | |
| "step": 80 | |
| }, | |
| { | |
| "epoch": 0.6113207547169811, | |
| "grad_norm": 6.1673102378845215, | |
| "learning_rate": 3.860759493670886e-06, | |
| "logps/chosen": -45.821449279785156, | |
| "logps/rejected": -57.79277420043945, | |
| "loss": 0.5474, | |
| "losses/dpo": 0.5481749773025513, | |
| "losses/sft": 1.9008748531341553, | |
| "losses/total": 0.5481749773025513, | |
| "ref_logps/chosen": -33.29500961303711, | |
| "ref_logps/rejected": -35.532379150390625, | |
| "rewards/accuracies": 0.734375, | |
| "rewards/chosen": -1.2526437044143677, | |
| "rewards/margins": 0.9733958840370178, | |
| "rewards/rejected": -2.2260396480560303, | |
| "step": 81 | |
| }, | |
| { | |
| "epoch": 0.6188679245283019, | |
| "grad_norm": 6.780162811279297, | |
| "learning_rate": 3.839662447257384e-06, | |
| "logps/chosen": -42.07637405395508, | |
| "logps/rejected": -53.19427490234375, | |
| "loss": 0.6369, | |
| "losses/dpo": 0.47502392530441284, | |
| "losses/sft": 1.4819515943527222, | |
| "losses/total": 0.47502392530441284, | |
| "ref_logps/chosen": -30.144392013549805, | |
| "ref_logps/rejected": -34.309967041015625, | |
| "rewards/accuracies": 0.6640625, | |
| "rewards/chosen": -1.193198323249817, | |
| "rewards/margins": 0.695232629776001, | |
| "rewards/rejected": -1.8884310722351074, | |
| "step": 82 | |
| }, | |
| { | |
| "epoch": 0.6264150943396226, | |
| "grad_norm": 6.206979751586914, | |
| "learning_rate": 3.818565400843882e-06, | |
| "logps/chosen": -41.30870056152344, | |
| "logps/rejected": -53.384010314941406, | |
| "loss": 0.5578, | |
| "losses/dpo": 0.7641366720199585, | |
| "losses/sft": 1.1807348728179932, | |
| "losses/total": 0.7641366720199585, | |
| "ref_logps/chosen": -30.49885368347168, | |
| "ref_logps/rejected": -34.54371643066406, | |
| "rewards/accuracies": 0.7421875, | |
| "rewards/chosen": -1.0809844732284546, | |
| "rewards/margins": 0.803044319152832, | |
| "rewards/rejected": -1.8840289115905762, | |
| "step": 83 | |
| }, | |
| { | |
| "epoch": 0.6339622641509434, | |
| "grad_norm": 6.179627418518066, | |
| "learning_rate": 3.7974683544303802e-06, | |
| "logps/chosen": -43.51046371459961, | |
| "logps/rejected": -57.6606330871582, | |
| "loss": 0.5393, | |
| "losses/dpo": 0.2461186796426773, | |
| "losses/sft": 1.4413666725158691, | |
| "losses/total": 0.2461186796426773, | |
| "ref_logps/chosen": -31.447532653808594, | |
| "ref_logps/rejected": -35.95327377319336, | |
| "rewards/accuracies": 0.71875, | |
| "rewards/chosen": -1.2062931060791016, | |
| "rewards/margins": 0.964443027973175, | |
| "rewards/rejected": -2.170736312866211, | |
| "step": 84 | |
| }, | |
| { | |
| "epoch": 0.6415094339622641, | |
| "grad_norm": 5.617598533630371, | |
| "learning_rate": 3.776371308016878e-06, | |
| "logps/chosen": -40.44061279296875, | |
| "logps/rejected": -58.36695098876953, | |
| "loss": 0.4637, | |
| "losses/dpo": 0.32039570808410645, | |
| "losses/sft": 1.308812141418457, | |
| "losses/total": 0.32039570808410645, | |
| "ref_logps/chosen": -30.45447540283203, | |
| "ref_logps/rejected": -36.56258010864258, | |
| "rewards/accuracies": 0.8203125, | |
| "rewards/chosen": -0.9986135363578796, | |
| "rewards/margins": 1.1818233728408813, | |
| "rewards/rejected": -2.180436849594116, | |
| "step": 85 | |
| }, | |
| { | |
| "epoch": 0.6490566037735849, | |
| "grad_norm": 6.853886604309082, | |
| "learning_rate": 3.755274261603376e-06, | |
| "logps/chosen": -42.52161407470703, | |
| "logps/rejected": -54.287841796875, | |
| "loss": 0.5991, | |
| "losses/dpo": 0.3581075370311737, | |
| "losses/sft": 1.5368932485580444, | |
| "losses/total": 0.3581075370311737, | |
| "ref_logps/chosen": -31.34189796447754, | |
| "ref_logps/rejected": -35.659629821777344, | |
| "rewards/accuracies": 0.671875, | |
| "rewards/chosen": -1.117971658706665, | |
| "rewards/margins": 0.7448497414588928, | |
| "rewards/rejected": -1.8628215789794922, | |
| "step": 86 | |
| }, | |
| { | |
| "epoch": 0.6566037735849056, | |
| "grad_norm": 6.692890167236328, | |
| "learning_rate": 3.7341772151898737e-06, | |
| "logps/chosen": -43.72019958496094, | |
| "logps/rejected": -58.138694763183594, | |
| "loss": 0.5519, | |
| "losses/dpo": 0.48806032538414, | |
| "losses/sft": 0.9289418458938599, | |
| "losses/total": 0.48806032538414, | |
| "ref_logps/chosen": -31.137048721313477, | |
| "ref_logps/rejected": -36.59523010253906, | |
| "rewards/accuracies": 0.75, | |
| "rewards/chosen": -1.2583153247833252, | |
| "rewards/margins": 0.8960307836532593, | |
| "rewards/rejected": -2.154345989227295, | |
| "step": 87 | |
| }, | |
| { | |
| "epoch": 0.6641509433962264, | |
| "grad_norm": 5.944717884063721, | |
| "learning_rate": 3.713080168776372e-06, | |
| "logps/chosen": -43.606327056884766, | |
| "logps/rejected": -54.38173294067383, | |
| "loss": 0.525, | |
| "losses/dpo": 0.4686363637447357, | |
| "losses/sft": 1.4131765365600586, | |
| "losses/total": 0.4686363637447357, | |
| "ref_logps/chosen": -32.12074279785156, | |
| "ref_logps/rejected": -35.17863464355469, | |
| "rewards/accuracies": 0.7421875, | |
| "rewards/chosen": -1.1485581398010254, | |
| "rewards/margins": 0.7717516422271729, | |
| "rewards/rejected": -1.9203099012374878, | |
| "step": 88 | |
| }, | |
| { | |
| "epoch": 0.6716981132075471, | |
| "grad_norm": 5.189677715301514, | |
| "learning_rate": 3.6919831223628693e-06, | |
| "logps/chosen": -41.43803787231445, | |
| "logps/rejected": -58.55611038208008, | |
| "loss": 0.4709, | |
| "losses/dpo": 0.27158284187316895, | |
| "losses/sft": 1.2001585960388184, | |
| "losses/total": 0.27158284187316895, | |
| "ref_logps/chosen": -30.10678482055664, | |
| "ref_logps/rejected": -36.89292907714844, | |
| "rewards/accuracies": 0.7578125, | |
| "rewards/chosen": -1.1331257820129395, | |
| "rewards/margins": 1.0331923961639404, | |
| "rewards/rejected": -2.16631817817688, | |
| "step": 89 | |
| }, | |
| { | |
| "epoch": 0.6792452830188679, | |
| "grad_norm": 5.98684024810791, | |
| "learning_rate": 3.6708860759493675e-06, | |
| "logps/chosen": -43.76606750488281, | |
| "logps/rejected": -52.34052276611328, | |
| "loss": 0.5686, | |
| "losses/dpo": 0.7345205545425415, | |
| "losses/sft": 1.5648329257965088, | |
| "losses/total": 0.7345205545425415, | |
| "ref_logps/chosen": -31.27000617980957, | |
| "ref_logps/rejected": -32.98979187011719, | |
| "rewards/accuracies": 0.71875, | |
| "rewards/chosen": -1.2496063709259033, | |
| "rewards/margins": 0.6854668855667114, | |
| "rewards/rejected": -1.9350732564926147, | |
| "step": 90 | |
| }, | |
| { | |
| "epoch": 0.6867924528301886, | |
| "grad_norm": 5.93943452835083, | |
| "learning_rate": 3.649789029535865e-06, | |
| "logps/chosen": -44.927276611328125, | |
| "logps/rejected": -56.36608123779297, | |
| "loss": 0.5478, | |
| "losses/dpo": 0.31034794449806213, | |
| "losses/sft": 1.5676113367080688, | |
| "losses/total": 0.31034794449806213, | |
| "ref_logps/chosen": -31.989208221435547, | |
| "ref_logps/rejected": -35.02646255493164, | |
| "rewards/accuracies": 0.75, | |
| "rewards/chosen": -1.2938066720962524, | |
| "rewards/margins": 0.8401551246643066, | |
| "rewards/rejected": -2.1339616775512695, | |
| "step": 91 | |
| }, | |
| { | |
| "epoch": 0.6943396226415094, | |
| "grad_norm": 5.51214075088501, | |
| "learning_rate": 3.628691983122363e-06, | |
| "logps/chosen": -44.41264343261719, | |
| "logps/rejected": -60.244510650634766, | |
| "loss": 0.4797, | |
| "losses/dpo": 0.42363566160202026, | |
| "losses/sft": 1.270397424697876, | |
| "losses/total": 0.42363566160202026, | |
| "ref_logps/chosen": -33.250125885009766, | |
| "ref_logps/rejected": -40.045772552490234, | |
| "rewards/accuracies": 0.75, | |
| "rewards/chosen": -1.1162512302398682, | |
| "rewards/margins": 0.9036226868629456, | |
| "rewards/rejected": -2.019874095916748, | |
| "step": 92 | |
| }, | |
| { | |
| "epoch": 0.7018867924528301, | |
| "grad_norm": 5.8951945304870605, | |
| "learning_rate": 3.607594936708861e-06, | |
| "logps/chosen": -41.33027648925781, | |
| "logps/rejected": -54.42854309082031, | |
| "loss": 0.5894, | |
| "losses/dpo": 0.31869563460350037, | |
| "losses/sft": 1.346259593963623, | |
| "losses/total": 0.31869563460350037, | |
| "ref_logps/chosen": -28.775463104248047, | |
| "ref_logps/rejected": -35.096275329589844, | |
| "rewards/accuracies": 0.703125, | |
| "rewards/chosen": -1.255481243133545, | |
| "rewards/margins": 0.6777457594871521, | |
| "rewards/rejected": -1.9332269430160522, | |
| "step": 93 | |
| }, | |
| { | |
| "epoch": 0.7094339622641509, | |
| "grad_norm": 6.491157531738281, | |
| "learning_rate": 3.586497890295359e-06, | |
| "logps/chosen": -43.51985168457031, | |
| "logps/rejected": -54.04357147216797, | |
| "loss": 0.5492, | |
| "losses/dpo": 0.6047073602676392, | |
| "losses/sft": 1.633582353591919, | |
| "losses/total": 0.6047073602676392, | |
| "ref_logps/chosen": -32.43757247924805, | |
| "ref_logps/rejected": -35.235679626464844, | |
| "rewards/accuracies": 0.6640625, | |
| "rewards/chosen": -1.1082279682159424, | |
| "rewards/margins": 0.7725614309310913, | |
| "rewards/rejected": -1.8807893991470337, | |
| "step": 94 | |
| }, | |
| { | |
| "epoch": 0.7169811320754716, | |
| "grad_norm": 5.706908702850342, | |
| "learning_rate": 3.5654008438818566e-06, | |
| "logps/chosen": -44.01639938354492, | |
| "logps/rejected": -56.538856506347656, | |
| "loss": 0.5286, | |
| "losses/dpo": 1.006984829902649, | |
| "losses/sft": 1.5528416633605957, | |
| "losses/total": 1.006984829902649, | |
| "ref_logps/chosen": -31.790197372436523, | |
| "ref_logps/rejected": -36.23029327392578, | |
| "rewards/accuracies": 0.7578125, | |
| "rewards/chosen": -1.2226204872131348, | |
| "rewards/margins": 0.8082359433174133, | |
| "rewards/rejected": -2.0308563709259033, | |
| "step": 95 | |
| }, | |
| { | |
| "epoch": 0.7245283018867924, | |
| "grad_norm": 5.353209018707275, | |
| "learning_rate": 3.544303797468355e-06, | |
| "logps/chosen": -39.303466796875, | |
| "logps/rejected": -55.11781311035156, | |
| "loss": 0.489, | |
| "losses/dpo": 0.3290543556213379, | |
| "losses/sft": 1.5605956315994263, | |
| "losses/total": 0.3290543556213379, | |
| "ref_logps/chosen": -30.248308181762695, | |
| "ref_logps/rejected": -37.410919189453125, | |
| "rewards/accuracies": 0.765625, | |
| "rewards/chosen": -0.9055157899856567, | |
| "rewards/margins": 0.8651739358901978, | |
| "rewards/rejected": -1.770689606666565, | |
| "step": 96 | |
| }, | |
| { | |
| "epoch": 0.7320754716981132, | |
| "grad_norm": 5.129034996032715, | |
| "learning_rate": 3.523206751054853e-06, | |
| "logps/chosen": -39.52154541015625, | |
| "logps/rejected": -57.900733947753906, | |
| "loss": 0.4432, | |
| "losses/dpo": 0.5874547958374023, | |
| "losses/sft": 2.055988073348999, | |
| "losses/total": 0.5874547958374023, | |
| "ref_logps/chosen": -30.57717514038086, | |
| "ref_logps/rejected": -37.92079544067383, | |
| "rewards/accuracies": 0.828125, | |
| "rewards/chosen": -0.8944366574287415, | |
| "rewards/margins": 1.1035571098327637, | |
| "rewards/rejected": -1.99799382686615, | |
| "step": 97 | |
| }, | |
| { | |
| "epoch": 0.7396226415094339, | |
| "grad_norm": 5.658941745758057, | |
| "learning_rate": 3.5021097046413504e-06, | |
| "logps/chosen": -42.83544158935547, | |
| "logps/rejected": -51.84044647216797, | |
| "loss": 0.5452, | |
| "losses/dpo": 0.711673378944397, | |
| "losses/sft": 1.195967674255371, | |
| "losses/total": 0.711673378944397, | |
| "ref_logps/chosen": -32.588661193847656, | |
| "ref_logps/rejected": -34.962364196777344, | |
| "rewards/accuracies": 0.7734375, | |
| "rewards/chosen": -1.0246777534484863, | |
| "rewards/margins": 0.6631301641464233, | |
| "rewards/rejected": -1.6878077983856201, | |
| "step": 98 | |
| }, | |
| { | |
| "epoch": 0.7471698113207547, | |
| "grad_norm": 5.521385669708252, | |
| "learning_rate": 3.4810126582278487e-06, | |
| "logps/chosen": -38.521541595458984, | |
| "logps/rejected": -50.79652786254883, | |
| "loss": 0.5304, | |
| "losses/dpo": 0.3267762064933777, | |
| "losses/sft": 1.4650697708129883, | |
| "losses/total": 0.3267762064933777, | |
| "ref_logps/chosen": -28.887462615966797, | |
| "ref_logps/rejected": -33.21889877319336, | |
| "rewards/accuracies": 0.71875, | |
| "rewards/chosen": -0.9634078741073608, | |
| "rewards/margins": 0.7943546772003174, | |
| "rewards/rejected": -1.7577626705169678, | |
| "step": 99 | |
| }, | |
| { | |
| "epoch": 0.7547169811320755, | |
| "grad_norm": 6.092041492462158, | |
| "learning_rate": 3.459915611814346e-06, | |
| "logps/chosen": -43.435054779052734, | |
| "logps/rejected": -48.984378814697266, | |
| "loss": 0.6331, | |
| "losses/dpo": 0.2698668837547302, | |
| "losses/sft": 1.1696505546569824, | |
| "losses/total": 0.2698668837547302, | |
| "ref_logps/chosen": -31.87053871154785, | |
| "ref_logps/rejected": -31.63926887512207, | |
| "rewards/accuracies": 0.6953125, | |
| "rewards/chosen": -1.1564514636993408, | |
| "rewards/margins": 0.5780597925186157, | |
| "rewards/rejected": -1.734511375427246, | |
| "step": 100 | |
| }, | |
| { | |
| "epoch": 0.7622641509433963, | |
| "grad_norm": 5.298862934112549, | |
| "learning_rate": 3.4388185654008443e-06, | |
| "logps/chosen": -38.33399200439453, | |
| "logps/rejected": -53.81504821777344, | |
| "loss": 0.478, | |
| "losses/dpo": 0.4309498071670532, | |
| "losses/sft": 1.7226063013076782, | |
| "losses/total": 0.4309498071670532, | |
| "ref_logps/chosen": -29.21530532836914, | |
| "ref_logps/rejected": -36.157432556152344, | |
| "rewards/accuracies": 0.7890625, | |
| "rewards/chosen": -0.9118687510490417, | |
| "rewards/margins": 0.8538926839828491, | |
| "rewards/rejected": -1.765761375427246, | |
| "step": 101 | |
| }, | |
| { | |
| "epoch": 0.769811320754717, | |
| "grad_norm": 5.454709529876709, | |
| "learning_rate": 3.417721518987342e-06, | |
| "logps/chosen": -36.00552749633789, | |
| "logps/rejected": -47.35252380371094, | |
| "loss": 0.5585, | |
| "losses/dpo": 0.3503888249397278, | |
| "losses/sft": 1.201081395149231, | |
| "losses/total": 0.3503888249397278, | |
| "ref_logps/chosen": -27.280101776123047, | |
| "ref_logps/rejected": -31.31577491760254, | |
| "rewards/accuracies": 0.75, | |
| "rewards/chosen": -0.8725426197052002, | |
| "rewards/margins": 0.7311323881149292, | |
| "rewards/rejected": -1.6036750078201294, | |
| "step": 102 | |
| }, | |
| { | |
| "epoch": 0.7773584905660378, | |
| "grad_norm": 5.91898250579834, | |
| "learning_rate": 3.39662447257384e-06, | |
| "logps/chosen": -41.72618103027344, | |
| "logps/rejected": -53.16327667236328, | |
| "loss": 0.5765, | |
| "losses/dpo": 0.3589654564857483, | |
| "losses/sft": 1.56475031375885, | |
| "losses/total": 0.3589654564857483, | |
| "ref_logps/chosen": -30.899045944213867, | |
| "ref_logps/rejected": -34.665802001953125, | |
| "rewards/accuracies": 0.6875, | |
| "rewards/chosen": -1.0827136039733887, | |
| "rewards/margins": 0.7670339941978455, | |
| "rewards/rejected": -1.8497475385665894, | |
| "step": 103 | |
| }, | |
| { | |
| "epoch": 0.7849056603773585, | |
| "grad_norm": 5.455533981323242, | |
| "learning_rate": 3.3755274261603377e-06, | |
| "logps/chosen": -41.35075378417969, | |
| "logps/rejected": -52.81592559814453, | |
| "loss": 0.5401, | |
| "losses/dpo": 0.39715278148651123, | |
| "losses/sft": 1.4784698486328125, | |
| "losses/total": 0.39715278148651123, | |
| "ref_logps/chosen": -30.155914306640625, | |
| "ref_logps/rejected": -34.45831298828125, | |
| "rewards/accuracies": 0.734375, | |
| "rewards/chosen": -1.1194841861724854, | |
| "rewards/margins": 0.7162774801254272, | |
| "rewards/rejected": -1.835761547088623, | |
| "step": 104 | |
| }, | |
| { | |
| "epoch": 0.7924528301886793, | |
| "grad_norm": 4.831524848937988, | |
| "learning_rate": 3.354430379746836e-06, | |
| "logps/chosen": -38.7393684387207, | |
| "logps/rejected": -53.984886169433594, | |
| "loss": 0.4876, | |
| "losses/dpo": 0.5145635604858398, | |
| "losses/sft": 1.3864595890045166, | |
| "losses/total": 0.5145635604858398, | |
| "ref_logps/chosen": -27.76490020751953, | |
| "ref_logps/rejected": -34.16275405883789, | |
| "rewards/accuracies": 0.7109375, | |
| "rewards/chosen": -1.0974467992782593, | |
| "rewards/margins": 0.8847663402557373, | |
| "rewards/rejected": -1.9822131395339966, | |
| "step": 105 | |
| }, | |
| { | |
| "epoch": 0.8, | |
| "grad_norm": 4.85020637512207, | |
| "learning_rate": 3.3333333333333333e-06, | |
| "logps/chosen": -34.77644729614258, | |
| "logps/rejected": -51.18222427368164, | |
| "loss": 0.5472, | |
| "losses/dpo": 0.6919156908988953, | |
| "losses/sft": 1.51556396484375, | |
| "losses/total": 0.6919156908988953, | |
| "ref_logps/chosen": -24.479114532470703, | |
| "ref_logps/rejected": -33.03002166748047, | |
| "rewards/accuracies": 0.703125, | |
| "rewards/chosen": -1.0297331809997559, | |
| "rewards/margins": 0.785487174987793, | |
| "rewards/rejected": -1.8152204751968384, | |
| "step": 106 | |
| }, | |
| { | |
| "epoch": 0.8075471698113208, | |
| "grad_norm": 5.955616474151611, | |
| "learning_rate": 3.3122362869198316e-06, | |
| "logps/chosen": -43.87641143798828, | |
| "logps/rejected": -54.51424789428711, | |
| "loss": 0.6114, | |
| "losses/dpo": 0.756654679775238, | |
| "losses/sft": 1.6842637062072754, | |
| "losses/total": 0.756654679775238, | |
| "ref_logps/chosen": -31.69654083251953, | |
| "ref_logps/rejected": -36.67472839355469, | |
| "rewards/accuracies": 0.734375, | |
| "rewards/chosen": -1.217987060546875, | |
| "rewards/margins": 0.565964937210083, | |
| "rewards/rejected": -1.7839521169662476, | |
| "step": 107 | |
| }, | |
| { | |
| "epoch": 0.8150943396226416, | |
| "grad_norm": 5.920444965362549, | |
| "learning_rate": 3.2911392405063294e-06, | |
| "logps/chosen": -40.982208251953125, | |
| "logps/rejected": -52.83149337768555, | |
| "loss": 0.5517, | |
| "losses/dpo": 0.65492182970047, | |
| "losses/sft": 1.6355092525482178, | |
| "losses/total": 0.65492182970047, | |
| "ref_logps/chosen": -29.412181854248047, | |
| "ref_logps/rejected": -34.126094818115234, | |
| "rewards/accuracies": 0.6796875, | |
| "rewards/chosen": -1.157002329826355, | |
| "rewards/margins": 0.7135379910469055, | |
| "rewards/rejected": -1.8705402612686157, | |
| "step": 108 | |
| }, | |
| { | |
| "epoch": 0.8226415094339623, | |
| "grad_norm": 5.697230339050293, | |
| "learning_rate": 3.270042194092827e-06, | |
| "logps/chosen": -42.04249954223633, | |
| "logps/rejected": -56.36012268066406, | |
| "loss": 0.5426, | |
| "losses/dpo": 0.3828045725822449, | |
| "losses/sft": 1.5319883823394775, | |
| "losses/total": 0.3828045725822449, | |
| "ref_logps/chosen": -31.600698471069336, | |
| "ref_logps/rejected": -36.47235107421875, | |
| "rewards/accuracies": 0.7265625, | |
| "rewards/chosen": -1.0441800355911255, | |
| "rewards/margins": 0.9445971846580505, | |
| "rewards/rejected": -1.9887771606445312, | |
| "step": 109 | |
| }, | |
| { | |
| "epoch": 0.8301886792452831, | |
| "grad_norm": 6.10832405090332, | |
| "learning_rate": 3.248945147679325e-06, | |
| "logps/chosen": -39.39381408691406, | |
| "logps/rejected": -50.33586120605469, | |
| "loss": 0.6518, | |
| "losses/dpo": 0.5053750872612, | |
| "losses/sft": 1.3503327369689941, | |
| "losses/total": 0.5053750872612, | |
| "ref_logps/chosen": -27.708271026611328, | |
| "ref_logps/rejected": -33.47583770751953, | |
| "rewards/accuracies": 0.640625, | |
| "rewards/chosen": -1.1685543060302734, | |
| "rewards/margins": 0.5174481272697449, | |
| "rewards/rejected": -1.6860023736953735, | |
| "step": 110 | |
| }, | |
| { | |
| "epoch": 0.8377358490566038, | |
| "grad_norm": 5.662174224853516, | |
| "learning_rate": 3.2278481012658232e-06, | |
| "logps/chosen": -41.917667388916016, | |
| "logps/rejected": -55.30170440673828, | |
| "loss": 0.5537, | |
| "losses/dpo": 0.9183385372161865, | |
| "losses/sft": 1.6308963298797607, | |
| "losses/total": 0.9183385372161865, | |
| "ref_logps/chosen": -29.052471160888672, | |
| "ref_logps/rejected": -34.99309539794922, | |
| "rewards/accuracies": 0.7109375, | |
| "rewards/chosen": -1.2865197658538818, | |
| "rewards/margins": 0.7443405985832214, | |
| "rewards/rejected": -2.030860424041748, | |
| "step": 111 | |
| }, | |
| { | |
| "epoch": 0.8452830188679246, | |
| "grad_norm": 5.386070728302002, | |
| "learning_rate": 3.206751054852321e-06, | |
| "logps/chosen": -43.091156005859375, | |
| "logps/rejected": -57.210243225097656, | |
| "loss": 0.5028, | |
| "losses/dpo": 0.3924194872379303, | |
| "losses/sft": 1.2852116823196411, | |
| "losses/total": 0.3924194872379303, | |
| "ref_logps/chosen": -30.833717346191406, | |
| "ref_logps/rejected": -36.638153076171875, | |
| "rewards/accuracies": 0.78125, | |
| "rewards/chosen": -1.2257441282272339, | |
| "rewards/margins": 0.8314655423164368, | |
| "rewards/rejected": -2.0572097301483154, | |
| "step": 112 | |
| }, | |
| { | |
| "epoch": 0.8528301886792453, | |
| "grad_norm": 6.078062057495117, | |
| "learning_rate": 3.185654008438819e-06, | |
| "logps/chosen": -44.7150764465332, | |
| "logps/rejected": -60.601261138916016, | |
| "loss": 0.5275, | |
| "losses/dpo": 0.3586404323577881, | |
| "losses/sft": 1.5925896167755127, | |
| "losses/total": 0.3586404323577881, | |
| "ref_logps/chosen": -31.377941131591797, | |
| "ref_logps/rejected": -38.768470764160156, | |
| "rewards/accuracies": 0.7265625, | |
| "rewards/chosen": -1.3337135314941406, | |
| "rewards/margins": 0.8495657444000244, | |
| "rewards/rejected": -2.183279037475586, | |
| "step": 113 | |
| }, | |
| { | |
| "epoch": 0.8603773584905661, | |
| "grad_norm": 5.526185035705566, | |
| "learning_rate": 3.164556962025317e-06, | |
| "logps/chosen": -42.03477478027344, | |
| "logps/rejected": -54.82069778442383, | |
| "loss": 0.5588, | |
| "losses/dpo": 1.1230486631393433, | |
| "losses/sft": 1.6963893175125122, | |
| "losses/total": 1.1230486631393433, | |
| "ref_logps/chosen": -28.538129806518555, | |
| "ref_logps/rejected": -33.739173889160156, | |
| "rewards/accuracies": 0.71875, | |
| "rewards/chosen": -1.3496648073196411, | |
| "rewards/margins": 0.75848788022995, | |
| "rewards/rejected": -2.1081528663635254, | |
| "step": 114 | |
| }, | |
| { | |
| "epoch": 0.8679245283018868, | |
| "grad_norm": 5.497169017791748, | |
| "learning_rate": 3.1434599156118145e-06, | |
| "logps/chosen": -42.89957046508789, | |
| "logps/rejected": -57.447914123535156, | |
| "loss": 0.523, | |
| "losses/dpo": 0.7913863658905029, | |
| "losses/sft": 1.356689453125, | |
| "losses/total": 0.7913863658905029, | |
| "ref_logps/chosen": -30.25906753540039, | |
| "ref_logps/rejected": -36.00640869140625, | |
| "rewards/accuracies": 0.78125, | |
| "rewards/chosen": -1.2640503644943237, | |
| "rewards/margins": 0.8801001906394958, | |
| "rewards/rejected": -2.144150733947754, | |
| "step": 115 | |
| }, | |
| { | |
| "epoch": 0.8754716981132076, | |
| "grad_norm": 5.6127824783325195, | |
| "learning_rate": 3.1223628691983127e-06, | |
| "logps/chosen": -46.81511688232422, | |
| "logps/rejected": -56.43955993652344, | |
| "loss": 0.5401, | |
| "losses/dpo": 0.7304984927177429, | |
| "losses/sft": 1.7529406547546387, | |
| "losses/total": 0.7304984927177429, | |
| "ref_logps/chosen": -32.37680435180664, | |
| "ref_logps/rejected": -34.742347717285156, | |
| "rewards/accuracies": 0.7421875, | |
| "rewards/chosen": -1.4438307285308838, | |
| "rewards/margins": 0.7258903980255127, | |
| "rewards/rejected": -2.1697211265563965, | |
| "step": 116 | |
| }, | |
| { | |
| "epoch": 0.8830188679245283, | |
| "grad_norm": 6.019126892089844, | |
| "learning_rate": 3.10126582278481e-06, | |
| "logps/chosen": -44.45075988769531, | |
| "logps/rejected": -56.06095886230469, | |
| "loss": 0.5493, | |
| "losses/dpo": 0.5076150298118591, | |
| "losses/sft": 1.855008602142334, | |
| "losses/total": 0.5076150298118591, | |
| "ref_logps/chosen": -29.896549224853516, | |
| "ref_logps/rejected": -33.72525405883789, | |
| "rewards/accuracies": 0.65625, | |
| "rewards/chosen": -1.4554212093353271, | |
| "rewards/margins": 0.7781496644020081, | |
| "rewards/rejected": -2.2335708141326904, | |
| "step": 117 | |
| }, | |
| { | |
| "epoch": 0.8905660377358491, | |
| "grad_norm": 4.897617816925049, | |
| "learning_rate": 3.0801687763713083e-06, | |
| "logps/chosen": -44.17365264892578, | |
| "logps/rejected": -59.890846252441406, | |
| "loss": 0.4334, | |
| "losses/dpo": 0.41634368896484375, | |
| "losses/sft": 1.3092498779296875, | |
| "losses/total": 0.41634368896484375, | |
| "ref_logps/chosen": -30.777435302734375, | |
| "ref_logps/rejected": -36.164852142333984, | |
| "rewards/accuracies": 0.859375, | |
| "rewards/chosen": -1.3396217823028564, | |
| "rewards/margins": 1.0329774618148804, | |
| "rewards/rejected": -2.3725991249084473, | |
| "step": 118 | |
| }, | |
| { | |
| "epoch": 0.8981132075471698, | |
| "grad_norm": 5.326719760894775, | |
| "learning_rate": 3.059071729957806e-06, | |
| "logps/chosen": -41.951332092285156, | |
| "logps/rejected": -55.92094039916992, | |
| "loss": 0.4896, | |
| "losses/dpo": 0.36023202538490295, | |
| "losses/sft": 1.5721536874771118, | |
| "losses/total": 0.36023202538490295, | |
| "ref_logps/chosen": -29.51433563232422, | |
| "ref_logps/rejected": -34.74650573730469, | |
| "rewards/accuracies": 0.796875, | |
| "rewards/chosen": -1.2436996698379517, | |
| "rewards/margins": 0.8737441301345825, | |
| "rewards/rejected": -2.1174440383911133, | |
| "step": 119 | |
| }, | |
| { | |
| "epoch": 0.9056603773584906, | |
| "grad_norm": 5.39622688293457, | |
| "learning_rate": 3.037974683544304e-06, | |
| "logps/chosen": -42.2480354309082, | |
| "logps/rejected": -58.74956512451172, | |
| "loss": 0.51, | |
| "losses/dpo": 0.6462827920913696, | |
| "losses/sft": 1.5755528211593628, | |
| "losses/total": 0.6462827920913696, | |
| "ref_logps/chosen": -28.274633407592773, | |
| "ref_logps/rejected": -35.56834411621094, | |
| "rewards/accuracies": 0.7421875, | |
| "rewards/chosen": -1.397340178489685, | |
| "rewards/margins": 0.9207824468612671, | |
| "rewards/rejected": -2.318122625350952, | |
| "step": 120 | |
| } | |
| ], | |
| "logging_steps": 1.0, | |
| "max_steps": 264, | |
| "num_input_tokens_seen": 0, | |
| "num_train_epochs": 2, | |
| "save_steps": 40, | |
| "total_flos": 0.0, | |
| "train_batch_size": 2, | |
| "trial_name": null, | |
| "trial_params": null | |
| } | |