diff --git "a/QLoRA_German-ORPO/trainer_state.json" "b/QLoRA_German-ORPO/trainer_state.json" new file mode 100644--- /dev/null +++ "b/QLoRA_German-ORPO/trainer_state.json" @@ -0,0 +1,23173 @@ +{ + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 1.0, + "eval_steps": 500, + "global_step": 12855, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.0007779074290159471, + "grad_norm": 5.248451232910156, + "learning_rate": 4.9999925343886e-05, + "logits/chosen": -0.08551748096942902, + "logits/rejected": -1.3160173892974854, + "logps/chosen": -0.9243558645248413, + "logps/rejected": -0.9934996366500854, + "loss": 0.9946, + "num_input_tokens_seen": 12288, + "odds_ratio_loss": 9.867249488830566, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.09243559837341309, + "rewards/margins": 0.00691437441855669, + "rewards/rejected": -0.0993499681353569, + "sft_loss": 0.007888625375926495, + "step": 10 + }, + { + "epoch": 0.0015558148580318942, + "grad_norm": 21.039575576782227, + "learning_rate": 4.999970137598988e-05, + "logits/chosen": -0.35449445247650146, + "logits/rejected": -0.5382658839225769, + "logps/chosen": -1.1569985151290894, + "logps/rejected": -1.1496927738189697, + "loss": 1.2427, + "num_input_tokens_seen": 26128, + "odds_ratio_loss": 11.52686882019043, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.11569985002279282, + "rewards/margins": -0.000730576750356704, + "rewards/rejected": -0.11496929079294205, + "sft_loss": 0.08998985588550568, + "step": 20 + }, + { + "epoch": 0.002333722287047841, + "grad_norm": 9.672579765319824, + "learning_rate": 4.999932809764929e-05, + "logits/chosen": -0.048610180616378784, + "logits/rejected": -1.061124563217163, + "logps/chosen": -1.2201629877090454, + "logps/rejected": -1.2427245378494263, + "loss": 1.2893, + "num_input_tokens_seen": 41488, + "odds_ratio_loss": 11.61978530883789, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.1220162883400917, + "rewards/margins": 0.0022561654914170504, + "rewards/rejected": -0.12427246570587158, + "sft_loss": 0.12736229598522186, + "step": 30 + }, + { + "epoch": 0.0031116297160637884, + "grad_norm": 3.708526372909546, + "learning_rate": 4.999880551109363e-05, + "logits/chosen": -0.2156565636396408, + "logits/rejected": -1.1819674968719482, + "logps/chosen": -1.1763250827789307, + "logps/rejected": -1.1008272171020508, + "loss": 1.2549, + "num_input_tokens_seen": 53664, + "odds_ratio_loss": 12.200116157531738, + "rewards/accuracies": 0.20000000298023224, + "rewards/chosen": -0.11763250827789307, + "rewards/margins": -0.007549784146249294, + "rewards/rejected": -0.1100827306509018, + "sft_loss": 0.03486320376396179, + "step": 40 + }, + { + "epoch": 0.0038895371450797353, + "grad_norm": 4.0965495109558105, + "learning_rate": 4.999813361944403e-05, + "logits/chosen": -0.061433859169483185, + "logits/rejected": -0.9057363271713257, + "logps/chosen": -0.7127934694290161, + "logps/rejected": -2.0076801776885986, + "loss": 0.7365, + "num_input_tokens_seen": 62688, + "odds_ratio_loss": 7.168131351470947, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.07127933204174042, + "rewards/margins": 0.12948869168758392, + "rewards/rejected": -0.20076803863048553, + "sft_loss": 0.01973068341612816, + "step": 50 + }, + { + "epoch": 0.004667444574095682, + "grad_norm": 2.570751190185547, + "learning_rate": 4.9997312426713375e-05, + "logits/chosen": -0.030793923884630203, + "logits/rejected": -0.8716487884521484, + "logps/chosen": -0.8243891596794128, + "logps/rejected": -1.0122798681259155, + "loss": 0.8873, + "num_input_tokens_seen": 76224, + "odds_ratio_loss": 8.698861122131348, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.08243891596794128, + "rewards/margins": 0.018789071589708328, + "rewards/rejected": -0.10122797638177872, + "sft_loss": 0.017441704869270325, + "step": 60 + }, + { + "epoch": 0.0054453520031116295, + "grad_norm": 4.000652313232422, + "learning_rate": 4.9996341937806215e-05, + "logits/chosen": -0.3607181906700134, + "logits/rejected": -0.8948682546615601, + "logps/chosen": -1.1096030473709106, + "logps/rejected": -1.1755973100662231, + "loss": 1.1815, + "num_input_tokens_seen": 85600, + "odds_ratio_loss": 9.79625415802002, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.11096030473709106, + "rewards/margins": 0.006599433720111847, + "rewards/rejected": -0.11755974590778351, + "sft_loss": 0.2019163817167282, + "step": 70 + }, + { + "epoch": 0.006223259432127577, + "grad_norm": 48.832618713378906, + "learning_rate": 4.999522215851879e-05, + "logits/chosen": 0.17318834364414215, + "logits/rejected": -1.169816493988037, + "logps/chosen": -1.2467583417892456, + "logps/rejected": -1.9135162830352783, + "loss": 1.3283, + "num_input_tokens_seen": 98944, + "odds_ratio_loss": 11.729827880859375, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.12467584758996964, + "rewards/margins": 0.06667578965425491, + "rewards/rejected": -0.19135163724422455, + "sft_loss": 0.15530584752559662, + "step": 80 + }, + { + "epoch": 0.007001166861143524, + "grad_norm": 2.759312629699707, + "learning_rate": 4.999395309553898e-05, + "logits/chosen": 0.00044447611435316503, + "logits/rejected": -1.0682010650634766, + "logps/chosen": -0.9102838635444641, + "logps/rejected": -1.688788652420044, + "loss": 0.9855, + "num_input_tokens_seen": 112032, + "odds_ratio_loss": 9.6449613571167, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.09102839231491089, + "rewards/margins": 0.07785046845674515, + "rewards/rejected": -0.16887886822223663, + "sft_loss": 0.02097008004784584, + "step": 90 + }, + { + "epoch": 0.007779074290159471, + "grad_norm": 2.1516456604003906, + "learning_rate": 4.999253475644623e-05, + "logits/chosen": -0.36004287004470825, + "logits/rejected": -0.6734232902526855, + "logps/chosen": -0.8154226541519165, + "logps/rejected": -1.0999035835266113, + "loss": 0.8797, + "num_input_tokens_seen": 123424, + "odds_ratio_loss": 8.56216049194336, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.08154226839542389, + "rewards/margins": 0.028448080644011497, + "rewards/rejected": -0.10999034345149994, + "sft_loss": 0.02346923016011715, + "step": 100 + }, + { + "epoch": 0.008556981719175419, + "grad_norm": 1.4805128574371338, + "learning_rate": 4.999096714971156e-05, + "logits/chosen": -0.04372026026248932, + "logits/rejected": -0.735072910785675, + "logps/chosen": -1.1332687139511108, + "logps/rejected": -1.4848557710647583, + "loss": 1.2075, + "num_input_tokens_seen": 138400, + "odds_ratio_loss": 11.999357223510742, + "rewards/accuracies": 0.4000000059604645, + "rewards/chosen": -0.11332686990499496, + "rewards/margins": 0.03515869751572609, + "rewards/rejected": -0.14848557114601135, + "sft_loss": 0.0076140896417200565, + "step": 110 + }, + { + "epoch": 0.009334889148191364, + "grad_norm": 3.1642253398895264, + "learning_rate": 4.99892502846975e-05, + "logits/chosen": 0.1289406716823578, + "logits/rejected": -0.9220048785209656, + "logps/chosen": -0.7731643319129944, + "logps/rejected": -1.410657525062561, + "loss": 0.8176, + "num_input_tokens_seen": 153360, + "odds_ratio_loss": 8.046560287475586, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.07731644809246063, + "rewards/margins": 0.06374931335449219, + "rewards/rejected": -0.14106574654579163, + "sft_loss": 0.01298043318092823, + "step": 120 + }, + { + "epoch": 0.010112796577207312, + "grad_norm": 2.5500879287719727, + "learning_rate": 4.9987384171657994e-05, + "logits/chosen": 0.19079621136188507, + "logits/rejected": -1.2353627681732178, + "logps/chosen": -0.9447005391120911, + "logps/rejected": -1.266804575920105, + "loss": 1.0001, + "num_input_tokens_seen": 165072, + "odds_ratio_loss": 9.937886238098145, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.0944700613617897, + "rewards/margins": 0.032210398465394974, + "rewards/rejected": -0.12668046355247498, + "sft_loss": 0.006324116140604019, + "step": 130 + }, + { + "epoch": 0.010890704006223259, + "grad_norm": 7.7927422523498535, + "learning_rate": 4.9985368821738385e-05, + "logits/chosen": -0.07595299184322357, + "logits/rejected": -0.8760563731193542, + "logps/chosen": -0.8572665452957153, + "logps/rejected": -2.1462793350219727, + "loss": 0.8903, + "num_input_tokens_seen": 175824, + "odds_ratio_loss": 8.764748573303223, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.08572666347026825, + "rewards/margins": 0.12890127301216125, + "rewards/rejected": -0.2146279513835907, + "sft_loss": 0.013779623433947563, + "step": 140 + }, + { + "epoch": 0.011668611435239206, + "grad_norm": 3.3617570400238037, + "learning_rate": 4.998320424697532e-05, + "logits/chosen": 0.15023799240589142, + "logits/rejected": -0.7425843477249146, + "logps/chosen": -0.9067550897598267, + "logps/rejected": -1.3671596050262451, + "loss": 0.9588, + "num_input_tokens_seen": 186320, + "odds_ratio_loss": 9.49233341217041, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.09067549556493759, + "rewards/margins": 0.046040456742048264, + "rewards/rejected": -0.13671596348285675, + "sft_loss": 0.009604133665561676, + "step": 150 + }, + { + "epoch": 0.012446518864255154, + "grad_norm": 2.341899871826172, + "learning_rate": 4.998089046029671e-05, + "logits/chosen": 0.06832513958215714, + "logits/rejected": -0.9852465391159058, + "logps/chosen": -0.9017775654792786, + "logps/rejected": -0.9677242040634155, + "loss": 0.9697, + "num_input_tokens_seen": 203072, + "odds_ratio_loss": 9.57034969329834, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.0901777595281601, + "rewards/margins": 0.006594663951545954, + "rewards/rejected": -0.09677241742610931, + "sft_loss": 0.012652057223021984, + "step": 160 + }, + { + "epoch": 0.013224426293271101, + "grad_norm": 7.437110900878906, + "learning_rate": 4.9978427475521616e-05, + "logits/chosen": -0.009473335929214954, + "logits/rejected": -0.7211953401565552, + "logps/chosen": -0.9088319540023804, + "logps/rejected": -1.050347089767456, + "loss": 0.9788, + "num_input_tokens_seen": 214336, + "odds_ratio_loss": 9.662178993225098, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.09088320285081863, + "rewards/margins": 0.014151516370475292, + "rewards/rejected": -0.1050347238779068, + "sft_loss": 0.012536203488707542, + "step": 170 + }, + { + "epoch": 0.014002333722287048, + "grad_norm": 3.429651975631714, + "learning_rate": 4.9975815307360194e-05, + "logits/chosen": -0.16578452289104462, + "logits/rejected": -0.4752795100212097, + "logps/chosen": -0.8099583387374878, + "logps/rejected": -0.9804008603096008, + "loss": 0.8951, + "num_input_tokens_seen": 232368, + "odds_ratio_loss": 8.755950927734375, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.0809958428144455, + "rewards/margins": 0.017044242471456528, + "rewards/rejected": -0.09804007411003113, + "sft_loss": 0.019553907215595245, + "step": 180 + }, + { + "epoch": 0.014780241151302996, + "grad_norm": 5.283464431762695, + "learning_rate": 4.997305397141357e-05, + "logits/chosen": -0.18071290850639343, + "logits/rejected": -0.5703662037849426, + "logps/chosen": -0.9150312542915344, + "logps/rejected": -1.393818974494934, + "loss": 0.9661, + "num_input_tokens_seen": 242960, + "odds_ratio_loss": 9.338902473449707, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.09150312840938568, + "rewards/margins": 0.04787876456975937, + "rewards/rejected": -0.13938190042972565, + "sft_loss": 0.03223598375916481, + "step": 190 + }, + { + "epoch": 0.015558148580318941, + "grad_norm": 2.1387197971343994, + "learning_rate": 4.997014348417382e-05, + "logits/chosen": 0.12889909744262695, + "logits/rejected": -1.100764513015747, + "logps/chosen": -0.8966400027275085, + "logps/rejected": -1.0923173427581787, + "loss": 0.9529, + "num_input_tokens_seen": 258080, + "odds_ratio_loss": 9.400256156921387, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.08966399729251862, + "rewards/margins": 0.019567731767892838, + "rewards/rejected": -0.10923173278570175, + "sft_loss": 0.012894442304968834, + "step": 200 + }, + { + "epoch": 0.01633605600933489, + "grad_norm": 4.736515998840332, + "learning_rate": 4.9967083863023776e-05, + "logits/chosen": -0.09360722452402115, + "logits/rejected": -1.2860757112503052, + "logps/chosen": -1.1631531715393066, + "logps/rejected": -1.368138074874878, + "loss": 1.2197, + "num_input_tokens_seen": 271920, + "odds_ratio_loss": 11.848650932312012, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.1163153275847435, + "rewards/margins": 0.020498499274253845, + "rewards/rejected": -0.13681381940841675, + "sft_loss": 0.034875985234975815, + "step": 210 + }, + { + "epoch": 0.017113963438350838, + "grad_norm": 4.38301944732666, + "learning_rate": 4.9963875126237016e-05, + "logits/chosen": -0.29661667346954346, + "logits/rejected": -0.6016083359718323, + "logps/chosen": -0.7634471654891968, + "logps/rejected": -0.9885290265083313, + "loss": 0.8338, + "num_input_tokens_seen": 286576, + "odds_ratio_loss": 8.109630584716797, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.07634471356868744, + "rewards/margins": 0.022508183494210243, + "rewards/rejected": -0.09885289520025253, + "sft_loss": 0.0228743739426136, + "step": 220 + }, + { + "epoch": 0.017891870867366783, + "grad_norm": 2.4642908573150635, + "learning_rate": 4.9960517292977655e-05, + "logits/chosen": -0.04434485360980034, + "logits/rejected": -0.9968097805976868, + "logps/chosen": -1.2226582765579224, + "logps/rejected": -2.301300525665283, + "loss": 1.2993, + "num_input_tokens_seen": 296896, + "odds_ratio_loss": 12.393956184387207, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.12226583063602448, + "rewards/margins": 0.10786418616771698, + "rewards/rejected": -0.23013003170490265, + "sft_loss": 0.05994652584195137, + "step": 230 + }, + { + "epoch": 0.01866977829638273, + "grad_norm": 2.916964054107666, + "learning_rate": 4.995701038330033e-05, + "logits/chosen": -0.3185642659664154, + "logits/rejected": -1.0956424474716187, + "logps/chosen": -1.0076221227645874, + "logps/rejected": -1.1618595123291016, + "loss": 1.0844, + "num_input_tokens_seen": 311296, + "odds_ratio_loss": 9.93652057647705, + "rewards/accuracies": 0.4000000059604645, + "rewards/chosen": -0.10076222568750381, + "rewards/margins": 0.01542372815310955, + "rewards/rejected": -0.11618594825267792, + "sft_loss": 0.09077021479606628, + "step": 240 + }, + { + "epoch": 0.019447685725398678, + "grad_norm": 2.7747602462768555, + "learning_rate": 4.995335441815003e-05, + "logits/chosen": 0.045975618064403534, + "logits/rejected": -1.4385907649993896, + "logps/chosen": -1.1574515104293823, + "logps/rejected": -1.4619978666305542, + "loss": 1.2243, + "num_input_tokens_seen": 321184, + "odds_ratio_loss": 12.18043327331543, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.11574514955282211, + "rewards/margins": 0.030454616993665695, + "rewards/rejected": -0.1461997926235199, + "sft_loss": 0.00630641495808959, + "step": 250 + }, + { + "epoch": 0.020225593154414623, + "grad_norm": 16.452539443969727, + "learning_rate": 4.9949549419361954e-05, + "logits/chosen": -0.2351895123720169, + "logits/rejected": -1.024817705154419, + "logps/chosen": -0.9535134434700012, + "logps/rejected": -1.9824984073638916, + "loss": 1.0084, + "num_input_tokens_seen": 331328, + "odds_ratio_loss": 9.948028564453125, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.09535135328769684, + "rewards/margins": 0.1028984934091568, + "rewards/rejected": -0.19824984669685364, + "sft_loss": 0.013568353839218616, + "step": 260 + }, + { + "epoch": 0.021003500583430573, + "grad_norm": 3.6468498706817627, + "learning_rate": 4.994559540966142e-05, + "logits/chosen": -0.022709887474775314, + "logits/rejected": -1.4626227617263794, + "logps/chosen": -0.9807567596435547, + "logps/rejected": -1.9314972162246704, + "loss": 1.0466, + "num_input_tokens_seen": 345456, + "odds_ratio_loss": 10.377405166625977, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.09807567298412323, + "rewards/margins": 0.09507404267787933, + "rewards/rejected": -0.19314971566200256, + "sft_loss": 0.008827922865748405, + "step": 270 + }, + { + "epoch": 0.021781408012446518, + "grad_norm": 6.5957465171813965, + "learning_rate": 4.994149241266371e-05, + "logits/chosen": -0.41178959608078003, + "logits/rejected": -0.37596777081489563, + "logps/chosen": -1.0016124248504639, + "logps/rejected": -0.9876958727836609, + "loss": 1.0896, + "num_input_tokens_seen": 357568, + "odds_ratio_loss": 10.677938461303711, + "rewards/accuracies": 0.4000000059604645, + "rewards/chosen": -0.10016125440597534, + "rewards/margins": -0.0013916685711592436, + "rewards/rejected": -0.09876958280801773, + "sft_loss": 0.02179970219731331, + "step": 280 + }, + { + "epoch": 0.022559315441462467, + "grad_norm": 1.5292377471923828, + "learning_rate": 4.993724045287392e-05, + "logits/chosen": 0.10405155271291733, + "logits/rejected": -1.1914265155792236, + "logps/chosen": -0.9256628751754761, + "logps/rejected": -1.3289026021957397, + "loss": 0.9888, + "num_input_tokens_seen": 373472, + "odds_ratio_loss": 9.648874282836914, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.09256630390882492, + "rewards/margins": 0.040323980152606964, + "rewards/rejected": -0.13289028406143188, + "sft_loss": 0.023866329342126846, + "step": 290 + }, + { + "epoch": 0.023337222870478413, + "grad_norm": 2.4046130180358887, + "learning_rate": 4.993283955568685e-05, + "logits/chosen": 0.010790085420012474, + "logits/rejected": -1.7810585498809814, + "logps/chosen": -0.8726987838745117, + "logps/rejected": -0.8844345211982727, + "loss": 0.9519, + "num_input_tokens_seen": 386080, + "odds_ratio_loss": 9.43667984008789, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.08726988732814789, + "rewards/margins": 0.001173576689325273, + "rewards/rejected": -0.08844345062971115, + "sft_loss": 0.008213376626372337, + "step": 300 + }, + { + "epoch": 0.024115130299494362, + "grad_norm": 3.012260913848877, + "learning_rate": 4.99282897473868e-05, + "logits/chosen": -0.04694366827607155, + "logits/rejected": -1.3969290256500244, + "logps/chosen": -0.9051028490066528, + "logps/rejected": -1.2001574039459229, + "loss": 0.961, + "num_input_tokens_seen": 396368, + "odds_ratio_loss": 9.4262056350708, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.0905102863907814, + "rewards/margins": 0.029505455866456032, + "rewards/rejected": -0.12001574039459229, + "sft_loss": 0.018341612070798874, + "step": 310 + }, + { + "epoch": 0.024893037728510307, + "grad_norm": 1.637371301651001, + "learning_rate": 4.9923591055147455e-05, + "logits/chosen": 0.16533169150352478, + "logits/rejected": -1.5700806379318237, + "logps/chosen": -1.0949220657348633, + "logps/rejected": -1.2998539209365845, + "loss": 1.1594, + "num_input_tokens_seen": 414368, + "odds_ratio_loss": 11.56415843963623, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.10949220508337021, + "rewards/margins": 0.020493196323513985, + "rewards/rejected": -0.12998540699481964, + "sft_loss": 0.0029837344773113728, + "step": 320 + }, + { + "epoch": 0.025670945157526253, + "grad_norm": 3.8987412452697754, + "learning_rate": 4.99187435070317e-05, + "logits/chosen": -0.1262105256319046, + "logits/rejected": -1.0445555448532104, + "logps/chosen": -0.9046597480773926, + "logps/rejected": -1.073553442955017, + "loss": 0.9619, + "num_input_tokens_seen": 424800, + "odds_ratio_loss": 9.364073753356934, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.0904659777879715, + "rewards/margins": 0.016889363527297974, + "rewards/rejected": -0.10735534131526947, + "sft_loss": 0.025530826300382614, + "step": 330 + }, + { + "epoch": 0.026448852586542202, + "grad_norm": 1.8967704772949219, + "learning_rate": 4.991374713199145e-05, + "logits/chosen": -0.1018231064081192, + "logits/rejected": -1.3850098848342896, + "logps/chosen": -1.0545568466186523, + "logps/rejected": -1.2990403175354004, + "loss": 1.1314, + "num_input_tokens_seen": 435984, + "odds_ratio_loss": 11.010573387145996, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.105455681681633, + "rewards/margins": 0.024448346346616745, + "rewards/rejected": -0.12990403175354004, + "sft_loss": 0.0303510669618845, + "step": 340 + }, + { + "epoch": 0.027226760015558148, + "grad_norm": 5.541660785675049, + "learning_rate": 4.990860195986753e-05, + "logits/chosen": -0.24207575619220734, + "logits/rejected": -1.1700140237808228, + "logps/chosen": -1.1388294696807861, + "logps/rejected": -2.223168134689331, + "loss": 1.2046, + "num_input_tokens_seen": 449136, + "odds_ratio_loss": 10.3714599609375, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.11388297379016876, + "rewards/margins": 0.10843388736248016, + "rewards/rejected": -0.22231683135032654, + "sft_loss": 0.16744010150432587, + "step": 350 + }, + { + "epoch": 0.028004667444574097, + "grad_norm": 1.861402153968811, + "learning_rate": 4.990330802138941e-05, + "logits/chosen": -0.31768426299095154, + "logits/rejected": -1.2700172662734985, + "logps/chosen": -0.8037441372871399, + "logps/rejected": -1.0384769439697266, + "loss": 0.8601, + "num_input_tokens_seen": 461712, + "odds_ratio_loss": 8.533435821533203, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.08037441223859787, + "rewards/margins": 0.023473283275961876, + "rewards/rejected": -0.10384770482778549, + "sft_loss": 0.006723404861986637, + "step": 360 + }, + { + "epoch": 0.028782574873590042, + "grad_norm": 8.835140228271484, + "learning_rate": 4.989786534817507e-05, + "logits/chosen": -0.3392910659313202, + "logits/rejected": -1.3058607578277588, + "logps/chosen": -0.8710435628890991, + "logps/rejected": -1.9576623439788818, + "loss": 0.9149, + "num_input_tokens_seen": 471328, + "odds_ratio_loss": 8.99726390838623, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.08710435032844543, + "rewards/margins": 0.10866190493106842, + "rewards/rejected": -0.19576624035835266, + "sft_loss": 0.015141059644520283, + "step": 370 + }, + { + "epoch": 0.02956048230260599, + "grad_norm": 2.149712562561035, + "learning_rate": 4.989227397273082e-05, + "logits/chosen": -0.45678621530532837, + "logits/rejected": -0.8773413896560669, + "logps/chosen": -0.9752458333969116, + "logps/rejected": -1.3521517515182495, + "loss": 1.0315, + "num_input_tokens_seen": 481728, + "odds_ratio_loss": 10.136479377746582, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.09752458333969116, + "rewards/margins": 0.03769059106707573, + "rewards/rejected": -0.13521519303321838, + "sft_loss": 0.017892986536026, + "step": 380 + }, + { + "epoch": 0.030338389731621937, + "grad_norm": 4.1422553062438965, + "learning_rate": 4.9886533928451104e-05, + "logits/chosen": -0.24568918347358704, + "logits/rejected": -1.1686650514602661, + "logps/chosen": -1.0748536586761475, + "logps/rejected": -1.366524577140808, + "loss": 1.1379, + "num_input_tokens_seen": 491856, + "odds_ratio_loss": 11.042202949523926, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.10748536884784698, + "rewards/margins": 0.029167091473937035, + "rewards/rejected": -0.13665245473384857, + "sft_loss": 0.03370191156864166, + "step": 390 + }, + { + "epoch": 0.031116297160637883, + "grad_norm": 3.0260448455810547, + "learning_rate": 4.9880645249618266e-05, + "logits/chosen": 0.010119600221514702, + "logits/rejected": -1.2736787796020508, + "logps/chosen": -0.9264433979988098, + "logps/rejected": -1.1216003894805908, + "loss": 0.9889, + "num_input_tokens_seen": 504080, + "odds_ratio_loss": 9.685811042785645, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.0926443412899971, + "rewards/margins": 0.01951570436358452, + "rewards/rejected": -0.11216004192829132, + "sft_loss": 0.020315980538725853, + "step": 400 + }, + { + "epoch": 0.03189420458965383, + "grad_norm": 2.714024782180786, + "learning_rate": 4.9874607971402366e-05, + "logits/chosen": -0.21029357612133026, + "logits/rejected": -1.3044781684875488, + "logps/chosen": -1.0438721179962158, + "logps/rejected": -1.292240858078003, + "loss": 1.1025, + "num_input_tokens_seen": 518688, + "odds_ratio_loss": 10.874516487121582, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.10438720881938934, + "rewards/margins": 0.024836886674165726, + "rewards/rejected": -0.12922410666942596, + "sft_loss": 0.015052947215735912, + "step": 410 + }, + { + "epoch": 0.03267211201866978, + "grad_norm": 3.6389522552490234, + "learning_rate": 4.986842212986099e-05, + "logits/chosen": -0.32333192229270935, + "logits/rejected": -1.2866684198379517, + "logps/chosen": -1.022554636001587, + "logps/rejected": -1.1091924905776978, + "loss": 1.0915, + "num_input_tokens_seen": 531904, + "odds_ratio_loss": 10.820093154907227, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.10225546360015869, + "rewards/margins": 0.008663791231811047, + "rewards/rejected": -0.11091925948858261, + "sft_loss": 0.009445266798138618, + "step": 420 + }, + { + "epoch": 0.033450019447685726, + "grad_norm": 5.962457656860352, + "learning_rate": 4.986208776193902e-05, + "logits/chosen": -0.07435014098882675, + "logits/rejected": -1.4142100811004639, + "logps/chosen": -1.0092417001724243, + "logps/rejected": -1.314469575881958, + "loss": 1.0653, + "num_input_tokens_seen": 544448, + "odds_ratio_loss": 10.6132173538208, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.10092417895793915, + "rewards/margins": 0.030522778630256653, + "rewards/rejected": -0.1314469575881958, + "sft_loss": 0.003954497165977955, + "step": 430 + }, + { + "epoch": 0.034227926876701675, + "grad_norm": 4.502202987670898, + "learning_rate": 4.985560490546839e-05, + "logits/chosen": -0.1421446055173874, + "logits/rejected": -1.3914377689361572, + "logps/chosen": -0.8869674801826477, + "logps/rejected": -2.607506275177002, + "loss": 0.9403, + "num_input_tokens_seen": 558576, + "odds_ratio_loss": 9.332474708557129, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.08869674056768417, + "rewards/margins": 0.17205388844013214, + "rewards/rejected": -0.2607506215572357, + "sft_loss": 0.007023976184427738, + "step": 440 + }, + { + "epoch": 0.03500583430571762, + "grad_norm": 2.3669939041137695, + "learning_rate": 4.984897359916788e-05, + "logits/chosen": -0.030847545713186264, + "logits/rejected": -0.9774616360664368, + "logps/chosen": -0.8356963992118835, + "logps/rejected": -0.9540478587150574, + "loss": 0.8978, + "num_input_tokens_seen": 567568, + "odds_ratio_loss": 8.91874885559082, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.08356963843107224, + "rewards/margins": 0.011835141107439995, + "rewards/rejected": -0.09540478140115738, + "sft_loss": 0.00597239937633276, + "step": 450 + }, + { + "epoch": 0.035783741734733567, + "grad_norm": 21.677845001220703, + "learning_rate": 4.9842193882642906e-05, + "logits/chosen": -0.2935951352119446, + "logits/rejected": -1.1201730966567993, + "logps/chosen": -1.3038840293884277, + "logps/rejected": -1.2353122234344482, + "loss": 1.3919, + "num_input_tokens_seen": 581632, + "odds_ratio_loss": 13.499517440795898, + "rewards/accuracies": 0.4000000059604645, + "rewards/chosen": -0.13038839399814606, + "rewards/margins": -0.006857173051685095, + "rewards/rejected": -0.12353122234344482, + "sft_loss": 0.041925784200429916, + "step": 460 + }, + { + "epoch": 0.036561649163749516, + "grad_norm": 2.8555731773376465, + "learning_rate": 4.9835265796385254e-05, + "logits/chosen": -0.1229843869805336, + "logits/rejected": -1.5147597789764404, + "logps/chosen": -1.1090964078903198, + "logps/rejected": -2.1519970893859863, + "loss": 1.163, + "num_input_tokens_seen": 594912, + "odds_ratio_loss": 11.57403564453125, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.11090965569019318, + "rewards/margins": 0.10429003089666367, + "rewards/rejected": -0.21519970893859863, + "sft_loss": 0.005632361862808466, + "step": 470 + }, + { + "epoch": 0.03733955659276546, + "grad_norm": 1.7752621173858643, + "learning_rate": 4.982818938177284e-05, + "logits/chosen": 0.023658571764826775, + "logits/rejected": -1.7298905849456787, + "logps/chosen": -1.1194090843200684, + "logps/rejected": -1.389075756072998, + "loss": 1.1806, + "num_input_tokens_seen": 607808, + "odds_ratio_loss": 11.7571439743042, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.11194092035293579, + "rewards/margins": 0.026966670528054237, + "rewards/rejected": -0.13890758156776428, + "sft_loss": 0.004934356547892094, + "step": 480 + }, + { + "epoch": 0.03811746402178141, + "grad_norm": 5.799675464630127, + "learning_rate": 4.982096468106947e-05, + "logits/chosen": -0.17389316856861115, + "logits/rejected": -1.2688932418823242, + "logps/chosen": -1.0437113046646118, + "logps/rejected": -1.5706325769424438, + "loss": 1.1041, + "num_input_tokens_seen": 625984, + "odds_ratio_loss": 10.988919258117676, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.10437111556529999, + "rewards/margins": 0.05269213393330574, + "rewards/rejected": -0.15706324577331543, + "sft_loss": 0.005206743720918894, + "step": 490 + }, + { + "epoch": 0.038895371450797356, + "grad_norm": 5.300241470336914, + "learning_rate": 4.9813591737424596e-05, + "logits/chosen": -0.4578474164009094, + "logits/rejected": -1.0031487941741943, + "logps/chosen": -0.8955472707748413, + "logps/rejected": -1.3793914318084717, + "loss": 0.9547, + "num_input_tokens_seen": 639856, + "odds_ratio_loss": 9.262350082397461, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.08955473452806473, + "rewards/margins": 0.0483844131231308, + "rewards/rejected": -0.13793914020061493, + "sft_loss": 0.02842315472662449, + "step": 500 + }, + { + "epoch": 0.039673278879813305, + "grad_norm": 20.943037033081055, + "learning_rate": 4.9806070594873037e-05, + "logits/chosen": -0.3508077561855316, + "logits/rejected": -1.051695466041565, + "logps/chosen": -0.9110622406005859, + "logps/rejected": -1.484948754310608, + "loss": 0.9541, + "num_input_tokens_seen": 650656, + "odds_ratio_loss": 9.375656127929688, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09110622853040695, + "rewards/margins": 0.05738866329193115, + "rewards/rejected": -0.1484948843717575, + "sft_loss": 0.016496889293193817, + "step": 510 + }, + { + "epoch": 0.04045118630882925, + "grad_norm": 3.846301555633545, + "learning_rate": 4.979840129833474e-05, + "logits/chosen": -0.4234844148159027, + "logits/rejected": -0.675470232963562, + "logps/chosen": -1.0217151641845703, + "logps/rejected": -1.2225643396377563, + "loss": 1.1046, + "num_input_tokens_seen": 661552, + "odds_ratio_loss": 10.809840202331543, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.10217152535915375, + "rewards/margins": 0.020084915682673454, + "rewards/rejected": -0.12225644290447235, + "sft_loss": 0.023616602644324303, + "step": 520 + }, + { + "epoch": 0.041229093737845196, + "grad_norm": 6.235986709594727, + "learning_rate": 4.97905838936145e-05, + "logits/chosen": 0.06146078556776047, + "logits/rejected": -1.7578405141830444, + "logps/chosen": -0.9877039194107056, + "logps/rejected": -1.5740721225738525, + "loss": 1.043, + "num_input_tokens_seen": 675936, + "odds_ratio_loss": 10.361382484436035, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.0987703949213028, + "rewards/margins": 0.05863680690526962, + "rewards/rejected": -0.15740719437599182, + "sft_loss": 0.006865262053906918, + "step": 530 + }, + { + "epoch": 0.042007001166861145, + "grad_norm": 1.6892807483673096, + "learning_rate": 4.978261842740166e-05, + "logits/chosen": -0.05991193652153015, + "logits/rejected": -1.4414308071136475, + "logps/chosen": -1.058107852935791, + "logps/rejected": -1.4011973142623901, + "loss": 1.1117, + "num_input_tokens_seen": 690112, + "odds_ratio_loss": 11.0405912399292, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.10581078380346298, + "rewards/margins": 0.03430896997451782, + "rewards/rejected": -0.1401197612285614, + "sft_loss": 0.007605980150401592, + "step": 540 + }, + { + "epoch": 0.04278490859587709, + "grad_norm": 2.5099849700927734, + "learning_rate": 4.9774504947269905e-05, + "logits/chosen": 0.03550032898783684, + "logits/rejected": -1.306727647781372, + "logps/chosen": -0.9737154245376587, + "logps/rejected": -1.42258882522583, + "loss": 1.0439, + "num_input_tokens_seen": 704480, + "odds_ratio_loss": 9.935417175292969, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.09737154096364975, + "rewards/margins": 0.04488734155893326, + "rewards/rejected": -0.14225886762142181, + "sft_loss": 0.05035661906003952, + "step": 550 + }, + { + "epoch": 0.043562816024893036, + "grad_norm": 2.5092949867248535, + "learning_rate": 4.97662435016769e-05, + "logits/chosen": 0.20719823241233826, + "logits/rejected": -1.6484529972076416, + "logps/chosen": -0.8146620988845825, + "logps/rejected": -1.2458853721618652, + "loss": 0.8628, + "num_input_tokens_seen": 720512, + "odds_ratio_loss": 8.611961364746094, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.08146621286869049, + "rewards/margins": 0.04312232881784439, + "rewards/rejected": -0.12458853423595428, + "sft_loss": 0.0016197937075048685, + "step": 560 + }, + { + "epoch": 0.044340723453908985, + "grad_norm": 7.840872764587402, + "learning_rate": 4.9757834139964035e-05, + "logits/chosen": -0.6304762363433838, + "logits/rejected": -0.601016640663147, + "logps/chosen": -0.9585424661636353, + "logps/rejected": -1.712816596031189, + "loss": 1.0203, + "num_input_tokens_seen": 731840, + "odds_ratio_loss": 9.28771686553955, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.0958542451262474, + "rewards/margins": 0.07542743533849716, + "rewards/rejected": -0.17128166556358337, + "sft_loss": 0.09156389534473419, + "step": 570 + }, + { + "epoch": 0.045118630882924934, + "grad_norm": 3.6173086166381836, + "learning_rate": 4.974927691235614e-05, + "logits/chosen": -0.04471359774470329, + "logits/rejected": -1.2187974452972412, + "logps/chosen": -0.8615912199020386, + "logps/rejected": -1.62959885597229, + "loss": 0.9147, + "num_input_tokens_seen": 742800, + "odds_ratio_loss": 9.08159065246582, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.0861591249704361, + "rewards/margins": 0.07680076360702515, + "rewards/rejected": -0.16295988857746124, + "sft_loss": 0.006538393907248974, + "step": 580 + }, + { + "epoch": 0.045896538311940877, + "grad_norm": 3.3760595321655273, + "learning_rate": 4.9740571869961137e-05, + "logits/chosen": -0.3412732481956482, + "logits/rejected": -1.3786067962646484, + "logps/chosen": -1.2360124588012695, + "logps/rejected": -1.6927540302276611, + "loss": 1.293, + "num_input_tokens_seen": 752016, + "odds_ratio_loss": 12.306550979614258, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.12360125780105591, + "rewards/margins": 0.04567413777112961, + "rewards/rejected": -0.16927538812160492, + "sft_loss": 0.06239398568868637, + "step": 590 + }, + { + "epoch": 0.046674445740956826, + "grad_norm": 3.992867946624756, + "learning_rate": 4.9731719064769833e-05, + "logits/chosen": -0.07947269827127457, + "logits/rejected": -1.2102258205413818, + "logps/chosen": -0.8327411413192749, + "logps/rejected": -1.5643784999847412, + "loss": 0.8797, + "num_input_tokens_seen": 763376, + "odds_ratio_loss": 8.484746932983398, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.08327411860227585, + "rewards/margins": 0.07316374778747559, + "rewards/rejected": -0.15643787384033203, + "sft_loss": 0.03125034272670746, + "step": 600 + }, + { + "epoch": 0.047452353169972775, + "grad_norm": 10.707353591918945, + "learning_rate": 4.972271854965549e-05, + "logits/chosen": 0.01606535166501999, + "logits/rejected": -1.0644062757492065, + "logps/chosen": -0.9305408596992493, + "logps/rejected": -1.4815337657928467, + "loss": 0.9802, + "num_input_tokens_seen": 775424, + "odds_ratio_loss": 9.663485527038574, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.09305407851934433, + "rewards/margins": 0.05509928986430168, + "rewards/rejected": -0.1481533646583557, + "sft_loss": 0.01387445442378521, + "step": 610 + }, + { + "epoch": 0.048230260598988724, + "grad_norm": 2.879865884780884, + "learning_rate": 4.9713570378373584e-05, + "logits/chosen": 0.09017481654882431, + "logits/rejected": -1.2260386943817139, + "logps/chosen": -0.9439705610275269, + "logps/rejected": -1.1371946334838867, + "loss": 1.0053, + "num_input_tokens_seen": 792192, + "odds_ratio_loss": 9.941380500793457, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.09439705312252045, + "rewards/margins": 0.01932242140173912, + "rewards/rejected": -0.11371947824954987, + "sft_loss": 0.011146670207381248, + "step": 620 + }, + { + "epoch": 0.049008168028004666, + "grad_norm": 2.4512083530426025, + "learning_rate": 4.9704274605561486e-05, + "logits/chosen": -0.06639071553945541, + "logits/rejected": -1.1143431663513184, + "logps/chosen": -0.8876552581787109, + "logps/rejected": -1.388165831565857, + "loss": 0.9485, + "num_input_tokens_seen": 803920, + "odds_ratio_loss": 9.404537200927734, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.08876553177833557, + "rewards/margins": 0.05005106329917908, + "rewards/rejected": -0.13881659507751465, + "sft_loss": 0.008023647591471672, + "step": 630 + }, + { + "epoch": 0.049786075457020615, + "grad_norm": 13.352177619934082, + "learning_rate": 4.969483128673808e-05, + "logits/chosen": -0.09846919775009155, + "logits/rejected": -0.6065551042556763, + "logps/chosen": -0.8927276730537415, + "logps/rejected": -1.3452032804489136, + "loss": 0.9619, + "num_input_tokens_seen": 817856, + "odds_ratio_loss": 9.278109550476074, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.08927276730537415, + "rewards/margins": 0.04524756222963333, + "rewards/rejected": -0.13452033698558807, + "sft_loss": 0.034084875136613846, + "step": 640 + }, + { + "epoch": 0.050563982886036564, + "grad_norm": 5.00535249710083, + "learning_rate": 4.96852404783035e-05, + "logits/chosen": 0.00019475221051834524, + "logits/rejected": -1.6706771850585938, + "logps/chosen": -1.0463913679122925, + "logps/rejected": -1.2002830505371094, + "loss": 1.1217, + "num_input_tokens_seen": 835488, + "odds_ratio_loss": 11.174688339233398, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.10463915020227432, + "rewards/margins": 0.015389163978397846, + "rewards/rejected": -0.12002831697463989, + "sft_loss": 0.004188282880932093, + "step": 650 + }, + { + "epoch": 0.051341890315052506, + "grad_norm": 2.9358901977539062, + "learning_rate": 4.967550223753873e-05, + "logits/chosen": -0.2236834019422531, + "logits/rejected": -0.8710101246833801, + "logps/chosen": -1.2662832736968994, + "logps/rejected": -2.0422096252441406, + "loss": 1.3282, + "num_input_tokens_seen": 852288, + "odds_ratio_loss": 12.909830093383789, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.1266283094882965, + "rewards/margins": 0.0775926411151886, + "rewards/rejected": -0.2042209655046463, + "sft_loss": 0.037256959825754166, + "step": 660 + }, + { + "epoch": 0.052119797744068455, + "grad_norm": 28.316844940185547, + "learning_rate": 4.966561662260532e-05, + "logits/chosen": -0.16411355137825012, + "logits/rejected": -0.9293942451477051, + "logps/chosen": -0.9301309585571289, + "logps/rejected": -1.9711850881576538, + "loss": 0.9846, + "num_input_tokens_seen": 861984, + "odds_ratio_loss": 9.354632377624512, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.09301309287548065, + "rewards/margins": 0.10410543531179428, + "rewards/rejected": -0.19711852073669434, + "sft_loss": 0.04918523132801056, + "step": 670 + }, + { + "epoch": 0.052897705173084404, + "grad_norm": 2.13519287109375, + "learning_rate": 4.9655583692544986e-05, + "logits/chosen": -0.29984229803085327, + "logits/rejected": -1.138826847076416, + "logps/chosen": -0.8817435503005981, + "logps/rejected": -1.1321008205413818, + "loss": 0.9382, + "num_input_tokens_seen": 875440, + "odds_ratio_loss": 9.27894115447998, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.08817435801029205, + "rewards/margins": 0.025035735219717026, + "rewards/rejected": -0.11321009695529938, + "sft_loss": 0.010337069630622864, + "step": 680 + }, + { + "epoch": 0.05367561260210035, + "grad_norm": 8.119619369506836, + "learning_rate": 4.96454035072793e-05, + "logits/chosen": -0.019457275047898293, + "logits/rejected": -1.780207633972168, + "logps/chosen": -1.1262990236282349, + "logps/rejected": -1.7122234106063843, + "loss": 1.1782, + "num_input_tokens_seen": 890912, + "odds_ratio_loss": 11.341121673583984, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.11262989044189453, + "rewards/margins": 0.05859243869781494, + "rewards/rejected": -0.17122234404087067, + "sft_loss": 0.044103674590587616, + "step": 690 + }, + { + "epoch": 0.054453520031116295, + "grad_norm": 2.790867805480957, + "learning_rate": 4.963507612760932e-05, + "logits/chosen": 0.0026435554027557373, + "logits/rejected": -1.221129298210144, + "logps/chosen": -1.1090561151504517, + "logps/rejected": -1.7434148788452148, + "loss": 1.1765, + "num_input_tokens_seen": 906464, + "odds_ratio_loss": 10.66307258605957, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.11090561002492905, + "rewards/margins": 0.06343585252761841, + "rewards/rejected": -0.17434147000312805, + "sft_loss": 0.11023900657892227, + "step": 700 + }, + { + "epoch": 0.055231427460132244, + "grad_norm": 8.295060157775879, + "learning_rate": 4.962460161521518e-05, + "logits/chosen": -0.43848055601119995, + "logits/rejected": -1.1596039533615112, + "logps/chosen": -1.5424635410308838, + "logps/rejected": -1.3884707689285278, + "loss": 1.659, + "num_input_tokens_seen": 919648, + "odds_ratio_loss": 15.883735656738281, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.15424636006355286, + "rewards/margins": -0.015399277210235596, + "rewards/rejected": -0.13884706795215607, + "sft_loss": 0.07059884071350098, + "step": 710 + }, + { + "epoch": 0.056009334889148193, + "grad_norm": 2.7219836711883545, + "learning_rate": 4.961398003265582e-05, + "logits/chosen": -0.0401337593793869, + "logits/rejected": -0.8950508832931519, + "logps/chosen": -0.9055600166320801, + "logps/rejected": -1.2787041664123535, + "loss": 0.9826, + "num_input_tokens_seen": 932144, + "odds_ratio_loss": 9.71101188659668, + "rewards/accuracies": 0.4000000059604645, + "rewards/chosen": -0.09055600315332413, + "rewards/margins": 0.03731442242860794, + "rewards/rejected": -0.12787041068077087, + "sft_loss": 0.011469913646578789, + "step": 720 + }, + { + "epoch": 0.056787242318164136, + "grad_norm": 9.607157707214355, + "learning_rate": 4.96032114433685e-05, + "logits/chosen": -0.20233073830604553, + "logits/rejected": -1.0893396139144897, + "logps/chosen": -1.086705207824707, + "logps/rejected": -1.1163735389709473, + "loss": 1.1655, + "num_input_tokens_seen": 944384, + "odds_ratio_loss": 11.479543685913086, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.10867051780223846, + "rewards/margins": 0.002966833533719182, + "rewards/rejected": -0.11163735389709473, + "sft_loss": 0.017573310062289238, + "step": 730 + }, + { + "epoch": 0.057565149747180085, + "grad_norm": 6.647035598754883, + "learning_rate": 4.9592295911668516e-05, + "logits/chosen": -0.1728004515171051, + "logits/rejected": -1.3159668445587158, + "logps/chosen": -0.8700982928276062, + "logps/rejected": -2.0540032386779785, + "loss": 0.9084, + "num_input_tokens_seen": 956528, + "odds_ratio_loss": 8.972583770751953, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.08700983226299286, + "rewards/margins": 0.1183905154466629, + "rewards/rejected": -0.20540031790733337, + "sft_loss": 0.011142651550471783, + "step": 740 + }, + { + "epoch": 0.058343057176196034, + "grad_norm": 1.6915847063064575, + "learning_rate": 4.958123350274878e-05, + "logits/chosen": 0.09090377390384674, + "logits/rejected": -1.1098421812057495, + "logps/chosen": -0.828323483467102, + "logps/rejected": -1.0652092695236206, + "loss": 0.8905, + "num_input_tokens_seen": 973072, + "odds_ratio_loss": 8.869556427001953, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.08283234387636185, + "rewards/margins": 0.02368859015405178, + "rewards/rejected": -0.10652093589305878, + "sft_loss": 0.003506676061078906, + "step": 750 + }, + { + "epoch": 0.05912096460521198, + "grad_norm": 4.410206317901611, + "learning_rate": 4.957002428267937e-05, + "logits/chosen": -0.12386639416217804, + "logits/rejected": -1.4178106784820557, + "logps/chosen": -1.006008505821228, + "logps/rejected": -1.5750558376312256, + "loss": 1.0587, + "num_input_tokens_seen": 990416, + "odds_ratio_loss": 10.437189102172852, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.10060086101293564, + "rewards/margins": 0.05690472200512886, + "rewards/rejected": -0.157505601644516, + "sft_loss": 0.01501976978033781, + "step": 760 + }, + { + "epoch": 0.059898872034227925, + "grad_norm": 4.2691192626953125, + "learning_rate": 4.9558668318407264e-05, + "logits/chosen": -0.0732850581407547, + "logits/rejected": -1.1064198017120361, + "logps/chosen": -0.8314032554626465, + "logps/rejected": -1.7381566762924194, + "loss": 0.8686, + "num_input_tokens_seen": 1000784, + "odds_ratio_loss": 8.609987258911133, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.08314032852649689, + "rewards/margins": 0.09067533910274506, + "rewards/rejected": -0.17381566762924194, + "sft_loss": 0.0076231942512094975, + "step": 770 + }, + { + "epoch": 0.060676779463243874, + "grad_norm": 2.8287811279296875, + "learning_rate": 4.954716567775582e-05, + "logits/chosen": -0.02032371237874031, + "logits/rejected": -1.1463935375213623, + "logps/chosen": -0.9587979316711426, + "logps/rejected": -1.1363391876220703, + "loss": 1.0232, + "num_input_tokens_seen": 1016992, + "odds_ratio_loss": 10.191497802734375, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.09587979316711426, + "rewards/margins": 0.017754126340150833, + "rewards/rejected": -0.11363391578197479, + "sft_loss": 0.00401442963629961, + "step": 780 + }, + { + "epoch": 0.06145468689225982, + "grad_norm": 3.20035719871521, + "learning_rate": 4.9535516429424444e-05, + "logits/chosen": -0.4328770041465759, + "logits/rejected": -0.9181963801383972, + "logps/chosen": -0.858111560344696, + "logps/rejected": -1.4384349584579468, + "loss": 0.8984, + "num_input_tokens_seen": 1027120, + "odds_ratio_loss": 8.835628509521484, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.08581115305423737, + "rewards/margins": 0.05803234130144119, + "rewards/rejected": -0.14384350180625916, + "sft_loss": 0.014793934300541878, + "step": 790 + }, + { + "epoch": 0.062232594321275765, + "grad_norm": 8.685051918029785, + "learning_rate": 4.952372064298813e-05, + "logits/chosen": -0.3011876940727234, + "logits/rejected": -0.7921189069747925, + "logps/chosen": -0.9806588888168335, + "logps/rejected": -1.0401127338409424, + "loss": 1.0546, + "num_input_tokens_seen": 1040192, + "odds_ratio_loss": 10.229253768920898, + "rewards/accuracies": 0.4000000059604645, + "rewards/chosen": -0.09806589782238007, + "rewards/margins": 0.005945396143943071, + "rewards/rejected": -0.10401128232479095, + "sft_loss": 0.03166633099317551, + "step": 800 + }, + { + "epoch": 0.06301050175029171, + "grad_norm": 13.73974323272705, + "learning_rate": 4.9511778388897094e-05, + "logits/chosen": -0.46943092346191406, + "logits/rejected": -1.2852362394332886, + "logps/chosen": -0.8816627264022827, + "logps/rejected": -1.0180727243423462, + "loss": 0.9553, + "num_input_tokens_seen": 1054160, + "odds_ratio_loss": 9.431961059570312, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.08816628158092499, + "rewards/margins": 0.013641009107232094, + "rewards/rejected": -0.10180728137493134, + "sft_loss": 0.012087122537195683, + "step": 810 + }, + { + "epoch": 0.06378840917930766, + "grad_norm": 2.1410396099090576, + "learning_rate": 4.949968973847632e-05, + "logits/chosen": -0.11463584750890732, + "logits/rejected": -1.3929070234298706, + "logps/chosen": -1.0623462200164795, + "logps/rejected": -1.6423686742782593, + "loss": 1.124, + "num_input_tokens_seen": 1067616, + "odds_ratio_loss": 10.206562995910645, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.10623463243246078, + "rewards/margins": 0.058002255856990814, + "rewards/rejected": -0.1642368733882904, + "sft_loss": 0.10329482704401016, + "step": 820 + }, + { + "epoch": 0.06456631660832361, + "grad_norm": 2.5342459678649902, + "learning_rate": 4.9487454763925135e-05, + "logits/chosen": 0.09601770341396332, + "logits/rejected": -1.3720271587371826, + "logps/chosen": -0.9235900044441223, + "logps/rejected": -1.3671214580535889, + "loss": 0.9705, + "num_input_tokens_seen": 1083808, + "odds_ratio_loss": 9.668932914733887, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.09235899150371552, + "rewards/margins": 0.044353142380714417, + "rewards/rejected": -0.13671213388442993, + "sft_loss": 0.0036094163078814745, + "step": 830 + }, + { + "epoch": 0.06534422403733955, + "grad_norm": 2.265723705291748, + "learning_rate": 4.94750735383168e-05, + "logits/chosen": -0.2784004509449005, + "logits/rejected": -1.390732765197754, + "logps/chosen": -1.2053892612457275, + "logps/rejected": -1.8831628561019897, + "loss": 1.2595, + "num_input_tokens_seen": 1095568, + "odds_ratio_loss": 12.26213264465332, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.12053891271352768, + "rewards/margins": 0.06777738034725189, + "rewards/rejected": -0.18831630051136017, + "sft_loss": 0.0333065465092659, + "step": 840 + }, + { + "epoch": 0.06612213146635551, + "grad_norm": 4.156375885009766, + "learning_rate": 4.946254613559803e-05, + "logits/chosen": -0.0888671875, + "logits/rejected": -1.1945364475250244, + "logps/chosen": -0.7892787456512451, + "logps/rejected": -2.2669517993927, + "loss": 0.8426, + "num_input_tokens_seen": 1110960, + "odds_ratio_loss": 8.253103256225586, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.07892787456512451, + "rewards/margins": 0.1477673053741455, + "rewards/rejected": -0.22669517993927002, + "sft_loss": 0.01730215921998024, + "step": 850 + }, + { + "epoch": 0.06690003889537145, + "grad_norm": 1.8222236633300781, + "learning_rate": 4.944987263058863e-05, + "logits/chosen": -0.27693435549736023, + "logits/rejected": -1.2136726379394531, + "logps/chosen": -0.9566059112548828, + "logps/rejected": -1.0269935131072998, + "loss": 1.0273, + "num_input_tokens_seen": 1121584, + "odds_ratio_loss": 10.1735200881958, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.09566058963537216, + "rewards/margins": 0.007038764655590057, + "rewards/rejected": -0.10269935429096222, + "sft_loss": 0.009919771924614906, + "step": 860 + }, + { + "epoch": 0.0676779463243874, + "grad_norm": 1.8523871898651123, + "learning_rate": 4.9437053098980956e-05, + "logits/chosen": -0.2266802042722702, + "logits/rejected": -1.2647974491119385, + "logps/chosen": -1.1253491640090942, + "logps/rejected": -1.2813799381256104, + "loss": 1.1889, + "num_input_tokens_seen": 1137296, + "odds_ratio_loss": 11.128896713256836, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.11253491789102554, + "rewards/margins": 0.01560310460627079, + "rewards/rejected": -0.12813802063465118, + "sft_loss": 0.0759621262550354, + "step": 870 + }, + { + "epoch": 0.06845585375340335, + "grad_norm": 8.056722640991211, + "learning_rate": 4.942408761733952e-05, + "logits/chosen": -0.10846817493438721, + "logits/rejected": -1.035222053527832, + "logps/chosen": -1.1279523372650146, + "logps/rejected": -1.899043321609497, + "loss": 1.1801, + "num_input_tokens_seen": 1152848, + "odds_ratio_loss": 10.552392959594727, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.11279523372650146, + "rewards/margins": 0.07710909843444824, + "rewards/rejected": -0.1899043172597885, + "sft_loss": 0.12487705796957016, + "step": 880 + }, + { + "epoch": 0.06923376118241929, + "grad_norm": 10.801746368408203, + "learning_rate": 4.9410976263100515e-05, + "logits/chosen": -0.4096245765686035, + "logits/rejected": -0.9306976199150085, + "logps/chosen": -0.8880082964897156, + "logps/rejected": -1.1635149717330933, + "loss": 0.9443, + "num_input_tokens_seen": 1164000, + "odds_ratio_loss": 9.348076820373535, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.0888008326292038, + "rewards/margins": 0.02755066193640232, + "rewards/rejected": -0.11635150760412216, + "sft_loss": 0.00947362557053566, + "step": 890 + }, + { + "epoch": 0.07001166861143523, + "grad_norm": 1.9704222679138184, + "learning_rate": 4.9397719114571386e-05, + "logits/chosen": 0.06543467938899994, + "logits/rejected": -1.1813740730285645, + "logps/chosen": -1.018379807472229, + "logps/rejected": -1.2177107334136963, + "loss": 1.0886, + "num_input_tokens_seen": 1174592, + "odds_ratio_loss": 10.819978713989258, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.10183799266815186, + "rewards/margins": 0.019933100789785385, + "rewards/rejected": -0.12177108228206635, + "sft_loss": 0.006566903553903103, + "step": 900 + }, + { + "epoch": 0.07078957604045119, + "grad_norm": 1.8523725271224976, + "learning_rate": 4.9384316250930276e-05, + "logits/chosen": -0.06241592764854431, + "logits/rejected": -1.6512749195098877, + "logps/chosen": -0.9898072481155396, + "logps/rejected": -1.2340342998504639, + "loss": 1.0652, + "num_input_tokens_seen": 1187040, + "odds_ratio_loss": 10.588815689086914, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.09898072481155396, + "rewards/margins": 0.02442272938787937, + "rewards/rejected": -0.12340345233678818, + "sft_loss": 0.006280974484980106, + "step": 910 + }, + { + "epoch": 0.07156748346946713, + "grad_norm": 5.821229934692383, + "learning_rate": 4.937076775222566e-05, + "logits/chosen": -0.2428424060344696, + "logits/rejected": -1.3016266822814941, + "logps/chosen": -0.9424579739570618, + "logps/rejected": -1.9924252033233643, + "loss": 0.9786, + "num_input_tokens_seen": 1195472, + "odds_ratio_loss": 9.530224800109863, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.0942457988858223, + "rewards/margins": 0.10499672591686249, + "rewards/rejected": -0.1992425173521042, + "sft_loss": 0.025574322789907455, + "step": 920 + }, + { + "epoch": 0.07234539089848308, + "grad_norm": 7.764154434204102, + "learning_rate": 4.93570736993758e-05, + "logits/chosen": -0.2199537456035614, + "logits/rejected": -0.9988893270492554, + "logps/chosen": -0.7839959263801575, + "logps/rejected": -1.4383164644241333, + "loss": 0.8377, + "num_input_tokens_seen": 1208128, + "odds_ratio_loss": 8.278602600097656, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.07839959859848022, + "rewards/margins": 0.06543205678462982, + "rewards/rejected": -0.14383165538311005, + "sft_loss": 0.009836836718022823, + "step": 930 + }, + { + "epoch": 0.07312329832749903, + "grad_norm": 3.802403211593628, + "learning_rate": 4.934323417416827e-05, + "logits/chosen": -0.15197499096393585, + "logits/rejected": -1.2443273067474365, + "logps/chosen": -0.9255944490432739, + "logps/rejected": -1.183050513267517, + "loss": 0.9847, + "num_input_tokens_seen": 1216976, + "odds_ratio_loss": 9.783621788024902, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.09255944192409515, + "rewards/margins": 0.025745611637830734, + "rewards/rejected": -0.11830506473779678, + "sft_loss": 0.0063563063740730286, + "step": 940 + }, + { + "epoch": 0.07390120575651497, + "grad_norm": 1.7451285123825073, + "learning_rate": 4.932924925925949e-05, + "logits/chosen": -0.1452958881855011, + "logits/rejected": -1.4032037258148193, + "logps/chosen": -1.0042587518692017, + "logps/rejected": -1.2171082496643066, + "loss": 1.0711, + "num_input_tokens_seen": 1230656, + "odds_ratio_loss": 10.356279373168945, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.10042587667703629, + "rewards/margins": 0.021284956485033035, + "rewards/rejected": -0.12171083688735962, + "sft_loss": 0.03550771623849869, + "step": 950 + }, + { + "epoch": 0.07467911318553092, + "grad_norm": 5.48258638381958, + "learning_rate": 4.9315119038174206e-05, + "logits/chosen": 0.07203695178031921, + "logits/rejected": -1.419390082359314, + "logps/chosen": -0.8708435893058777, + "logps/rejected": -2.2798571586608887, + "loss": 0.9308, + "num_input_tokens_seen": 1245024, + "odds_ratio_loss": 8.826228141784668, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.08708436042070389, + "rewards/margins": 0.1409013420343399, + "rewards/rejected": -0.2279857099056244, + "sft_loss": 0.04818565398454666, + "step": 960 + }, + { + "epoch": 0.07545702061454687, + "grad_norm": 5.357232570648193, + "learning_rate": 4.930084359530502e-05, + "logits/chosen": -0.06561549007892609, + "logits/rejected": -1.3615022897720337, + "logps/chosen": -0.9727898836135864, + "logps/rejected": -1.3841956853866577, + "loss": 1.0159, + "num_input_tokens_seen": 1260400, + "odds_ratio_loss": 9.965991973876953, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09727897495031357, + "rewards/margins": 0.04114057868719101, + "rewards/rejected": -0.13841956853866577, + "sft_loss": 0.01931620016694069, + "step": 970 + }, + { + "epoch": 0.07623492804356281, + "grad_norm": 6.6753435134887695, + "learning_rate": 4.9286423015911845e-05, + "logits/chosen": -0.3039540946483612, + "logits/rejected": -0.6542695760726929, + "logps/chosen": -0.9842222929000854, + "logps/rejected": -1.1113169193267822, + "loss": 1.0535, + "num_input_tokens_seen": 1272128, + "odds_ratio_loss": 10.407747268676758, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.09842222929000854, + "rewards/margins": 0.012709471397101879, + "rewards/rejected": -0.1111316904425621, + "sft_loss": 0.012736847624182701, + "step": 980 + }, + { + "epoch": 0.07701283547257877, + "grad_norm": 1.6576621532440186, + "learning_rate": 4.927185738612145e-05, + "logits/chosen": -0.2928544282913208, + "logits/rejected": -1.067185878753662, + "logps/chosen": -0.835498034954071, + "logps/rejected": -1.6010229587554932, + "loss": 0.8751, + "num_input_tokens_seen": 1285616, + "odds_ratio_loss": 8.540815353393555, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.08354981243610382, + "rewards/margins": 0.07655248790979385, + "rewards/rejected": -0.16010229289531708, + "sft_loss": 0.021057048812508583, + "step": 990 + }, + { + "epoch": 0.07779074290159471, + "grad_norm": 2.4466707706451416, + "learning_rate": 4.925714679292689e-05, + "logits/chosen": -0.13626167178153992, + "logits/rejected": -0.7506991624832153, + "logps/chosen": -1.0066719055175781, + "logps/rejected": -1.3263317346572876, + "loss": 1.0701, + "num_input_tokens_seen": 1304160, + "odds_ratio_loss": 10.451330184936523, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.10066720098257065, + "rewards/margins": 0.03196597844362259, + "rewards/rejected": -0.13263316452503204, + "sft_loss": 0.024980902671813965, + "step": 1000 + }, + { + "epoch": 0.07856865033061065, + "grad_norm": 8.611452102661133, + "learning_rate": 4.9242291324187024e-05, + "logits/chosen": -0.006370346061885357, + "logits/rejected": -1.4868528842926025, + "logps/chosen": -0.871174693107605, + "logps/rejected": -1.2081753015518188, + "loss": 0.9235, + "num_input_tokens_seen": 1317664, + "odds_ratio_loss": 9.131412506103516, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.08711747825145721, + "rewards/margins": 0.03370004519820213, + "rewards/rejected": -0.12081752717494965, + "sft_loss": 0.01033018808811903, + "step": 1010 + }, + { + "epoch": 0.07934655775962661, + "grad_norm": 9.985763549804688, + "learning_rate": 4.9227291068625983e-05, + "logits/chosen": -0.009476041421294212, + "logits/rejected": -1.3522567749023438, + "logps/chosen": -1.0734918117523193, + "logps/rejected": -1.6543095111846924, + "loss": 1.1205, + "num_input_tokens_seen": 1335264, + "odds_ratio_loss": 11.110357284545898, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.10734919458627701, + "rewards/margins": 0.05808175355195999, + "rewards/rejected": -0.1654309332370758, + "sft_loss": 0.00942428968846798, + "step": 1020 + }, + { + "epoch": 0.08012446518864255, + "grad_norm": 2.383880138397217, + "learning_rate": 4.921214611583262e-05, + "logits/chosen": -0.02636711858212948, + "logits/rejected": -1.0757944583892822, + "logps/chosen": -0.8740970492362976, + "logps/rejected": -1.1859877109527588, + "loss": 0.9276, + "num_input_tokens_seen": 1350464, + "odds_ratio_loss": 9.17290210723877, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.08740971237421036, + "rewards/margins": 0.031189054250717163, + "rewards/rejected": -0.11859877407550812, + "sft_loss": 0.010294022038578987, + "step": 1030 + }, + { + "epoch": 0.0809023726176585, + "grad_norm": 1.8636724948883057, + "learning_rate": 4.9196856556260004e-05, + "logits/chosen": 0.16306543350219727, + "logits/rejected": -1.4347612857818604, + "logps/chosen": -0.8971742391586304, + "logps/rejected": -0.8668204545974731, + "loss": 0.9785, + "num_input_tokens_seen": 1368112, + "odds_ratio_loss": 9.759934425354004, + "rewards/accuracies": 0.4000000059604645, + "rewards/chosen": -0.08971743285655975, + "rewards/margins": -0.0030353828333318233, + "rewards/rejected": -0.0866820439696312, + "sft_loss": 0.002534131519496441, + "step": 1040 + }, + { + "epoch": 0.08168028004667445, + "grad_norm": 17.53750228881836, + "learning_rate": 4.918142248122488e-05, + "logits/chosen": -0.10349367558956146, + "logits/rejected": -1.0907751321792603, + "logps/chosen": -0.8256258964538574, + "logps/rejected": -1.1939427852630615, + "loss": 0.8723, + "num_input_tokens_seen": 1380816, + "odds_ratio_loss": 8.530784606933594, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.08256258070468903, + "rewards/margins": 0.03683171421289444, + "rewards/rejected": -0.11939430236816406, + "sft_loss": 0.019271310418844223, + "step": 1050 + }, + { + "epoch": 0.08245818747569039, + "grad_norm": 3.393805742263794, + "learning_rate": 4.916584398290707e-05, + "logits/chosen": 0.09704780578613281, + "logits/rejected": -1.274310827255249, + "logps/chosen": -0.9823578000068665, + "logps/rejected": -2.0368709564208984, + "loss": 1.0289, + "num_input_tokens_seen": 1394336, + "odds_ratio_loss": 10.22205924987793, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.09823577105998993, + "rewards/margins": 0.1054513230919838, + "rewards/rejected": -0.20368710160255432, + "sft_loss": 0.006736454553902149, + "step": 1060 + }, + { + "epoch": 0.08323609490470633, + "grad_norm": 1.6751441955566406, + "learning_rate": 4.915012115434899e-05, + "logits/chosen": 0.005007547326385975, + "logits/rejected": -1.2583626508712769, + "logps/chosen": -0.8267304301261902, + "logps/rejected": -1.207726240158081, + "loss": 0.8746, + "num_input_tokens_seen": 1408224, + "odds_ratio_loss": 8.649084091186523, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.08267304301261902, + "rewards/margins": 0.03809957951307297, + "rewards/rejected": -0.12077263742685318, + "sft_loss": 0.009731943719089031, + "step": 1070 + }, + { + "epoch": 0.08401400233372229, + "grad_norm": 33.09058380126953, + "learning_rate": 4.913425408945508e-05, + "logits/chosen": -0.19444052875041962, + "logits/rejected": -0.9926478266716003, + "logps/chosen": -1.0728189945220947, + "logps/rejected": -1.1655198335647583, + "loss": 1.15, + "num_input_tokens_seen": 1418224, + "odds_ratio_loss": 10.621275901794434, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.10728190094232559, + "rewards/margins": 0.009270080365240574, + "rewards/rejected": -0.11655198037624359, + "sft_loss": 0.08791467547416687, + "step": 1080 + }, + { + "epoch": 0.08479190976273823, + "grad_norm": 2.9355344772338867, + "learning_rate": 4.911824288299119e-05, + "logits/chosen": -0.01051928848028183, + "logits/rejected": -1.152430534362793, + "logps/chosen": -1.1754026412963867, + "logps/rejected": -1.0413397550582886, + "loss": 1.2581, + "num_input_tokens_seen": 1430928, + "odds_ratio_loss": 12.01326847076416, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.11754026263952255, + "rewards/margins": -0.01340627670288086, + "rewards/rejected": -0.1041339859366417, + "sft_loss": 0.056798458099365234, + "step": 1090 + }, + { + "epoch": 0.08556981719175417, + "grad_norm": 3.590717077255249, + "learning_rate": 4.9102087630584096e-05, + "logits/chosen": -0.20410041511058807, + "logits/rejected": -0.8998339772224426, + "logps/chosen": -0.8227648735046387, + "logps/rejected": -1.7237087488174438, + "loss": 0.878, + "num_input_tokens_seen": 1440912, + "odds_ratio_loss": 8.626799583435059, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.08227649331092834, + "rewards/margins": 0.09009438753128052, + "rewards/rejected": -0.17237088084220886, + "sft_loss": 0.015276836231350899, + "step": 1100 + }, + { + "epoch": 0.08634772462077013, + "grad_norm": 2.8673641681671143, + "learning_rate": 4.9085788428720845e-05, + "logits/chosen": -0.22915875911712646, + "logits/rejected": -0.9836863279342651, + "logps/chosen": -1.005216360092163, + "logps/rejected": -1.4263298511505127, + "loss": 1.06, + "num_input_tokens_seen": 1450960, + "odds_ratio_loss": 10.289142608642578, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.10052163898944855, + "rewards/margins": 0.0421113483607769, + "rewards/rejected": -0.14263299107551575, + "sft_loss": 0.03111717477440834, + "step": 1110 + }, + { + "epoch": 0.08712563204978607, + "grad_norm": 3.8207547664642334, + "learning_rate": 4.9069345374748265e-05, + "logits/chosen": -0.14207658171653748, + "logits/rejected": -1.0518529415130615, + "logps/chosen": -0.7499372959136963, + "logps/rejected": -1.2938522100448608, + "loss": 0.7924, + "num_input_tokens_seen": 1463232, + "odds_ratio_loss": 7.8319525718688965, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.07499372959136963, + "rewards/margins": 0.05439149588346481, + "rewards/rejected": -0.12938521802425385, + "sft_loss": 0.009246365167200565, + "step": 1120 + }, + { + "epoch": 0.08790353947880203, + "grad_norm": 2.379305839538574, + "learning_rate": 4.9052758566872305e-05, + "logits/chosen": -0.0017435208428651094, + "logits/rejected": -1.5067884922027588, + "logps/chosen": -0.9974363446235657, + "logps/rejected": -1.6765762567520142, + "loss": 1.0578, + "num_input_tokens_seen": 1478576, + "odds_ratio_loss": 10.503931045532227, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.09974364191293716, + "rewards/margins": 0.0679139792919159, + "rewards/rejected": -0.16765762865543365, + "sft_loss": 0.007378022186458111, + "step": 1130 + }, + { + "epoch": 0.08868144690781797, + "grad_norm": 2.0183286666870117, + "learning_rate": 4.903602810415749e-05, + "logits/chosen": -0.038505952805280685, + "logits/rejected": -1.4772956371307373, + "logps/chosen": -1.0257680416107178, + "logps/rejected": -1.224268913269043, + "loss": 1.0967, + "num_input_tokens_seen": 1489696, + "odds_ratio_loss": 10.111272811889648, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.10257680714130402, + "rewards/margins": 0.01985008828341961, + "rewards/rejected": -0.12242688983678818, + "sft_loss": 0.08557184785604477, + "step": 1140 + }, + { + "epoch": 0.08945935433683391, + "grad_norm": 15.378406524658203, + "learning_rate": 4.901915408652635e-05, + "logits/chosen": -0.26834017038345337, + "logits/rejected": -0.5739148855209351, + "logps/chosen": -0.7833901643753052, + "logps/rejected": -1.7032454013824463, + "loss": 0.8442, + "num_input_tokens_seen": 1503280, + "odds_ratio_loss": 8.320596694946289, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.07833901792764664, + "rewards/margins": 0.09198550879955292, + "rewards/rejected": -0.17032451927661896, + "sft_loss": 0.01215345785021782, + "step": 1150 + }, + { + "epoch": 0.09023726176584987, + "grad_norm": 3.9435245990753174, + "learning_rate": 4.900213661475874e-05, + "logits/chosen": -0.365293949842453, + "logits/rejected": -0.7508081197738647, + "logps/chosen": -1.0041760206222534, + "logps/rejected": -1.228610873222351, + "loss": 1.0602, + "num_input_tokens_seen": 1518464, + "odds_ratio_loss": 10.431593894958496, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.1004176139831543, + "rewards/margins": 0.022443488240242004, + "rewards/rejected": -0.1228611022233963, + "sft_loss": 0.01701296493411064, + "step": 1160 + }, + { + "epoch": 0.09101516919486581, + "grad_norm": 4.047309398651123, + "learning_rate": 4.898497579049135e-05, + "logits/chosen": -0.20835068821907043, + "logits/rejected": -1.1838566064834595, + "logps/chosen": -1.052578091621399, + "logps/rejected": -1.1051318645477295, + "loss": 1.1269, + "num_input_tokens_seen": 1534032, + "odds_ratio_loss": 10.748819351196289, + "rewards/accuracies": 0.4000000059604645, + "rewards/chosen": -0.10525782406330109, + "rewards/margins": 0.005255373660475016, + "rewards/rejected": -0.11051319539546967, + "sft_loss": 0.052055876702070236, + "step": 1170 + }, + { + "epoch": 0.09179307662388175, + "grad_norm": 5.090392589569092, + "learning_rate": 4.8967671716217e-05, + "logits/chosen": -0.4639959931373596, + "logits/rejected": -0.8973940014839172, + "logps/chosen": -0.7618675231933594, + "logps/rejected": -1.0851995944976807, + "loss": 0.8111, + "num_input_tokens_seen": 1549760, + "odds_ratio_loss": 8.026649475097656, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.07618675380945206, + "rewards/margins": 0.03233320266008377, + "rewards/rejected": -0.10851994901895523, + "sft_loss": 0.008396802470088005, + "step": 1180 + }, + { + "epoch": 0.09257098405289771, + "grad_norm": 2.946673631668091, + "learning_rate": 4.8950224495284095e-05, + "logits/chosen": 0.011005115695297718, + "logits/rejected": -1.8329797983169556, + "logps/chosen": -1.0231001377105713, + "logps/rejected": -1.3699067831039429, + "loss": 1.0743, + "num_input_tokens_seen": 1568048, + "odds_ratio_loss": 10.699010848999023, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.10231000185012817, + "rewards/margins": 0.034680671989917755, + "rewards/rejected": -0.13699068129062653, + "sft_loss": 0.004405025392770767, + "step": 1190 + }, + { + "epoch": 0.09334889148191365, + "grad_norm": 4.360876560211182, + "learning_rate": 4.893263423189597e-05, + "logits/chosen": -0.48811715841293335, + "logits/rejected": -0.7015589475631714, + "logps/chosen": -0.983946681022644, + "logps/rejected": -1.409174919128418, + "loss": 1.0317, + "num_input_tokens_seen": 1580768, + "odds_ratio_loss": 9.939173698425293, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.09839466959238052, + "rewards/margins": 0.04252283275127411, + "rewards/rejected": -0.14091750979423523, + "sft_loss": 0.03777718171477318, + "step": 1200 + }, + { + "epoch": 0.0941267989109296, + "grad_norm": 3.2938485145568848, + "learning_rate": 4.891490103111027e-05, + "logits/chosen": -0.3066580891609192, + "logits/rejected": -1.1762993335723877, + "logps/chosen": -0.8579041361808777, + "logps/rejected": -1.3002510070800781, + "loss": 0.9136, + "num_input_tokens_seen": 1592448, + "odds_ratio_loss": 9.070494651794434, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.08579041808843613, + "rewards/margins": 0.04423467814922333, + "rewards/rejected": -0.13002508878707886, + "sft_loss": 0.006512727588415146, + "step": 1210 + }, + { + "epoch": 0.09490470633994555, + "grad_norm": 25.387502670288086, + "learning_rate": 4.8897024998838363e-05, + "logits/chosen": -0.6901842355728149, + "logits/rejected": -1.5692954063415527, + "logps/chosen": -0.918140709400177, + "logps/rejected": -1.9830029010772705, + "loss": 0.9607, + "num_input_tokens_seen": 1605392, + "odds_ratio_loss": 8.890172958374023, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.0918140709400177, + "rewards/margins": 0.10648620128631592, + "rewards/rejected": -0.1983002871274948, + "sft_loss": 0.07163845747709274, + "step": 1220 + }, + { + "epoch": 0.09568261376896149, + "grad_norm": 2.1307079792022705, + "learning_rate": 4.887900624184465e-05, + "logits/chosen": -0.3583410382270813, + "logits/rejected": -1.3679367303848267, + "logps/chosen": -0.8472444415092468, + "logps/rejected": -2.5962655544281006, + "loss": 0.8907, + "num_input_tokens_seen": 1619680, + "odds_ratio_loss": 8.66484546661377, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.08472445607185364, + "rewards/margins": 0.17490212619304657, + "rewards/rejected": -0.259626567363739, + "sft_loss": 0.024249475449323654, + "step": 1230 + }, + { + "epoch": 0.09646052119797745, + "grad_norm": 4.3973283767700195, + "learning_rate": 4.886084486774596e-05, + "logits/chosen": 0.027990514412522316, + "logits/rejected": -1.3642051219940186, + "logps/chosen": -0.9216273427009583, + "logps/rejected": -1.4038372039794922, + "loss": 0.9652, + "num_input_tokens_seen": 1635408, + "odds_ratio_loss": 9.592658996582031, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.09216274321079254, + "rewards/margins": 0.04822098836302757, + "rewards/rejected": -0.1403837502002716, + "sft_loss": 0.00590053666383028, + "step": 1240 + }, + { + "epoch": 0.09723842862699339, + "grad_norm": 1.9861226081848145, + "learning_rate": 4.8842540985010896e-05, + "logits/chosen": -0.05565056949853897, + "logits/rejected": -1.8242801427841187, + "logps/chosen": -0.8096386194229126, + "logps/rejected": -2.4190220832824707, + "loss": 0.8558, + "num_input_tokens_seen": 1650128, + "odds_ratio_loss": 8.537267684936523, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.0809638649225235, + "rewards/margins": 0.1609383374452591, + "rewards/rejected": -0.2419021874666214, + "sft_loss": 0.0020962427370250225, + "step": 1250 + }, + { + "epoch": 0.09801633605600933, + "grad_norm": 4.605903148651123, + "learning_rate": 4.882409470295921e-05, + "logits/chosen": -0.8531273007392883, + "logits/rejected": -0.8438383936882019, + "logps/chosen": -1.0195088386535645, + "logps/rejected": -1.2654592990875244, + "loss": 1.0799, + "num_input_tokens_seen": 1660384, + "odds_ratio_loss": 10.63732624053955, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.10195086896419525, + "rewards/margins": 0.024595048278570175, + "rewards/rejected": -0.12654592096805573, + "sft_loss": 0.016118118539452553, + "step": 1260 + }, + { + "epoch": 0.09879424348502529, + "grad_norm": 34.797637939453125, + "learning_rate": 4.880550613176111e-05, + "logits/chosen": -0.18267883360385895, + "logits/rejected": -1.4181299209594727, + "logps/chosen": -1.1587470769882202, + "logps/rejected": -2.900524854660034, + "loss": 1.1986, + "num_input_tokens_seen": 1679104, + "odds_ratio_loss": 11.581075668334961, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.11587472259998322, + "rewards/margins": 0.17417779564857483, + "rewards/rejected": -0.29005253314971924, + "sft_loss": 0.04045722261071205, + "step": 1270 + }, + { + "epoch": 0.09957215091404123, + "grad_norm": 2.3860249519348145, + "learning_rate": 4.8786775382436646e-05, + "logits/chosen": 0.012228338047862053, + "logits/rejected": -1.394362449645996, + "logps/chosen": -1.030656099319458, + "logps/rejected": -2.4141783714294434, + "loss": 1.0842, + "num_input_tokens_seen": 1694560, + "odds_ratio_loss": 10.41369342803955, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.1030656099319458, + "rewards/margins": 0.13835224509239197, + "rewards/rejected": -0.24141785502433777, + "sft_loss": 0.04287029802799225, + "step": 1280 + }, + { + "epoch": 0.10035005834305717, + "grad_norm": 1.527199625968933, + "learning_rate": 4.8767902566855004e-05, + "logits/chosen": -0.12059100717306137, + "logits/rejected": -1.3332120180130005, + "logps/chosen": -0.8000995516777039, + "logps/rejected": -1.1178014278411865, + "loss": 0.8518, + "num_input_tokens_seen": 1708288, + "odds_ratio_loss": 8.446915626525879, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.08000995963811874, + "rewards/margins": 0.03177017718553543, + "rewards/rejected": -0.11178014427423477, + "sft_loss": 0.0071195634081959724, + "step": 1290 + }, + { + "epoch": 0.10112796577207313, + "grad_norm": 4.501274108886719, + "learning_rate": 4.874888779773388e-05, + "logits/chosen": -0.2184199094772339, + "logits/rejected": -1.169959306716919, + "logps/chosen": -0.9113515615463257, + "logps/rejected": -1.130590796470642, + "loss": 0.9644, + "num_input_tokens_seen": 1723024, + "odds_ratio_loss": 9.515974044799805, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.09113515913486481, + "rewards/margins": 0.02192392572760582, + "rewards/rejected": -0.11305908113718033, + "sft_loss": 0.012761697173118591, + "step": 1300 + }, + { + "epoch": 0.10190587320108907, + "grad_norm": 2.2603354454040527, + "learning_rate": 4.872973118863877e-05, + "logits/chosen": -0.058025557547807693, + "logits/rejected": -0.956487774848938, + "logps/chosen": -0.8199183344841003, + "logps/rejected": -1.225602388381958, + "loss": 0.8717, + "num_input_tokens_seen": 1731728, + "odds_ratio_loss": 8.62109375, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.08199183642864227, + "rewards/margins": 0.04056840017437935, + "rewards/rejected": -0.12256024032831192, + "sft_loss": 0.00954120047390461, + "step": 1310 + }, + { + "epoch": 0.10268378063010501, + "grad_norm": 27.04770278930664, + "learning_rate": 4.871043285398231e-05, + "logits/chosen": -0.09762338548898697, + "logits/rejected": -1.3713834285736084, + "logps/chosen": -0.8707898855209351, + "logps/rejected": -1.5118682384490967, + "loss": 0.9145, + "num_input_tokens_seen": 1743568, + "odds_ratio_loss": 8.92141056060791, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.0870789885520935, + "rewards/margins": 0.06410785019397736, + "rewards/rejected": -0.15118683874607086, + "sft_loss": 0.02232534810900688, + "step": 1320 + }, + { + "epoch": 0.10346168805912097, + "grad_norm": 2.578408718109131, + "learning_rate": 4.8690992909023606e-05, + "logits/chosen": -0.3252541124820709, + "logits/rejected": -1.0389803647994995, + "logps/chosen": -0.9129737019538879, + "logps/rejected": -1.4556406736373901, + "loss": 0.954, + "num_input_tokens_seen": 1756272, + "odds_ratio_loss": 8.928561210632324, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.09129737317562103, + "rewards/margins": 0.05426669865846634, + "rewards/rejected": -0.14556407928466797, + "sft_loss": 0.0611644871532917, + "step": 1330 + }, + { + "epoch": 0.10423959548813691, + "grad_norm": 2.151125907897949, + "learning_rate": 4.86714114698675e-05, + "logits/chosen": -0.3139194846153259, + "logits/rejected": -1.2926034927368164, + "logps/chosen": -0.7979762554168701, + "logps/rejected": -1.579063057899475, + "loss": 0.8471, + "num_input_tokens_seen": 1769328, + "odds_ratio_loss": 8.359407424926758, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.0797976404428482, + "rewards/margins": 0.07810865342617035, + "rewards/rejected": -0.15790629386901855, + "sft_loss": 0.011173962615430355, + "step": 1340 + }, + { + "epoch": 0.10501750291715285, + "grad_norm": 3.7997851371765137, + "learning_rate": 4.865168865346393e-05, + "logits/chosen": -0.1871212124824524, + "logits/rejected": -1.3270195722579956, + "logps/chosen": -0.9265115857124329, + "logps/rejected": -1.0720789432525635, + "loss": 0.9921, + "num_input_tokens_seen": 1783504, + "odds_ratio_loss": 9.808382034301758, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.09265115857124329, + "rewards/margins": 0.014556738547980785, + "rewards/rejected": -0.10720790922641754, + "sft_loss": 0.011273539625108242, + "step": 1350 + }, + { + "epoch": 0.10579541034616881, + "grad_norm": 9.577467918395996, + "learning_rate": 4.863182457760722e-05, + "logits/chosen": -0.16290147602558136, + "logits/rejected": -1.5737485885620117, + "logps/chosen": -1.00246000289917, + "logps/rejected": -1.5742182731628418, + "loss": 1.0608, + "num_input_tokens_seen": 1804496, + "odds_ratio_loss": 10.485246658325195, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.10024599730968475, + "rewards/margins": 0.057175833731889725, + "rewards/rejected": -0.15742184221744537, + "sft_loss": 0.012254795990884304, + "step": 1360 + }, + { + "epoch": 0.10657331777518475, + "grad_norm": 3.4173192977905273, + "learning_rate": 4.861181936093533e-05, + "logits/chosen": 0.01892375573515892, + "logits/rejected": -1.1257383823394775, + "logps/chosen": -0.9024341702461243, + "logps/rejected": -1.1515446901321411, + "loss": 0.9693, + "num_input_tokens_seen": 1817024, + "odds_ratio_loss": 9.511759757995605, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.09024341404438019, + "rewards/margins": 0.02491104230284691, + "rewards/rejected": -0.11515446752309799, + "sft_loss": 0.018130768090486526, + "step": 1370 + }, + { + "epoch": 0.1073512252042007, + "grad_norm": 62.1481819152832, + "learning_rate": 4.859167312292919e-05, + "logits/chosen": -0.0794883444905281, + "logits/rejected": -1.1580173969268799, + "logps/chosen": -1.090137243270874, + "logps/rejected": -1.5018608570098877, + "loss": 1.1456, + "num_input_tokens_seen": 1824992, + "odds_ratio_loss": 10.897436141967773, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.10901372134685516, + "rewards/margins": 0.04117236286401749, + "rewards/rejected": -0.15018606185913086, + "sft_loss": 0.055823612958192825, + "step": 1380 + }, + { + "epoch": 0.10812913263321665, + "grad_norm": 6.845315456390381, + "learning_rate": 4.857138598391201e-05, + "logits/chosen": -0.37189120054244995, + "logits/rejected": -0.7980047464370728, + "logps/chosen": -0.8744696378707886, + "logps/rejected": -1.1153911352157593, + "loss": 0.9373, + "num_input_tokens_seen": 1836608, + "odds_ratio_loss": 9.201163291931152, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.08744696527719498, + "rewards/margins": 0.02409215085208416, + "rewards/rejected": -0.11153911054134369, + "sft_loss": 0.017146050930023193, + "step": 1390 + }, + { + "epoch": 0.10890704006223259, + "grad_norm": 3.4855692386627197, + "learning_rate": 4.855095806504849e-05, + "logits/chosen": -0.17076408863067627, + "logits/rejected": -1.1082465648651123, + "logps/chosen": -0.8685711026191711, + "logps/rejected": -1.8029320240020752, + "loss": 0.9069, + "num_input_tokens_seen": 1850608, + "odds_ratio_loss": 8.906305313110352, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.08685711771249771, + "rewards/margins": 0.0934360921382904, + "rewards/rejected": -0.1802932173013687, + "sft_loss": 0.016220781952142715, + "step": 1400 + }, + { + "epoch": 0.10968494749124855, + "grad_norm": 2.627373695373535, + "learning_rate": 4.8530389488344176e-05, + "logits/chosen": -0.3422343134880066, + "logits/rejected": -0.7871395945549011, + "logps/chosen": -1.2754361629486084, + "logps/rejected": -2.4510159492492676, + "loss": 1.3462, + "num_input_tokens_seen": 1863232, + "odds_ratio_loss": 12.735507011413574, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.1275436282157898, + "rewards/margins": 0.11755795776844025, + "rewards/rejected": -0.24510157108306885, + "sft_loss": 0.07264798879623413, + "step": 1410 + }, + { + "epoch": 0.11046285492026449, + "grad_norm": 6.909982681274414, + "learning_rate": 4.850968037664463e-05, + "logits/chosen": 0.03238892927765846, + "logits/rejected": -1.570744514465332, + "logps/chosen": -0.822226881980896, + "logps/rejected": -2.0880284309387207, + "loss": 0.8495, + "num_input_tokens_seen": 1874480, + "odds_ratio_loss": 8.436205863952637, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.08222270011901855, + "rewards/margins": 0.1265801340341568, + "rewards/rejected": -0.20880281925201416, + "sft_loss": 0.005918358452618122, + "step": 1420 + }, + { + "epoch": 0.11124076234928043, + "grad_norm": 2.540519952774048, + "learning_rate": 4.8488830853634835e-05, + "logits/chosen": -0.6312833428382874, + "logits/rejected": -0.735874354839325, + "logps/chosen": -0.9134576916694641, + "logps/rejected": -2.435875177383423, + "loss": 0.9673, + "num_input_tokens_seen": 1884768, + "odds_ratio_loss": 9.412214279174805, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.09134577214717865, + "rewards/margins": 0.1522417664527893, + "rewards/rejected": -0.24358752369880676, + "sft_loss": 0.026076728478074074, + "step": 1430 + }, + { + "epoch": 0.11201866977829639, + "grad_norm": 2.2013466358184814, + "learning_rate": 4.8467841043838315e-05, + "logits/chosen": -0.2646324932575226, + "logits/rejected": -1.3403586149215698, + "logps/chosen": -1.020293116569519, + "logps/rejected": -2.1002237796783447, + "loss": 1.0608, + "num_input_tokens_seen": 1897216, + "odds_ratio_loss": 10.047431945800781, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.10202930867671967, + "rewards/margins": 0.10799306631088257, + "rewards/rejected": -0.21002237498760223, + "sft_loss": 0.05609869956970215, + "step": 1440 + }, + { + "epoch": 0.11279657720731233, + "grad_norm": 6.7772908210754395, + "learning_rate": 4.844671107261649e-05, + "logits/chosen": -0.565276026725769, + "logits/rejected": -1.2158313989639282, + "logps/chosen": -1.1465237140655518, + "logps/rejected": -2.0157454013824463, + "loss": 1.1853, + "num_input_tokens_seen": 1908384, + "odds_ratio_loss": 11.634048461914062, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.1146523728966713, + "rewards/margins": 0.08692215383052826, + "rewards/rejected": -0.20157453417778015, + "sft_loss": 0.021862532943487167, + "step": 1450 + }, + { + "epoch": 0.11357448463632827, + "grad_norm": 1.6834242343902588, + "learning_rate": 4.842544106616789e-05, + "logits/chosen": 0.08155706524848938, + "logits/rejected": -1.4333988428115845, + "logps/chosen": -0.9613820910453796, + "logps/rejected": -1.2888075113296509, + "loss": 1.0162, + "num_input_tokens_seen": 1923600, + "odds_ratio_loss": 10.128491401672363, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.09613820910453796, + "rewards/margins": 0.032742537558078766, + "rewards/rejected": -0.12888075411319733, + "sft_loss": 0.0033340167719870806, + "step": 1460 + }, + { + "epoch": 0.11435239206534423, + "grad_norm": 1.6985890865325928, + "learning_rate": 4.8404031151527375e-05, + "logits/chosen": 0.07832728326320648, + "logits/rejected": -1.7995054721832275, + "logps/chosen": -0.9585608243942261, + "logps/rejected": -2.9924867153167725, + "loss": 0.9941, + "num_input_tokens_seen": 1939600, + "odds_ratio_loss": 9.870729446411133, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.09585610032081604, + "rewards/margins": 0.20339255034923553, + "rewards/rejected": -0.2992486357688904, + "sft_loss": 0.007033943198621273, + "step": 1470 + }, + { + "epoch": 0.11513029949436017, + "grad_norm": 3.885937452316284, + "learning_rate": 4.838248145656545e-05, + "logits/chosen": -0.3160240650177002, + "logits/rejected": -1.2407915592193604, + "logps/chosen": -1.0432136058807373, + "logps/rejected": -1.7586313486099243, + "loss": 1.0903, + "num_input_tokens_seen": 1950864, + "odds_ratio_loss": 10.701848983764648, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.10432137548923492, + "rewards/margins": 0.07154176384210587, + "rewards/rejected": -0.175863116979599, + "sft_loss": 0.020145347341895103, + "step": 1480 + }, + { + "epoch": 0.11590820692337611, + "grad_norm": 6.555927753448486, + "learning_rate": 4.836079210998743e-05, + "logits/chosen": -0.38438206911087036, + "logits/rejected": -1.0794670581817627, + "logps/chosen": -1.1527174711227417, + "logps/rejected": -1.513710618019104, + "loss": 1.2121, + "num_input_tokens_seen": 1960960, + "odds_ratio_loss": 10.904621124267578, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.11527176201343536, + "rewards/margins": 0.036099307239055634, + "rewards/rejected": -0.1513710618019104, + "sft_loss": 0.12164745479822159, + "step": 1490 + }, + { + "epoch": 0.11668611435239207, + "grad_norm": 4.298940181732178, + "learning_rate": 4.833896324133269e-05, + "logits/chosen": -0.027753740549087524, + "logits/rejected": -1.0681359767913818, + "logps/chosen": -0.6166560649871826, + "logps/rejected": -1.9046653509140015, + "loss": 0.661, + "num_input_tokens_seen": 1974304, + "odds_ratio_loss": 6.545144081115723, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.0616656132042408, + "rewards/margins": 0.1288009136915207, + "rewards/rejected": -0.1904665231704712, + "sft_loss": 0.006462976336479187, + "step": 1500 + }, + { + "epoch": 0.11746402178140801, + "grad_norm": 3.952348470687866, + "learning_rate": 4.8316994980973924e-05, + "logits/chosen": -0.04972705617547035, + "logits/rejected": -1.4225316047668457, + "logps/chosen": -0.8579155206680298, + "logps/rejected": -0.9653614163398743, + "loss": 0.9237, + "num_input_tokens_seen": 1987552, + "odds_ratio_loss": 9.174779891967773, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.08579154312610626, + "rewards/margins": 0.010744589380919933, + "rewards/rejected": -0.09653614461421967, + "sft_loss": 0.006242315750569105, + "step": 1510 + }, + { + "epoch": 0.11824192921042397, + "grad_norm": 2.8112475872039795, + "learning_rate": 4.8294887460116315e-05, + "logits/chosen": -0.1582581251859665, + "logits/rejected": -1.0091254711151123, + "logps/chosen": -0.8943713903427124, + "logps/rejected": -1.2091583013534546, + "loss": 0.9495, + "num_input_tokens_seen": 2004064, + "odds_ratio_loss": 9.371780395507812, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.08943714946508408, + "rewards/margins": 0.03147869184613228, + "rewards/rejected": -0.12091584503650665, + "sft_loss": 0.012294212356209755, + "step": 1520 + }, + { + "epoch": 0.11901983663943991, + "grad_norm": 2.3471946716308594, + "learning_rate": 4.82726408107968e-05, + "logits/chosen": -0.16725249588489532, + "logits/rejected": -1.5413681268692017, + "logps/chosen": -1.0477161407470703, + "logps/rejected": -2.4742424488067627, + "loss": 1.0902, + "num_input_tokens_seen": 2016864, + "odds_ratio_loss": 10.714027404785156, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.10477161407470703, + "rewards/margins": 0.14265263080596924, + "rewards/rejected": -0.24742427468299866, + "sft_loss": 0.01874869130551815, + "step": 1530 + }, + { + "epoch": 0.11979774406845585, + "grad_norm": 3.0610897541046143, + "learning_rate": 4.825025516588325e-05, + "logits/chosen": -0.3162890076637268, + "logits/rejected": -1.1895010471343994, + "logps/chosen": -1.181998610496521, + "logps/rejected": -1.7462644577026367, + "loss": 1.2516, + "num_input_tokens_seen": 2034576, + "odds_ratio_loss": 11.94798469543457, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.11819986253976822, + "rewards/margins": 0.056426577270030975, + "rewards/rejected": -0.1746264398097992, + "sft_loss": 0.05677152797579765, + "step": 1540 + }, + { + "epoch": 0.1205756514974718, + "grad_norm": 3.618778944015503, + "learning_rate": 4.822773065907368e-05, + "logits/chosen": -0.20199429988861084, + "logits/rejected": -1.1273068189620972, + "logps/chosen": -0.8808591961860657, + "logps/rejected": -2.315542221069336, + "loss": 0.9078, + "num_input_tokens_seen": 2048816, + "odds_ratio_loss": 8.197305679321289, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.08808592706918716, + "rewards/margins": 0.14346830546855927, + "rewards/rejected": -0.23155423998832703, + "sft_loss": 0.08805806934833527, + "step": 1550 + }, + { + "epoch": 0.12135355892648775, + "grad_norm": 2.7620632648468018, + "learning_rate": 4.8205067424895464e-05, + "logits/chosen": -0.1513085812330246, + "logits/rejected": -1.329477071762085, + "logps/chosen": -1.0271084308624268, + "logps/rejected": -1.7614002227783203, + "loss": 1.0768, + "num_input_tokens_seen": 2059440, + "odds_ratio_loss": 10.682302474975586, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.10271084308624268, + "rewards/margins": 0.0734291821718216, + "rewards/rejected": -0.17614004015922546, + "sft_loss": 0.008544063195586205, + "step": 1560 + }, + { + "epoch": 0.12213146635550369, + "grad_norm": 3.6041388511657715, + "learning_rate": 4.8182265598704536e-05, + "logits/chosen": 0.1286245584487915, + "logits/rejected": -1.131695032119751, + "logps/chosen": -1.0837154388427734, + "logps/rejected": -1.2388603687286377, + "loss": 1.1554, + "num_input_tokens_seen": 2076736, + "odds_ratio_loss": 11.479883193969727, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.1083715409040451, + "rewards/margins": 0.015514492988586426, + "rewards/rejected": -0.12388603389263153, + "sft_loss": 0.007456196006387472, + "step": 1570 + }, + { + "epoch": 0.12290937378451965, + "grad_norm": 3.0888729095458984, + "learning_rate": 4.8159325316684524e-05, + "logits/chosen": 0.005407279822975397, + "logits/rejected": -1.2279163599014282, + "logps/chosen": -0.6726976633071899, + "logps/rejected": -1.9678910970687866, + "loss": 0.7018, + "num_input_tokens_seen": 2087664, + "odds_ratio_loss": 6.969337463378906, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.06726975739002228, + "rewards/margins": 0.12951937317848206, + "rewards/rejected": -0.19678911566734314, + "sft_loss": 0.0048288991674780846, + "step": 1580 + }, + { + "epoch": 0.12368728121353559, + "grad_norm": 1.3733124732971191, + "learning_rate": 4.813624671584604e-05, + "logits/chosen": -0.06323923170566559, + "logits/rejected": -1.087624430656433, + "logps/chosen": -0.7786732912063599, + "logps/rejected": -1.5691801309585571, + "loss": 0.8094, + "num_input_tokens_seen": 2101888, + "odds_ratio_loss": 7.827001094818115, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.07786732167005539, + "rewards/margins": 0.0790506973862648, + "rewards/rejected": -0.1569180190563202, + "sft_loss": 0.026730451732873917, + "step": 1590 + }, + { + "epoch": 0.12446518864255153, + "grad_norm": 1.6684848070144653, + "learning_rate": 4.811302993402576e-05, + "logits/chosen": -0.01859951578080654, + "logits/rejected": -1.343358039855957, + "logps/chosen": -0.8631114959716797, + "logps/rejected": -1.6435956954956055, + "loss": 0.9019, + "num_input_tokens_seen": 2113792, + "odds_ratio_loss": 8.9646577835083, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.08631114661693573, + "rewards/margins": 0.07804843038320541, + "rewards/rejected": -0.16435958445072174, + "sft_loss": 0.005461776163429022, + "step": 1600 + }, + { + "epoch": 0.12524309607156747, + "grad_norm": 2.922821283340454, + "learning_rate": 4.808967510988567e-05, + "logits/chosen": -0.4168400764465332, + "logits/rejected": -1.010283350944519, + "logps/chosen": -0.817193865776062, + "logps/rejected": -1.413362741470337, + "loss": 0.8562, + "num_input_tokens_seen": 2128176, + "odds_ratio_loss": 8.470296859741211, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.08171939849853516, + "rewards/margins": 0.059616874903440475, + "rewards/rejected": -0.14133629202842712, + "sft_loss": 0.009124183095991611, + "step": 1610 + }, + { + "epoch": 0.12602100350058343, + "grad_norm": 2.7644245624542236, + "learning_rate": 4.806618238291221e-05, + "logits/chosen": -0.06522510945796967, + "logits/rejected": -1.4439966678619385, + "logps/chosen": -0.9762029647827148, + "logps/rejected": -1.2714521884918213, + "loss": 1.0377, + "num_input_tokens_seen": 2137808, + "odds_ratio_loss": 10.251848220825195, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.09762029349803925, + "rewards/margins": 0.029524922370910645, + "rewards/rejected": -0.1271452009677887, + "sft_loss": 0.012493262067437172, + "step": 1620 + }, + { + "epoch": 0.12679891092959938, + "grad_norm": 9.554707527160645, + "learning_rate": 4.8042551893415413e-05, + "logits/chosen": -0.07397669553756714, + "logits/rejected": -1.4646739959716797, + "logps/chosen": -0.9141004681587219, + "logps/rejected": -1.7428028583526611, + "loss": 0.9564, + "num_input_tokens_seen": 2149376, + "odds_ratio_loss": 9.50117015838623, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.09141004830598831, + "rewards/margins": 0.0828702375292778, + "rewards/rejected": -0.17428025603294373, + "sft_loss": 0.006242091301828623, + "step": 1630 + }, + { + "epoch": 0.1275768183586153, + "grad_norm": 11.70290756225586, + "learning_rate": 4.801878378252814e-05, + "logits/chosen": -0.33434975147247314, + "logits/rejected": -1.3799288272857666, + "logps/chosen": -0.9573726654052734, + "logps/rejected": -1.7814598083496094, + "loss": 0.9976, + "num_input_tokens_seen": 2162304, + "odds_ratio_loss": 9.793394088745117, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.0957372635602951, + "rewards/margins": 0.08240871876478195, + "rewards/rejected": -0.17814597487449646, + "sft_loss": 0.018307171761989594, + "step": 1640 + }, + { + "epoch": 0.12835472578763127, + "grad_norm": 2.2834181785583496, + "learning_rate": 4.799487819220516e-05, + "logits/chosen": -0.09276885539293289, + "logits/rejected": -1.5006898641586304, + "logps/chosen": -0.8897638320922852, + "logps/rejected": -1.636853575706482, + "loss": 0.9256, + "num_input_tokens_seen": 2174896, + "odds_ratio_loss": 9.220222473144531, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.08897639065980911, + "rewards/margins": 0.0747089758515358, + "rewards/rejected": -0.16368535161018372, + "sft_loss": 0.0035331719554960728, + "step": 1650 + }, + { + "epoch": 0.12913263321664722, + "grad_norm": 2.08956241607666, + "learning_rate": 4.7970835265222374e-05, + "logits/chosen": -0.12462923675775528, + "logits/rejected": -1.3280928134918213, + "logps/chosen": -0.8080765008926392, + "logps/rejected": -1.0823023319244385, + "loss": 0.8614, + "num_input_tokens_seen": 2189168, + "odds_ratio_loss": 8.544610023498535, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.0808076411485672, + "rewards/margins": 0.0274225864559412, + "rewards/rejected": -0.10823021829128265, + "sft_loss": 0.006928306072950363, + "step": 1660 + }, + { + "epoch": 0.12991054064566315, + "grad_norm": 2.577354907989502, + "learning_rate": 4.794665514517588e-05, + "logits/chosen": -0.08472862094640732, + "logits/rejected": -1.416602373123169, + "logps/chosen": -0.7574108242988586, + "logps/rejected": -1.9338436126708984, + "loss": 0.7921, + "num_input_tokens_seen": 2200080, + "odds_ratio_loss": 7.881539821624756, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.07574108242988586, + "rewards/margins": 0.1176433190703392, + "rewards/rejected": -0.19338437914848328, + "sft_loss": 0.003906466998159885, + "step": 1670 + }, + { + "epoch": 0.1306884480746791, + "grad_norm": 2.179121255874634, + "learning_rate": 4.792233797648119e-05, + "logits/chosen": -0.22027191519737244, + "logits/rejected": -1.1554373502731323, + "logps/chosen": -0.9135554432868958, + "logps/rejected": -1.2012169361114502, + "loss": 0.9724, + "num_input_tokens_seen": 2210304, + "odds_ratio_loss": 9.670771598815918, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.09135554730892181, + "rewards/margins": 0.028766151517629623, + "rewards/rejected": -0.12012170255184174, + "sft_loss": 0.005340161733329296, + "step": 1680 + }, + { + "epoch": 0.13146635550369506, + "grad_norm": 2.312490940093994, + "learning_rate": 4.789788390437233e-05, + "logits/chosen": -0.25304338335990906, + "logits/rejected": -0.7925938367843628, + "logps/chosen": -0.8756221532821655, + "logps/rejected": -1.050024390220642, + "loss": 0.9342, + "num_input_tokens_seen": 2222928, + "odds_ratio_loss": 8.787582397460938, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.0875622034072876, + "rewards/margins": 0.017440233379602432, + "rewards/rejected": -0.10500244051218033, + "sft_loss": 0.05540759488940239, + "step": 1690 + }, + { + "epoch": 0.13224426293271102, + "grad_norm": 1.9695780277252197, + "learning_rate": 4.7873293074900984e-05, + "logits/chosen": 0.15910422801971436, + "logits/rejected": -1.55144464969635, + "logps/chosen": -0.951107382774353, + "logps/rejected": -1.2606693506240845, + "loss": 1.0041, + "num_input_tokens_seen": 2235984, + "odds_ratio_loss": 9.915519714355469, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.09511072933673859, + "rewards/margins": 0.030956193804740906, + "rewards/rejected": -0.12606693804264069, + "sft_loss": 0.012589936144649982, + "step": 1700 + }, + { + "epoch": 0.13302217036172695, + "grad_norm": 2.928964853286743, + "learning_rate": 4.7848565634935605e-05, + "logits/chosen": -0.24512776732444763, + "logits/rejected": -1.1192513704299927, + "logps/chosen": -1.0003267526626587, + "logps/rejected": -1.929778814315796, + "loss": 1.0628, + "num_input_tokens_seen": 2245280, + "odds_ratio_loss": 9.978555679321289, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.10003267228603363, + "rewards/margins": 0.09294520318508148, + "rewards/rejected": -0.1929778754711151, + "sft_loss": 0.06496123969554901, + "step": 1710 + }, + { + "epoch": 0.1338000777907429, + "grad_norm": 2.1121087074279785, + "learning_rate": 4.782370173216056e-05, + "logits/chosen": -0.08122909069061279, + "logits/rejected": -1.1934936046600342, + "logps/chosen": -0.8961094617843628, + "logps/rejected": -1.861797571182251, + "loss": 0.9457, + "num_input_tokens_seen": 2259712, + "odds_ratio_loss": 9.394018173217773, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.08961095660924911, + "rewards/margins": 0.09656880795955658, + "rewards/rejected": -0.1861797571182251, + "sft_loss": 0.006347438786178827, + "step": 1720 + }, + { + "epoch": 0.13457798521975886, + "grad_norm": 1.7788792848587036, + "learning_rate": 4.7798701515075236e-05, + "logits/chosen": 0.067621149122715, + "logits/rejected": -1.4974925518035889, + "logps/chosen": -0.9736189842224121, + "logps/rejected": -1.7845993041992188, + "loss": 1.0156, + "num_input_tokens_seen": 2275328, + "odds_ratio_loss": 10.119224548339844, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.09736189991235733, + "rewards/margins": 0.0810980275273323, + "rewards/rejected": -0.17845991253852844, + "sft_loss": 0.003652214305475354, + "step": 1730 + }, + { + "epoch": 0.1353558926487748, + "grad_norm": 6.806885242462158, + "learning_rate": 4.777356513299317e-05, + "logits/chosen": -0.2556004524230957, + "logits/rejected": -1.1190215349197388, + "logps/chosen": -1.2167608737945557, + "logps/rejected": -1.4703128337860107, + "loss": 1.2947, + "num_input_tokens_seen": 2285040, + "odds_ratio_loss": 12.760774612426758, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.12167608737945557, + "rewards/margins": 0.025355201214551926, + "rewards/rejected": -0.1470312774181366, + "sft_loss": 0.01857621595263481, + "step": 1740 + }, + { + "epoch": 0.13613380007779075, + "grad_norm": 3.1653451919555664, + "learning_rate": 4.7748292736041114e-05, + "logits/chosen": -0.3644051253795624, + "logits/rejected": -1.1646645069122314, + "logps/chosen": -1.006195068359375, + "logps/rejected": -1.2280415296554565, + "loss": 1.0739, + "num_input_tokens_seen": 2296960, + "odds_ratio_loss": 10.570137023925781, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.10061951726675034, + "rewards/margins": 0.022184643894433975, + "rewards/rejected": -0.12280416488647461, + "sft_loss": 0.016888169571757317, + "step": 1750 + }, + { + "epoch": 0.1369117075068067, + "grad_norm": 7.864021301269531, + "learning_rate": 4.77228844751582e-05, + "logits/chosen": -0.11170890182256699, + "logits/rejected": -0.6443509459495544, + "logps/chosen": -0.7633668780326843, + "logps/rejected": -1.3804105520248413, + "loss": 0.811, + "num_input_tokens_seen": 2307584, + "odds_ratio_loss": 7.9055681228637695, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.07633669674396515, + "rewards/margins": 0.0617043562233448, + "rewards/rejected": -0.13804103434085846, + "sft_loss": 0.020423702895641327, + "step": 1760 + }, + { + "epoch": 0.13768961493582263, + "grad_norm": 22.770339965820312, + "learning_rate": 4.769734050209498e-05, + "logits/chosen": 0.022691845893859863, + "logits/rejected": -1.144308090209961, + "logps/chosen": -0.9744008183479309, + "logps/rejected": -2.4538748264312744, + "loss": 1.0189, + "num_input_tokens_seen": 2322576, + "odds_ratio_loss": 9.946294784545898, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.09744006395339966, + "rewards/margins": 0.14794743061065674, + "rewards/rejected": -0.2453875094652176, + "sft_loss": 0.024235859513282776, + "step": 1770 + }, + { + "epoch": 0.13846752236483859, + "grad_norm": 1.8606911897659302, + "learning_rate": 4.7671660969412556e-05, + "logits/chosen": -0.13664504885673523, + "logits/rejected": -0.7414021492004395, + "logps/chosen": -0.9381047487258911, + "logps/rejected": -1.4822548627853394, + "loss": 0.9899, + "num_input_tokens_seen": 2332432, + "odds_ratio_loss": 9.61351203918457, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.09381048381328583, + "rewards/margins": 0.0544150173664093, + "rewards/rejected": -0.14822550117969513, + "sft_loss": 0.028502438217401505, + "step": 1780 + }, + { + "epoch": 0.13924542979385454, + "grad_norm": 2.0753164291381836, + "learning_rate": 4.7645846030481656e-05, + "logits/chosen": -0.0015734911430627108, + "logits/rejected": -1.2405633926391602, + "logps/chosen": -0.9424698948860168, + "logps/rejected": -1.425398349761963, + "loss": 0.9868, + "num_input_tokens_seen": 2345360, + "odds_ratio_loss": 9.795587539672852, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.09424698352813721, + "rewards/margins": 0.0482928566634655, + "rewards/rejected": -0.1425398290157318, + "sft_loss": 0.0072590867057442665, + "step": 1790 + }, + { + "epoch": 0.14002333722287047, + "grad_norm": 1.9002023935317993, + "learning_rate": 4.761989583948173e-05, + "logits/chosen": 0.12105624377727509, + "logits/rejected": -1.2195909023284912, + "logps/chosen": -1.0945359468460083, + "logps/rejected": -1.4879124164581299, + "loss": 1.2197, + "num_input_tokens_seen": 2356688, + "odds_ratio_loss": 10.811766624450684, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.10945358127355576, + "rewards/margins": 0.0393376424908638, + "rewards/rejected": -0.14879122376441956, + "sft_loss": 0.1385701596736908, + "step": 1800 + }, + { + "epoch": 0.14080124465188643, + "grad_norm": 4.353161811828613, + "learning_rate": 4.7593810551399996e-05, + "logits/chosen": -0.2872072756290436, + "logits/rejected": -1.187946081161499, + "logps/chosen": -1.1624119281768799, + "logps/rejected": -2.9942376613616943, + "loss": 1.2146, + "num_input_tokens_seen": 2368128, + "odds_ratio_loss": 11.870499610900879, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.11624119430780411, + "rewards/margins": 0.18318259716033936, + "rewards/rejected": -0.29942378401756287, + "sft_loss": 0.027569111436605453, + "step": 1810 + }, + { + "epoch": 0.14157915208090238, + "grad_norm": 2.369858741760254, + "learning_rate": 4.756759032203058e-05, + "logits/chosen": -0.4844452440738678, + "logits/rejected": -1.1404305696487427, + "logps/chosen": -0.870577335357666, + "logps/rejected": -2.190336227416992, + "loss": 0.8944, + "num_input_tokens_seen": 2379456, + "odds_ratio_loss": 8.819035530090332, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.08705773204565048, + "rewards/margins": 0.13197587430477142, + "rewards/rejected": -0.2190335988998413, + "sft_loss": 0.012462143786251545, + "step": 1820 + }, + { + "epoch": 0.1423570595099183, + "grad_norm": 2.999417304992676, + "learning_rate": 4.754123530797349e-05, + "logits/chosen": -0.22438016533851624, + "logits/rejected": -1.3170385360717773, + "logps/chosen": -0.9477856755256653, + "logps/rejected": -1.3005917072296143, + "loss": 1.0135, + "num_input_tokens_seen": 2390576, + "odds_ratio_loss": 10.019603729248047, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.09477856755256653, + "rewards/margins": 0.03528060019016266, + "rewards/rejected": -0.1300591677427292, + "sft_loss": 0.011541618965566158, + "step": 1830 + }, + { + "epoch": 0.14313496693893427, + "grad_norm": 9.854547500610352, + "learning_rate": 4.7514745666633765e-05, + "logits/chosen": -0.17480389773845673, + "logits/rejected": -1.8269084692001343, + "logps/chosen": -0.9835509061813354, + "logps/rejected": -1.5236918926239014, + "loss": 1.0258, + "num_input_tokens_seen": 2403664, + "odds_ratio_loss": 10.115317344665527, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.09835509955883026, + "rewards/margins": 0.05401410534977913, + "rewards/rejected": -0.1523691713809967, + "sft_loss": 0.01426604948937893, + "step": 1840 + }, + { + "epoch": 0.14391287436795022, + "grad_norm": 1.7128846645355225, + "learning_rate": 4.748812155622051e-05, + "logits/chosen": -0.6130805015563965, + "logits/rejected": -1.215207815170288, + "logps/chosen": -1.1691943407058716, + "logps/rejected": -2.4545435905456543, + "loss": 1.2283, + "num_input_tokens_seen": 2412992, + "odds_ratio_loss": 11.835399627685547, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.11691941320896149, + "rewards/margins": 0.1285349428653717, + "rewards/rejected": -0.245454341173172, + "sft_loss": 0.04480103775858879, + "step": 1850 + }, + { + "epoch": 0.14469078179696615, + "grad_norm": 2.1353652477264404, + "learning_rate": 4.746136313574593e-05, + "logits/chosen": -0.2938610017299652, + "logits/rejected": -1.362804651260376, + "logps/chosen": -0.9929152727127075, + "logps/rejected": -2.309945583343506, + "loss": 1.0447, + "num_input_tokens_seen": 2425600, + "odds_ratio_loss": 10.25415325164795, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.09929152578115463, + "rewards/margins": 0.13170306384563446, + "rewards/rejected": -0.23099461197853088, + "sft_loss": 0.019318608567118645, + "step": 1860 + }, + { + "epoch": 0.1454686892259821, + "grad_norm": 5.580964088439941, + "learning_rate": 4.74344705650244e-05, + "logits/chosen": -0.2848891615867615, + "logits/rejected": -1.2315860986709595, + "logps/chosen": -0.9934043884277344, + "logps/rejected": -1.4588521718978882, + "loss": 1.0503, + "num_input_tokens_seen": 2434992, + "odds_ratio_loss": 10.418456077575684, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.09934045374393463, + "rewards/margins": 0.046544767916202545, + "rewards/rejected": -0.14588521420955658, + "sft_loss": 0.008504530414938927, + "step": 1870 + }, + { + "epoch": 0.14624659665499806, + "grad_norm": 3.9102535247802734, + "learning_rate": 4.74074440046715e-05, + "logits/chosen": -0.15559545159339905, + "logits/rejected": -1.0929044485092163, + "logps/chosen": -0.9797903299331665, + "logps/rejected": -1.7228037118911743, + "loss": 1.0229, + "num_input_tokens_seen": 2449712, + "odds_ratio_loss": 10.148366928100586, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.09797903150320053, + "rewards/margins": 0.0743013322353363, + "rewards/rejected": -0.17228034138679504, + "sft_loss": 0.008058695122599602, + "step": 1880 + }, + { + "epoch": 0.147024504084014, + "grad_norm": 2.150550603866577, + "learning_rate": 4.738028361610308e-05, + "logits/chosen": -0.06973959505558014, + "logits/rejected": -1.7949050664901733, + "logps/chosen": -0.9789167642593384, + "logps/rejected": -1.9717823266983032, + "loss": 1.0202, + "num_input_tokens_seen": 2466224, + "odds_ratio_loss": 9.753425598144531, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.0978916734457016, + "rewards/margins": 0.09928654879331589, + "rewards/rejected": -0.19717822968959808, + "sft_loss": 0.04484931379556656, + "step": 1890 + }, + { + "epoch": 0.14780241151302995, + "grad_norm": 1.9614287614822388, + "learning_rate": 4.7352989561534254e-05, + "logits/chosen": -0.08428236842155457, + "logits/rejected": -1.1122157573699951, + "logps/chosen": -0.8200074434280396, + "logps/rejected": -1.4336493015289307, + "loss": 0.8636, + "num_input_tokens_seen": 2481952, + "odds_ratio_loss": 8.588362693786621, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.0820007398724556, + "rewards/margins": 0.06136419251561165, + "rewards/rejected": -0.14336493611335754, + "sft_loss": 0.0047540245577692986, + "step": 1900 + }, + { + "epoch": 0.1485803189420459, + "grad_norm": 2.2471156120300293, + "learning_rate": 4.732556200397848e-05, + "logits/chosen": 0.047021109610795975, + "logits/rejected": -1.0816609859466553, + "logps/chosen": -0.7682046294212341, + "logps/rejected": -0.7830635905265808, + "loss": 0.8538, + "num_input_tokens_seen": 2495664, + "odds_ratio_loss": 8.502764701843262, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.07682047039270401, + "rewards/margins": 0.0014858979266136885, + "rewards/rejected": -0.07830636203289032, + "sft_loss": 0.0034969509579241276, + "step": 1910 + }, + { + "epoch": 0.14935822637106183, + "grad_norm": 1.4071708917617798, + "learning_rate": 4.729800110724654e-05, + "logits/chosen": -0.16252145171165466, + "logits/rejected": -1.0662734508514404, + "logps/chosen": -0.9402019381523132, + "logps/rejected": -1.3474619388580322, + "loss": 1.0023, + "num_input_tokens_seen": 2512992, + "odds_ratio_loss": 9.984285354614258, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.09402020275592804, + "rewards/margins": 0.040725983679294586, + "rewards/rejected": -0.13474617898464203, + "sft_loss": 0.0038742502219974995, + "step": 1920 + }, + { + "epoch": 0.1501361338000778, + "grad_norm": 2.530600070953369, + "learning_rate": 4.727030703594557e-05, + "logits/chosen": -0.022717636078596115, + "logits/rejected": -1.2906973361968994, + "logps/chosen": -0.9707010388374329, + "logps/rejected": -1.3414376974105835, + "loss": 1.0211, + "num_input_tokens_seen": 2529264, + "odds_ratio_loss": 10.094282150268555, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.09707011282444, + "rewards/margins": 0.037073664367198944, + "rewards/rejected": -0.13414378464221954, + "sft_loss": 0.011715163476765156, + "step": 1930 + }, + { + "epoch": 0.15091404122909374, + "grad_norm": 6.173549652099609, + "learning_rate": 4.7242479955478145e-05, + "logits/chosen": -0.04072573408484459, + "logits/rejected": -0.8758264780044556, + "logps/chosen": -0.844284176826477, + "logps/rejected": -1.7218849658966064, + "loss": 0.8893, + "num_input_tokens_seen": 2537824, + "odds_ratio_loss": 8.815264701843262, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.08442841470241547, + "rewards/margins": 0.0877600908279419, + "rewards/rejected": -0.17218850553035736, + "sft_loss": 0.007810283452272415, + "step": 1940 + }, + { + "epoch": 0.1516919486581097, + "grad_norm": 3.6885008811950684, + "learning_rate": 4.721452003204118e-05, + "logits/chosen": -0.03177540749311447, + "logits/rejected": -1.4413988590240479, + "logps/chosen": -0.8689084053039551, + "logps/rejected": -2.869048595428467, + "loss": 0.8987, + "num_input_tokens_seen": 2551184, + "odds_ratio_loss": 8.938272476196289, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.08689083904027939, + "rewards/margins": 0.20001402497291565, + "rewards/rejected": -0.28690487146377563, + "sft_loss": 0.004882193636149168, + "step": 1950 + }, + { + "epoch": 0.15246985608712563, + "grad_norm": 10.387799263000488, + "learning_rate": 4.7186427432625005e-05, + "logits/chosen": -0.015120279975235462, + "logits/rejected": -0.9145073890686035, + "logps/chosen": -0.6851322054862976, + "logps/rejected": -1.577131986618042, + "loss": 0.7254, + "num_input_tokens_seen": 2568528, + "odds_ratio_loss": 7.129334449768066, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.06851321458816528, + "rewards/margins": 0.0891999751329422, + "rewards/rejected": -0.15771318972110748, + "sft_loss": 0.012512214481830597, + "step": 1960 + }, + { + "epoch": 0.15324776351614158, + "grad_norm": 12.012364387512207, + "learning_rate": 4.715820232501239e-05, + "logits/chosen": -0.10855318605899811, + "logits/rejected": -1.3677846193313599, + "logps/chosen": -0.998216450214386, + "logps/rejected": -1.1140133142471313, + "loss": 1.064, + "num_input_tokens_seen": 2582928, + "odds_ratio_loss": 10.501612663269043, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.09982164204120636, + "rewards/margins": 0.01157968956977129, + "rewards/rejected": -0.11140134185552597, + "sft_loss": 0.013802697882056236, + "step": 1970 + }, + { + "epoch": 0.15402567094515754, + "grad_norm": 2.588481903076172, + "learning_rate": 4.7129844877777456e-05, + "logits/chosen": -0.28296536207199097, + "logits/rejected": -1.1763520240783691, + "logps/chosen": -0.9495035409927368, + "logps/rejected": -1.0533324480056763, + "loss": 1.0163, + "num_input_tokens_seen": 2595600, + "odds_ratio_loss": 10.042470932006836, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.0949503630399704, + "rewards/margins": 0.010382888838648796, + "rewards/rejected": -0.10533325374126434, + "sft_loss": 0.01205428782850504, + "step": 1980 + }, + { + "epoch": 0.15480357837417347, + "grad_norm": 5.5801897048950195, + "learning_rate": 4.710135526028477e-05, + "logits/chosen": 0.04890453815460205, + "logits/rejected": -1.548763632774353, + "logps/chosen": -0.9930458068847656, + "logps/rejected": -1.474796175956726, + "loss": 1.0394, + "num_input_tokens_seen": 2616560, + "odds_ratio_loss": 10.355806350708008, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.09930458664894104, + "rewards/margins": 0.048175033181905746, + "rewards/rejected": -0.14747962355613708, + "sft_loss": 0.0038034829776734114, + "step": 1990 + }, + { + "epoch": 0.15558148580318942, + "grad_norm": 1.9893940687179565, + "learning_rate": 4.707273364268825e-05, + "logits/chosen": -0.15720073878765106, + "logits/rejected": -0.8983931541442871, + "logps/chosen": -1.2000892162322998, + "logps/rejected": -1.751647710800171, + "loss": 1.2625, + "num_input_tokens_seen": 2629616, + "odds_ratio_loss": 11.613740921020508, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.1200089082121849, + "rewards/margins": 0.055155880749225616, + "rewards/rejected": -0.17516478896141052, + "sft_loss": 0.10116754472255707, + "step": 2000 + }, + { + "epoch": 0.15635939323220538, + "grad_norm": 1.1382113695144653, + "learning_rate": 4.70439801959302e-05, + "logits/chosen": -0.15353144705295563, + "logits/rejected": -1.406969428062439, + "logps/chosen": -1.0247169733047485, + "logps/rejected": -1.8631995916366577, + "loss": 1.0967, + "num_input_tokens_seen": 2643328, + "odds_ratio_loss": 10.818486213684082, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.10247169435024261, + "rewards/margins": 0.08384827524423599, + "rewards/rejected": -0.1863199770450592, + "sft_loss": 0.01486129779368639, + "step": 2010 + }, + { + "epoch": 0.1571373006612213, + "grad_norm": 9.748759269714355, + "learning_rate": 4.701509509174027e-05, + "logits/chosen": 0.012250833213329315, + "logits/rejected": -1.0942684412002563, + "logps/chosen": -0.9024358987808228, + "logps/rejected": -1.7884238958358765, + "loss": 0.9446, + "num_input_tokens_seen": 2653136, + "odds_ratio_loss": 9.02326488494873, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.09024359285831451, + "rewards/margins": 0.08859880268573761, + "rewards/rejected": -0.17884239554405212, + "sft_loss": 0.042305268347263336, + "step": 2020 + }, + { + "epoch": 0.15791520809023726, + "grad_norm": 2.3677189350128174, + "learning_rate": 4.698607850263442e-05, + "logits/chosen": -0.1969408094882965, + "logits/rejected": -1.3645201921463013, + "logps/chosen": -0.9819061160087585, + "logps/rejected": -1.4384701251983643, + "loss": 1.036, + "num_input_tokens_seen": 2662192, + "odds_ratio_loss": 10.110983848571777, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.09819060564041138, + "rewards/margins": 0.045656394213438034, + "rewards/rejected": -0.1438470035791397, + "sft_loss": 0.024949556216597557, + "step": 2030 + }, + { + "epoch": 0.15869311551925322, + "grad_norm": 1.7309666872024536, + "learning_rate": 4.6956930601913924e-05, + "logits/chosen": 0.028863852843642235, + "logits/rejected": -1.6371591091156006, + "logps/chosen": -1.029279112815857, + "logps/rejected": -2.1302683353424072, + "loss": 1.069, + "num_input_tokens_seen": 2677456, + "odds_ratio_loss": 10.366458892822266, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.10292792320251465, + "rewards/margins": 0.11009888350963593, + "rewards/rejected": -0.21302680671215057, + "sft_loss": 0.032364509999752045, + "step": 2040 + }, + { + "epoch": 0.15947102294826915, + "grad_norm": 2.105114698410034, + "learning_rate": 4.6927651563664296e-05, + "logits/chosen": -0.2948930859565735, + "logits/rejected": -0.6920993328094482, + "logps/chosen": -0.723526656627655, + "logps/rejected": -1.1117141246795654, + "loss": 0.7734, + "num_input_tokens_seen": 2692944, + "odds_ratio_loss": 7.633302211761475, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.07235266268253326, + "rewards/margins": 0.03881874680519104, + "rewards/rejected": -0.1111714243888855, + "sft_loss": 0.010118861682713032, + "step": 2050 + }, + { + "epoch": 0.1602489303772851, + "grad_norm": 8.95576000213623, + "learning_rate": 4.689824156275428e-05, + "logits/chosen": -0.4269348680973053, + "logits/rejected": -0.9338502883911133, + "logps/chosen": -0.8120001554489136, + "logps/rejected": -1.077885389328003, + "loss": 0.8686, + "num_input_tokens_seen": 2703904, + "odds_ratio_loss": 8.558115005493164, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.0812000185251236, + "rewards/margins": 0.02658851072192192, + "rewards/rejected": -0.10778852552175522, + "sft_loss": 0.01276195328682661, + "step": 2060 + }, + { + "epoch": 0.16102683780630106, + "grad_norm": 7.518883228302002, + "learning_rate": 4.686870077483478e-05, + "logits/chosen": -0.5116538405418396, + "logits/rejected": -0.652612566947937, + "logps/chosen": -1.0511877536773682, + "logps/rejected": -1.480696439743042, + "loss": 1.1021, + "num_input_tokens_seen": 2714112, + "odds_ratio_loss": 10.629408836364746, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.1051187738776207, + "rewards/margins": 0.04295086860656738, + "rewards/rejected": -0.14806964993476868, + "sft_loss": 0.03919699788093567, + "step": 2070 + }, + { + "epoch": 0.161804745235317, + "grad_norm": 2.5984585285186768, + "learning_rate": 4.6839029376337825e-05, + "logits/chosen": -0.25364458560943604, + "logits/rejected": -1.1314189434051514, + "logps/chosen": -0.9039451479911804, + "logps/rejected": -3.1899101734161377, + "loss": 0.9435, + "num_input_tokens_seen": 2728544, + "odds_ratio_loss": 9.305583000183105, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.0903945118188858, + "rewards/margins": 0.22859647870063782, + "rewards/rejected": -0.3189910054206848, + "sft_loss": 0.012947039678692818, + "step": 2080 + }, + { + "epoch": 0.16258265266433294, + "grad_norm": 2.2118406295776367, + "learning_rate": 4.680922754447554e-05, + "logits/chosen": -0.28953272104263306, + "logits/rejected": -0.8960913419723511, + "logps/chosen": -1.0112407207489014, + "logps/rejected": -1.2518157958984375, + "loss": 1.0744, + "num_input_tokens_seen": 2742592, + "odds_ratio_loss": 9.534306526184082, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.10112407058477402, + "rewards/margins": 0.02405751682817936, + "rewards/rejected": -0.12518158555030823, + "sft_loss": 0.12095620483160019, + "step": 2090 + }, + { + "epoch": 0.1633605600933489, + "grad_norm": 1.8755844831466675, + "learning_rate": 4.6779295457239025e-05, + "logits/chosen": -0.12629246711730957, + "logits/rejected": -1.1058032512664795, + "logps/chosen": -0.8406488299369812, + "logps/rejected": -1.691583275794983, + "loss": 0.8857, + "num_input_tokens_seen": 2754208, + "odds_ratio_loss": 8.595414161682129, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.08406487852334976, + "rewards/margins": 0.0850934311747551, + "rewards/rejected": -0.16915829479694366, + "sft_loss": 0.026196395978331566, + "step": 2100 + }, + { + "epoch": 0.16413846752236483, + "grad_norm": 5.349771022796631, + "learning_rate": 4.6749233293397346e-05, + "logits/chosen": -0.8121733665466309, + "logits/rejected": -0.8647876977920532, + "logps/chosen": -0.8223133087158203, + "logps/rejected": -1.5897068977355957, + "loss": 0.8577, + "num_input_tokens_seen": 2763648, + "odds_ratio_loss": 8.505762100219727, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.08223132789134979, + "rewards/margins": 0.0767393708229065, + "rewards/rejected": -0.1589706987142563, + "sft_loss": 0.007078877184540033, + "step": 2110 + }, + { + "epoch": 0.16491637495138078, + "grad_norm": 0.6865819692611694, + "learning_rate": 4.671904123249645e-05, + "logits/chosen": -0.2070043534040451, + "logits/rejected": -1.5442055463790894, + "logps/chosen": -0.7468840479850769, + "logps/rejected": -1.994861364364624, + "loss": 0.7813, + "num_input_tokens_seen": 2778096, + "odds_ratio_loss": 7.69118595123291, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.07468841224908829, + "rewards/margins": 0.12479771673679352, + "rewards/rejected": -0.1994861364364624, + "sft_loss": 0.012225426733493805, + "step": 2120 + }, + { + "epoch": 0.16569428238039674, + "grad_norm": 5.290821552276611, + "learning_rate": 4.66887194548581e-05, + "logits/chosen": 0.052118025720119476, + "logits/rejected": -1.3986949920654297, + "logps/chosen": -0.9030756950378418, + "logps/rejected": -1.7299425601959229, + "loss": 0.9546, + "num_input_tokens_seen": 2789712, + "odds_ratio_loss": 9.505033493041992, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.0903075709939003, + "rewards/margins": 0.08268668502569199, + "rewards/rejected": -0.17299427092075348, + "sft_loss": 0.00411962578073144, + "step": 2130 + }, + { + "epoch": 0.16647218980941267, + "grad_norm": 4.0711798667907715, + "learning_rate": 4.665826814157878e-05, + "logits/chosen": -0.04582834988832474, + "logits/rejected": -1.3113749027252197, + "logps/chosen": -1.0264580249786377, + "logps/rejected": -1.1542977094650269, + "loss": 1.0925, + "num_input_tokens_seen": 2805600, + "odds_ratio_loss": 10.873018264770508, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.10264579206705093, + "rewards/margins": 0.01278398372232914, + "rewards/rejected": -0.11542978137731552, + "sft_loss": 0.005238743498921394, + "step": 2140 + }, + { + "epoch": 0.16725009723842862, + "grad_norm": 1.4560567140579224, + "learning_rate": 4.662768747452862e-05, + "logits/chosen": -0.21474024653434753, + "logits/rejected": -1.1197646856307983, + "logps/chosen": -0.805280864238739, + "logps/rejected": -1.3285717964172363, + "loss": 0.86, + "num_input_tokens_seen": 2816848, + "odds_ratio_loss": 7.857496738433838, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.08052808791399002, + "rewards/margins": 0.05232909321784973, + "rewards/rejected": -0.13285718858242035, + "sft_loss": 0.07425809651613235, + "step": 2150 + }, + { + "epoch": 0.16802800466744458, + "grad_norm": 1.6971755027770996, + "learning_rate": 4.6596977636350325e-05, + "logits/chosen": -0.24752207100391388, + "logits/rejected": -1.3036912679672241, + "logps/chosen": -1.1429294347763062, + "logps/rejected": -3.008378267288208, + "loss": 1.2139, + "num_input_tokens_seen": 2827472, + "odds_ratio_loss": 11.748734474182129, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.1142929419875145, + "rewards/margins": 0.18654491007328033, + "rewards/rejected": -0.30083784461021423, + "sft_loss": 0.03905164450407028, + "step": 2160 + }, + { + "epoch": 0.1688059120964605, + "grad_norm": 3.530958652496338, + "learning_rate": 4.656613881045808e-05, + "logits/chosen": -0.20504558086395264, + "logits/rejected": -1.0912846326828003, + "logps/chosen": -0.8128896951675415, + "logps/rejected": -1.1443490982055664, + "loss": 0.8659, + "num_input_tokens_seen": 2839984, + "odds_ratio_loss": 8.59929084777832, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.08128897100687027, + "rewards/margins": 0.03314594551920891, + "rewards/rejected": -0.11443491280078888, + "sft_loss": 0.005995646119117737, + "step": 2170 + }, + { + "epoch": 0.16958381952547646, + "grad_norm": 2.934725761413574, + "learning_rate": 4.653517118103642e-05, + "logits/chosen": -0.06856352090835571, + "logits/rejected": -1.1658178567886353, + "logps/chosen": -0.7763811349868774, + "logps/rejected": -3.0190224647521973, + "loss": 0.8124, + "num_input_tokens_seen": 2851120, + "odds_ratio_loss": 7.866454124450684, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.07763811200857162, + "rewards/margins": 0.22426410019397736, + "rewards/rejected": -0.30190223455429077, + "sft_loss": 0.025785360485315323, + "step": 2180 + }, + { + "epoch": 0.17036172695449242, + "grad_norm": 31.268569946289062, + "learning_rate": 4.6504074933039185e-05, + "logits/chosen": -0.23514564335346222, + "logits/rejected": -0.8977664709091187, + "logps/chosen": -0.7419293522834778, + "logps/rejected": -1.1732724905014038, + "loss": 0.7893, + "num_input_tokens_seen": 2859984, + "odds_ratio_loss": 7.76514196395874, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.07419292628765106, + "rewards/margins": 0.04313432425260544, + "rewards/rejected": -0.1173272579908371, + "sft_loss": 0.012793630361557007, + "step": 2190 + }, + { + "epoch": 0.17113963438350835, + "grad_norm": 3.4862051010131836, + "learning_rate": 4.647285025218837e-05, + "logits/chosen": 0.15971137583255768, + "logits/rejected": -1.107314109802246, + "logps/chosen": -1.1586195230484009, + "logps/rejected": -1.0516465902328491, + "loss": 1.2555, + "num_input_tokens_seen": 2876672, + "odds_ratio_loss": 11.734086036682129, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.11586196720600128, + "rewards/margins": -0.010697288438677788, + "rewards/rejected": -0.10516466945409775, + "sft_loss": 0.08209719508886337, + "step": 2200 + }, + { + "epoch": 0.1719175418125243, + "grad_norm": 2.9709460735321045, + "learning_rate": 4.644149732497305e-05, + "logits/chosen": -0.35991913080215454, + "logits/rejected": -0.9069361686706543, + "logps/chosen": -0.9952637553215027, + "logps/rejected": -1.203416109085083, + "loss": 1.0537, + "num_input_tokens_seen": 2887568, + "odds_ratio_loss": 9.413312911987305, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.09952638298273087, + "rewards/margins": 0.020815229043364525, + "rewards/rejected": -0.12034161388874054, + "sft_loss": 0.11237955093383789, + "step": 2210 + }, + { + "epoch": 0.17269544924154026, + "grad_norm": 2.8426527976989746, + "learning_rate": 4.6410016338648246e-05, + "logits/chosen": -0.5125675201416016, + "logits/rejected": -0.7049168348312378, + "logps/chosen": -0.9495420455932617, + "logps/rejected": -1.2189608812332153, + "loss": 1.0015, + "num_input_tokens_seen": 2901824, + "odds_ratio_loss": 9.808638572692871, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.09495420008897781, + "rewards/margins": 0.026941895484924316, + "rewards/rejected": -0.12189608812332153, + "sft_loss": 0.02062970958650112, + "step": 2220 + }, + { + "epoch": 0.17347335667055622, + "grad_norm": 3.1397180557250977, + "learning_rate": 4.637840748123379e-05, + "logits/chosen": -0.31803905963897705, + "logits/rejected": -1.4113385677337646, + "logps/chosen": -1.1904270648956299, + "logps/rejected": -1.7130476236343384, + "loss": 1.2448, + "num_input_tokens_seen": 2914160, + "odds_ratio_loss": 12.293588638305664, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.11904270946979523, + "rewards/margins": 0.052262045443058014, + "rewards/rejected": -0.17130474746227264, + "sft_loss": 0.015424099750816822, + "step": 2230 + }, + { + "epoch": 0.17425126409957215, + "grad_norm": 1.6466132402420044, + "learning_rate": 4.634667094151325e-05, + "logits/chosen": -0.2235032320022583, + "logits/rejected": -0.9990476369857788, + "logps/chosen": -0.9258815050125122, + "logps/rejected": -1.9906742572784424, + "loss": 0.9599, + "num_input_tokens_seen": 2925072, + "odds_ratio_loss": 9.475238800048828, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.0925881564617157, + "rewards/margins": 0.10647926479578018, + "rewards/rejected": -0.19906741380691528, + "sft_loss": 0.012406408786773682, + "step": 2240 + }, + { + "epoch": 0.1750291715285881, + "grad_norm": 8.311766624450684, + "learning_rate": 4.6314806909032766e-05, + "logits/chosen": 0.008072186261415482, + "logits/rejected": -1.0863254070281982, + "logps/chosen": -0.8214001655578613, + "logps/rejected": -1.5726441144943237, + "loss": 0.8628, + "num_input_tokens_seen": 2940336, + "odds_ratio_loss": 8.559614181518555, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.08214002847671509, + "rewards/margins": 0.07512439787387848, + "rewards/rejected": -0.15726442635059357, + "sft_loss": 0.006882502697408199, + "step": 2250 + }, + { + "epoch": 0.17580707895760406, + "grad_norm": 3.6163852214813232, + "learning_rate": 4.6282815574099915e-05, + "logits/chosen": -0.03153163567185402, + "logits/rejected": -1.0020339488983154, + "logps/chosen": -0.9039045572280884, + "logps/rejected": -1.3647565841674805, + "loss": 0.9506, + "num_input_tokens_seen": 2955120, + "odds_ratio_loss": 9.3183012008667, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.09039046615362167, + "rewards/margins": 0.04608519747853279, + "rewards/rejected": -0.13647565245628357, + "sft_loss": 0.018744414672255516, + "step": 2260 + }, + { + "epoch": 0.17658498638661999, + "grad_norm": 4.02025032043457, + "learning_rate": 4.625069712778262e-05, + "logits/chosen": 0.02350408211350441, + "logits/rejected": -1.1788727045059204, + "logps/chosen": -1.0226409435272217, + "logps/rejected": -1.3775674104690552, + "loss": 1.0807, + "num_input_tokens_seen": 2971120, + "odds_ratio_loss": 10.762960433959961, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.10226409137248993, + "rewards/margins": 0.03549265116453171, + "rewards/rejected": -0.13775673508644104, + "sft_loss": 0.004435698967427015, + "step": 2270 + }, + { + "epoch": 0.17736289381563594, + "grad_norm": 3.7961761951446533, + "learning_rate": 4.6218451761907916e-05, + "logits/chosen": -0.44552960991859436, + "logits/rejected": -1.1484557390213013, + "logps/chosen": -0.9121963381767273, + "logps/rejected": -1.7838424444198608, + "loss": 0.948, + "num_input_tokens_seen": 2983664, + "odds_ratio_loss": 9.294628143310547, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.09121964126825333, + "rewards/margins": 0.08716461062431335, + "rewards/rejected": -0.17838425934314728, + "sft_loss": 0.018568426370620728, + "step": 2280 + }, + { + "epoch": 0.1781408012446519, + "grad_norm": 2.345093250274658, + "learning_rate": 4.618607966906093e-05, + "logits/chosen": 0.13724735379219055, + "logits/rejected": -1.7830671072006226, + "logps/chosen": -1.0102989673614502, + "logps/rejected": -1.6919376850128174, + "loss": 1.072, + "num_input_tokens_seen": 2994544, + "odds_ratio_loss": 10.666772842407227, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.1010298952460289, + "rewards/margins": 0.06816387176513672, + "rewards/rejected": -0.16919377446174622, + "sft_loss": 0.005309919826686382, + "step": 2290 + }, + { + "epoch": 0.17891870867366783, + "grad_norm": 46.41415786743164, + "learning_rate": 4.615358104258362e-05, + "logits/chosen": -0.37161797285079956, + "logits/rejected": -1.3046677112579346, + "logps/chosen": -0.9496458768844604, + "logps/rejected": -1.2153384685516357, + "loss": 1.0113, + "num_input_tokens_seen": 3007296, + "odds_ratio_loss": 9.624669075012207, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.09496460109949112, + "rewards/margins": 0.02656925842165947, + "rewards/rejected": -0.12153385579586029, + "sft_loss": 0.04888255149126053, + "step": 2300 + }, + { + "epoch": 0.17969661610268378, + "grad_norm": 3.0734081268310547, + "learning_rate": 4.612095607657368e-05, + "logits/chosen": -0.11081395298242569, + "logits/rejected": -1.347336769104004, + "logps/chosen": -0.8333578109741211, + "logps/rejected": -1.7155996561050415, + "loss": 0.8714, + "num_input_tokens_seen": 3019632, + "odds_ratio_loss": 8.64091682434082, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.08333578705787659, + "rewards/margins": 0.08822415769100189, + "rewards/rejected": -0.17155995965003967, + "sft_loss": 0.007296589203178883, + "step": 2310 + }, + { + "epoch": 0.18047452353169974, + "grad_norm": 6.051811218261719, + "learning_rate": 4.6088204965883374e-05, + "logits/chosen": 0.131056547164917, + "logits/rejected": -0.9874318242073059, + "logps/chosen": -0.9227386713027954, + "logps/rejected": -1.371084451675415, + "loss": 0.9708, + "num_input_tokens_seen": 3035568, + "odds_ratio_loss": 9.314332962036133, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.09227386862039566, + "rewards/margins": 0.044834576547145844, + "rewards/rejected": -0.1371084451675415, + "sft_loss": 0.03936336562037468, + "step": 2320 + }, + { + "epoch": 0.18125243096071567, + "grad_norm": 12.885236740112305, + "learning_rate": 4.605532790611834e-05, + "logits/chosen": -0.13883432745933533, + "logits/rejected": -1.3708261251449585, + "logps/chosen": -1.0801494121551514, + "logps/rejected": -1.9117374420166016, + "loss": 1.128, + "num_input_tokens_seen": 3050976, + "odds_ratio_loss": 11.087190628051758, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.10801491886377335, + "rewards/margins": 0.08315880596637726, + "rewards/rejected": -0.1911737322807312, + "sft_loss": 0.019236447289586067, + "step": 2330 + }, + { + "epoch": 0.18203033838973162, + "grad_norm": 1.722999930381775, + "learning_rate": 4.602232509363647e-05, + "logits/chosen": -0.15571023523807526, + "logits/rejected": -1.4897704124450684, + "logps/chosen": -0.9368125796318054, + "logps/rejected": -1.0839000940322876, + "loss": 1.0032, + "num_input_tokens_seen": 3068576, + "odds_ratio_loss": 9.988349914550781, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.09368126094341278, + "rewards/margins": 0.014708745293319225, + "rewards/rejected": -0.10839001089334488, + "sft_loss": 0.004401323851197958, + "step": 2340 + }, + { + "epoch": 0.18280824581874758, + "grad_norm": 2.3113698959350586, + "learning_rate": 4.5989196725546704e-05, + "logits/chosen": -0.5129882097244263, + "logits/rejected": -0.7843137979507446, + "logps/chosen": -0.8895527124404907, + "logps/rejected": -1.5355644226074219, + "loss": 0.9253, + "num_input_tokens_seen": 3080592, + "odds_ratio_loss": 8.968894958496094, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.08895527571439743, + "rewards/margins": 0.06460118293762207, + "rewards/rejected": -0.1535564512014389, + "sft_loss": 0.028372284024953842, + "step": 2350 + }, + { + "epoch": 0.1835861532477635, + "grad_norm": 2.3606832027435303, + "learning_rate": 4.595594299970786e-05, + "logits/chosen": -0.12805475294589996, + "logits/rejected": -1.2973390817642212, + "logps/chosen": -0.7868680953979492, + "logps/rejected": -2.545942783355713, + "loss": 0.842, + "num_input_tokens_seen": 3096016, + "odds_ratio_loss": 8.354960441589355, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.07868681102991104, + "rewards/margins": 0.17590750753879547, + "rewards/rejected": -0.2545942962169647, + "sft_loss": 0.006476639304310083, + "step": 2360 + }, + { + "epoch": 0.18436406067677946, + "grad_norm": 2.4345719814300537, + "learning_rate": 4.592256411472744e-05, + "logits/chosen": 0.039739884436130524, + "logits/rejected": -1.3224941492080688, + "logps/chosen": -0.7756458520889282, + "logps/rejected": -2.3631529808044434, + "loss": 0.7966, + "num_input_tokens_seen": 3107456, + "odds_ratio_loss": 7.849300384521484, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.07756458222866058, + "rewards/margins": 0.1587507277727127, + "rewards/rejected": -0.23631532490253448, + "sft_loss": 0.011640993878245354, + "step": 2370 + }, + { + "epoch": 0.18514196810579542, + "grad_norm": 4.071404457092285, + "learning_rate": 4.5889060269960484e-05, + "logits/chosen": 0.11023125797510147, + "logits/rejected": -1.3383567333221436, + "logps/chosen": -0.9944393038749695, + "logps/rejected": -1.7868140935897827, + "loss": 1.0393, + "num_input_tokens_seen": 3118288, + "odds_ratio_loss": 10.298423767089844, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.09944392740726471, + "rewards/margins": 0.07923749089241028, + "rewards/rejected": -0.17868143320083618, + "sft_loss": 0.009473122656345367, + "step": 2380 + }, + { + "epoch": 0.18591987553481135, + "grad_norm": 3.297738552093506, + "learning_rate": 4.5855431665508345e-05, + "logits/chosen": 0.050270576030015945, + "logits/rejected": -1.6520401239395142, + "logps/chosen": -1.0888586044311523, + "logps/rejected": -1.9133188724517822, + "loss": 1.1342, + "num_input_tokens_seen": 3134016, + "odds_ratio_loss": 11.285490036010742, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.10888586193323135, + "rewards/margins": 0.08244602382183075, + "rewards/rejected": -0.1913318932056427, + "sft_loss": 0.0056497156620025635, + "step": 2390 + }, + { + "epoch": 0.1866977829638273, + "grad_norm": 9.202101707458496, + "learning_rate": 4.582167850221748e-05, + "logits/chosen": 0.002509523881599307, + "logits/rejected": -1.233533263206482, + "logps/chosen": -0.9214789271354675, + "logps/rejected": -0.9710084795951843, + "loss": 1.0009, + "num_input_tokens_seen": 3146544, + "odds_ratio_loss": 9.797624588012695, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.09214789420366287, + "rewards/margins": 0.0049529531970620155, + "rewards/rejected": -0.09710085391998291, + "sft_loss": 0.021165817975997925, + "step": 2400 + }, + { + "epoch": 0.18747569039284326, + "grad_norm": 3.42102313041687, + "learning_rate": 4.5787800981678296e-05, + "logits/chosen": 0.04520236700773239, + "logits/rejected": -1.4154117107391357, + "logps/chosen": -0.6949530839920044, + "logps/rejected": -1.560598611831665, + "loss": 0.7382, + "num_input_tokens_seen": 3164224, + "odds_ratio_loss": 7.338306427001953, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.0694953054189682, + "rewards/margins": 0.0865645632147789, + "rewards/rejected": -0.1560598909854889, + "sft_loss": 0.0043824478052556515, + "step": 2410 + }, + { + "epoch": 0.1882535978218592, + "grad_norm": 4.071820259094238, + "learning_rate": 4.5753799306223925e-05, + "logits/chosen": -0.16165246069431305, + "logits/rejected": -1.2863690853118896, + "logps/chosen": -1.1060092449188232, + "logps/rejected": -1.3174792528152466, + "loss": 1.1651, + "num_input_tokens_seen": 3179568, + "odds_ratio_loss": 11.241264343261719, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.11060092598199844, + "rewards/margins": 0.021147005259990692, + "rewards/rejected": -0.13174793124198914, + "sft_loss": 0.04098730534315109, + "step": 2420 + }, + { + "epoch": 0.18903150525087514, + "grad_norm": 2.3402061462402344, + "learning_rate": 4.5719673678929e-05, + "logits/chosen": -0.33619269728660583, + "logits/rejected": -0.7193960547447205, + "logps/chosen": -0.7185854911804199, + "logps/rejected": -1.4699690341949463, + "loss": 0.7549, + "num_input_tokens_seen": 3192832, + "odds_ratio_loss": 7.497372627258301, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.07185854017734528, + "rewards/margins": 0.07513834536075592, + "rewards/rejected": -0.1469969004392624, + "sft_loss": 0.005116223357617855, + "step": 2430 + }, + { + "epoch": 0.1898094126798911, + "grad_norm": 3.9390127658843994, + "learning_rate": 4.568542430360846e-05, + "logits/chosen": -0.3282054662704468, + "logits/rejected": -1.3228813409805298, + "logps/chosen": -1.1381219625473022, + "logps/rejected": -1.7825075387954712, + "loss": 1.1856, + "num_input_tokens_seen": 3204080, + "odds_ratio_loss": 11.355585098266602, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.11381220817565918, + "rewards/margins": 0.06443854421377182, + "rewards/rejected": -0.1782507598400116, + "sft_loss": 0.05002690106630325, + "step": 2440 + }, + { + "epoch": 0.19058732010890703, + "grad_norm": 3.2009530067443848, + "learning_rate": 4.565105138481631e-05, + "logits/chosen": -0.47313547134399414, + "logits/rejected": -0.8150053024291992, + "logps/chosen": -1.141099214553833, + "logps/rejected": -1.3412197828292847, + "loss": 1.2029, + "num_input_tokens_seen": 3218432, + "odds_ratio_loss": 11.356019973754883, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.11410991847515106, + "rewards/margins": 0.02001207135617733, + "rewards/rejected": -0.13412198424339294, + "sft_loss": 0.06727258861064911, + "step": 2450 + }, + { + "epoch": 0.19136522753792298, + "grad_norm": 2.887192726135254, + "learning_rate": 4.5616555127844455e-05, + "logits/chosen": -0.710000216960907, + "logits/rejected": -0.689724326133728, + "logps/chosen": -0.7518829107284546, + "logps/rejected": -1.4371519088745117, + "loss": 0.7949, + "num_input_tokens_seen": 3231408, + "odds_ratio_loss": 7.861302852630615, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.0751882940530777, + "rewards/margins": 0.06852690875530243, + "rewards/rejected": -0.14371520280838013, + "sft_loss": 0.00877359975129366, + "step": 2460 + }, + { + "epoch": 0.19214313496693894, + "grad_norm": 5.989782333374023, + "learning_rate": 4.55819357387214e-05, + "logits/chosen": -0.17640532553195953, + "logits/rejected": -1.0639609098434448, + "logps/chosen": -1.0197169780731201, + "logps/rejected": -1.1484568119049072, + "loss": 1.091, + "num_input_tokens_seen": 3243872, + "odds_ratio_loss": 10.208763122558594, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.10197170823812485, + "rewards/margins": 0.012873975560069084, + "rewards/rejected": -0.11484567821025848, + "sft_loss": 0.07008983194828033, + "step": 2470 + }, + { + "epoch": 0.1929210423959549, + "grad_norm": 2.637138605117798, + "learning_rate": 4.554719342421108e-05, + "logits/chosen": -0.2950691282749176, + "logits/rejected": -1.0944535732269287, + "logps/chosen": -0.8581943511962891, + "logps/rejected": -0.9821914434432983, + "loss": 0.9194, + "num_input_tokens_seen": 3255296, + "odds_ratio_loss": 9.095025062561035, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.08581943809986115, + "rewards/margins": 0.012399706058204174, + "rewards/rejected": -0.098219133913517, + "sft_loss": 0.009927446953952312, + "step": 2480 + }, + { + "epoch": 0.19369894982497082, + "grad_norm": 3.5724494457244873, + "learning_rate": 4.551232839181159e-05, + "logits/chosen": -0.1675427407026291, + "logits/rejected": -1.0505664348602295, + "logps/chosen": -1.0215306282043457, + "logps/rejected": -1.6657826900482178, + "loss": 1.0779, + "num_input_tokens_seen": 3273408, + "odds_ratio_loss": 10.157814025878906, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.10215306282043457, + "rewards/margins": 0.06442520767450333, + "rewards/rejected": -0.1665782630443573, + "sft_loss": 0.06208919361233711, + "step": 2490 + }, + { + "epoch": 0.19447685725398678, + "grad_norm": 9.58640193939209, + "learning_rate": 4.5477340849753944e-05, + "logits/chosen": -0.2685242295265198, + "logits/rejected": -0.9891644716262817, + "logps/chosen": -0.9026187062263489, + "logps/rejected": -1.8491036891937256, + "loss": 0.9503, + "num_input_tokens_seen": 3284784, + "odds_ratio_loss": 8.730802536010742, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.09026187658309937, + "rewards/margins": 0.09464851021766663, + "rewards/rejected": -0.184910386800766, + "sft_loss": 0.07725141942501068, + "step": 2500 + }, + { + "epoch": 0.19525476468300274, + "grad_norm": 0.9771826863288879, + "learning_rate": 4.544223100700087e-05, + "logits/chosen": -0.17309458553791046, + "logits/rejected": -1.0318530797958374, + "logps/chosen": -0.9268582463264465, + "logps/rejected": -1.784379243850708, + "loss": 0.9709, + "num_input_tokens_seen": 3298928, + "odds_ratio_loss": 9.390172004699707, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.09268582612276077, + "rewards/margins": 0.08575211465358734, + "rewards/rejected": -0.17843793332576752, + "sft_loss": 0.03189669921994209, + "step": 2510 + }, + { + "epoch": 0.19603267211201866, + "grad_norm": 3.369408130645752, + "learning_rate": 4.540699907324552e-05, + "logits/chosen": 0.031320180743932724, + "logits/rejected": -1.3592791557312012, + "logps/chosen": -0.9459033012390137, + "logps/rejected": -1.9444615840911865, + "loss": 1.0037, + "num_input_tokens_seen": 3308880, + "odds_ratio_loss": 9.935675621032715, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.09459032863378525, + "rewards/margins": 0.09985584020614624, + "rewards/rejected": -0.19444617629051208, + "sft_loss": 0.01017682533711195, + "step": 2520 + }, + { + "epoch": 0.19681057954103462, + "grad_norm": 5.482847690582275, + "learning_rate": 4.537164525891023e-05, + "logits/chosen": -0.1788056343793869, + "logits/rejected": -1.1290700435638428, + "logps/chosen": -0.8016532063484192, + "logps/rejected": -2.0308566093444824, + "loss": 0.8472, + "num_input_tokens_seen": 3324896, + "odds_ratio_loss": 8.375109672546387, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.0801653191447258, + "rewards/margins": 0.12292035669088364, + "rewards/rejected": -0.20308569073677063, + "sft_loss": 0.009639225900173187, + "step": 2530 + }, + { + "epoch": 0.19758848697005058, + "grad_norm": 10.093277931213379, + "learning_rate": 4.533616977514527e-05, + "logits/chosen": -0.10968391597270966, + "logits/rejected": -1.4015908241271973, + "logps/chosen": -0.8579193353652954, + "logps/rejected": -1.047619104385376, + "loss": 0.9156, + "num_input_tokens_seen": 3341072, + "odds_ratio_loss": 8.98912239074707, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.0857919305562973, + "rewards/margins": 0.01896999403834343, + "rewards/rejected": -0.10476192086935043, + "sft_loss": 0.0166663508862257, + "step": 2540 + }, + { + "epoch": 0.1983663943990665, + "grad_norm": 1.3628586530685425, + "learning_rate": 4.530057283382759e-05, + "logits/chosen": -0.13702496886253357, + "logits/rejected": -1.4749879837036133, + "logps/chosen": -0.6388250589370728, + "logps/rejected": -1.442430853843689, + "loss": 0.6655, + "num_input_tokens_seen": 3354048, + "odds_ratio_loss": 6.584555149078369, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.06388251483440399, + "rewards/margins": 0.08036056905984879, + "rewards/rejected": -0.14424307644367218, + "sft_loss": 0.007017000578343868, + "step": 2550 + }, + { + "epoch": 0.19914430182808246, + "grad_norm": 2.2133822441101074, + "learning_rate": 4.5264854647559516e-05, + "logits/chosen": -0.0047277300618588924, + "logits/rejected": -2.1251773834228516, + "logps/chosen": -0.9321849942207336, + "logps/rejected": -1.8874273300170898, + "loss": 0.9926, + "num_input_tokens_seen": 3367584, + "odds_ratio_loss": 9.882081031799316, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.09321849048137665, + "rewards/margins": 0.09552423655986786, + "rewards/rejected": -0.1887427270412445, + "sft_loss": 0.004392318893224001, + "step": 2560 + }, + { + "epoch": 0.19992220925709842, + "grad_norm": 3.6601076126098633, + "learning_rate": 4.522901542966756e-05, + "logits/chosen": -0.11102037131786346, + "logits/rejected": -0.8737829923629761, + "logps/chosen": -0.8281256556510925, + "logps/rejected": -1.7336536645889282, + "loss": 0.8642, + "num_input_tokens_seen": 3375024, + "odds_ratio_loss": 8.55268383026123, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.08281257003545761, + "rewards/margins": 0.09055278450250626, + "rewards/rejected": -0.17336536943912506, + "sft_loss": 0.008942141197621822, + "step": 2570 + }, + { + "epoch": 0.20070011668611434, + "grad_norm": 4.413498401641846, + "learning_rate": 4.5193055394201025e-05, + "logits/chosen": -0.2287522554397583, + "logits/rejected": -1.363667607307434, + "logps/chosen": -1.1148064136505127, + "logps/rejected": -1.7411060333251953, + "loss": 1.1658, + "num_input_tokens_seen": 3387376, + "odds_ratio_loss": 11.551223754882812, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.11148063838481903, + "rewards/margins": 0.06262998282909393, + "rewards/rejected": -0.17411060631275177, + "sft_loss": 0.01069558784365654, + "step": 2580 + }, + { + "epoch": 0.2014780241151303, + "grad_norm": 4.33030366897583, + "learning_rate": 4.515697475593085e-05, + "logits/chosen": -0.03919697552919388, + "logits/rejected": -1.1795130968093872, + "logps/chosen": -0.8254842758178711, + "logps/rejected": -0.8753021955490112, + "loss": 0.909, + "num_input_tokens_seen": 3403952, + "odds_ratio_loss": 9.031672477722168, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.08254842460155487, + "rewards/margins": 0.0049817985855042934, + "rewards/rejected": -0.0875302106142044, + "sft_loss": 0.0058304606936872005, + "step": 2590 + }, + { + "epoch": 0.20225593154414626, + "grad_norm": 6.164618968963623, + "learning_rate": 4.512077373034825e-05, + "logits/chosen": -0.1275373250246048, + "logits/rejected": -1.0006495714187622, + "logps/chosen": -0.8229962587356567, + "logps/rejected": -2.5332837104797363, + "loss": 0.8639, + "num_input_tokens_seen": 3415264, + "odds_ratio_loss": 8.567689895629883, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.0822996199131012, + "rewards/margins": 0.1710287630558014, + "rewards/rejected": -0.2533283829689026, + "sft_loss": 0.007158086635172367, + "step": 2600 + }, + { + "epoch": 0.20303383897316218, + "grad_norm": 20.658157348632812, + "learning_rate": 4.5084452533663466e-05, + "logits/chosen": -0.0677039623260498, + "logits/rejected": -1.151421308517456, + "logps/chosen": -1.050412654876709, + "logps/rejected": -2.1510255336761475, + "loss": 1.1177, + "num_input_tokens_seen": 3427392, + "odds_ratio_loss": 10.269535064697266, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.1050412654876709, + "rewards/margins": 0.11006130278110504, + "rewards/rejected": -0.21510258316993713, + "sft_loss": 0.09070286899805069, + "step": 2610 + }, + { + "epoch": 0.20381174640217814, + "grad_norm": 6.2879533767700195, + "learning_rate": 4.504801138280445e-05, + "logits/chosen": -0.12882912158966064, + "logits/rejected": -1.148313283920288, + "logps/chosen": -1.0179685354232788, + "logps/rejected": -1.5564950704574585, + "loss": 1.0858, + "num_input_tokens_seen": 3440992, + "odds_ratio_loss": 10.7903470993042, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.10179685056209564, + "rewards/margins": 0.053852636367082596, + "rewards/rejected": -0.15564949810504913, + "sft_loss": 0.006785678677260876, + "step": 2620 + }, + { + "epoch": 0.2045896538311941, + "grad_norm": 47.05371856689453, + "learning_rate": 4.5011450495415555e-05, + "logits/chosen": 0.04895355924963951, + "logits/rejected": -1.5381346940994263, + "logps/chosen": -0.958077073097229, + "logps/rejected": -2.2323126792907715, + "loss": 1.008, + "num_input_tokens_seen": 3455888, + "odds_ratio_loss": 9.713939666748047, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.09580769389867783, + "rewards/margins": 0.12742355465888977, + "rewards/rejected": -0.223231241106987, + "sft_loss": 0.03663123771548271, + "step": 2630 + }, + { + "epoch": 0.20536756126021002, + "grad_norm": 2.227385997772217, + "learning_rate": 4.49747700898563e-05, + "logits/chosen": -0.019439058378338814, + "logits/rejected": -1.1940211057662964, + "logps/chosen": -0.8006144762039185, + "logps/rejected": -1.171351671218872, + "loss": 0.8501, + "num_input_tokens_seen": 3472864, + "odds_ratio_loss": 8.339761734008789, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.08006144315004349, + "rewards/margins": 0.03707372397184372, + "rewards/rejected": -0.11713516712188721, + "sft_loss": 0.016088882461190224, + "step": 2640 + }, + { + "epoch": 0.20614546868922598, + "grad_norm": 9.593066215515137, + "learning_rate": 4.493797038520001e-05, + "logits/chosen": -0.08355199545621872, + "logits/rejected": -1.4018335342407227, + "logps/chosen": -0.9090895652770996, + "logps/rejected": -2.0214409828186035, + "loss": 0.9523, + "num_input_tokens_seen": 3487200, + "odds_ratio_loss": 9.115594863891602, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.0909089595079422, + "rewards/margins": 0.11123514175415039, + "rewards/rejected": -0.2021441012620926, + "sft_loss": 0.04074696823954582, + "step": 2650 + }, + { + "epoch": 0.20692337611824194, + "grad_norm": 5.4159932136535645, + "learning_rate": 4.4901051601232526e-05, + "logits/chosen": -0.22638556361198425, + "logits/rejected": -1.0507686138153076, + "logps/chosen": -1.0690250396728516, + "logps/rejected": -1.880934476852417, + "loss": 1.1143, + "num_input_tokens_seen": 3502288, + "odds_ratio_loss": 10.672109603881836, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.10690250247716904, + "rewards/margins": 0.08119096606969833, + "rewards/rejected": -0.18809348344802856, + "sft_loss": 0.047121696174144745, + "step": 2660 + }, + { + "epoch": 0.20770128354725786, + "grad_norm": 4.875866413116455, + "learning_rate": 4.4864013958450867e-05, + "logits/chosen": -0.07939404249191284, + "logits/rejected": -1.7361027002334595, + "logps/chosen": -1.1061766147613525, + "logps/rejected": -1.8076751232147217, + "loss": 1.1485, + "num_input_tokens_seen": 3518000, + "odds_ratio_loss": 11.435917854309082, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.11061765998601913, + "rewards/margins": 0.07014986127614975, + "rewards/rejected": -0.18076752126216888, + "sft_loss": 0.004879302810877562, + "step": 2670 + }, + { + "epoch": 0.20847919097627382, + "grad_norm": 1.931626319885254, + "learning_rate": 4.482685767806196e-05, + "logits/chosen": -0.18810833990573883, + "logits/rejected": -1.2008379697799683, + "logps/chosen": -1.0920370817184448, + "logps/rejected": -1.5815891027450562, + "loss": 1.1486, + "num_input_tokens_seen": 3528432, + "odds_ratio_loss": 11.227903366088867, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.10920371860265732, + "rewards/margins": 0.04895520210266113, + "rewards/rejected": -0.15815891325473785, + "sft_loss": 0.025777261704206467, + "step": 2680 + }, + { + "epoch": 0.20925709840528978, + "grad_norm": 7.632688045501709, + "learning_rate": 4.4789582981981294e-05, + "logits/chosen": -0.1754075586795807, + "logits/rejected": -1.7076215744018555, + "logps/chosen": -0.9906984567642212, + "logps/rejected": -2.3729805946350098, + "loss": 1.0319, + "num_input_tokens_seen": 3539520, + "odds_ratio_loss": 10.129727363586426, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.09906984865665436, + "rewards/margins": 0.13822820782661438, + "rewards/rejected": -0.23729804158210754, + "sft_loss": 0.018918436020612717, + "step": 2690 + }, + { + "epoch": 0.2100350058343057, + "grad_norm": 2.336216688156128, + "learning_rate": 4.475219009283157e-05, + "logits/chosen": -0.4126604199409485, + "logits/rejected": -1.1251232624053955, + "logps/chosen": -0.8320616483688354, + "logps/rejected": -1.44077730178833, + "loss": 0.8774, + "num_input_tokens_seen": 3553856, + "odds_ratio_loss": 8.672832489013672, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.0832061693072319, + "rewards/margins": 0.06087157130241394, + "rewards/rejected": -0.14407773315906525, + "sft_loss": 0.01009486522525549, + "step": 2700 + }, + { + "epoch": 0.21081291326332166, + "grad_norm": 10.049501419067383, + "learning_rate": 4.471467923394142e-05, + "logits/chosen": -0.47965916991233826, + "logits/rejected": -0.8203874826431274, + "logps/chosen": -0.8291003108024597, + "logps/rejected": -1.9053233861923218, + "loss": 0.8601, + "num_input_tokens_seen": 3566720, + "odds_ratio_loss": 7.984526634216309, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.08291003853082657, + "rewards/margins": 0.10762230306863785, + "rewards/rejected": -0.19053234159946442, + "sft_loss": 0.061681754887104034, + "step": 2710 + }, + { + "epoch": 0.21159082069233762, + "grad_norm": 1.9754427671432495, + "learning_rate": 4.467705062934405e-05, + "logits/chosen": -0.33071309328079224, + "logits/rejected": -1.6625646352767944, + "logps/chosen": -0.7561646699905396, + "logps/rejected": -2.0466246604919434, + "loss": 0.7865, + "num_input_tokens_seen": 3579088, + "odds_ratio_loss": 7.769416809082031, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.07561646401882172, + "rewards/margins": 0.1290459930896759, + "rewards/rejected": -0.20466244220733643, + "sft_loss": 0.00955834798514843, + "step": 2720 + }, + { + "epoch": 0.21236872812135357, + "grad_norm": 8.636311531066895, + "learning_rate": 4.463930450377587e-05, + "logits/chosen": -0.21290338039398193, + "logits/rejected": -1.0868850946426392, + "logps/chosen": -0.8203240633010864, + "logps/rejected": -1.304052710533142, + "loss": 0.8716, + "num_input_tokens_seen": 3591184, + "odds_ratio_loss": 8.614351272583008, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.08203241229057312, + "rewards/margins": 0.04837285727262497, + "rewards/rejected": -0.1304052770137787, + "sft_loss": 0.010142195969820023, + "step": 2730 + }, + { + "epoch": 0.2131466355503695, + "grad_norm": 2.121227979660034, + "learning_rate": 4.460144108267522e-05, + "logits/chosen": -0.5269980430603027, + "logits/rejected": -1.0767323970794678, + "logps/chosen": -0.8903768658638, + "logps/rejected": -2.5388858318328857, + "loss": 0.9273, + "num_input_tokens_seen": 3600656, + "odds_ratio_loss": 9.161144256591797, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.08903767913579941, + "rewards/margins": 0.16485092043876648, + "rewards/rejected": -0.2538886070251465, + "sft_loss": 0.011165516451001167, + "step": 2740 + }, + { + "epoch": 0.21392454297938546, + "grad_norm": 3.538132667541504, + "learning_rate": 4.4563460592180966e-05, + "logits/chosen": -0.1817234754562378, + "logits/rejected": -0.8993843793869019, + "logps/chosen": -0.9267904162406921, + "logps/rejected": -2.0416767597198486, + "loss": 0.9811, + "num_input_tokens_seen": 3611760, + "odds_ratio_loss": 9.679262161254883, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.09267903864383698, + "rewards/margins": 0.11148865520954132, + "rewards/rejected": -0.2041676789522171, + "sft_loss": 0.013199281878769398, + "step": 2750 + }, + { + "epoch": 0.2147024504084014, + "grad_norm": 1.918135166168213, + "learning_rate": 4.4525363259131175e-05, + "logits/chosen": -0.20659813284873962, + "logits/rejected": -1.5829854011535645, + "logps/chosen": -1.042778730392456, + "logps/rejected": -2.7298874855041504, + "loss": 1.0853, + "num_input_tokens_seen": 3624352, + "odds_ratio_loss": 10.755561828613281, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.10427787154912949, + "rewards/margins": 0.1687108874320984, + "rewards/rejected": -0.2729887366294861, + "sft_loss": 0.009703483432531357, + "step": 2760 + }, + { + "epoch": 0.21548035783741734, + "grad_norm": 1.848811149597168, + "learning_rate": 4.4487149311061754e-05, + "logits/chosen": -0.48155227303504944, + "logits/rejected": -0.7977389097213745, + "logps/chosen": -0.7917124032974243, + "logps/rejected": -1.8758751153945923, + "loss": 0.8297, + "num_input_tokens_seen": 3636736, + "odds_ratio_loss": 7.977292537689209, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.07917123287916183, + "rewards/margins": 0.10841628164052963, + "rewards/rejected": -0.18758751451969147, + "sft_loss": 0.0319630466401577, + "step": 2770 + }, + { + "epoch": 0.2162582652664333, + "grad_norm": 9.27741527557373, + "learning_rate": 4.444881897620509e-05, + "logits/chosen": -0.4793930649757385, + "logits/rejected": -0.9707077145576477, + "logps/chosen": -0.9283552169799805, + "logps/rejected": -1.2709600925445557, + "loss": 0.9828, + "num_input_tokens_seen": 3649920, + "odds_ratio_loss": 9.733741760253906, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.09283552318811417, + "rewards/margins": 0.034260500222444534, + "rewards/rejected": -0.127096027135849, + "sft_loss": 0.009438985027372837, + "step": 2780 + }, + { + "epoch": 0.21703617269544925, + "grad_norm": 5.2187066078186035, + "learning_rate": 4.441037248348869e-05, + "logits/chosen": -0.03495123237371445, + "logits/rejected": -1.3832604885101318, + "logps/chosen": -0.9088050723075867, + "logps/rejected": -1.8502451181411743, + "loss": 0.9442, + "num_input_tokens_seen": 3662992, + "odds_ratio_loss": 9.403382301330566, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.09088051319122314, + "rewards/margins": 0.09414400160312653, + "rewards/rejected": -0.18502449989318848, + "sft_loss": 0.0038636140525341034, + "step": 2790 + }, + { + "epoch": 0.21781408012446518, + "grad_norm": 1.8032280206680298, + "learning_rate": 4.437181006253382e-05, + "logits/chosen": -0.29241687059402466, + "logits/rejected": -1.4174654483795166, + "logps/chosen": -0.9323759078979492, + "logps/rejected": -2.653144121170044, + "loss": 0.9668, + "num_input_tokens_seen": 3674288, + "odds_ratio_loss": 9.554191589355469, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09323760122060776, + "rewards/margins": 0.17207682132720947, + "rewards/rejected": -0.26531440019607544, + "sft_loss": 0.011398935690522194, + "step": 2800 + }, + { + "epoch": 0.21859198755348114, + "grad_norm": 6.935772895812988, + "learning_rate": 4.433313194365411e-05, + "logits/chosen": -0.11382085084915161, + "logits/rejected": -1.1239991188049316, + "logps/chosen": -1.0345927476882935, + "logps/rejected": -2.4821836948394775, + "loss": 1.0816, + "num_input_tokens_seen": 3683472, + "odds_ratio_loss": 10.74571418762207, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.10345926135778427, + "rewards/margins": 0.14475905895233154, + "rewards/rejected": -0.24821837246418, + "sft_loss": 0.00707204919308424, + "step": 2810 + }, + { + "epoch": 0.2193698949824971, + "grad_norm": 1.740234375, + "learning_rate": 4.429433835785422e-05, + "logits/chosen": -0.09553475677967072, + "logits/rejected": -1.2517316341400146, + "logps/chosen": -1.050550937652588, + "logps/rejected": -1.3104311227798462, + "loss": 1.1106, + "num_input_tokens_seen": 3695840, + "odds_ratio_loss": 10.7747220993042, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.10505510866641998, + "rewards/margins": 0.025988012552261353, + "rewards/rejected": -0.13104310631752014, + "sft_loss": 0.033088210970163345, + "step": 2820 + }, + { + "epoch": 0.22014780241151302, + "grad_norm": 5.372836112976074, + "learning_rate": 4.425542953682841e-05, + "logits/chosen": -0.15488819777965546, + "logits/rejected": -1.968838095664978, + "logps/chosen": -0.8588301539421082, + "logps/rejected": -1.6684284210205078, + "loss": 0.9, + "num_input_tokens_seen": 3709792, + "odds_ratio_loss": 8.847394943237305, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.08588302135467529, + "rewards/margins": 0.08095981180667877, + "rewards/rejected": -0.16684284806251526, + "sft_loss": 0.01527967769652605, + "step": 2830 + }, + { + "epoch": 0.22092570984052898, + "grad_norm": 2.368696689605713, + "learning_rate": 4.421640571295918e-05, + "logits/chosen": -0.1854361593723297, + "logits/rejected": -0.9668446779251099, + "logps/chosen": -1.0571156740188599, + "logps/rejected": -2.4429931640625, + "loss": 1.1238, + "num_input_tokens_seen": 3721664, + "odds_ratio_loss": 10.837104797363281, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.10571156442165375, + "rewards/margins": 0.13858774304389954, + "rewards/rejected": -0.24429932236671448, + "sft_loss": 0.04005955904722214, + "step": 2840 + }, + { + "epoch": 0.22170361726954493, + "grad_norm": 3.42175555229187, + "learning_rate": 4.4177267119315905e-05, + "logits/chosen": -0.17983666062355042, + "logits/rejected": -0.6118065118789673, + "logps/chosen": -0.9091208577156067, + "logps/rejected": -1.3174176216125488, + "loss": 0.9641, + "num_input_tokens_seen": 3729968, + "odds_ratio_loss": 9.469156265258789, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.09091208875179291, + "rewards/margins": 0.040829677134752274, + "rewards/rejected": -0.13174176216125488, + "sft_loss": 0.017219817265868187, + "step": 2850 + }, + { + "epoch": 0.22248152469856086, + "grad_norm": 5.314608573913574, + "learning_rate": 4.4138013989653406e-05, + "logits/chosen": -0.027306431904435158, + "logits/rejected": -0.819806694984436, + "logps/chosen": -0.8254560232162476, + "logps/rejected": -2.61415433883667, + "loss": 0.886, + "num_input_tokens_seen": 3744544, + "odds_ratio_loss": 8.529707908630371, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.08254560083150864, + "rewards/margins": 0.1788698434829712, + "rewards/rejected": -0.26141542196273804, + "sft_loss": 0.03305988386273384, + "step": 2860 + }, + { + "epoch": 0.22325943212757682, + "grad_norm": 2.7968525886535645, + "learning_rate": 4.409864655841058e-05, + "logits/chosen": 0.05771436542272568, + "logits/rejected": -1.489516019821167, + "logps/chosen": -1.114302396774292, + "logps/rejected": -2.0234618186950684, + "loss": 1.1547, + "num_input_tokens_seen": 3760112, + "odds_ratio_loss": 11.147990226745605, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.11143024265766144, + "rewards/margins": 0.09091595560312271, + "rewards/rejected": -0.20234617590904236, + "sft_loss": 0.039897941052913666, + "step": 2870 + }, + { + "epoch": 0.22403733955659277, + "grad_norm": 2.459502696990967, + "learning_rate": 4.405916506070897e-05, + "logits/chosen": -0.17634642124176025, + "logits/rejected": -1.053168535232544, + "logps/chosen": -0.7258324027061462, + "logps/rejected": -1.2297321557998657, + "loss": 0.7769, + "num_input_tokens_seen": 3774064, + "odds_ratio_loss": 7.716550350189209, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.07258324325084686, + "rewards/margins": 0.050389982759952545, + "rewards/rejected": -0.12297321856021881, + "sft_loss": 0.005258309654891491, + "step": 2880 + }, + { + "epoch": 0.2248152469856087, + "grad_norm": 2.5929694175720215, + "learning_rate": 4.40195697323514e-05, + "logits/chosen": -0.31808415055274963, + "logits/rejected": -1.3807660341262817, + "logps/chosen": -0.8598939776420593, + "logps/rejected": -1.0913549661636353, + "loss": 0.9157, + "num_input_tokens_seen": 3787120, + "odds_ratio_loss": 9.119691848754883, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.08598940074443817, + "rewards/margins": 0.023146091029047966, + "rewards/rejected": -0.10913548618555069, + "sft_loss": 0.003712741192430258, + "step": 2890 + }, + { + "epoch": 0.22559315441462466, + "grad_norm": 1.6673979759216309, + "learning_rate": 4.397986080982053e-05, + "logits/chosen": -0.18936365842819214, + "logits/rejected": -0.9067906141281128, + "logps/chosen": -0.811468780040741, + "logps/rejected": -1.1548564434051514, + "loss": 0.8606, + "num_input_tokens_seen": 3797968, + "odds_ratio_loss": 8.536993026733398, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.08114688098430634, + "rewards/margins": 0.034338753670454025, + "rewards/rejected": -0.11548563092947006, + "sft_loss": 0.006941842380911112, + "step": 2900 + }, + { + "epoch": 0.22637106184364061, + "grad_norm": 2.810910224914551, + "learning_rate": 4.394003853027748e-05, + "logits/chosen": -0.534250020980835, + "logits/rejected": -0.8749955296516418, + "logps/chosen": -0.7439308762550354, + "logps/rejected": -1.425173282623291, + "loss": 0.7953, + "num_input_tokens_seen": 3812112, + "odds_ratio_loss": 7.768838405609131, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.07439308613538742, + "rewards/margins": 0.06812424957752228, + "rewards/rejected": -0.1425173431634903, + "sft_loss": 0.018367454409599304, + "step": 2910 + }, + { + "epoch": 0.22714896927265654, + "grad_norm": 1.7785518169403076, + "learning_rate": 4.3900103131560376e-05, + "logits/chosen": -0.4530204236507416, + "logits/rejected": -1.1013813018798828, + "logps/chosen": -0.804338276386261, + "logps/rejected": -1.8796924352645874, + "loss": 0.8429, + "num_input_tokens_seen": 3824768, + "odds_ratio_loss": 8.357107162475586, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.08043383061885834, + "rewards/margins": 0.10753540694713593, + "rewards/rejected": -0.18796923756599426, + "sft_loss": 0.0071527608670294285, + "step": 2920 + }, + { + "epoch": 0.2279268767016725, + "grad_norm": 2.90664005279541, + "learning_rate": 4.386005485218294e-05, + "logits/chosen": -0.10354918241500854, + "logits/rejected": -1.3656952381134033, + "logps/chosen": -0.8719033002853394, + "logps/rejected": -5.1970534324646, + "loss": 0.9075, + "num_input_tokens_seen": 3836960, + "odds_ratio_loss": 8.931937217712402, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.08719031512737274, + "rewards/margins": 0.43251505494117737, + "rewards/rejected": -0.5197054147720337, + "sft_loss": 0.014338955283164978, + "step": 2930 + }, + { + "epoch": 0.22870478413068845, + "grad_norm": 2.312278985977173, + "learning_rate": 4.3819893931333106e-05, + "logits/chosen": -0.1507093757390976, + "logits/rejected": -1.4808213710784912, + "logps/chosen": -0.8520559072494507, + "logps/rejected": -2.1594061851501465, + "loss": 0.8842, + "num_input_tokens_seen": 3852752, + "odds_ratio_loss": 8.767353057861328, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.08520559221506119, + "rewards/margins": 0.13073505461215973, + "rewards/rejected": -0.21594062447547913, + "sft_loss": 0.007421643938869238, + "step": 2940 + }, + { + "epoch": 0.22948269155970438, + "grad_norm": 2.6451447010040283, + "learning_rate": 4.3779620608871526e-05, + "logits/chosen": -0.20507876574993134, + "logits/rejected": -0.8062494397163391, + "logps/chosen": -1.0439388751983643, + "logps/rejected": -1.4790273904800415, + "loss": 1.0918, + "num_input_tokens_seen": 3863120, + "odds_ratio_loss": 10.622690200805664, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.10439388453960419, + "rewards/margins": 0.043508853763341904, + "rewards/rejected": -0.1479027271270752, + "sft_loss": 0.029571745544672012, + "step": 2950 + }, + { + "epoch": 0.23026059898872034, + "grad_norm": 2.9421136379241943, + "learning_rate": 4.373923512533018e-05, + "logits/chosen": -0.11698918044567108, + "logits/rejected": -1.4051399230957031, + "logps/chosen": -0.987946629524231, + "logps/rejected": -1.9638170003890991, + "loss": 1.0268, + "num_input_tokens_seen": 3876384, + "odds_ratio_loss": 9.981897354125977, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09879465401172638, + "rewards/margins": 0.09758704155683517, + "rewards/rejected": -0.19638168811798096, + "sft_loss": 0.028579894453287125, + "step": 2960 + }, + { + "epoch": 0.2310385064177363, + "grad_norm": 4.920382976531982, + "learning_rate": 4.369873772191092e-05, + "logits/chosen": -0.18340758979320526, + "logits/rejected": -1.0555959939956665, + "logps/chosen": -0.994560718536377, + "logps/rejected": -3.392359495162964, + "loss": 1.0257, + "num_input_tokens_seen": 3887008, + "odds_ratio_loss": 9.342180252075195, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.09945607930421829, + "rewards/margins": 0.23977987468242645, + "rewards/rejected": -0.33923593163490295, + "sft_loss": 0.09144171327352524, + "step": 2970 + }, + { + "epoch": 0.23181641384675222, + "grad_norm": 2.079035997390747, + "learning_rate": 4.365812864048407e-05, + "logits/chosen": 0.05672953277826309, + "logits/rejected": -1.5294697284698486, + "logps/chosen": -0.996350884437561, + "logps/rejected": -1.1568883657455444, + "loss": 1.0754, + "num_input_tokens_seen": 3902128, + "odds_ratio_loss": 10.721492767333984, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.09963510185480118, + "rewards/margins": 0.016053764149546623, + "rewards/rejected": -0.11568886041641235, + "sft_loss": 0.0032391403801739216, + "step": 2980 + }, + { + "epoch": 0.23259432127576818, + "grad_norm": 2.5082170963287354, + "learning_rate": 4.3617408123586913e-05, + "logits/chosen": 0.05996738746762276, + "logits/rejected": -1.517051100730896, + "logps/chosen": -1.1436173915863037, + "logps/rejected": -2.44610595703125, + "loss": 1.1817, + "num_input_tokens_seen": 3917712, + "odds_ratio_loss": 10.85283088684082, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.11436174064874649, + "rewards/margins": 0.13024888932704926, + "rewards/rejected": -0.24461062252521515, + "sft_loss": 0.09644796699285507, + "step": 2990 + }, + { + "epoch": 0.23337222870478413, + "grad_norm": 2.336026668548584, + "learning_rate": 4.357657641442229e-05, + "logits/chosen": -0.17219378054141998, + "logits/rejected": -1.2242201566696167, + "logps/chosen": -0.9584628939628601, + "logps/rejected": -1.6986106634140015, + "loss": 1.0064, + "num_input_tokens_seen": 3930480, + "odds_ratio_loss": 9.946453094482422, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.09584628790616989, + "rewards/margins": 0.07401478290557861, + "rewards/rejected": -0.1698610782623291, + "sft_loss": 0.011790582910180092, + "step": 3000 + }, + { + "epoch": 0.2341501361338001, + "grad_norm": 3.9625983238220215, + "learning_rate": 4.3535633756857136e-05, + "logits/chosen": -0.14156615734100342, + "logits/rejected": -0.9244439005851746, + "logps/chosen": -0.9330800771713257, + "logps/rejected": -1.97588312625885, + "loss": 0.9786, + "num_input_tokens_seen": 3939456, + "odds_ratio_loss": 9.251434326171875, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.09330800920724869, + "rewards/margins": 0.10428030788898468, + "rewards/rejected": -0.19758832454681396, + "sft_loss": 0.053438376635313034, + "step": 3010 + }, + { + "epoch": 0.23492804356281602, + "grad_norm": 4.78399658203125, + "learning_rate": 4.349458039542105e-05, + "logits/chosen": -0.0739276185631752, + "logits/rejected": -1.4249768257141113, + "logps/chosen": -1.1414638757705688, + "logps/rejected": -2.1657207012176514, + "loss": 1.1815, + "num_input_tokens_seen": 3948896, + "odds_ratio_loss": 11.685748100280762, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.114146389067173, + "rewards/margins": 0.10242567211389542, + "rewards/rejected": -0.21657204627990723, + "sft_loss": 0.012904743663966656, + "step": 3020 + }, + { + "epoch": 0.23570595099183198, + "grad_norm": 4.978982925415039, + "learning_rate": 4.345341657530476e-05, + "logits/chosen": -0.0407116636633873, + "logits/rejected": -1.3743739128112793, + "logps/chosen": -0.8405038714408875, + "logps/rejected": -2.2632803916931152, + "loss": 0.8875, + "num_input_tokens_seen": 3963056, + "odds_ratio_loss": 8.778888702392578, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.08405039459466934, + "rewards/margins": 0.14227768778800964, + "rewards/rejected": -0.2263280600309372, + "sft_loss": 0.009616486728191376, + "step": 3030 + }, + { + "epoch": 0.23648385842084793, + "grad_norm": 2.736375570297241, + "learning_rate": 4.3412142542358746e-05, + "logits/chosen": -0.09749384224414825, + "logits/rejected": -1.2443464994430542, + "logps/chosen": -0.9987101554870605, + "logps/rejected": -3.874497652053833, + "loss": 1.0336, + "num_input_tokens_seen": 3973632, + "odds_ratio_loss": 10.03593635559082, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.09987101703882217, + "rewards/margins": 0.287578821182251, + "rewards/rejected": -0.38744983077049255, + "sft_loss": 0.030052388086915016, + "step": 3040 + }, + { + "epoch": 0.23726176584986386, + "grad_norm": 1.4619851112365723, + "learning_rate": 4.3370758543091735e-05, + "logits/chosen": -0.030552173033356667, + "logits/rejected": -1.4240920543670654, + "logps/chosen": -0.9915860295295715, + "logps/rejected": -1.7709944248199463, + "loss": 1.0508, + "num_input_tokens_seen": 3983488, + "odds_ratio_loss": 10.395299911499023, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.0991586297750473, + "rewards/margins": 0.07794082909822464, + "rewards/rejected": -0.17709943652153015, + "sft_loss": 0.011277599260210991, + "step": 3050 + }, + { + "epoch": 0.23803967327887982, + "grad_norm": 4.666402816772461, + "learning_rate": 4.332926482466919e-05, + "logits/chosen": -0.033102355897426605, + "logits/rejected": -0.9652382731437683, + "logps/chosen": -0.8036381006240845, + "logps/rejected": -1.126878023147583, + "loss": 0.8586, + "num_input_tokens_seen": 3993264, + "odds_ratio_loss": 8.400969505310059, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.08036380261182785, + "rewards/margins": 0.03232399746775627, + "rewards/rejected": -0.11268781125545502, + "sft_loss": 0.018524659797549248, + "step": 3060 + }, + { + "epoch": 0.23881758070789577, + "grad_norm": 7.239573955535889, + "learning_rate": 4.32876616349119e-05, + "logits/chosen": -0.0428139865398407, + "logits/rejected": -1.262446641921997, + "logps/chosen": -1.015485405921936, + "logps/rejected": -1.9241611957550049, + "loss": 1.0565, + "num_input_tokens_seen": 4006208, + "odds_ratio_loss": 10.502401351928711, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.10154855251312256, + "rewards/margins": 0.09086756408214569, + "rewards/rejected": -0.19241611659526825, + "sft_loss": 0.006284940056502819, + "step": 3070 + }, + { + "epoch": 0.2395954881369117, + "grad_norm": 2.8917922973632812, + "learning_rate": 4.3245949222294466e-05, + "logits/chosen": -0.06223181635141373, + "logits/rejected": -1.25765061378479, + "logps/chosen": -0.9540655016899109, + "logps/rejected": -2.0520453453063965, + "loss": 1.0042, + "num_input_tokens_seen": 4019776, + "odds_ratio_loss": 9.981868743896484, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.09540656208992004, + "rewards/margins": 0.10979799926280975, + "rewards/rejected": -0.2052045613527298, + "sft_loss": 0.006059139966964722, + "step": 3080 + }, + { + "epoch": 0.24037339556592766, + "grad_norm": 3.6924211978912354, + "learning_rate": 4.320412783594381e-05, + "logits/chosen": 0.03607861325144768, + "logits/rejected": -1.6893059015274048, + "logps/chosen": -0.9528789520263672, + "logps/rejected": -1.5469565391540527, + "loss": 0.9954, + "num_input_tokens_seen": 4031968, + "odds_ratio_loss": 9.90943717956543, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09528790414333344, + "rewards/margins": 0.059407759457826614, + "rewards/rejected": -0.15469565987586975, + "sft_loss": 0.004419064149260521, + "step": 3090 + }, + { + "epoch": 0.2411513029949436, + "grad_norm": 2.1363964080810547, + "learning_rate": 4.316219772563772e-05, + "logits/chosen": -0.01864052750170231, + "logits/rejected": -1.3548976182937622, + "logps/chosen": -0.8682738542556763, + "logps/rejected": -1.7034642696380615, + "loss": 0.9118, + "num_input_tokens_seen": 4044592, + "odds_ratio_loss": 8.747833251953125, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.08682739734649658, + "rewards/margins": 0.08351902663707733, + "rewards/rejected": -0.1703464239835739, + "sft_loss": 0.037029556930065155, + "step": 3100 + }, + { + "epoch": 0.24192921042395954, + "grad_norm": 6.060778617858887, + "learning_rate": 4.312015914180332e-05, + "logits/chosen": -0.16518588364124298, + "logits/rejected": -0.9924762845039368, + "logps/chosen": -0.9183260798454285, + "logps/rejected": -3.4362235069274902, + "loss": 0.9622, + "num_input_tokens_seen": 4056096, + "odds_ratio_loss": 9.315229415893555, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.09183261543512344, + "rewards/margins": 0.25178977847099304, + "rewards/rejected": -0.3436223864555359, + "sft_loss": 0.030677739530801773, + "step": 3110 + }, + { + "epoch": 0.2427071178529755, + "grad_norm": 3.24570631980896, + "learning_rate": 4.307801233551558e-05, + "logits/chosen": -0.4022963047027588, + "logits/rejected": -0.7498866319656372, + "logps/chosen": -0.9533295631408691, + "logps/rejected": -2.0013861656188965, + "loss": 1.0044, + "num_input_tokens_seen": 4068272, + "odds_ratio_loss": 9.946795463562012, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.09533295035362244, + "rewards/margins": 0.10480568557977676, + "rewards/rejected": -0.2001386433839798, + "sft_loss": 0.009699802845716476, + "step": 3120 + }, + { + "epoch": 0.24348502528199145, + "grad_norm": 2.2562637329101562, + "learning_rate": 4.303575755849587e-05, + "logits/chosen": -0.20986226201057434, + "logits/rejected": -0.7615283727645874, + "logps/chosen": -1.099443793296814, + "logps/rejected": -2.7838354110717773, + "loss": 1.1389, + "num_input_tokens_seen": 4079664, + "odds_ratio_loss": 11.104766845703125, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.10994438081979752, + "rewards/margins": 0.16843917965888977, + "rewards/rejected": -0.2783835530281067, + "sft_loss": 0.028436914086341858, + "step": 3130 + }, + { + "epoch": 0.24426293271100738, + "grad_norm": 7.0951409339904785, + "learning_rate": 4.2993395063110365e-05, + "logits/chosen": 0.050548214465379715, + "logits/rejected": -1.5015652179718018, + "logps/chosen": -0.9225507974624634, + "logps/rejected": -2.1379103660583496, + "loss": 0.9769, + "num_input_tokens_seen": 4094448, + "odds_ratio_loss": 9.712604522705078, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.09225507825613022, + "rewards/margins": 0.12153597921133041, + "rewards/rejected": -0.21379104256629944, + "sft_loss": 0.005675177555531263, + "step": 3140 + }, + { + "epoch": 0.24504084014002334, + "grad_norm": 2.7226779460906982, + "learning_rate": 4.2950925102368605e-05, + "logits/chosen": -0.22441892325878143, + "logits/rejected": -0.9779049754142761, + "logps/chosen": -1.048175573348999, + "logps/rejected": -1.4846611022949219, + "loss": 1.1078, + "num_input_tokens_seen": 4108192, + "odds_ratio_loss": 10.27183723449707, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.10481754690408707, + "rewards/margins": 0.043648574501276016, + "rewards/rejected": -0.14846612513065338, + "sft_loss": 0.08061198890209198, + "step": 3150 + }, + { + "epoch": 0.2458187475690393, + "grad_norm": 1.8000941276550293, + "learning_rate": 4.290834792992198e-05, + "logits/chosen": 0.08007173985242844, + "logits/rejected": -1.388127326965332, + "logps/chosen": -0.8269979357719421, + "logps/rejected": -1.2204464673995972, + "loss": 0.8765, + "num_input_tokens_seen": 4127088, + "odds_ratio_loss": 8.728238105773926, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.08269979804754257, + "rewards/margins": 0.03934483975172043, + "rewards/rejected": -0.122044637799263, + "sft_loss": 0.0036289836280047894, + "step": 3160 + }, + { + "epoch": 0.24659665499805522, + "grad_norm": 1.884130835533142, + "learning_rate": 4.286566380006218e-05, + "logits/chosen": -0.21565194427967072, + "logits/rejected": -1.293933391571045, + "logps/chosen": -0.9503879547119141, + "logps/rejected": -1.7476370334625244, + "loss": 0.9898, + "num_input_tokens_seen": 4140960, + "odds_ratio_loss": 9.796585083007812, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.09503878653049469, + "rewards/margins": 0.07972491532564163, + "rewards/rejected": -0.17476370930671692, + "sft_loss": 0.010092451237142086, + "step": 3170 + }, + { + "epoch": 0.24737456242707118, + "grad_norm": 2.0614566802978516, + "learning_rate": 4.282287296771972e-05, + "logits/chosen": 0.061348915100097656, + "logits/rejected": -1.3977020978927612, + "logps/chosen": -0.9699851870536804, + "logps/rejected": -1.573901891708374, + "loss": 1.0286, + "num_input_tokens_seen": 4155904, + "odds_ratio_loss": 10.165769577026367, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.09699852019548416, + "rewards/margins": 0.06039166450500488, + "rewards/rejected": -0.15739016234874725, + "sft_loss": 0.012014877051115036, + "step": 3180 + }, + { + "epoch": 0.24815246985608713, + "grad_norm": 4.029314994812012, + "learning_rate": 4.277997568846237e-05, + "logits/chosen": 0.060758672654628754, + "logits/rejected": -1.4077775478363037, + "logps/chosen": -0.9179569482803345, + "logps/rejected": -1.417022466659546, + "loss": 0.9598, + "num_input_tokens_seen": 4174832, + "odds_ratio_loss": 9.570338249206543, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.09179569035768509, + "rewards/margins": 0.04990655183792114, + "rewards/rejected": -0.14170224964618683, + "sft_loss": 0.0027877301909029484, + "step": 3190 + }, + { + "epoch": 0.24893037728510306, + "grad_norm": 1.9069956541061401, + "learning_rate": 4.2736972218493664e-05, + "logits/chosen": -0.34009242057800293, + "logits/rejected": -0.9457812309265137, + "logps/chosen": -0.9835821390151978, + "logps/rejected": -3.5627548694610596, + "loss": 1.0203, + "num_input_tokens_seen": 4191232, + "odds_ratio_loss": 10.105257987976074, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.09835819900035858, + "rewards/margins": 0.25791728496551514, + "rewards/rejected": -0.3562754988670349, + "sft_loss": 0.009779800660908222, + "step": 3200 + }, + { + "epoch": 0.24970828471411902, + "grad_norm": 11.421212196350098, + "learning_rate": 4.269386281465136e-05, + "logits/chosen": -0.2749151587486267, + "logits/rejected": -0.9223438501358032, + "logps/chosen": -1.0342686176300049, + "logps/rejected": -1.3730922937393188, + "loss": 1.0954, + "num_input_tokens_seen": 4204640, + "odds_ratio_loss": 10.71926212310791, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.10342685878276825, + "rewards/margins": 0.033882349729537964, + "rewards/rejected": -0.1373092234134674, + "sft_loss": 0.023464282974600792, + "step": 3210 + }, + { + "epoch": 0.25048619214313494, + "grad_norm": 2.6410937309265137, + "learning_rate": 4.2650647734405914e-05, + "logits/chosen": 0.08165550976991653, + "logits/rejected": -1.680293083190918, + "logps/chosen": -0.9836196899414062, + "logps/rejected": -2.458129644393921, + "loss": 1.0289, + "num_input_tokens_seen": 4220752, + "odds_ratio_loss": 10.125057220458984, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.09836197644472122, + "rewards/margins": 0.1474509984254837, + "rewards/rejected": -0.24581298232078552, + "sft_loss": 0.016405515372753143, + "step": 3220 + }, + { + "epoch": 0.2512640995721509, + "grad_norm": 1.7691044807434082, + "learning_rate": 4.2607327235858894e-05, + "logits/chosen": -0.3390534520149231, + "logits/rejected": -1.4492483139038086, + "logps/chosen": -0.9570733308792114, + "logps/rejected": -2.46934175491333, + "loss": 0.9947, + "num_input_tokens_seen": 4233920, + "odds_ratio_loss": 9.843822479248047, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.09570731967687607, + "rewards/margins": 0.15122681856155396, + "rewards/rejected": -0.24693414568901062, + "sft_loss": 0.010312440805137157, + "step": 3230 + }, + { + "epoch": 0.25204200700116686, + "grad_norm": 4.76840353012085, + "learning_rate": 4.256390157774154e-05, + "logits/chosen": 0.003717652056366205, + "logits/rejected": -1.1586898565292358, + "logps/chosen": -1.0405880212783813, + "logps/rejected": -2.021322250366211, + "loss": 1.0714, + "num_input_tokens_seen": 4243920, + "odds_ratio_loss": 10.504170417785645, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.10405881702899933, + "rewards/margins": 0.09807340800762177, + "rewards/rejected": -0.2021322250366211, + "sft_loss": 0.02094918303191662, + "step": 3240 + }, + { + "epoch": 0.2528199144301828, + "grad_norm": 3.3458433151245117, + "learning_rate": 4.2520371019413095e-05, + "logits/chosen": 0.016684969887137413, + "logits/rejected": -1.4637718200683594, + "logps/chosen": -0.9979241490364075, + "logps/rejected": -1.426872730255127, + "loss": 1.0548, + "num_input_tokens_seen": 4256112, + "odds_ratio_loss": 10.42685317993164, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.09979242086410522, + "rewards/margins": 0.04289485886693001, + "rewards/rejected": -0.14268726110458374, + "sft_loss": 0.012084713205695152, + "step": 3250 + }, + { + "epoch": 0.25359782185919877, + "grad_norm": 3.482081174850464, + "learning_rate": 4.2476735820859374e-05, + "logits/chosen": 0.01486960332840681, + "logits/rejected": -1.3319334983825684, + "logps/chosen": -1.0822274684906006, + "logps/rejected": -1.4299288988113403, + "loss": 1.1476, + "num_input_tokens_seen": 4270368, + "odds_ratio_loss": 11.378705978393555, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.10822276026010513, + "rewards/margins": 0.034770142287015915, + "rewards/rejected": -0.14299288392066956, + "sft_loss": 0.00969963613897562, + "step": 3260 + }, + { + "epoch": 0.2543757292882147, + "grad_norm": 5.3337721824646, + "learning_rate": 4.2432996242691095e-05, + "logits/chosen": -0.19184128940105438, + "logits/rejected": -1.325873851776123, + "logps/chosen": -0.8774449229240417, + "logps/rejected": -1.518665075302124, + "loss": 0.9217, + "num_input_tokens_seen": 4283568, + "odds_ratio_loss": 9.148040771484375, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.08774449676275253, + "rewards/margins": 0.06412201374769211, + "rewards/rejected": -0.15186652541160583, + "sft_loss": 0.006918758153915405, + "step": 3270 + }, + { + "epoch": 0.2551536367172306, + "grad_norm": 6.184454441070557, + "learning_rate": 4.238915254614243e-05, + "logits/chosen": -0.2082027643918991, + "logits/rejected": -1.2313646078109741, + "logps/chosen": -1.2307305335998535, + "logps/rejected": -1.9771459102630615, + "loss": 1.2814, + "num_input_tokens_seen": 4293056, + "odds_ratio_loss": 12.01191520690918, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.1230730414390564, + "rewards/margins": 0.07464151829481125, + "rewards/rejected": -0.19771459698677063, + "sft_loss": 0.0802154690027237, + "step": 3280 + }, + { + "epoch": 0.2559315441462466, + "grad_norm": 1.5774378776550293, + "learning_rate": 4.2345204993069376e-05, + "logits/chosen": -0.09383687376976013, + "logits/rejected": -1.0173933506011963, + "logps/chosen": -0.8213062286376953, + "logps/rejected": -1.9807208776474, + "loss": 0.8583, + "num_input_tokens_seen": 4307728, + "odds_ratio_loss": 8.337804794311523, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.08213062584400177, + "rewards/margins": 0.11594147980213165, + "rewards/rejected": -0.19807210564613342, + "sft_loss": 0.02448856830596924, + "step": 3290 + }, + { + "epoch": 0.25670945157526254, + "grad_norm": 2.8531248569488525, + "learning_rate": 4.230115384594821e-05, + "logits/chosen": 0.13179947435855865, + "logits/rejected": -1.6411634683609009, + "logps/chosen": -0.7820327281951904, + "logps/rejected": -2.090890884399414, + "loss": 0.8122, + "num_input_tokens_seen": 4320736, + "odds_ratio_loss": 8.024853706359863, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.07820327579975128, + "rewards/margins": 0.13088582456111908, + "rewards/rejected": -0.20908911526203156, + "sft_loss": 0.009706547483801842, + "step": 3300 + }, + { + "epoch": 0.2574873590042785, + "grad_norm": 4.10869026184082, + "learning_rate": 4.225699936787394e-05, + "logits/chosen": -0.10895247757434845, + "logits/rejected": -1.4337221384048462, + "logps/chosen": -0.8080259561538696, + "logps/rejected": -2.6543774604797363, + "loss": 0.8513, + "num_input_tokens_seen": 4332432, + "odds_ratio_loss": 8.121633529663086, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.08080259710550308, + "rewards/margins": 0.18463513255119324, + "rewards/rejected": -0.2654377222061157, + "sft_loss": 0.039158955216407776, + "step": 3310 + }, + { + "epoch": 0.25826526643329445, + "grad_norm": 2.5322458744049072, + "learning_rate": 4.22127418225587e-05, + "logits/chosen": 0.08826179057359695, + "logits/rejected": -1.37474524974823, + "logps/chosen": -0.9683723449707031, + "logps/rejected": -1.6476446390151978, + "loss": 1.0142, + "num_input_tokens_seen": 4348576, + "odds_ratio_loss": 10.062582015991211, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.09683723747730255, + "rewards/margins": 0.06792722642421722, + "rewards/rejected": -0.16476444900035858, + "sft_loss": 0.007978400215506554, + "step": 3320 + }, + { + "epoch": 0.2590431738623104, + "grad_norm": 6.394929885864258, + "learning_rate": 4.21683814743302e-05, + "logits/chosen": -0.4122388958930969, + "logits/rejected": -1.254586935043335, + "logps/chosen": -0.7898470759391785, + "logps/rejected": -1.3862065076828003, + "loss": 0.8378, + "num_input_tokens_seen": 4361584, + "odds_ratio_loss": 8.037008285522461, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.07898470759391785, + "rewards/margins": 0.0596359483897686, + "rewards/rejected": -0.13862064480781555, + "sft_loss": 0.03409110754728317, + "step": 3330 + }, + { + "epoch": 0.2598210812913263, + "grad_norm": 3.825871706008911, + "learning_rate": 4.212391858813013e-05, + "logits/chosen": -0.12090665102005005, + "logits/rejected": -1.2372742891311646, + "logps/chosen": -0.8906979560852051, + "logps/rejected": -3.0938544273376465, + "loss": 0.9238, + "num_input_tokens_seen": 4372672, + "odds_ratio_loss": 9.202934265136719, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.08906979858875275, + "rewards/margins": 0.22031569480895996, + "rewards/rejected": -0.3093855082988739, + "sft_loss": 0.0035380299668759108, + "step": 3340 + }, + { + "epoch": 0.26059898872034226, + "grad_norm": 2.6060073375701904, + "learning_rate": 4.207935342951261e-05, + "logits/chosen": -0.35914385318756104, + "logits/rejected": -1.3731544017791748, + "logps/chosen": -1.066075325012207, + "logps/rejected": -1.582295536994934, + "loss": 1.11, + "num_input_tokens_seen": 4382704, + "odds_ratio_loss": 11.008880615234375, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.10660751909017563, + "rewards/margins": 0.05162200331687927, + "rewards/rejected": -0.15822955965995789, + "sft_loss": 0.009122246876358986, + "step": 3350 + }, + { + "epoch": 0.2613768961493582, + "grad_norm": 4.739326477050781, + "learning_rate": 4.203468626464254e-05, + "logits/chosen": -0.03921103477478027, + "logits/rejected": -1.5998603105545044, + "logps/chosen": -0.8556955456733704, + "logps/rejected": -1.8652175664901733, + "loss": 0.8993, + "num_input_tokens_seen": 4394320, + "odds_ratio_loss": 8.893582344055176, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.08556955307722092, + "rewards/margins": 0.10095222294330597, + "rewards/rejected": -0.18652178347110748, + "sft_loss": 0.009982654824852943, + "step": 3360 + }, + { + "epoch": 0.2621548035783742, + "grad_norm": 6.477309226989746, + "learning_rate": 4.1989917360294106e-05, + "logits/chosen": -0.5154345035552979, + "logits/rejected": -0.8112955093383789, + "logps/chosen": -1.1205177307128906, + "logps/rejected": -2.7822554111480713, + "loss": 1.1625, + "num_input_tokens_seen": 4401664, + "odds_ratio_loss": 11.260440826416016, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.11205177009105682, + "rewards/margins": 0.1661737859249115, + "rewards/rejected": -0.27822554111480713, + "sft_loss": 0.03643777221441269, + "step": 3370 + }, + { + "epoch": 0.26293271100739013, + "grad_norm": 4.272067546844482, + "learning_rate": 4.194504698384908e-05, + "logits/chosen": -0.19038191437721252, + "logits/rejected": -1.9764163494110107, + "logps/chosen": -1.1597676277160645, + "logps/rejected": -3.2375998497009277, + "loss": 1.184, + "num_input_tokens_seen": 4413920, + "odds_ratio_loss": 11.342437744140625, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.11597676575183868, + "rewards/margins": 0.20778322219848633, + "rewards/rejected": -0.3237599730491638, + "sft_loss": 0.04977894946932793, + "step": 3380 + }, + { + "epoch": 0.2637106184364061, + "grad_norm": 7.2915191650390625, + "learning_rate": 4.1900075403295304e-05, + "logits/chosen": -0.2795480191707611, + "logits/rejected": -0.964443027973175, + "logps/chosen": -0.9262790679931641, + "logps/rejected": -1.7858781814575195, + "loss": 0.9663, + "num_input_tokens_seen": 4424784, + "odds_ratio_loss": 9.245964050292969, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.09262792021036148, + "rewards/margins": 0.08595990389585495, + "rewards/rejected": -0.17858782410621643, + "sft_loss": 0.041728947311639786, + "step": 3390 + }, + { + "epoch": 0.26448852586542204, + "grad_norm": 0.9598113894462585, + "learning_rate": 4.185500288722505e-05, + "logits/chosen": -0.24663646519184113, + "logits/rejected": -1.0148065090179443, + "logps/chosen": -0.7349613904953003, + "logps/rejected": -1.3731105327606201, + "loss": 0.7776, + "num_input_tokens_seen": 4433632, + "odds_ratio_loss": 7.688754081726074, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.07349614799022675, + "rewards/margins": 0.0638149231672287, + "rewards/rejected": -0.13731107115745544, + "sft_loss": 0.00868178904056549, + "step": 3400 + }, + { + "epoch": 0.26526643329443794, + "grad_norm": 1.427416443824768, + "learning_rate": 4.180982970483344e-05, + "logits/chosen": -0.0015018790727481246, + "logits/rejected": -1.599158763885498, + "logps/chosen": -0.9013495445251465, + "logps/rejected": -2.0213136672973633, + "loss": 0.9434, + "num_input_tokens_seen": 4446192, + "odds_ratio_loss": 9.38914680480957, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.09013496339321136, + "rewards/margins": 0.11199643462896347, + "rewards/rejected": -0.20213139057159424, + "sft_loss": 0.004515963606536388, + "step": 3410 + }, + { + "epoch": 0.2660443407234539, + "grad_norm": 5.408295154571533, + "learning_rate": 4.176455612591681e-05, + "logits/chosen": -0.4608125686645508, + "logits/rejected": -0.97273188829422, + "logps/chosen": -0.7134547233581543, + "logps/rejected": -1.6708390712738037, + "loss": 0.7579, + "num_input_tokens_seen": 4456784, + "odds_ratio_loss": 7.530245304107666, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.07134547084569931, + "rewards/margins": 0.09573844820261002, + "rewards/rejected": -0.16708391904830933, + "sft_loss": 0.0048301443457603455, + "step": 3420 + }, + { + "epoch": 0.26682224815246985, + "grad_norm": 5.093200206756592, + "learning_rate": 4.1719182420871104e-05, + "logits/chosen": -0.10298941284418106, + "logits/rejected": -1.2602466344833374, + "logps/chosen": -0.9793552160263062, + "logps/rejected": -1.680318832397461, + "loss": 1.0241, + "num_input_tokens_seen": 4468704, + "odds_ratio_loss": 10.133760452270508, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.09793553501367569, + "rewards/margins": 0.07009634375572205, + "rewards/rejected": -0.16803188621997833, + "sft_loss": 0.010695156641304493, + "step": 3430 + }, + { + "epoch": 0.2676001555814858, + "grad_norm": 4.929412364959717, + "learning_rate": 4.16737088606903e-05, + "logits/chosen": -0.06423575431108475, + "logits/rejected": -1.2866432666778564, + "logps/chosen": -0.8811386227607727, + "logps/rejected": -1.5141222476959229, + "loss": 0.9317, + "num_input_tokens_seen": 4480144, + "odds_ratio_loss": 8.985457420349121, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.08811385929584503, + "rewards/margins": 0.06329836696386337, + "rewards/rejected": -0.151412233710289, + "sft_loss": 0.03313466161489487, + "step": 3440 + }, + { + "epoch": 0.26837806301050177, + "grad_norm": 2.390888214111328, + "learning_rate": 4.1628135716964734e-05, + "logits/chosen": -0.019254203885793686, + "logits/rejected": -1.3756763935089111, + "logps/chosen": -0.9164663553237915, + "logps/rejected": -2.7706151008605957, + "loss": 0.9583, + "num_input_tokens_seen": 4493504, + "odds_ratio_loss": 8.82702350616455, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.09164663404226303, + "rewards/margins": 0.1854148954153061, + "rewards/rejected": -0.2770615518093109, + "sft_loss": 0.07561619579792023, + "step": 3450 + }, + { + "epoch": 0.2691559704395177, + "grad_norm": 2.5836713314056396, + "learning_rate": 4.158246326187951e-05, + "logits/chosen": -0.09574378281831741, + "logits/rejected": -1.3053169250488281, + "logps/chosen": -0.9305857419967651, + "logps/rejected": -2.1507930755615234, + "loss": 0.9748, + "num_input_tokens_seen": 4507200, + "odds_ratio_loss": 9.583921432495117, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.09305857867002487, + "rewards/margins": 0.12202074378728867, + "rewards/rejected": -0.21507930755615234, + "sft_loss": 0.01643938571214676, + "step": 3460 + }, + { + "epoch": 0.2699338778685336, + "grad_norm": 2.783560276031494, + "learning_rate": 4.153669176821288e-05, + "logits/chosen": -0.2140583097934723, + "logits/rejected": -0.7443190813064575, + "logps/chosen": -1.1720969676971436, + "logps/rejected": -2.1571438312530518, + "loss": 1.2102, + "num_input_tokens_seen": 4519392, + "odds_ratio_loss": 11.70689868927002, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.11720969527959824, + "rewards/margins": 0.0985046997666359, + "rewards/rejected": -0.21571438014507294, + "sft_loss": 0.03948521986603737, + "step": 3470 + }, + { + "epoch": 0.2707117852975496, + "grad_norm": 12.581844329833984, + "learning_rate": 4.149082150933456e-05, + "logits/chosen": -0.15463215112686157, + "logits/rejected": -1.3547630310058594, + "logps/chosen": -1.0013000965118408, + "logps/rejected": -2.429577589035034, + "loss": 1.0537, + "num_input_tokens_seen": 4531984, + "odds_ratio_loss": 10.430364608764648, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.10013000667095184, + "rewards/margins": 0.14282776415348053, + "rewards/rejected": -0.24295778572559357, + "sft_loss": 0.01061457023024559, + "step": 3480 + }, + { + "epoch": 0.27148969272656553, + "grad_norm": 4.4241533279418945, + "learning_rate": 4.144485275920422e-05, + "logits/chosen": -0.17204315960407257, + "logits/rejected": -0.9298747777938843, + "logps/chosen": -1.1662946939468384, + "logps/rejected": -1.887099027633667, + "loss": 1.2139, + "num_input_tokens_seen": 4547792, + "odds_ratio_loss": 11.961484909057617, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.11662948131561279, + "rewards/margins": 0.07208044826984406, + "rewards/rejected": -0.18870989978313446, + "sft_loss": 0.017767272889614105, + "step": 3490 + }, + { + "epoch": 0.2722676001555815, + "grad_norm": 3.9844746589660645, + "learning_rate": 4.1398785792369676e-05, + "logits/chosen": -0.10230766236782074, + "logits/rejected": -1.183638334274292, + "logps/chosen": -1.080854892730713, + "logps/rejected": -3.9099273681640625, + "loss": 1.1075, + "num_input_tokens_seen": 4560016, + "odds_ratio_loss": 10.963944435119629, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.108085498213768, + "rewards/margins": 0.28290724754333496, + "rewards/rejected": -0.39099276065826416, + "sft_loss": 0.011096212081611156, + "step": 3500 + }, + { + "epoch": 0.27304550758459745, + "grad_norm": 3.405622959136963, + "learning_rate": 4.135262088396542e-05, + "logits/chosen": -0.11642368882894516, + "logits/rejected": -1.573893427848816, + "logps/chosen": -1.002684235572815, + "logps/rejected": -2.7077748775482178, + "loss": 1.0274, + "num_input_tokens_seen": 4573536, + "odds_ratio_loss": 9.916679382324219, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1002684235572815, + "rewards/margins": 0.17050907015800476, + "rewards/rejected": -0.27077749371528625, + "sft_loss": 0.035697899758815765, + "step": 3510 + }, + { + "epoch": 0.2738234150136134, + "grad_norm": 1.425681710243225, + "learning_rate": 4.130635830971084e-05, + "logits/chosen": -0.13541904091835022, + "logits/rejected": -1.7754377126693726, + "logps/chosen": -0.8972693681716919, + "logps/rejected": -2.1450066566467285, + "loss": 0.931, + "num_input_tokens_seen": 4592768, + "odds_ratio_loss": 9.266926765441895, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.08972693979740143, + "rewards/margins": 0.12477374076843262, + "rewards/rejected": -0.21450069546699524, + "sft_loss": 0.004262088797986507, + "step": 3520 + }, + { + "epoch": 0.2746013224426293, + "grad_norm": 2.133974075317383, + "learning_rate": 4.125999834590867e-05, + "logits/chosen": -0.5876361131668091, + "logits/rejected": -1.141478180885315, + "logps/chosen": -1.6601444482803345, + "logps/rejected": -1.2758491039276123, + "loss": 1.7911, + "num_input_tokens_seen": 4602480, + "odds_ratio_loss": 16.37868881225586, + "rewards/accuracies": 0.4000000059604645, + "rewards/chosen": -0.16601444780826569, + "rewards/margins": -0.03842953220009804, + "rewards/rejected": -0.12758490443229675, + "sft_loss": 0.15324907004833221, + "step": 3530 + }, + { + "epoch": 0.27537922987164526, + "grad_norm": 4.674381256103516, + "learning_rate": 4.121354126944329e-05, + "logits/chosen": -0.5263653993606567, + "logits/rejected": -1.006824254989624, + "logps/chosen": -1.0107089281082153, + "logps/rejected": -1.957911729812622, + "loss": 1.0487, + "num_input_tokens_seen": 4614832, + "odds_ratio_loss": 10.238679885864258, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.10107089579105377, + "rewards/margins": 0.0947202816605568, + "rewards/rejected": -0.19579115509986877, + "sft_loss": 0.024857979267835617, + "step": 3540 + }, + { + "epoch": 0.2761571373006612, + "grad_norm": 8.11523723602295, + "learning_rate": 4.116698735777907e-05, + "logits/chosen": -0.3436376452445984, + "logits/rejected": -1.0312341451644897, + "logps/chosen": -0.7612732648849487, + "logps/rejected": -2.2395224571228027, + "loss": 0.8017, + "num_input_tokens_seen": 4626304, + "odds_ratio_loss": 7.937858581542969, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.07612732797861099, + "rewards/margins": 0.14782488346099854, + "rewards/rejected": -0.2239522486925125, + "sft_loss": 0.007900206372141838, + "step": 3550 + }, + { + "epoch": 0.27693504472967717, + "grad_norm": 2.475008010864258, + "learning_rate": 4.1120336888958766e-05, + "logits/chosen": -0.2083013951778412, + "logits/rejected": -1.0854121446609497, + "logps/chosen": -0.9013233184814453, + "logps/rejected": -2.847856044769287, + "loss": 0.9514, + "num_input_tokens_seen": 4638624, + "odds_ratio_loss": 9.257806777954102, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.09013234078884125, + "rewards/margins": 0.19465331733226776, + "rewards/rejected": -0.284785658121109, + "sft_loss": 0.025621671229600906, + "step": 3560 + }, + { + "epoch": 0.2777129521586931, + "grad_norm": 8.027059555053711, + "learning_rate": 4.107359014160177e-05, + "logits/chosen": -0.2505590319633484, + "logits/rejected": -1.458212971687317, + "logps/chosen": -1.0533485412597656, + "logps/rejected": -2.902940511703491, + "loss": 1.0792, + "num_input_tokens_seen": 4647168, + "odds_ratio_loss": 10.279277801513672, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.10533484071493149, + "rewards/margins": 0.18495921790599823, + "rewards/rejected": -0.2902940809726715, + "sft_loss": 0.05122938007116318, + "step": 3570 + }, + { + "epoch": 0.2784908595877091, + "grad_norm": 1.9160383939743042, + "learning_rate": 4.102674739490254e-05, + "logits/chosen": -0.05519244819879532, + "logits/rejected": -1.2347227334976196, + "logps/chosen": -0.9562760591506958, + "logps/rejected": -2.191115379333496, + "loss": 1.0222, + "num_input_tokens_seen": 4659312, + "odds_ratio_loss": 9.945977210998535, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.09562759846448898, + "rewards/margins": 0.12348394095897675, + "rewards/rejected": -0.21911153197288513, + "sft_loss": 0.027616357430815697, + "step": 3580 + }, + { + "epoch": 0.279268767016725, + "grad_norm": 2.969160556793213, + "learning_rate": 4.097980892862886e-05, + "logits/chosen": -0.13975301384925842, + "logits/rejected": -0.8550666570663452, + "logps/chosen": -0.8890474438667297, + "logps/rejected": -1.838004469871521, + "loss": 0.923, + "num_input_tokens_seen": 4674992, + "odds_ratio_loss": 9.136037826538086, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.0889047384262085, + "rewards/margins": 0.09489572048187256, + "rewards/rejected": -0.18380047380924225, + "sft_loss": 0.009357091039419174, + "step": 3590 + }, + { + "epoch": 0.28004667444574094, + "grad_norm": 2.5685956478118896, + "learning_rate": 4.093277502312022e-05, + "logits/chosen": -0.6892591118812561, + "logits/rejected": -0.5192093849182129, + "logps/chosen": -1.035064458847046, + "logps/rejected": -2.2198333740234375, + "loss": 1.0698, + "num_input_tokens_seen": 4684448, + "odds_ratio_loss": 9.483429908752441, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.10350646078586578, + "rewards/margins": 0.11847686767578125, + "rewards/rejected": -0.22198334336280823, + "sft_loss": 0.12144489586353302, + "step": 3600 + }, + { + "epoch": 0.2808245818747569, + "grad_norm": 2.3054659366607666, + "learning_rate": 4.088564595928609e-05, + "logits/chosen": -0.18362614512443542, + "logits/rejected": -0.8535488247871399, + "logps/chosen": -1.262632131576538, + "logps/rejected": -2.090200424194336, + "loss": 1.3427, + "num_input_tokens_seen": 4695472, + "odds_ratio_loss": 12.58495044708252, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.12626323103904724, + "rewards/margins": 0.08275681734085083, + "rewards/rejected": -0.20902001857757568, + "sft_loss": 0.0841931700706482, + "step": 3610 + }, + { + "epoch": 0.28160248930377285, + "grad_norm": 3.6128058433532715, + "learning_rate": 4.083842201860431e-05, + "logits/chosen": -0.11260410398244858, + "logits/rejected": -0.9792426228523254, + "logps/chosen": -1.070449948310852, + "logps/rejected": -2.4821507930755615, + "loss": 1.1047, + "num_input_tokens_seen": 4708896, + "odds_ratio_loss": 10.800509452819824, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.1070449948310852, + "rewards/margins": 0.14117011427879333, + "rewards/rejected": -0.24821512401103973, + "sft_loss": 0.02461850829422474, + "step": 3620 + }, + { + "epoch": 0.2823803967327888, + "grad_norm": 2.64975643157959, + "learning_rate": 4.079110348311934e-05, + "logits/chosen": 0.1377231627702713, + "logits/rejected": -1.7623564004898071, + "logps/chosen": -0.8980050086975098, + "logps/rejected": -1.417351484298706, + "loss": 0.9384, + "num_input_tokens_seen": 4723376, + "odds_ratio_loss": 9.318737030029297, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.08980049937963486, + "rewards/margins": 0.05193463712930679, + "rewards/rejected": -0.14173513650894165, + "sft_loss": 0.006572979036718607, + "step": 3630 + }, + { + "epoch": 0.28315830416180476, + "grad_norm": 6.5229411125183105, + "learning_rate": 4.0743690635440634e-05, + "logits/chosen": 0.1733946055173874, + "logits/rejected": -1.2729114294052124, + "logps/chosen": -0.9009236097335815, + "logps/rejected": -2.0939249992370605, + "loss": 0.9396, + "num_input_tokens_seen": 4734944, + "odds_ratio_loss": 9.328462600708008, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.09009236097335815, + "rewards/margins": 0.11930011212825775, + "rewards/rejected": -0.2093924731016159, + "sft_loss": 0.0067521268501877785, + "step": 3640 + }, + { + "epoch": 0.2839362115908207, + "grad_norm": 2.036282539367676, + "learning_rate": 4.0696183758740894e-05, + "logits/chosen": 0.10636024177074432, + "logits/rejected": -1.588081955909729, + "logps/chosen": -0.9661546945571899, + "logps/rejected": -2.7496097087860107, + "loss": 0.9965, + "num_input_tokens_seen": 4748816, + "odds_ratio_loss": 9.889032363891602, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09661546349525452, + "rewards/margins": 0.17834553122520447, + "rewards/rejected": -0.274960994720459, + "sft_loss": 0.0075736879371106625, + "step": 3650 + }, + { + "epoch": 0.2847141190198366, + "grad_norm": 4.932629585266113, + "learning_rate": 4.064858313675443e-05, + "logits/chosen": -0.26715725660324097, + "logits/rejected": -0.7281715273857117, + "logps/chosen": -1.0199708938598633, + "logps/rejected": -2.7163097858428955, + "loss": 1.0673, + "num_input_tokens_seen": 4762544, + "odds_ratio_loss": 10.461103439331055, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.10199709981679916, + "rewards/margins": 0.16963385045528412, + "rewards/rejected": -0.2716309726238251, + "sft_loss": 0.02118172124028206, + "step": 3660 + }, + { + "epoch": 0.2854920264488526, + "grad_norm": 7.332515239715576, + "learning_rate": 4.060088905377543e-05, + "logits/chosen": -0.023127097636461258, + "logits/rejected": -1.021946668624878, + "logps/chosen": -1.0226608514785767, + "logps/rejected": -2.9361438751220703, + "loss": 1.0629, + "num_input_tokens_seen": 4778464, + "odds_ratio_loss": 10.557701110839844, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.10226607322692871, + "rewards/margins": 0.19134829938411713, + "rewards/rejected": -0.29361438751220703, + "sft_loss": 0.0071670361794531345, + "step": 3670 + }, + { + "epoch": 0.28626993387786853, + "grad_norm": 9.548507690429688, + "learning_rate": 4.05531017946563e-05, + "logits/chosen": -0.19037041068077087, + "logits/rejected": -0.8707745671272278, + "logps/chosen": -0.9563606381416321, + "logps/rejected": -3.273907423019409, + "loss": 0.9816, + "num_input_tokens_seen": 4790656, + "odds_ratio_loss": 9.66557502746582, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.09563605487346649, + "rewards/margins": 0.23175469040870667, + "rewards/rejected": -0.32739073038101196, + "sft_loss": 0.0150833148509264, + "step": 3680 + }, + { + "epoch": 0.2870478413068845, + "grad_norm": 2.4525821208953857, + "learning_rate": 4.0505221644805915e-05, + "logits/chosen": -0.003124893642961979, + "logits/rejected": -1.4493802785873413, + "logps/chosen": -0.9687921404838562, + "logps/rejected": -4.383993148803711, + "loss": 1.011, + "num_input_tokens_seen": 4803440, + "odds_ratio_loss": 9.884014129638672, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.09687922894954681, + "rewards/margins": 0.3415201008319855, + "rewards/rejected": -0.43839937448501587, + "sft_loss": 0.022603165358304977, + "step": 3690 + }, + { + "epoch": 0.28782574873590044, + "grad_norm": 5.859448432922363, + "learning_rate": 4.045724889018795e-05, + "logits/chosen": -0.5072391629219055, + "logits/rejected": -0.9983925819396973, + "logps/chosen": -0.9392580986022949, + "logps/rejected": -1.865422010421753, + "loss": 0.9823, + "num_input_tokens_seen": 4817680, + "odds_ratio_loss": 9.259554862976074, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.09392580389976501, + "rewards/margins": 0.09261641651391983, + "rewards/rejected": -0.18654224276542664, + "sft_loss": 0.05638272687792778, + "step": 3700 + }, + { + "epoch": 0.2886036561649164, + "grad_norm": 2.2526004314422607, + "learning_rate": 4.040918381731916e-05, + "logits/chosen": -0.21405701339244843, + "logits/rejected": -1.030743956565857, + "logps/chosen": -1.2350473403930664, + "logps/rejected": -1.3367514610290527, + "loss": 1.3073, + "num_input_tokens_seen": 4830976, + "odds_ratio_loss": 12.2762451171875, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.12350473552942276, + "rewards/margins": 0.010170431807637215, + "rewards/rejected": -0.13367517292499542, + "sft_loss": 0.07963992655277252, + "step": 3710 + }, + { + "epoch": 0.2893815635939323, + "grad_norm": 10.45372486114502, + "learning_rate": 4.036102671326768e-05, + "logits/chosen": -0.540435791015625, + "logits/rejected": -0.8520223498344421, + "logps/chosen": -0.8511611819267273, + "logps/rejected": -1.6462376117706299, + "loss": 0.8937, + "num_input_tokens_seen": 4839504, + "odds_ratio_loss": 8.561546325683594, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.08511612564325333, + "rewards/margins": 0.07950763404369354, + "rewards/rejected": -0.16462375223636627, + "sft_loss": 0.03758342191576958, + "step": 3720 + }, + { + "epoch": 0.29015947102294826, + "grad_norm": 28.474945068359375, + "learning_rate": 4.031277786565128e-05, + "logits/chosen": 0.027608368545770645, + "logits/rejected": -1.2352272272109985, + "logps/chosen": -0.9236048460006714, + "logps/rejected": -1.9401235580444336, + "loss": 0.971, + "num_input_tokens_seen": 4851552, + "odds_ratio_loss": 9.641461372375488, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.0923604816198349, + "rewards/margins": 0.1016518846154213, + "rewards/rejected": -0.1940123587846756, + "sft_loss": 0.006871473044157028, + "step": 3730 + }, + { + "epoch": 0.2909373784519642, + "grad_norm": 5.415572166442871, + "learning_rate": 4.0264437562635674e-05, + "logits/chosen": -0.164834126830101, + "logits/rejected": -1.1962740421295166, + "logps/chosen": -0.8205951452255249, + "logps/rejected": -2.995657444000244, + "loss": 0.8648, + "num_input_tokens_seen": 4861456, + "odds_ratio_loss": 8.583596229553223, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.08205951750278473, + "rewards/margins": 0.21750621497631073, + "rewards/rejected": -0.29956573247909546, + "sft_loss": 0.006485482212156057, + "step": 3740 + }, + { + "epoch": 0.29171528588098017, + "grad_norm": 3.854557514190674, + "learning_rate": 4.0216006092932815e-05, + "logits/chosen": -0.38443735241889954, + "logits/rejected": -0.8124167323112488, + "logps/chosen": -0.7476634979248047, + "logps/rejected": -1.6348378658294678, + "loss": 0.7778, + "num_input_tokens_seen": 4872080, + "odds_ratio_loss": 7.580167293548584, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.07476634532213211, + "rewards/margins": 0.08871743828058243, + "rewards/rejected": -0.16348378360271454, + "sft_loss": 0.019787630066275597, + "step": 3750 + }, + { + "epoch": 0.2924931933099961, + "grad_norm": 3.8585402965545654, + "learning_rate": 4.01674837457991e-05, + "logits/chosen": -0.4993743896484375, + "logits/rejected": -1.3653934001922607, + "logps/chosen": -0.9110993146896362, + "logps/rejected": -1.640768051147461, + "loss": 0.9672, + "num_input_tokens_seen": 4881072, + "odds_ratio_loss": 9.55540943145752, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.09110994637012482, + "rewards/margins": 0.07296689599752426, + "rewards/rejected": -0.1640768200159073, + "sft_loss": 0.01162740308791399, + "step": 3760 + }, + { + "epoch": 0.2932711007390121, + "grad_norm": 2.339550256729126, + "learning_rate": 4.011887081103374e-05, + "logits/chosen": 0.10720139741897583, + "logits/rejected": -1.6887003183364868, + "logps/chosen": -0.9534517526626587, + "logps/rejected": -2.0855872631073, + "loss": 0.9834, + "num_input_tokens_seen": 4891296, + "odds_ratio_loss": 9.790858268737793, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.0953451618552208, + "rewards/margins": 0.11321356147527695, + "rewards/rejected": -0.20855875313282013, + "sft_loss": 0.004304885398596525, + "step": 3770 + }, + { + "epoch": 0.294049008168028, + "grad_norm": 2.4522814750671387, + "learning_rate": 4.007016757897696e-05, + "logits/chosen": -0.20804552733898163, + "logits/rejected": -0.932209849357605, + "logps/chosen": -0.971185564994812, + "logps/rejected": -1.918980598449707, + "loss": 1.0289, + "num_input_tokens_seen": 4904416, + "odds_ratio_loss": 10.114686965942383, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.0971185564994812, + "rewards/margins": 0.09477949887514114, + "rewards/rejected": -0.19189807772636414, + "sft_loss": 0.01742658205330372, + "step": 3780 + }, + { + "epoch": 0.29482691559704394, + "grad_norm": 2.2531633377075195, + "learning_rate": 4.0021374340508266e-05, + "logits/chosen": -0.1509544998407364, + "logits/rejected": -1.0268183946609497, + "logps/chosen": -1.0117778778076172, + "logps/rejected": -2.7240467071533203, + "loss": 1.0588, + "num_input_tokens_seen": 4915760, + "odds_ratio_loss": 10.479829788208008, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.10117778927087784, + "rewards/margins": 0.1712268888950348, + "rewards/rejected": -0.27240467071533203, + "sft_loss": 0.010849922895431519, + "step": 3790 + }, + { + "epoch": 0.2956048230260599, + "grad_norm": 4.629971981048584, + "learning_rate": 3.997249138704476e-05, + "logits/chosen": -0.4955291152000427, + "logits/rejected": -0.7846877574920654, + "logps/chosen": -0.9722871780395508, + "logps/rejected": -2.4349236488342285, + "loss": 1.0018, + "num_input_tokens_seen": 4928688, + "odds_ratio_loss": 9.719146728515625, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.09722872078418732, + "rewards/margins": 0.14626367390155792, + "rewards/rejected": -0.24349236488342285, + "sft_loss": 0.029885223135352135, + "step": 3800 + }, + { + "epoch": 0.29638273045507585, + "grad_norm": 2.5899016857147217, + "learning_rate": 3.992351901053934e-05, + "logits/chosen": -0.21175506711006165, + "logits/rejected": -1.361531138420105, + "logps/chosen": -0.9431182146072388, + "logps/rejected": -1.7130239009857178, + "loss": 0.9906, + "num_input_tokens_seen": 4942880, + "odds_ratio_loss": 9.751505851745605, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.09431181848049164, + "rewards/margins": 0.07699058949947357, + "rewards/rejected": -0.1713024079799652, + "sft_loss": 0.015456100925803185, + "step": 3810 + }, + { + "epoch": 0.2971606378840918, + "grad_norm": 2.118806838989258, + "learning_rate": 3.9874457503479e-05, + "logits/chosen": 0.025537211447954178, + "logits/rejected": -1.928781509399414, + "logps/chosen": -1.0842783451080322, + "logps/rejected": -2.6051692962646484, + "loss": 1.1504, + "num_input_tokens_seen": 4954816, + "odds_ratio_loss": 11.466569900512695, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.10842782258987427, + "rewards/margins": 0.15208911895751953, + "rewards/rejected": -0.2605169415473938, + "sft_loss": 0.0037107430398464203, + "step": 3820 + }, + { + "epoch": 0.29793854531310776, + "grad_norm": 10.088512420654297, + "learning_rate": 3.982530715888306e-05, + "logits/chosen": -0.16875715553760529, + "logits/rejected": -1.5386735200881958, + "logps/chosen": -1.2174850702285767, + "logps/rejected": -2.4324543476104736, + "loss": 1.2595, + "num_input_tokens_seen": 4965600, + "odds_ratio_loss": 12.139209747314453, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.12174850702285767, + "rewards/margins": 0.12149693816900253, + "rewards/rejected": -0.2432454377412796, + "sft_loss": 0.045557282865047455, + "step": 3830 + }, + { + "epoch": 0.29871645274212366, + "grad_norm": 3.1706979274749756, + "learning_rate": 3.9776068270301406e-05, + "logits/chosen": -0.016481909900903702, + "logits/rejected": -0.8835228085517883, + "logps/chosen": -0.6874436140060425, + "logps/rejected": -2.4763224124908447, + "loss": 0.7151, + "num_input_tokens_seen": 4976512, + "odds_ratio_loss": 7.089423179626465, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.06874434649944305, + "rewards/margins": 0.17888788878917694, + "rewards/rejected": -0.24763223528862, + "sft_loss": 0.006194235756993294, + "step": 3840 + }, + { + "epoch": 0.2994943601711396, + "grad_norm": 2.881826877593994, + "learning_rate": 3.972674113181277e-05, + "logits/chosen": -0.12652811408042908, + "logits/rejected": -1.2228888273239136, + "logps/chosen": -0.8278632164001465, + "logps/rejected": -1.1442887783050537, + "loss": 0.8862, + "num_input_tokens_seen": 4992464, + "odds_ratio_loss": 8.791101455688477, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.08278631418943405, + "rewards/margins": 0.03164254501461983, + "rewards/rejected": -0.11442887783050537, + "sft_loss": 0.007116052322089672, + "step": 3850 + }, + { + "epoch": 0.3002722676001556, + "grad_norm": 1.8632662296295166, + "learning_rate": 3.9677326038022955e-05, + "logits/chosen": -0.12285123020410538, + "logits/rejected": -1.1610023975372314, + "logps/chosen": -0.922530472278595, + "logps/rejected": -1.739963173866272, + "loss": 0.9586, + "num_input_tokens_seen": 5006896, + "odds_ratio_loss": 9.477170944213867, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09225304424762726, + "rewards/margins": 0.0817432776093483, + "rewards/rejected": -0.17399632930755615, + "sft_loss": 0.01084755826741457, + "step": 3860 + }, + { + "epoch": 0.30105017502917153, + "grad_norm": 9.572855949401855, + "learning_rate": 3.962782328406306e-05, + "logits/chosen": -0.06145212799310684, + "logits/rejected": -1.5028049945831299, + "logps/chosen": -0.9595915675163269, + "logps/rejected": -2.8785479068756104, + "loss": 0.9902, + "num_input_tokens_seen": 5016528, + "odds_ratio_loss": 9.786698341369629, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.09595916420221329, + "rewards/margins": 0.19189564883708954, + "rewards/rejected": -0.28785479068756104, + "sft_loss": 0.011535370722413063, + "step": 3870 + }, + { + "epoch": 0.3018280824581875, + "grad_norm": 3.2435624599456787, + "learning_rate": 3.9578233165587755e-05, + "logits/chosen": -0.2727770209312439, + "logits/rejected": -1.1245701313018799, + "logps/chosen": -1.0421762466430664, + "logps/rejected": -1.6434085369110107, + "loss": 1.0913, + "num_input_tokens_seen": 5032528, + "odds_ratio_loss": 10.329371452331543, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.10421763360500336, + "rewards/margins": 0.060123227536678314, + "rewards/rejected": -0.16434086859226227, + "sft_loss": 0.05831795930862427, + "step": 3880 + }, + { + "epoch": 0.30260598988720344, + "grad_norm": 2.970083236694336, + "learning_rate": 3.9528555978773476e-05, + "logits/chosen": -0.0969950333237648, + "logits/rejected": -1.2138078212738037, + "logps/chosen": -0.8614737391471863, + "logps/rejected": -2.951115846633911, + "loss": 0.8901, + "num_input_tokens_seen": 5043472, + "odds_ratio_loss": 8.799966812133789, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.08614736795425415, + "rewards/margins": 0.20896419882774353, + "rewards/rejected": -0.29511159658432007, + "sft_loss": 0.010101880878210068, + "step": 3890 + }, + { + "epoch": 0.3033838973162194, + "grad_norm": 3.662118434906006, + "learning_rate": 3.9478792020316685e-05, + "logits/chosen": -0.12759213149547577, + "logits/rejected": -1.413557767868042, + "logps/chosen": -0.9169372320175171, + "logps/rejected": -1.3593989610671997, + "loss": 0.9658, + "num_input_tokens_seen": 5060352, + "odds_ratio_loss": 9.604103088378906, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.09169372916221619, + "rewards/margins": 0.044246163219213486, + "rewards/rejected": -0.13593989610671997, + "sft_loss": 0.005390730686485767, + "step": 3900 + }, + { + "epoch": 0.3041618047452353, + "grad_norm": 5.8356614112854, + "learning_rate": 3.9428941587432084e-05, + "logits/chosen": -0.12662261724472046, + "logits/rejected": -1.0600976943969727, + "logps/chosen": -0.8249911069869995, + "logps/rejected": -2.696192502975464, + "loss": 0.8534, + "num_input_tokens_seen": 5069808, + "odds_ratio_loss": 8.45701789855957, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.08249910920858383, + "rewards/margins": 0.18712015450000763, + "rewards/rejected": -0.26961925625801086, + "sft_loss": 0.007720357738435268, + "step": 3910 + }, + { + "epoch": 0.30493971217425125, + "grad_norm": 2.135950803756714, + "learning_rate": 3.937900497785083e-05, + "logits/chosen": 0.018372492864727974, + "logits/rejected": -1.4753367900848389, + "logps/chosen": -0.8515621423721313, + "logps/rejected": -2.814161777496338, + "loss": 0.8767, + "num_input_tokens_seen": 5084512, + "odds_ratio_loss": 8.717264175415039, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.08515620976686478, + "rewards/margins": 0.19625994563102722, + "rewards/rejected": -0.2814161777496338, + "sft_loss": 0.004960400518029928, + "step": 3920 + }, + { + "epoch": 0.3057176196032672, + "grad_norm": 3.209294557571411, + "learning_rate": 3.93289824898188e-05, + "logits/chosen": 0.11812561750411987, + "logits/rejected": -1.5049880743026733, + "logps/chosen": -0.8617255091667175, + "logps/rejected": -2.3110384941101074, + "loss": 0.9163, + "num_input_tokens_seen": 5104352, + "odds_ratio_loss": 9.127669334411621, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.08617255091667175, + "rewards/margins": 0.14493130147457123, + "rewards/rejected": -0.23110385239124298, + "sft_loss": 0.003509619738906622, + "step": 3930 + }, + { + "epoch": 0.30649552703228317, + "grad_norm": 3.6836912631988525, + "learning_rate": 3.9278874422094735e-05, + "logits/chosen": -0.19296979904174805, + "logits/rejected": -1.0401495695114136, + "logps/chosen": -0.9293330907821655, + "logps/rejected": -2.3793680667877197, + "loss": 0.9711, + "num_input_tokens_seen": 5115648, + "odds_ratio_loss": 9.585695266723633, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.09293331205844879, + "rewards/margins": 0.14500349760055542, + "rewards/rejected": -0.23793677985668182, + "sft_loss": 0.01250266470015049, + "step": 3940 + }, + { + "epoch": 0.3072734344612991, + "grad_norm": 10.170770645141602, + "learning_rate": 3.9228681073948545e-05, + "logits/chosen": -0.37889277935028076, + "logits/rejected": -1.431139349937439, + "logps/chosen": -0.9039124250411987, + "logps/rejected": -2.598503589630127, + "loss": 0.9282, + "num_input_tokens_seen": 5138352, + "odds_ratio_loss": 9.163290023803711, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09039124101400375, + "rewards/margins": 0.16945910453796387, + "rewards/rejected": -0.2598503530025482, + "sft_loss": 0.011916982010006905, + "step": 3950 + }, + { + "epoch": 0.3080513418903151, + "grad_norm": 1.9521749019622803, + "learning_rate": 3.917840274515945e-05, + "logits/chosen": -0.17992599308490753, + "logits/rejected": -1.6150553226470947, + "logps/chosen": -1.086472749710083, + "logps/rejected": -1.8796952962875366, + "loss": 1.1391, + "num_input_tokens_seen": 5156576, + "odds_ratio_loss": 11.34900951385498, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.10864728689193726, + "rewards/margins": 0.07932224869728088, + "rewards/rejected": -0.18796953558921814, + "sft_loss": 0.004219838418066502, + "step": 3960 + }, + { + "epoch": 0.308829249319331, + "grad_norm": 3.9300570487976074, + "learning_rate": 3.9128039736014225e-05, + "logits/chosen": -0.24541409313678741, + "logits/rejected": -1.1223763227462769, + "logps/chosen": -0.98028165102005, + "logps/rejected": -1.4245949983596802, + "loss": 1.0275, + "num_input_tokens_seen": 5167008, + "odds_ratio_loss": 10.201057434082031, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.09802816808223724, + "rewards/margins": 0.04443133622407913, + "rewards/rejected": -0.14245949685573578, + "sft_loss": 0.007406421005725861, + "step": 3970 + }, + { + "epoch": 0.30960715674834693, + "grad_norm": 1.8432157039642334, + "learning_rate": 3.907759234730537e-05, + "logits/chosen": -0.21457970142364502, + "logits/rejected": -1.5651886463165283, + "logps/chosen": -0.992315948009491, + "logps/rejected": -1.2637860774993896, + "loss": 1.0453, + "num_input_tokens_seen": 5179360, + "odds_ratio_loss": 10.372383117675781, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.09923159331083298, + "rewards/margins": 0.02714700624346733, + "rewards/rejected": -0.12637859582901, + "sft_loss": 0.008084376342594624, + "step": 3980 + }, + { + "epoch": 0.3103850641773629, + "grad_norm": 2.259423017501831, + "learning_rate": 3.9027060880329394e-05, + "logits/chosen": -0.20980079472064972, + "logits/rejected": -1.3999924659729004, + "logps/chosen": -1.0523958206176758, + "logps/rejected": -1.4480522871017456, + "loss": 1.1037, + "num_input_tokens_seen": 5193008, + "odds_ratio_loss": 10.699328422546387, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.10523958504199982, + "rewards/margins": 0.039565641433000565, + "rewards/rejected": -0.14480522274971008, + "sft_loss": 0.033770475536584854, + "step": 3990 + }, + { + "epoch": 0.31116297160637885, + "grad_norm": 2.654867649078369, + "learning_rate": 3.897644563688493e-05, + "logits/chosen": -0.07522716373205185, + "logits/rejected": -1.35951566696167, + "logps/chosen": -0.9276152849197388, + "logps/rejected": -1.5378810167312622, + "loss": 0.9933, + "num_input_tokens_seen": 5205440, + "odds_ratio_loss": 9.885408401489258, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.09276153147220612, + "rewards/margins": 0.06102656200528145, + "rewards/rejected": -0.15378808975219727, + "sft_loss": 0.004794456530362368, + "step": 4000 + }, + { + "epoch": 0.3119408790353948, + "grad_norm": 3.115788698196411, + "learning_rate": 3.892574691927096e-05, + "logits/chosen": 0.01536107063293457, + "logits/rejected": -1.748814582824707, + "logps/chosen": -1.3016383647918701, + "logps/rejected": -1.8964124917984009, + "loss": 1.3451, + "num_input_tokens_seen": 5218096, + "odds_ratio_loss": 13.239151000976562, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.13016386330127716, + "rewards/margins": 0.05947741121053696, + "rewards/rejected": -0.18964125216007233, + "sft_loss": 0.021146830171346664, + "step": 4010 + }, + { + "epoch": 0.31271878646441076, + "grad_norm": 3.643676996231079, + "learning_rate": 3.887496503028501e-05, + "logits/chosen": -0.5208491683006287, + "logits/rejected": -1.0714832544326782, + "logps/chosen": -1.2488865852355957, + "logps/rejected": -1.78982412815094, + "loss": 1.3005, + "num_input_tokens_seen": 5230224, + "odds_ratio_loss": 12.145222663879395, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.12488865852355957, + "rewards/margins": 0.054093748331069946, + "rewards/rejected": -0.17898240685462952, + "sft_loss": 0.08594609797000885, + "step": 4020 + }, + { + "epoch": 0.31349669389342666, + "grad_norm": 1.7634806632995605, + "learning_rate": 3.88241002732214e-05, + "logits/chosen": -0.26208943128585815, + "logits/rejected": -1.0980716943740845, + "logps/chosen": -1.0256167650222778, + "logps/rejected": -1.8397510051727295, + "loss": 1.0805, + "num_input_tokens_seen": 5244752, + "odds_ratio_loss": 10.735319137573242, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.10256167501211166, + "rewards/margins": 0.08141344785690308, + "rewards/rejected": -0.18397513031959534, + "sft_loss": 0.006956270430237055, + "step": 4030 + }, + { + "epoch": 0.3142746013224426, + "grad_norm": 2.526923179626465, + "learning_rate": 3.87731529518693e-05, + "logits/chosen": -0.37667712569236755, + "logits/rejected": -1.2309520244598389, + "logps/chosen": -1.100372076034546, + "logps/rejected": -1.6383819580078125, + "loss": 1.1504, + "num_input_tokens_seen": 5258256, + "odds_ratio_loss": 11.36614990234375, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.11003722995519638, + "rewards/margins": 0.05380095914006233, + "rewards/rejected": -0.16383817791938782, + "sft_loss": 0.013796374201774597, + "step": 4040 + }, + { + "epoch": 0.31505250875145857, + "grad_norm": 2.473900079727173, + "learning_rate": 3.8722123370511045e-05, + "logits/chosen": -0.17751950025558472, + "logits/rejected": -1.7557871341705322, + "logps/chosen": -0.9826386570930481, + "logps/rejected": -1.5881037712097168, + "loss": 1.0416, + "num_input_tokens_seen": 5270784, + "odds_ratio_loss": 10.001214981079102, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.09826386719942093, + "rewards/margins": 0.06054651737213135, + "rewards/rejected": -0.15881037712097168, + "sft_loss": 0.04149065166711807, + "step": 4050 + }, + { + "epoch": 0.3158304161804745, + "grad_norm": 2.0946033000946045, + "learning_rate": 3.8671011833920253e-05, + "logits/chosen": -0.45757579803466797, + "logits/rejected": -1.309895634651184, + "logps/chosen": -0.8470166325569153, + "logps/rejected": -1.6577746868133545, + "loss": 0.8825, + "num_input_tokens_seen": 5283760, + "odds_ratio_loss": 8.305177688598633, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.08470165729522705, + "rewards/margins": 0.08107583224773407, + "rewards/rejected": -0.16577747464179993, + "sft_loss": 0.051960818469524384, + "step": 4060 + }, + { + "epoch": 0.3166083236094905, + "grad_norm": 5.183399677276611, + "learning_rate": 3.8619818647360036e-05, + "logits/chosen": -0.13480818271636963, + "logits/rejected": -0.695132851600647, + "logps/chosen": -0.6039600968360901, + "logps/rejected": -2.416942596435547, + "loss": 0.656, + "num_input_tokens_seen": 5294784, + "odds_ratio_loss": 6.408038139343262, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.060396015644073486, + "rewards/margins": 0.18129824101924896, + "rewards/rejected": -0.24169425666332245, + "sft_loss": 0.015180411748588085, + "step": 4070 + }, + { + "epoch": 0.31738623103850644, + "grad_norm": 3.9851224422454834, + "learning_rate": 3.856854411658112e-05, + "logits/chosen": -0.4109697937965393, + "logits/rejected": -0.8812214136123657, + "logps/chosen": -0.8279398083686829, + "logps/rejected": -1.6647628545761108, + "loss": 0.8613, + "num_input_tokens_seen": 5305744, + "odds_ratio_loss": 8.519083976745605, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.08279397338628769, + "rewards/margins": 0.08368229866027832, + "rewards/rejected": -0.1664762794971466, + "sft_loss": 0.00942680612206459, + "step": 4080 + }, + { + "epoch": 0.31816413846752234, + "grad_norm": 1.8268203735351562, + "learning_rate": 3.85171885478201e-05, + "logits/chosen": -0.12919794023036957, + "logits/rejected": -1.543035626411438, + "logps/chosen": -0.8919490575790405, + "logps/rejected": -2.169306993484497, + "loss": 0.9306, + "num_input_tokens_seen": 5319440, + "odds_ratio_loss": 8.916237831115723, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.08919490873813629, + "rewards/margins": 0.12773580849170685, + "rewards/rejected": -0.21693070232868195, + "sft_loss": 0.03893839567899704, + "step": 4090 + }, + { + "epoch": 0.3189420458965383, + "grad_norm": 55.99069595336914, + "learning_rate": 3.846575224779754e-05, + "logits/chosen": -0.34843164682388306, + "logits/rejected": -0.8747657537460327, + "logps/chosen": -1.342206358909607, + "logps/rejected": -2.3212666511535645, + "loss": 1.4213, + "num_input_tokens_seen": 5332400, + "odds_ratio_loss": 13.317713737487793, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.13422062993049622, + "rewards/margins": 0.09790606051683426, + "rewards/rejected": -0.23212668299674988, + "sft_loss": 0.08950953185558319, + "step": 4100 + }, + { + "epoch": 0.31971995332555425, + "grad_norm": 3.501796245574951, + "learning_rate": 3.841423552371619e-05, + "logits/chosen": -0.2077409327030182, + "logits/rejected": -1.3427925109863281, + "logps/chosen": -0.8884612917900085, + "logps/rejected": -1.882891058921814, + "loss": 0.9302, + "num_input_tokens_seen": 5345552, + "odds_ratio_loss": 9.219976425170898, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.0888461321592331, + "rewards/margins": 0.0994429737329483, + "rewards/rejected": -0.1882891207933426, + "sft_loss": 0.0081623625010252, + "step": 4110 + }, + { + "epoch": 0.3204978607545702, + "grad_norm": 15.101680755615234, + "learning_rate": 3.836263868325911e-05, + "logits/chosen": -0.25319308042526245, + "logits/rejected": -0.8677579164505005, + "logps/chosen": -0.7892023324966431, + "logps/rejected": -3.2379326820373535, + "loss": 0.8212, + "num_input_tokens_seen": 5359504, + "odds_ratio_loss": 8.134763717651367, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.07892023026943207, + "rewards/margins": 0.244873046875, + "rewards/rejected": -0.32379329204559326, + "sft_loss": 0.007730512414127588, + "step": 4120 + }, + { + "epoch": 0.32127576818358616, + "grad_norm": 2.807835817337036, + "learning_rate": 3.831096203458788e-05, + "logits/chosen": -0.41573959589004517, + "logits/rejected": -1.315179705619812, + "logps/chosen": -0.9759588241577148, + "logps/rejected": -1.5809203386306763, + "loss": 1.0161, + "num_input_tokens_seen": 5373568, + "odds_ratio_loss": 9.983091354370117, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.09759588539600372, + "rewards/margins": 0.06049615144729614, + "rewards/rejected": -0.15809203684329987, + "sft_loss": 0.01783646084368229, + "step": 4130 + }, + { + "epoch": 0.3220536756126021, + "grad_norm": 3.5603480339050293, + "learning_rate": 3.825920588634073e-05, + "logits/chosen": -0.29883021116256714, + "logits/rejected": -0.9933277368545532, + "logps/chosen": -0.990680992603302, + "logps/rejected": -2.809013843536377, + "loss": 1.0337, + "num_input_tokens_seen": 5383136, + "odds_ratio_loss": 10.23255729675293, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.09906809777021408, + "rewards/margins": 0.18183325231075287, + "rewards/rejected": -0.28090134263038635, + "sft_loss": 0.01048289518803358, + "step": 4140 + }, + { + "epoch": 0.322831583041618, + "grad_norm": 3.0425772666931152, + "learning_rate": 3.820737054763068e-05, + "logits/chosen": -0.035363975912332535, + "logits/rejected": -1.1433265209197998, + "logps/chosen": -0.9977181553840637, + "logps/rejected": -2.1278040409088135, + "loss": 1.0341, + "num_input_tokens_seen": 5397440, + "odds_ratio_loss": 10.24191665649414, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.09977181255817413, + "rewards/margins": 0.11300859600305557, + "rewards/rejected": -0.2127804309129715, + "sft_loss": 0.009899460710585117, + "step": 4150 + }, + { + "epoch": 0.323609490470634, + "grad_norm": 3.0186002254486084, + "learning_rate": 3.8155456328043735e-05, + "logits/chosen": -0.27061909437179565, + "logits/rejected": -1.1650453805923462, + "logps/chosen": -0.9972823262214661, + "logps/rejected": -2.490751266479492, + "loss": 1.0286, + "num_input_tokens_seen": 5408160, + "odds_ratio_loss": 10.072731018066406, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09972824156284332, + "rewards/margins": 0.14934687316417694, + "rewards/rejected": -0.24907509982585907, + "sft_loss": 0.02131205052137375, + "step": 4160 + }, + { + "epoch": 0.32438739789964993, + "grad_norm": 1.8467899560928345, + "learning_rate": 3.8103463537636996e-05, + "logits/chosen": -0.05058874562382698, + "logits/rejected": -1.356675624847412, + "logps/chosen": -0.7572735548019409, + "logps/rejected": -1.5102696418762207, + "loss": 0.8184, + "num_input_tokens_seen": 5418992, + "odds_ratio_loss": 7.95639181137085, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.07572735100984573, + "rewards/margins": 0.07529960572719574, + "rewards/rejected": -0.15102696418762207, + "sft_loss": 0.02272329106926918, + "step": 4170 + }, + { + "epoch": 0.3251653053286659, + "grad_norm": 2.185946464538574, + "learning_rate": 3.8051392486936844e-05, + "logits/chosen": -0.08555354177951813, + "logits/rejected": -0.8780487179756165, + "logps/chosen": -1.0520031452178955, + "logps/rejected": -2.304363489151001, + "loss": 1.1085, + "num_input_tokens_seen": 5435808, + "odds_ratio_loss": 11.030229568481445, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.10520030558109283, + "rewards/margins": 0.12523601949214935, + "rewards/rejected": -0.2304363250732422, + "sft_loss": 0.005523464176803827, + "step": 4180 + }, + { + "epoch": 0.32594321275768184, + "grad_norm": 1.7623631954193115, + "learning_rate": 3.799924348693706e-05, + "logits/chosen": -0.41470474004745483, + "logits/rejected": -0.833422839641571, + "logps/chosen": -0.9697521328926086, + "logps/rejected": -2.2706055641174316, + "loss": 1.0086, + "num_input_tokens_seen": 5449056, + "odds_ratio_loss": 9.848416328430176, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.09697521477937698, + "rewards/margins": 0.1300853192806244, + "rewards/rejected": -0.22706055641174316, + "sft_loss": 0.023781007155776024, + "step": 4190 + }, + { + "epoch": 0.3267211201866978, + "grad_norm": 15.622406005859375, + "learning_rate": 3.794701684909698e-05, + "logits/chosen": -0.2555413544178009, + "logits/rejected": -1.5905277729034424, + "logps/chosen": -1.1539592742919922, + "logps/rejected": -3.2055859565734863, + "loss": 1.2011, + "num_input_tokens_seen": 5460400, + "odds_ratio_loss": 11.80296802520752, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.1153959184885025, + "rewards/margins": 0.2051626741886139, + "rewards/rejected": -0.3205585777759552, + "sft_loss": 0.02078629657626152, + "step": 4200 + }, + { + "epoch": 0.32749902761571376, + "grad_norm": 5.139983654022217, + "learning_rate": 3.7894712885339634e-05, + "logits/chosen": -0.08099023252725601, + "logits/rejected": -1.3205289840698242, + "logps/chosen": -0.9128491282463074, + "logps/rejected": -4.242014408111572, + "loss": 0.9318, + "num_input_tokens_seen": 5478928, + "odds_ratio_loss": 9.188984870910645, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.09128491580486298, + "rewards/margins": 0.3329164683818817, + "rewards/rejected": -0.4242013990879059, + "sft_loss": 0.012938452884554863, + "step": 4210 + }, + { + "epoch": 0.32827693504472966, + "grad_norm": 4.9469099044799805, + "learning_rate": 3.784233190804988e-05, + "logits/chosen": -0.1827361285686493, + "logits/rejected": -1.0644224882125854, + "logps/chosen": -0.8118671178817749, + "logps/rejected": -1.7130250930786133, + "loss": 0.8587, + "num_input_tokens_seen": 5497600, + "odds_ratio_loss": 8.533797264099121, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.08118671178817749, + "rewards/margins": 0.09011580049991608, + "rewards/rejected": -0.17130251228809357, + "sft_loss": 0.005304677411913872, + "step": 4220 + }, + { + "epoch": 0.3290548424737456, + "grad_norm": 3.007436752319336, + "learning_rate": 3.778987423007252e-05, + "logits/chosen": -0.6660521626472473, + "logits/rejected": -0.7182964086532593, + "logps/chosen": -0.8447635769844055, + "logps/rejected": -1.9354795217514038, + "loss": 0.8841, + "num_input_tokens_seen": 5510512, + "odds_ratio_loss": 8.513199806213379, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.08447636663913727, + "rewards/margins": 0.10907159745693207, + "rewards/rejected": -0.19354796409606934, + "sft_loss": 0.032828912138938904, + "step": 4230 + }, + { + "epoch": 0.32983274990276157, + "grad_norm": 3.3129470348358154, + "learning_rate": 3.7737340164710474e-05, + "logits/chosen": -0.3203393816947937, + "logits/rejected": -0.8482300639152527, + "logps/chosen": -0.9250307083129883, + "logps/rejected": -2.723344326019287, + "loss": 0.9538, + "num_input_tokens_seen": 5522992, + "odds_ratio_loss": 9.43076229095459, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.09250307828187943, + "rewards/margins": 0.1798313558101654, + "rewards/rejected": -0.2723344564437866, + "sft_loss": 0.010698824189603329, + "step": 4240 + }, + { + "epoch": 0.3306106573317775, + "grad_norm": 9.28976821899414, + "learning_rate": 3.7684730025722884e-05, + "logits/chosen": -0.1900298297405243, + "logits/rejected": -0.8809641003608704, + "logps/chosen": -0.9689387083053589, + "logps/rejected": -1.8893556594848633, + "loss": 1.0076, + "num_input_tokens_seen": 5539344, + "odds_ratio_loss": 10.023103713989258, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.09689386934041977, + "rewards/margins": 0.09204168617725372, + "rewards/rejected": -0.1889355629682541, + "sft_loss": 0.005292147863656282, + "step": 4250 + }, + { + "epoch": 0.3313885647607935, + "grad_norm": 1.712719440460205, + "learning_rate": 3.7632044127323216e-05, + "logits/chosen": 0.1145520806312561, + "logits/rejected": -1.800127625465393, + "logps/chosen": -0.9322274923324585, + "logps/rejected": -1.7594856023788452, + "loss": 0.9729, + "num_input_tokens_seen": 5555152, + "odds_ratio_loss": 9.706247329711914, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.09322275966405869, + "rewards/margins": 0.08272580802440643, + "rewards/rejected": -0.17594856023788452, + "sft_loss": 0.0022511687129735947, + "step": 4260 + }, + { + "epoch": 0.33216647218980944, + "grad_norm": 1.8186136484146118, + "learning_rate": 3.7579282784177436e-05, + "logits/chosen": -0.21599099040031433, + "logits/rejected": -1.5185308456420898, + "logps/chosen": -0.8938544988632202, + "logps/rejected": -1.7810747623443604, + "loss": 0.9353, + "num_input_tokens_seen": 5566192, + "odds_ratio_loss": 9.18561840057373, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.08938544988632202, + "rewards/margins": 0.08872202783823013, + "rewards/rejected": -0.17810747027397156, + "sft_loss": 0.016736816614866257, + "step": 4270 + }, + { + "epoch": 0.33294437961882534, + "grad_norm": 2.4397411346435547, + "learning_rate": 3.7526446311402095e-05, + "logits/chosen": -0.014413388445973396, + "logits/rejected": -1.4831351041793823, + "logps/chosen": -0.7523231506347656, + "logps/rejected": -2.23205828666687, + "loss": 0.7786, + "num_input_tokens_seen": 5580640, + "odds_ratio_loss": 7.747849464416504, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.07523230463266373, + "rewards/margins": 0.14797353744506836, + "rewards/rejected": -0.2232058346271515, + "sft_loss": 0.0037966384552419186, + "step": 4280 + }, + { + "epoch": 0.3337222870478413, + "grad_norm": 1.911331295967102, + "learning_rate": 3.7473535024562446e-05, + "logits/chosen": -0.1065264493227005, + "logits/rejected": -1.228137731552124, + "logps/chosen": -0.8722216486930847, + "logps/rejected": -1.2977032661437988, + "loss": 0.9205, + "num_input_tokens_seen": 5591376, + "odds_ratio_loss": 9.155069351196289, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.08722217381000519, + "rewards/margins": 0.042548153549432755, + "rewards/rejected": -0.12977032363414764, + "sft_loss": 0.005033262073993683, + "step": 4290 + }, + { + "epoch": 0.33450019447685725, + "grad_norm": 1.7152159214019775, + "learning_rate": 3.742054923967057e-05, + "logits/chosen": 0.07246137410402298, + "logits/rejected": -1.9047914743423462, + "logps/chosen": -0.8156659007072449, + "logps/rejected": -3.5807089805603027, + "loss": 0.8458, + "num_input_tokens_seen": 5606752, + "odds_ratio_loss": 8.434159278869629, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.08156659454107285, + "rewards/margins": 0.2765043377876282, + "rewards/rejected": -0.35807090997695923, + "sft_loss": 0.002411172492429614, + "step": 4300 + }, + { + "epoch": 0.3352781019058732, + "grad_norm": 1.6043893098831177, + "learning_rate": 3.7367489273183495e-05, + "logits/chosen": -0.13491670787334442, + "logits/rejected": -1.4279476404190063, + "logps/chosen": -0.675383448600769, + "logps/rejected": -2.471374034881592, + "loss": 0.7059, + "num_input_tokens_seen": 5618800, + "odds_ratio_loss": 6.939095497131348, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.06753835082054138, + "rewards/margins": 0.1795990765094757, + "rewards/rejected": -0.2471374273300171, + "sft_loss": 0.01202771533280611, + "step": 4310 + }, + { + "epoch": 0.33605600933488916, + "grad_norm": 6.193589687347412, + "learning_rate": 3.7314355442001294e-05, + "logits/chosen": -0.4281120300292969, + "logits/rejected": -0.8102872967720032, + "logps/chosen": -1.042340636253357, + "logps/rejected": -2.1481173038482666, + "loss": 1.079, + "num_input_tokens_seen": 5630448, + "odds_ratio_loss": 10.683089256286621, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.10423406213521957, + "rewards/margins": 0.11057764291763306, + "rewards/rejected": -0.21481171250343323, + "sft_loss": 0.010718410834670067, + "step": 4320 + }, + { + "epoch": 0.3368339167639051, + "grad_norm": 3.1446692943573, + "learning_rate": 3.726114806346518e-05, + "logits/chosen": -0.5779768228530884, + "logits/rejected": -1.1927872896194458, + "logps/chosen": -0.9345456957817078, + "logps/rejected": -2.156569004058838, + "loss": 0.9621, + "num_input_tokens_seen": 5643584, + "odds_ratio_loss": 9.511880874633789, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.09345457702875137, + "rewards/margins": 0.12220235913991928, + "rewards/rejected": -0.21565695106983185, + "sft_loss": 0.010866018943488598, + "step": 4330 + }, + { + "epoch": 0.337611824192921, + "grad_norm": 2.2310080528259277, + "learning_rate": 3.720786745535568e-05, + "logits/chosen": 0.017054501920938492, + "logits/rejected": -1.5218160152435303, + "logps/chosen": -0.9744178652763367, + "logps/rejected": -2.552828311920166, + "loss": 1.0142, + "num_input_tokens_seen": 5656624, + "odds_ratio_loss": 10.09164810180664, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.09744178503751755, + "rewards/margins": 0.1578410416841507, + "rewards/rejected": -0.25528278946876526, + "sft_loss": 0.005024941172450781, + "step": 4340 + }, + { + "epoch": 0.338389731621937, + "grad_norm": 2.9606218338012695, + "learning_rate": 3.7154513935890614e-05, + "logits/chosen": -0.18985190987586975, + "logits/rejected": -1.7492328882217407, + "logps/chosen": -0.8841496706008911, + "logps/rejected": -2.6282975673675537, + "loss": 0.9231, + "num_input_tokens_seen": 5666960, + "odds_ratio_loss": 9.061014175415039, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.08841496706008911, + "rewards/margins": 0.17441478371620178, + "rewards/rejected": -0.2628297507762909, + "sft_loss": 0.0170004703104496, + "step": 4350 + }, + { + "epoch": 0.33916763905095293, + "grad_norm": 12.98637866973877, + "learning_rate": 3.710108782372331e-05, + "logits/chosen": -0.5041710138320923, + "logits/rejected": -0.9508134722709656, + "logps/chosen": -1.128469705581665, + "logps/rejected": -1.6678993701934814, + "loss": 1.18, + "num_input_tokens_seen": 5679232, + "odds_ratio_loss": 11.528806686401367, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.11284694820642471, + "rewards/margins": 0.053942978382110596, + "rewards/rejected": -0.1667899340391159, + "sft_loss": 0.027155181393027306, + "step": 4360 + }, + { + "epoch": 0.3399455464799689, + "grad_norm": 31.380678176879883, + "learning_rate": 3.7047589437940636e-05, + "logits/chosen": -0.7777343988418579, + "logits/rejected": -1.0380951166152954, + "logps/chosen": -0.94134920835495, + "logps/rejected": -2.7589385509490967, + "loss": 0.9895, + "num_input_tokens_seen": 5690656, + "odds_ratio_loss": 9.376608848571777, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.09413491189479828, + "rewards/margins": 0.18175895512104034, + "rewards/rejected": -0.27589383721351624, + "sft_loss": 0.05186940357089043, + "step": 4370 + }, + { + "epoch": 0.34072345390898484, + "grad_norm": 5.7405595779418945, + "learning_rate": 3.6994019098061123e-05, + "logits/chosen": -0.3143104910850525, + "logits/rejected": -0.915662407875061, + "logps/chosen": -0.9450228810310364, + "logps/rejected": -1.476238489151001, + "loss": 0.9972, + "num_input_tokens_seen": 5702304, + "odds_ratio_loss": 9.882279396057129, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.0945022851228714, + "rewards/margins": 0.05312156677246094, + "rewards/rejected": -0.14762386679649353, + "sft_loss": 0.008927664719522, + "step": 4380 + }, + { + "epoch": 0.3415013613380008, + "grad_norm": 2.7874977588653564, + "learning_rate": 3.6940377124033033e-05, + "logits/chosen": -0.540741503238678, + "logits/rejected": -1.1405988931655884, + "logps/chosen": -0.9208908081054688, + "logps/rejected": -2.2547502517700195, + "loss": 0.9975, + "num_input_tokens_seen": 5713360, + "odds_ratio_loss": 9.85312271118164, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.09208908677101135, + "rewards/margins": 0.13338592648506165, + "rewards/rejected": -0.2254750281572342, + "sft_loss": 0.012210485525429249, + "step": 4390 + }, + { + "epoch": 0.3422792687670167, + "grad_norm": 1.7215228080749512, + "learning_rate": 3.688666383623249e-05, + "logits/chosen": -0.3327955901622772, + "logits/rejected": -1.1232503652572632, + "logps/chosen": -1.0700501203536987, + "logps/rejected": -1.328126311302185, + "loss": 1.1692, + "num_input_tokens_seen": 5728016, + "odds_ratio_loss": 11.548150062561035, + "rewards/accuracies": 0.4000000059604645, + "rewards/chosen": -0.10700502246618271, + "rewards/margins": 0.025807607918977737, + "rewards/rejected": -0.13281263411045074, + "sft_loss": 0.014384521171450615, + "step": 4400 + }, + { + "epoch": 0.34305717619603265, + "grad_norm": 2.786660671234131, + "learning_rate": 3.6832879555461507e-05, + "logits/chosen": -0.06084407493472099, + "logits/rejected": -1.2869935035705566, + "logps/chosen": -0.970739483833313, + "logps/rejected": -2.362373113632202, + "loss": 0.9998, + "num_input_tokens_seen": 5742624, + "odds_ratio_loss": 9.313217163085938, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09707394987344742, + "rewards/margins": 0.13916337490081787, + "rewards/rejected": -0.2362373322248459, + "sft_loss": 0.06852734088897705, + "step": 4410 + }, + { + "epoch": 0.3438350836250486, + "grad_norm": 11.903273582458496, + "learning_rate": 3.677902460294612e-05, + "logits/chosen": 0.05326567962765694, + "logits/rejected": -1.6322355270385742, + "logps/chosen": -0.9381688833236694, + "logps/rejected": -1.8394731283187866, + "loss": 0.9744, + "num_input_tokens_seen": 5756800, + "odds_ratio_loss": 9.685869216918945, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.09381689876317978, + "rewards/margins": 0.09013044089078903, + "rewards/rejected": -0.18394732475280762, + "sft_loss": 0.005861429963260889, + "step": 4420 + }, + { + "epoch": 0.34461299105406457, + "grad_norm": 4.127033710479736, + "learning_rate": 3.6725099300334446e-05, + "logits/chosen": -0.16183969378471375, + "logits/rejected": -1.6115763187408447, + "logps/chosen": -0.887466311454773, + "logps/rejected": -2.323113441467285, + "loss": 0.9084, + "num_input_tokens_seen": 5769152, + "odds_ratio_loss": 9.000121116638184, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.08874664455652237, + "rewards/margins": 0.14356471598148346, + "rewards/rejected": -0.23231136798858643, + "sft_loss": 0.008424920961260796, + "step": 4430 + }, + { + "epoch": 0.3453908984830805, + "grad_norm": 2.3415582180023193, + "learning_rate": 3.667110396969477e-05, + "logits/chosen": -0.02460600808262825, + "logits/rejected": -1.9235773086547852, + "logps/chosen": -0.9934428930282593, + "logps/rejected": -2.2113564014434814, + "loss": 1.0324, + "num_input_tokens_seen": 5786576, + "odds_ratio_loss": 10.282426834106445, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09934429824352264, + "rewards/margins": 0.12179134041070938, + "rewards/rejected": -0.22113564610481262, + "sft_loss": 0.004182109609246254, + "step": 4440 + }, + { + "epoch": 0.3461688059120965, + "grad_norm": 2.3995091915130615, + "learning_rate": 3.661703893351361e-05, + "logits/chosen": 0.07914382219314575, + "logits/rejected": -1.6564337015151978, + "logps/chosen": -0.9553931951522827, + "logps/rejected": -1.6407089233398438, + "loss": 0.9945, + "num_input_tokens_seen": 5799632, + "odds_ratio_loss": 9.90433406829834, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.09553932398557663, + "rewards/margins": 0.0685315653681755, + "rewards/rejected": -0.16407087445259094, + "sft_loss": 0.004019489046186209, + "step": 4450 + }, + { + "epoch": 0.34694671334111243, + "grad_norm": 4.87782621383667, + "learning_rate": 3.656290451469382e-05, + "logits/chosen": -0.36600252985954285, + "logits/rejected": -0.7956069707870483, + "logps/chosen": -0.8138600587844849, + "logps/rejected": -2.0378212928771973, + "loss": 0.8456, + "num_input_tokens_seen": 5811184, + "odds_ratio_loss": 8.397814750671387, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.08138599991798401, + "rewards/margins": 0.12239613384008408, + "rewards/rejected": -0.2037821263074875, + "sft_loss": 0.005857658572494984, + "step": 4460 + }, + { + "epoch": 0.34772462077012833, + "grad_norm": 4.152865886688232, + "learning_rate": 3.6508701036552624e-05, + "logits/chosen": 0.022718677297234535, + "logits/rejected": -1.404191017150879, + "logps/chosen": -0.9380876421928406, + "logps/rejected": -1.9611479043960571, + "loss": 0.9886, + "num_input_tokens_seen": 5823296, + "odds_ratio_loss": 9.800165176391602, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.09380875527858734, + "rewards/margins": 0.10230604559183121, + "rewards/rejected": -0.19611480832099915, + "sft_loss": 0.008533725515007973, + "step": 4470 + }, + { + "epoch": 0.3485025281991443, + "grad_norm": 1.5638813972473145, + "learning_rate": 3.6454428822819695e-05, + "logits/chosen": -0.2070436030626297, + "logits/rejected": -1.384708046913147, + "logps/chosen": -1.103393793106079, + "logps/rejected": -2.512498140335083, + "loss": 1.1549, + "num_input_tokens_seen": 5836768, + "odds_ratio_loss": 11.028097152709961, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.11033938825130463, + "rewards/margins": 0.14091041684150696, + "rewards/rejected": -0.25124984979629517, + "sft_loss": 0.05206207558512688, + "step": 4480 + }, + { + "epoch": 0.34928043562816025, + "grad_norm": 5.045617580413818, + "learning_rate": 3.640008819763525e-05, + "logits/chosen": -0.35230201482772827, + "logits/rejected": -1.3696832656860352, + "logps/chosen": -1.197439432144165, + "logps/rejected": -2.184636116027832, + "loss": 1.2687, + "num_input_tokens_seen": 5847728, + "odds_ratio_loss": 11.410408973693848, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.11974392831325531, + "rewards/margins": 0.09871968626976013, + "rewards/rejected": -0.21846361458301544, + "sft_loss": 0.12763787806034088, + "step": 4490 + }, + { + "epoch": 0.3500583430571762, + "grad_norm": 1.8348246812820435, + "learning_rate": 3.634567948554807e-05, + "logits/chosen": -0.05636955052614212, + "logits/rejected": -1.405261516571045, + "logps/chosen": -1.0076950788497925, + "logps/rejected": -1.3658123016357422, + "loss": 1.0606, + "num_input_tokens_seen": 5861776, + "odds_ratio_loss": 10.531423568725586, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.1007695198059082, + "rewards/margins": 0.03581172972917557, + "rewards/rejected": -0.13658125698566437, + "sft_loss": 0.007421608082950115, + "step": 4500 + }, + { + "epoch": 0.35083625048619216, + "grad_norm": 1.8715535402297974, + "learning_rate": 3.62912030115136e-05, + "logits/chosen": -0.3818695545196533, + "logits/rejected": -1.6756805181503296, + "logps/chosen": -1.0263088941574097, + "logps/rejected": -2.775115966796875, + "loss": 1.0608, + "num_input_tokens_seen": 5872640, + "odds_ratio_loss": 10.09156608581543, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.10263089835643768, + "rewards/margins": 0.174880713224411, + "rewards/rejected": -0.2775115966796875, + "sft_loss": 0.05163625627756119, + "step": 4510 + }, + { + "epoch": 0.3516141579152081, + "grad_norm": 1.4873770475387573, + "learning_rate": 3.6236659100892e-05, + "logits/chosen": 0.11474569141864777, + "logits/rejected": -1.988440752029419, + "logps/chosen": -0.980634868144989, + "logps/rejected": -1.8994451761245728, + "loss": 1.0176, + "num_input_tokens_seen": 5892208, + "odds_ratio_loss": 10.155592918395996, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09806347638368607, + "rewards/margins": 0.09188102185726166, + "rewards/rejected": -0.18994450569152832, + "sft_loss": 0.0020749946124851704, + "step": 4520 + }, + { + "epoch": 0.352392065344224, + "grad_norm": 3.3406612873077393, + "learning_rate": 3.618204807944617e-05, + "logits/chosen": 0.010874845087528229, + "logits/rejected": -1.3346697092056274, + "logps/chosen": -0.8537413477897644, + "logps/rejected": -3.3422837257385254, + "loss": 0.8963, + "num_input_tokens_seen": 5903216, + "odds_ratio_loss": 8.909745216369629, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.0853741317987442, + "rewards/margins": 0.24885420501232147, + "rewards/rejected": -0.3342283368110657, + "sft_loss": 0.005339351482689381, + "step": 4530 + }, + { + "epoch": 0.35316997277323997, + "grad_norm": 1.2649593353271484, + "learning_rate": 3.612737027333984e-05, + "logits/chosen": -0.04491196200251579, + "logits/rejected": -1.3434958457946777, + "logps/chosen": -0.8880999684333801, + "logps/rejected": -1.592533826828003, + "loss": 0.921, + "num_input_tokens_seen": 5918576, + "odds_ratio_loss": 9.174939155578613, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.08880998939275742, + "rewards/margins": 0.07044339925050735, + "rewards/rejected": -0.15925338864326477, + "sft_loss": 0.0034705442376434803, + "step": 4540 + }, + { + "epoch": 0.3539478802022559, + "grad_norm": 2.581486225128174, + "learning_rate": 3.607262600913563e-05, + "logits/chosen": -0.402034193277359, + "logits/rejected": -0.3176518380641937, + "logps/chosen": -0.7697569727897644, + "logps/rejected": -1.591722011566162, + "loss": 0.8021, + "num_input_tokens_seen": 5931248, + "odds_ratio_loss": 7.6087541580200195, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.07697569578886032, + "rewards/margins": 0.08219649642705917, + "rewards/rejected": -0.1591722071170807, + "sft_loss": 0.041197795420885086, + "step": 4550 + }, + { + "epoch": 0.3547257876312719, + "grad_norm": 3.187448263168335, + "learning_rate": 3.601781561379304e-05, + "logits/chosen": -0.0909782350063324, + "logits/rejected": -1.6023838520050049, + "logps/chosen": -1.0848517417907715, + "logps/rejected": -1.7551847696304321, + "loss": 1.1528, + "num_input_tokens_seen": 5945392, + "odds_ratio_loss": 11.288145065307617, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.10848517715930939, + "rewards/margins": 0.0670332983136177, + "rewards/rejected": -0.1755184680223465, + "sft_loss": 0.02397894114255905, + "step": 4560 + }, + { + "epoch": 0.35550369506028784, + "grad_norm": 6.8400373458862305, + "learning_rate": 3.596293941466658e-05, + "logits/chosen": -0.3084474205970764, + "logits/rejected": -1.0309423208236694, + "logps/chosen": -0.8194421529769897, + "logps/rejected": -2.439091205596924, + "loss": 0.8622, + "num_input_tokens_seen": 5956272, + "odds_ratio_loss": 8.532068252563477, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.08194421231746674, + "rewards/margins": 0.16196492314338684, + "rewards/rejected": -0.24390912055969238, + "sft_loss": 0.008951345458626747, + "step": 4570 + }, + { + "epoch": 0.3562816024893038, + "grad_norm": 5.4224772453308105, + "learning_rate": 3.5907997739503735e-05, + "logits/chosen": 0.02636636421084404, + "logits/rejected": -1.5030062198638916, + "logps/chosen": -0.7921457290649414, + "logps/rejected": -4.366400718688965, + "loss": 0.811, + "num_input_tokens_seen": 5968640, + "odds_ratio_loss": 8.064387321472168, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.07921457290649414, + "rewards/margins": 0.3574255406856537, + "rewards/rejected": -0.4366401135921478, + "sft_loss": 0.0045320941135287285, + "step": 4580 + }, + { + "epoch": 0.3570595099183197, + "grad_norm": 30.135780334472656, + "learning_rate": 3.585299091644307e-05, + "logits/chosen": -0.31513237953186035, + "logits/rejected": -1.236054539680481, + "logps/chosen": -0.8894597887992859, + "logps/rejected": -4.156570911407471, + "loss": 0.9329, + "num_input_tokens_seen": 5977824, + "odds_ratio_loss": 8.80221176147461, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.08894598484039307, + "rewards/margins": 0.32671108841896057, + "rewards/rejected": -0.41565704345703125, + "sft_loss": 0.05266100913286209, + "step": 4590 + }, + { + "epoch": 0.35783741734733565, + "grad_norm": 3.0575132369995117, + "learning_rate": 3.5797919274012235e-05, + "logits/chosen": -0.0275888592004776, + "logits/rejected": -1.386785864830017, + "logps/chosen": -1.0150604248046875, + "logps/rejected": -2.2971482276916504, + "loss": 1.056, + "num_input_tokens_seen": 5985584, + "odds_ratio_loss": 10.500213623046875, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.1015060544013977, + "rewards/margins": 0.12820878624916077, + "rewards/rejected": -0.22971484065055847, + "sft_loss": 0.005982295144349337, + "step": 4600 + }, + { + "epoch": 0.3586153247763516, + "grad_norm": 6.465326309204102, + "learning_rate": 3.5742783141126016e-05, + "logits/chosen": -0.48850011825561523, + "logits/rejected": -0.8825419545173645, + "logps/chosen": -0.9011627435684204, + "logps/rejected": -1.7979437112808228, + "loss": 0.9462, + "num_input_tokens_seen": 5999360, + "odds_ratio_loss": 9.355947494506836, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.09011626988649368, + "rewards/margins": 0.08967811614274979, + "rewards/rejected": -0.17979438602924347, + "sft_loss": 0.010608791373670101, + "step": 4610 + }, + { + "epoch": 0.35939323220536756, + "grad_norm": 2.4869587421417236, + "learning_rate": 3.568758284708437e-05, + "logits/chosen": -0.01620178297162056, + "logits/rejected": -1.0039026737213135, + "logps/chosen": -0.8032919764518738, + "logps/rejected": -2.2006430625915527, + "loss": 0.8354, + "num_input_tokens_seen": 6010208, + "odds_ratio_loss": 8.280613899230957, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.08032919466495514, + "rewards/margins": 0.13973508775234222, + "rewards/rejected": -0.22006428241729736, + "sft_loss": 0.0073692104779183865, + "step": 4620 + }, + { + "epoch": 0.3601711396343835, + "grad_norm": 33.07954025268555, + "learning_rate": 3.563231872157045e-05, + "logits/chosen": -0.32904794812202454, + "logits/rejected": -1.1162834167480469, + "logps/chosen": -1.023409128189087, + "logps/rejected": -2.7538928985595703, + "loss": 1.0469, + "num_input_tokens_seen": 6021760, + "odds_ratio_loss": 10.05885124206543, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.10234091430902481, + "rewards/margins": 0.17304839193820953, + "rewards/rejected": -0.27538928389549255, + "sft_loss": 0.04098767414689064, + "step": 4630 + }, + { + "epoch": 0.3609490470633995, + "grad_norm": 5.191165447235107, + "learning_rate": 3.557699109464865e-05, + "logits/chosen": -0.34737399220466614, + "logits/rejected": -0.7770891189575195, + "logps/chosen": -0.8472837209701538, + "logps/rejected": -2.7982988357543945, + "loss": 0.8721, + "num_input_tokens_seen": 6036992, + "odds_ratio_loss": 8.06086254119873, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.08472837507724762, + "rewards/margins": 0.19510149955749512, + "rewards/rejected": -0.27982988953590393, + "sft_loss": 0.06597577035427094, + "step": 4640 + }, + { + "epoch": 0.3617269544924154, + "grad_norm": 4.145506858825684, + "learning_rate": 3.55216002967626e-05, + "logits/chosen": -0.5288735628128052, + "logits/rejected": -0.6832670569419861, + "logps/chosen": -0.841774582862854, + "logps/rejected": -1.3679779767990112, + "loss": 0.9025, + "num_input_tokens_seen": 6048112, + "odds_ratio_loss": 8.584033012390137, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.08417744934558868, + "rewards/margins": 0.05262034386396408, + "rewards/rejected": -0.13679781556129456, + "sft_loss": 0.04414389654994011, + "step": 4650 + }, + { + "epoch": 0.36250486192143133, + "grad_norm": 1.8345520496368408, + "learning_rate": 3.546614665873326e-05, + "logits/chosen": -0.19236251711845398, + "logits/rejected": -1.0131992101669312, + "logps/chosen": -0.9838958978652954, + "logps/rejected": -1.544071912765503, + "loss": 1.0348, + "num_input_tokens_seen": 6064512, + "odds_ratio_loss": 10.184259414672852, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.09838961064815521, + "rewards/margins": 0.05601758882403374, + "rewards/rejected": -0.15440720319747925, + "sft_loss": 0.016333328559994698, + "step": 4660 + }, + { + "epoch": 0.3632827693504473, + "grad_norm": 1.775390863418579, + "learning_rate": 3.5410630511756874e-05, + "logits/chosen": 0.026402678340673447, + "logits/rejected": -1.7419201135635376, + "logps/chosen": -0.9833968877792358, + "logps/rejected": -3.8465347290039062, + "loss": 1.0076, + "num_input_tokens_seen": 6083280, + "odds_ratio_loss": 9.9813814163208, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.09833969175815582, + "rewards/margins": 0.2863138020038605, + "rewards/rejected": -0.3846535086631775, + "sft_loss": 0.009477846324443817, + "step": 4670 + }, + { + "epoch": 0.36406067677946324, + "grad_norm": 9.401809692382812, + "learning_rate": 3.535505218740303e-05, + "logits/chosen": -0.07243140041828156, + "logits/rejected": -1.3489840030670166, + "logps/chosen": -0.9240199327468872, + "logps/rejected": -1.2963130474090576, + "loss": 0.9727, + "num_input_tokens_seen": 6098496, + "odds_ratio_loss": 9.667537689208984, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.09240199625492096, + "rewards/margins": 0.037229303270578384, + "rewards/rejected": -0.12963131070137024, + "sft_loss": 0.005986756179481745, + "step": 4680 + }, + { + "epoch": 0.3648385842084792, + "grad_norm": 5.5551581382751465, + "learning_rate": 3.529941201761264e-05, + "logits/chosen": -0.06656485050916672, + "logits/rejected": -1.1562308073043823, + "logps/chosen": -0.8639378547668457, + "logps/rejected": -2.3257360458374023, + "loss": 0.8889, + "num_input_tokens_seen": 6111008, + "odds_ratio_loss": 8.844882011413574, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.08639378100633621, + "rewards/margins": 0.14617982506752014, + "rewards/rejected": -0.23257359862327576, + "sft_loss": 0.004383668769150972, + "step": 4690 + }, + { + "epoch": 0.36561649163749516, + "grad_norm": 12.059032440185547, + "learning_rate": 3.5243710334696047e-05, + "logits/chosen": -0.08709411323070526, + "logits/rejected": -1.2754627466201782, + "logps/chosen": -0.921644389629364, + "logps/rejected": -2.91294002532959, + "loss": 0.9603, + "num_input_tokens_seen": 6123424, + "odds_ratio_loss": 9.507317543029785, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.09216442704200745, + "rewards/margins": 0.19912958145141602, + "rewards/rejected": -0.29129400849342346, + "sft_loss": 0.009597374126315117, + "step": 4700 + }, + { + "epoch": 0.3663943990665111, + "grad_norm": 1.602284550666809, + "learning_rate": 3.5187947471330925e-05, + "logits/chosen": -0.22274041175842285, + "logits/rejected": -1.458547830581665, + "logps/chosen": -0.9226606488227844, + "logps/rejected": -2.3675448894500732, + "loss": 0.9565, + "num_input_tokens_seen": 6138416, + "odds_ratio_loss": 9.507163047790527, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.09226606041193008, + "rewards/margins": 0.14448843896389008, + "rewards/rejected": -0.23675450682640076, + "sft_loss": 0.005814017727971077, + "step": 4710 + }, + { + "epoch": 0.367172306495527, + "grad_norm": 9.358242988586426, + "learning_rate": 3.5132123760560375e-05, + "logits/chosen": -0.4783569872379303, + "logits/rejected": -0.9221259355545044, + "logps/chosen": -0.9382320642471313, + "logps/rejected": -3.0921168327331543, + "loss": 0.9693, + "num_input_tokens_seen": 6153056, + "odds_ratio_loss": 8.790300369262695, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.09382319450378418, + "rewards/margins": 0.2153884917497635, + "rewards/rejected": -0.30921173095703125, + "sft_loss": 0.09023318439722061, + "step": 4720 + }, + { + "epoch": 0.36795021392454297, + "grad_norm": 6.939538478851318, + "learning_rate": 3.507623953579091e-05, + "logits/chosen": -0.3538041114807129, + "logits/rejected": -0.9801716804504395, + "logps/chosen": -1.0121508836746216, + "logps/rejected": -1.9558436870574951, + "loss": 1.049, + "num_input_tokens_seen": 6161680, + "odds_ratio_loss": 10.354455947875977, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.10121508687734604, + "rewards/margins": 0.09436929225921631, + "rewards/rejected": -0.19558438658714294, + "sft_loss": 0.013524286448955536, + "step": 4730 + }, + { + "epoch": 0.3687281213535589, + "grad_norm": 4.77536678314209, + "learning_rate": 3.502029513079043e-05, + "logits/chosen": -0.03763290494680405, + "logits/rejected": -1.1161601543426514, + "logps/chosen": -0.8724561929702759, + "logps/rejected": -1.8304494619369507, + "loss": 0.9243, + "num_input_tokens_seen": 6180208, + "odds_ratio_loss": 9.193095207214355, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.08724562078714371, + "rewards/margins": 0.09579931199550629, + "rewards/rejected": -0.18304495513439178, + "sft_loss": 0.004998179618269205, + "step": 4740 + }, + { + "epoch": 0.3695060287825749, + "grad_norm": 3.624525308609009, + "learning_rate": 3.496429087968631e-05, + "logits/chosen": -0.18473979830741882, + "logits/rejected": -0.7089244723320007, + "logps/chosen": -0.718346118927002, + "logps/rejected": -2.575134038925171, + "loss": 0.7439, + "num_input_tokens_seen": 6191808, + "odds_ratio_loss": 7.302772521972656, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.07183460891246796, + "rewards/margins": 0.18567879498004913, + "rewards/rejected": -0.2575134336948395, + "sft_loss": 0.013608465902507305, + "step": 4750 + }, + { + "epoch": 0.37028393621159084, + "grad_norm": 1.931633710861206, + "learning_rate": 3.490822711696333e-05, + "logits/chosen": 0.20464691519737244, + "logits/rejected": -1.3728229999542236, + "logps/chosen": -0.8623288869857788, + "logps/rejected": -1.52870774269104, + "loss": 0.9078, + "num_input_tokens_seen": 6206112, + "odds_ratio_loss": 9.046833038330078, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.08623289316892624, + "rewards/margins": 0.06663788855075836, + "rewards/rejected": -0.152870774269104, + "sft_loss": 0.003099628258496523, + "step": 4760 + }, + { + "epoch": 0.3710618436406068, + "grad_norm": 16.25254249572754, + "learning_rate": 3.4852104177461685e-05, + "logits/chosen": -0.030558357015252113, + "logits/rejected": -1.134142279624939, + "logps/chosen": -0.9946354627609253, + "logps/rejected": -3.872499942779541, + "loss": 1.0323, + "num_input_tokens_seen": 6223760, + "odds_ratio_loss": 10.13349437713623, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.099463552236557, + "rewards/margins": 0.28778642416000366, + "rewards/rejected": -0.38725000619888306, + "sft_loss": 0.018958527594804764, + "step": 4770 + }, + { + "epoch": 0.3718397510696227, + "grad_norm": 6.025865077972412, + "learning_rate": 3.4795922396375034e-05, + "logits/chosen": -0.03270745649933815, + "logits/rejected": -1.0162115097045898, + "logps/chosen": -0.7854177951812744, + "logps/rejected": -2.39085054397583, + "loss": 0.8294, + "num_input_tokens_seen": 6232512, + "odds_ratio_loss": 8.224783897399902, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.07854177802801132, + "rewards/margins": 0.1605432778596878, + "rewards/rejected": -0.23908504843711853, + "sft_loss": 0.0069030774757266045, + "step": 4780 + }, + { + "epoch": 0.37261765849863865, + "grad_norm": 1.815285563468933, + "learning_rate": 3.473968210924845e-05, + "logits/chosen": 0.16790811717510223, + "logits/rejected": -1.867071509361267, + "logps/chosen": -0.8914972543716431, + "logps/rejected": -2.30869197845459, + "loss": 0.9189, + "num_input_tokens_seen": 6250160, + "odds_ratio_loss": 9.165290832519531, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.08914972096681595, + "rewards/margins": 0.14171946048736572, + "rewards/rejected": -0.23086917400360107, + "sft_loss": 0.002407553605735302, + "step": 4790 + }, + { + "epoch": 0.3733955659276546, + "grad_norm": 5.934152603149414, + "learning_rate": 3.468338365197643e-05, + "logits/chosen": -0.04665079340338707, + "logits/rejected": -1.3595021963119507, + "logps/chosen": -0.8833147883415222, + "logps/rejected": -2.2141613960266113, + "loss": 0.9124, + "num_input_tokens_seen": 6262560, + "odds_ratio_loss": 9.068648338317871, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.08833148330450058, + "rewards/margins": 0.13308466970920563, + "rewards/rejected": -0.2214161604642868, + "sft_loss": 0.005574858281761408, + "step": 4800 + }, + { + "epoch": 0.37417347335667056, + "grad_norm": 1.7159769535064697, + "learning_rate": 3.4627027360800914e-05, + "logits/chosen": -0.22343473136425018, + "logits/rejected": -1.1393592357635498, + "logps/chosen": -0.8822799921035767, + "logps/rejected": -3.1561760902404785, + "loss": 0.9097, + "num_input_tokens_seen": 6276928, + "odds_ratio_loss": 8.91008186340332, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.08822799474000931, + "rewards/margins": 0.2273896187543869, + "rewards/rejected": -0.3156176209449768, + "sft_loss": 0.018716171383857727, + "step": 4810 + }, + { + "epoch": 0.3749513807856865, + "grad_norm": 2.7350761890411377, + "learning_rate": 3.457061357230922e-05, + "logits/chosen": 0.11034146696329117, + "logits/rejected": -1.421015739440918, + "logps/chosen": -0.9038509130477905, + "logps/rejected": -1.2649097442626953, + "loss": 0.9624, + "num_input_tokens_seen": 6293984, + "odds_ratio_loss": 9.54714298248291, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.09038509428501129, + "rewards/margins": 0.03610587492585182, + "rewards/rejected": -0.12649096548557281, + "sft_loss": 0.0076478696428239346, + "step": 4820 + }, + { + "epoch": 0.3757292882147025, + "grad_norm": 4.121143817901611, + "learning_rate": 3.451414262343209e-05, + "logits/chosen": -0.07758097350597382, + "logits/rejected": -1.2049821615219116, + "logps/chosen": -0.9723950624465942, + "logps/rejected": -2.1746673583984375, + "loss": 1.0058, + "num_input_tokens_seen": 6309888, + "odds_ratio_loss": 10.01242733001709, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.09723950177431107, + "rewards/margins": 0.12022723257541656, + "rewards/rejected": -0.21746671199798584, + "sft_loss": 0.004573773592710495, + "step": 4830 + }, + { + "epoch": 0.3765071956437184, + "grad_norm": 3.1243743896484375, + "learning_rate": 3.4457614851441664e-05, + "logits/chosen": -0.06046917289495468, + "logits/rejected": -1.2953927516937256, + "logps/chosen": -0.972058892250061, + "logps/rejected": -2.9306788444519043, + "loss": 1.0167, + "num_input_tokens_seen": 6319312, + "odds_ratio_loss": 10.093836784362793, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.09720588475465775, + "rewards/margins": 0.19586201012134552, + "rewards/rejected": -0.2930678725242615, + "sft_loss": 0.0073084523901343346, + "step": 4840 + }, + { + "epoch": 0.37728510307273433, + "grad_norm": 21.9497127532959, + "learning_rate": 3.440103059394942e-05, + "logits/chosen": -0.3451583981513977, + "logits/rejected": -1.1908886432647705, + "logps/chosen": -1.0274622440338135, + "logps/rejected": -2.5549333095550537, + "loss": 1.0703, + "num_input_tokens_seen": 6331040, + "odds_ratio_loss": 10.037371635437012, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.10274622589349747, + "rewards/margins": 0.15274709463119507, + "rewards/rejected": -0.25549331307411194, + "sft_loss": 0.06652641296386719, + "step": 4850 + }, + { + "epoch": 0.3780630105017503, + "grad_norm": 2.5502912998199463, + "learning_rate": 3.434439018890425e-05, + "logits/chosen": -0.2128363847732544, + "logits/rejected": -1.292725920677185, + "logps/chosen": -1.1600662469863892, + "logps/rejected": -2.37194561958313, + "loss": 1.1907, + "num_input_tokens_seen": 6345360, + "odds_ratio_loss": 11.305562973022461, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.11600663512945175, + "rewards/margins": 0.12118792533874512, + "rewards/rejected": -0.23719453811645508, + "sft_loss": 0.060100257396698, + "step": 4860 + }, + { + "epoch": 0.37884091793076624, + "grad_norm": 2.542813777923584, + "learning_rate": 3.428769397459034e-05, + "logits/chosen": -0.11289496719837189, + "logits/rejected": -1.2843087911605835, + "logps/chosen": -1.1906862258911133, + "logps/rejected": -4.575067043304443, + "loss": 1.2152, + "num_input_tokens_seen": 6358656, + "odds_ratio_loss": 11.52924633026123, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.11906862258911133, + "rewards/margins": 0.3384380340576172, + "rewards/rejected": -0.4575066566467285, + "sft_loss": 0.062293488532304764, + "step": 4870 + }, + { + "epoch": 0.3796188253597822, + "grad_norm": 4.05035924911499, + "learning_rate": 3.423094228962522e-05, + "logits/chosen": -0.1508137285709381, + "logits/rejected": -1.2871469259262085, + "logps/chosen": -1.0064743757247925, + "logps/rejected": -2.946434497833252, + "loss": 1.0262, + "num_input_tokens_seen": 6373184, + "odds_ratio_loss": 10.089973449707031, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1006474494934082, + "rewards/margins": 0.19399599730968475, + "rewards/rejected": -0.29464346170425415, + "sft_loss": 0.017155107110738754, + "step": 4880 + }, + { + "epoch": 0.38039673278879815, + "grad_norm": 0.47668442130088806, + "learning_rate": 3.41741354729577e-05, + "logits/chosen": -0.6911668181419373, + "logits/rejected": -0.9342254400253296, + "logps/chosen": -0.6154254078865051, + "logps/rejected": -2.7076973915100098, + "loss": 0.6428, + "num_input_tokens_seen": 6388032, + "odds_ratio_loss": 6.27427339553833, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.061542533338069916, + "rewards/margins": 0.20922720432281494, + "rewards/rejected": -0.27076974511146545, + "sft_loss": 0.01537760067731142, + "step": 4890 + }, + { + "epoch": 0.38117464021781405, + "grad_norm": 2.141674280166626, + "learning_rate": 3.411727386386589e-05, + "logits/chosen": -0.0400090292096138, + "logits/rejected": -1.7212409973144531, + "logps/chosen": -0.8987911343574524, + "logps/rejected": -1.8310871124267578, + "loss": 0.9343, + "num_input_tokens_seen": 6406128, + "odds_ratio_loss": 9.297147750854492, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.089879110455513, + "rewards/margins": 0.09322960674762726, + "rewards/rejected": -0.18310873210430145, + "sft_loss": 0.004615682177245617, + "step": 4900 + }, + { + "epoch": 0.38195254764683, + "grad_norm": 7.255565166473389, + "learning_rate": 3.406035780195512e-05, + "logits/chosen": -0.299398273229599, + "logits/rejected": -1.0692157745361328, + "logps/chosen": -0.8373295664787292, + "logps/rejected": -1.964667558670044, + "loss": 0.8708, + "num_input_tokens_seen": 6417088, + "odds_ratio_loss": 8.478549003601074, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.0837329626083374, + "rewards/margins": 0.11273376643657684, + "rewards/rejected": -0.19646672904491425, + "sft_loss": 0.02297946810722351, + "step": 4910 + }, + { + "epoch": 0.38273045507584597, + "grad_norm": 1.2640591859817505, + "learning_rate": 3.400338762715595e-05, + "logits/chosen": -0.13642269372940063, + "logits/rejected": -1.1650359630584717, + "logps/chosen": -1.0347317457199097, + "logps/rejected": -3.073350429534912, + "loss": 1.0928, + "num_input_tokens_seen": 6432048, + "odds_ratio_loss": 10.22927474975586, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.10347318649291992, + "rewards/margins": 0.20386183261871338, + "rewards/rejected": -0.3073350489139557, + "sft_loss": 0.06988820433616638, + "step": 4920 + }, + { + "epoch": 0.3835083625048619, + "grad_norm": 2.732785940170288, + "learning_rate": 3.394636367972214e-05, + "logits/chosen": -0.21746814250946045, + "logits/rejected": -0.9812092781066895, + "logps/chosen": -0.8451567888259888, + "logps/rejected": -5.4030632972717285, + "loss": 0.8965, + "num_input_tokens_seen": 6444384, + "odds_ratio_loss": 8.777750015258789, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.08451567590236664, + "rewards/margins": 0.4557906687259674, + "rewards/rejected": -0.5403063297271729, + "sft_loss": 0.018716566264629364, + "step": 4930 + }, + { + "epoch": 0.3842862699338779, + "grad_norm": 1.8364019393920898, + "learning_rate": 3.388928630022858e-05, + "logits/chosen": -0.25838810205459595, + "logits/rejected": -1.060112476348877, + "logps/chosen": -0.9007120132446289, + "logps/rejected": -2.080401659011841, + "loss": 0.9343, + "num_input_tokens_seen": 6458848, + "odds_ratio_loss": 9.283602714538574, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.09007120132446289, + "rewards/margins": 0.11796896159648895, + "rewards/rejected": -0.20804016292095184, + "sft_loss": 0.005900096148252487, + "step": 4940 + }, + { + "epoch": 0.38506417736289383, + "grad_norm": 4.771830081939697, + "learning_rate": 3.383215582956931e-05, + "logits/chosen": -0.3711922764778137, + "logits/rejected": -0.9806662797927856, + "logps/chosen": -0.7544740438461304, + "logps/rejected": -3.114260196685791, + "loss": 0.7843, + "num_input_tokens_seen": 6472848, + "odds_ratio_loss": 7.793927192687988, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.07544741779565811, + "rewards/margins": 0.2359786480665207, + "rewards/rejected": -0.311426043510437, + "sft_loss": 0.004883417394012213, + "step": 4950 + }, + { + "epoch": 0.3858420847919098, + "grad_norm": 2.1738204956054688, + "learning_rate": 3.377497260895545e-05, + "logits/chosen": -0.3498469293117523, + "logits/rejected": -1.3903651237487793, + "logps/chosen": -0.7919679880142212, + "logps/rejected": -3.7652854919433594, + "loss": 0.817, + "num_input_tokens_seen": 6489472, + "odds_ratio_loss": 8.089635848999023, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.07919679582118988, + "rewards/margins": 0.2973317503929138, + "rewards/rejected": -0.3765285611152649, + "sft_loss": 0.008001464419066906, + "step": 4960 + }, + { + "epoch": 0.3866199922209257, + "grad_norm": 3.357419013977051, + "learning_rate": 3.371773697991315e-05, + "logits/chosen": -0.10142382234334946, + "logits/rejected": -1.1423218250274658, + "logps/chosen": -0.9958864450454712, + "logps/rejected": -2.3799166679382324, + "loss": 1.0363, + "num_input_tokens_seen": 6504976, + "odds_ratio_loss": 10.3013277053833, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.09958864003419876, + "rewards/margins": 0.1384030282497406, + "rewards/rejected": -0.23799166083335876, + "sft_loss": 0.006184398662298918, + "step": 4970 + }, + { + "epoch": 0.38739789964994165, + "grad_norm": 2.800766944885254, + "learning_rate": 3.366044928428158e-05, + "logits/chosen": -0.24856014549732208, + "logits/rejected": -1.6147916316986084, + "logps/chosen": -1.0334672927856445, + "logps/rejected": -2.046121835708618, + "loss": 1.0743, + "num_input_tokens_seen": 6516480, + "odds_ratio_loss": 10.621127128601074, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.10334674268960953, + "rewards/margins": 0.10126541554927826, + "rewards/rejected": -0.20461216568946838, + "sft_loss": 0.012207184918224812, + "step": 4980 + }, + { + "epoch": 0.3881758070789576, + "grad_norm": 3.6391632556915283, + "learning_rate": 3.3603109864210885e-05, + "logits/chosen": -0.26488277316093445, + "logits/rejected": -1.168951392173767, + "logps/chosen": -0.7505983114242554, + "logps/rejected": -3.6049633026123047, + "loss": 0.7821, + "num_input_tokens_seen": 6527696, + "odds_ratio_loss": 7.635157585144043, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.07505983859300613, + "rewards/margins": 0.2854365110397339, + "rewards/rejected": -0.3604963719844818, + "sft_loss": 0.018557529896497726, + "step": 4990 + }, + { + "epoch": 0.38895371450797356, + "grad_norm": 10.413762092590332, + "learning_rate": 3.354571906216013e-05, + "logits/chosen": -0.7141574621200562, + "logits/rejected": -0.7490317821502686, + "logps/chosen": -0.8036062121391296, + "logps/rejected": -2.085695266723633, + "loss": 0.8372, + "num_input_tokens_seen": 6539104, + "odds_ratio_loss": 7.791023254394531, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.08036062121391296, + "rewards/margins": 0.12820890545845032, + "rewards/rejected": -0.20856952667236328, + "sft_loss": 0.05809778720140457, + "step": 5000 + }, + { + "epoch": 0.3897316219369895, + "grad_norm": 13.636780738830566, + "learning_rate": 3.3488277220895255e-05, + "logits/chosen": -0.1624249964952469, + "logits/rejected": -1.1657661199569702, + "logps/chosen": -0.842096209526062, + "logps/rejected": -3.2395873069763184, + "loss": 0.8656, + "num_input_tokens_seen": 6551712, + "odds_ratio_loss": 8.55160903930664, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.0842096209526062, + "rewards/margins": 0.23974916338920593, + "rewards/rejected": -0.32395878434181213, + "sft_loss": 0.010446015745401382, + "step": 5010 + }, + { + "epoch": 0.39050952936600547, + "grad_norm": 1.952941656112671, + "learning_rate": 3.343078468348703e-05, + "logits/chosen": -0.17302784323692322, + "logits/rejected": -1.580682635307312, + "logps/chosen": -1.2223979234695435, + "logps/rejected": -2.8610918521881104, + "loss": 1.2445, + "num_input_tokens_seen": 6568784, + "odds_ratio_loss": 11.82412052154541, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.12223976850509644, + "rewards/margins": 0.16386938095092773, + "rewards/rejected": -0.28610917925834656, + "sft_loss": 0.06208343058824539, + "step": 5020 + }, + { + "epoch": 0.39128743679502137, + "grad_norm": 2.657878875732422, + "learning_rate": 3.337324179330901e-05, + "logits/chosen": -0.16119320690631866, + "logits/rejected": -1.2853561639785767, + "logps/chosen": -0.680398166179657, + "logps/rejected": -2.567169666290283, + "loss": 0.7005, + "num_input_tokens_seen": 6583136, + "odds_ratio_loss": 6.976306915283203, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.06803981959819794, + "rewards/margins": 0.18867716193199158, + "rewards/rejected": -0.2567169964313507, + "sft_loss": 0.0028994691092520952, + "step": 5030 + }, + { + "epoch": 0.3920653442240373, + "grad_norm": 1.8662550449371338, + "learning_rate": 3.331564889403547e-05, + "logits/chosen": -0.13829556107521057, + "logits/rejected": -1.6017764806747437, + "logps/chosen": -1.0351202487945557, + "logps/rejected": -3.8283722400665283, + "loss": 1.0596, + "num_input_tokens_seen": 6596496, + "odds_ratio_loss": 10.485677719116211, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.10351201146841049, + "rewards/margins": 0.2793252468109131, + "rewards/rejected": -0.3828372657299042, + "sft_loss": 0.010996273718774319, + "step": 5040 + }, + { + "epoch": 0.3928432516530533, + "grad_norm": 6.15413236618042, + "learning_rate": 3.3258006329639394e-05, + "logits/chosen": -0.13947170972824097, + "logits/rejected": -1.517034888267517, + "logps/chosen": -0.8672447204589844, + "logps/rejected": -1.9955562353134155, + "loss": 0.9209, + "num_input_tokens_seen": 6610672, + "odds_ratio_loss": 9.139081954956055, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.08672447502613068, + "rewards/margins": 0.11283116042613983, + "rewards/rejected": -0.1995556354522705, + "sft_loss": 0.006971079856157303, + "step": 5050 + }, + { + "epoch": 0.39362115908206924, + "grad_norm": 10.247533798217773, + "learning_rate": 3.320031444439036e-05, + "logits/chosen": -0.1946418732404709, + "logits/rejected": -1.4651105403900146, + "logps/chosen": -0.9956845045089722, + "logps/rejected": -3.3512864112854004, + "loss": 1.0244, + "num_input_tokens_seen": 6626704, + "odds_ratio_loss": 10.028584480285645, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.0995684489607811, + "rewards/margins": 0.23556020855903625, + "rewards/rejected": -0.33512866497039795, + "sft_loss": 0.021575702354311943, + "step": 5060 + }, + { + "epoch": 0.3943990665110852, + "grad_norm": 4.458131790161133, + "learning_rate": 3.314257358285253e-05, + "logits/chosen": -0.11686799675226212, + "logits/rejected": -1.1872190237045288, + "logps/chosen": -0.8236813545227051, + "logps/rejected": -4.407199382781982, + "loss": 0.8377, + "num_input_tokens_seen": 6644080, + "odds_ratio_loss": 8.333956718444824, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.0823681429028511, + "rewards/margins": 0.35835182666778564, + "rewards/rejected": -0.44071999192237854, + "sft_loss": 0.004319559782743454, + "step": 5070 + }, + { + "epoch": 0.39517697394010115, + "grad_norm": 4.913539886474609, + "learning_rate": 3.308478408988256e-05, + "logits/chosen": 0.11457274109125137, + "logits/rejected": -1.8358453512191772, + "logps/chosen": -0.9558965563774109, + "logps/rejected": -2.9340109825134277, + "loss": 0.9787, + "num_input_tokens_seen": 6656352, + "odds_ratio_loss": 9.736452102661133, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09558966010808945, + "rewards/margins": 0.19781143963336945, + "rewards/rejected": -0.2934010624885559, + "sft_loss": 0.0050883106887340546, + "step": 5080 + }, + { + "epoch": 0.39595488136911705, + "grad_norm": 2.4029388427734375, + "learning_rate": 3.3026946310627575e-05, + "logits/chosen": -0.24575765430927277, + "logits/rejected": -1.149924397468567, + "logps/chosen": -0.8266646265983582, + "logps/rejected": -2.8694467544555664, + "loss": 0.8558, + "num_input_tokens_seen": 6671216, + "odds_ratio_loss": 8.494571685791016, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.08266647160053253, + "rewards/margins": 0.20427823066711426, + "rewards/rejected": -0.2869446873664856, + "sft_loss": 0.006306462921202183, + "step": 5090 + }, + { + "epoch": 0.396732788798133, + "grad_norm": 2.4077677726745605, + "learning_rate": 3.296906059052308e-05, + "logits/chosen": 0.02295600436627865, + "logits/rejected": -1.56650710105896, + "logps/chosen": -0.9330233335494995, + "logps/rejected": -3.9007136821746826, + "loss": 0.9509, + "num_input_tokens_seen": 6686496, + "odds_ratio_loss": 9.481006622314453, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.09330235421657562, + "rewards/margins": 0.29676908254623413, + "rewards/rejected": -0.39007142186164856, + "sft_loss": 0.002811084035784006, + "step": 5100 + }, + { + "epoch": 0.39751069622714896, + "grad_norm": 6.374454498291016, + "learning_rate": 3.2911127275290923e-05, + "logits/chosen": -0.3088083863258362, + "logits/rejected": -1.3737026453018188, + "logps/chosen": -0.9243408441543579, + "logps/rejected": -4.967630386352539, + "loss": 0.9321, + "num_input_tokens_seen": 6696800, + "odds_ratio_loss": 9.218812942504883, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09243406355381012, + "rewards/margins": 0.40432897210121155, + "rewards/rejected": -0.49676305055618286, + "sft_loss": 0.010226202197372913, + "step": 5110 + }, + { + "epoch": 0.3982886036561649, + "grad_norm": 2.183419704437256, + "learning_rate": 3.285314671093717e-05, + "logits/chosen": -0.22676067054271698, + "logits/rejected": -0.9388538599014282, + "logps/chosen": -0.8574481010437012, + "logps/rejected": -2.431346893310547, + "loss": 0.8923, + "num_input_tokens_seen": 6711424, + "odds_ratio_loss": 8.803481101989746, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.08574481308460236, + "rewards/margins": 0.15738989412784576, + "rewards/rejected": -0.24313469231128693, + "sft_loss": 0.011917488649487495, + "step": 5120 + }, + { + "epoch": 0.3990665110851809, + "grad_norm": 3.2639870643615723, + "learning_rate": 3.279511924375014e-05, + "logits/chosen": -0.2976762056350708, + "logits/rejected": -0.8617954254150391, + "logps/chosen": -0.9393598437309265, + "logps/rejected": -2.5974528789520264, + "loss": 0.9994, + "num_input_tokens_seen": 6721808, + "odds_ratio_loss": 9.516226768493652, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.09393598884344101, + "rewards/margins": 0.16580930352210999, + "rewards/rejected": -0.2597452998161316, + "sft_loss": 0.04779388755559921, + "step": 5130 + }, + { + "epoch": 0.39984441851419683, + "grad_norm": 3.7727150917053223, + "learning_rate": 3.273704522029824e-05, + "logits/chosen": -0.0710880383849144, + "logits/rejected": -1.1556459665298462, + "logps/chosen": -0.9698354601860046, + "logps/rejected": -2.05775785446167, + "loss": 0.9936, + "num_input_tokens_seen": 6738656, + "odds_ratio_loss": 9.846353530883789, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09698355197906494, + "rewards/margins": 0.1087922677397728, + "rewards/rejected": -0.20577578246593475, + "sft_loss": 0.009006194770336151, + "step": 5140 + }, + { + "epoch": 0.40062232594321273, + "grad_norm": 2.765150547027588, + "learning_rate": 3.2678924987427935e-05, + "logits/chosen": -0.22569546103477478, + "logits/rejected": -1.0096803903579712, + "logps/chosen": -1.295796513557434, + "logps/rejected": -4.616843223571777, + "loss": 1.35, + "num_input_tokens_seen": 6748608, + "odds_ratio_loss": 11.873702049255371, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.12957964837551117, + "rewards/margins": 0.3321046829223633, + "rewards/rejected": -0.46168431639671326, + "sft_loss": 0.1626376509666443, + "step": 5150 + }, + { + "epoch": 0.4014002333722287, + "grad_norm": 3.987420082092285, + "learning_rate": 3.2620758892261684e-05, + "logits/chosen": -0.13436201214790344, + "logits/rejected": -1.4836794137954712, + "logps/chosen": -1.3089988231658936, + "logps/rejected": -6.551393985748291, + "loss": 1.3293, + "num_input_tokens_seen": 6759200, + "odds_ratio_loss": 13.168505668640137, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.13089989125728607, + "rewards/margins": 0.5242395401000977, + "rewards/rejected": -0.6551393866539001, + "sft_loss": 0.012489492073655128, + "step": 5160 + }, + { + "epoch": 0.40217814080124464, + "grad_norm": 1.4179848432540894, + "learning_rate": 3.2562547282195864e-05, + "logits/chosen": -0.013448494486510754, + "logits/rejected": -1.2190428972244263, + "logps/chosen": -0.8156849145889282, + "logps/rejected": -3.563967227935791, + "loss": 0.8285, + "num_input_tokens_seen": 6774192, + "odds_ratio_loss": 8.089681625366211, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.08156848698854446, + "rewards/margins": 0.2748281955718994, + "rewards/rejected": -0.35639673471450806, + "sft_loss": 0.019520580768585205, + "step": 5170 + }, + { + "epoch": 0.4029560482302606, + "grad_norm": 3.3582563400268555, + "learning_rate": 3.250429050489868e-05, + "logits/chosen": -0.08996443450450897, + "logits/rejected": -1.5054981708526611, + "logps/chosen": -0.9165959358215332, + "logps/rejected": -2.1653242111206055, + "loss": 0.9387, + "num_input_tokens_seen": 6785632, + "odds_ratio_loss": 9.320246696472168, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09165959060192108, + "rewards/margins": 0.12487280368804932, + "rewards/rejected": -0.2165323942899704, + "sft_loss": 0.006640143692493439, + "step": 5180 + }, + { + "epoch": 0.40373395565927656, + "grad_norm": 2.3919262886047363, + "learning_rate": 3.244598890830811e-05, + "logits/chosen": 0.04672724008560181, + "logits/rejected": -1.244020700454712, + "logps/chosen": -1.1241716146469116, + "logps/rejected": -3.711254835128784, + "loss": 1.1392, + "num_input_tokens_seen": 6796816, + "odds_ratio_loss": 11.346402168273926, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.11241716146469116, + "rewards/margins": 0.25870823860168457, + "rewards/rejected": -0.3711254298686981, + "sft_loss": 0.004553156904876232, + "step": 5190 + }, + { + "epoch": 0.4045118630882925, + "grad_norm": 3.7833569049835205, + "learning_rate": 3.2387642840629795e-05, + "logits/chosen": -0.17067503929138184, + "logits/rejected": -1.752317190170288, + "logps/chosen": -0.9981927871704102, + "logps/rejected": -4.065258502960205, + "loss": 1.0181, + "num_input_tokens_seen": 6807600, + "odds_ratio_loss": 10.092321395874023, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.09981928765773773, + "rewards/margins": 0.3067065477371216, + "rewards/rejected": -0.40652579069137573, + "sft_loss": 0.008861953392624855, + "step": 5200 + }, + { + "epoch": 0.40528977051730847, + "grad_norm": 2.497227191925049, + "learning_rate": 3.232925265033499e-05, + "logits/chosen": -0.17980130016803741, + "logits/rejected": -1.2274338006973267, + "logps/chosen": -1.2723392248153687, + "logps/rejected": -2.9400558471679688, + "loss": 1.2985, + "num_input_tokens_seen": 6818256, + "odds_ratio_loss": 12.26331901550293, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.12723393738269806, + "rewards/margins": 0.16677165031433105, + "rewards/rejected": -0.2940055727958679, + "sft_loss": 0.07213227450847626, + "step": 5210 + }, + { + "epoch": 0.40606767794632437, + "grad_norm": 4.260927200317383, + "learning_rate": 3.227081868615847e-05, + "logits/chosen": -0.17838871479034424, + "logits/rejected": -1.1781151294708252, + "logps/chosen": -0.8753217458724976, + "logps/rejected": -3.9808316230773926, + "loss": 0.9, + "num_input_tokens_seen": 6827504, + "odds_ratio_loss": 8.838985443115234, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.0875321626663208, + "rewards/margins": 0.3105510175228119, + "rewards/rejected": -0.3980831503868103, + "sft_loss": 0.016073519363999367, + "step": 5220 + }, + { + "epoch": 0.4068455853753403, + "grad_norm": 7.187779903411865, + "learning_rate": 3.221234129709645e-05, + "logits/chosen": -0.019155466929078102, + "logits/rejected": -1.0761711597442627, + "logps/chosen": -0.9712806940078735, + "logps/rejected": -3.239480495452881, + "loss": 0.9842, + "num_input_tokens_seen": 6838800, + "odds_ratio_loss": 9.74677848815918, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09712807089090347, + "rewards/margins": 0.2268199920654297, + "rewards/rejected": -0.32394805550575256, + "sft_loss": 0.00948383565992117, + "step": 5230 + }, + { + "epoch": 0.4076234928043563, + "grad_norm": 7.241087436676025, + "learning_rate": 3.215382083240451e-05, + "logits/chosen": -0.1222321018576622, + "logits/rejected": -1.066963791847229, + "logps/chosen": -1.1054328680038452, + "logps/rejected": -7.408546447753906, + "loss": 1.1193, + "num_input_tokens_seen": 6850048, + "odds_ratio_loss": 11.099000930786133, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.11054328829050064, + "rewards/margins": 0.6303113102912903, + "rewards/rejected": -0.7408546209335327, + "sft_loss": 0.009413559921085835, + "step": 5240 + }, + { + "epoch": 0.40840140023337224, + "grad_norm": 2.376088857650757, + "learning_rate": 3.209525764159548e-05, + "logits/chosen": -0.11934573948383331, + "logits/rejected": -1.205902338027954, + "logps/chosen": -0.6773741245269775, + "logps/rejected": -2.1595306396484375, + "loss": 0.702, + "num_input_tokens_seen": 6863520, + "odds_ratio_loss": 6.923120021820068, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.06773741543292999, + "rewards/margins": 0.148215651512146, + "rewards/rejected": -0.215953066945076, + "sft_loss": 0.009694124571979046, + "step": 5250 + }, + { + "epoch": 0.4091793076623882, + "grad_norm": 2.091048240661621, + "learning_rate": 3.203665207443738e-05, + "logits/chosen": -0.08640505373477936, + "logits/rejected": -1.4066154956817627, + "logps/chosen": -0.7765898704528809, + "logps/rejected": -5.977321624755859, + "loss": 0.788, + "num_input_tokens_seen": 6873776, + "odds_ratio_loss": 7.773428916931152, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.0776589885354042, + "rewards/margins": 0.5200731754302979, + "rewards/rejected": -0.5977321267127991, + "sft_loss": 0.010621096938848495, + "step": 5260 + }, + { + "epoch": 0.40995721509140415, + "grad_norm": 3.409365177154541, + "learning_rate": 3.197800448095133e-05, + "logits/chosen": -0.14782679080963135, + "logits/rejected": -1.2552828788757324, + "logps/chosen": -0.9665555953979492, + "logps/rejected": -6.867901802062988, + "loss": 0.9847, + "num_input_tokens_seen": 6890400, + "odds_ratio_loss": 9.714597702026367, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09665556252002716, + "rewards/margins": 0.5901346206665039, + "rewards/rejected": -0.6867901682853699, + "sft_loss": 0.013249441981315613, + "step": 5270 + }, + { + "epoch": 0.41073512252042005, + "grad_norm": 2.304940938949585, + "learning_rate": 3.191931521140943e-05, + "logits/chosen": -0.18876931071281433, + "logits/rejected": -1.5803499221801758, + "logps/chosen": -1.3756680488586426, + "logps/rejected": -6.449460029602051, + "loss": 1.4162, + "num_input_tokens_seen": 6906960, + "odds_ratio_loss": 14.041711807250977, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.13756680488586426, + "rewards/margins": 0.5073791742324829, + "rewards/rejected": -0.6449459791183472, + "sft_loss": 0.012073583900928497, + "step": 5280 + }, + { + "epoch": 0.411513029949436, + "grad_norm": 10.533405303955078, + "learning_rate": 3.186058461633271e-05, + "logits/chosen": -0.13900762796401978, + "logits/rejected": -1.0330493450164795, + "logps/chosen": -0.9527713656425476, + "logps/rejected": -3.1085875034332275, + "loss": 0.9869, + "num_input_tokens_seen": 6917440, + "odds_ratio_loss": 9.745183944702148, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.09527714550495148, + "rewards/margins": 0.21558161079883575, + "rewards/rejected": -0.31085875630378723, + "sft_loss": 0.012343376874923706, + "step": 5290 + }, + { + "epoch": 0.41229093737845196, + "grad_norm": 11.712982177734375, + "learning_rate": 3.180181304648902e-05, + "logits/chosen": -0.20885948836803436, + "logits/rejected": -0.5603643655776978, + "logps/chosen": -0.7855948805809021, + "logps/rejected": -1.346688985824585, + "loss": 0.8738, + "num_input_tokens_seen": 6925824, + "odds_ratio_loss": 8.597783088684082, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.07855948805809021, + "rewards/margins": 0.056109391152858734, + "rewards/rejected": -0.13466887176036835, + "sft_loss": 0.01399481762200594, + "step": 5300 + }, + { + "epoch": 0.4130688448074679, + "grad_norm": 2.2742912769317627, + "learning_rate": 3.17430008528909e-05, + "logits/chosen": -0.27529218792915344, + "logits/rejected": -1.3195812702178955, + "logps/chosen": -0.8954521417617798, + "logps/rejected": -2.502065896987915, + "loss": 0.9261, + "num_input_tokens_seen": 6937024, + "odds_ratio_loss": 9.151135444641113, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.08954521268606186, + "rewards/margins": 0.16066136956214905, + "rewards/rejected": -0.2502065598964691, + "sft_loss": 0.011016755364835262, + "step": 5310 + }, + { + "epoch": 0.4138467522364839, + "grad_norm": 3.655043125152588, + "learning_rate": 3.168414838679356e-05, + "logits/chosen": -0.16698357462882996, + "logits/rejected": -1.3797894716262817, + "logps/chosen": -0.9265694618225098, + "logps/rejected": -3.8845913410186768, + "loss": 0.962, + "num_input_tokens_seen": 6949456, + "odds_ratio_loss": 9.402085304260254, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.0926569476723671, + "rewards/margins": 0.29580220580101013, + "rewards/rejected": -0.388459175825119, + "sft_loss": 0.02179957926273346, + "step": 5320 + }, + { + "epoch": 0.41462465966549983, + "grad_norm": 4.217288017272949, + "learning_rate": 3.1625255999692707e-05, + "logits/chosen": -0.21513691544532776, + "logits/rejected": -1.0598417520523071, + "logps/chosen": -1.184991478919983, + "logps/rejected": -3.253505229949951, + "loss": 1.2351, + "num_input_tokens_seen": 6965536, + "odds_ratio_loss": 12.20081615447998, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.11849913746118546, + "rewards/margins": 0.20685140788555145, + "rewards/rejected": -0.32535049319267273, + "sft_loss": 0.014983227476477623, + "step": 5330 + }, + { + "epoch": 0.41540256709451573, + "grad_norm": 115.32563018798828, + "learning_rate": 3.156632404332247e-05, + "logits/chosen": -0.0027507960330694914, + "logits/rejected": -1.1567466259002686, + "logps/chosen": -1.2281917333602905, + "logps/rejected": -4.739614009857178, + "loss": 1.2494, + "num_input_tokens_seen": 6982160, + "odds_ratio_loss": 11.43757152557373, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.12281918525695801, + "rewards/margins": 0.3511422574520111, + "rewards/rejected": -0.4739614427089691, + "sft_loss": 0.10563137382268906, + "step": 5340 + }, + { + "epoch": 0.4161804745235317, + "grad_norm": 5.8340163230896, + "learning_rate": 3.1507352869653326e-05, + "logits/chosen": -0.37204450368881226, + "logits/rejected": -0.7582011222839355, + "logps/chosen": -0.8406723737716675, + "logps/rejected": -2.9044442176818848, + "loss": 0.8583, + "num_input_tokens_seen": 7000208, + "odds_ratio_loss": 8.48363971710205, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.08406723290681839, + "rewards/margins": 0.20637719333171844, + "rewards/rejected": -0.29044443368911743, + "sft_loss": 0.009952530264854431, + "step": 5350 + }, + { + "epoch": 0.41695838195254764, + "grad_norm": 6.446136951446533, + "learning_rate": 3.144834283088997e-05, + "logits/chosen": -0.38958272337913513, + "logits/rejected": -0.5921080708503723, + "logps/chosen": -0.6633752584457397, + "logps/rejected": -2.818526268005371, + "loss": 0.6832, + "num_input_tokens_seen": 7015120, + "odds_ratio_loss": 6.703489780426025, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.06633754074573517, + "rewards/margins": 0.21551513671875, + "rewards/rejected": -0.2818526327610016, + "sft_loss": 0.012843889184296131, + "step": 5360 + }, + { + "epoch": 0.4177362893815636, + "grad_norm": 2.075299024581909, + "learning_rate": 3.1389294279469215e-05, + "logits/chosen": -0.20397326350212097, + "logits/rejected": -1.4514131546020508, + "logps/chosen": -0.9150875806808472, + "logps/rejected": -4.986923694610596, + "loss": 0.9423, + "num_input_tokens_seen": 7028880, + "odds_ratio_loss": 9.248687744140625, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.09150876104831696, + "rewards/margins": 0.4071836471557617, + "rewards/rejected": -0.49869242310523987, + "sft_loss": 0.017469432204961777, + "step": 5370 + }, + { + "epoch": 0.41851419681057955, + "grad_norm": 7.3845295906066895, + "learning_rate": 3.133020756805789e-05, + "logits/chosen": -0.1727880984544754, + "logits/rejected": -0.7701871395111084, + "logps/chosen": -0.9352876543998718, + "logps/rejected": -1.9459960460662842, + "loss": 0.984, + "num_input_tokens_seen": 7038720, + "odds_ratio_loss": 9.56579303741455, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.09352876245975494, + "rewards/margins": 0.1010708436369896, + "rewards/rejected": -0.19459959864616394, + "sft_loss": 0.027424853295087814, + "step": 5380 + }, + { + "epoch": 0.4192921042395955, + "grad_norm": 1.9074008464813232, + "learning_rate": 3.1271083049550736e-05, + "logits/chosen": 0.10264843702316284, + "logits/rejected": -1.5509620904922485, + "logps/chosen": -0.8807462453842163, + "logps/rejected": -4.2532196044921875, + "loss": 0.9056, + "num_input_tokens_seen": 7052992, + "odds_ratio_loss": 9.034521102905273, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.08807463198900223, + "rewards/margins": 0.337247371673584, + "rewards/rejected": -0.4253219962120056, + "sft_loss": 0.0021968758665025234, + "step": 5390 + }, + { + "epoch": 0.4200700116686114, + "grad_norm": 1.974965214729309, + "learning_rate": 3.12119210770683e-05, + "logits/chosen": -0.13084354996681213, + "logits/rejected": -1.0727672576904297, + "logps/chosen": -0.8137876391410828, + "logps/rejected": -3.66515851020813, + "loss": 0.837, + "num_input_tokens_seen": 7071088, + "odds_ratio_loss": 8.31655216217041, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.08137877285480499, + "rewards/margins": 0.2851371169090271, + "rewards/rejected": -0.3665158450603485, + "sft_loss": 0.005319107323884964, + "step": 5400 + }, + { + "epoch": 0.42084791909762737, + "grad_norm": 3.967224359512329, + "learning_rate": 3.1152722003954815e-05, + "logits/chosen": -0.14671170711517334, + "logits/rejected": -0.9043130874633789, + "logps/chosen": -0.9685951471328735, + "logps/rejected": -2.1605186462402344, + "loss": 1.0268, + "num_input_tokens_seen": 7081392, + "odds_ratio_loss": 10.207342147827148, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.09685951471328735, + "rewards/margins": 0.11919237673282623, + "rewards/rejected": -0.2160518616437912, + "sft_loss": 0.006040588952600956, + "step": 5410 + }, + { + "epoch": 0.4216258265266433, + "grad_norm": 4.425365447998047, + "learning_rate": 3.10934861837761e-05, + "logits/chosen": -0.38159123063087463, + "logits/rejected": -1.5197417736053467, + "logps/chosen": -0.9468908309936523, + "logps/rejected": -5.209157466888428, + "loss": 0.9555, + "num_input_tokens_seen": 7093248, + "odds_ratio_loss": 9.486285209655762, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09468908607959747, + "rewards/margins": 0.4262266159057617, + "rewards/rejected": -0.520915687084198, + "sft_loss": 0.006851466838270426, + "step": 5420 + }, + { + "epoch": 0.4224037339556593, + "grad_norm": 30.725133895874023, + "learning_rate": 3.103421397031745e-05, + "logits/chosen": -0.34320956468582153, + "logits/rejected": -1.181138038635254, + "logps/chosen": -1.0038588047027588, + "logps/rejected": -3.035968065261841, + "loss": 1.0209, + "num_input_tokens_seen": 7112240, + "odds_ratio_loss": 9.553422927856445, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.1003858894109726, + "rewards/margins": 0.20321090519428253, + "rewards/rejected": -0.3035968244075775, + "sft_loss": 0.06559192389249802, + "step": 5430 + }, + { + "epoch": 0.42318164138467523, + "grad_norm": 2.8233392238616943, + "learning_rate": 3.097490571758151e-05, + "logits/chosen": -0.029369115829467773, + "logits/rejected": -1.5265640020370483, + "logps/chosen": -0.7838415503501892, + "logps/rejected": -4.505312919616699, + "loss": 0.7893, + "num_input_tokens_seen": 7131280, + "odds_ratio_loss": 7.869359016418457, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.0783841460943222, + "rewards/margins": 0.37214717268943787, + "rewards/rejected": -0.45053133368492126, + "sft_loss": 0.0024120414163917303, + "step": 5440 + }, + { + "epoch": 0.4239595488136912, + "grad_norm": 10.336220741271973, + "learning_rate": 3.091556177978619e-05, + "logits/chosen": -0.2264358550310135, + "logits/rejected": -1.415867805480957, + "logps/chosen": -0.71709144115448, + "logps/rejected": -7.780030727386475, + "loss": 0.7227, + "num_input_tokens_seen": 7145920, + "odds_ratio_loss": 7.129467010498047, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.07170914113521576, + "rewards/margins": 0.706294059753418, + "rewards/rejected": -0.7780030965805054, + "sft_loss": 0.009785325266420841, + "step": 5450 + }, + { + "epoch": 0.42473745624270715, + "grad_norm": 1.6062244176864624, + "learning_rate": 3.08561825113625e-05, + "logits/chosen": -0.037660397589206696, + "logits/rejected": -1.3419077396392822, + "logps/chosen": -0.9172475934028625, + "logps/rejected": -2.8567147254943848, + "loss": 0.9378, + "num_input_tokens_seen": 7161216, + "odds_ratio_loss": 9.303727149963379, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09172476828098297, + "rewards/margins": 0.1939467191696167, + "rewards/rejected": -0.28567150235176086, + "sft_loss": 0.007443229667842388, + "step": 5460 + }, + { + "epoch": 0.42551536367172305, + "grad_norm": 6.955587387084961, + "learning_rate": 3.079676826695248e-05, + "logits/chosen": -0.2916269302368164, + "logits/rejected": -0.9078566431999207, + "logps/chosen": -0.8469215631484985, + "logps/rejected": -4.218659400939941, + "loss": 0.8656, + "num_input_tokens_seen": 7172848, + "odds_ratio_loss": 8.542197227478027, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.08469214290380478, + "rewards/margins": 0.3371737599372864, + "rewards/rejected": -0.42186588048934937, + "sft_loss": 0.011388765648007393, + "step": 5470 + }, + { + "epoch": 0.426293271100739, + "grad_norm": 6.72831392288208, + "learning_rate": 3.0737319401407037e-05, + "logits/chosen": -0.1381830871105194, + "logits/rejected": -0.9215238690376282, + "logps/chosen": -0.926182746887207, + "logps/rejected": -2.7554633617401123, + "loss": 0.9704, + "num_input_tokens_seen": 7187648, + "odds_ratio_loss": 9.581047058105469, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.09261827915906906, + "rewards/margins": 0.18292807042598724, + "rewards/rejected": -0.2755463719367981, + "sft_loss": 0.01227156538516283, + "step": 5480 + }, + { + "epoch": 0.42707117852975496, + "grad_norm": 1.813859224319458, + "learning_rate": 3.0677836269783895e-05, + "logits/chosen": -0.4149433672428131, + "logits/rejected": -1.3414149284362793, + "logps/chosen": -1.15546715259552, + "logps/rejected": -4.314212799072266, + "loss": 1.1696, + "num_input_tokens_seen": 7200368, + "odds_ratio_loss": 11.48894214630127, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.11554670333862305, + "rewards/margins": 0.3158745765686035, + "rewards/rejected": -0.4314212203025818, + "sft_loss": 0.020742299035191536, + "step": 5490 + }, + { + "epoch": 0.4278490859587709, + "grad_norm": 2.206411361694336, + "learning_rate": 3.061831922734541e-05, + "logits/chosen": -0.4122841954231262, + "logits/rejected": -1.3862360715866089, + "logps/chosen": -0.9372841715812683, + "logps/rejected": -3.5864288806915283, + "loss": 0.9635, + "num_input_tokens_seen": 7211472, + "odds_ratio_loss": 9.549125671386719, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.09372840821743011, + "rewards/margins": 0.26491445302963257, + "rewards/rejected": -0.3586428761482239, + "sft_loss": 0.008612281642854214, + "step": 5500 + }, + { + "epoch": 0.42862699338778687, + "grad_norm": 7.141944885253906, + "learning_rate": 3.055876862955647e-05, + "logits/chosen": 0.02164841815829277, + "logits/rejected": -1.631679892539978, + "logps/chosen": -0.8621139526367188, + "logps/rejected": -6.096848487854004, + "loss": 0.8666, + "num_input_tokens_seen": 7222832, + "odds_ratio_loss": 8.421870231628418, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.08621139079332352, + "rewards/margins": 0.5234734416007996, + "rewards/rejected": -0.6096848249435425, + "sft_loss": 0.024400796741247177, + "step": 5510 + }, + { + "epoch": 0.4294049008168028, + "grad_norm": 2.589207410812378, + "learning_rate": 3.0499184832082362e-05, + "logits/chosen": -0.3766138255596161, + "logits/rejected": -1.7519257068634033, + "logps/chosen": -0.8969801664352417, + "logps/rejected": -3.5163254737854004, + "loss": 0.9139, + "num_input_tokens_seen": 7232992, + "odds_ratio_loss": 8.830495834350586, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.08969802409410477, + "rewards/margins": 0.2619345486164093, + "rewards/rejected": -0.3516325354576111, + "sft_loss": 0.030821654945611954, + "step": 5520 + }, + { + "epoch": 0.4301828082458187, + "grad_norm": 2.4507274627685547, + "learning_rate": 3.0439568190786682e-05, + "logits/chosen": -0.02348879538476467, + "logits/rejected": -1.166263461112976, + "logps/chosen": -0.8205503225326538, + "logps/rejected": -5.710233688354492, + "loss": 0.8556, + "num_input_tokens_seen": 7241760, + "odds_ratio_loss": 8.516347885131836, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.08205503225326538, + "rewards/margins": 0.4889683723449707, + "rewards/rejected": -0.5710233449935913, + "sft_loss": 0.003920835442841053, + "step": 5530 + }, + { + "epoch": 0.4309607156748347, + "grad_norm": 2.328852891921997, + "learning_rate": 3.0379919061729168e-05, + "logits/chosen": 0.01200108602643013, + "logits/rejected": -1.959764838218689, + "logps/chosen": -0.8829509019851685, + "logps/rejected": -3.345860004425049, + "loss": 0.8981, + "num_input_tokens_seen": 7254960, + "odds_ratio_loss": 8.910298347473145, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.08829507976770401, + "rewards/margins": 0.2462908923625946, + "rewards/rejected": -0.3345859944820404, + "sft_loss": 0.007096708752214909, + "step": 5540 + }, + { + "epoch": 0.43173862310385064, + "grad_norm": 9.930980682373047, + "learning_rate": 3.032023780116358e-05, + "logits/chosen": -0.35067591071128845, + "logits/rejected": -0.4748639166355133, + "logps/chosen": -0.743221640586853, + "logps/rejected": -5.461892127990723, + "loss": 0.7485, + "num_input_tokens_seen": 7273088, + "odds_ratio_loss": 7.39886999130249, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.0743221715092659, + "rewards/margins": 0.47186699509620667, + "rewards/rejected": -0.5461891889572144, + "sft_loss": 0.008651467971503735, + "step": 5550 + }, + { + "epoch": 0.4325165305328666, + "grad_norm": 1.6604907512664795, + "learning_rate": 3.026052476553561e-05, + "logits/chosen": 0.056398797780275345, + "logits/rejected": -1.9943794012069702, + "logps/chosen": -0.8136641383171082, + "logps/rejected": -6.4685492515563965, + "loss": 0.8194, + "num_input_tokens_seen": 7286160, + "odds_ratio_loss": 8.159880638122559, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.08136641979217529, + "rewards/margins": 0.5654885172843933, + "rewards/rejected": -0.6468549966812134, + "sft_loss": 0.0034152872394770384, + "step": 5560 + }, + { + "epoch": 0.43329443796188255, + "grad_norm": 17.088098526000977, + "learning_rate": 3.0200780311480716e-05, + "logits/chosen": -0.6167064309120178, + "logits/rejected": -1.160718321800232, + "logps/chosen": -1.155081868171692, + "logps/rejected": -6.337732315063477, + "loss": 1.1695, + "num_input_tokens_seen": 7297920, + "odds_ratio_loss": 11.395452499389648, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.11550818383693695, + "rewards/margins": 0.5182650685310364, + "rewards/rejected": -0.6337732076644897, + "sft_loss": 0.029999535530805588, + "step": 5570 + }, + { + "epoch": 0.4340723453908985, + "grad_norm": 3.5572893619537354, + "learning_rate": 3.0141004795821992e-05, + "logits/chosen": -0.5635565519332886, + "logits/rejected": -0.6509861350059509, + "logps/chosen": -1.1619627475738525, + "logps/rejected": -8.93120002746582, + "loss": 1.1741, + "num_input_tokens_seen": 7311472, + "odds_ratio_loss": 11.183347702026367, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.11619627475738525, + "rewards/margins": 0.7769237756729126, + "rewards/rejected": -0.8931199312210083, + "sft_loss": 0.05573863908648491, + "step": 5580 + }, + { + "epoch": 0.4348502528199144, + "grad_norm": 2.9649763107299805, + "learning_rate": 3.008119857556807e-05, + "logits/chosen": -0.1862354576587677, + "logits/rejected": -1.009856939315796, + "logps/chosen": -0.8901270627975464, + "logps/rejected": -2.387753963470459, + "loss": 0.9199, + "num_input_tokens_seen": 7322080, + "odds_ratio_loss": 8.999714851379395, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.08901270478963852, + "rewards/margins": 0.14976267516613007, + "rewards/rejected": -0.23877540230751038, + "sft_loss": 0.019966933876276016, + "step": 5590 + }, + { + "epoch": 0.43562816024893036, + "grad_norm": 6.203644752502441, + "learning_rate": 3.002136200791093e-05, + "logits/chosen": -0.3672073185443878, + "logits/rejected": -1.2089344263076782, + "logps/chosen": -1.1483100652694702, + "logps/rejected": -6.217892646789551, + "loss": 1.1625, + "num_input_tokens_seen": 7337232, + "odds_ratio_loss": 10.810770034790039, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.11483101546764374, + "rewards/margins": 0.5069582462310791, + "rewards/rejected": -0.6217892169952393, + "sft_loss": 0.08141560107469559, + "step": 5600 + }, + { + "epoch": 0.4364060676779463, + "grad_norm": 1.8932578563690186, + "learning_rate": 2.996149545022383e-05, + "logits/chosen": -0.06057189032435417, + "logits/rejected": -1.631291389465332, + "logps/chosen": -0.9629873037338257, + "logps/rejected": -7.250810146331787, + "loss": 0.9769, + "num_input_tokens_seen": 7347408, + "odds_ratio_loss": 9.381027221679688, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.09629872441291809, + "rewards/margins": 0.628782331943512, + "rewards/rejected": -0.7250810265541077, + "sft_loss": 0.03883747756481171, + "step": 5610 + }, + { + "epoch": 0.4371839751069623, + "grad_norm": 25.1219425201416, + "learning_rate": 2.990159926005913e-05, + "logits/chosen": -0.252627432346344, + "logits/rejected": -1.3857918977737427, + "logps/chosen": -1.0234429836273193, + "logps/rejected": -6.393345832824707, + "loss": 1.0359, + "num_input_tokens_seen": 7362448, + "odds_ratio_loss": 9.738905906677246, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.10234431177377701, + "rewards/margins": 0.5369902849197388, + "rewards/rejected": -0.6393346190452576, + "sft_loss": 0.06196347624063492, + "step": 5620 + }, + { + "epoch": 0.43796188253597823, + "grad_norm": 9.728668212890625, + "learning_rate": 2.984167379514618e-05, + "logits/chosen": -0.07025805860757828, + "logits/rejected": -1.309217929840088, + "logps/chosen": -1.034991979598999, + "logps/rejected": -6.303004741668701, + "loss": 1.0562, + "num_input_tokens_seen": 7377168, + "odds_ratio_loss": 10.495607376098633, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.10349918901920319, + "rewards/margins": 0.5268012881278992, + "rewards/rejected": -0.6303004622459412, + "sft_loss": 0.0066757239401340485, + "step": 5630 + }, + { + "epoch": 0.4387397899649942, + "grad_norm": 2.4033548831939697, + "learning_rate": 2.978171941338917e-05, + "logits/chosen": 0.06687622517347336, + "logits/rejected": -1.3250255584716797, + "logps/chosen": -0.8779948949813843, + "logps/rejected": -5.355679512023926, + "loss": 0.9189, + "num_input_tokens_seen": 7391184, + "odds_ratio_loss": 9.139822006225586, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.08779950439929962, + "rewards/margins": 0.4477684497833252, + "rewards/rejected": -0.5355679392814636, + "sft_loss": 0.004900778643786907, + "step": 5640 + }, + { + "epoch": 0.4395176973940101, + "grad_norm": 4.902503490447998, + "learning_rate": 2.972173647286499e-05, + "logits/chosen": -0.16414262354373932, + "logits/rejected": -1.3518102169036865, + "logps/chosen": -1.7802966833114624, + "logps/rejected": -11.64123249053955, + "loss": 1.8715, + "num_input_tokens_seen": 7405376, + "odds_ratio_loss": 18.06220245361328, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.17802968621253967, + "rewards/margins": 0.9860936403274536, + "rewards/rejected": -1.1641231775283813, + "sft_loss": 0.0653175562620163, + "step": 5650 + }, + { + "epoch": 0.44029560482302604, + "grad_norm": 7.490935325622559, + "learning_rate": 2.96617253318211e-05, + "logits/chosen": -0.11388234794139862, + "logits/rejected": -1.323212742805481, + "logps/chosen": -1.8963305950164795, + "logps/rejected": -6.981576442718506, + "loss": 1.9488, + "num_input_tokens_seen": 7419312, + "odds_ratio_loss": 18.875633239746094, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.1896330714225769, + "rewards/margins": 0.5085245966911316, + "rewards/rejected": -0.6981576681137085, + "sft_loss": 0.061241038143634796, + "step": 5660 + }, + { + "epoch": 0.441073512252042, + "grad_norm": 3.349161148071289, + "learning_rate": 2.960168634867338e-05, + "logits/chosen": -0.3609488606452942, + "logits/rejected": -0.9301692247390747, + "logps/chosen": -1.1135082244873047, + "logps/rejected": -6.0065412521362305, + "loss": 1.1235, + "num_input_tokens_seen": 7432704, + "odds_ratio_loss": 10.837892532348633, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.11135081946849823, + "rewards/margins": 0.48930326104164124, + "rewards/rejected": -0.600654125213623, + "sft_loss": 0.039745159447193146, + "step": 5670 + }, + { + "epoch": 0.44185141968105796, + "grad_norm": 7.245255947113037, + "learning_rate": 2.9541619882004007e-05, + "logits/chosen": -0.09249483048915863, + "logits/rejected": -1.1441720724105835, + "logps/chosen": -0.8567533493041992, + "logps/rejected": -4.396549701690674, + "loss": 0.8774, + "num_input_tokens_seen": 7447376, + "odds_ratio_loss": 8.738390922546387, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.08567533642053604, + "rewards/margins": 0.3539796471595764, + "rewards/rejected": -0.43965497612953186, + "sft_loss": 0.003584805177524686, + "step": 5680 + }, + { + "epoch": 0.4426293271100739, + "grad_norm": 1.8555481433868408, + "learning_rate": 2.9481526290559298e-05, + "logits/chosen": -0.10176386684179306, + "logits/rejected": -1.3324403762817383, + "logps/chosen": -1.0124648809432983, + "logps/rejected": -4.5107245445251465, + "loss": 1.026, + "num_input_tokens_seen": 7457856, + "odds_ratio_loss": 9.876537322998047, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.10124649107456207, + "rewards/margins": 0.34982600808143616, + "rewards/rejected": -0.4510725140571594, + "sft_loss": 0.038344431668519974, + "step": 5690 + }, + { + "epoch": 0.44340723453908987, + "grad_norm": 1.75498366355896, + "learning_rate": 2.9421405933247582e-05, + "logits/chosen": -0.010723483748733997, + "logits/rejected": -1.041904091835022, + "logps/chosen": -0.8967369794845581, + "logps/rejected": -5.024321556091309, + "loss": 0.9163, + "num_input_tokens_seen": 7469360, + "odds_ratio_loss": 9.101985931396484, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.089673712849617, + "rewards/margins": 0.4127584397792816, + "rewards/rejected": -0.502432107925415, + "sft_loss": 0.006076131947338581, + "step": 5700 + }, + { + "epoch": 0.44418514196810577, + "grad_norm": 3.833069086074829, + "learning_rate": 2.936125916913704e-05, + "logits/chosen": -0.18692442774772644, + "logits/rejected": -1.2350839376449585, + "logps/chosen": -1.1202170848846436, + "logps/rejected": -4.992281913757324, + "loss": 1.139, + "num_input_tokens_seen": 7478880, + "odds_ratio_loss": 11.292125701904297, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.11202170699834824, + "rewards/margins": 0.38720643520355225, + "rewards/rejected": -0.49922817945480347, + "sft_loss": 0.00977136380970478, + "step": 5710 + }, + { + "epoch": 0.4449630493971217, + "grad_norm": 1.6511999368667603, + "learning_rate": 2.9301086357453557e-05, + "logits/chosen": -0.22126416862010956, + "logits/rejected": -0.9571665525436401, + "logps/chosen": -1.0049254894256592, + "logps/rejected": -3.3358731269836426, + "loss": 1.0214, + "num_input_tokens_seen": 7495696, + "odds_ratio_loss": 10.157580375671387, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.10049253702163696, + "rewards/margins": 0.23309478163719177, + "rewards/rejected": -0.33358731865882874, + "sft_loss": 0.005630288738757372, + "step": 5720 + }, + { + "epoch": 0.4457409568261377, + "grad_norm": 2.4155290126800537, + "learning_rate": 2.9240887857578597e-05, + "logits/chosen": -0.13161590695381165, + "logits/rejected": -0.9564615488052368, + "logps/chosen": -0.7802937626838684, + "logps/rejected": -6.504634857177734, + "loss": 0.7909, + "num_input_tokens_seen": 7507056, + "odds_ratio_loss": 7.859260559082031, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.07802937924861908, + "rewards/margins": 0.57243412733078, + "rewards/rejected": -0.6504634022712708, + "sft_loss": 0.004980637691915035, + "step": 5730 + }, + { + "epoch": 0.44651886425515364, + "grad_norm": 3.0114312171936035, + "learning_rate": 2.918066402904705e-05, + "logits/chosen": 0.14526775479316711, + "logits/rejected": -1.194178581237793, + "logps/chosen": -0.7590327858924866, + "logps/rejected": -11.647214889526367, + "loss": 0.7787, + "num_input_tokens_seen": 7517136, + "odds_ratio_loss": 7.698354244232178, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.0759032815694809, + "rewards/margins": 1.0888183116912842, + "rewards/rejected": -1.1647217273712158, + "sft_loss": 0.008911113254725933, + "step": 5740 + }, + { + "epoch": 0.4472967716841696, + "grad_norm": 2.222205638885498, + "learning_rate": 2.9120415231545066e-05, + "logits/chosen": 0.02403981052339077, + "logits/rejected": -1.470847487449646, + "logps/chosen": -0.9761005640029907, + "logps/rejected": -8.883180618286133, + "loss": 0.9802, + "num_input_tokens_seen": 7530208, + "odds_ratio_loss": 9.752192497253418, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09761004894971848, + "rewards/margins": 0.7907080054283142, + "rewards/rejected": -0.8883180618286133, + "sft_loss": 0.00500735267996788, + "step": 5750 + }, + { + "epoch": 0.44807467911318555, + "grad_norm": 2.7538716793060303, + "learning_rate": 2.906014182490795e-05, + "logits/chosen": -0.13734382390975952, + "logits/rejected": -1.3238089084625244, + "logps/chosen": -0.9427657127380371, + "logps/rejected": -5.268524169921875, + "loss": 0.9545, + "num_input_tokens_seen": 7544272, + "odds_ratio_loss": 9.417902946472168, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.094276562333107, + "rewards/margins": 0.43257585167884827, + "rewards/rejected": -0.5268524289131165, + "sft_loss": 0.01273438148200512, + "step": 5760 + }, + { + "epoch": 0.4488525865422015, + "grad_norm": 2.635197877883911, + "learning_rate": 2.8999844169117958e-05, + "logits/chosen": -0.20401224493980408, + "logits/rejected": -0.8030599355697632, + "logps/chosen": -0.8307873606681824, + "logps/rejected": -4.622495651245117, + "loss": 0.8435, + "num_input_tokens_seen": 7558480, + "odds_ratio_loss": 8.378512382507324, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.08307874202728271, + "rewards/margins": 0.37917080521583557, + "rewards/rejected": -0.4622495770454407, + "sft_loss": 0.0056395018473267555, + "step": 5770 + }, + { + "epoch": 0.4496304939712174, + "grad_norm": 10.95548152923584, + "learning_rate": 2.893952262430219e-05, + "logits/chosen": -0.2868979871273041, + "logits/rejected": -1.515904426574707, + "logps/chosen": -1.033301830291748, + "logps/rejected": -6.378268718719482, + "loss": 1.0583, + "num_input_tokens_seen": 7572112, + "odds_ratio_loss": 9.888237953186035, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.10333018004894257, + "rewards/margins": 0.5344967246055603, + "rewards/rejected": -0.6378269195556641, + "sft_loss": 0.06948982179164886, + "step": 5780 + }, + { + "epoch": 0.45040840140023336, + "grad_norm": 2.2623848915100098, + "learning_rate": 2.8879177550730407e-05, + "logits/chosen": -0.10103366523981094, + "logits/rejected": -1.231601595878601, + "logps/chosen": -0.8214209675788879, + "logps/rejected": -4.604743480682373, + "loss": 0.847, + "num_input_tokens_seen": 7585360, + "odds_ratio_loss": 8.389901161193848, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.08214209973812103, + "rewards/margins": 0.3783322274684906, + "rewards/rejected": -0.4604743421077728, + "sft_loss": 0.007993288338184357, + "step": 5790 + }, + { + "epoch": 0.4511863088292493, + "grad_norm": 2.8834474086761475, + "learning_rate": 2.8818809308812906e-05, + "logits/chosen": -0.21622204780578613, + "logits/rejected": -1.0297644138336182, + "logps/chosen": -0.9987231492996216, + "logps/rejected": -5.402844429016113, + "loss": 1.0204, + "num_input_tokens_seen": 7599648, + "odds_ratio_loss": 10.143254280090332, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.09987231343984604, + "rewards/margins": 0.44041213393211365, + "rewards/rejected": -0.5402845144271851, + "sft_loss": 0.006070182658731937, + "step": 5800 + }, + { + "epoch": 0.45196421625826527, + "grad_norm": 7.1191864013671875, + "learning_rate": 2.875841825909835e-05, + "logits/chosen": -0.04676201939582825, + "logits/rejected": -1.37875235080719, + "logps/chosen": -0.7726394534111023, + "logps/rejected": -5.68399715423584, + "loss": 0.7841, + "num_input_tokens_seen": 7607328, + "odds_ratio_loss": 7.746439456939697, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.07726395130157471, + "rewards/margins": 0.49113577604293823, + "rewards/rejected": -0.5683996677398682, + "sft_loss": 0.009432977065443993, + "step": 5810 + }, + { + "epoch": 0.45274212368728123, + "grad_norm": 2.4421451091766357, + "learning_rate": 2.8698004762271634e-05, + "logits/chosen": -0.13264720141887665, + "logits/rejected": -1.1827833652496338, + "logps/chosen": -1.1583963632583618, + "logps/rejected": -6.693757057189941, + "loss": 1.161, + "num_input_tokens_seen": 7619840, + "odds_ratio_loss": 11.536027908325195, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1158396452665329, + "rewards/margins": 0.553536057472229, + "rewards/rejected": -0.6693757176399231, + "sft_loss": 0.007445620838552713, + "step": 5820 + }, + { + "epoch": 0.4535200311162972, + "grad_norm": 2.106879234313965, + "learning_rate": 2.8637569179151714e-05, + "logits/chosen": -0.2481449842453003, + "logits/rejected": -0.9333834648132324, + "logps/chosen": -0.8990824818611145, + "logps/rejected": -7.192356109619141, + "loss": 0.9073, + "num_input_tokens_seen": 7632912, + "odds_ratio_loss": 8.996527671813965, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.08990823477506638, + "rewards/margins": 0.6293273568153381, + "rewards/rejected": -0.7192355990409851, + "sft_loss": 0.007650734391063452, + "step": 5830 + }, + { + "epoch": 0.4542979385453131, + "grad_norm": 2.554652214050293, + "learning_rate": 2.8577111870689454e-05, + "logits/chosen": -0.3907225728034973, + "logits/rejected": -1.3036072254180908, + "logps/chosen": -0.8883325457572937, + "logps/rejected": -4.797537803649902, + "loss": 0.9031, + "num_input_tokens_seen": 7645040, + "odds_ratio_loss": 8.559049606323242, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.0888332650065422, + "rewards/margins": 0.39092057943344116, + "rewards/rejected": -0.4797538220882416, + "sft_loss": 0.04718649387359619, + "step": 5840 + }, + { + "epoch": 0.45507584597432904, + "grad_norm": 2.30387020111084, + "learning_rate": 2.8516633197965458e-05, + "logits/chosen": -0.20695964992046356, + "logits/rejected": -0.8214203715324402, + "logps/chosen": -1.097795844078064, + "logps/rejected": -4.349630355834961, + "loss": 1.116, + "num_input_tokens_seen": 7659872, + "odds_ratio_loss": 11.061562538146973, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.10977959632873535, + "rewards/margins": 0.3251834511756897, + "rewards/rejected": -0.43496307730674744, + "sft_loss": 0.009882526472210884, + "step": 5850 + }, + { + "epoch": 0.455853753403345, + "grad_norm": 2.312739372253418, + "learning_rate": 2.845613352218795e-05, + "logits/chosen": 0.01900560036301613, + "logits/rejected": -1.2111179828643799, + "logps/chosen": -0.8945795893669128, + "logps/rejected": -6.274838447570801, + "loss": 0.904, + "num_input_tokens_seen": 7675120, + "odds_ratio_loss": 8.90841293334961, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.08945796638727188, + "rewards/margins": 0.5380259156227112, + "rewards/rejected": -0.6274839639663696, + "sft_loss": 0.01311540324240923, + "step": 5860 + }, + { + "epoch": 0.45663166083236095, + "grad_norm": 1.5392405986785889, + "learning_rate": 2.839561320469058e-05, + "logits/chosen": -0.062470972537994385, + "logits/rejected": -1.4860166311264038, + "logps/chosen": -0.806922435760498, + "logps/rejected": -5.0439581871032715, + "loss": 0.8221, + "num_input_tokens_seen": 7690256, + "odds_ratio_loss": 8.170222282409668, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.08069224655628204, + "rewards/margins": 0.4237035810947418, + "rewards/rejected": -0.5043958425521851, + "sft_loss": 0.005118037573993206, + "step": 5870 + }, + { + "epoch": 0.4574095682613769, + "grad_norm": 2.0916812419891357, + "learning_rate": 2.8335072606930287e-05, + "logits/chosen": -0.2488502562046051, + "logits/rejected": -1.1231826543807983, + "logps/chosen": -1.1016508340835571, + "logps/rejected": -6.482033729553223, + "loss": 1.1136, + "num_input_tokens_seen": 7702640, + "odds_ratio_loss": 10.864334106445312, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.11016509681940079, + "rewards/margins": 0.5380383133888245, + "rewards/rejected": -0.6482033729553223, + "sft_loss": 0.027160849422216415, + "step": 5880 + }, + { + "epoch": 0.45818747569039286, + "grad_norm": 2.859052896499634, + "learning_rate": 2.827451209048514e-05, + "logits/chosen": -0.2412732094526291, + "logits/rejected": -1.3144216537475586, + "logps/chosen": -0.961597740650177, + "logps/rejected": -2.7586493492126465, + "loss": 0.996, + "num_input_tokens_seen": 7713392, + "odds_ratio_loss": 9.801130294799805, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.09615977108478546, + "rewards/margins": 0.1797051727771759, + "rewards/rejected": -0.27586492896080017, + "sft_loss": 0.0158519484102726, + "step": 5890 + }, + { + "epoch": 0.45896538311940877, + "grad_norm": 6.613534450531006, + "learning_rate": 2.8213932017052164e-05, + "logits/chosen": -0.15394410490989685, + "logits/rejected": -1.5015206336975098, + "logps/chosen": -0.8291906118392944, + "logps/rejected": -4.777556419372559, + "loss": 0.843, + "num_input_tokens_seen": 7723536, + "odds_ratio_loss": 8.361349105834961, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.08291905373334885, + "rewards/margins": 0.3948366343975067, + "rewards/rejected": -0.4777556359767914, + "sft_loss": 0.006913746241480112, + "step": 5900 + }, + { + "epoch": 0.4597432905484247, + "grad_norm": 2.7631146907806396, + "learning_rate": 2.8153332748445178e-05, + "logits/chosen": -0.13939619064331055, + "logits/rejected": -1.1255842447280884, + "logps/chosen": -0.849032998085022, + "logps/rejected": -3.494724988937378, + "loss": 0.8648, + "num_input_tokens_seen": 7733392, + "odds_ratio_loss": 8.560256004333496, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.08490331470966339, + "rewards/margins": 0.2645691931247711, + "rewards/rejected": -0.3494724929332733, + "sft_loss": 0.008783399127423763, + "step": 5910 + }, + { + "epoch": 0.4605211979774407, + "grad_norm": 7.09406852722168, + "learning_rate": 2.8092714646592654e-05, + "logits/chosen": -0.1654086410999298, + "logits/rejected": -1.4517545700073242, + "logps/chosen": -0.8999756574630737, + "logps/rejected": -4.131761074066162, + "loss": 0.9214, + "num_input_tokens_seen": 7745856, + "odds_ratio_loss": 9.104066848754883, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.08999757468700409, + "rewards/margins": 0.32317858934402466, + "rewards/rejected": -0.41317611932754517, + "sft_loss": 0.01101828645914793, + "step": 5920 + }, + { + "epoch": 0.46129910540645663, + "grad_norm": 2.690000295639038, + "learning_rate": 2.803207807353556e-05, + "logits/chosen": -0.2681902348995209, + "logits/rejected": -0.8913981318473816, + "logps/chosen": -0.6731475591659546, + "logps/rejected": -5.323077201843262, + "loss": 0.6876, + "num_input_tokens_seen": 7759568, + "odds_ratio_loss": 6.8018670082092285, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.0673147663474083, + "rewards/margins": 0.46499305963516235, + "rewards/rejected": -0.5323078036308289, + "sft_loss": 0.007376432418823242, + "step": 5930 + }, + { + "epoch": 0.4620770128354726, + "grad_norm": 3.621859073638916, + "learning_rate": 2.7971423391425154e-05, + "logits/chosen": 0.03137628361582756, + "logits/rejected": -1.338469386100769, + "logps/chosen": -1.0437556505203247, + "logps/rejected": -8.069334983825684, + "loss": 1.0659, + "num_input_tokens_seen": 7774960, + "odds_ratio_loss": 9.991592407226562, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.10437557846307755, + "rewards/margins": 0.7025579214096069, + "rewards/rejected": -0.806933581829071, + "sft_loss": 0.06670835614204407, + "step": 5940 + }, + { + "epoch": 0.46285492026448855, + "grad_norm": 2.3327841758728027, + "learning_rate": 2.7910750962520872e-05, + "logits/chosen": 0.03029645048081875, + "logits/rejected": -1.226078748703003, + "logps/chosen": -0.6837007403373718, + "logps/rejected": -5.380637168884277, + "loss": 0.6976, + "num_input_tokens_seen": 7786784, + "odds_ratio_loss": 6.941661834716797, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.06837008148431778, + "rewards/margins": 0.469693660736084, + "rewards/rejected": -0.5380637645721436, + "sft_loss": 0.003387672360986471, + "step": 5950 + }, + { + "epoch": 0.46363282769350445, + "grad_norm": 90.8195571899414, + "learning_rate": 2.7850061149188146e-05, + "logits/chosen": -0.4019021987915039, + "logits/rejected": -1.172957181930542, + "logps/chosen": -1.0341510772705078, + "logps/rejected": -9.447542190551758, + "loss": 1.0446, + "num_input_tokens_seen": 7799360, + "odds_ratio_loss": 10.295927047729492, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.1034151166677475, + "rewards/margins": 0.841339111328125, + "rewards/rejected": -0.9447541236877441, + "sft_loss": 0.014985822141170502, + "step": 5960 + }, + { + "epoch": 0.4644107351225204, + "grad_norm": 5.015258312225342, + "learning_rate": 2.7789354313896203e-05, + "logits/chosen": -0.1693064272403717, + "logits/rejected": -1.2654229402542114, + "logps/chosen": -0.8729956746101379, + "logps/rejected": -7.783166408538818, + "loss": 0.8972, + "num_input_tokens_seen": 7815408, + "odds_ratio_loss": 8.893301010131836, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.08729957044124603, + "rewards/margins": 0.691017210483551, + "rewards/rejected": -0.7783167362213135, + "sft_loss": 0.007835522294044495, + "step": 5970 + }, + { + "epoch": 0.46518864255153636, + "grad_norm": 8.604205131530762, + "learning_rate": 2.7728630819215978e-05, + "logits/chosen": -0.21987919509410858, + "logits/rejected": -1.4255279302597046, + "logps/chosen": -1.0022478103637695, + "logps/rejected": -6.178910255432129, + "loss": 1.0319, + "num_input_tokens_seen": 7826352, + "odds_ratio_loss": 10.210367202758789, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.10022477805614471, + "rewards/margins": 0.517666220664978, + "rewards/rejected": -0.6178910136222839, + "sft_loss": 0.010909122414886951, + "step": 5980 + }, + { + "epoch": 0.4659665499805523, + "grad_norm": 5.230869770050049, + "learning_rate": 2.7667891027817865e-05, + "logits/chosen": -0.3464639186859131, + "logits/rejected": -1.0649701356887817, + "logps/chosen": -0.7585294842720032, + "logps/rejected": -5.745346546173096, + "loss": 0.7815, + "num_input_tokens_seen": 7833904, + "odds_ratio_loss": 7.651752471923828, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.07585294544696808, + "rewards/margins": 0.4986816942691803, + "rewards/rejected": -0.5745345950126648, + "sft_loss": 0.016333168372511864, + "step": 5990 + }, + { + "epoch": 0.46674445740956827, + "grad_norm": 507.97247314453125, + "learning_rate": 2.7607135302469615e-05, + "logits/chosen": -0.22762298583984375, + "logits/rejected": -1.140218734741211, + "logps/chosen": -1.5735101699829102, + "logps/rejected": -3.470374345779419, + "loss": 1.6485, + "num_input_tokens_seen": 7850688, + "odds_ratio_loss": 15.951501846313477, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.1573510318994522, + "rewards/margins": 0.1896863877773285, + "rewards/rejected": -0.3470374047756195, + "sft_loss": 0.053358934819698334, + "step": 6000 + }, + { + "epoch": 0.4675223648385842, + "grad_norm": 2.535000801086426, + "learning_rate": 2.7546364006034137e-05, + "logits/chosen": -0.06629699468612671, + "logits/rejected": -1.4996873140335083, + "logps/chosen": -0.8568462133407593, + "logps/rejected": -9.733963012695312, + "loss": 0.8854, + "num_input_tokens_seen": 7862704, + "odds_ratio_loss": 8.79551887512207, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.085684634745121, + "rewards/margins": 0.8877116441726685, + "rewards/rejected": -0.9733963012695312, + "sft_loss": 0.005818786099553108, + "step": 6010 + }, + { + "epoch": 0.4683002722676002, + "grad_norm": 2.4148144721984863, + "learning_rate": 2.7485577501467337e-05, + "logits/chosen": -0.3303348124027252, + "logits/rejected": -0.8454314470291138, + "logps/chosen": -0.819682240486145, + "logps/rejected": -3.0566112995147705, + "loss": 0.849, + "num_input_tokens_seen": 7876240, + "odds_ratio_loss": 8.1669921875, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.08196821808815002, + "rewards/margins": 0.22369292378425598, + "rewards/rejected": -0.305661141872406, + "sft_loss": 0.03231917321681976, + "step": 6020 + }, + { + "epoch": 0.4690781796966161, + "grad_norm": 2.606917381286621, + "learning_rate": 2.742477615181596e-05, + "logits/chosen": 0.09807735681533813, + "logits/rejected": -1.4461350440979004, + "logps/chosen": -1.050130844116211, + "logps/rejected": -6.8379645347595215, + "loss": 1.0526, + "num_input_tokens_seen": 7885536, + "odds_ratio_loss": 10.428014755249023, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.10501308739185333, + "rewards/margins": 0.578783392906189, + "rewards/rejected": -0.6837964653968811, + "sft_loss": 0.009837312623858452, + "step": 6030 + }, + { + "epoch": 0.46985608712563204, + "grad_norm": 1.6283973455429077, + "learning_rate": 2.7363960320215393e-05, + "logits/chosen": -0.24384596943855286, + "logits/rejected": -1.2933290004730225, + "logps/chosen": -0.591637134552002, + "logps/rejected": -6.946697235107422, + "loss": 0.6009, + "num_input_tokens_seen": 7898160, + "odds_ratio_loss": 5.6380205154418945, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.059163711965084076, + "rewards/margins": 0.6355060338973999, + "rewards/rejected": -0.694669783115387, + "sft_loss": 0.0371074303984642, + "step": 6040 + }, + { + "epoch": 0.470633994554648, + "grad_norm": 5.1402177810668945, + "learning_rate": 2.730313036988753e-05, + "logits/chosen": -0.12846294045448303, + "logits/rejected": -1.385025143623352, + "logps/chosen": -0.8894141912460327, + "logps/rejected": -3.4371979236602783, + "loss": 0.9212, + "num_input_tokens_seen": 7910608, + "odds_ratio_loss": 8.882787704467773, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.08894142508506775, + "rewards/margins": 0.2547783851623535, + "rewards/rejected": -0.34371981024742126, + "sft_loss": 0.03289348632097244, + "step": 6050 + }, + { + "epoch": 0.47141190198366395, + "grad_norm": 2.494810104370117, + "learning_rate": 2.7242286664138595e-05, + "logits/chosen": 0.08889412134885788, + "logits/rejected": -1.808688759803772, + "logps/chosen": -0.7223685383796692, + "logps/rejected": -4.377908229827881, + "loss": 0.7449, + "num_input_tokens_seen": 7923776, + "odds_ratio_loss": 7.426810264587402, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.07223685085773468, + "rewards/margins": 0.36555397510528564, + "rewards/rejected": -0.43779081106185913, + "sft_loss": 0.0022290514316409826, + "step": 6060 + }, + { + "epoch": 0.4721898094126799, + "grad_norm": 0.6981421113014221, + "learning_rate": 2.7181429566356952e-05, + "logits/chosen": -0.44528698921203613, + "logits/rejected": -1.303655743598938, + "logps/chosen": -0.7890397310256958, + "logps/rejected": -4.5952653884887695, + "loss": 0.8009, + "num_input_tokens_seen": 7934848, + "odds_ratio_loss": 7.944195747375488, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.07890397310256958, + "rewards/margins": 0.38062259554862976, + "rewards/rejected": -0.45952659845352173, + "sft_loss": 0.006529558449983597, + "step": 6070 + }, + { + "epoch": 0.47296771684169586, + "grad_norm": 2.9179821014404297, + "learning_rate": 2.712055944001095e-05, + "logits/chosen": -0.22472205758094788, + "logits/rejected": -1.4862409830093384, + "logps/chosen": -0.9329856038093567, + "logps/rejected": -12.334473609924316, + "loss": 0.9441, + "num_input_tokens_seen": 7946752, + "odds_ratio_loss": 9.35265064239502, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.0932985469698906, + "rewards/margins": 1.1401488780975342, + "rewards/rejected": -1.233447551727295, + "sft_loss": 0.008851487189531326, + "step": 6080 + }, + { + "epoch": 0.47374562427071176, + "grad_norm": 3.321930408477783, + "learning_rate": 2.705967664864678e-05, + "logits/chosen": -0.2719866633415222, + "logits/rejected": -1.5798949003219604, + "logps/chosen": -0.9579636454582214, + "logps/rejected": -11.035733222961426, + "loss": 0.9719, + "num_input_tokens_seen": 7959040, + "odds_ratio_loss": 9.649393081665039, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.0957963690161705, + "rewards/margins": 1.0077769756317139, + "rewards/rejected": -1.1035733222961426, + "sft_loss": 0.006914190948009491, + "step": 6090 + }, + { + "epoch": 0.4745235316997277, + "grad_norm": 7.015444278717041, + "learning_rate": 2.6998781555886227e-05, + "logits/chosen": -0.2518048584461212, + "logits/rejected": -1.1595853567123413, + "logps/chosen": -0.810549259185791, + "logps/rejected": -17.622051239013672, + "loss": 0.8161, + "num_input_tokens_seen": 7972240, + "odds_ratio_loss": 8.099400520324707, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.0810549259185791, + "rewards/margins": 1.6811498403549194, + "rewards/rejected": -1.762204885482788, + "sft_loss": 0.006167970597743988, + "step": 6100 + }, + { + "epoch": 0.4753014391287437, + "grad_norm": 1.4676387310028076, + "learning_rate": 2.6937874525424577e-05, + "logits/chosen": -0.2762657403945923, + "logits/rejected": -1.0840905904769897, + "logps/chosen": -1.2074869871139526, + "logps/rejected": -11.498342514038086, + "loss": 1.2286, + "num_input_tokens_seen": 7987488, + "odds_ratio_loss": 10.559377670288086, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.12074869871139526, + "rewards/margins": 1.029085636138916, + "rewards/rejected": -1.149834394454956, + "sft_loss": 0.1726953685283661, + "step": 6110 + }, + { + "epoch": 0.47607934655775963, + "grad_norm": 1.908224105834961, + "learning_rate": 2.6876955921028408e-05, + "logits/chosen": -0.20587265491485596, + "logits/rejected": -1.3624060153961182, + "logps/chosen": -0.7148422002792358, + "logps/rejected": -7.384983062744141, + "loss": 0.7285, + "num_input_tokens_seen": 7998720, + "odds_ratio_loss": 7.2302398681640625, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.07148422300815582, + "rewards/margins": 0.6670140027999878, + "rewards/rejected": -0.7384982705116272, + "sft_loss": 0.005456112790852785, + "step": 6120 + }, + { + "epoch": 0.4768572539867756, + "grad_norm": 1.6788196563720703, + "learning_rate": 2.6816026106533428e-05, + "logits/chosen": -0.14552515745162964, + "logits/rejected": -1.3555206060409546, + "logps/chosen": -0.8666656613349915, + "logps/rejected": -16.5738582611084, + "loss": 0.878, + "num_input_tokens_seen": 8016608, + "odds_ratio_loss": 8.666595458984375, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.08666656166315079, + "rewards/margins": 1.5707193613052368, + "rewards/rejected": -1.6573858261108398, + "sft_loss": 0.01131691224873066, + "step": 6130 + }, + { + "epoch": 0.47763516141579154, + "grad_norm": 2.0778162479400635, + "learning_rate": 2.6755085445842276e-05, + "logits/chosen": -0.06541848182678223, + "logits/rejected": -1.4086366891860962, + "logps/chosen": -0.8241838216781616, + "logps/rejected": -12.803787231445312, + "loss": 0.8433, + "num_input_tokens_seen": 8029872, + "odds_ratio_loss": 8.388916969299316, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.08241839706897736, + "rewards/margins": 1.1979604959487915, + "rewards/rejected": -1.2803786993026733, + "sft_loss": 0.0044305888004601, + "step": 6140 + }, + { + "epoch": 0.47841306884480744, + "grad_norm": 7.2569074630737305, + "learning_rate": 2.6694134302922396e-05, + "logits/chosen": -0.15363359451293945, + "logits/rejected": -1.0825797319412231, + "logps/chosen": -0.9479478001594543, + "logps/rejected": -9.695534706115723, + "loss": 0.9548, + "num_input_tokens_seen": 8045200, + "odds_ratio_loss": 9.402900695800781, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09479478001594543, + "rewards/margins": 0.8747587203979492, + "rewards/rejected": -0.9695534706115723, + "sft_loss": 0.014496704563498497, + "step": 6150 + }, + { + "epoch": 0.4791909762738234, + "grad_norm": 1.3309544324874878, + "learning_rate": 2.6633173041803828e-05, + "logits/chosen": -0.1747881919145584, + "logits/rejected": -1.319799542427063, + "logps/chosen": -0.8536643981933594, + "logps/rejected": -5.712998867034912, + "loss": 0.8676, + "num_input_tokens_seen": 8060896, + "odds_ratio_loss": 8.596722602844238, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.08536644279956818, + "rewards/margins": 0.48593348264694214, + "rewards/rejected": -0.5712999105453491, + "sft_loss": 0.00788833387196064, + "step": 6160 + }, + { + "epoch": 0.47996888370283936, + "grad_norm": 4.741382122039795, + "learning_rate": 2.6572202026577036e-05, + "logits/chosen": -0.07354072481393814, + "logits/rejected": -1.1875712871551514, + "logps/chosen": -0.7627614736557007, + "logps/rejected": -10.352548599243164, + "loss": 0.7677, + "num_input_tokens_seen": 8074224, + "odds_ratio_loss": 7.301914215087891, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.07627616077661514, + "rewards/margins": 0.9589789509773254, + "rewards/rejected": -1.035254955291748, + "sft_loss": 0.037500251084566116, + "step": 6170 + }, + { + "epoch": 0.4807467911318553, + "grad_norm": 22.816436767578125, + "learning_rate": 2.651122162139074e-05, + "logits/chosen": -0.5151816606521606, + "logits/rejected": -1.1198275089263916, + "logps/chosen": -0.9731024503707886, + "logps/rejected": -12.562901496887207, + "loss": 0.996, + "num_input_tokens_seen": 8088144, + "odds_ratio_loss": 9.769756317138672, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.09731025993824005, + "rewards/margins": 1.1589797735214233, + "rewards/rejected": -1.256290078163147, + "sft_loss": 0.019033921882510185, + "step": 6180 + }, + { + "epoch": 0.48152469856087127, + "grad_norm": 3.49373722076416, + "learning_rate": 2.6450232190449758e-05, + "logits/chosen": -0.05174265056848526, + "logits/rejected": -1.6913894414901733, + "logps/chosen": -0.8371627926826477, + "logps/rejected": -12.714179039001465, + "loss": 0.8474, + "num_input_tokens_seen": 8101824, + "odds_ratio_loss": 8.428500175476074, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.08371627330780029, + "rewards/margins": 1.18770170211792, + "rewards/rejected": -1.2714178562164307, + "sft_loss": 0.0045602647587656975, + "step": 6190 + }, + { + "epoch": 0.4823026059898872, + "grad_norm": 3.0217878818511963, + "learning_rate": 2.6389234098012792e-05, + "logits/chosen": -0.1306189000606537, + "logits/rejected": -1.3144865036010742, + "logps/chosen": -1.0501823425292969, + "logps/rejected": -17.49735450744629, + "loss": 1.0605, + "num_input_tokens_seen": 8119472, + "odds_ratio_loss": 10.536087989807129, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.1050182357430458, + "rewards/margins": 1.6447172164916992, + "rewards/rejected": -1.7497354745864868, + "sft_loss": 0.006926544010639191, + "step": 6200 + }, + { + "epoch": 0.4830805134189031, + "grad_norm": 2.083096981048584, + "learning_rate": 2.6328227708390297e-05, + "logits/chosen": -0.15976101160049438, + "logits/rejected": -1.337317705154419, + "logps/chosen": -0.7609018087387085, + "logps/rejected": -17.045391082763672, + "loss": 0.7782, + "num_input_tokens_seen": 8133248, + "odds_ratio_loss": 7.667045593261719, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.07609017193317413, + "rewards/margins": 1.6284490823745728, + "rewards/rejected": -1.7045392990112305, + "sft_loss": 0.0114522036164999, + "step": 6210 + }, + { + "epoch": 0.4838584208479191, + "grad_norm": 72.3661117553711, + "learning_rate": 2.6267213385942262e-05, + "logits/chosen": -0.5410548448562622, + "logits/rejected": -1.1711819171905518, + "logps/chosen": -1.275607943534851, + "logps/rejected": -17.424577713012695, + "loss": 1.2817, + "num_input_tokens_seen": 8149296, + "odds_ratio_loss": 11.32263469696045, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1275607943534851, + "rewards/margins": 1.6148971319198608, + "rewards/rejected": -1.7424579858779907, + "sft_loss": 0.14939887821674347, + "step": 6220 + }, + { + "epoch": 0.48463632827693504, + "grad_norm": 2.0818560123443604, + "learning_rate": 2.6206191495076065e-05, + "logits/chosen": -0.006120137870311737, + "logits/rejected": -1.480681300163269, + "logps/chosen": -0.987036406993866, + "logps/rejected": -8.894469261169434, + "loss": 0.9965, + "num_input_tokens_seen": 8165152, + "odds_ratio_loss": 9.909078598022461, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09870363026857376, + "rewards/margins": 0.7907432913780212, + "rewards/rejected": -0.8894469141960144, + "sft_loss": 0.0056028286926448345, + "step": 6230 + }, + { + "epoch": 0.485414235705951, + "grad_norm": 2.703519582748413, + "learning_rate": 2.614516240024429e-05, + "logits/chosen": -0.06827685236930847, + "logits/rejected": -1.7971550226211548, + "logps/chosen": -0.9212660789489746, + "logps/rejected": -12.907064437866211, + "loss": 0.9536, + "num_input_tokens_seen": 8177776, + "odds_ratio_loss": 9.493310928344727, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.09212660044431686, + "rewards/margins": 1.1985799074172974, + "rewards/rejected": -1.2907065153121948, + "sft_loss": 0.00423536216840148, + "step": 6240 + }, + { + "epoch": 0.48619214313496695, + "grad_norm": 1.8637549877166748, + "learning_rate": 2.608412646594254e-05, + "logits/chosen": 0.13315926492214203, + "logits/rejected": -1.7044929265975952, + "logps/chosen": -0.945042610168457, + "logps/rejected": -4.411226272583008, + "loss": 0.9641, + "num_input_tokens_seen": 8192080, + "odds_ratio_loss": 9.610013008117676, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.09450425952672958, + "rewards/margins": 0.3466184139251709, + "rewards/rejected": -0.4411226212978363, + "sft_loss": 0.0030971444211900234, + "step": 6250 + }, + { + "epoch": 0.4869700505639829, + "grad_norm": 1.8692630529403687, + "learning_rate": 2.6023084056707258e-05, + "logits/chosen": -0.03700752183794975, + "logits/rejected": -1.5287142992019653, + "logps/chosen": -0.7440109848976135, + "logps/rejected": -6.019461154937744, + "loss": 0.7637, + "num_input_tokens_seen": 8204656, + "odds_ratio_loss": 7.601499080657959, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.07440110296010971, + "rewards/margins": 0.5275450944900513, + "rewards/rejected": -0.6019461750984192, + "sft_loss": 0.003576572285965085, + "step": 6260 + }, + { + "epoch": 0.48774795799299886, + "grad_norm": 2.0554590225219727, + "learning_rate": 2.596203553711359e-05, + "logits/chosen": -0.07751911878585815, + "logits/rejected": -1.502213478088379, + "logps/chosen": -0.9608389735221863, + "logps/rejected": -7.2761054039001465, + "loss": 0.9785, + "num_input_tokens_seen": 8216896, + "odds_ratio_loss": 9.725781440734863, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.09608390182256699, + "rewards/margins": 0.63152676820755, + "rewards/rejected": -0.7276106476783752, + "sft_loss": 0.0058743832632899284, + "step": 6270 + }, + { + "epoch": 0.48852586542201476, + "grad_norm": 13.380386352539062, + "learning_rate": 2.5900981271773133e-05, + "logits/chosen": -0.20113714039325714, + "logits/rejected": -1.1153767108917236, + "logps/chosen": -0.8374971151351929, + "logps/rejected": -4.752079010009766, + "loss": 0.8551, + "num_input_tokens_seen": 8232096, + "odds_ratio_loss": 8.423782348632812, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.08374972641468048, + "rewards/margins": 0.3914581835269928, + "rewards/rejected": -0.4752078950405121, + "sft_loss": 0.012685893103480339, + "step": 6280 + }, + { + "epoch": 0.4893037728510307, + "grad_norm": 11.337404251098633, + "learning_rate": 2.5839921625331842e-05, + "logits/chosen": -0.49898165464401245, + "logits/rejected": -0.7197166085243225, + "logps/chosen": -2.081268310546875, + "logps/rejected": -3.8042755126953125, + "loss": 2.2166, + "num_input_tokens_seen": 8241072, + "odds_ratio_loss": 21.73521614074707, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.20812682807445526, + "rewards/margins": 0.17230072617530823, + "rewards/rejected": -0.3804275393486023, + "sft_loss": 0.04303508251905441, + "step": 6290 + }, + { + "epoch": 0.49008168028004667, + "grad_norm": 3.3102755546569824, + "learning_rate": 2.5778856962467777e-05, + "logits/chosen": -0.13400593400001526, + "logits/rejected": -1.450063705444336, + "logps/chosen": -0.9296320080757141, + "logps/rejected": -4.581965446472168, + "loss": 0.9603, + "num_input_tokens_seen": 8249264, + "odds_ratio_loss": 9.388015747070312, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.09296319633722305, + "rewards/margins": 0.3652333617210388, + "rewards/rejected": -0.4581965506076813, + "sft_loss": 0.021517466753721237, + "step": 6300 + }, + { + "epoch": 0.49085958770906263, + "grad_norm": 2.8110644817352295, + "learning_rate": 2.571778764788898e-05, + "logits/chosen": -0.0194260124117136, + "logits/rejected": -1.2334833145141602, + "logps/chosen": -0.8915126919746399, + "logps/rejected": -6.6634955406188965, + "loss": 0.9124, + "num_input_tokens_seen": 8262368, + "odds_ratio_loss": 9.059114456176758, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.08915125578641891, + "rewards/margins": 0.5771982669830322, + "rewards/rejected": -0.6663495302200317, + "sft_loss": 0.00646891538053751, + "step": 6310 + }, + { + "epoch": 0.4916374951380786, + "grad_norm": 4.480525493621826, + "learning_rate": 2.5656714046331265e-05, + "logits/chosen": -0.4957144856452942, + "logits/rejected": -0.8260617256164551, + "logps/chosen": -0.9069435000419617, + "logps/rejected": -3.975454807281494, + "loss": 0.9271, + "num_input_tokens_seen": 8273760, + "odds_ratio_loss": 8.90278434753418, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.0906943529844284, + "rewards/margins": 0.3068511486053467, + "rewards/rejected": -0.3975455164909363, + "sft_loss": 0.03677966445684433, + "step": 6320 + }, + { + "epoch": 0.49241540256709454, + "grad_norm": 1.9025019407272339, + "learning_rate": 2.559563652255605e-05, + "logits/chosen": -0.33826085925102234, + "logits/rejected": -1.4704886674880981, + "logps/chosen": -0.8480048179626465, + "logps/rejected": -6.5116119384765625, + "loss": 0.8607, + "num_input_tokens_seen": 8285680, + "odds_ratio_loss": 8.557178497314453, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.08480048179626465, + "rewards/margins": 0.5663608312606812, + "rewards/rejected": -0.6511613130569458, + "sft_loss": 0.0049712504260241985, + "step": 6330 + }, + { + "epoch": 0.49319330999611044, + "grad_norm": 2.8888540267944336, + "learning_rate": 2.553455544134819e-05, + "logits/chosen": -0.09928639233112335, + "logits/rejected": -0.8209127187728882, + "logps/chosen": -0.9104410409927368, + "logps/rejected": -4.182798385620117, + "loss": 0.9345, + "num_input_tokens_seen": 8297488, + "odds_ratio_loss": 9.230264663696289, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.0910440981388092, + "rewards/margins": 0.32723572850227356, + "rewards/rejected": -0.41827982664108276, + "sft_loss": 0.011449308134615421, + "step": 6340 + }, + { + "epoch": 0.4939712174251264, + "grad_norm": 2.2666444778442383, + "learning_rate": 2.547347116751377e-05, + "logits/chosen": -0.2674545645713806, + "logits/rejected": -1.1496336460113525, + "logps/chosen": -0.921932578086853, + "logps/rejected": -8.103863716125488, + "loss": 0.9277, + "num_input_tokens_seen": 8308768, + "odds_ratio_loss": 9.053686141967773, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09219325333833694, + "rewards/margins": 0.7181931138038635, + "rewards/rejected": -0.8103863596916199, + "sft_loss": 0.022327039390802383, + "step": 6350 + }, + { + "epoch": 0.49474912485414235, + "grad_norm": 7.034699440002441, + "learning_rate": 2.5412384065877954e-05, + "logits/chosen": -0.3206288516521454, + "logits/rejected": -1.109317421913147, + "logps/chosen": -0.9621008634567261, + "logps/rejected": -7.422767639160156, + "loss": 0.9766, + "num_input_tokens_seen": 8321168, + "odds_ratio_loss": 9.610101699829102, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.09621009230613708, + "rewards/margins": 0.6460667848587036, + "rewards/rejected": -0.7422767877578735, + "sft_loss": 0.015624640509486198, + "step": 6360 + }, + { + "epoch": 0.4955270322831583, + "grad_norm": 7.01314640045166, + "learning_rate": 2.5351294501282785e-05, + "logits/chosen": -0.2326393872499466, + "logits/rejected": -0.9409756660461426, + "logps/chosen": -1.0383962392807007, + "logps/rejected": -15.200935363769531, + "loss": 1.0673, + "num_input_tokens_seen": 8337456, + "odds_ratio_loss": 10.551141738891602, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.10383962094783783, + "rewards/margins": 1.4162540435791016, + "rewards/rejected": -1.5200936794281006, + "sft_loss": 0.012215681374073029, + "step": 6370 + }, + { + "epoch": 0.49630493971217426, + "grad_norm": 4.612608432769775, + "learning_rate": 2.5290202838585036e-05, + "logits/chosen": -0.2782391309738159, + "logits/rejected": -0.8692216873168945, + "logps/chosen": -1.1118863821029663, + "logps/rejected": -4.572246551513672, + "loss": 1.1561, + "num_input_tokens_seen": 8347088, + "odds_ratio_loss": 11.084630966186523, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.11118865013122559, + "rewards/margins": 0.34603601694107056, + "rewards/rejected": -0.45722466707229614, + "sft_loss": 0.04765518382191658, + "step": 6380 + }, + { + "epoch": 0.4970828471411902, + "grad_norm": 2.985591411590576, + "learning_rate": 2.522910944265399e-05, + "logits/chosen": -0.1539156287908554, + "logits/rejected": -1.1583564281463623, + "logps/chosen": -0.8345049619674683, + "logps/rejected": -7.536215305328369, + "loss": 0.8428, + "num_input_tokens_seen": 8361040, + "odds_ratio_loss": 8.301528930664062, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.08345050364732742, + "rewards/margins": 0.6701710820198059, + "rewards/rejected": -0.7536215782165527, + "sft_loss": 0.012616393156349659, + "step": 6390 + }, + { + "epoch": 0.4978607545702061, + "grad_norm": 3.878408670425415, + "learning_rate": 2.516801467836929e-05, + "logits/chosen": -0.3584039807319641, + "logits/rejected": -1.179455041885376, + "logps/chosen": -0.9978312253952026, + "logps/rejected": -10.517958641052246, + "loss": 1.0255, + "num_input_tokens_seen": 8369088, + "odds_ratio_loss": 9.225652694702148, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.09978312253952026, + "rewards/margins": 0.9520127177238464, + "rewards/rejected": -1.0517958402633667, + "sft_loss": 0.10292162746191025, + "step": 6400 + }, + { + "epoch": 0.4986386619992221, + "grad_norm": 2.9756100177764893, + "learning_rate": 2.510691891061875e-05, + "logits/chosen": 0.055911578238010406, + "logits/rejected": -1.6097557544708252, + "logps/chosen": -1.0666486024856567, + "logps/rejected": -9.044986724853516, + "loss": 1.0829, + "num_input_tokens_seen": 8378512, + "odds_ratio_loss": 10.744181632995605, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.10666485875844955, + "rewards/margins": 0.7978337407112122, + "rewards/rejected": -0.9044985771179199, + "sft_loss": 0.008470003493130207, + "step": 6410 + }, + { + "epoch": 0.49941656942823803, + "grad_norm": 2.4883270263671875, + "learning_rate": 2.504582250429618e-05, + "logits/chosen": -0.19540779292583466, + "logits/rejected": -0.9103869199752808, + "logps/chosen": -0.8133610486984253, + "logps/rejected": -8.337348937988281, + "loss": 0.8362, + "num_input_tokens_seen": 8393488, + "odds_ratio_loss": 8.27368450164795, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.08133610337972641, + "rewards/margins": 0.7523988485336304, + "rewards/rejected": -0.8337349891662598, + "sft_loss": 0.008857587352395058, + "step": 6420 + }, + { + "epoch": 0.5001944768572539, + "grad_norm": 2.813175678253174, + "learning_rate": 2.498472582429921e-05, + "logits/chosen": -0.30721643567085266, + "logits/rejected": -1.406435251235962, + "logps/chosen": -0.8761197924613953, + "logps/rejected": -6.059558868408203, + "loss": 0.899, + "num_input_tokens_seen": 8407136, + "odds_ratio_loss": 8.8612699508667, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.08761197328567505, + "rewards/margins": 0.5183438658714294, + "rewards/rejected": -0.6059558391571045, + "sft_loss": 0.012886581011116505, + "step": 6430 + }, + { + "epoch": 0.5009723842862699, + "grad_norm": 1.4266654253005981, + "learning_rate": 2.4923629235527078e-05, + "logits/chosen": -0.1768713891506195, + "logits/rejected": -1.1466429233551025, + "logps/chosen": -0.9270246624946594, + "logps/rejected": -6.692111968994141, + "loss": 0.9596, + "num_input_tokens_seen": 8425696, + "odds_ratio_loss": 9.194589614868164, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.0927024707198143, + "rewards/margins": 0.5765087008476257, + "rewards/rejected": -0.669211208820343, + "sft_loss": 0.0400998480618, + "step": 6440 + }, + { + "epoch": 0.5017502917152858, + "grad_norm": 2.516162395477295, + "learning_rate": 2.486253310287851e-05, + "logits/chosen": -0.2516472041606903, + "logits/rejected": -1.1252797842025757, + "logps/chosen": -1.5495781898498535, + "logps/rejected": -10.633633613586426, + "loss": 1.5676, + "num_input_tokens_seen": 8439280, + "odds_ratio_loss": 15.6168212890625, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.1549578160047531, + "rewards/margins": 0.908405601978302, + "rewards/rejected": -1.0633634328842163, + "sft_loss": 0.005875526927411556, + "step": 6450 + }, + { + "epoch": 0.5025281991443018, + "grad_norm": 2.324152946472168, + "learning_rate": 2.4801437791249482e-05, + "logits/chosen": -0.0015714972978457808, + "logits/rejected": -1.7600247859954834, + "logps/chosen": -0.7816251516342163, + "logps/rejected": -7.274757385253906, + "loss": 0.7902, + "num_input_tokens_seen": 8456080, + "odds_ratio_loss": 7.856690883636475, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.07816250622272491, + "rewards/margins": 0.64931321144104, + "rewards/rejected": -0.7274757027626038, + "sft_loss": 0.004567933268845081, + "step": 6460 + }, + { + "epoch": 0.5033061065733178, + "grad_norm": 3.4712696075439453, + "learning_rate": 2.4740343665531092e-05, + "logits/chosen": -0.2848728597164154, + "logits/rejected": -1.1505284309387207, + "logps/chosen": -0.8179367184638977, + "logps/rejected": -6.20058012008667, + "loss": 0.8267, + "num_input_tokens_seen": 8468992, + "odds_ratio_loss": 8.157306671142578, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.08179367333650589, + "rewards/margins": 0.5382643342018127, + "rewards/rejected": -0.6200579404830933, + "sft_loss": 0.0109955919906497, + "step": 6470 + }, + { + "epoch": 0.5040840140023337, + "grad_norm": 3.0974578857421875, + "learning_rate": 2.4679251090607333e-05, + "logits/chosen": -0.22295601665973663, + "logits/rejected": -1.3508847951889038, + "logps/chosen": -0.932137668132782, + "logps/rejected": -8.735374450683594, + "loss": 0.9399, + "num_input_tokens_seen": 8481024, + "odds_ratio_loss": 9.306537628173828, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.093213751912117, + "rewards/margins": 0.7803236246109009, + "rewards/rejected": -0.8735373616218567, + "sft_loss": 0.009225866757333279, + "step": 6480 + }, + { + "epoch": 0.5048619214313497, + "grad_norm": 5.221313953399658, + "learning_rate": 2.4618160431352946e-05, + "logits/chosen": 0.10161104053258896, + "logits/rejected": -1.5568485260009766, + "logps/chosen": -0.8982173204421997, + "logps/rejected": -7.640667915344238, + "loss": 0.9011, + "num_input_tokens_seen": 8492816, + "odds_ratio_loss": 8.938986778259277, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.0898217260837555, + "rewards/margins": 0.6742449998855591, + "rewards/rejected": -0.764066755771637, + "sft_loss": 0.007185804657638073, + "step": 6490 + }, + { + "epoch": 0.5056398288603656, + "grad_norm": 1.990058183670044, + "learning_rate": 2.455707205263123e-05, + "logits/chosen": -0.3942679166793823, + "logits/rejected": -0.9469472765922546, + "logps/chosen": -1.034597635269165, + "logps/rejected": -9.053838729858398, + "loss": 1.0465, + "num_input_tokens_seen": 8509136, + "odds_ratio_loss": 10.092089653015137, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.10345976054668427, + "rewards/margins": 0.8019241094589233, + "rewards/rejected": -0.9053838849067688, + "sft_loss": 0.037330999970436096, + "step": 6500 + }, + { + "epoch": 0.5064177362893816, + "grad_norm": 2.9650418758392334, + "learning_rate": 2.4495986319291857e-05, + "logits/chosen": -0.54425448179245, + "logits/rejected": -0.8309041261672974, + "logps/chosen": -0.9747613668441772, + "logps/rejected": -16.209012985229492, + "loss": 0.9967, + "num_input_tokens_seen": 8522096, + "odds_ratio_loss": 9.86829948425293, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.09747613966464996, + "rewards/margins": 1.5234253406524658, + "rewards/rejected": -1.6209014654159546, + "sft_loss": 0.009864835068583488, + "step": 6510 + }, + { + "epoch": 0.5071956437183975, + "grad_norm": 2.1551918983459473, + "learning_rate": 2.443490359616871e-05, + "logits/chosen": -0.22963781654834747, + "logits/rejected": -0.9503774642944336, + "logps/chosen": -0.8328534364700317, + "logps/rejected": -11.546224594116211, + "loss": 0.838, + "num_input_tokens_seen": 8538672, + "odds_ratio_loss": 7.670839786529541, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.08328534662723541, + "rewards/margins": 1.071337103843689, + "rewards/rejected": -1.1546225547790527, + "sft_loss": 0.07093828916549683, + "step": 6520 + }, + { + "epoch": 0.5079735511474135, + "grad_norm": 3.307870864868164, + "learning_rate": 2.437382424807769e-05, + "logits/chosen": -0.43049168586730957, + "logits/rejected": -1.3492554426193237, + "logps/chosen": -1.0302252769470215, + "logps/rejected": -15.203390121459961, + "loss": 1.0419, + "num_input_tokens_seen": 8550192, + "odds_ratio_loss": 10.329671859741211, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.10302253067493439, + "rewards/margins": 1.4173165559768677, + "rewards/rejected": -1.520339012145996, + "sft_loss": 0.00895792618393898, + "step": 6530 + }, + { + "epoch": 0.5087514585764294, + "grad_norm": 4.686474323272705, + "learning_rate": 2.431274863981453e-05, + "logits/chosen": -0.265191912651062, + "logits/rejected": -0.9243543744087219, + "logps/chosen": -0.8716975450515747, + "logps/rejected": -12.998971939086914, + "loss": 0.8795, + "num_input_tokens_seen": 8561584, + "odds_ratio_loss": 8.608287811279297, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.08716975152492523, + "rewards/margins": 1.212727427482605, + "rewards/rejected": -1.2998971939086914, + "sft_loss": 0.0186309851706028, + "step": 6540 + }, + { + "epoch": 0.5095293660054454, + "grad_norm": 4.554167747497559, + "learning_rate": 2.4251677136152642e-05, + "logits/chosen": -0.5132981538772583, + "logits/rejected": -1.1222872734069824, + "logps/chosen": -0.8228030204772949, + "logps/rejected": -17.707448959350586, + "loss": 0.823, + "num_input_tokens_seen": 8570304, + "odds_ratio_loss": 8.138057708740234, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.08228030055761337, + "rewards/margins": 1.688464879989624, + "rewards/rejected": -1.770745038986206, + "sft_loss": 0.009233796037733555, + "step": 6550 + }, + { + "epoch": 0.5103072734344613, + "grad_norm": 2.7331156730651855, + "learning_rate": 2.419061010184093e-05, + "logits/chosen": -0.09426519274711609, + "logits/rejected": -1.7221357822418213, + "logps/chosen": -0.9415035247802734, + "logps/rejected": -3.5511512756347656, + "loss": 0.976, + "num_input_tokens_seen": 8580016, + "odds_ratio_loss": 9.654105186462402, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.0941503494977951, + "rewards/margins": 0.2609647810459137, + "rewards/rejected": -0.3551151156425476, + "sft_loss": 0.010551346465945244, + "step": 6560 + }, + { + "epoch": 0.5110851808634772, + "grad_norm": 4.815445423126221, + "learning_rate": 2.412954790160157e-05, + "logits/chosen": -0.09299831092357635, + "logits/rejected": -1.0223968029022217, + "logps/chosen": -0.6498953104019165, + "logps/rejected": -7.632620334625244, + "loss": 0.6732, + "num_input_tokens_seen": 8592208, + "odds_ratio_loss": 6.660180568695068, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.06498952209949493, + "rewards/margins": 0.6982723474502563, + "rewards/rejected": -0.7632620334625244, + "sft_loss": 0.007177567575126886, + "step": 6570 + }, + { + "epoch": 0.5118630882924932, + "grad_norm": 29.947223663330078, + "learning_rate": 2.4068490900127894e-05, + "logits/chosen": -0.11313676834106445, + "logits/rejected": -1.2990573644638062, + "logps/chosen": -1.1999033689498901, + "logps/rejected": -9.722343444824219, + "loss": 1.2255, + "num_input_tokens_seen": 8603552, + "odds_ratio_loss": 11.905380249023438, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.11999032646417618, + "rewards/margins": 0.8522440195083618, + "rewards/rejected": -0.9722343683242798, + "sft_loss": 0.03497845306992531, + "step": 6580 + }, + { + "epoch": 0.5126409957215091, + "grad_norm": 3.5857746601104736, + "learning_rate": 2.4007439462082175e-05, + "logits/chosen": -0.06995495408773422, + "logits/rejected": -1.8384273052215576, + "logps/chosen": -0.9535351991653442, + "logps/rejected": -23.40558624267578, + "loss": 0.9537, + "num_input_tokens_seen": 8620960, + "odds_ratio_loss": 9.443254470825195, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09535352140665054, + "rewards/margins": 2.2452054023742676, + "rewards/rejected": -2.3405587673187256, + "sft_loss": 0.00940064899623394, + "step": 6590 + }, + { + "epoch": 0.5134189031505251, + "grad_norm": 2.141690731048584, + "learning_rate": 2.394639395209348e-05, + "logits/chosen": -0.028879564255475998, + "logits/rejected": -1.3578704595565796, + "logps/chosen": -0.9493895769119263, + "logps/rejected": -11.105527877807617, + "loss": 0.9641, + "num_input_tokens_seen": 8630704, + "odds_ratio_loss": 9.563076972961426, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09493895620107651, + "rewards/margins": 1.0156137943267822, + "rewards/rejected": -1.1105527877807617, + "sft_loss": 0.007841681130230427, + "step": 6600 + }, + { + "epoch": 0.514196810579541, + "grad_norm": 2.8887338638305664, + "learning_rate": 2.388535473475544e-05, + "logits/chosen": 0.045301564037799835, + "logits/rejected": -1.2772947549819946, + "logps/chosen": -0.8695799708366394, + "logps/rejected": -12.75346565246582, + "loss": 0.9, + "num_input_tokens_seen": 8645168, + "odds_ratio_loss": 8.117189407348633, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.08695799112319946, + "rewards/margins": 1.188388466835022, + "rewards/rejected": -1.2753465175628662, + "sft_loss": 0.08828867971897125, + "step": 6610 + }, + { + "epoch": 0.514974718008557, + "grad_norm": 3.8750803470611572, + "learning_rate": 2.382432217462412e-05, + "logits/chosen": -0.0706484317779541, + "logits/rejected": -1.0864431858062744, + "logps/chosen": -0.6691504716873169, + "logps/rejected": -10.930901527404785, + "loss": 0.679, + "num_input_tokens_seen": 8659424, + "odds_ratio_loss": 6.66359806060791, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.06691503524780273, + "rewards/margins": 1.0261751413345337, + "rewards/rejected": -1.0930901765823364, + "sft_loss": 0.012643078342080116, + "step": 6620 + }, + { + "epoch": 0.5157526254375729, + "grad_norm": 1.7773513793945312, + "learning_rate": 2.3763296636215824e-05, + "logits/chosen": -0.02193978801369667, + "logits/rejected": -1.34123957157135, + "logps/chosen": -0.9752591252326965, + "logps/rejected": -7.491532325744629, + "loss": 0.9884, + "num_input_tokens_seen": 8671056, + "odds_ratio_loss": 9.739108085632324, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09752590954303741, + "rewards/margins": 0.6516273021697998, + "rewards/rejected": -0.749153196811676, + "sft_loss": 0.014478230848908424, + "step": 6630 + }, + { + "epoch": 0.5165305328665889, + "grad_norm": 5.180094242095947, + "learning_rate": 2.3702278484004907e-05, + "logits/chosen": -0.1238371953368187, + "logits/rejected": -1.0661619901657104, + "logps/chosen": -2.1213927268981934, + "logps/rejected": -16.54400062561035, + "loss": 2.1394, + "num_input_tokens_seen": 8683952, + "odds_ratio_loss": 21.279338836669922, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.212139293551445, + "rewards/margins": 1.442260980606079, + "rewards/rejected": -1.6544002294540405, + "sft_loss": 0.011500345543026924, + "step": 6640 + }, + { + "epoch": 0.5173084402956049, + "grad_norm": 4.295303821563721, + "learning_rate": 2.364126808242163e-05, + "logits/chosen": -0.8342688679695129, + "logits/rejected": -0.7375487089157104, + "logps/chosen": -0.8082100749015808, + "logps/rejected": -7.813617706298828, + "loss": 0.8247, + "num_input_tokens_seen": 8696416, + "odds_ratio_loss": 8.126741409301758, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.08082100749015808, + "rewards/margins": 0.7005407214164734, + "rewards/rejected": -0.7813617587089539, + "sft_loss": 0.012039626948535442, + "step": 6650 + }, + { + "epoch": 0.5180863477246208, + "grad_norm": 3.939998149871826, + "learning_rate": 2.3580265795849944e-05, + "logits/chosen": -0.10337956249713898, + "logits/rejected": -1.3135967254638672, + "logps/chosen": -1.0097706317901611, + "logps/rejected": -9.967623710632324, + "loss": 1.0107, + "num_input_tokens_seen": 8707424, + "odds_ratio_loss": 10.029942512512207, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.10097707808017731, + "rewards/margins": 0.8957854509353638, + "rewards/rejected": -0.9967623949050903, + "sft_loss": 0.00767454644665122, + "step": 6660 + }, + { + "epoch": 0.5188642551536368, + "grad_norm": 2.4933345317840576, + "learning_rate": 2.3519271988625344e-05, + "logits/chosen": -0.10118494182825089, + "logits/rejected": -1.1729929447174072, + "logps/chosen": -0.7244973182678223, + "logps/rejected": -13.689013481140137, + "loss": 0.7268, + "num_input_tokens_seen": 8721984, + "odds_ratio_loss": 7.2344560623168945, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.07244972884654999, + "rewards/margins": 1.2964515686035156, + "rewards/rejected": -1.368901252746582, + "sft_loss": 0.003336378838866949, + "step": 6670 + }, + { + "epoch": 0.5196421625826526, + "grad_norm": 2.1454243659973145, + "learning_rate": 2.3458287025032697e-05, + "logits/chosen": -0.136545330286026, + "logits/rejected": -0.8210135698318481, + "logps/chosen": -0.8043525815010071, + "logps/rejected": -7.613142967224121, + "loss": 0.8181, + "num_input_tokens_seen": 8730688, + "odds_ratio_loss": 8.056388854980469, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.08043525367975235, + "rewards/margins": 0.6808789968490601, + "rewards/rejected": -0.7613142132759094, + "sft_loss": 0.012427089735865593, + "step": 6680 + }, + { + "epoch": 0.5204200700116686, + "grad_norm": 3.140146017074585, + "learning_rate": 2.3397311269304006e-05, + "logits/chosen": -0.10605098307132721, + "logits/rejected": -1.4007622003555298, + "logps/chosen": -0.9359437823295593, + "logps/rejected": -8.159541130065918, + "loss": 0.9587, + "num_input_tokens_seen": 8744160, + "odds_ratio_loss": 9.49116325378418, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.09359437227249146, + "rewards/margins": 0.7223597764968872, + "rewards/rejected": -0.8159540891647339, + "sft_loss": 0.009535567834973335, + "step": 6690 + }, + { + "epoch": 0.5211979774406845, + "grad_norm": 3.360002279281616, + "learning_rate": 2.3336345085616324e-05, + "logits/chosen": 0.030445415526628494, + "logits/rejected": -1.2207574844360352, + "logps/chosen": -0.8218139410018921, + "logps/rejected": -9.092556953430176, + "loss": 0.8714, + "num_input_tokens_seen": 8756560, + "odds_ratio_loss": 8.25312614440918, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.0821814090013504, + "rewards/margins": 0.8270743489265442, + "rewards/rejected": -0.9092556834220886, + "sft_loss": 0.04608220234513283, + "step": 6700 + }, + { + "epoch": 0.5219758848697005, + "grad_norm": 8.89459228515625, + "learning_rate": 2.3275388838089517e-05, + "logits/chosen": -0.21345002949237823, + "logits/rejected": -1.5567723512649536, + "logps/chosen": -1.1306450366973877, + "logps/rejected": -20.125442504882812, + "loss": 1.1347, + "num_input_tokens_seen": 8767408, + "odds_ratio_loss": 11.194887161254883, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.11306451261043549, + "rewards/margins": 1.899479627609253, + "rewards/rejected": -2.0125441551208496, + "sft_loss": 0.015241652727127075, + "step": 6710 + }, + { + "epoch": 0.5227537922987164, + "grad_norm": 2.6667776107788086, + "learning_rate": 2.3214442890784107e-05, + "logits/chosen": 0.04326433688402176, + "logits/rejected": -1.349846363067627, + "logps/chosen": -0.8453485369682312, + "logps/rejected": -11.233430862426758, + "loss": 0.8482, + "num_input_tokens_seen": 8785664, + "odds_ratio_loss": 8.441164016723633, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.08453484624624252, + "rewards/margins": 1.0388084650039673, + "rewards/rejected": -1.1233432292938232, + "sft_loss": 0.004091930575668812, + "step": 6720 + }, + { + "epoch": 0.5235316997277324, + "grad_norm": 2.6269643306732178, + "learning_rate": 2.315350760769911e-05, + "logits/chosen": 0.026680314913392067, + "logits/rejected": -1.3829847574234009, + "logps/chosen": -0.7016235589981079, + "logps/rejected": -21.766069412231445, + "loss": 0.7021, + "num_input_tokens_seen": 8795008, + "odds_ratio_loss": 6.94488000869751, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.07016235589981079, + "rewards/margins": 2.1064445972442627, + "rewards/rejected": -2.1766068935394287, + "sft_loss": 0.007626746781170368, + "step": 6730 + }, + { + "epoch": 0.5243096071567483, + "grad_norm": 3.123906373977661, + "learning_rate": 2.3092583352769832e-05, + "logits/chosen": -0.20880265533924103, + "logits/rejected": -1.134286642074585, + "logps/chosen": -1.1254082918167114, + "logps/rejected": -16.223125457763672, + "loss": 1.1258, + "num_input_tokens_seen": 8807312, + "odds_ratio_loss": 11.091835975646973, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1125408411026001, + "rewards/margins": 1.5097719430923462, + "rewards/rejected": -1.6223127841949463, + "sft_loss": 0.016620712354779243, + "step": 6740 + }, + { + "epoch": 0.5250875145857643, + "grad_norm": 2.3707122802734375, + "learning_rate": 2.303167048986573e-05, + "logits/chosen": -0.48967304825782776, + "logits/rejected": -1.3673675060272217, + "logps/chosen": -1.1683456897735596, + "logps/rejected": -14.066203117370605, + "loss": 1.1774, + "num_input_tokens_seen": 8821056, + "odds_ratio_loss": 10.931964874267578, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.11683456599712372, + "rewards/margins": 1.2897858619689941, + "rewards/rejected": -1.4066202640533447, + "sft_loss": 0.08424625545740128, + "step": 6750 + }, + { + "epoch": 0.5258654220147803, + "grad_norm": 1.509379506111145, + "learning_rate": 2.2970769382788215e-05, + "logits/chosen": -0.026856541633605957, + "logits/rejected": -1.7084585428237915, + "logps/chosen": -1.0115071535110474, + "logps/rejected": -9.703714370727539, + "loss": 1.0464, + "num_input_tokens_seen": 8831952, + "odds_ratio_loss": 10.413983345031738, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.10115071386098862, + "rewards/margins": 0.8692208528518677, + "rewards/rejected": -0.970371425151825, + "sft_loss": 0.004966267850250006, + "step": 6760 + }, + { + "epoch": 0.5266433294437962, + "grad_norm": 6.818657398223877, + "learning_rate": 2.2909880395268478e-05, + "logits/chosen": -0.456535279750824, + "logits/rejected": -0.9776288866996765, + "logps/chosen": -0.6454017162322998, + "logps/rejected": -11.684601783752441, + "loss": 0.6572, + "num_input_tokens_seen": 8844432, + "odds_ratio_loss": 6.454319000244141, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.06454017758369446, + "rewards/margins": 1.1039199829101562, + "rewards/rejected": -1.1684601306915283, + "sft_loss": 0.011763582937419415, + "step": 6770 + }, + { + "epoch": 0.5274212368728122, + "grad_norm": 1.824180245399475, + "learning_rate": 2.2849003890965347e-05, + "logits/chosen": 0.14084644615650177, + "logits/rejected": -1.8677068948745728, + "logps/chosen": -0.9107322692871094, + "logps/rejected": -9.742560386657715, + "loss": 0.9214, + "num_input_tokens_seen": 8855632, + "odds_ratio_loss": 9.149304389953613, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09107323735952377, + "rewards/margins": 0.8831828236579895, + "rewards/rejected": -0.974256157875061, + "sft_loss": 0.0064893984235823154, + "step": 6780 + }, + { + "epoch": 0.5281991443018281, + "grad_norm": 3.3605880737304688, + "learning_rate": 2.278814023346307e-05, + "logits/chosen": -0.04147498309612274, + "logits/rejected": -1.2981659173965454, + "logps/chosen": -1.005117654800415, + "logps/rejected": -12.193399429321289, + "loss": 1.0254, + "num_input_tokens_seen": 8867264, + "odds_ratio_loss": 10.19458293914795, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.10051177442073822, + "rewards/margins": 1.118828296661377, + "rewards/rejected": -1.2193399667739868, + "sft_loss": 0.005910595878958702, + "step": 6790 + }, + { + "epoch": 0.5289770517308441, + "grad_norm": 1.5352081060409546, + "learning_rate": 2.2727289786269177e-05, + "logits/chosen": 0.0757211297750473, + "logits/rejected": -1.747073769569397, + "logps/chosen": -0.9647790193557739, + "logps/rejected": -19.082246780395508, + "loss": 0.9748, + "num_input_tokens_seen": 8879312, + "odds_ratio_loss": 9.69461727142334, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09647790342569351, + "rewards/margins": 1.8117469549179077, + "rewards/rejected": -1.9082248210906982, + "sft_loss": 0.005362776108086109, + "step": 6800 + }, + { + "epoch": 0.5297549591598599, + "grad_norm": 35.42118453979492, + "learning_rate": 2.2666452912812326e-05, + "logits/chosen": -0.15618260204792023, + "logits/rejected": -0.8608878254890442, + "logps/chosen": -0.9781033396720886, + "logps/rejected": -8.846330642700195, + "loss": 0.9884, + "num_input_tokens_seen": 8889840, + "odds_ratio_loss": 9.386763572692871, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09781033545732498, + "rewards/margins": 0.7868226766586304, + "rewards/rejected": -0.8846330642700195, + "sft_loss": 0.04968716949224472, + "step": 6810 + }, + { + "epoch": 0.5305328665888759, + "grad_norm": 1.5546804666519165, + "learning_rate": 2.2605629976440054e-05, + "logits/chosen": -0.008410501293838024, + "logits/rejected": -1.3464069366455078, + "logps/chosen": -1.0442028045654297, + "logps/rejected": -10.094877243041992, + "loss": 1.0548, + "num_input_tokens_seen": 8900288, + "odds_ratio_loss": 10.442464828491211, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.10442028194665909, + "rewards/margins": 0.9050676226615906, + "rewards/rejected": -1.0094878673553467, + "sft_loss": 0.01057764608412981, + "step": 6820 + }, + { + "epoch": 0.5313107740178918, + "grad_norm": 5.027902603149414, + "learning_rate": 2.2544821340416696e-05, + "logits/chosen": -0.05764826387166977, + "logits/rejected": -1.2585020065307617, + "logps/chosen": -0.7766581773757935, + "logps/rejected": -2.905539035797119, + "loss": 0.8124, + "num_input_tokens_seen": 8913792, + "odds_ratio_loss": 7.814454078674316, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.07766580581665039, + "rewards/margins": 0.21288807690143585, + "rewards/rejected": -0.29055389761924744, + "sft_loss": 0.030926724895834923, + "step": 6830 + }, + { + "epoch": 0.5320886814469078, + "grad_norm": 1.7245932817459106, + "learning_rate": 2.2484027367921172e-05, + "logits/chosen": -0.09873903542757034, + "logits/rejected": -0.891947865486145, + "logps/chosen": -0.8979700803756714, + "logps/rejected": -4.629054069519043, + "loss": 0.9119, + "num_input_tokens_seen": 8924320, + "odds_ratio_loss": 8.985692977905273, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.0897970125079155, + "rewards/margins": 0.3731083571910858, + "rewards/rejected": -0.4629054069519043, + "sft_loss": 0.013306763954460621, + "step": 6840 + }, + { + "epoch": 0.5328665888759238, + "grad_norm": 3.8417139053344727, + "learning_rate": 2.2423248422044812e-05, + "logits/chosen": 0.05378826707601547, + "logits/rejected": -1.0258357524871826, + "logps/chosen": -0.8549526929855347, + "logps/rejected": -4.404053211212158, + "loss": 0.8929, + "num_input_tokens_seen": 8933520, + "odds_ratio_loss": 8.802249908447266, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.08549527078866959, + "rewards/margins": 0.3549100458621979, + "rewards/rejected": -0.44040530920028687, + "sft_loss": 0.012637937441468239, + "step": 6850 + }, + { + "epoch": 0.5336444963049397, + "grad_norm": 22.083005905151367, + "learning_rate": 2.2362484865789222e-05, + "logits/chosen": -0.08559930324554443, + "logits/rejected": -1.1315510272979736, + "logps/chosen": -0.810233473777771, + "logps/rejected": -7.210709571838379, + "loss": 0.8283, + "num_input_tokens_seen": 8947824, + "odds_ratio_loss": 8.138298034667969, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.08102335035800934, + "rewards/margins": 0.6400476098060608, + "rewards/rejected": -0.7210709452629089, + "sft_loss": 0.0144663006067276, + "step": 6860 + }, + { + "epoch": 0.5344224037339557, + "grad_norm": 1.6675682067871094, + "learning_rate": 2.230173706206408e-05, + "logits/chosen": -0.026315787807106972, + "logits/rejected": -1.3517112731933594, + "logps/chosen": -0.9361854791641235, + "logps/rejected": -12.261492729187012, + "loss": 0.9492, + "num_input_tokens_seen": 8963552, + "odds_ratio_loss": 9.391185760498047, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09361854940652847, + "rewards/margins": 1.132530689239502, + "rewards/rejected": -1.226149320602417, + "sft_loss": 0.010050268843770027, + "step": 6870 + }, + { + "epoch": 0.5352003111629716, + "grad_norm": 88.71906280517578, + "learning_rate": 2.2241005373684973e-05, + "logits/chosen": 0.005768129136413336, + "logits/rejected": -1.5856225490570068, + "logps/chosen": -1.0896073579788208, + "logps/rejected": -15.783429145812988, + "loss": 1.0958, + "num_input_tokens_seen": 8973680, + "odds_ratio_loss": 10.530699729919434, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.10896072536706924, + "rewards/margins": 1.4693820476531982, + "rewards/rejected": -1.5783427953720093, + "sft_loss": 0.04275606572628021, + "step": 6880 + }, + { + "epoch": 0.5359782185919876, + "grad_norm": 1.807417392730713, + "learning_rate": 2.2180290163371254e-05, + "logits/chosen": -0.3520210087299347, + "logits/rejected": -0.8492413759231567, + "logps/chosen": -0.7934950590133667, + "logps/rejected": -12.94706916809082, + "loss": 0.8035, + "num_input_tokens_seen": 8986320, + "odds_ratio_loss": 7.927926540374756, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.07934950292110443, + "rewards/margins": 1.2153574228286743, + "rewards/rejected": -1.29470694065094, + "sft_loss": 0.010721179656684399, + "step": 6890 + }, + { + "epoch": 0.5367561260210035, + "grad_norm": 4.75670051574707, + "learning_rate": 2.2119591793743858e-05, + "logits/chosen": -0.14193794131278992, + "logits/rejected": -0.600124716758728, + "logps/chosen": -0.9928326606750488, + "logps/rejected": -6.36657190322876, + "loss": 1.0727, + "num_input_tokens_seen": 8997776, + "odds_ratio_loss": 10.262323379516602, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.09928327053785324, + "rewards/margins": 0.5373739004135132, + "rewards/rejected": -0.636657178401947, + "sft_loss": 0.046507202088832855, + "step": 6900 + }, + { + "epoch": 0.5375340334500195, + "grad_norm": 2.5006494522094727, + "learning_rate": 2.205891062732313e-05, + "logits/chosen": -0.08175057172775269, + "logits/rejected": -1.2389705181121826, + "logps/chosen": -0.8751962780952454, + "logps/rejected": -6.338713645935059, + "loss": 0.9051, + "num_input_tokens_seen": 9008288, + "odds_ratio_loss": 8.97519588470459, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.08751963078975677, + "rewards/margins": 0.5463517904281616, + "rewards/rejected": -0.6338714361190796, + "sft_loss": 0.007537661585956812, + "step": 6910 + }, + { + "epoch": 0.5383119408790354, + "grad_norm": 1.7010674476623535, + "learning_rate": 2.1998247026526687e-05, + "logits/chosen": -0.10393454879522324, + "logits/rejected": -1.0789237022399902, + "logps/chosen": -0.9110797047615051, + "logps/rejected": -10.882912635803223, + "loss": 0.9325, + "num_input_tokens_seen": 9024400, + "odds_ratio_loss": 9.21080207824707, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.09110797941684723, + "rewards/margins": 0.9971832036972046, + "rewards/rejected": -1.0882911682128906, + "sft_loss": 0.011449300684034824, + "step": 6920 + }, + { + "epoch": 0.5390898483080513, + "grad_norm": 4.3742451667785645, + "learning_rate": 2.1937601353667224e-05, + "logits/chosen": -0.06702017784118652, + "logits/rejected": -1.0037449598312378, + "logps/chosen": -1.159448266029358, + "logps/rejected": -8.16427993774414, + "loss": 1.2041, + "num_input_tokens_seen": 9034432, + "odds_ratio_loss": 11.199224472045898, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.11594481766223907, + "rewards/margins": 0.7004832029342651, + "rewards/rejected": -0.8164280652999878, + "sft_loss": 0.08420418202877045, + "step": 6930 + }, + { + "epoch": 0.5398677557370672, + "grad_norm": 5.265347003936768, + "learning_rate": 2.187697397095035e-05, + "logits/chosen": -0.058852922171354294, + "logits/rejected": -1.5332764387130737, + "logps/chosen": -0.9087274670600891, + "logps/rejected": -7.820371150970459, + "loss": 0.9098, + "num_input_tokens_seen": 9046656, + "odds_ratio_loss": 9.041991233825684, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09087274968624115, + "rewards/margins": 0.6911643743515015, + "rewards/rejected": -0.782037079334259, + "sft_loss": 0.005645034834742546, + "step": 6940 + }, + { + "epoch": 0.5406456631660832, + "grad_norm": 16.496196746826172, + "learning_rate": 2.1816365240472458e-05, + "logits/chosen": -0.05866460129618645, + "logits/rejected": -1.5211254358291626, + "logps/chosen": -0.8906657099723816, + "logps/rejected": -15.501606941223145, + "loss": 0.9026, + "num_input_tokens_seen": 9061104, + "odds_ratio_loss": 8.945243835449219, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.08906657993793488, + "rewards/margins": 1.4610940217971802, + "rewards/rejected": -1.5501604080200195, + "sft_loss": 0.008040891028940678, + "step": 6950 + }, + { + "epoch": 0.5414235705950992, + "grad_norm": 19.269718170166016, + "learning_rate": 2.175577552421853e-05, + "logits/chosen": 0.027465347200632095, + "logits/rejected": -1.0681571960449219, + "logps/chosen": -1.0010095834732056, + "logps/rejected": -6.01483154296875, + "loss": 1.0318, + "num_input_tokens_seen": 9075488, + "odds_ratio_loss": 10.191933631896973, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.10010097175836563, + "rewards/margins": 0.5013822317123413, + "rewards/rejected": -0.601483166217804, + "sft_loss": 0.012642273679375648, + "step": 6960 + }, + { + "epoch": 0.5422014780241151, + "grad_norm": 5.147210121154785, + "learning_rate": 2.1695205184059987e-05, + "logits/chosen": -0.010642116889357567, + "logits/rejected": -1.368715763092041, + "logps/chosen": -0.741737961769104, + "logps/rejected": -6.325432777404785, + "loss": 0.7524, + "num_input_tokens_seen": 9087680, + "odds_ratio_loss": 7.441653251647949, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.07417380064725876, + "rewards/margins": 0.558369517326355, + "rewards/rejected": -0.6325433254241943, + "sft_loss": 0.008275500498712063, + "step": 6970 + }, + { + "epoch": 0.5429793854531311, + "grad_norm": 1.992269515991211, + "learning_rate": 2.1634654581752517e-05, + "logits/chosen": -0.05781219154596329, + "logits/rejected": -1.7768752574920654, + "logps/chosen": -1.0271333456039429, + "logps/rejected": -15.123313903808594, + "loss": 1.0431, + "num_input_tokens_seen": 9103792, + "odds_ratio_loss": 10.401407241821289, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.10271334648132324, + "rewards/margins": 1.4096179008483887, + "rewards/rejected": -1.512331247329712, + "sft_loss": 0.0029968966264277697, + "step": 6980 + }, + { + "epoch": 0.543757292882147, + "grad_norm": 4.649837493896484, + "learning_rate": 2.1574124078933953e-05, + "logits/chosen": -0.3127630352973938, + "logits/rejected": -1.208951711654663, + "logps/chosen": -0.952308177947998, + "logps/rejected": -6.729982852935791, + "loss": 0.9666, + "num_input_tokens_seen": 9119392, + "odds_ratio_loss": 9.563682556152344, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.0952308177947998, + "rewards/margins": 0.5777674913406372, + "rewards/rejected": -0.672998309135437, + "sft_loss": 0.010236539877951145, + "step": 6990 + }, + { + "epoch": 0.544535200311163, + "grad_norm": 3.9194202423095703, + "learning_rate": 2.151361403712205e-05, + "logits/chosen": -0.03569873049855232, + "logits/rejected": -1.1331626176834106, + "logps/chosen": -0.8960397839546204, + "logps/rejected": -8.979105949401855, + "loss": 0.9071, + "num_input_tokens_seen": 9131184, + "odds_ratio_loss": 9.027724266052246, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.0896039754152298, + "rewards/margins": 0.8083066940307617, + "rewards/rejected": -0.8979107141494751, + "sft_loss": 0.004334039054811001, + "step": 7000 + }, + { + "epoch": 0.5453131077401789, + "grad_norm": 4.821101665496826, + "learning_rate": 2.1453124817712382e-05, + "logits/chosen": -0.1305716633796692, + "logits/rejected": -1.4796316623687744, + "logps/chosen": -1.0244022607803345, + "logps/rejected": -13.108938217163086, + "loss": 1.0268, + "num_input_tokens_seen": 9145184, + "odds_ratio_loss": 10.204194068908691, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.10244021564722061, + "rewards/margins": 1.2084537744522095, + "rewards/rejected": -1.3108938932418823, + "sft_loss": 0.0063963839784264565, + "step": 7010 + }, + { + "epoch": 0.5460910151691949, + "grad_norm": 2.2801997661590576, + "learning_rate": 2.1392656781976147e-05, + "logits/chosen": 0.051787324249744415, + "logits/rejected": -1.641805648803711, + "logps/chosen": -0.9896619915962219, + "logps/rejected": -13.139961242675781, + "loss": 1.0347, + "num_input_tokens_seen": 9161856, + "odds_ratio_loss": 10.30107593536377, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.09896618872880936, + "rewards/margins": 1.2150300741195679, + "rewards/rejected": -1.3139961957931519, + "sft_loss": 0.004545119125396013, + "step": 7020 + }, + { + "epoch": 0.5468689225982108, + "grad_norm": 5.194355010986328, + "learning_rate": 2.1332210291058037e-05, + "logits/chosen": -0.1215142235159874, + "logits/rejected": -1.6807050704956055, + "logps/chosen": -1.1550666093826294, + "logps/rejected": -11.24571418762207, + "loss": 1.1721, + "num_input_tokens_seen": 9177152, + "odds_ratio_loss": 11.612730026245117, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.11550666391849518, + "rewards/margins": 1.0090649127960205, + "rewards/rejected": -1.1245715618133545, + "sft_loss": 0.010867111384868622, + "step": 7030 + }, + { + "epoch": 0.5476468300272268, + "grad_norm": 2.0088090896606445, + "learning_rate": 2.1271785705974058e-05, + "logits/chosen": -0.05257201939821243, + "logits/rejected": -1.5861520767211914, + "logps/chosen": -0.8975200653076172, + "logps/rejected": -5.729624271392822, + "loss": 0.9255, + "num_input_tokens_seen": 9191776, + "odds_ratio_loss": 9.136106491088867, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.08975200355052948, + "rewards/margins": 0.48321041464805603, + "rewards/rejected": -0.5729624629020691, + "sft_loss": 0.01193344034254551, + "step": 7040 + }, + { + "epoch": 0.5484247374562428, + "grad_norm": 1.584776520729065, + "learning_rate": 2.1211383387609383e-05, + "logits/chosen": 0.05494903400540352, + "logits/rejected": -1.6136630773544312, + "logps/chosen": -0.849213719367981, + "logps/rejected": -6.129418849945068, + "loss": 0.8748, + "num_input_tokens_seen": 9208192, + "odds_ratio_loss": 8.697319030761719, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.08492136001586914, + "rewards/margins": 0.5280205607414246, + "rewards/rejected": -0.6129419207572937, + "sft_loss": 0.005068082828074694, + "step": 7050 + }, + { + "epoch": 0.5492026448852586, + "grad_norm": 1.491456389427185, + "learning_rate": 2.1151003696716217e-05, + "logits/chosen": -0.013895402662456036, + "logits/rejected": -1.5359677076339722, + "logps/chosen": -1.0025429725646973, + "logps/rejected": -2.2590880393981934, + "loss": 1.0409, + "num_input_tokens_seen": 9220816, + "odds_ratio_loss": 10.346986770629883, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.10025429725646973, + "rewards/margins": 0.12565451860427856, + "rewards/rejected": -0.22590883076190948, + "sft_loss": 0.006227378733456135, + "step": 7060 + }, + { + "epoch": 0.5499805523142746, + "grad_norm": 12.248387336730957, + "learning_rate": 2.1090646993911588e-05, + "logits/chosen": -0.38075366616249084, + "logits/rejected": -0.7645701169967651, + "logps/chosen": -0.8709882497787476, + "logps/rejected": -11.054000854492188, + "loss": 0.8847, + "num_input_tokens_seen": 9233984, + "odds_ratio_loss": 8.402530670166016, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.08709883689880371, + "rewards/margins": 1.018301248550415, + "rewards/rejected": -1.1054000854492188, + "sft_loss": 0.04444083198904991, + "step": 7070 + }, + { + "epoch": 0.5507584597432905, + "grad_norm": 7.897884845733643, + "learning_rate": 2.1030313639675253e-05, + "logits/chosen": -0.16736795008182526, + "logits/rejected": -1.2365648746490479, + "logps/chosen": -0.9248377084732056, + "logps/rejected": -7.286402702331543, + "loss": 0.9708, + "num_input_tokens_seen": 9245328, + "odds_ratio_loss": 9.52192497253418, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.0924837738275528, + "rewards/margins": 0.6361564993858337, + "rewards/rejected": -0.7286401987075806, + "sft_loss": 0.01863746903836727, + "step": 7080 + }, + { + "epoch": 0.5515363671723065, + "grad_norm": 5.933419227600098, + "learning_rate": 2.0970003994347514e-05, + "logits/chosen": 0.006171123590320349, + "logits/rejected": -1.7836297750473022, + "logps/chosen": -0.9997663497924805, + "logps/rejected": -8.293216705322266, + "loss": 1.0218, + "num_input_tokens_seen": 9259760, + "odds_ratio_loss": 10.131877899169922, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.09997662901878357, + "rewards/margins": 0.7293451428413391, + "rewards/rejected": -0.8293216824531555, + "sft_loss": 0.008609093725681305, + "step": 7090 + }, + { + "epoch": 0.5523142746013224, + "grad_norm": 1.800391674041748, + "learning_rate": 2.0909718418127077e-05, + "logits/chosen": -0.3593955636024475, + "logits/rejected": -1.4708583354949951, + "logps/chosen": -1.018897533416748, + "logps/rejected": -9.909947395324707, + "loss": 1.0323, + "num_input_tokens_seen": 9273216, + "odds_ratio_loss": 9.919496536254883, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.10188976675271988, + "rewards/margins": 0.8891050219535828, + "rewards/rejected": -0.9909947514533997, + "sft_loss": 0.04033954441547394, + "step": 7100 + }, + { + "epoch": 0.5530921820303384, + "grad_norm": 6.053109645843506, + "learning_rate": 2.084945727106888e-05, + "logits/chosen": -0.3147508502006531, + "logits/rejected": -1.4761817455291748, + "logps/chosen": -0.9336416125297546, + "logps/rejected": -16.528560638427734, + "loss": 0.9503, + "num_input_tokens_seen": 9284144, + "odds_ratio_loss": 9.027544975280762, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.0933641642332077, + "rewards/margins": 1.5594919919967651, + "rewards/rejected": -1.6528562307357788, + "sft_loss": 0.04757608473300934, + "step": 7110 + }, + { + "epoch": 0.5538700894593543, + "grad_norm": 3.530287742614746, + "learning_rate": 2.0789220913081973e-05, + "logits/chosen": -0.07793496549129486, + "logits/rejected": -1.2224290370941162, + "logps/chosen": -2.3578381538391113, + "logps/rejected": -11.467536926269531, + "loss": 2.4995, + "num_input_tokens_seen": 9294624, + "odds_ratio_loss": 24.468048095703125, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.23578378558158875, + "rewards/margins": 0.9109698534011841, + "rewards/rejected": -1.1467535495758057, + "sft_loss": 0.052667904645204544, + "step": 7120 + }, + { + "epoch": 0.5546479968883703, + "grad_norm": 5.699097633361816, + "learning_rate": 2.0729009703927353e-05, + "logits/chosen": -0.17937228083610535, + "logits/rejected": -1.003334403038025, + "logps/chosen": -0.9862130880355835, + "logps/rejected": -17.108503341674805, + "loss": 0.9927, + "num_input_tokens_seen": 9308192, + "odds_ratio_loss": 9.582437515258789, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09862131625413895, + "rewards/margins": 1.6122291088104248, + "rewards/rejected": -1.7108503580093384, + "sft_loss": 0.03447087109088898, + "step": 7130 + }, + { + "epoch": 0.5554259043173863, + "grad_norm": 3.0519697666168213, + "learning_rate": 2.066882400321581e-05, + "logits/chosen": -0.1276940554380417, + "logits/rejected": -1.6967575550079346, + "logps/chosen": -0.9447711706161499, + "logps/rejected": -10.891839981079102, + "loss": 0.9522, + "num_input_tokens_seen": 9323920, + "odds_ratio_loss": 9.481612205505371, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09447711706161499, + "rewards/margins": 0.9947069883346558, + "rewards/rejected": -1.089184045791626, + "sft_loss": 0.004014966078102589, + "step": 7140 + }, + { + "epoch": 0.5562038117464022, + "grad_norm": 2.3701045513153076, + "learning_rate": 2.0608664170405783e-05, + "logits/chosen": -0.0037519708275794983, + "logits/rejected": -1.6001594066619873, + "logps/chosen": -0.8713818788528442, + "logps/rejected": -8.277201652526855, + "loss": 0.8797, + "num_input_tokens_seen": 9339744, + "odds_ratio_loss": 8.76332950592041, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.08713819831609726, + "rewards/margins": 0.7405818700790405, + "rewards/rejected": -0.827720046043396, + "sft_loss": 0.003366539254784584, + "step": 7150 + }, + { + "epoch": 0.5569817191754182, + "grad_norm": 4.49266242980957, + "learning_rate": 2.054853056480122e-05, + "logits/chosen": -0.19142615795135498, + "logits/rejected": -1.3163836002349854, + "logps/chosen": -1.1975313425064087, + "logps/rejected": -18.15427017211914, + "loss": 1.199, + "num_input_tokens_seen": 9356800, + "odds_ratio_loss": 10.61500358581543, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.11975312232971191, + "rewards/margins": 1.6956737041473389, + "rewards/rejected": -1.8154270648956299, + "sft_loss": 0.1374645233154297, + "step": 7160 + }, + { + "epoch": 0.5577596266044341, + "grad_norm": 4.971721172332764, + "learning_rate": 2.048842354554943e-05, + "logits/chosen": -0.2915323078632355, + "logits/rejected": -0.9623371362686157, + "logps/chosen": -0.7749730944633484, + "logps/rejected": -16.453466415405273, + "loss": 0.7792, + "num_input_tokens_seen": 9370544, + "odds_ratio_loss": 7.725368499755859, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.07749731838703156, + "rewards/margins": 1.5678495168685913, + "rewards/rejected": -1.6453468799591064, + "sft_loss": 0.006691871676594019, + "step": 7170 + }, + { + "epoch": 0.55853753403345, + "grad_norm": 1.6422734260559082, + "learning_rate": 2.0428343471638933e-05, + "logits/chosen": -0.09886634349822998, + "logits/rejected": -1.1431877613067627, + "logps/chosen": -0.7844544649124146, + "logps/rejected": -10.52696418762207, + "loss": 0.8138, + "num_input_tokens_seen": 9380608, + "odds_ratio_loss": 7.857357978820801, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.0784454494714737, + "rewards/margins": 0.9742509722709656, + "rewards/rejected": -1.0526965856552124, + "sft_loss": 0.028091808781027794, + "step": 7180 + }, + { + "epoch": 0.5593154414624659, + "grad_norm": 4.934447288513184, + "learning_rate": 2.0368290701897306e-05, + "logits/chosen": -0.01598060131072998, + "logits/rejected": -1.2897093296051025, + "logps/chosen": -0.9649337530136108, + "logps/rejected": -7.158110618591309, + "loss": 0.9823, + "num_input_tokens_seen": 9389424, + "odds_ratio_loss": 9.752803802490234, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.09649336338043213, + "rewards/margins": 0.6193177103996277, + "rewards/rejected": -0.7158111333847046, + "sft_loss": 0.007014032453298569, + "step": 7190 + }, + { + "epoch": 0.5600933488914819, + "grad_norm": 16.234329223632812, + "learning_rate": 2.030826559498907e-05, + "logits/chosen": 0.006356333382427692, + "logits/rejected": -1.6087524890899658, + "logps/chosen": -0.8868484497070312, + "logps/rejected": -19.774511337280273, + "loss": 0.8881, + "num_input_tokens_seen": 9405392, + "odds_ratio_loss": 8.79615592956543, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.08868485689163208, + "rewards/margins": 1.8887665271759033, + "rewards/rejected": -1.9774510860443115, + "sft_loss": 0.008442886173725128, + "step": 7200 + }, + { + "epoch": 0.5608712563204978, + "grad_norm": 3.596815347671509, + "learning_rate": 2.0248268509413522e-05, + "logits/chosen": -0.15051034092903137, + "logits/rejected": -1.5102225542068481, + "logps/chosen": -1.2850151062011719, + "logps/rejected": -18.5378360748291, + "loss": 1.3145, + "num_input_tokens_seen": 9419088, + "odds_ratio_loss": 12.647809982299805, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.1285015046596527, + "rewards/margins": 1.7252819538116455, + "rewards/rejected": -1.8537836074829102, + "sft_loss": 0.0497058629989624, + "step": 7210 + }, + { + "epoch": 0.5616491637495138, + "grad_norm": 4.4439496994018555, + "learning_rate": 2.01882998035026e-05, + "logits/chosen": -0.6091213226318359, + "logits/rejected": -0.9413172602653503, + "logps/chosen": -1.053614854812622, + "logps/rejected": -25.349395751953125, + "loss": 1.061, + "num_input_tokens_seen": 9439232, + "odds_ratio_loss": 10.500904083251953, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.10536149889230728, + "rewards/margins": 2.4295783042907715, + "rewards/rejected": -2.5349395275115967, + "sft_loss": 0.010934138670563698, + "step": 7220 + }, + { + "epoch": 0.5624270711785297, + "grad_norm": 7.545828819274902, + "learning_rate": 2.012835983541876e-05, + "logits/chosen": -0.1955385059118271, + "logits/rejected": -1.245243787765503, + "logps/chosen": -0.8543983697891235, + "logps/rejected": -22.91373634338379, + "loss": 0.8583, + "num_input_tokens_seen": 9456352, + "odds_ratio_loss": 8.325008392333984, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.08543984591960907, + "rewards/margins": 2.2059340476989746, + "rewards/rejected": -2.2913737297058105, + "sft_loss": 0.02578604780137539, + "step": 7230 + }, + { + "epoch": 0.5632049786075457, + "grad_norm": 2.2859387397766113, + "learning_rate": 2.0068448963152786e-05, + "logits/chosen": -0.06819014251232147, + "logits/rejected": -1.5684789419174194, + "logps/chosen": -0.9775391817092896, + "logps/rejected": -9.996573448181152, + "loss": 0.9786, + "num_input_tokens_seen": 9467104, + "odds_ratio_loss": 9.728795051574707, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09775391221046448, + "rewards/margins": 0.9019034504890442, + "rewards/rejected": -0.9996573328971863, + "sft_loss": 0.0056895045563578606, + "step": 7240 + }, + { + "epoch": 0.5639828860365617, + "grad_norm": 4.704026222229004, + "learning_rate": 2.000856754452173e-05, + "logits/chosen": -0.1956000179052353, + "logits/rejected": -0.9999753832817078, + "logps/chosen": -0.9804566502571106, + "logps/rejected": -5.157223224639893, + "loss": 0.989, + "num_input_tokens_seen": 9480304, + "odds_ratio_loss": 9.775531768798828, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09804566949605942, + "rewards/margins": 0.4176766276359558, + "rewards/rejected": -0.5157222747802734, + "sft_loss": 0.01142149604856968, + "step": 7250 + }, + { + "epoch": 0.5647607934655776, + "grad_norm": 1.6146222352981567, + "learning_rate": 1.9948715937166708e-05, + "logits/chosen": -0.12992647290229797, + "logits/rejected": -1.6173648834228516, + "logps/chosen": -0.927302360534668, + "logps/rejected": -11.353559494018555, + "loss": 0.9374, + "num_input_tokens_seen": 9496448, + "odds_ratio_loss": 9.286779403686523, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.09273023903369904, + "rewards/margins": 1.0426256656646729, + "rewards/rejected": -1.1353559494018555, + "sft_loss": 0.008731330744922161, + "step": 7260 + }, + { + "epoch": 0.5655387008945936, + "grad_norm": 0.37701746821403503, + "learning_rate": 1.9888894498550798e-05, + "logits/chosen": -0.46010270714759827, + "logits/rejected": -1.1463847160339355, + "logps/chosen": -0.7036060690879822, + "logps/rejected": -12.497151374816895, + "loss": 0.7117, + "num_input_tokens_seen": 9509680, + "odds_ratio_loss": 6.6658935546875, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.07036060094833374, + "rewards/margins": 1.1793544292449951, + "rewards/rejected": -1.2497152090072632, + "sft_loss": 0.04514772444963455, + "step": 7270 + }, + { + "epoch": 0.5663166083236095, + "grad_norm": 2.3157222270965576, + "learning_rate": 1.9829103585956882e-05, + "logits/chosen": -0.15636548399925232, + "logits/rejected": -1.0944286584854126, + "logps/chosen": -0.8795326352119446, + "logps/rejected": -13.466470718383789, + "loss": 0.8988, + "num_input_tokens_seen": 9523712, + "odds_ratio_loss": 8.924982070922852, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.08795326948165894, + "rewards/margins": 1.258693814277649, + "rewards/rejected": -1.3466471433639526, + "sft_loss": 0.0063031939789652824, + "step": 7280 + }, + { + "epoch": 0.5670945157526255, + "grad_norm": 6.568509578704834, + "learning_rate": 1.9769343556485544e-05, + "logits/chosen": -0.7389642000198364, + "logits/rejected": -0.5600911974906921, + "logps/chosen": -0.8247898817062378, + "logps/rejected": -9.657159805297852, + "loss": 0.8419, + "num_input_tokens_seen": 9533104, + "odds_ratio_loss": 8.1779146194458, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.08247898519039154, + "rewards/margins": 0.8832368850708008, + "rewards/rejected": -0.965715765953064, + "sft_loss": 0.024064451456069946, + "step": 7290 + }, + { + "epoch": 0.5678724231816414, + "grad_norm": 5.062512397766113, + "learning_rate": 1.97096147670529e-05, + "logits/chosen": -0.3354889750480652, + "logits/rejected": -1.1099668741226196, + "logps/chosen": -2.140015125274658, + "logps/rejected": -15.030134201049805, + "loss": 2.1731, + "num_input_tokens_seen": 9544176, + "odds_ratio_loss": 21.28677749633789, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.21400149166584015, + "rewards/margins": 1.2890119552612305, + "rewards/rejected": -1.5030133724212646, + "sft_loss": 0.04440547153353691, + "step": 7300 + }, + { + "epoch": 0.5686503306106573, + "grad_norm": 3.2668755054473877, + "learning_rate": 1.964991757438851e-05, + "logits/chosen": -0.19430990517139435, + "logits/rejected": -1.2478229999542236, + "logps/chosen": -0.8429762721061707, + "logps/rejected": -13.245553970336914, + "loss": 0.8518, + "num_input_tokens_seen": 9554544, + "odds_ratio_loss": 8.369302749633789, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.08429761976003647, + "rewards/margins": 1.240257740020752, + "rewards/rejected": -1.3245553970336914, + "sft_loss": 0.014858727343380451, + "step": 7310 + }, + { + "epoch": 0.5694282380396732, + "grad_norm": 2.498728036880493, + "learning_rate": 1.9590252335033193e-05, + "logits/chosen": -0.21325652301311493, + "logits/rejected": -1.3000379800796509, + "logps/chosen": -0.7684622406959534, + "logps/rejected": -12.937002182006836, + "loss": 0.7806, + "num_input_tokens_seen": 9564480, + "odds_ratio_loss": 7.7429962158203125, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.07684622704982758, + "rewards/margins": 1.2168540954589844, + "rewards/rejected": -1.2937002182006836, + "sft_loss": 0.006269586272537708, + "step": 7320 + }, + { + "epoch": 0.5702061454686892, + "grad_norm": 1.7052494287490845, + "learning_rate": 1.953061940533695e-05, + "logits/chosen": -0.06910298019647598, + "logits/rejected": -1.5475032329559326, + "logps/chosen": -0.8709535598754883, + "logps/rejected": -8.465534210205078, + "loss": 0.875, + "num_input_tokens_seen": 9574912, + "odds_ratio_loss": 8.691818237304688, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.08709535002708435, + "rewards/margins": 0.759458065032959, + "rewards/rejected": -0.8465534448623657, + "sft_loss": 0.005867579486221075, + "step": 7330 + }, + { + "epoch": 0.5709840528977052, + "grad_norm": 4.655317306518555, + "learning_rate": 1.94710191414568e-05, + "logits/chosen": -0.4469028115272522, + "logits/rejected": -0.8740050196647644, + "logps/chosen": -0.9266998171806335, + "logps/rejected": -10.280158996582031, + "loss": 0.95, + "num_input_tokens_seen": 9587824, + "odds_ratio_loss": 9.35964298248291, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.09266997873783112, + "rewards/margins": 0.9353460073471069, + "rewards/rejected": -1.0280159711837769, + "sft_loss": 0.014021886512637138, + "step": 7340 + }, + { + "epoch": 0.5717619603267211, + "grad_norm": 3.7156572341918945, + "learning_rate": 1.941145189935467e-05, + "logits/chosen": 0.022521795704960823, + "logits/rejected": -1.2985093593597412, + "logps/chosen": -0.9652273058891296, + "logps/rejected": -13.660550117492676, + "loss": 0.9723, + "num_input_tokens_seen": 9599248, + "odds_ratio_loss": 9.660152435302734, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.0965227335691452, + "rewards/margins": 1.269532322883606, + "rewards/rejected": -1.3660550117492676, + "sft_loss": 0.006298977881669998, + "step": 7350 + }, + { + "epoch": 0.5725398677557371, + "grad_norm": 8.959341049194336, + "learning_rate": 1.935191803479527e-05, + "logits/chosen": -0.09870149940252304, + "logits/rejected": -1.3404889106750488, + "logps/chosen": -0.7295455932617188, + "logps/rejected": -3.998936891555786, + "loss": 0.7599, + "num_input_tokens_seen": 9609584, + "odds_ratio_loss": 7.533753871917725, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.07295455038547516, + "rewards/margins": 0.3269391655921936, + "rewards/rejected": -0.39989370107650757, + "sft_loss": 0.006560014095157385, + "step": 7360 + }, + { + "epoch": 0.573317775184753, + "grad_norm": 14.540144920349121, + "learning_rate": 1.9292417903343953e-05, + "logits/chosen": -0.09635542333126068, + "logits/rejected": -1.358878493309021, + "logps/chosen": -1.100290298461914, + "logps/rejected": -12.779852867126465, + "loss": 1.1131, + "num_input_tokens_seen": 9629072, + "odds_ratio_loss": 11.04706859588623, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.11002902686595917, + "rewards/margins": 1.1679562330245972, + "rewards/rejected": -1.2779853343963623, + "sft_loss": 0.008365782909095287, + "step": 7370 + }, + { + "epoch": 0.574095682613769, + "grad_norm": 2.4772889614105225, + "learning_rate": 1.923295186036461e-05, + "logits/chosen": -0.40384602546691895, + "logits/rejected": -0.952344536781311, + "logps/chosen": -0.9728520512580872, + "logps/rejected": -9.531290054321289, + "loss": 0.9861, + "num_input_tokens_seen": 9641760, + "odds_ratio_loss": 9.743693351745605, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.0972852036356926, + "rewards/margins": 0.8558439016342163, + "rewards/rejected": -0.9531289935112, + "sft_loss": 0.011719195172190666, + "step": 7380 + }, + { + "epoch": 0.5748735900427849, + "grad_norm": 94.43122100830078, + "learning_rate": 1.917352026101754e-05, + "logits/chosen": 0.10378698259592056, + "logits/rejected": -1.9902175664901733, + "logps/chosen": -3.0623791217803955, + "logps/rejected": -19.109020233154297, + "loss": 3.1164, + "num_input_tokens_seen": 9659728, + "odds_ratio_loss": 31.07373046875, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.30623793601989746, + "rewards/margins": 1.6046640872955322, + "rewards/rejected": -1.9109020233154297, + "sft_loss": 0.009036138653755188, + "step": 7390 + }, + { + "epoch": 0.5756514974718009, + "grad_norm": 2.3216805458068848, + "learning_rate": 1.9114123460257317e-05, + "logits/chosen": -0.5298863053321838, + "logits/rejected": -1.2425428628921509, + "logps/chosen": -0.8780226707458496, + "logps/rejected": -13.617281913757324, + "loss": 0.8919, + "num_input_tokens_seen": 9669920, + "odds_ratio_loss": 8.812639236450195, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.08780227601528168, + "rewards/margins": 1.27392578125, + "rewards/rejected": -1.3617281913757324, + "sft_loss": 0.010591082274913788, + "step": 7400 + }, + { + "epoch": 0.5764294049008168, + "grad_norm": 2.2508273124694824, + "learning_rate": 1.905476181283069e-05, + "logits/chosen": -0.08014758676290512, + "logits/rejected": -1.3329795598983765, + "logps/chosen": -1.058363676071167, + "logps/rejected": -8.324371337890625, + "loss": 1.1206, + "num_input_tokens_seen": 9679776, + "odds_ratio_loss": 11.035293579101562, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.10583637654781342, + "rewards/margins": 0.7266008853912354, + "rewards/rejected": -0.8324371576309204, + "sft_loss": 0.01711886003613472, + "step": 7410 + }, + { + "epoch": 0.5772073123298328, + "grad_norm": 8.43524169921875, + "learning_rate": 1.8995435673274452e-05, + "logits/chosen": -0.00843315850943327, + "logits/rejected": -1.1164019107818604, + "logps/chosen": -0.9222790002822876, + "logps/rejected": -8.81623363494873, + "loss": 0.9453, + "num_input_tokens_seen": 9693408, + "odds_ratio_loss": 9.378290176391602, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.09222790598869324, + "rewards/margins": 0.7893955111503601, + "rewards/rejected": -0.881623387336731, + "sft_loss": 0.0074365464970469475, + "step": 7420 + }, + { + "epoch": 0.5779852197588486, + "grad_norm": 4.231116771697998, + "learning_rate": 1.8936145395913336e-05, + "logits/chosen": 0.08308565616607666, + "logits/rejected": -1.8230034112930298, + "logps/chosen": -0.9667071104049683, + "logps/rejected": -7.038580417633057, + "loss": 0.9898, + "num_input_tokens_seen": 9708144, + "odds_ratio_loss": 9.870607376098633, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.0966707244515419, + "rewards/margins": 0.6071873307228088, + "rewards/rejected": -0.7038580179214478, + "sft_loss": 0.002733974251896143, + "step": 7430 + }, + { + "epoch": 0.5787631271878646, + "grad_norm": 4.452478885650635, + "learning_rate": 1.8876891334857858e-05, + "logits/chosen": -0.3030247390270233, + "logits/rejected": -1.1824462413787842, + "logps/chosen": -1.0096709728240967, + "logps/rejected": -5.993814468383789, + "loss": 1.0364, + "num_input_tokens_seen": 9717872, + "odds_ratio_loss": 9.989054679870605, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.10096710920333862, + "rewards/margins": 0.49841442704200745, + "rewards/rejected": -0.5993814468383789, + "sft_loss": 0.037446800619363785, + "step": 7440 + }, + { + "epoch": 0.5795410346168806, + "grad_norm": 1.922385573387146, + "learning_rate": 1.8817673844002264e-05, + "logits/chosen": -0.2290133237838745, + "logits/rejected": -1.22044837474823, + "logps/chosen": -0.9724160432815552, + "logps/rejected": -17.186552047729492, + "loss": 0.984, + "num_input_tokens_seen": 9727408, + "odds_ratio_loss": 9.41455078125, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.0972416028380394, + "rewards/margins": 1.6214138269424438, + "rewards/rejected": -1.7186553478240967, + "sft_loss": 0.042552731931209564, + "step": 7450 + }, + { + "epoch": 0.5803189420458965, + "grad_norm": 1.7939796447753906, + "learning_rate": 1.8758493277022377e-05, + "logits/chosen": 0.06815723329782486, + "logits/rejected": -1.399775743484497, + "logps/chosen": -0.7001609802246094, + "logps/rejected": -17.41643524169922, + "loss": 0.7025, + "num_input_tokens_seen": 9741200, + "odds_ratio_loss": 6.983880043029785, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.07001610100269318, + "rewards/margins": 1.671627402305603, + "rewards/rejected": -1.7416435480117798, + "sft_loss": 0.004157370887696743, + "step": 7460 + }, + { + "epoch": 0.5810968494749125, + "grad_norm": 24.65456771850586, + "learning_rate": 1.8699349987373482e-05, + "logits/chosen": -0.09804271161556244, + "logits/rejected": -1.4231456518173218, + "logps/chosen": -1.0049656629562378, + "logps/rejected": -9.602811813354492, + "loss": 1.0205, + "num_input_tokens_seen": 9756272, + "odds_ratio_loss": 9.92216682434082, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.1004965677857399, + "rewards/margins": 0.8597846031188965, + "rewards/rejected": -0.9602810740470886, + "sft_loss": 0.028289655223488808, + "step": 7470 + }, + { + "epoch": 0.5818747569039284, + "grad_norm": 10.561646461486816, + "learning_rate": 1.8640244328288237e-05, + "logits/chosen": -0.5321280360221863, + "logits/rejected": -0.9137857556343079, + "logps/chosen": -1.0886926651000977, + "logps/rejected": -14.622525215148926, + "loss": 1.0946, + "num_input_tokens_seen": 9770016, + "odds_ratio_loss": 10.751686096191406, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.108869269490242, + "rewards/margins": 1.3533833026885986, + "rewards/rejected": -1.4622526168823242, + "sft_loss": 0.019413534551858902, + "step": 7480 + }, + { + "epoch": 0.5826526643329444, + "grad_norm": 2.4361612796783447, + "learning_rate": 1.8581176652774552e-05, + "logits/chosen": -0.15811732411384583, + "logits/rejected": -1.6251471042633057, + "logps/chosen": -1.0075920820236206, + "logps/rejected": -9.168794631958008, + "loss": 1.0262, + "num_input_tokens_seen": 9785408, + "odds_ratio_loss": 10.2182035446167, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.10075920820236206, + "rewards/margins": 0.8161203265190125, + "rewards/rejected": -0.9168795347213745, + "sft_loss": 0.004419906530529261, + "step": 7490 + }, + { + "epoch": 0.5834305717619603, + "grad_norm": 2.919311285018921, + "learning_rate": 1.8522147313613463e-05, + "logits/chosen": -0.3516610264778137, + "logits/rejected": -1.0093597173690796, + "logps/chosen": -0.8289676904678345, + "logps/rejected": -9.245644569396973, + "loss": 0.8332, + "num_input_tokens_seen": 9796592, + "odds_ratio_loss": 7.4911017417907715, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.08289675414562225, + "rewards/margins": 0.8416677713394165, + "rewards/rejected": -0.9245645403862, + "sft_loss": 0.08413822203874588, + "step": 7500 + }, + { + "epoch": 0.5842084791909763, + "grad_norm": 2.8248913288116455, + "learning_rate": 1.8463156663357067e-05, + "logits/chosen": -0.018082041293382645, + "logits/rejected": -1.0669796466827393, + "logps/chosen": -0.9453287124633789, + "logps/rejected": -9.931035041809082, + "loss": 0.9489, + "num_input_tokens_seen": 9814848, + "odds_ratio_loss": 8.671426773071289, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09453286975622177, + "rewards/margins": 0.8985706567764282, + "rewards/rejected": -0.9931035041809082, + "sft_loss": 0.081757552921772, + "step": 7510 + }, + { + "epoch": 0.5849863866199922, + "grad_norm": 5.032744407653809, + "learning_rate": 1.8404205054326385e-05, + "logits/chosen": -0.20301274955272675, + "logits/rejected": -1.5665793418884277, + "logps/chosen": -1.1178547143936157, + "logps/rejected": -12.492952346801758, + "loss": 1.1481, + "num_input_tokens_seen": 9829264, + "odds_ratio_loss": 11.17640209197998, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.11178547143936157, + "rewards/margins": 1.137509822845459, + "rewards/rejected": -1.2492953538894653, + "sft_loss": 0.030486587435007095, + "step": 7520 + }, + { + "epoch": 0.5857642940490082, + "grad_norm": 7.078161239624023, + "learning_rate": 1.834529283860925e-05, + "logits/chosen": -0.34447169303894043, + "logits/rejected": -1.1488739252090454, + "logps/chosen": -0.9062315225601196, + "logps/rejected": -7.6972527503967285, + "loss": 0.917, + "num_input_tokens_seen": 9838160, + "odds_ratio_loss": 8.966435432434082, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.0906231552362442, + "rewards/margins": 0.6791021823883057, + "rewards/rejected": -0.7697253227233887, + "sft_loss": 0.020331600680947304, + "step": 7530 + }, + { + "epoch": 0.5865422014780242, + "grad_norm": 6.74251127243042, + "learning_rate": 1.828642036805823e-05, + "logits/chosen": -0.0008529677870683372, + "logits/rejected": -1.2243870496749878, + "logps/chosen": -0.7502158284187317, + "logps/rejected": -7.243124485015869, + "loss": 0.7729, + "num_input_tokens_seen": 9848976, + "odds_ratio_loss": 7.561089992523193, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.07502157986164093, + "rewards/margins": 0.649290919303894, + "rewards/rejected": -0.7243124842643738, + "sft_loss": 0.016804486513137817, + "step": 7540 + }, + { + "epoch": 0.5873201089070401, + "grad_norm": 1.7365477085113525, + "learning_rate": 1.8227587994288527e-05, + "logits/chosen": -0.33039507269859314, + "logits/rejected": -1.2971482276916504, + "logps/chosen": -0.8243374824523926, + "logps/rejected": -11.704877853393555, + "loss": 0.8381, + "num_input_tokens_seen": 9860736, + "odds_ratio_loss": 8.30073070526123, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.08243375271558762, + "rewards/margins": 1.0880540609359741, + "rewards/rejected": -1.1704877614974976, + "sft_loss": 0.008057581260800362, + "step": 7550 + }, + { + "epoch": 0.588098016336056, + "grad_norm": 4.24249792098999, + "learning_rate": 1.8168796068675846e-05, + "logits/chosen": -0.17244823276996613, + "logits/rejected": -1.3585816621780396, + "logps/chosen": -0.9519235491752625, + "logps/rejected": -13.540249824523926, + "loss": 0.9545, + "num_input_tokens_seen": 9872784, + "odds_ratio_loss": 9.490447998046875, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09519235044717789, + "rewards/margins": 1.2588326930999756, + "rewards/rejected": -1.35402512550354, + "sft_loss": 0.005409600213170052, + "step": 7560 + }, + { + "epoch": 0.5888759237650719, + "grad_norm": 4.247993469238281, + "learning_rate": 1.8110044942354316e-05, + "logits/chosen": 0.012781450524926186, + "logits/rejected": -1.249723196029663, + "logps/chosen": -0.8717182874679565, + "logps/rejected": -8.16511058807373, + "loss": 0.8773, + "num_input_tokens_seen": 9883712, + "odds_ratio_loss": 8.610315322875977, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.08717183768749237, + "rewards/margins": 0.7293392419815063, + "rewards/rejected": -0.8165111541748047, + "sft_loss": 0.01628163829445839, + "step": 7570 + }, + { + "epoch": 0.5896538311940879, + "grad_norm": 10.727156639099121, + "learning_rate": 1.8051334966214407e-05, + "logits/chosen": -0.2588588297367096, + "logits/rejected": -1.214369535446167, + "logps/chosen": -0.7946221828460693, + "logps/rejected": -10.378890991210938, + "loss": 0.8084, + "num_input_tokens_seen": 9899888, + "odds_ratio_loss": 7.984610557556152, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.07946222275495529, + "rewards/margins": 0.9584269523620605, + "rewards/rejected": -1.0378892421722412, + "sft_loss": 0.009910033084452152, + "step": 7580 + }, + { + "epoch": 0.5904317386231038, + "grad_norm": 2.202627182006836, + "learning_rate": 1.799266649090081e-05, + "logits/chosen": 0.1137768030166626, + "logits/rejected": -1.546129822731018, + "logps/chosen": -0.8701799511909485, + "logps/rejected": -12.834440231323242, + "loss": 0.875, + "num_input_tokens_seen": 9910800, + "odds_ratio_loss": 8.691085815429688, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.08701799809932709, + "rewards/margins": 1.1964260339736938, + "rewards/rejected": -1.2834439277648926, + "sft_loss": 0.0058501651510596275, + "step": 7590 + }, + { + "epoch": 0.5912096460521198, + "grad_norm": 17.521636962890625, + "learning_rate": 1.7934039866810355e-05, + "logits/chosen": -0.06771423667669296, + "logits/rejected": -1.020412802696228, + "logps/chosen": -0.9952315092086792, + "logps/rejected": -9.883779525756836, + "loss": 1.0155, + "num_input_tokens_seen": 9919408, + "odds_ratio_loss": 9.928845405578613, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.0995231419801712, + "rewards/margins": 0.8888548016548157, + "rewards/rejected": -0.9883779287338257, + "sft_loss": 0.022567380219697952, + "step": 7600 + }, + { + "epoch": 0.5919875534811357, + "grad_norm": 2.3257410526275635, + "learning_rate": 1.7875455444089924e-05, + "logits/chosen": -0.028352582827210426, + "logits/rejected": -1.1464836597442627, + "logps/chosen": -0.9791489839553833, + "logps/rejected": -11.992913246154785, + "loss": 0.9998, + "num_input_tokens_seen": 9927168, + "odds_ratio_loss": 9.925604820251465, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.09791488945484161, + "rewards/margins": 1.1013762950897217, + "rewards/rejected": -1.1992913484573364, + "sft_loss": 0.007277652621269226, + "step": 7610 + }, + { + "epoch": 0.5927654609101517, + "grad_norm": 11.161792755126953, + "learning_rate": 1.781691357263433e-05, + "logits/chosen": -0.30510300397872925, + "logits/rejected": -0.8714715838432312, + "logps/chosen": -0.7402487993240356, + "logps/rejected": -8.375066757202148, + "loss": 0.7437, + "num_input_tokens_seen": 9935056, + "odds_ratio_loss": 7.290592193603516, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.07402488589286804, + "rewards/margins": 0.7634817361831665, + "rewards/rejected": -0.8375066518783569, + "sft_loss": 0.0146897342056036, + "step": 7620 + }, + { + "epoch": 0.5935433683391677, + "grad_norm": 3.6200902462005615, + "learning_rate": 1.7758414602084278e-05, + "logits/chosen": -0.22039294242858887, + "logits/rejected": -1.0717424154281616, + "logps/chosen": -0.8429325819015503, + "logps/rejected": -16.25857925415039, + "loss": 0.8539, + "num_input_tokens_seen": 9946608, + "odds_ratio_loss": 8.18214225769043, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.08429326117038727, + "rewards/margins": 1.541564702987671, + "rewards/rejected": -1.625857949256897, + "sft_loss": 0.03569791093468666, + "step": 7630 + }, + { + "epoch": 0.5943212757681836, + "grad_norm": 5.2960615158081055, + "learning_rate": 1.7699958881824217e-05, + "logits/chosen": -0.37763914465904236, + "logits/rejected": -0.9068030118942261, + "logps/chosen": -0.9599825143814087, + "logps/rejected": -8.023133277893066, + "loss": 0.9926, + "num_input_tokens_seen": 9961696, + "odds_ratio_loss": 9.783538818359375, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.09599826484918594, + "rewards/margins": 0.7063151597976685, + "rewards/rejected": -0.8023134469985962, + "sft_loss": 0.014255953952670097, + "step": 7640 + }, + { + "epoch": 0.5950991831971996, + "grad_norm": 2.2685546875, + "learning_rate": 1.7641546760980322e-05, + "logits/chosen": -0.32570111751556396, + "logits/rejected": -1.069291114807129, + "logps/chosen": -0.8563793897628784, + "logps/rejected": -11.079155921936035, + "loss": 0.8647, + "num_input_tokens_seen": 9975360, + "odds_ratio_loss": 8.590951919555664, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.08563794195652008, + "rewards/margins": 1.022277593612671, + "rewards/rejected": -1.1079155206680298, + "sft_loss": 0.005621200893074274, + "step": 7650 + }, + { + "epoch": 0.5958770906262155, + "grad_norm": 3.5751876831054688, + "learning_rate": 1.7583178588418336e-05, + "logits/chosen": -0.34155577421188354, + "logits/rejected": -0.8890758752822876, + "logps/chosen": -0.8219859004020691, + "logps/rejected": -6.9678239822387695, + "loss": 0.8456, + "num_input_tokens_seen": 9988336, + "odds_ratio_loss": 8.05842399597168, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.0821986049413681, + "rewards/margins": 0.6145837903022766, + "rewards/rejected": -0.6967824101448059, + "sft_loss": 0.03971128538250923, + "step": 7660 + }, + { + "epoch": 0.5966549980552315, + "grad_norm": 4.907100200653076, + "learning_rate": 1.7524854712741535e-05, + "logits/chosen": 0.010171901434659958, + "logits/rejected": -1.6238276958465576, + "logps/chosen": -0.9058274030685425, + "logps/rejected": -14.971672058105469, + "loss": 0.9245, + "num_input_tokens_seen": 9998880, + "odds_ratio_loss": 9.062615394592285, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.09058274328708649, + "rewards/margins": 1.4065845012664795, + "rewards/rejected": -1.4971672296524048, + "sft_loss": 0.018236646428704262, + "step": 7670 + }, + { + "epoch": 0.5974329054842473, + "grad_norm": 1.8566153049468994, + "learning_rate": 1.7466575482288642e-05, + "logits/chosen": -0.10399254411458969, + "logits/rejected": -1.25956130027771, + "logps/chosen": -1.007875919342041, + "logps/rejected": -13.468701362609863, + "loss": 1.0377, + "num_input_tokens_seen": 10014272, + "odds_ratio_loss": 9.983131408691406, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.10078759491443634, + "rewards/margins": 1.2460824251174927, + "rewards/rejected": -1.3468698263168335, + "sft_loss": 0.039339579641819, + "step": 7680 + }, + { + "epoch": 0.5982108129132633, + "grad_norm": 3.9419636726379395, + "learning_rate": 1.7408341245131704e-05, + "logits/chosen": -0.35398897528648376, + "logits/rejected": -1.1243394613265991, + "logps/chosen": -0.9453102946281433, + "logps/rejected": -10.045394897460938, + "loss": 0.9588, + "num_input_tokens_seen": 10026672, + "odds_ratio_loss": 9.469941139221191, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.09453103691339493, + "rewards/margins": 0.910008430480957, + "rewards/rejected": -1.0045394897460938, + "sft_loss": 0.011818505823612213, + "step": 7690 + }, + { + "epoch": 0.5989887203422792, + "grad_norm": 2.3425137996673584, + "learning_rate": 1.7350152349074083e-05, + "logits/chosen": -0.12070386111736298, + "logits/rejected": -1.457331895828247, + "logps/chosen": -0.8209397196769714, + "logps/rejected": -12.298772811889648, + "loss": 0.8349, + "num_input_tokens_seen": 10040640, + "odds_ratio_loss": 8.312410354614258, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.0820939689874649, + "rewards/margins": 1.1477832794189453, + "rewards/rejected": -1.229877233505249, + "sft_loss": 0.003699915949255228, + "step": 7700 + }, + { + "epoch": 0.5997666277712952, + "grad_norm": 2.7194995880126953, + "learning_rate": 1.7292009141648335e-05, + "logits/chosen": -0.28672319650650024, + "logits/rejected": -1.3284246921539307, + "logps/chosen": -0.7340124845504761, + "logps/rejected": -9.563413619995117, + "loss": 0.7401, + "num_input_tokens_seen": 10051408, + "odds_ratio_loss": 7.306893825531006, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.07340125739574432, + "rewards/margins": 0.8829402923583984, + "rewards/rejected": -0.9563414454460144, + "sft_loss": 0.009374911896884441, + "step": 7710 + }, + { + "epoch": 0.6005445352003111, + "grad_norm": 2.2125983238220215, + "learning_rate": 1.723391197011412e-05, + "logits/chosen": 0.05357031896710396, + "logits/rejected": -1.37665593624115, + "logps/chosen": -0.8307100534439087, + "logps/rejected": -7.978257656097412, + "loss": 0.8534, + "num_input_tokens_seen": 10064832, + "odds_ratio_loss": 8.482166290283203, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.08307100087404251, + "rewards/margins": 0.7147548794746399, + "rewards/rejected": -0.7978259325027466, + "sft_loss": 0.005207848735153675, + "step": 7720 + }, + { + "epoch": 0.6013224426293271, + "grad_norm": 4.190229415893555, + "learning_rate": 1.717586118145617e-05, + "logits/chosen": 0.09481069445610046, + "logits/rejected": -1.8997443914413452, + "logps/chosen": -1.0379550457000732, + "logps/rejected": -10.335776329040527, + "loss": 1.0536, + "num_input_tokens_seen": 10078064, + "odds_ratio_loss": 10.46440315246582, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.10379551351070404, + "rewards/margins": 0.9297822117805481, + "rewards/rejected": -1.0335776805877686, + "sft_loss": 0.0071298182010650635, + "step": 7730 + }, + { + "epoch": 0.6021003500583431, + "grad_norm": 1.5671765804290771, + "learning_rate": 1.711785712238219e-05, + "logits/chosen": -0.21722228825092316, + "logits/rejected": -1.2630269527435303, + "logps/chosen": -0.782172679901123, + "logps/rejected": -13.395368576049805, + "loss": 0.7957, + "num_input_tokens_seen": 10093328, + "odds_ratio_loss": 7.910400390625, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.07821726053953171, + "rewards/margins": 1.261319637298584, + "rewards/rejected": -1.3395369052886963, + "sft_loss": 0.0046241264790296555, + "step": 7740 + }, + { + "epoch": 0.602878257487359, + "grad_norm": 4.133308410644531, + "learning_rate": 1.7059900139320788e-05, + "logits/chosen": 0.09541847556829453, + "logits/rejected": -1.746908187866211, + "logps/chosen": -0.7769309282302856, + "logps/rejected": -16.251314163208008, + "loss": 0.7803, + "num_input_tokens_seen": 10104752, + "odds_ratio_loss": 7.769944667816162, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.07769308984279633, + "rewards/margins": 1.547438383102417, + "rewards/rejected": -1.625131607055664, + "sft_loss": 0.003262136597186327, + "step": 7750 + }, + { + "epoch": 0.603656164916375, + "grad_norm": 2.230847120285034, + "learning_rate": 1.700199057841942e-05, + "logits/chosen": -0.17776095867156982, + "logits/rejected": -1.3732407093048096, + "logps/chosen": -1.0622332096099854, + "logps/rejected": -13.337030410766602, + "loss": 1.0745, + "num_input_tokens_seen": 10117184, + "odds_ratio_loss": 10.697388648986816, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.10622330754995346, + "rewards/margins": 1.2274798154830933, + "rewards/rejected": -1.3337030410766602, + "sft_loss": 0.00480131758376956, + "step": 7760 + }, + { + "epoch": 0.6044340723453909, + "grad_norm": 5.956446647644043, + "learning_rate": 1.694412878554229e-05, + "logits/chosen": -0.011517253704369068, + "logits/rejected": -1.2620794773101807, + "logps/chosen": -1.0131255388259888, + "logps/rejected": -16.502269744873047, + "loss": 1.0178, + "num_input_tokens_seen": 10129328, + "odds_ratio_loss": 10.132184028625488, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.101312555372715, + "rewards/margins": 1.5489144325256348, + "rewards/rejected": -1.6502269506454468, + "sft_loss": 0.004577781073749065, + "step": 7770 + }, + { + "epoch": 0.6052119797744069, + "grad_norm": 7.566381454467773, + "learning_rate": 1.6886315106268356e-05, + "logits/chosen": -0.6538764238357544, + "logits/rejected": -1.3822630643844604, + "logps/chosen": -1.214513897895813, + "logps/rejected": -17.518587112426758, + "loss": 1.2516, + "num_input_tokens_seen": 10142144, + "odds_ratio_loss": 12.307482719421387, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.12145139276981354, + "rewards/margins": 1.6304075717926025, + "rewards/rejected": -1.7518587112426758, + "sft_loss": 0.02084279991686344, + "step": 7780 + }, + { + "epoch": 0.6059898872034228, + "grad_norm": 1.904586672782898, + "learning_rate": 1.6828549885889168e-05, + "logits/chosen": -0.040725018829107285, + "logits/rejected": -1.3858834505081177, + "logps/chosen": -0.8801521062850952, + "logps/rejected": -8.406266212463379, + "loss": 0.8922, + "num_input_tokens_seen": 10153456, + "odds_ratio_loss": 8.876083374023438, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.08801521360874176, + "rewards/margins": 0.7526113986968994, + "rewards/rejected": -0.8406265377998352, + "sft_loss": 0.004563276655972004, + "step": 7790 + }, + { + "epoch": 0.6067677946324388, + "grad_norm": 1.555030107498169, + "learning_rate": 1.677083346940688e-05, + "logits/chosen": -0.042568035423755646, + "logits/rejected": -1.6122106313705444, + "logps/chosen": -0.903033435344696, + "logps/rejected": -12.794513702392578, + "loss": 0.9184, + "num_input_tokens_seen": 10167520, + "odds_ratio_loss": 9.131217956542969, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.09030334651470184, + "rewards/margins": 1.1891480684280396, + "rewards/rejected": -1.2794514894485474, + "sft_loss": 0.005324669647961855, + "step": 7800 + }, + { + "epoch": 0.6075457020614546, + "grad_norm": 6.690324783325195, + "learning_rate": 1.671316620153218e-05, + "logits/chosen": -0.06983957439661026, + "logits/rejected": -1.5677844285964966, + "logps/chosen": -0.8355168104171753, + "logps/rejected": -13.89329719543457, + "loss": 0.8519, + "num_input_tokens_seen": 10179824, + "odds_ratio_loss": 8.46717643737793, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.08355168998241425, + "rewards/margins": 1.3057780265808105, + "rewards/rejected": -1.3893296718597412, + "sft_loss": 0.005196014419198036, + "step": 7810 + }, + { + "epoch": 0.6083236094904706, + "grad_norm": 9.257512092590332, + "learning_rate": 1.665554842668216e-05, + "logits/chosen": -0.1954742968082428, + "logits/rejected": -1.2004436254501343, + "logps/chosen": -0.7859827876091003, + "logps/rejected": -12.374399185180664, + "loss": 0.7911, + "num_input_tokens_seen": 10200016, + "odds_ratio_loss": 7.818950653076172, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.0785982757806778, + "rewards/margins": 1.1588417291641235, + "rewards/rejected": -1.2374398708343506, + "sft_loss": 0.00916118361055851, + "step": 7820 + }, + { + "epoch": 0.6091015169194866, + "grad_norm": 5.428943634033203, + "learning_rate": 1.6597980488978383e-05, + "logits/chosen": -0.22211293876171112, + "logits/rejected": -1.2993932962417603, + "logps/chosen": -1.1758582592010498, + "logps/rejected": -11.725682258605957, + "loss": 1.1917, + "num_input_tokens_seen": 10209760, + "odds_ratio_loss": 10.848090171813965, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.11758583784103394, + "rewards/margins": 1.054982304573059, + "rewards/rejected": -1.1725682020187378, + "sft_loss": 0.1069209948182106, + "step": 7830 + }, + { + "epoch": 0.6098794243485025, + "grad_norm": 40.075374603271484, + "learning_rate": 1.654046273224472e-05, + "logits/chosen": -0.19958052039146423, + "logits/rejected": -0.9851436614990234, + "logps/chosen": -1.0347871780395508, + "logps/rejected": -14.101142883300781, + "loss": 1.0491, + "num_input_tokens_seen": 10224176, + "odds_ratio_loss": 9.863958358764648, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.10347872972488403, + "rewards/margins": 1.3066356182098389, + "rewards/rejected": -1.4101145267486572, + "sft_loss": 0.06271268427371979, + "step": 7840 + }, + { + "epoch": 0.6106573317775185, + "grad_norm": 7.990603446960449, + "learning_rate": 1.648299550000535e-05, + "logits/chosen": -0.23650245368480682, + "logits/rejected": -1.7011165618896484, + "logps/chosen": -1.1272385120391846, + "logps/rejected": -12.137048721313477, + "loss": 1.1298, + "num_input_tokens_seen": 10236544, + "odds_ratio_loss": 11.069969177246094, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.11272384971380234, + "rewards/margins": 1.1009809970855713, + "rewards/rejected": -1.2137049436569214, + "sft_loss": 0.022801917046308517, + "step": 7850 + }, + { + "epoch": 0.6114352392065344, + "grad_norm": 3.8824405670166016, + "learning_rate": 1.6425579135482693e-05, + "logits/chosen": -0.032112546265125275, + "logits/rejected": -1.6255460977554321, + "logps/chosen": -0.8585487604141235, + "logps/rejected": -12.598340034484863, + "loss": 0.8854, + "num_input_tokens_seen": 10247648, + "odds_ratio_loss": 8.798824310302734, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.08585488051176071, + "rewards/margins": 1.1739790439605713, + "rewards/rejected": -1.2598340511322021, + "sft_loss": 0.0054823956452310085, + "step": 7860 + }, + { + "epoch": 0.6122131466355504, + "grad_norm": 15.668688774108887, + "learning_rate": 1.636821398159536e-05, + "logits/chosen": -0.41416144371032715, + "logits/rejected": -1.1996614933013916, + "logps/chosen": -0.9376810193061829, + "logps/rejected": -13.07994556427002, + "loss": 0.9465, + "num_input_tokens_seen": 10259424, + "odds_ratio_loss": 9.251453399658203, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.09376810491085052, + "rewards/margins": 1.2142266035079956, + "rewards/rejected": -1.3079947233200073, + "sft_loss": 0.021313045173883438, + "step": 7870 + }, + { + "epoch": 0.6129910540645663, + "grad_norm": 5.761885643005371, + "learning_rate": 1.6310900380956107e-05, + "logits/chosen": -0.03201372176408768, + "logits/rejected": -1.7225326299667358, + "logps/chosen": -0.733839213848114, + "logps/rejected": -21.30135726928711, + "loss": 0.7377, + "num_input_tokens_seen": 10272656, + "odds_ratio_loss": 7.32309627532959, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.07338393479585648, + "rewards/margins": 2.0567519664764404, + "rewards/rejected": -2.1301357746124268, + "sft_loss": 0.00536178145557642, + "step": 7880 + }, + { + "epoch": 0.6137689614935823, + "grad_norm": 2.743339776992798, + "learning_rate": 1.625363867586979e-05, + "logits/chosen": -0.18800285458564758, + "logits/rejected": -1.488651990890503, + "logps/chosen": -1.2328776121139526, + "logps/rejected": -16.766881942749023, + "loss": 1.2778, + "num_input_tokens_seen": 10283520, + "odds_ratio_loss": 12.68993854522705, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.12328775227069855, + "rewards/margins": 1.5534005165100098, + "rewards/rejected": -1.6766881942749023, + "sft_loss": 0.008772713132202625, + "step": 7890 + }, + { + "epoch": 0.6145468689225982, + "grad_norm": 1.8026765584945679, + "learning_rate": 1.619642920833132e-05, + "logits/chosen": 0.11679697036743164, + "logits/rejected": -1.7344423532485962, + "logps/chosen": -1.8516442775726318, + "logps/rejected": -18.924882888793945, + "loss": 1.8524, + "num_input_tokens_seen": 10300160, + "odds_ratio_loss": 18.48210906982422, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1851644217967987, + "rewards/margins": 1.7073237895965576, + "rewards/rejected": -1.8924882411956787, + "sft_loss": 0.004218951798975468, + "step": 7900 + }, + { + "epoch": 0.6153247763516142, + "grad_norm": 3.9778380393981934, + "learning_rate": 1.6139272320023623e-05, + "logits/chosen": -0.24164679646492004, + "logits/rejected": -0.9055463075637817, + "logps/chosen": -0.7773641347885132, + "logps/rejected": -5.6745500564575195, + "loss": 0.7941, + "num_input_tokens_seen": 10314528, + "odds_ratio_loss": 7.78795862197876, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.07773640006780624, + "rewards/margins": 0.48971861600875854, + "rewards/rejected": -0.567454993724823, + "sft_loss": 0.015351097099483013, + "step": 7910 + }, + { + "epoch": 0.6161026837806302, + "grad_norm": 1.4372708797454834, + "learning_rate": 1.6082168352315595e-05, + "logits/chosen": -0.12904110550880432, + "logits/rejected": -1.357168436050415, + "logps/chosen": -1.0113813877105713, + "logps/rejected": -11.638294219970703, + "loss": 1.0317, + "num_input_tokens_seen": 10327776, + "odds_ratio_loss": 10.201348304748535, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.10113813728094101, + "rewards/margins": 1.062691330909729, + "rewards/rejected": -1.1638294458389282, + "sft_loss": 0.01153038814663887, + "step": 7920 + }, + { + "epoch": 0.616880591209646, + "grad_norm": 62.50389862060547, + "learning_rate": 1.6025117646260064e-05, + "logits/chosen": -0.26859474182128906, + "logits/rejected": -1.2755110263824463, + "logps/chosen": -1.3069630861282349, + "logps/rejected": -12.296360969543457, + "loss": 1.34, + "num_input_tokens_seen": 10344272, + "odds_ratio_loss": 11.950390815734863, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.13069631159305573, + "rewards/margins": 1.0989396572113037, + "rewards/rejected": -1.2296360731124878, + "sft_loss": 0.14492058753967285, + "step": 7930 + }, + { + "epoch": 0.617658498638662, + "grad_norm": 1.6316852569580078, + "learning_rate": 1.5968120542591737e-05, + "logits/chosen": 0.008070459589362144, + "logits/rejected": -1.9038267135620117, + "logps/chosen": -0.8354727625846863, + "logps/rejected": -16.232791900634766, + "loss": 0.8567, + "num_input_tokens_seen": 10357504, + "odds_ratio_loss": 8.52620792388916, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.08354727923870087, + "rewards/margins": 1.5397322177886963, + "rewards/rejected": -1.6232792139053345, + "sft_loss": 0.004094335250556469, + "step": 7940 + }, + { + "epoch": 0.6184364060676779, + "grad_norm": 3.011404275894165, + "learning_rate": 1.5911177381725194e-05, + "logits/chosen": -0.2592969536781311, + "logits/rejected": -1.011600375175476, + "logps/chosen": -1.0075538158416748, + "logps/rejected": -10.504826545715332, + "loss": 1.011, + "num_input_tokens_seen": 10370448, + "odds_ratio_loss": 9.980040550231934, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.10075537860393524, + "rewards/margins": 0.9497271776199341, + "rewards/rejected": -1.0504825115203857, + "sft_loss": 0.01301506906747818, + "step": 7950 + }, + { + "epoch": 0.6192143134966939, + "grad_norm": 2.981290102005005, + "learning_rate": 1.585428850375286e-05, + "logits/chosen": -0.11600154638290405, + "logits/rejected": -1.1014983654022217, + "logps/chosen": -0.946338951587677, + "logps/rejected": -15.650550842285156, + "loss": 0.98, + "num_input_tokens_seen": 10381040, + "odds_ratio_loss": 9.58875846862793, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.09463390707969666, + "rewards/margins": 1.4704211950302124, + "rewards/rejected": -1.565055012702942, + "sft_loss": 0.02110918052494526, + "step": 7960 + }, + { + "epoch": 0.6199922209257098, + "grad_norm": 7.606637477874756, + "learning_rate": 1.5797454248442932e-05, + "logits/chosen": -0.11081588268280029, + "logits/rejected": -1.375892162322998, + "logps/chosen": -0.8813081979751587, + "logps/rejected": -9.27597427368164, + "loss": 0.8969, + "num_input_tokens_seen": 10392496, + "odds_ratio_loss": 8.776693344116211, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.08813081681728363, + "rewards/margins": 0.8394666910171509, + "rewards/rejected": -0.9275975227355957, + "sft_loss": 0.01920653134584427, + "step": 7970 + }, + { + "epoch": 0.6207701283547258, + "grad_norm": 1.7592698335647583, + "learning_rate": 1.5740674955237383e-05, + "logits/chosen": -0.22309553623199463, + "logits/rejected": -1.9166593551635742, + "logps/chosen": -0.9842703938484192, + "logps/rejected": -14.545404434204102, + "loss": 0.9891, + "num_input_tokens_seen": 10406624, + "odds_ratio_loss": 9.730588912963867, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09842703491449356, + "rewards/margins": 1.3561134338378906, + "rewards/rejected": -1.4545403718948364, + "sft_loss": 0.016037700697779655, + "step": 7980 + }, + { + "epoch": 0.6215480357837417, + "grad_norm": 4.15437650680542, + "learning_rate": 1.568395096324992e-05, + "logits/chosen": -0.41758909821510315, + "logits/rejected": -0.8042207956314087, + "logps/chosen": -0.787390947341919, + "logps/rejected": -20.126354217529297, + "loss": 0.7915, + "num_input_tokens_seen": 10420240, + "odds_ratio_loss": 7.432351589202881, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.07873909175395966, + "rewards/margins": 1.9338963031768799, + "rewards/rejected": -2.0126352310180664, + "sft_loss": 0.04827570170164108, + "step": 7990 + }, + { + "epoch": 0.6223259432127577, + "grad_norm": 2.4224088191986084, + "learning_rate": 1.562728261126398e-05, + "logits/chosen": -0.1612803190946579, + "logits/rejected": -1.6714760065078735, + "logps/chosen": -1.6590150594711304, + "logps/rejected": -12.734283447265625, + "loss": 1.7128, + "num_input_tokens_seen": 10434064, + "odds_ratio_loss": 16.774051666259766, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.16590151190757751, + "rewards/margins": 1.1075268983840942, + "rewards/rejected": -1.2734283208847046, + "sft_loss": 0.03537450358271599, + "step": 8000 + }, + { + "epoch": 0.6231038506417736, + "grad_norm": 3.840738296508789, + "learning_rate": 1.557067023773066e-05, + "logits/chosen": -0.06671886146068573, + "logits/rejected": -1.5114303827285767, + "logps/chosen": -0.9673206210136414, + "logps/rejected": -10.580313682556152, + "loss": 0.9822, + "num_input_tokens_seen": 10449616, + "odds_ratio_loss": 9.774526596069336, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.09673206508159637, + "rewards/margins": 0.9612993001937866, + "rewards/rejected": -1.0580313205718994, + "sft_loss": 0.004785637836903334, + "step": 8010 + }, + { + "epoch": 0.6238817580707896, + "grad_norm": 3.5444207191467285, + "learning_rate": 1.5514114180766758e-05, + "logits/chosen": -0.044100046157836914, + "logits/rejected": -1.8607912063598633, + "logps/chosen": -1.0431252717971802, + "logps/rejected": -11.461140632629395, + "loss": 1.0497, + "num_input_tokens_seen": 10462192, + "odds_ratio_loss": 10.443166732788086, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.10431252419948578, + "rewards/margins": 1.0418015718460083, + "rewards/rejected": -1.1461141109466553, + "sft_loss": 0.005407185293734074, + "step": 8020 + }, + { + "epoch": 0.6246596654998056, + "grad_norm": 3.0164756774902344, + "learning_rate": 1.5457614778152696e-05, + "logits/chosen": -0.2252255231142044, + "logits/rejected": -1.2585619688034058, + "logps/chosen": -0.9918392896652222, + "logps/rejected": -18.422239303588867, + "loss": 0.994, + "num_input_tokens_seen": 10475648, + "odds_ratio_loss": 9.408598899841309, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09918393194675446, + "rewards/margins": 1.743039846420288, + "rewards/rejected": -1.842223882675171, + "sft_loss": 0.05314243957400322, + "step": 8030 + }, + { + "epoch": 0.6254375729288215, + "grad_norm": 3.0151381492614746, + "learning_rate": 1.5401172367330563e-05, + "logits/chosen": -0.320361852645874, + "logits/rejected": -0.9782750010490417, + "logps/chosen": -0.9511677622795105, + "logps/rejected": -12.32588005065918, + "loss": 0.9753, + "num_input_tokens_seen": 10487008, + "odds_ratio_loss": 9.589314460754395, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.09511677920818329, + "rewards/margins": 1.1374711990356445, + "rewards/rejected": -1.2325880527496338, + "sft_loss": 0.016387831419706345, + "step": 8040 + }, + { + "epoch": 0.6262154803578374, + "grad_norm": 13.852144241333008, + "learning_rate": 1.5344787285402036e-05, + "logits/chosen": 0.040863268077373505, + "logits/rejected": -1.9360154867172241, + "logps/chosen": -0.9229300618171692, + "logps/rejected": -14.831850051879883, + "loss": 0.9275, + "num_input_tokens_seen": 10499472, + "odds_ratio_loss": 9.100309371948242, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09229300916194916, + "rewards/margins": 1.3908920288085938, + "rewards/rejected": -1.483185052871704, + "sft_loss": 0.01749167963862419, + "step": 8050 + }, + { + "epoch": 0.6269933877868533, + "grad_norm": 5.214956760406494, + "learning_rate": 1.528845986912639e-05, + "logits/chosen": -0.15219607949256897, + "logits/rejected": -1.030737280845642, + "logps/chosen": -0.9987784624099731, + "logps/rejected": -13.901496887207031, + "loss": 1.0396, + "num_input_tokens_seen": 10515952, + "odds_ratio_loss": 10.083951950073242, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.09987784177064896, + "rewards/margins": 1.2902718782424927, + "rewards/rejected": -1.3901498317718506, + "sft_loss": 0.03117167390882969, + "step": 8060 + }, + { + "epoch": 0.6277712952158693, + "grad_norm": 1.3211863040924072, + "learning_rate": 1.5232190454918518e-05, + "logits/chosen": -0.3884350657463074, + "logits/rejected": -1.1633074283599854, + "logps/chosen": -0.9224312901496887, + "logps/rejected": -15.697378158569336, + "loss": 0.933, + "num_input_tokens_seen": 10529248, + "odds_ratio_loss": 8.834152221679688, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.09224313497543335, + "rewards/margins": 1.4774949550628662, + "rewards/rejected": -1.5697380304336548, + "sft_loss": 0.049607302993535995, + "step": 8070 + }, + { + "epoch": 0.6285492026448852, + "grad_norm": 7.5865702629089355, + "learning_rate": 1.5175979378846874e-05, + "logits/chosen": -0.14658516645431519, + "logits/rejected": -1.6767528057098389, + "logps/chosen": -1.00062894821167, + "logps/rejected": -8.337983131408691, + "loss": 1.017, + "num_input_tokens_seen": 10542224, + "odds_ratio_loss": 10.107731819152832, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.10006289184093475, + "rewards/margins": 0.7337354421615601, + "rewards/rejected": -0.8337982892990112, + "sft_loss": 0.006207920610904694, + "step": 8080 + }, + { + "epoch": 0.6293271100739012, + "grad_norm": 1.4249829053878784, + "learning_rate": 1.511982697663151e-05, + "logits/chosen": -0.07819326221942902, + "logits/rejected": -1.4027016162872314, + "logps/chosen": -0.9682968854904175, + "logps/rejected": -8.290817260742188, + "loss": 0.9892, + "num_input_tokens_seen": 10551648, + "odds_ratio_loss": 9.809918403625488, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.09682969748973846, + "rewards/margins": 0.7322520017623901, + "rewards/rejected": -0.8290818333625793, + "sft_loss": 0.008252739906311035, + "step": 8090 + }, + { + "epoch": 0.6301050175029171, + "grad_norm": 3.2043020725250244, + "learning_rate": 1.5063733583642031e-05, + "logits/chosen": -0.21396026015281677, + "logits/rejected": -1.2566635608673096, + "logps/chosen": -1.3404967784881592, + "logps/rejected": -9.582209587097168, + "loss": 1.3798, + "num_input_tokens_seen": 10562768, + "odds_ratio_loss": 13.57678508758545, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.1340496838092804, + "rewards/margins": 0.8241713643074036, + "rewards/rejected": -0.9582209587097168, + "sft_loss": 0.022097880020737648, + "step": 8100 + }, + { + "epoch": 0.6308829249319331, + "grad_norm": 2.394761323928833, + "learning_rate": 1.500769953489562e-05, + "logits/chosen": -0.2207452952861786, + "logits/rejected": -1.4108034372329712, + "logps/chosen": -0.9100346565246582, + "logps/rejected": -10.773725509643555, + "loss": 0.9213, + "num_input_tokens_seen": 10580176, + "odds_ratio_loss": 8.980841636657715, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.09100347757339478, + "rewards/margins": 0.9863690137863159, + "rewards/rejected": -1.0773725509643555, + "sft_loss": 0.023260122165083885, + "step": 8110 + }, + { + "epoch": 0.631660832360949, + "grad_norm": 2.3065590858459473, + "learning_rate": 1.495172516505502e-05, + "logits/chosen": -0.026962101459503174, + "logits/rejected": -1.4148756265640259, + "logps/chosen": -1.1877614259719849, + "logps/rejected": -13.182968139648438, + "loss": 1.1974, + "num_input_tokens_seen": 10595664, + "odds_ratio_loss": 11.025718688964844, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.11877612769603729, + "rewards/margins": 1.199520468711853, + "rewards/rejected": -1.3182967901229858, + "sft_loss": 0.09481464326381683, + "step": 8120 + }, + { + "epoch": 0.632438739789965, + "grad_norm": 2.421651601791382, + "learning_rate": 1.4895810808426547e-05, + "logits/chosen": -0.3440100848674774, + "logits/rejected": -1.4067785739898682, + "logps/chosen": -0.742415726184845, + "logps/rejected": -27.202030181884766, + "loss": 0.7445, + "num_input_tokens_seen": 10609008, + "odds_ratio_loss": 7.3816704750061035, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.07424157112836838, + "rewards/margins": 2.645961284637451, + "rewards/rejected": -2.720202922821045, + "sft_loss": 0.0063473558984696865, + "step": 8130 + }, + { + "epoch": 0.633216647218981, + "grad_norm": 3.3853743076324463, + "learning_rate": 1.4839956798958088e-05, + "logits/chosen": -0.15349504351615906, + "logits/rejected": -1.580263614654541, + "logps/chosen": -0.9464637637138367, + "logps/rejected": -19.33546257019043, + "loss": 0.9567, + "num_input_tokens_seen": 10622096, + "odds_ratio_loss": 9.516892433166504, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.0946463793516159, + "rewards/margins": 1.8388999700546265, + "rewards/rejected": -1.9335464239120483, + "sft_loss": 0.005022524856030941, + "step": 8140 + }, + { + "epoch": 0.6339945546479969, + "grad_norm": 3.37499737739563, + "learning_rate": 1.4784163470237105e-05, + "logits/chosen": 0.038762547075748444, + "logits/rejected": -1.8333539962768555, + "logps/chosen": -0.8853127360343933, + "logps/rejected": -19.017131805419922, + "loss": 0.8871, + "num_input_tokens_seen": 10635136, + "odds_ratio_loss": 8.821528434753418, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.08853127062320709, + "rewards/margins": 1.8131821155548096, + "rewards/rejected": -1.9017131328582764, + "sft_loss": 0.004929094575345516, + "step": 8150 + }, + { + "epoch": 0.6347724620770129, + "grad_norm": 2.917329788208008, + "learning_rate": 1.4728431155488648e-05, + "logits/chosen": -0.06749407947063446, + "logits/rejected": -1.3514297008514404, + "logps/chosen": -0.7193356156349182, + "logps/rejected": -10.20820140838623, + "loss": 0.7239, + "num_input_tokens_seen": 10645920, + "odds_ratio_loss": 7.190954685211182, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.0719335600733757, + "rewards/margins": 0.9488865733146667, + "rewards/rejected": -1.020820140838623, + "sft_loss": 0.004786919802427292, + "step": 8160 + }, + { + "epoch": 0.6355503695060288, + "grad_norm": 17.307886123657227, + "learning_rate": 1.4672760187573361e-05, + "logits/chosen": 0.02161557599902153, + "logits/rejected": -1.5112015008926392, + "logps/chosen": -1.0977082252502441, + "logps/rejected": -11.884286880493164, + "loss": 1.1202, + "num_input_tokens_seen": 10657040, + "odds_ratio_loss": 11.025915145874023, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.10977083444595337, + "rewards/margins": 1.078657865524292, + "rewards/rejected": -1.1884286403656006, + "sft_loss": 0.01760939136147499, + "step": 8170 + }, + { + "epoch": 0.6363282769350447, + "grad_norm": 2.5105154514312744, + "learning_rate": 1.4617150898985504e-05, + "logits/chosen": -0.2205604761838913, + "logits/rejected": -1.1295816898345947, + "logps/chosen": -0.6735633611679077, + "logps/rejected": -12.9937744140625, + "loss": 0.6829, + "num_input_tokens_seen": 10673296, + "odds_ratio_loss": 6.721777439117432, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.06735634058713913, + "rewards/margins": 1.2320213317871094, + "rewards/rejected": -1.29937744140625, + "sft_loss": 0.010709892027080059, + "step": 8180 + }, + { + "epoch": 0.6371061843640606, + "grad_norm": 14.385485649108887, + "learning_rate": 1.4561603621850933e-05, + "logits/chosen": -0.20852942764759064, + "logits/rejected": -1.5024021863937378, + "logps/chosen": -1.0381171703338623, + "logps/rejected": -20.33262062072754, + "loss": 1.039, + "num_input_tokens_seen": 10686848, + "odds_ratio_loss": 10.2601900100708, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.10381171852350235, + "rewards/margins": 1.9294503927230835, + "rewards/rejected": -2.033262014389038, + "sft_loss": 0.012959579937160015, + "step": 8190 + }, + { + "epoch": 0.6378840917930766, + "grad_norm": 4.269653797149658, + "learning_rate": 1.4506118687925158e-05, + "logits/chosen": -0.19100096821784973, + "logits/rejected": -1.4904474020004272, + "logps/chosen": -0.8303168416023254, + "logps/rejected": -18.543073654174805, + "loss": 0.8505, + "num_input_tokens_seen": 10699184, + "odds_ratio_loss": 8.387426376342773, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.08303168416023254, + "rewards/margins": 1.7712757587432861, + "rewards/rejected": -1.8543074131011963, + "sft_loss": 0.011750212870538235, + "step": 8200 + }, + { + "epoch": 0.6386619992220925, + "grad_norm": 1.870347261428833, + "learning_rate": 1.4450696428591346e-05, + "logits/chosen": -0.008823069743812084, + "logits/rejected": -2.0110087394714355, + "logps/chosen": -0.8827959895133972, + "logps/rejected": -27.75350570678711, + "loss": 0.8932, + "num_input_tokens_seen": 10714544, + "odds_ratio_loss": 8.899144172668457, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.088279590010643, + "rewards/margins": 2.6870710849761963, + "rewards/rejected": -2.77535080909729, + "sft_loss": 0.003280414966866374, + "step": 8210 + }, + { + "epoch": 0.6394399066511085, + "grad_norm": 1.4320480823516846, + "learning_rate": 1.4395337174858336e-05, + "logits/chosen": -0.2863291800022125, + "logits/rejected": -1.024493932723999, + "logps/chosen": -0.9053533673286438, + "logps/rejected": -8.440168380737305, + "loss": 0.9208, + "num_input_tokens_seen": 10729824, + "odds_ratio_loss": 8.99482536315918, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.09053533524274826, + "rewards/margins": 0.7534816861152649, + "rewards/rejected": -0.8440170288085938, + "sft_loss": 0.021277474239468575, + "step": 8220 + }, + { + "epoch": 0.6402178140801245, + "grad_norm": 14.688983917236328, + "learning_rate": 1.4340041257358678e-05, + "logits/chosen": -0.026918869465589523, + "logits/rejected": -1.681409478187561, + "logps/chosen": -1.0343601703643799, + "logps/rejected": -15.40973949432373, + "loss": 1.0354, + "num_input_tokens_seen": 10742416, + "odds_ratio_loss": 10.315384864807129, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.10343600809574127, + "rewards/margins": 1.4375379085540771, + "rewards/rejected": -1.5409739017486572, + "sft_loss": 0.0038158062379807234, + "step": 8230 + }, + { + "epoch": 0.6409957215091404, + "grad_norm": 5.5650153160095215, + "learning_rate": 1.4284809006346625e-05, + "logits/chosen": -0.07448375225067139, + "logits/rejected": -1.6491645574569702, + "logps/chosen": -0.9802107810974121, + "logps/rejected": -20.54385757446289, + "loss": 0.9911, + "num_input_tokens_seen": 10758448, + "odds_ratio_loss": 9.796810150146484, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.09802107512950897, + "rewards/margins": 1.956364631652832, + "rewards/rejected": -2.0543856620788574, + "sft_loss": 0.01137588545680046, + "step": 8240 + }, + { + "epoch": 0.6417736289381564, + "grad_norm": 1.2503637075424194, + "learning_rate": 1.4229640751696194e-05, + "logits/chosen": -0.11880844831466675, + "logits/rejected": -1.7745939493179321, + "logps/chosen": -0.8601085543632507, + "logps/rejected": -17.31156349182129, + "loss": 0.8629, + "num_input_tokens_seen": 10770160, + "odds_ratio_loss": 8.552189826965332, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.08601085841655731, + "rewards/margins": 1.6451454162597656, + "rewards/rejected": -1.731156349182129, + "sft_loss": 0.0076408893801271915, + "step": 8250 + }, + { + "epoch": 0.6425515363671723, + "grad_norm": 2.0524260997772217, + "learning_rate": 1.4174536822899201e-05, + "logits/chosen": -0.13288657367229462, + "logits/rejected": -1.4250755310058594, + "logps/chosen": -0.7484859228134155, + "logps/rejected": -5.569182395935059, + "loss": 0.7704, + "num_input_tokens_seen": 10778368, + "odds_ratio_loss": 7.6065993309021, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.07484859228134155, + "rewards/margins": 0.48206964135169983, + "rewards/rejected": -0.5569182634353638, + "sft_loss": 0.009762279689311981, + "step": 8260 + }, + { + "epoch": 0.6433294437961883, + "grad_norm": 3.603144884109497, + "learning_rate": 1.4119497549063254e-05, + "logits/chosen": -0.36294257640838623, + "logits/rejected": -0.45799484848976135, + "logps/chosen": -0.6412696838378906, + "logps/rejected": -5.427443981170654, + "loss": 0.6591, + "num_input_tokens_seen": 10793760, + "odds_ratio_loss": 6.5315985679626465, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.06412696838378906, + "rewards/margins": 0.47861751914024353, + "rewards/rejected": -0.5427444577217102, + "sft_loss": 0.005894807167351246, + "step": 8270 + }, + { + "epoch": 0.6441073512252042, + "grad_norm": 39.19443893432617, + "learning_rate": 1.4064523258909812e-05, + "logits/chosen": -0.17389366030693054, + "logits/rejected": -1.5737632513046265, + "logps/chosen": -1.433529257774353, + "logps/rejected": -17.10720443725586, + "loss": 1.4628, + "num_input_tokens_seen": 10806208, + "odds_ratio_loss": 13.620051383972168, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.1433529406785965, + "rewards/margins": 1.5673677921295166, + "rewards/rejected": -1.7107206583023071, + "sft_loss": 0.10076556354761124, + "step": 8280 + }, + { + "epoch": 0.6448852586542202, + "grad_norm": 3.161728858947754, + "learning_rate": 1.4009614280772232e-05, + "logits/chosen": -0.25053030252456665, + "logits/rejected": -1.4600354433059692, + "logps/chosen": -0.9723641276359558, + "logps/rejected": -8.221105575561523, + "loss": 0.9929, + "num_input_tokens_seen": 10821328, + "odds_ratio_loss": 9.868061065673828, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.09723640978336334, + "rewards/margins": 0.7248741388320923, + "rewards/rejected": -0.8221105337142944, + "sft_loss": 0.006079370621591806, + "step": 8290 + }, + { + "epoch": 0.645663166083236, + "grad_norm": 2.4112048149108887, + "learning_rate": 1.3954770942593799e-05, + "logits/chosen": -0.09294062852859497, + "logits/rejected": -1.1986109018325806, + "logps/chosen": -0.8862360119819641, + "logps/rejected": -10.496671676635742, + "loss": 0.9003, + "num_input_tokens_seen": 10829936, + "odds_ratio_loss": 8.87278938293457, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.08862359821796417, + "rewards/margins": 0.961043655872345, + "rewards/rejected": -1.049667239189148, + "sft_loss": 0.012980528175830841, + "step": 8300 + }, + { + "epoch": 0.646441073512252, + "grad_norm": 9.586309432983398, + "learning_rate": 1.3899993571925723e-05, + "logits/chosen": -0.4729840159416199, + "logits/rejected": -1.2745640277862549, + "logps/chosen": -0.9752651453018188, + "logps/rejected": -20.047935485839844, + "loss": 0.9982, + "num_input_tokens_seen": 10841696, + "odds_ratio_loss": 9.761887550354004, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.09752652049064636, + "rewards/margins": 1.9072672128677368, + "rewards/rejected": -2.004793882369995, + "sft_loss": 0.02201143279671669, + "step": 8310 + }, + { + "epoch": 0.647218980941268, + "grad_norm": 3.2549245357513428, + "learning_rate": 1.3845282495925277e-05, + "logits/chosen": 0.06686688214540482, + "logits/rejected": -1.5578182935714722, + "logps/chosen": -0.7869054675102234, + "logps/rejected": -15.545036315917969, + "loss": 0.7966, + "num_input_tokens_seen": 10858688, + "odds_ratio_loss": 7.923579216003418, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.0786905512213707, + "rewards/margins": 1.4758131504058838, + "rewards/rejected": -1.5545036792755127, + "sft_loss": 0.0042589083313941956, + "step": 8320 + }, + { + "epoch": 0.6479968883702839, + "grad_norm": 1.3830910921096802, + "learning_rate": 1.379063804135376e-05, + "logits/chosen": 0.07782074809074402, + "logits/rejected": -1.69546639919281, + "logps/chosen": -1.0121997594833374, + "logps/rejected": -18.158201217651367, + "loss": 1.0353, + "num_input_tokens_seen": 10873904, + "odds_ratio_loss": 10.268865585327148, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.10121996700763702, + "rewards/margins": 1.7145999670028687, + "rewards/rejected": -1.8158200979232788, + "sft_loss": 0.008449789136648178, + "step": 8330 + }, + { + "epoch": 0.6487747957992999, + "grad_norm": 5.086806774139404, + "learning_rate": 1.373606053457458e-05, + "logits/chosen": -0.14860081672668457, + "logits/rejected": -0.8564743995666504, + "logps/chosen": -1.0690841674804688, + "logps/rejected": -14.578874588012695, + "loss": 1.0799, + "num_input_tokens_seen": 10894304, + "odds_ratio_loss": 10.592825889587402, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.106908418238163, + "rewards/margins": 1.3509790897369385, + "rewards/rejected": -1.4578876495361328, + "sft_loss": 0.020593276247382164, + "step": 8340 + }, + { + "epoch": 0.6495527032283158, + "grad_norm": 2.1482231616973877, + "learning_rate": 1.368155030155131e-05, + "logits/chosen": 0.05506908893585205, + "logits/rejected": -1.4965693950653076, + "logps/chosen": -0.7515527009963989, + "logps/rejected": -9.797318458557129, + "loss": 0.7538, + "num_input_tokens_seen": 10908752, + "odds_ratio_loss": 7.45101261138916, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.07515527307987213, + "rewards/margins": 0.9045765995979309, + "rewards/rejected": -0.9797319173812866, + "sft_loss": 0.008677107281982899, + "step": 8350 + }, + { + "epoch": 0.6503306106573318, + "grad_norm": 2.6537070274353027, + "learning_rate": 1.3627107667845717e-05, + "logits/chosen": -0.24765677750110626, + "logits/rejected": -1.157957673072815, + "logps/chosen": -0.8520873188972473, + "logps/rejected": -6.802451133728027, + "loss": 0.8713, + "num_input_tokens_seen": 10922384, + "odds_ratio_loss": 8.526128768920898, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.08520873636007309, + "rewards/margins": 0.595036506652832, + "rewards/rejected": -0.6802451014518738, + "sft_loss": 0.01870984025299549, + "step": 8360 + }, + { + "epoch": 0.6511085180863477, + "grad_norm": 4.838310718536377, + "learning_rate": 1.3572732958615847e-05, + "logits/chosen": -0.16812340915203094, + "logits/rejected": -1.4218782186508179, + "logps/chosen": -0.890440821647644, + "logps/rejected": -16.090251922607422, + "loss": 0.8994, + "num_input_tokens_seen": 10935392, + "odds_ratio_loss": 8.929390907287598, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.08904408663511276, + "rewards/margins": 1.519981026649475, + "rewards/rejected": -1.609025239944458, + "sft_loss": 0.006434135138988495, + "step": 8370 + }, + { + "epoch": 0.6518864255153637, + "grad_norm": 2.035271406173706, + "learning_rate": 1.3518426498614039e-05, + "logits/chosen": -0.36989468336105347, + "logits/rejected": -1.2575443983078003, + "logps/chosen": -0.9331470727920532, + "logps/rejected": -11.751790046691895, + "loss": 0.9469, + "num_input_tokens_seen": 10950480, + "odds_ratio_loss": 9.325362205505371, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09331471472978592, + "rewards/margins": 1.081864356994629, + "rewards/rejected": -1.1751790046691895, + "sft_loss": 0.01432580966502428, + "step": 8380 + }, + { + "epoch": 0.6526643329443796, + "grad_norm": 3.222139835357666, + "learning_rate": 1.3464188612185058e-05, + "logits/chosen": -0.20379066467285156, + "logits/rejected": -1.3362772464752197, + "logps/chosen": -0.7404529452323914, + "logps/rejected": -11.38569450378418, + "loss": 0.7573, + "num_input_tokens_seen": 10965280, + "odds_ratio_loss": 7.494593143463135, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.07404529303312302, + "rewards/margins": 1.0645242929458618, + "rewards/rejected": -1.1385693550109863, + "sft_loss": 0.007848866283893585, + "step": 8390 + }, + { + "epoch": 0.6534422403733956, + "grad_norm": 3.6170237064361572, + "learning_rate": 1.3410019623264085e-05, + "logits/chosen": -0.2227931022644043, + "logits/rejected": -1.260690689086914, + "logps/chosen": -1.0277048349380493, + "logps/rejected": -14.743139266967773, + "loss": 1.0475, + "num_input_tokens_seen": 10977376, + "odds_ratio_loss": 10.209665298461914, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.10277047008275986, + "rewards/margins": 1.371543526649475, + "rewards/rejected": -1.4743139743804932, + "sft_loss": 0.026581311598420143, + "step": 8400 + }, + { + "epoch": 0.6542201478024116, + "grad_norm": 2.616525650024414, + "learning_rate": 1.3355919855374816e-05, + "logits/chosen": -0.297484815120697, + "logits/rejected": -1.609807014465332, + "logps/chosen": -0.8975094556808472, + "logps/rejected": -12.006285667419434, + "loss": 0.9106, + "num_input_tokens_seen": 10986112, + "odds_ratio_loss": 8.958974838256836, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.08975095301866531, + "rewards/margins": 1.1108777523040771, + "rewards/rejected": -1.2006287574768066, + "sft_loss": 0.014737960882484913, + "step": 8410 + }, + { + "epoch": 0.6549980552314275, + "grad_norm": 3.0650668144226074, + "learning_rate": 1.3301889631627517e-05, + "logits/chosen": 0.17799173295497894, + "logits/rejected": -2.1459360122680664, + "logps/chosen": -0.8587332963943481, + "logps/rejected": -16.28403663635254, + "loss": 0.8656, + "num_input_tokens_seen": 11001168, + "odds_ratio_loss": 8.636218070983887, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.0858733281493187, + "rewards/margins": 1.5425302982330322, + "rewards/rejected": -1.628403902053833, + "sft_loss": 0.0019893767312169075, + "step": 8420 + }, + { + "epoch": 0.6557759626604434, + "grad_norm": 7.51597785949707, + "learning_rate": 1.3247929274717134e-05, + "logits/chosen": -0.13804951310157776, + "logits/rejected": -1.4197338819503784, + "logps/chosen": -0.6933324933052063, + "logps/rejected": -13.103666305541992, + "loss": 0.7249, + "num_input_tokens_seen": 11011248, + "odds_ratio_loss": 7.01900577545166, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.06933324784040451, + "rewards/margins": 1.2410335540771484, + "rewards/rejected": -1.3103668689727783, + "sft_loss": 0.02296510711312294, + "step": 8430 + }, + { + "epoch": 0.6565538700894593, + "grad_norm": 2.676499366760254, + "learning_rate": 1.3194039106921285e-05, + "logits/chosen": -0.29988133907318115, + "logits/rejected": -1.4580739736557007, + "logps/chosen": -0.8631059527397156, + "logps/rejected": -20.36659049987793, + "loss": 0.8746, + "num_input_tokens_seen": 11028512, + "odds_ratio_loss": 8.61852741241455, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.08631060272455215, + "rewards/margins": 1.9503482580184937, + "rewards/rejected": -2.036658763885498, + "sft_loss": 0.012698397040367126, + "step": 8440 + }, + { + "epoch": 0.6573317775184753, + "grad_norm": 13.87569808959961, + "learning_rate": 1.3140219450098418e-05, + "logits/chosen": -0.4483930468559265, + "logits/rejected": -1.1865017414093018, + "logps/chosen": -0.8578631281852722, + "logps/rejected": -11.311476707458496, + "loss": 0.8759, + "num_input_tokens_seen": 11040240, + "odds_ratio_loss": 8.652433395385742, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.08578631281852722, + "rewards/margins": 1.0453612804412842, + "rewards/rejected": -1.1311476230621338, + "sft_loss": 0.010646623559296131, + "step": 8450 + }, + { + "epoch": 0.6581096849474912, + "grad_norm": 3.449667453765869, + "learning_rate": 1.3086470625685854e-05, + "logits/chosen": 0.011820548214018345, + "logits/rejected": -2.015455722808838, + "logps/chosen": -0.992369532585144, + "logps/rejected": -19.755306243896484, + "loss": 1.0005, + "num_input_tokens_seen": 11054704, + "odds_ratio_loss": 9.969560623168945, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09923695772886276, + "rewards/margins": 1.8762937784194946, + "rewards/rejected": -1.975530982017517, + "sft_loss": 0.003559226170182228, + "step": 8460 + }, + { + "epoch": 0.6588875923765072, + "grad_norm": 1.5920977592468262, + "learning_rate": 1.3032792954697862e-05, + "logits/chosen": 0.04607129842042923, + "logits/rejected": -1.4695940017700195, + "logps/chosen": -0.8204959034919739, + "logps/rejected": -20.647167205810547, + "loss": 0.8206, + "num_input_tokens_seen": 11070592, + "odds_ratio_loss": 8.174798965454102, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.08204958587884903, + "rewards/margins": 1.9826672077178955, + "rewards/rejected": -2.0647168159484863, + "sft_loss": 0.0030748662538826466, + "step": 8470 + }, + { + "epoch": 0.6596654998055231, + "grad_norm": 65.4085922241211, + "learning_rate": 1.2979186757723744e-05, + "logits/chosen": -0.40389958024024963, + "logits/rejected": -1.273033857345581, + "logps/chosen": -0.9728702306747437, + "logps/rejected": -11.638945579528809, + "loss": 0.982, + "num_input_tokens_seen": 11080528, + "odds_ratio_loss": 9.580873489379883, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09728703647851944, + "rewards/margins": 1.0666075944900513, + "rewards/rejected": -1.163894534111023, + "sft_loss": 0.023938244208693504, + "step": 8480 + }, + { + "epoch": 0.6604434072345391, + "grad_norm": 3.122735023498535, + "learning_rate": 1.2925652354925927e-05, + "logits/chosen": -0.05589236691594124, + "logits/rejected": -1.5410343408584595, + "logps/chosen": -0.8380798101425171, + "logps/rejected": -19.911548614501953, + "loss": 0.8463, + "num_input_tokens_seen": 11090880, + "odds_ratio_loss": 8.223835945129395, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.08380798250436783, + "rewards/margins": 1.9073469638824463, + "rewards/rejected": -1.9911549091339111, + "sft_loss": 0.023908449336886406, + "step": 8490 + }, + { + "epoch": 0.661221314663555, + "grad_norm": 3.586082935333252, + "learning_rate": 1.2872190066038056e-05, + "logits/chosen": -0.19564349949359894, + "logits/rejected": -1.6615159511566162, + "logps/chosen": -0.9936773180961609, + "logps/rejected": -6.101696491241455, + "loss": 1.0242, + "num_input_tokens_seen": 11104016, + "odds_ratio_loss": 10.164363861083984, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.09936773777008057, + "rewards/margins": 0.5108018517494202, + "rewards/rejected": -0.6101695895195007, + "sft_loss": 0.0077539486810564995, + "step": 8500 + }, + { + "epoch": 0.661999222092571, + "grad_norm": 3.438119888305664, + "learning_rate": 1.2818800210363063e-05, + "logits/chosen": -0.04792151600122452, + "logits/rejected": -1.3143954277038574, + "logps/chosen": -0.8305968046188354, + "logps/rejected": -12.585901260375977, + "loss": 0.8629, + "num_input_tokens_seen": 11117712, + "odds_ratio_loss": 8.393974304199219, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.08305969089269638, + "rewards/margins": 1.1755305528640747, + "rewards/rejected": -1.2585902214050293, + "sft_loss": 0.023479651659727097, + "step": 8510 + }, + { + "epoch": 0.662777129521587, + "grad_norm": 1.5884816646575928, + "learning_rate": 1.2765483106771275e-05, + "logits/chosen": -0.11270128190517426, + "logits/rejected": -1.8259063959121704, + "logps/chosen": -0.9288307428359985, + "logps/rejected": -12.536392211914062, + "loss": 0.9358, + "num_input_tokens_seen": 11131488, + "odds_ratio_loss": 9.29153823852539, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09288307279348373, + "rewards/margins": 1.1607561111450195, + "rewards/rejected": -1.2536393404006958, + "sft_loss": 0.006653589196503162, + "step": 8520 + }, + { + "epoch": 0.6635550369506029, + "grad_norm": 1.9690076112747192, + "learning_rate": 1.271223907369853e-05, + "logits/chosen": -0.41172775626182556, + "logits/rejected": -1.3405262231826782, + "logps/chosen": -0.9557029604911804, + "logps/rejected": -12.360254287719727, + "loss": 0.9689, + "num_input_tokens_seen": 11141600, + "odds_ratio_loss": 9.551517486572266, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.09557029604911804, + "rewards/margins": 1.1404552459716797, + "rewards/rejected": -1.2360254526138306, + "sft_loss": 0.013766573742032051, + "step": 8530 + }, + { + "epoch": 0.6643329443796189, + "grad_norm": 2.0265607833862305, + "learning_rate": 1.2659068429144228e-05, + "logits/chosen": -0.4004293382167816, + "logits/rejected": -1.3477731943130493, + "logps/chosen": -0.8370245099067688, + "logps/rejected": -14.558506965637207, + "loss": 0.8566, + "num_input_tokens_seen": 11155056, + "odds_ratio_loss": 8.357165336608887, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.0837024673819542, + "rewards/margins": 1.3721481561660767, + "rewards/rejected": -1.45585036277771, + "sft_loss": 0.02086927741765976, + "step": 8540 + }, + { + "epoch": 0.6651108518086347, + "grad_norm": 1.8647454977035522, + "learning_rate": 1.2605971490669471e-05, + "logits/chosen": -0.2736256420612335, + "logits/rejected": -1.54111647605896, + "logps/chosen": -0.7586938738822937, + "logps/rejected": -18.400957107543945, + "loss": 0.7653, + "num_input_tokens_seen": 11171168, + "odds_ratio_loss": 7.573142051696777, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.07586938887834549, + "rewards/margins": 1.7642263174057007, + "rewards/rejected": -1.8400957584381104, + "sft_loss": 0.00795892346650362, + "step": 8550 + }, + { + "epoch": 0.6658887592376507, + "grad_norm": 5.958194255828857, + "learning_rate": 1.2552948575395135e-05, + "logits/chosen": -0.16555894911289215, + "logits/rejected": -1.250742793083191, + "logps/chosen": -0.975231945514679, + "logps/rejected": -8.279606819152832, + "loss": 1.0221, + "num_input_tokens_seen": 11181328, + "odds_ratio_loss": 10.13719367980957, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.09752318263053894, + "rewards/margins": 0.7304375171661377, + "rewards/rejected": -0.8279607892036438, + "sft_loss": 0.008426772430539131, + "step": 8560 + }, + { + "epoch": 0.6666666666666666, + "grad_norm": 6.090371608734131, + "learning_rate": 1.2500000000000006e-05, + "logits/chosen": -0.26315200328826904, + "logits/rejected": -1.2788677215576172, + "logps/chosen": -0.8311313390731812, + "logps/rejected": -16.073301315307617, + "loss": 0.8437, + "num_input_tokens_seen": 11191392, + "odds_ratio_loss": 8.264939308166504, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.08311314135789871, + "rewards/margins": 1.5242170095443726, + "rewards/rejected": -1.607330322265625, + "sft_loss": 0.017202703282237053, + "step": 8570 + }, + { + "epoch": 0.6674445740956826, + "grad_norm": 2.4439313411712646, + "learning_rate": 1.2447126080718876e-05, + "logits/chosen": -0.11930867284536362, + "logits/rejected": -1.3498518466949463, + "logps/chosen": -0.8209142684936523, + "logps/rejected": -9.882196426391602, + "loss": 0.8278, + "num_input_tokens_seen": 11207888, + "odds_ratio_loss": 8.222280502319336, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.08209143579006195, + "rewards/margins": 0.9061282277107239, + "rewards/rejected": -0.9882196187973022, + "sft_loss": 0.005587009247392416, + "step": 8580 + }, + { + "epoch": 0.6682224815246985, + "grad_norm": 13.487736701965332, + "learning_rate": 1.2394327133340658e-05, + "logits/chosen": -0.32252174615859985, + "logits/rejected": -1.2589683532714844, + "logps/chosen": -0.8944998979568481, + "logps/rejected": -17.84218978881836, + "loss": 0.9001, + "num_input_tokens_seen": 11220624, + "odds_ratio_loss": 8.879426002502441, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.08944999426603317, + "rewards/margins": 1.6947689056396484, + "rewards/rejected": -1.7842190265655518, + "sft_loss": 0.012113329954445362, + "step": 8590 + }, + { + "epoch": 0.6690003889537145, + "grad_norm": 3.8251841068267822, + "learning_rate": 1.2341603473206486e-05, + "logits/chosen": -0.5418458580970764, + "logits/rejected": -1.5960874557495117, + "logps/chosen": -1.3800010681152344, + "logps/rejected": -14.487066268920898, + "loss": 1.4506, + "num_input_tokens_seen": 11232528, + "odds_ratio_loss": 14.339879035949707, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.13800010085105896, + "rewards/margins": 1.310706377029419, + "rewards/rejected": -1.4487067461013794, + "sft_loss": 0.016607291996479034, + "step": 8600 + }, + { + "epoch": 0.6697782963827305, + "grad_norm": 12.286779403686523, + "learning_rate": 1.2288955415207848e-05, + "logits/chosen": -0.3529645800590515, + "logits/rejected": -1.4659942388534546, + "logps/chosen": -0.9061535000801086, + "logps/rejected": -27.138967514038086, + "loss": 0.9102, + "num_input_tokens_seen": 11244800, + "odds_ratio_loss": 9.013811111450195, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09061536192893982, + "rewards/margins": 2.623281717300415, + "rewards/rejected": -2.7138967514038086, + "sft_loss": 0.008838102221488953, + "step": 8610 + }, + { + "epoch": 0.6705562038117464, + "grad_norm": 3.788783073425293, + "learning_rate": 1.2236383273784696e-05, + "logits/chosen": -0.4239223599433899, + "logits/rejected": -1.192138910293579, + "logps/chosen": -0.9421186447143555, + "logps/rejected": -14.43946647644043, + "loss": 0.9622, + "num_input_tokens_seen": 11254816, + "odds_ratio_loss": 9.390454292297363, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.0942118763923645, + "rewards/margins": 1.3497350215911865, + "rewards/rejected": -1.4439469575881958, + "sft_loss": 0.023183036595582962, + "step": 8620 + }, + { + "epoch": 0.6713341112407624, + "grad_norm": 1.819946527481079, + "learning_rate": 1.2183887362923577e-05, + "logits/chosen": -0.10835647583007812, + "logits/rejected": -1.4449514150619507, + "logps/chosen": -0.9603055715560913, + "logps/rejected": -14.043817520141602, + "loss": 0.9647, + "num_input_tokens_seen": 11264864, + "odds_ratio_loss": 9.427949905395508, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09603056311607361, + "rewards/margins": 1.3083511590957642, + "rewards/rejected": -1.4043816328048706, + "sft_loss": 0.021934445947408676, + "step": 8630 + }, + { + "epoch": 0.6721120186697783, + "grad_norm": 1.8487447500228882, + "learning_rate": 1.213146799615574e-05, + "logits/chosen": 0.02626688778400421, + "logits/rejected": -1.3874980211257935, + "logps/chosen": -0.8622997403144836, + "logps/rejected": -8.232680320739746, + "loss": 0.88, + "num_input_tokens_seen": 11275808, + "odds_ratio_loss": 8.757139205932617, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.08622998744249344, + "rewards/margins": 0.7370380759239197, + "rewards/rejected": -0.8232681155204773, + "sft_loss": 0.004265786614269018, + "step": 8640 + }, + { + "epoch": 0.6728899260987943, + "grad_norm": 1.3646889925003052, + "learning_rate": 1.2079125486555279e-05, + "logits/chosen": -0.10303683578968048, + "logits/rejected": -1.6437324285507202, + "logps/chosen": -0.8649684190750122, + "logps/rejected": -11.726491928100586, + "loss": 0.8771, + "num_input_tokens_seen": 11289264, + "odds_ratio_loss": 8.71319580078125, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.08649684488773346, + "rewards/margins": 1.0861523151397705, + "rewards/rejected": -1.1726491451263428, + "sft_loss": 0.0057486421428620815, + "step": 8650 + }, + { + "epoch": 0.6736678335278102, + "grad_norm": 2.016709804534912, + "learning_rate": 1.2026860146737281e-05, + "logits/chosen": -0.7974362969398499, + "logits/rejected": -1.0762536525726318, + "logps/chosen": -0.8179294466972351, + "logps/rejected": -10.683052062988281, + "loss": 0.8308, + "num_input_tokens_seen": 11301696, + "odds_ratio_loss": 8.224560737609863, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.08179295808076859, + "rewards/margins": 0.9865123629570007, + "rewards/rejected": -1.0683053731918335, + "sft_loss": 0.008364574983716011, + "step": 8660 + }, + { + "epoch": 0.6744457409568262, + "grad_norm": 1.868152379989624, + "learning_rate": 1.1974672288855912e-05, + "logits/chosen": -0.24265766143798828, + "logits/rejected": -2.250298023223877, + "logps/chosen": -0.9682738184928894, + "logps/rejected": -21.465303421020508, + "loss": 0.9782, + "num_input_tokens_seen": 11315600, + "odds_ratio_loss": 9.635416030883789, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.09682738035917282, + "rewards/margins": 2.0497031211853027, + "rewards/rejected": -2.1465301513671875, + "sft_loss": 0.014634659513831139, + "step": 8670 + }, + { + "epoch": 0.675223648385842, + "grad_norm": 1.5183236598968506, + "learning_rate": 1.192256222460259e-05, + "logits/chosen": -0.0016524322563782334, + "logits/rejected": -1.906431794166565, + "logps/chosen": -0.9167082905769348, + "logps/rejected": -12.803431510925293, + "loss": 0.9351, + "num_input_tokens_seen": 11329840, + "odds_ratio_loss": 9.321020126342773, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.09167083352804184, + "rewards/margins": 1.1886723041534424, + "rewards/rejected": -1.2803431749343872, + "sft_loss": 0.0030473307706415653, + "step": 8680 + }, + { + "epoch": 0.676001555814858, + "grad_norm": 2.5191078186035156, + "learning_rate": 1.1870530265204091e-05, + "logits/chosen": -0.3458879590034485, + "logits/rejected": -1.210057020187378, + "logps/chosen": -0.9681374430656433, + "logps/rejected": -17.70801544189453, + "loss": 0.9687, + "num_input_tokens_seen": 11342304, + "odds_ratio_loss": 9.132461547851562, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09681375324726105, + "rewards/margins": 1.6739877462387085, + "rewards/rejected": -1.7708015441894531, + "sft_loss": 0.05550052598118782, + "step": 8690 + }, + { + "epoch": 0.676779463243874, + "grad_norm": 11.486089706420898, + "learning_rate": 1.1818576721420734e-05, + "logits/chosen": -0.07480702549219131, + "logits/rejected": -1.7288659811019897, + "logps/chosen": -0.9699427485466003, + "logps/rejected": -11.04030990600586, + "loss": 0.9763, + "num_input_tokens_seen": 11360608, + "odds_ratio_loss": 9.68722152709961, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09699428081512451, + "rewards/margins": 1.007036566734314, + "rewards/rejected": -1.104030966758728, + "sft_loss": 0.007532379124313593, + "step": 8700 + }, + { + "epoch": 0.6775573706728899, + "grad_norm": 3.7408041954040527, + "learning_rate": 1.1766701903544497e-05, + "logits/chosen": -0.5660278797149658, + "logits/rejected": -1.1048133373260498, + "logps/chosen": -0.7535013556480408, + "logps/rejected": -13.154385566711426, + "loss": 0.7904, + "num_input_tokens_seen": 11374576, + "odds_ratio_loss": 7.830130100250244, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.07535013556480408, + "rewards/margins": 1.2400884628295898, + "rewards/rejected": -1.3154386281967163, + "sft_loss": 0.007340291980654001, + "step": 8710 + }, + { + "epoch": 0.6783352781019059, + "grad_norm": 1.3837281465530396, + "learning_rate": 1.1714906121397162e-05, + "logits/chosen": -0.07832653075456619, + "logits/rejected": -1.9233297109603882, + "logps/chosen": -1.125610589981079, + "logps/rejected": -14.071528434753418, + "loss": 1.1281, + "num_input_tokens_seen": 11391488, + "odds_ratio_loss": 11.223248481750488, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.11256106942892075, + "rewards/margins": 1.2945916652679443, + "rewards/rejected": -1.407152771949768, + "sft_loss": 0.005823501385748386, + "step": 8720 + }, + { + "epoch": 0.6791131855309218, + "grad_norm": 1.702384114265442, + "learning_rate": 1.166318968432848e-05, + "logits/chosen": -0.033839546144008636, + "logits/rejected": -1.8430595397949219, + "logps/chosen": -0.812944233417511, + "logps/rejected": -22.31185531616211, + "loss": 0.8148, + "num_input_tokens_seen": 11404816, + "odds_ratio_loss": 8.115874290466309, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.08129443228244781, + "rewards/margins": 2.1498911380767822, + "rewards/rejected": -2.2311854362487793, + "sft_loss": 0.0032049964647740126, + "step": 8730 + }, + { + "epoch": 0.6798910929599378, + "grad_norm": 2.200639009475708, + "learning_rate": 1.16115529012143e-05, + "logits/chosen": 0.025216102600097656, + "logits/rejected": -1.8154652118682861, + "logps/chosen": -1.0452499389648438, + "logps/rejected": -17.805185317993164, + "loss": 1.0578, + "num_input_tokens_seen": 11423280, + "odds_ratio_loss": 10.54097843170166, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.10452499240636826, + "rewards/margins": 1.67599356174469, + "rewards/rejected": -1.7805185317993164, + "sft_loss": 0.003734355792403221, + "step": 8740 + }, + { + "epoch": 0.6806690003889537, + "grad_norm": 5.125865459442139, + "learning_rate": 1.1559996080454752e-05, + "logits/chosen": -0.46686309576034546, + "logits/rejected": -1.0633283853530884, + "logps/chosen": -1.0171476602554321, + "logps/rejected": -12.088407516479492, + "loss": 1.0336, + "num_input_tokens_seen": 11435520, + "odds_ratio_loss": 9.8032865524292, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.10171475261449814, + "rewards/margins": 1.1071261167526245, + "rewards/rejected": -1.2088408470153809, + "sft_loss": 0.053316712379455566, + "step": 8750 + }, + { + "epoch": 0.6814469078179697, + "grad_norm": 2.6860620975494385, + "learning_rate": 1.150851952997239e-05, + "logits/chosen": -0.27116113901138306, + "logits/rejected": -1.396429419517517, + "logps/chosen": -0.8566263318061829, + "logps/rejected": -12.005642890930176, + "loss": 0.8583, + "num_input_tokens_seen": 11448432, + "odds_ratio_loss": 8.500247955322266, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.08566261827945709, + "rewards/margins": 1.1149016618728638, + "rewards/rejected": -1.2005643844604492, + "sft_loss": 0.00828742515295744, + "step": 8760 + }, + { + "epoch": 0.6822248152469856, + "grad_norm": 3.973215341567993, + "learning_rate": 1.1457123557210347e-05, + "logits/chosen": -0.6482058763504028, + "logits/rejected": -0.9667502641677856, + "logps/chosen": -0.8053656816482544, + "logps/rejected": -10.670485496520996, + "loss": 0.8311, + "num_input_tokens_seen": 11457328, + "odds_ratio_loss": 8.189837455749512, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.08053655922412872, + "rewards/margins": 0.9865120053291321, + "rewards/rejected": -1.0670485496520996, + "sft_loss": 0.012087037786841393, + "step": 8770 + }, + { + "epoch": 0.6830027226760016, + "grad_norm": 1.684739351272583, + "learning_rate": 1.1405808469130504e-05, + "logits/chosen": -0.32509225606918335, + "logits/rejected": -1.5012362003326416, + "logps/chosen": -0.9674826860427856, + "logps/rejected": -18.22718048095703, + "loss": 0.9736, + "num_input_tokens_seen": 11471040, + "odds_ratio_loss": 9.610559463500977, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09674827009439468, + "rewards/margins": 1.725969910621643, + "rewards/rejected": -1.8227182626724243, + "sft_loss": 0.012532609514892101, + "step": 8780 + }, + { + "epoch": 0.6837806301050176, + "grad_norm": 1.870661973953247, + "learning_rate": 1.1354574572211685e-05, + "logits/chosen": -0.03116082027554512, + "logits/rejected": -1.4862477779388428, + "logps/chosen": -0.7511264681816101, + "logps/rejected": -10.093324661254883, + "loss": 0.7804, + "num_input_tokens_seen": 11483152, + "odds_ratio_loss": 7.756175994873047, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.07511265575885773, + "rewards/margins": 0.9342197179794312, + "rewards/rejected": -1.0093324184417725, + "sft_loss": 0.004796931054443121, + "step": 8790 + }, + { + "epoch": 0.6845585375340334, + "grad_norm": 7.217814922332764, + "learning_rate": 1.1303422172447781e-05, + "logits/chosen": -0.1332283765077591, + "logits/rejected": -1.6998363733291626, + "logps/chosen": -0.7651488780975342, + "logps/rejected": -13.795843124389648, + "loss": 0.7664, + "num_input_tokens_seen": 11494192, + "odds_ratio_loss": 7.457201957702637, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.07651487737894058, + "rewards/margins": 1.3030694723129272, + "rewards/rejected": -1.3795844316482544, + "sft_loss": 0.02067355439066887, + "step": 8800 + }, + { + "epoch": 0.6853364449630494, + "grad_norm": 4.628015041351318, + "learning_rate": 1.1252351575345923e-05, + "logits/chosen": -0.05683094263076782, + "logits/rejected": -1.7242136001586914, + "logps/chosen": -0.7889854311943054, + "logps/rejected": -22.717567443847656, + "loss": 0.7949, + "num_input_tokens_seen": 11509200, + "odds_ratio_loss": 7.907063961029053, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.07889854162931442, + "rewards/margins": 2.1928582191467285, + "rewards/rejected": -2.271756649017334, + "sft_loss": 0.004231655970215797, + "step": 8810 + }, + { + "epoch": 0.6861143523920653, + "grad_norm": 2.469001531600952, + "learning_rate": 1.1201363085924709e-05, + "logits/chosen": -0.21477165818214417, + "logits/rejected": -0.7675528526306152, + "logps/chosen": -0.9782177805900574, + "logps/rejected": -21.221256256103516, + "loss": 1.0021, + "num_input_tokens_seen": 11522208, + "odds_ratio_loss": 9.50112247467041, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.09782177209854126, + "rewards/margins": 2.024303913116455, + "rewards/rejected": -2.1221256256103516, + "sft_loss": 0.052003372460603714, + "step": 8820 + }, + { + "epoch": 0.6868922598210813, + "grad_norm": 2.0622456073760986, + "learning_rate": 1.1150457008712337e-05, + "logits/chosen": 0.011802531778812408, + "logits/rejected": -1.9207899570465088, + "logps/chosen": -0.9674263000488281, + "logps/rejected": -21.708913803100586, + "loss": 0.9748, + "num_input_tokens_seen": 11538160, + "odds_ratio_loss": 9.71261978149414, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09674263000488281, + "rewards/margins": 2.074148654937744, + "rewards/rejected": -2.170891284942627, + "sft_loss": 0.0035202347207814455, + "step": 8830 + }, + { + "epoch": 0.6876701672500972, + "grad_norm": 2.0015759468078613, + "learning_rate": 1.1099633647744797e-05, + "logits/chosen": -0.19650664925575256, + "logits/rejected": -1.1552398204803467, + "logps/chosen": -2.600151538848877, + "logps/rejected": -9.30272102355957, + "loss": 2.6104, + "num_input_tokens_seen": 11552928, + "odds_ratio_loss": 25.9542179107666, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.26001518964767456, + "rewards/margins": 0.6702569723129272, + "rewards/rejected": -0.9302722215652466, + "sft_loss": 0.014955626800656319, + "step": 8840 + }, + { + "epoch": 0.6884480746791132, + "grad_norm": 2.001830577850342, + "learning_rate": 1.1048893306564062e-05, + "logits/chosen": -0.30013757944107056, + "logits/rejected": -1.426927924156189, + "logps/chosen": -0.8858613967895508, + "logps/rejected": -11.036964416503906, + "loss": 0.892, + "num_input_tokens_seen": 11564832, + "odds_ratio_loss": 8.833036422729492, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.08858613669872284, + "rewards/margins": 1.0151102542877197, + "rewards/rejected": -1.103696346282959, + "sft_loss": 0.008729054592549801, + "step": 8850 + }, + { + "epoch": 0.6892259821081291, + "grad_norm": 1.8071199655532837, + "learning_rate": 1.0998236288216265e-05, + "logits/chosen": 0.06675352901220322, + "logits/rejected": -1.804593801498413, + "logps/chosen": -0.8722532987594604, + "logps/rejected": -10.574170112609863, + "loss": 0.8744, + "num_input_tokens_seen": 11579696, + "odds_ratio_loss": 8.713655471801758, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.08722533285617828, + "rewards/margins": 0.9701916575431824, + "rewards/rejected": -1.0574170351028442, + "sft_loss": 0.0030626547522842884, + "step": 8860 + }, + { + "epoch": 0.6900038895371451, + "grad_norm": 2.2742514610290527, + "learning_rate": 1.0947662895249896e-05, + "logits/chosen": 0.007434540893882513, + "logits/rejected": -1.6352884769439697, + "logps/chosen": -0.8696354031562805, + "logps/rejected": -18.652408599853516, + "loss": 0.87, + "num_input_tokens_seen": 11596480, + "odds_ratio_loss": 8.673111915588379, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.08696354180574417, + "rewards/margins": 1.7782771587371826, + "rewards/rejected": -1.8652406930923462, + "sft_loss": 0.002644627820700407, + "step": 8870 + }, + { + "epoch": 0.690781796966161, + "grad_norm": 3.000932455062866, + "learning_rate": 1.0897173429714e-05, + "logits/chosen": -0.13386768102645874, + "logits/rejected": -1.3530477285385132, + "logps/chosen": -0.9897897839546204, + "logps/rejected": -15.522348403930664, + "loss": 0.9965, + "num_input_tokens_seen": 11615344, + "odds_ratio_loss": 9.91053581237793, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09897898137569427, + "rewards/margins": 1.4532560110092163, + "rewards/rejected": -1.5522348880767822, + "sft_loss": 0.005432153586298227, + "step": 8880 + }, + { + "epoch": 0.691559704395177, + "grad_norm": 1.9952462911605835, + "learning_rate": 1.0846768193156357e-05, + "logits/chosen": -0.7853596210479736, + "logits/rejected": -1.0207130908966064, + "logps/chosen": -0.8637072443962097, + "logps/rejected": -9.85428237915039, + "loss": 0.8733, + "num_input_tokens_seen": 11626128, + "odds_ratio_loss": 8.149516105651855, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.08637072145938873, + "rewards/margins": 0.8990575075149536, + "rewards/rejected": -0.9854282140731812, + "sft_loss": 0.058370787650346756, + "step": 8890 + }, + { + "epoch": 0.692337611824193, + "grad_norm": 11.468011856079102, + "learning_rate": 1.0796447486621694e-05, + "logits/chosen": -0.0654086098074913, + "logits/rejected": -1.213006615638733, + "logps/chosen": -0.8232144117355347, + "logps/rejected": -9.072195053100586, + "loss": 0.8323, + "num_input_tokens_seen": 11635776, + "odds_ratio_loss": 8.218587875366211, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.08232144266366959, + "rewards/margins": 0.8248981237411499, + "rewards/rejected": -0.9072195291519165, + "sft_loss": 0.010446458123624325, + "step": 8900 + }, + { + "epoch": 0.6931155192532089, + "grad_norm": 1.6070795059204102, + "learning_rate": 1.0746211610649873e-05, + "logits/chosen": 0.10436618328094482, + "logits/rejected": -1.6011590957641602, + "logps/chosen": -0.7685268521308899, + "logps/rejected": -2.133186101913452, + "loss": 0.7968, + "num_input_tokens_seen": 11648464, + "odds_ratio_loss": 7.8565216064453125, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.07685267925262451, + "rewards/margins": 0.13646593689918518, + "rewards/rejected": -0.2133186161518097, + "sft_loss": 0.011129858903586864, + "step": 8910 + }, + { + "epoch": 0.6938934266822249, + "grad_norm": 19.818248748779297, + "learning_rate": 1.0696060865274139e-05, + "logits/chosen": -0.2180163860321045, + "logits/rejected": -1.0414024591445923, + "logps/chosen": -0.8535948991775513, + "logps/rejected": -6.71603536605835, + "loss": 0.8646, + "num_input_tokens_seen": 11663152, + "odds_ratio_loss": 8.475765228271484, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.08535948395729065, + "rewards/margins": 0.5862440466880798, + "rewards/rejected": -0.6716035008430481, + "sft_loss": 0.01703769713640213, + "step": 8920 + }, + { + "epoch": 0.6946713341112407, + "grad_norm": 12.964266777038574, + "learning_rate": 1.0645995550019266e-05, + "logits/chosen": -0.14962530136108398, + "logits/rejected": -1.0459420680999756, + "logps/chosen": -0.8487846255302429, + "logps/rejected": -11.240068435668945, + "loss": 0.8549, + "num_input_tokens_seen": 11679952, + "odds_ratio_loss": 8.277114868164062, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.08487847447395325, + "rewards/margins": 1.039128303527832, + "rewards/rejected": -1.124006748199463, + "sft_loss": 0.027226870879530907, + "step": 8930 + }, + { + "epoch": 0.6954492415402567, + "grad_norm": 1.5695898532867432, + "learning_rate": 1.0596015963899792e-05, + "logits/chosen": -0.05155446380376816, + "logits/rejected": -1.243017554283142, + "logps/chosen": -0.868207573890686, + "logps/rejected": -7.0557861328125, + "loss": 0.8793, + "num_input_tokens_seen": 11694208, + "odds_ratio_loss": 8.7078218460083, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.08682074397802353, + "rewards/margins": 0.6187578439712524, + "rewards/rejected": -0.705578625202179, + "sft_loss": 0.008534679189324379, + "step": 8940 + }, + { + "epoch": 0.6962271489692726, + "grad_norm": 2.6625661849975586, + "learning_rate": 1.054612240541826e-05, + "logits/chosen": -0.12724608182907104, + "logits/rejected": -1.3886604309082031, + "logps/chosen": -0.8681074976921082, + "logps/rejected": -17.198060989379883, + "loss": 0.8759, + "num_input_tokens_seen": 11705008, + "odds_ratio_loss": 8.679101943969727, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.08681075274944305, + "rewards/margins": 1.6329952478408813, + "rewards/rejected": -1.7198059558868408, + "sft_loss": 0.00800777692347765, + "step": 8950 + }, + { + "epoch": 0.6970050563982886, + "grad_norm": 5.75851583480835, + "learning_rate": 1.0496315172563404e-05, + "logits/chosen": 0.010910524055361748, + "logits/rejected": -1.5142662525177002, + "logps/chosen": -0.9781953692436218, + "logps/rejected": -10.615856170654297, + "loss": 0.9931, + "num_input_tokens_seen": 11716576, + "odds_ratio_loss": 9.874059677124023, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09781954437494278, + "rewards/margins": 0.9637660980224609, + "rewards/rejected": -1.061585783958435, + "sft_loss": 0.005718961823731661, + "step": 8960 + }, + { + "epoch": 0.6977829638273045, + "grad_norm": 1.7544348239898682, + "learning_rate": 1.0446594562808382e-05, + "logits/chosen": -0.22071878612041473, + "logits/rejected": -1.1810603141784668, + "logps/chosen": -0.7846574783325195, + "logps/rejected": -7.4228668212890625, + "loss": 0.8171, + "num_input_tokens_seen": 11732480, + "odds_ratio_loss": 8.076042175292969, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.07846574485301971, + "rewards/margins": 0.6638209819793701, + "rewards/rejected": -0.742286741733551, + "sft_loss": 0.009545383974909782, + "step": 8970 + }, + { + "epoch": 0.6985608712563205, + "grad_norm": 4.993774890899658, + "learning_rate": 1.0396960873108992e-05, + "logits/chosen": -0.11778923124074936, + "logits/rejected": -1.838526964187622, + "logps/chosen": -0.943067193031311, + "logps/rejected": -21.153854370117188, + "loss": 0.9468, + "num_input_tokens_seen": 11744784, + "odds_ratio_loss": 9.421133041381836, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09430670738220215, + "rewards/margins": 2.0210790634155273, + "rewards/rejected": -2.1153855323791504, + "sft_loss": 0.004669210873544216, + "step": 8980 + }, + { + "epoch": 0.6993387786853364, + "grad_norm": 3.575345993041992, + "learning_rate": 1.0347414399901908e-05, + "logits/chosen": -0.47023558616638184, + "logits/rejected": -1.3087602853775024, + "logps/chosen": -1.0606311559677124, + "logps/rejected": -14.766632080078125, + "loss": 1.062, + "num_input_tokens_seen": 11757712, + "odds_ratio_loss": 10.034512519836426, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1060631051659584, + "rewards/margins": 1.3705999851226807, + "rewards/rejected": -1.4766628742218018, + "sft_loss": 0.058538712561130524, + "step": 8990 + }, + { + "epoch": 0.7001166861143524, + "grad_norm": 4.2185163497924805, + "learning_rate": 1.0297955439102899e-05, + "logits/chosen": -0.19833922386169434, + "logits/rejected": -1.2449114322662354, + "logps/chosen": -0.9985941648483276, + "logps/rejected": -22.485218048095703, + "loss": 0.999, + "num_input_tokens_seen": 11767792, + "odds_ratio_loss": 9.736976623535156, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09985942393541336, + "rewards/margins": 2.1486623287200928, + "rewards/rejected": -2.248521566390991, + "sft_loss": 0.025351542979478836, + "step": 9000 + }, + { + "epoch": 0.7008945935433684, + "grad_norm": 5.648533821105957, + "learning_rate": 1.0248584286105077e-05, + "logits/chosen": -0.11871405690908432, + "logits/rejected": -1.4459521770477295, + "logps/chosen": -0.7193819880485535, + "logps/rejected": -9.93423080444336, + "loss": 0.7289, + "num_input_tokens_seen": 11778512, + "odds_ratio_loss": 7.127845764160156, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.07193820178508759, + "rewards/margins": 0.9214849472045898, + "rewards/rejected": -0.9934231042861938, + "sft_loss": 0.01614934578537941, + "step": 9010 + }, + { + "epoch": 0.7016725009723843, + "grad_norm": 1.579898476600647, + "learning_rate": 1.0199301235777106e-05, + "logits/chosen": 0.014426630921661854, + "logits/rejected": -1.645169973373413, + "logps/chosen": -1.0434606075286865, + "logps/rejected": -11.167806625366211, + "loss": 1.0792, + "num_input_tokens_seen": 11794976, + "odds_ratio_loss": 9.845039367675781, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.10434605926275253, + "rewards/margins": 1.012434720993042, + "rewards/rejected": -1.1167807579040527, + "sft_loss": 0.09464666247367859, + "step": 9020 + }, + { + "epoch": 0.7024504084014003, + "grad_norm": 19.114221572875977, + "learning_rate": 1.0150106582461477e-05, + "logits/chosen": -0.24926729500293732, + "logits/rejected": -1.216196060180664, + "logps/chosen": -0.697177529335022, + "logps/rejected": -14.336259841918945, + "loss": 0.7016, + "num_input_tokens_seen": 11806400, + "odds_ratio_loss": 6.796858310699463, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.06971774995326996, + "rewards/margins": 1.3639085292816162, + "rewards/rejected": -1.4336262941360474, + "sft_loss": 0.021893560886383057, + "step": 9030 + }, + { + "epoch": 0.7032283158304162, + "grad_norm": 4.123728275299072, + "learning_rate": 1.0101000619972708e-05, + "logits/chosen": -0.24322524666786194, + "logits/rejected": -1.1093257665634155, + "logps/chosen": -0.8399659991264343, + "logps/rejected": -11.511434555053711, + "loss": 0.8557, + "num_input_tokens_seen": 11818976, + "odds_ratio_loss": 8.476297378540039, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.08399660885334015, + "rewards/margins": 1.0671470165252686, + "rewards/rejected": -1.1511435508728027, + "sft_loss": 0.008043365553021431, + "step": 9040 + }, + { + "epoch": 0.7040062232594321, + "grad_norm": 2.056776523590088, + "learning_rate": 1.0051983641595658e-05, + "logits/chosen": -0.26209187507629395, + "logits/rejected": -1.374551773071289, + "logps/chosen": -0.8355258107185364, + "logps/rejected": -15.121932983398438, + "loss": 0.8437, + "num_input_tokens_seen": 11830848, + "odds_ratio_loss": 8.353340148925781, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.08355257660150528, + "rewards/margins": 1.428640604019165, + "rewards/rejected": -1.512193202972412, + "sft_loss": 0.008403485640883446, + "step": 9050 + }, + { + "epoch": 0.704784130688448, + "grad_norm": 1.8826544284820557, + "learning_rate": 1.0003055940083661e-05, + "logits/chosen": -0.6671730279922485, + "logits/rejected": -1.399988055229187, + "logps/chosen": -0.7520816922187805, + "logps/rejected": -17.23227310180664, + "loss": 0.7572, + "num_input_tokens_seen": 11840480, + "odds_ratio_loss": 7.3093414306640625, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.07520817220211029, + "rewards/margins": 1.6480194330215454, + "rewards/rejected": -1.7232272624969482, + "sft_loss": 0.02628893218934536, + "step": 9060 + }, + { + "epoch": 0.705562038117464, + "grad_norm": 11.5000638961792, + "learning_rate": 9.954217807656898e-06, + "logits/chosen": -0.24273736774921417, + "logits/rejected": -1.2386590242385864, + "logps/chosen": -3.6087188720703125, + "logps/rejected": -13.7881441116333, + "loss": 3.6154, + "num_input_tokens_seen": 11851088, + "odds_ratio_loss": 34.90401077270508, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.36087191104888916, + "rewards/margins": 1.0179424285888672, + "rewards/rejected": -1.3788143396377563, + "sft_loss": 0.12496994435787201, + "step": 9070 + }, + { + "epoch": 0.7063399455464799, + "grad_norm": 6.532557964324951, + "learning_rate": 9.905469536000583e-06, + "logits/chosen": -0.4153507649898529, + "logits/rejected": -1.344873070716858, + "logps/chosen": -0.8688298463821411, + "logps/rejected": -10.471773147583008, + "loss": 0.8814, + "num_input_tokens_seen": 11862384, + "odds_ratio_loss": 8.589521408081055, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.08688297867774963, + "rewards/margins": 0.960294246673584, + "rewards/rejected": -1.0471774339675903, + "sft_loss": 0.022413309663534164, + "step": 9080 + }, + { + "epoch": 0.7071178529754959, + "grad_norm": 0.696464478969574, + "learning_rate": 9.856811416263234e-06, + "logits/chosen": -0.236698180437088, + "logits/rejected": -1.628491997718811, + "logps/chosen": -0.6710294485092163, + "logps/rejected": -18.233524322509766, + "loss": 0.678, + "num_input_tokens_seen": 11876304, + "odds_ratio_loss": 6.749547004699707, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.06710294634103775, + "rewards/margins": 1.7562497854232788, + "rewards/rejected": -1.823352575302124, + "sft_loss": 0.003054919885471463, + "step": 9090 + }, + { + "epoch": 0.7078957604045119, + "grad_norm": 3.7164294719696045, + "learning_rate": 9.808243739054945e-06, + "logits/chosen": -0.09606461226940155, + "logits/rejected": -1.6019067764282227, + "logps/chosen": -1.0349172353744507, + "logps/rejected": -13.971864700317383, + "loss": 1.0452, + "num_input_tokens_seen": 11889872, + "odds_ratio_loss": 10.380487442016602, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.10349173843860626, + "rewards/margins": 1.2936947345733643, + "rewards/rejected": -1.397186517715454, + "sft_loss": 0.007199641317129135, + "step": 9100 + }, + { + "epoch": 0.7086736678335278, + "grad_norm": 2.627157211303711, + "learning_rate": 9.75976679444564e-06, + "logits/chosen": -0.26083993911743164, + "logits/rejected": -1.5357798337936401, + "logps/chosen": -0.9003885388374329, + "logps/rejected": -10.310569763183594, + "loss": 0.9186, + "num_input_tokens_seen": 11898656, + "odds_ratio_loss": 8.754205703735352, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.09003885090351105, + "rewards/margins": 0.9410181045532227, + "rewards/rejected": -1.0310567617416382, + "sft_loss": 0.04318580403923988, + "step": 9110 + }, + { + "epoch": 0.7094515752625438, + "grad_norm": 1.5432356595993042, + "learning_rate": 9.711380871963343e-06, + "logits/chosen": -0.36176571249961853, + "logits/rejected": -1.313044548034668, + "logps/chosen": -0.6721317172050476, + "logps/rejected": -17.497119903564453, + "loss": 0.6738, + "num_input_tokens_seen": 11912496, + "odds_ratio_loss": 6.629150390625, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.06721317768096924, + "rewards/margins": 1.682498574256897, + "rewards/rejected": -1.7497117519378662, + "sft_loss": 0.010848169215023518, + "step": 9120 + }, + { + "epoch": 0.7102294826915597, + "grad_norm": 4.048521995544434, + "learning_rate": 9.663086260592447e-06, + "logits/chosen": -0.3890684247016907, + "logits/rejected": -1.5890153646469116, + "logps/chosen": -0.7857718467712402, + "logps/rejected": -24.208412170410156, + "loss": 0.7861, + "num_input_tokens_seen": 11925536, + "odds_ratio_loss": 7.81637716293335, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.0785771906375885, + "rewards/margins": 2.342264175415039, + "rewards/rejected": -2.4208412170410156, + "sft_loss": 0.004494016990065575, + "step": 9130 + }, + { + "epoch": 0.7110073901205757, + "grad_norm": 2.3026251792907715, + "learning_rate": 9.614883248771996e-06, + "logits/chosen": -0.17405235767364502, + "logits/rejected": -1.2665290832519531, + "logps/chosen": -0.9642848968505859, + "logps/rejected": -14.18018913269043, + "loss": 0.9792, + "num_input_tokens_seen": 11938576, + "odds_ratio_loss": 9.722308158874512, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.09642849117517471, + "rewards/margins": 1.321590542793274, + "rewards/rejected": -1.4180189371109009, + "sft_loss": 0.006958406418561935, + "step": 9140 + }, + { + "epoch": 0.7117852975495916, + "grad_norm": 1.8911758661270142, + "learning_rate": 9.566772124393952e-06, + "logits/chosen": -0.11972236633300781, + "logits/rejected": -2.1072115898132324, + "logps/chosen": -1.1833240985870361, + "logps/rejected": -21.958148956298828, + "loss": 1.193, + "num_input_tokens_seen": 11952224, + "odds_ratio_loss": 11.897111892700195, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.1183324009180069, + "rewards/margins": 2.0774824619293213, + "rewards/rejected": -2.195814847946167, + "sft_loss": 0.003329962957650423, + "step": 9150 + }, + { + "epoch": 0.7125632049786076, + "grad_norm": 2.6128060817718506, + "learning_rate": 9.518753174801484e-06, + "logits/chosen": -0.2604648768901825, + "logits/rejected": -1.4065415859222412, + "logps/chosen": -0.8946755528450012, + "logps/rejected": -10.743023872375488, + "loss": 0.9171, + "num_input_tokens_seen": 11967408, + "odds_ratio_loss": 9.056489944458008, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.08946755528450012, + "rewards/margins": 0.9848347902297974, + "rewards/rejected": -1.074302315711975, + "sft_loss": 0.011441009119153023, + "step": 9160 + }, + { + "epoch": 0.7133411124076235, + "grad_norm": 3.6682426929473877, + "learning_rate": 9.470826686787235e-06, + "logits/chosen": -0.0443451926112175, + "logits/rejected": -1.7861120700836182, + "logps/chosen": -1.0642540454864502, + "logps/rejected": -9.87019157409668, + "loss": 1.0997, + "num_input_tokens_seen": 11980944, + "odds_ratio_loss": 10.783086776733398, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.10642538964748383, + "rewards/margins": 0.8805936574935913, + "rewards/rejected": -0.9870191812515259, + "sft_loss": 0.021341949701309204, + "step": 9170 + }, + { + "epoch": 0.7141190198366394, + "grad_norm": 2.1391987800598145, + "learning_rate": 9.42299294659166e-06, + "logits/chosen": -0.08032462000846863, + "logits/rejected": -1.516364336013794, + "logps/chosen": -0.9333599209785461, + "logps/rejected": -15.166709899902344, + "loss": 0.9447, + "num_input_tokens_seen": 11992208, + "odds_ratio_loss": 9.370416641235352, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09333600103855133, + "rewards/margins": 1.4233348369598389, + "rewards/rejected": -1.5166709423065186, + "sft_loss": 0.0076144421473145485, + "step": 9180 + }, + { + "epoch": 0.7148969272656553, + "grad_norm": 13.14161491394043, + "learning_rate": 9.375252239901225e-06, + "logits/chosen": -0.20920245349407196, + "logits/rejected": -1.2209100723266602, + "logps/chosen": -0.9390764236450195, + "logps/rejected": -7.180729866027832, + "loss": 0.9496, + "num_input_tokens_seen": 12003008, + "odds_ratio_loss": 9.222626686096191, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09390763938426971, + "rewards/margins": 0.6241652369499207, + "rewards/rejected": -0.7180729508399963, + "sft_loss": 0.027297839522361755, + "step": 9190 + }, + { + "epoch": 0.7156748346946713, + "grad_norm": 1.4618985652923584, + "learning_rate": 9.327604851846791e-06, + "logits/chosen": -0.06988964974880219, + "logits/rejected": -1.7549928426742554, + "logps/chosen": -0.7257611751556396, + "logps/rejected": -17.123815536499023, + "loss": 0.7286, + "num_input_tokens_seen": 12016240, + "odds_ratio_loss": 7.260410308837891, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.0725761204957962, + "rewards/margins": 1.6398054361343384, + "rewards/rejected": -1.7123816013336182, + "sft_loss": 0.002546419855207205, + "step": 9200 + }, + { + "epoch": 0.7164527421236873, + "grad_norm": 2.0492186546325684, + "learning_rate": 9.280051067001865e-06, + "logits/chosen": -0.11623696237802505, + "logits/rejected": -1.9431318044662476, + "logps/chosen": -0.816973865032196, + "logps/rejected": -23.292240142822266, + "loss": 0.8197, + "num_input_tokens_seen": 12026816, + "odds_ratio_loss": 7.786641597747803, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.08169738948345184, + "rewards/margins": 2.2475266456604004, + "rewards/rejected": -2.329224109649658, + "sft_loss": 0.04104030877351761, + "step": 9210 + }, + { + "epoch": 0.7172306495527032, + "grad_norm": 1.845423698425293, + "learning_rate": 9.232591169380912e-06, + "logits/chosen": -0.03926096111536026, + "logits/rejected": -1.3892351388931274, + "logps/chosen": -0.8934553861618042, + "logps/rejected": -15.741470336914062, + "loss": 0.9061, + "num_input_tokens_seen": 12040848, + "odds_ratio_loss": 8.62596321105957, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.0893455296754837, + "rewards/margins": 1.4848015308380127, + "rewards/rejected": -1.5741469860076904, + "sft_loss": 0.04347647354006767, + "step": 9220 + }, + { + "epoch": 0.7180085569817192, + "grad_norm": 3.811688184738159, + "learning_rate": 9.185225442437648e-06, + "logits/chosen": -0.5131353735923767, + "logits/rejected": -1.2000887393951416, + "logps/chosen": -0.810464084148407, + "logps/rejected": -12.801618576049805, + "loss": 0.8145, + "num_input_tokens_seen": 12052192, + "odds_ratio_loss": 7.7980546951293945, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.08104641735553741, + "rewards/margins": 1.199115514755249, + "rewards/rejected": -1.2801618576049805, + "sft_loss": 0.03466106206178665, + "step": 9230 + }, + { + "epoch": 0.7187864644107351, + "grad_norm": 8.794293403625488, + "learning_rate": 9.137954169063365e-06, + "logits/chosen": -0.12427148967981339, + "logits/rejected": -1.897454857826233, + "logps/chosen": -0.7215501666069031, + "logps/rejected": -11.68718147277832, + "loss": 0.7312, + "num_input_tokens_seen": 12067040, + "odds_ratio_loss": 7.176033973693848, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.07215501368045807, + "rewards/margins": 1.0965631008148193, + "rewards/rejected": -1.1687180995941162, + "sft_loss": 0.013598406687378883, + "step": 9240 + }, + { + "epoch": 0.7195643718397511, + "grad_norm": 1.840759038925171, + "learning_rate": 9.09077763158523e-06, + "logits/chosen": -0.05737997964024544, + "logits/rejected": -1.8592857122421265, + "logps/chosen": -0.8444797396659851, + "logps/rejected": -23.798778533935547, + "loss": 0.8557, + "num_input_tokens_seen": 12083264, + "odds_ratio_loss": 8.520927429199219, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.08444797247648239, + "rewards/margins": 2.2954299449920654, + "rewards/rejected": -2.379878044128418, + "sft_loss": 0.0036380328238010406, + "step": 9250 + }, + { + "epoch": 0.720342279268767, + "grad_norm": 1.4600502252578735, + "learning_rate": 9.043696111764598e-06, + "logits/chosen": -0.12732329964637756, + "logits/rejected": -1.7196537256240845, + "logps/chosen": -0.8292292356491089, + "logps/rejected": -17.078582763671875, + "loss": 0.8535, + "num_input_tokens_seen": 12100672, + "odds_ratio_loss": 8.224623680114746, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.08292293548583984, + "rewards/margins": 1.6249353885650635, + "rewards/rejected": -1.7078583240509033, + "sft_loss": 0.031013712286949158, + "step": 9260 + }, + { + "epoch": 0.721120186697783, + "grad_norm": 47.71871566772461, + "learning_rate": 8.996709890795338e-06, + "logits/chosen": -0.3540479242801666, + "logits/rejected": -1.249452829360962, + "logps/chosen": -0.9343889355659485, + "logps/rejected": -9.030014038085938, + "loss": 0.9404, + "num_input_tokens_seen": 12111344, + "odds_ratio_loss": 8.981085777282715, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09343889355659485, + "rewards/margins": 0.8095625638961792, + "rewards/rejected": -0.9030014872550964, + "sft_loss": 0.04224556311964989, + "step": 9270 + }, + { + "epoch": 0.721898094126799, + "grad_norm": 1.8473955392837524, + "learning_rate": 8.949819249302138e-06, + "logits/chosen": -0.08845169842243195, + "logits/rejected": -1.9312286376953125, + "logps/chosen": -0.8921386003494263, + "logps/rejected": -14.9306640625, + "loss": 0.9023, + "num_input_tokens_seen": 12126928, + "odds_ratio_loss": 8.729538917541504, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.08921386301517487, + "rewards/margins": 1.4038527011871338, + "rewards/rejected": -1.4930665493011475, + "sft_loss": 0.029351364821195602, + "step": 9280 + }, + { + "epoch": 0.7226760015558149, + "grad_norm": 4.839386463165283, + "learning_rate": 8.903024467338846e-06, + "logits/chosen": -0.5877536535263062, + "logits/rejected": -1.3349168300628662, + "logps/chosen": -1.2354286909103394, + "logps/rejected": -17.281709671020508, + "loss": 1.2405, + "num_input_tokens_seen": 12138704, + "odds_ratio_loss": 11.595271110534668, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.12354286760091782, + "rewards/margins": 1.6046279668807983, + "rewards/rejected": -1.7281709909439087, + "sft_loss": 0.0809488296508789, + "step": 9290 + }, + { + "epoch": 0.7234539089848308, + "grad_norm": 2.161583185195923, + "learning_rate": 8.856325824386792e-06, + "logits/chosen": -0.2935566306114197, + "logits/rejected": -1.9119758605957031, + "logps/chosen": -0.9439760446548462, + "logps/rejected": -15.964810371398926, + "loss": 0.9524, + "num_input_tokens_seen": 12147984, + "odds_ratio_loss": 9.423673629760742, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09439760446548462, + "rewards/margins": 1.5020835399627686, + "rewards/rejected": -1.5964810848236084, + "sft_loss": 0.010013467632234097, + "step": 9300 + }, + { + "epoch": 0.7242318164138467, + "grad_norm": 3.4106974601745605, + "learning_rate": 8.809723599353106e-06, + "logits/chosen": -0.6587649583816528, + "logits/rejected": -0.8816956281661987, + "logps/chosen": -0.93231201171875, + "logps/rejected": -10.11970329284668, + "loss": 0.9529, + "num_input_tokens_seen": 12161680, + "odds_ratio_loss": 9.175849914550781, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.0932312160730362, + "rewards/margins": 0.9187391400337219, + "rewards/rejected": -1.0119702816009521, + "sft_loss": 0.0353216677904129, + "step": 9310 + }, + { + "epoch": 0.7250097238428627, + "grad_norm": 2.1136739253997803, + "learning_rate": 8.763218070569073e-06, + "logits/chosen": -0.294426292181015, + "logits/rejected": -1.9087531566619873, + "logps/chosen": -0.9304768443107605, + "logps/rejected": -20.47693634033203, + "loss": 0.9382, + "num_input_tokens_seen": 12174112, + "odds_ratio_loss": 9.206003189086914, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09304769337177277, + "rewards/margins": 1.954646110534668, + "rewards/rejected": -2.0476937294006348, + "sft_loss": 0.017637435346841812, + "step": 9320 + }, + { + "epoch": 0.7257876312718786, + "grad_norm": 1.225160002708435, + "learning_rate": 8.71680951578846e-06, + "logits/chosen": -0.24428872764110565, + "logits/rejected": -1.6217687129974365, + "logps/chosen": -0.9168788194656372, + "logps/rejected": -13.614445686340332, + "loss": 0.9378, + "num_input_tokens_seen": 12188144, + "odds_ratio_loss": 9.270367622375488, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.091687873005867, + "rewards/margins": 1.2697566747665405, + "rewards/rejected": -1.3614444732666016, + "sft_loss": 0.010737092234194279, + "step": 9330 + }, + { + "epoch": 0.7265655387008946, + "grad_norm": 14.630156517028809, + "learning_rate": 8.670498212185854e-06, + "logits/chosen": -0.17345094680786133, + "logits/rejected": -1.7669017314910889, + "logps/chosen": -0.8515311479568481, + "logps/rejected": -16.09687042236328, + "loss": 0.8719, + "num_input_tokens_seen": 12202320, + "odds_ratio_loss": 8.373773574829102, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.08515311032533646, + "rewards/margins": 1.524533748626709, + "rewards/rejected": -1.609687089920044, + "sft_loss": 0.03454557806253433, + "step": 9340 + }, + { + "epoch": 0.7273434461299105, + "grad_norm": 1.8369165658950806, + "learning_rate": 8.624284436355012e-06, + "logits/chosen": -0.3347812294960022, + "logits/rejected": -1.5777273178100586, + "logps/chosen": -0.7993783950805664, + "logps/rejected": -20.731544494628906, + "loss": 0.8026, + "num_input_tokens_seen": 12218336, + "odds_ratio_loss": 7.981850624084473, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.07993783801794052, + "rewards/margins": 1.9932165145874023, + "rewards/rejected": -2.0731544494628906, + "sft_loss": 0.0044608041644096375, + "step": 9350 + }, + { + "epoch": 0.7281213535589265, + "grad_norm": 2.1387698650360107, + "learning_rate": 8.578168464307206e-06, + "logits/chosen": -0.47583675384521484, + "logits/rejected": -1.1676183938980103, + "logps/chosen": -0.6008586287498474, + "logps/rejected": -16.23210334777832, + "loss": 0.6062, + "num_input_tokens_seen": 12230592, + "odds_ratio_loss": 5.891040325164795, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.060085855424404144, + "rewards/margins": 1.563124418258667, + "rewards/rejected": -1.6232101917266846, + "sft_loss": 0.017101094126701355, + "step": 9360 + }, + { + "epoch": 0.7288992609879424, + "grad_norm": 4.9119110107421875, + "learning_rate": 8.53215057146958e-06, + "logits/chosen": -0.31183579564094543, + "logits/rejected": -1.394217848777771, + "logps/chosen": -1.281356930732727, + "logps/rejected": -8.785355567932129, + "loss": 1.3069, + "num_input_tokens_seen": 12242032, + "odds_ratio_loss": 12.887720108032227, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.12813569605350494, + "rewards/margins": 0.7503998279571533, + "rewards/rejected": -0.8785354495048523, + "sft_loss": 0.018101245164871216, + "step": 9370 + }, + { + "epoch": 0.7296771684169584, + "grad_norm": 6.540828704833984, + "learning_rate": 8.486231032683498e-06, + "logits/chosen": -0.2462223321199417, + "logits/rejected": -1.6916329860687256, + "logps/chosen": -0.9182268977165222, + "logps/rejected": -23.39679527282715, + "loss": 0.9206, + "num_input_tokens_seen": 12259872, + "odds_ratio_loss": 9.148286819458008, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09182268381118774, + "rewards/margins": 2.247856616973877, + "rewards/rejected": -2.339679479598999, + "sft_loss": 0.005803549196571112, + "step": 9380 + }, + { + "epoch": 0.7304550758459744, + "grad_norm": 3.1595852375030518, + "learning_rate": 8.440410122202902e-06, + "logits/chosen": -0.285053551197052, + "logits/rejected": -1.308659315109253, + "logps/chosen": -0.9199910163879395, + "logps/rejected": -17.944133758544922, + "loss": 0.9393, + "num_input_tokens_seen": 12274096, + "odds_ratio_loss": 9.294504165649414, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.09199909120798111, + "rewards/margins": 1.7024141550064087, + "rewards/rejected": -1.7944132089614868, + "sft_loss": 0.009822076186537743, + "step": 9390 + }, + { + "epoch": 0.7312329832749903, + "grad_norm": 3.8401002883911133, + "learning_rate": 8.394688113692684e-06, + "logits/chosen": -0.17555391788482666, + "logits/rejected": -1.3844785690307617, + "logps/chosen": -0.9503070712089539, + "logps/rejected": -15.155308723449707, + "loss": 0.9588, + "num_input_tokens_seen": 12288448, + "odds_ratio_loss": 9.44873332977295, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09503071010112762, + "rewards/margins": 1.4205002784729004, + "rewards/rejected": -1.5155308246612549, + "sft_loss": 0.013938379473984241, + "step": 9400 + }, + { + "epoch": 0.7320108907040063, + "grad_norm": 7.067780017852783, + "learning_rate": 8.349065280227043e-06, + "logits/chosen": -0.2124090939760208, + "logits/rejected": -1.7307307720184326, + "logps/chosen": -0.9263704419136047, + "logps/rejected": -14.757944107055664, + "loss": 0.9409, + "num_input_tokens_seen": 12300544, + "odds_ratio_loss": 9.307869911193848, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.09263704717159271, + "rewards/margins": 1.3831573724746704, + "rewards/rejected": -1.4757944345474243, + "sft_loss": 0.010076623409986496, + "step": 9410 + }, + { + "epoch": 0.7327887981330222, + "grad_norm": 1.2005184888839722, + "learning_rate": 8.303541894287853e-06, + "logits/chosen": -0.2487785816192627, + "logits/rejected": -1.5799301862716675, + "logps/chosen": -0.972506046295166, + "logps/rejected": -15.554588317871094, + "loss": 0.9839, + "num_input_tokens_seen": 12310352, + "odds_ratio_loss": 9.77066707611084, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09725061058998108, + "rewards/margins": 1.4582083225250244, + "rewards/rejected": -1.5554590225219727, + "sft_loss": 0.006865913979709148, + "step": 9420 + }, + { + "epoch": 0.7335667055620381, + "grad_norm": 23.47117805480957, + "learning_rate": 8.258118227763042e-06, + "logits/chosen": -0.18120557069778442, + "logits/rejected": -1.3460719585418701, + "logps/chosen": -0.9152958989143372, + "logps/rejected": -7.53924036026001, + "loss": 0.9301, + "num_input_tokens_seen": 12325184, + "odds_ratio_loss": 8.720853805541992, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.09152959287166595, + "rewards/margins": 0.6623944044113159, + "rewards/rejected": -0.7539240121841431, + "sft_loss": 0.058042317628860474, + "step": 9430 + }, + { + "epoch": 0.734344612991054, + "grad_norm": 2.655810594558716, + "learning_rate": 8.212794551944968e-06, + "logits/chosen": -0.3864406645298004, + "logits/rejected": -1.237755298614502, + "logps/chosen": -1.3730939626693726, + "logps/rejected": -14.318827629089355, + "loss": 1.3908, + "num_input_tokens_seen": 12338624, + "odds_ratio_loss": 13.32928466796875, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.13730938732624054, + "rewards/margins": 1.2945735454559326, + "rewards/rejected": -1.4318828582763672, + "sft_loss": 0.05790317803621292, + "step": 9440 + }, + { + "epoch": 0.73512252042007, + "grad_norm": 2.26202392578125, + "learning_rate": 8.167571137528788e-06, + "logits/chosen": -0.0685519352555275, + "logits/rejected": -1.5295946598052979, + "logps/chosen": -0.7410327792167664, + "logps/rejected": -16.688684463500977, + "loss": 0.7493, + "num_input_tokens_seen": 12351760, + "odds_ratio_loss": 7.458738803863525, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.07410328090190887, + "rewards/margins": 1.5947654247283936, + "rewards/rejected": -1.6688687801361084, + "sft_loss": 0.003421871457248926, + "step": 9450 + }, + { + "epoch": 0.7359004278490859, + "grad_norm": 1.3735111951828003, + "learning_rate": 8.122448254610853e-06, + "logits/chosen": -0.22400489449501038, + "logits/rejected": -1.5753123760223389, + "logps/chosen": -0.7171843647956848, + "logps/rejected": -7.419655799865723, + "loss": 0.7375, + "num_input_tokens_seen": 12365376, + "odds_ratio_loss": 7.273862361907959, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.07171843200922012, + "rewards/margins": 0.6702471971511841, + "rewards/rejected": -0.741965651512146, + "sft_loss": 0.010080719366669655, + "step": 9460 + }, + { + "epoch": 0.7366783352781019, + "grad_norm": 1.3101513385772705, + "learning_rate": 8.077426172687092e-06, + "logits/chosen": -0.2681163251399994, + "logits/rejected": -1.6242246627807617, + "logps/chosen": -0.9232928156852722, + "logps/rejected": -21.1041202545166, + "loss": 0.935, + "num_input_tokens_seen": 12382208, + "odds_ratio_loss": 9.310216903686523, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.09232927858829498, + "rewards/margins": 2.018082857131958, + "rewards/rejected": -2.110412120819092, + "sft_loss": 0.003992895595729351, + "step": 9470 + }, + { + "epoch": 0.7374562427071178, + "grad_norm": 3.2835025787353516, + "learning_rate": 8.032505160651393e-06, + "logits/chosen": -0.5149093866348267, + "logits/rejected": -1.5588371753692627, + "logps/chosen": -0.9430519938468933, + "logps/rejected": -17.860750198364258, + "loss": 0.9444, + "num_input_tokens_seen": 12394752, + "odds_ratio_loss": 9.215082168579102, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09430520981550217, + "rewards/margins": 1.6917698383331299, + "rewards/rejected": -1.7860748767852783, + "sft_loss": 0.02293788641691208, + "step": 9480 + }, + { + "epoch": 0.7382341501361338, + "grad_norm": 2.113330841064453, + "learning_rate": 7.987685486794021e-06, + "logits/chosen": -0.11559674888849258, + "logits/rejected": -1.5787172317504883, + "logps/chosen": -0.7669419646263123, + "logps/rejected": -13.251312255859375, + "loss": 0.7709, + "num_input_tokens_seen": 12408784, + "odds_ratio_loss": 7.644850254058838, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.07669420540332794, + "rewards/margins": 1.2484371662139893, + "rewards/rejected": -1.3251312971115112, + "sft_loss": 0.006389009766280651, + "step": 9490 + }, + { + "epoch": 0.7390120575651498, + "grad_norm": 3.922805070877075, + "learning_rate": 7.942967418799987e-06, + "logits/chosen": -0.2295842468738556, + "logits/rejected": -1.2077304124832153, + "logps/chosen": -0.8419455289840698, + "logps/rejected": -8.025991439819336, + "loss": 0.8703, + "num_input_tokens_seen": 12420784, + "odds_ratio_loss": 8.591384887695312, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.08419455587863922, + "rewards/margins": 0.7184046506881714, + "rewards/rejected": -0.8025991320610046, + "sft_loss": 0.011183908209204674, + "step": 9500 + }, + { + "epoch": 0.7397899649941657, + "grad_norm": 2.023562431335449, + "learning_rate": 7.898351223747463e-06, + "logits/chosen": -0.47920385003089905, + "logits/rejected": -1.178824543952942, + "logps/chosen": -0.8258765935897827, + "logps/rejected": -21.803810119628906, + "loss": 0.8267, + "num_input_tokens_seen": 12437792, + "odds_ratio_loss": 8.039857864379883, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.08258765190839767, + "rewards/margins": 2.0977935791015625, + "rewards/rejected": -2.1803812980651855, + "sft_loss": 0.022705521434545517, + "step": 9510 + }, + { + "epoch": 0.7405678724231817, + "grad_norm": 2.659808874130249, + "learning_rate": 7.85383716810619e-06, + "logits/chosen": -0.14984755218029022, + "logits/rejected": -1.4295680522918701, + "logps/chosen": -0.8580425381660461, + "logps/rejected": -13.875532150268555, + "loss": 0.8718, + "num_input_tokens_seen": 12450464, + "odds_ratio_loss": 8.642005920410156, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.08580423891544342, + "rewards/margins": 1.3017492294311523, + "rewards/rejected": -1.3875534534454346, + "sft_loss": 0.007576326839625835, + "step": 9520 + }, + { + "epoch": 0.7413457798521976, + "grad_norm": 7.571407794952393, + "learning_rate": 7.809425517735882e-06, + "logits/chosen": -0.4199492931365967, + "logits/rejected": -1.2727622985839844, + "logps/chosen": -0.7845070362091064, + "logps/rejected": -7.90404748916626, + "loss": 0.8038, + "num_input_tokens_seen": 12460944, + "odds_ratio_loss": 7.95632791519165, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.07845070958137512, + "rewards/margins": 0.7119540572166443, + "rewards/rejected": -0.790404736995697, + "sft_loss": 0.008128446526825428, + "step": 9530 + }, + { + "epoch": 0.7421236872812136, + "grad_norm": 2.4146907329559326, + "learning_rate": 7.765116537884638e-06, + "logits/chosen": -0.10155007988214493, + "logits/rejected": -1.724957823753357, + "logps/chosen": -0.7110482454299927, + "logps/rejected": -16.225696563720703, + "loss": 0.7145, + "num_input_tokens_seen": 12476384, + "odds_ratio_loss": 7.10809850692749, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.07110483199357986, + "rewards/margins": 1.5514647960662842, + "rewards/rejected": -1.6225694417953491, + "sft_loss": 0.0037299140822142363, + "step": 9540 + }, + { + "epoch": 0.7429015947102294, + "grad_norm": 4.062788009643555, + "learning_rate": 7.720910493187353e-06, + "logits/chosen": -0.11513806879520416, + "logits/rejected": -1.4933720827102661, + "logps/chosen": -0.8890712857246399, + "logps/rejected": -15.664164543151855, + "loss": 0.9005, + "num_input_tokens_seen": 12491440, + "odds_ratio_loss": 8.813547134399414, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.08890712261199951, + "rewards/margins": 1.4775094985961914, + "rewards/rejected": -1.5664165019989014, + "sft_loss": 0.01918819174170494, + "step": 9550 + }, + { + "epoch": 0.7436795021392454, + "grad_norm": 17.29952049255371, + "learning_rate": 7.676807647664156e-06, + "logits/chosen": -0.6568484306335449, + "logits/rejected": -1.1108663082122803, + "logps/chosen": -0.9456981420516968, + "logps/rejected": -18.975109100341797, + "loss": 0.9569, + "num_input_tokens_seen": 12499264, + "odds_ratio_loss": 9.273554801940918, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09456981718540192, + "rewards/margins": 1.802941083908081, + "rewards/rejected": -1.8975108861923218, + "sft_loss": 0.029570013284683228, + "step": 9560 + }, + { + "epoch": 0.7444574095682613, + "grad_norm": 2.4996085166931152, + "learning_rate": 7.632808264718805e-06, + "logits/chosen": -0.0891091451048851, + "logits/rejected": -1.5486935377120972, + "logps/chosen": -0.9733103513717651, + "logps/rejected": -11.993257522583008, + "loss": 0.9894, + "num_input_tokens_seen": 12513984, + "odds_ratio_loss": 9.830720901489258, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.09733103215694427, + "rewards/margins": 1.1019947528839111, + "rewards/rejected": -1.199325680732727, + "sft_loss": 0.006290743593126535, + "step": 9570 + }, + { + "epoch": 0.7452353169972773, + "grad_norm": 12.195624351501465, + "learning_rate": 7.588912607137141e-06, + "logits/chosen": -0.19729329645633698, + "logits/rejected": -1.429392695426941, + "logps/chosen": -0.8851596117019653, + "logps/rejected": -10.769949913024902, + "loss": 0.9099, + "num_input_tokens_seen": 12527952, + "odds_ratio_loss": 9.021488189697266, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.08851595968008041, + "rewards/margins": 0.9884791374206543, + "rewards/rejected": -1.0769951343536377, + "sft_loss": 0.007774298544973135, + "step": 9580 + }, + { + "epoch": 0.7460132244262933, + "grad_norm": 2.7442383766174316, + "learning_rate": 7.545120937085498e-06, + "logits/chosen": -0.2869792878627777, + "logits/rejected": -1.7530750036239624, + "logps/chosen": -0.8515239953994751, + "logps/rejected": -15.776219367980957, + "loss": 0.8742, + "num_input_tokens_seen": 12542816, + "odds_ratio_loss": 8.661226272583008, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.08515240252017975, + "rewards/margins": 1.4924695491790771, + "rewards/rejected": -1.5776218175888062, + "sft_loss": 0.008089212700724602, + "step": 9590 + }, + { + "epoch": 0.7467911318553092, + "grad_norm": 8.69823169708252, + "learning_rate": 7.501433516109152e-06, + "logits/chosen": -0.3283597230911255, + "logits/rejected": -1.450851321220398, + "logps/chosen": -1.2167384624481201, + "logps/rejected": -15.752424240112305, + "loss": 1.2204, + "num_input_tokens_seen": 12557728, + "odds_ratio_loss": 10.35692024230957, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.12167384475469589, + "rewards/margins": 1.4535688161849976, + "rewards/rejected": -1.575242519378662, + "sft_loss": 0.18467479944229126, + "step": 9600 + }, + { + "epoch": 0.7475690392843252, + "grad_norm": 3.905773878097534, + "learning_rate": 7.457850605130745e-06, + "logits/chosen": -0.31992048025131226, + "logits/rejected": -0.8100858926773071, + "logps/chosen": -0.751301646232605, + "logps/rejected": -14.805696487426758, + "loss": 0.7602, + "num_input_tokens_seen": 12571680, + "odds_ratio_loss": 7.564237117767334, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.0751301646232605, + "rewards/margins": 1.4054394960403442, + "rewards/rejected": -1.48056960105896, + "sft_loss": 0.003806706052273512, + "step": 9610 + }, + { + "epoch": 0.7483469467133411, + "grad_norm": 2.715651035308838, + "learning_rate": 7.414372464448743e-06, + "logits/chosen": -0.25902432203292847, + "logits/rejected": -1.471724271774292, + "logps/chosen": -0.9195705652236938, + "logps/rejected": -12.037912368774414, + "loss": 0.9254, + "num_input_tokens_seen": 12580880, + "odds_ratio_loss": 9.145597457885742, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09195704758167267, + "rewards/margins": 1.111834168434143, + "rewards/rejected": -1.2037912607192993, + "sft_loss": 0.010806088335812092, + "step": 9620 + }, + { + "epoch": 0.7491248541423571, + "grad_norm": 4.897205352783203, + "learning_rate": 7.370999353735869e-06, + "logits/chosen": -0.6254599690437317, + "logits/rejected": -0.6810623407363892, + "logps/chosen": -0.8146526217460632, + "logps/rejected": -13.211095809936523, + "loss": 0.8365, + "num_input_tokens_seen": 12591248, + "odds_ratio_loss": 8.088262557983398, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.08146525919437408, + "rewards/margins": 1.2396444082260132, + "rewards/rejected": -1.321109652519226, + "sft_loss": 0.027669047936797142, + "step": 9630 + }, + { + "epoch": 0.749902761571373, + "grad_norm": 2.603041410446167, + "learning_rate": 7.327731532037549e-06, + "logits/chosen": -0.34888967871665955, + "logits/rejected": -1.740108847618103, + "logps/chosen": -1.137001633644104, + "logps/rejected": -20.085874557495117, + "loss": 1.148, + "num_input_tokens_seen": 12606272, + "odds_ratio_loss": 11.355992317199707, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.11370013654232025, + "rewards/margins": 1.8948875665664673, + "rewards/rejected": -2.008587598800659, + "sft_loss": 0.012404505163431168, + "step": 9640 + }, + { + "epoch": 0.750680669000389, + "grad_norm": 3.0989139080047607, + "learning_rate": 7.284569257770382e-06, + "logits/chosen": -0.3092551529407501, + "logits/rejected": -1.3083518743515015, + "logps/chosen": -1.007351279258728, + "logps/rejected": -14.896029472351074, + "loss": 1.0127, + "num_input_tokens_seen": 12620336, + "odds_ratio_loss": 10.002145767211914, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1007351279258728, + "rewards/margins": 1.388867974281311, + "rewards/rejected": -1.4896031618118286, + "sft_loss": 0.012452312745153904, + "step": 9650 + }, + { + "epoch": 0.751458576429405, + "grad_norm": 2.0114264488220215, + "learning_rate": 7.241512788720578e-06, + "logits/chosen": -0.3267592787742615, + "logits/rejected": -1.3846229314804077, + "logps/chosen": -0.8181226849555969, + "logps/rejected": -11.771913528442383, + "loss": 0.8315, + "num_input_tokens_seen": 12630912, + "odds_ratio_loss": 8.077900886535645, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.08181225508451462, + "rewards/margins": 1.095379114151001, + "rewards/rejected": -1.1771913766860962, + "sft_loss": 0.023709949105978012, + "step": 9660 + }, + { + "epoch": 0.7522364838584209, + "grad_norm": 1.943953037261963, + "learning_rate": 7.198562382042434e-06, + "logits/chosen": -0.1044231653213501, + "logits/rejected": -1.8825289011001587, + "logps/chosen": -0.9438576698303223, + "logps/rejected": -14.98261833190918, + "loss": 0.9609, + "num_input_tokens_seen": 12644560, + "odds_ratio_loss": 9.543760299682617, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.09438576549291611, + "rewards/margins": 1.4038760662078857, + "rewards/rejected": -1.4982619285583496, + "sft_loss": 0.006542509887367487, + "step": 9670 + }, + { + "epoch": 0.7530143912874367, + "grad_norm": 2.966200828552246, + "learning_rate": 7.155718294256783e-06, + "logits/chosen": 0.012543360702693462, + "logits/rejected": -1.677483320236206, + "logps/chosen": -1.0234817266464233, + "logps/rejected": -11.815591812133789, + "loss": 1.0293, + "num_input_tokens_seen": 12659456, + "odds_ratio_loss": 10.21452808380127, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.10234817117452621, + "rewards/margins": 1.0792109966278076, + "rewards/rejected": -1.1815592050552368, + "sft_loss": 0.00782776903361082, + "step": 9680 + }, + { + "epoch": 0.7537922987164527, + "grad_norm": 7.487515449523926, + "learning_rate": 7.112980781249478e-06, + "logits/chosen": -0.5516456961631775, + "logits/rejected": -0.7924268841743469, + "logps/chosen": -0.9835079312324524, + "logps/rejected": -9.512467384338379, + "loss": 1.0017, + "num_input_tokens_seen": 12670160, + "odds_ratio_loss": 9.811184883117676, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.09835079312324524, + "rewards/margins": 0.8528959155082703, + "rewards/rejected": -0.9512467384338379, + "sft_loss": 0.020590314641594887, + "step": 9690 + }, + { + "epoch": 0.7545702061454687, + "grad_norm": 2.7716739177703857, + "learning_rate": 7.070350098269845e-06, + "logits/chosen": -0.24449455738067627, + "logits/rejected": -2.006038188934326, + "logps/chosen": -0.8078185319900513, + "logps/rejected": -17.701519012451172, + "loss": 0.8097, + "num_input_tokens_seen": 12680560, + "odds_ratio_loss": 8.062870025634766, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.08078185468912125, + "rewards/margins": 1.6893701553344727, + "rewards/rejected": -1.7701518535614014, + "sft_loss": 0.003373973071575165, + "step": 9700 + }, + { + "epoch": 0.7553481135744846, + "grad_norm": 2.9411494731903076, + "learning_rate": 7.027826499929177e-06, + "logits/chosen": -0.18928129971027374, + "logits/rejected": -1.3758265972137451, + "logps/chosen": -0.9373753666877747, + "logps/rejected": -15.49957275390625, + "loss": 0.9539, + "num_input_tokens_seen": 12694608, + "odds_ratio_loss": 9.455991744995117, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.09373755007982254, + "rewards/margins": 1.4562196731567383, + "rewards/rejected": -1.549957275390625, + "sft_loss": 0.008260851725935936, + "step": 9710 + }, + { + "epoch": 0.7561260210035006, + "grad_norm": 23.82400131225586, + "learning_rate": 6.985410240199203e-06, + "logits/chosen": -0.6237587928771973, + "logits/rejected": -1.1116478443145752, + "logps/chosen": -0.8208237886428833, + "logps/rejected": -14.505549430847168, + "loss": 0.8352, + "num_input_tokens_seen": 12704256, + "odds_ratio_loss": 7.775033473968506, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.08208237588405609, + "rewards/margins": 1.368472695350647, + "rewards/rejected": -1.4505550861358643, + "sft_loss": 0.05770029500126839, + "step": 9720 + }, + { + "epoch": 0.7569039284325165, + "grad_norm": 5.56984281539917, + "learning_rate": 6.943101572410571e-06, + "logits/chosen": -0.3685595393180847, + "logits/rejected": -1.0919402837753296, + "logps/chosen": -0.7797693014144897, + "logps/rejected": -13.029932975769043, + "loss": 0.7932, + "num_input_tokens_seen": 12713744, + "odds_ratio_loss": 7.839241027832031, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.07797692716121674, + "rewards/margins": 1.2250163555145264, + "rewards/rejected": -1.3029931783676147, + "sft_loss": 0.009250042960047722, + "step": 9730 + }, + { + "epoch": 0.7576818358615325, + "grad_norm": 1.6570754051208496, + "learning_rate": 6.900900749251338e-06, + "logits/chosen": -0.5102152228355408, + "logits/rejected": -1.114384651184082, + "logps/chosen": -0.7536827325820923, + "logps/rejected": -12.92182731628418, + "loss": 0.7615, + "num_input_tokens_seen": 12727072, + "odds_ratio_loss": 6.868859767913818, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.07536827772855759, + "rewards/margins": 1.2168145179748535, + "rewards/rejected": -1.2921828031539917, + "sft_loss": 0.0746065154671669, + "step": 9740 + }, + { + "epoch": 0.7584597432905484, + "grad_norm": 4.804180145263672, + "learning_rate": 6.858808022765467e-06, + "logits/chosen": -0.0006758645176887512, + "logits/rejected": -1.5899875164031982, + "logps/chosen": -0.8684228658676147, + "logps/rejected": -13.665643692016602, + "loss": 0.8872, + "num_input_tokens_seen": 12737776, + "odds_ratio_loss": 8.773876190185547, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.08684228360652924, + "rewards/margins": 1.2797222137451172, + "rewards/rejected": -1.3665645122528076, + "sft_loss": 0.009812915697693825, + "step": 9750 + }, + { + "epoch": 0.7592376507195644, + "grad_norm": 1.4910658597946167, + "learning_rate": 6.816823644351303e-06, + "logits/chosen": -0.5923045873641968, + "logits/rejected": -1.2481071949005127, + "logps/chosen": -0.7165006995201111, + "logps/rejected": -9.771490097045898, + "loss": 0.7448, + "num_input_tokens_seen": 12749344, + "odds_ratio_loss": 7.370150566101074, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.07165006548166275, + "rewards/margins": 0.9054989814758301, + "rewards/rejected": -0.9771490097045898, + "sft_loss": 0.007765809539705515, + "step": 9760 + }, + { + "epoch": 0.7600155581485804, + "grad_norm": 5.410368919372559, + "learning_rate": 6.774947864760089e-06, + "logits/chosen": -0.1797337532043457, + "logits/rejected": -1.679751992225647, + "logps/chosen": -1.0861485004425049, + "logps/rejected": -14.003316879272461, + "loss": 1.1028, + "num_input_tokens_seen": 12760368, + "odds_ratio_loss": 10.247908592224121, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.10861483961343765, + "rewards/margins": 1.2917169332504272, + "rewards/rejected": -1.400331735610962, + "sft_loss": 0.07805103808641434, + "step": 9770 + }, + { + "epoch": 0.7607934655775963, + "grad_norm": 17.647127151489258, + "learning_rate": 6.733180934094466e-06, + "logits/chosen": -0.40942397713661194, + "logits/rejected": -1.2711374759674072, + "logps/chosen": -0.953356921672821, + "logps/rejected": -14.270413398742676, + "loss": 0.9616, + "num_input_tokens_seen": 12775856, + "odds_ratio_loss": 9.490220069885254, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.0953356921672821, + "rewards/margins": 1.3317056894302368, + "rewards/rejected": -1.4270412921905518, + "sft_loss": 0.012609261088073254, + "step": 9780 + }, + { + "epoch": 0.7615713730066123, + "grad_norm": 1.5215857028961182, + "learning_rate": 6.691523101806976e-06, + "logits/chosen": -0.06620067358016968, + "logits/rejected": -1.9449646472930908, + "logps/chosen": -0.9097517728805542, + "logps/rejected": -17.92283058166504, + "loss": 0.9124, + "num_input_tokens_seen": 12790816, + "odds_ratio_loss": 8.980363845825195, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09097517281770706, + "rewards/margins": 1.7013078927993774, + "rewards/rejected": -1.792283058166504, + "sft_loss": 0.014339657500386238, + "step": 9790 + }, + { + "epoch": 0.7623492804356281, + "grad_norm": 15.784709930419922, + "learning_rate": 6.649974616698562e-06, + "logits/chosen": -0.2136622965335846, + "logits/rejected": -1.6949361562728882, + "logps/chosen": -1.03670334815979, + "logps/rejected": -13.867294311523438, + "loss": 1.0583, + "num_input_tokens_seen": 12802160, + "odds_ratio_loss": 10.493218421936035, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.10367034375667572, + "rewards/margins": 1.2830591201782227, + "rewards/rejected": -1.3867294788360596, + "sft_loss": 0.008986649103462696, + "step": 9800 + }, + { + "epoch": 0.7631271878646441, + "grad_norm": 15.24134635925293, + "learning_rate": 6.6085357269171035e-06, + "logits/chosen": -0.48242124915122986, + "logits/rejected": -0.9146297574043274, + "logps/chosen": -0.6837570071220398, + "logps/rejected": -10.073492050170898, + "loss": 0.7005, + "num_input_tokens_seen": 12814800, + "odds_ratio_loss": 6.878556728363037, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.06837569177150726, + "rewards/margins": 0.938973605632782, + "rewards/rejected": -1.0073492527008057, + "sft_loss": 0.012609479017555714, + "step": 9810 + }, + { + "epoch": 0.76390509529366, + "grad_norm": 5.1652512550354, + "learning_rate": 6.567206679955917e-06, + "logits/chosen": -0.18169178068637848, + "logits/rejected": -1.6934398412704468, + "logps/chosen": -0.8636601567268372, + "logps/rejected": -13.6743803024292, + "loss": 0.8859, + "num_input_tokens_seen": 12827936, + "odds_ratio_loss": 8.807854652404785, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.08636601269245148, + "rewards/margins": 1.2810719013214111, + "rewards/rejected": -1.3674379587173462, + "sft_loss": 0.005116607528179884, + "step": 9820 + }, + { + "epoch": 0.764683002722676, + "grad_norm": 7.437309265136719, + "learning_rate": 6.525987722652286e-06, + "logits/chosen": -0.21487879753112793, + "logits/rejected": -1.2891901731491089, + "logps/chosen": -0.8677453994750977, + "logps/rejected": -15.33686351776123, + "loss": 0.8864, + "num_input_tokens_seen": 12842480, + "odds_ratio_loss": 8.776076316833496, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.08677453547716141, + "rewards/margins": 1.4469118118286133, + "rewards/rejected": -1.5336863994598389, + "sft_loss": 0.008795716799795628, + "step": 9830 + }, + { + "epoch": 0.7654609101516919, + "grad_norm": 1.8497710227966309, + "learning_rate": 6.484879101185986e-06, + "logits/chosen": -0.2847957909107208, + "logits/rejected": -2.0163838863372803, + "logps/chosen": -0.8841788172721863, + "logps/rejected": -16.62775421142578, + "loss": 0.8887, + "num_input_tokens_seen": 12857024, + "odds_ratio_loss": 8.826251983642578, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.0884178951382637, + "rewards/margins": 1.5743576288223267, + "rewards/rejected": -1.6627753973007202, + "sft_loss": 0.0060475775972008705, + "step": 9840 + }, + { + "epoch": 0.7662388175807079, + "grad_norm": 2.8241870403289795, + "learning_rate": 6.443881061077814e-06, + "logits/chosen": -0.13205505907535553, + "logits/rejected": -1.3500169515609741, + "logps/chosen": -1.0866869688034058, + "logps/rejected": -17.037189483642578, + "loss": 1.0873, + "num_input_tokens_seen": 12870720, + "odds_ratio_loss": 10.3355131149292, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.10866870731115341, + "rewards/margins": 1.5950504541397095, + "rewards/rejected": -1.703718900680542, + "sft_loss": 0.05375038459897041, + "step": 9850 + }, + { + "epoch": 0.7670167250097238, + "grad_norm": 8.120436668395996, + "learning_rate": 6.402993847188113e-06, + "logits/chosen": -0.07916440069675446, + "logits/rejected": -1.727804183959961, + "logps/chosen": -0.8412625193595886, + "logps/rejected": -12.190113067626953, + "loss": 0.8442, + "num_input_tokens_seen": 12886128, + "odds_ratio_loss": 8.385359764099121, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.08412624895572662, + "rewards/margins": 1.1348850727081299, + "rewards/rejected": -1.2190113067626953, + "sft_loss": 0.005621061660349369, + "step": 9860 + }, + { + "epoch": 0.7677946324387398, + "grad_norm": 8.273092269897461, + "learning_rate": 6.3622177037153255e-06, + "logits/chosen": -0.30196234583854675, + "logits/rejected": -1.4886324405670166, + "logps/chosen": -0.9431349039077759, + "logps/rejected": -11.96256160736084, + "loss": 0.9606, + "num_input_tokens_seen": 12902208, + "odds_ratio_loss": 9.528372764587402, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.09431348741054535, + "rewards/margins": 1.101942777633667, + "rewards/rejected": -1.196256160736084, + "sft_loss": 0.007743525318801403, + "step": 9870 + }, + { + "epoch": 0.7685725398677558, + "grad_norm": 4.558796405792236, + "learning_rate": 6.3215528741945225e-06, + "logits/chosen": -0.49385198950767517, + "logits/rejected": -0.8861545324325562, + "logps/chosen": -0.8803753852844238, + "logps/rejected": -10.397488594055176, + "loss": 0.8937, + "num_input_tokens_seen": 12915488, + "odds_ratio_loss": 8.755577087402344, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.08803753554821014, + "rewards/margins": 0.9517113566398621, + "rewards/rejected": -1.0397489070892334, + "sft_loss": 0.018163876608014107, + "step": 9880 + }, + { + "epoch": 0.7693504472967717, + "grad_norm": 1.6330955028533936, + "learning_rate": 6.28099960149596e-06, + "logits/chosen": -0.6630735397338867, + "logits/rejected": -1.1244785785675049, + "logps/chosen": -0.8058063387870789, + "logps/rejected": -22.111560821533203, + "loss": 0.8059, + "num_input_tokens_seen": 12929568, + "odds_ratio_loss": 7.960448265075684, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.08058063685894012, + "rewards/margins": 2.130575656890869, + "rewards/rejected": -2.211156129837036, + "sft_loss": 0.009810159914195538, + "step": 9890 + }, + { + "epoch": 0.7701283547257877, + "grad_norm": 7.713956832885742, + "learning_rate": 6.2405581278236145e-06, + "logits/chosen": -0.44286996126174927, + "logits/rejected": -1.3790740966796875, + "logps/chosen": -0.9783164858818054, + "logps/rejected": -20.651592254638672, + "loss": 0.9787, + "num_input_tokens_seen": 12940400, + "odds_ratio_loss": 9.46540641784668, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09783165901899338, + "rewards/margins": 1.9673277139663696, + "rewards/rejected": -2.065159320831299, + "sft_loss": 0.032199710607528687, + "step": 9900 + }, + { + "epoch": 0.7709062621548036, + "grad_norm": 3.927487850189209, + "learning_rate": 6.200228694713753e-06, + "logits/chosen": -0.042903829365968704, + "logits/rejected": -1.6210275888442993, + "logps/chosen": -0.8930376172065735, + "logps/rejected": -16.273311614990234, + "loss": 0.8973, + "num_input_tokens_seen": 12955424, + "odds_ratio_loss": 8.926946640014648, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.08930376172065735, + "rewards/margins": 1.538027286529541, + "rewards/rejected": -1.6273311376571655, + "sft_loss": 0.0045908852480351925, + "step": 9910 + }, + { + "epoch": 0.7716841695838196, + "grad_norm": 1.831764817237854, + "learning_rate": 6.160011543033467e-06, + "logits/chosen": -0.25490206480026245, + "logits/rejected": -1.7062351703643799, + "logps/chosen": -1.1320891380310059, + "logps/rejected": -13.286028861999512, + "loss": 1.1717, + "num_input_tokens_seen": 12965680, + "odds_ratio_loss": 11.279840469360352, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.11320891231298447, + "rewards/margins": 1.2153937816619873, + "rewards/rejected": -1.3286025524139404, + "sft_loss": 0.043743547052145004, + "step": 9920 + }, + { + "epoch": 0.7724620770128354, + "grad_norm": 4.164346694946289, + "learning_rate": 6.119906912979264e-06, + "logits/chosen": -0.2769322097301483, + "logits/rejected": -1.6600967645645142, + "logps/chosen": -0.7944921255111694, + "logps/rejected": -17.34960174560547, + "loss": 0.7974, + "num_input_tokens_seen": 12976944, + "odds_ratio_loss": 7.91311502456665, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.07944921404123306, + "rewards/margins": 1.6555111408233643, + "rewards/rejected": -1.7349601984024048, + "sft_loss": 0.006089845206588507, + "step": 9930 + }, + { + "epoch": 0.7732399844418514, + "grad_norm": 4.3693318367004395, + "learning_rate": 6.0799150440756095e-06, + "logits/chosen": -0.1835547685623169, + "logits/rejected": -1.7030483484268188, + "logps/chosen": -0.8427737951278687, + "logps/rejected": -18.01861000061035, + "loss": 0.8444, + "num_input_tokens_seen": 12992592, + "odds_ratio_loss": 8.359333992004395, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.08427738398313522, + "rewards/margins": 1.7175838947296143, + "rewards/rejected": -1.801861047744751, + "sft_loss": 0.008451655507087708, + "step": 9940 + }, + { + "epoch": 0.7740178918708673, + "grad_norm": 1.813942790031433, + "learning_rate": 6.040036175173505e-06, + "logits/chosen": -0.07210500538349152, + "logits/rejected": -1.649038314819336, + "logps/chosen": -0.8709640502929688, + "logps/rejected": -17.332408905029297, + "loss": 0.8763, + "num_input_tokens_seen": 13007312, + "odds_ratio_loss": 8.719454765319824, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.08709640055894852, + "rewards/margins": 1.6461445093154907, + "rewards/rejected": -1.7332408428192139, + "sft_loss": 0.004315356258302927, + "step": 9950 + }, + { + "epoch": 0.7747957992998833, + "grad_norm": 3.831186532974243, + "learning_rate": 6.00027054444906e-06, + "logits/chosen": -0.3818754553794861, + "logits/rejected": -1.2485878467559814, + "logps/chosen": -0.7479159832000732, + "logps/rejected": -11.673700332641602, + "loss": 0.7542, + "num_input_tokens_seen": 13020864, + "odds_ratio_loss": 7.26510763168335, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.07479159533977509, + "rewards/margins": 1.0925782918930054, + "rewards/rejected": -1.167370080947876, + "sft_loss": 0.02773687243461609, + "step": 9960 + }, + { + "epoch": 0.7755737067288992, + "grad_norm": 1.1907448768615723, + "learning_rate": 5.960618389402075e-06, + "logits/chosen": -0.005200075916945934, + "logits/rejected": -1.7917149066925049, + "logps/chosen": -1.2306063175201416, + "logps/rejected": -19.59311294555664, + "loss": 1.231, + "num_input_tokens_seen": 13034928, + "odds_ratio_loss": 12.255085945129395, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.12306062877178192, + "rewards/margins": 1.836250901222229, + "rewards/rejected": -1.9593114852905273, + "sft_loss": 0.0055275773629546165, + "step": 9970 + }, + { + "epoch": 0.7763516141579152, + "grad_norm": 5.450308799743652, + "learning_rate": 5.92107994685461e-06, + "logits/chosen": -0.529719889163971, + "logits/rejected": -1.083274483680725, + "logps/chosen": -0.8484355211257935, + "logps/rejected": -10.391220092773438, + "loss": 0.857, + "num_input_tokens_seen": 13046672, + "odds_ratio_loss": 8.189764022827148, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.08484355360269547, + "rewards/margins": 0.9542785882949829, + "rewards/rejected": -1.0391219854354858, + "sft_loss": 0.03801070898771286, + "step": 9980 + }, + { + "epoch": 0.7771295215869312, + "grad_norm": 7.595885753631592, + "learning_rate": 5.881655452949583e-06, + "logits/chosen": 0.03671069070696831, + "logits/rejected": -2.0498228073120117, + "logps/chosen": -0.898884654045105, + "logps/rejected": -14.653169631958008, + "loss": 0.9055, + "num_input_tokens_seen": 13056912, + "odds_ratio_loss": 9.006585121154785, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.08988846838474274, + "rewards/margins": 1.3754284381866455, + "rewards/rejected": -1.4653170108795166, + "sft_loss": 0.0048610251396894455, + "step": 9990 + }, + { + "epoch": 0.7779074290159471, + "grad_norm": 3.6287105083465576, + "learning_rate": 5.842345143149361e-06, + "logits/chosen": -0.1152871698141098, + "logits/rejected": -1.3964219093322754, + "logps/chosen": -1.1272786855697632, + "logps/rejected": -9.357419967651367, + "loss": 1.1474, + "num_input_tokens_seen": 13068000, + "odds_ratio_loss": 10.209806442260742, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.11272789537906647, + "rewards/margins": 0.8230140805244446, + "rewards/rejected": -0.9357420206069946, + "sft_loss": 0.12642648816108704, + "step": 10000 + }, + { + "epoch": 0.7786853364449631, + "grad_norm": 4.439057350158691, + "learning_rate": 5.803149252234336e-06, + "logits/chosen": -0.22541113197803497, + "logits/rejected": -1.4121778011322021, + "logps/chosen": -1.1307504177093506, + "logps/rejected": -19.595867156982422, + "loss": 1.132, + "num_input_tokens_seen": 13083008, + "odds_ratio_loss": 9.42105484008789, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.11307503283023834, + "rewards/margins": 1.8465116024017334, + "rewards/rejected": -1.9595867395401, + "sft_loss": 0.18993499875068665, + "step": 10010 + }, + { + "epoch": 0.779463243873979, + "grad_norm": 1.8150360584259033, + "learning_rate": 5.764068014301538e-06, + "logits/chosen": -0.14730560779571533, + "logits/rejected": -1.2366430759429932, + "logps/chosen": -0.7479221224784851, + "logps/rejected": -6.61281681060791, + "loss": 0.7628, + "num_input_tokens_seen": 13090528, + "odds_ratio_loss": 7.487229824066162, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.07479222118854523, + "rewards/margins": 0.5864894986152649, + "rewards/rejected": -0.6612816452980042, + "sft_loss": 0.014043653383851051, + "step": 10020 + }, + { + "epoch": 0.780241151302995, + "grad_norm": 11.748605728149414, + "learning_rate": 5.7251016627632435e-06, + "logits/chosen": -0.6228640079498291, + "logits/rejected": -1.2003719806671143, + "logps/chosen": -0.8472782373428345, + "logps/rejected": -18.236339569091797, + "loss": 0.8512, + "num_input_tokens_seen": 13106288, + "odds_ratio_loss": 8.328832626342773, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.08472783118486404, + "rewards/margins": 1.7389062643051147, + "rewards/rejected": -1.8236339092254639, + "sft_loss": 0.018275240436196327, + "step": 10030 + }, + { + "epoch": 0.7810190587320109, + "grad_norm": 8.976465225219727, + "learning_rate": 5.686250430345555e-06, + "logits/chosen": -0.40403372049331665, + "logits/rejected": -1.1737014055252075, + "logps/chosen": -0.8802043795585632, + "logps/rejected": -11.703226089477539, + "loss": 0.8928, + "num_input_tokens_seen": 13119776, + "odds_ratio_loss": 8.639188766479492, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.0880204439163208, + "rewards/margins": 1.0823020935058594, + "rewards/rejected": -1.1703224182128906, + "sft_loss": 0.028895627707242966, + "step": 10040 + }, + { + "epoch": 0.7817969661610268, + "grad_norm": 11.301222801208496, + "learning_rate": 5.647514549087046e-06, + "logits/chosen": -0.41604989767074585, + "logits/rejected": -1.3308502435684204, + "logps/chosen": -1.0168416500091553, + "logps/rejected": -14.803606986999512, + "loss": 1.0266, + "num_input_tokens_seen": 13132656, + "odds_ratio_loss": 10.065681457519531, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.10168415307998657, + "rewards/margins": 1.3786766529083252, + "rewards/rejected": -1.480360984802246, + "sft_loss": 0.019993456080555916, + "step": 10050 + }, + { + "epoch": 0.7825748735900427, + "grad_norm": 3.9200751781463623, + "learning_rate": 5.6088942503373265e-06, + "logits/chosen": -0.00039936602115631104, + "logits/rejected": -1.7556613683700562, + "logps/chosen": -0.8142244219779968, + "logps/rejected": -15.882316589355469, + "loss": 0.8212, + "num_input_tokens_seen": 13148624, + "odds_ratio_loss": 8.17491340637207, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.08142244815826416, + "rewards/margins": 1.506809115409851, + "rewards/rejected": -1.5882318019866943, + "sft_loss": 0.003715798957273364, + "step": 10060 + }, + { + "epoch": 0.7833527810190587, + "grad_norm": 1.288907527923584, + "learning_rate": 5.570389764755729e-06, + "logits/chosen": -0.10343638807535172, + "logits/rejected": -1.422821044921875, + "logps/chosen": -0.8510880470275879, + "logps/rejected": -13.416641235351562, + "loss": 0.8577, + "num_input_tokens_seen": 13168432, + "odds_ratio_loss": 8.532673835754395, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.08510880172252655, + "rewards/margins": 1.256555438041687, + "rewards/rejected": -1.3416640758514404, + "sft_loss": 0.004443835467100143, + "step": 10070 + }, + { + "epoch": 0.7841306884480747, + "grad_norm": 8.386462211608887, + "learning_rate": 5.5320013223098684e-06, + "logits/chosen": -0.10891640186309814, + "logits/rejected": -1.3848743438720703, + "logps/chosen": -0.6136741638183594, + "logps/rejected": -19.104137420654297, + "loss": 0.6158, + "num_input_tokens_seen": 13179840, + "odds_ratio_loss": 6.0931010246276855, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.061367422342300415, + "rewards/margins": 1.8490467071533203, + "rewards/rejected": -1.9104140996932983, + "sft_loss": 0.006519814021885395, + "step": 10080 + }, + { + "epoch": 0.7849085958770906, + "grad_norm": 1.7663533687591553, + "learning_rate": 5.493729152274302e-06, + "logits/chosen": -0.22560708224773407, + "logits/rejected": -1.5404101610183716, + "logps/chosen": -1.1439088582992554, + "logps/rejected": -20.378692626953125, + "loss": 1.1557, + "num_input_tokens_seen": 13200592, + "odds_ratio_loss": 11.499128341674805, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.11439088732004166, + "rewards/margins": 1.923478364944458, + "rewards/rejected": -2.0378692150115967, + "sft_loss": 0.005769290495663881, + "step": 10090 + }, + { + "epoch": 0.7856865033061066, + "grad_norm": 2.094831705093384, + "learning_rate": 5.455573483229143e-06, + "logits/chosen": -0.20372407138347626, + "logits/rejected": -1.6087785959243774, + "logps/chosen": -0.8583130836486816, + "logps/rejected": -9.61543083190918, + "loss": 0.8677, + "num_input_tokens_seen": 13217472, + "odds_ratio_loss": 8.596895217895508, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.08583130687475204, + "rewards/margins": 0.8757117390632629, + "rewards/rejected": -0.961543083190918, + "sft_loss": 0.00801418349146843, + "step": 10100 + }, + { + "epoch": 0.7864644107351225, + "grad_norm": 2.245335578918457, + "learning_rate": 5.417534543058716e-06, + "logits/chosen": -0.3722991347312927, + "logits/rejected": -1.39009428024292, + "logps/chosen": -0.7631933689117432, + "logps/rejected": -14.828773498535156, + "loss": 0.7698, + "num_input_tokens_seen": 13233008, + "odds_ratio_loss": 7.651087760925293, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.07631933689117432, + "rewards/margins": 1.4065582752227783, + "rewards/rejected": -1.482877492904663, + "sft_loss": 0.004690065048635006, + "step": 10110 + }, + { + "epoch": 0.7872423181641385, + "grad_norm": 1.768862009048462, + "learning_rate": 5.379612558950173e-06, + "logits/chosen": 0.006290731020271778, + "logits/rejected": -1.8056621551513672, + "logps/chosen": -0.8933433294296265, + "logps/rejected": -10.847860336303711, + "loss": 0.9274, + "num_input_tokens_seen": 13250944, + "odds_ratio_loss": 9.240643501281738, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.08933433145284653, + "rewards/margins": 0.995451807975769, + "rewards/rejected": -1.0847861766815186, + "sft_loss": 0.00336031848564744, + "step": 10120 + }, + { + "epoch": 0.7880202255931544, + "grad_norm": 1.7466740608215332, + "learning_rate": 5.341807757392153e-06, + "logits/chosen": -0.26185980439186096, + "logits/rejected": -1.625367522239685, + "logps/chosen": -0.7085305452346802, + "logps/rejected": -20.165281295776367, + "loss": 0.7105, + "num_input_tokens_seen": 13263552, + "odds_ratio_loss": 7.040585994720459, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.07085305452346802, + "rewards/margins": 1.9456751346588135, + "rewards/rejected": -2.0165281295776367, + "sft_loss": 0.006464931182563305, + "step": 10130 + }, + { + "epoch": 0.7887981330221704, + "grad_norm": 1.9766432046890259, + "learning_rate": 5.30412036417342e-06, + "logits/chosen": -0.15813367068767548, + "logits/rejected": -1.8555446863174438, + "logps/chosen": -0.9007395505905151, + "logps/rejected": -18.488134384155273, + "loss": 0.9119, + "num_input_tokens_seen": 13274960, + "odds_ratio_loss": 9.050884246826172, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09007396548986435, + "rewards/margins": 1.7587394714355469, + "rewards/rejected": -1.8488136529922485, + "sft_loss": 0.006835309322923422, + "step": 10140 + }, + { + "epoch": 0.7895760404511863, + "grad_norm": 4.220992088317871, + "learning_rate": 5.266550604381526e-06, + "logits/chosen": -0.2565006613731384, + "logits/rejected": -1.4154245853424072, + "logps/chosen": -1.000119686126709, + "logps/rejected": -15.364748001098633, + "loss": 1.0084, + "num_input_tokens_seen": 13288768, + "odds_ratio_loss": 9.917895317077637, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.10001198202371597, + "rewards/margins": 1.4364628791809082, + "rewards/rejected": -1.5364747047424316, + "sft_loss": 0.016632404178380966, + "step": 10150 + }, + { + "epoch": 0.7903539478802023, + "grad_norm": 55.403568267822266, + "learning_rate": 5.229098702401447e-06, + "logits/chosen": -0.535290539264679, + "logits/rejected": -0.9979928731918335, + "logps/chosen": -1.0686589479446411, + "logps/rejected": -15.240943908691406, + "loss": 1.0706, + "num_input_tokens_seen": 13303632, + "odds_ratio_loss": 10.46064281463623, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.10686589777469635, + "rewards/margins": 1.4172285795211792, + "rewards/rejected": -1.524094581604004, + "sft_loss": 0.024510297924280167, + "step": 10160 + }, + { + "epoch": 0.7911318553092183, + "grad_norm": 2.378926992416382, + "learning_rate": 5.19176488191426e-06, + "logits/chosen": -0.089039646089077, + "logits/rejected": -1.5092610120773315, + "logps/chosen": -1.036505937576294, + "logps/rejected": -14.474766731262207, + "loss": 1.0391, + "num_input_tokens_seen": 13317664, + "odds_ratio_loss": 10.324877738952637, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.10365059226751328, + "rewards/margins": 1.343826174736023, + "rewards/rejected": -1.4474767446517944, + "sft_loss": 0.00660539697855711, + "step": 10170 + }, + { + "epoch": 0.7919097627382341, + "grad_norm": 3.639150619506836, + "learning_rate": 5.154549365895814e-06, + "logits/chosen": -0.09354257583618164, + "logits/rejected": -1.6912329196929932, + "logps/chosen": -0.8237224817276001, + "logps/rejected": -22.405752182006836, + "loss": 0.8254, + "num_input_tokens_seen": 13334128, + "odds_ratio_loss": 8.221444129943848, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.08237224817276001, + "rewards/margins": 2.158203125, + "rewards/rejected": -2.2405755519866943, + "sft_loss": 0.0032429061830043793, + "step": 10180 + }, + { + "epoch": 0.7926876701672501, + "grad_norm": 2.631995916366577, + "learning_rate": 5.117452376615345e-06, + "logits/chosen": -0.07670159637928009, + "logits/rejected": -1.7550842761993408, + "logps/chosen": -1.188686490058899, + "logps/rejected": -8.534611701965332, + "loss": 1.2081, + "num_input_tokens_seen": 13346160, + "odds_ratio_loss": 11.93842887878418, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.11886866390705109, + "rewards/margins": 0.7345924377441406, + "rewards/rejected": -0.8534610867500305, + "sft_loss": 0.014260287396609783, + "step": 10190 + }, + { + "epoch": 0.793465577596266, + "grad_norm": 31.077972412109375, + "learning_rate": 5.0804741356342445e-06, + "logits/chosen": -0.42552217841148376, + "logits/rejected": -0.9647568464279175, + "logps/chosen": -1.0847182273864746, + "logps/rejected": -12.32005500793457, + "loss": 1.1025, + "num_input_tokens_seen": 13361056, + "odds_ratio_loss": 9.90789794921875, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.1084718257188797, + "rewards/margins": 1.1235337257385254, + "rewards/rejected": -1.2320055961608887, + "sft_loss": 0.11174285411834717, + "step": 10200 + }, + { + "epoch": 0.794243485025282, + "grad_norm": 2.0199506282806396, + "learning_rate": 5.043614863804646e-06, + "logits/chosen": -0.3819964826107025, + "logits/rejected": -1.6330349445343018, + "logps/chosen": -1.033983588218689, + "logps/rejected": -24.93582534790039, + "loss": 1.034, + "num_input_tokens_seen": 13373008, + "odds_ratio_loss": 9.945966720581055, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.10339836031198502, + "rewards/margins": 2.390184164047241, + "rewards/rejected": -2.4935827255249023, + "sft_loss": 0.039412155747413635, + "step": 10210 + }, + { + "epoch": 0.7950213924542979, + "grad_norm": 11.4447021484375, + "learning_rate": 5.0068747812681505e-06, + "logits/chosen": -0.1652594357728958, + "logits/rejected": -1.1705518960952759, + "logps/chosen": -0.7428327202796936, + "logps/rejected": -13.094820022583008, + "loss": 0.7596, + "num_input_tokens_seen": 13383968, + "odds_ratio_loss": 7.454339504241943, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.07428327202796936, + "rewards/margins": 1.235198736190796, + "rewards/rejected": -1.3094819784164429, + "sft_loss": 0.014146876521408558, + "step": 10220 + }, + { + "epoch": 0.7957992998833139, + "grad_norm": 7.466521739959717, + "learning_rate": 4.970254107454497e-06, + "logits/chosen": -0.33368849754333496, + "logits/rejected": -1.018107533454895, + "logps/chosen": -0.7646960616111755, + "logps/rejected": -17.289798736572266, + "loss": 0.7803, + "num_input_tokens_seen": 13400320, + "odds_ratio_loss": 7.598325252532959, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.07646960765123367, + "rewards/margins": 1.6525102853775024, + "rewards/rejected": -1.7289798259735107, + "sft_loss": 0.020495515316724777, + "step": 10230 + }, + { + "epoch": 0.7965772073123298, + "grad_norm": 3.564819812774658, + "learning_rate": 4.933753061080265e-06, + "logits/chosen": -0.4034608006477356, + "logits/rejected": -1.3804579973220825, + "logps/chosen": -1.7706609964370728, + "logps/rejected": -17.545896530151367, + "loss": 1.8045, + "num_input_tokens_seen": 13419232, + "odds_ratio_loss": 15.623266220092773, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.17706608772277832, + "rewards/margins": 1.5775234699249268, + "rewards/rejected": -1.7545896768569946, + "sft_loss": 0.24220208823680878, + "step": 10240 + }, + { + "epoch": 0.7973551147413458, + "grad_norm": 1.9408292770385742, + "learning_rate": 4.897371860147556e-06, + "logits/chosen": -0.10395392030477524, + "logits/rejected": -1.6356542110443115, + "logps/chosen": -1.13056218624115, + "logps/rejected": -25.447980880737305, + "loss": 1.1486, + "num_input_tokens_seen": 13432848, + "odds_ratio_loss": 11.428488731384277, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.1130562275648117, + "rewards/margins": 2.43174147605896, + "rewards/rejected": -2.544797897338867, + "sft_loss": 0.005717330612242222, + "step": 10250 + }, + { + "epoch": 0.7981330221703618, + "grad_norm": 1.4113094806671143, + "learning_rate": 4.8611107219426996e-06, + "logits/chosen": -0.5167861580848694, + "logits/rejected": -1.1747467517852783, + "logps/chosen": -0.9015966653823853, + "logps/rejected": -15.947178840637207, + "loss": 0.9024, + "num_input_tokens_seen": 13445440, + "odds_ratio_loss": 8.933000564575195, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09015966206789017, + "rewards/margins": 1.5045583248138428, + "rewards/rejected": -1.5947179794311523, + "sft_loss": 0.009095687419176102, + "step": 10260 + }, + { + "epoch": 0.7989109295993777, + "grad_norm": 6.226866245269775, + "learning_rate": 4.824969863034942e-06, + "logits/chosen": -0.2744348645210266, + "logits/rejected": -1.526656985282898, + "logps/chosen": -1.152288794517517, + "logps/rejected": -21.838542938232422, + "loss": 1.1687, + "num_input_tokens_seen": 13459184, + "odds_ratio_loss": 11.03105640411377, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.11522886902093887, + "rewards/margins": 2.0686254501342773, + "rewards/rejected": -2.1838538646698, + "sft_loss": 0.0656442791223526, + "step": 10270 + }, + { + "epoch": 0.7996888370283937, + "grad_norm": 3.539475202560425, + "learning_rate": 4.788949499275177e-06, + "logits/chosen": -0.30656367540359497, + "logits/rejected": -0.8668183088302612, + "logps/chosen": -0.8456679582595825, + "logps/rejected": -16.91873550415039, + "loss": 0.8515, + "num_input_tokens_seen": 13470288, + "odds_ratio_loss": 8.222433090209961, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.08456679433584213, + "rewards/margins": 1.6073070764541626, + "rewards/rejected": -1.6918739080429077, + "sft_loss": 0.029214253649115562, + "step": 10280 + }, + { + "epoch": 0.8004667444574096, + "grad_norm": 1.8849167823791504, + "learning_rate": 4.7530498457946325e-06, + "logits/chosen": -0.3800256848335266, + "logits/rejected": -1.3928248882293701, + "logps/chosen": -0.7146269083023071, + "logps/rejected": -13.00952434539795, + "loss": 0.7249, + "num_input_tokens_seen": 13484928, + "odds_ratio_loss": 7.169774532318115, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.07146269083023071, + "rewards/margins": 1.229489803314209, + "rewards/rejected": -1.300952672958374, + "sft_loss": 0.007878500036895275, + "step": 10290 + }, + { + "epoch": 0.8012446518864255, + "grad_norm": 28.979467391967773, + "learning_rate": 4.717271117003605e-06, + "logits/chosen": -0.49607008695602417, + "logits/rejected": -1.2129285335540771, + "logps/chosen": -1.0439517498016357, + "logps/rejected": -9.6376953125, + "loss": 1.07, + "num_input_tokens_seen": 13495120, + "odds_ratio_loss": 10.457507133483887, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.10439516603946686, + "rewards/margins": 0.8593744039535522, + "rewards/rejected": -0.9637695550918579, + "sft_loss": 0.024267733097076416, + "step": 10300 + }, + { + "epoch": 0.8020225593154414, + "grad_norm": 0.30100616812705994, + "learning_rate": 4.68161352659015e-06, + "logits/chosen": -0.3668210804462433, + "logits/rejected": -0.8683077096939087, + "logps/chosen": -0.7981551885604858, + "logps/rejected": -14.805597305297852, + "loss": 0.8126, + "num_input_tokens_seen": 13506912, + "odds_ratio_loss": 8.055696487426758, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.07981552928686142, + "rewards/margins": 1.400744080543518, + "rewards/rejected": -1.4805595874786377, + "sft_loss": 0.007010910660028458, + "step": 10310 + }, + { + "epoch": 0.8028004667444574, + "grad_norm": 11.078510284423828, + "learning_rate": 4.646077287518838e-06, + "logits/chosen": -0.39960020780563354, + "logits/rejected": -1.750201940536499, + "logps/chosen": -0.8384559750556946, + "logps/rejected": -13.797350883483887, + "loss": 0.8468, + "num_input_tokens_seen": 13516128, + "odds_ratio_loss": 8.331451416015625, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.0838456004858017, + "rewards/margins": 1.2958896160125732, + "rewards/rejected": -1.3797352313995361, + "sft_loss": 0.013614359311759472, + "step": 10320 + }, + { + "epoch": 0.8035783741734733, + "grad_norm": 1.1385927200317383, + "learning_rate": 4.610662612029482e-06, + "logits/chosen": -0.13649150729179382, + "logits/rejected": -1.0758001804351807, + "logps/chosen": -0.8508375287055969, + "logps/rejected": -12.534235954284668, + "loss": 0.8695, + "num_input_tokens_seen": 13528368, + "odds_ratio_loss": 8.62982177734375, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.08508376032114029, + "rewards/margins": 1.1683398485183716, + "rewards/rejected": -1.2534236907958984, + "sft_loss": 0.0065626101568341255, + "step": 10330 + }, + { + "epoch": 0.8043562816024893, + "grad_norm": 2.9482533931732178, + "learning_rate": 4.575369711635843e-06, + "logits/chosen": -0.10335323959589005, + "logits/rejected": -1.4621038436889648, + "logps/chosen": -0.7176315188407898, + "logps/rejected": -24.72249984741211, + "loss": 0.7204, + "num_input_tokens_seen": 13545872, + "odds_ratio_loss": 7.180578708648682, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.07176315784454346, + "rewards/margins": 2.400486707687378, + "rewards/rejected": -2.47225022315979, + "sft_loss": 0.002294877776876092, + "step": 10340 + }, + { + "epoch": 0.8051341890315052, + "grad_norm": 16.524131774902344, + "learning_rate": 4.540198797124382e-06, + "logits/chosen": -0.5208472013473511, + "logits/rejected": -0.6951860189437866, + "logps/chosen": -0.9725438952445984, + "logps/rejected": -15.746805191040039, + "loss": 0.9747, + "num_input_tokens_seen": 13562416, + "odds_ratio_loss": 9.463030815124512, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09725438058376312, + "rewards/margins": 1.4774260520935059, + "rewards/rejected": -1.5746803283691406, + "sft_loss": 0.028433341532945633, + "step": 10350 + }, + { + "epoch": 0.8059120964605212, + "grad_norm": 1.489711880683899, + "learning_rate": 4.5051500785530024e-06, + "logits/chosen": -0.309017539024353, + "logits/rejected": -1.6297719478607178, + "logps/chosen": -0.9229093790054321, + "logps/rejected": -20.863962173461914, + "loss": 0.9229, + "num_input_tokens_seen": 13579456, + "odds_ratio_loss": 9.180902481079102, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09229093790054321, + "rewards/margins": 1.9941049814224243, + "rewards/rejected": -2.086395740509033, + "sft_loss": 0.0048326728865504265, + "step": 10360 + }, + { + "epoch": 0.8066900038895372, + "grad_norm": 2.2888967990875244, + "learning_rate": 4.470223765249796e-06, + "logits/chosen": -0.23541298508644104, + "logits/rejected": -1.2439138889312744, + "logps/chosen": -0.7586043477058411, + "logps/rejected": -13.460233688354492, + "loss": 0.7655, + "num_input_tokens_seen": 13594272, + "odds_ratio_loss": 7.621101379394531, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.07586044073104858, + "rewards/margins": 1.2701631784439087, + "rewards/rejected": -1.346023440361023, + "sft_loss": 0.0033806138671934605, + "step": 10370 + }, + { + "epoch": 0.8074679113185531, + "grad_norm": 1.8455400466918945, + "learning_rate": 4.435420065811791e-06, + "logits/chosen": -0.10371305793523788, + "logits/rejected": -1.3010910749435425, + "logps/chosen": -0.8560296297073364, + "logps/rejected": -5.94932746887207, + "loss": 0.8874, + "num_input_tokens_seen": 13610336, + "odds_ratio_loss": 8.841263771057129, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.08560295403003693, + "rewards/margins": 0.5093298554420471, + "rewards/rejected": -0.5949327349662781, + "sft_loss": 0.003240591613575816, + "step": 10380 + }, + { + "epoch": 0.8082458187475691, + "grad_norm": 2.436837673187256, + "learning_rate": 4.4007391881036975e-06, + "logits/chosen": -0.5878714919090271, + "logits/rejected": -1.4302802085876465, + "logps/chosen": -1.0162811279296875, + "logps/rejected": -21.074613571166992, + "loss": 1.0227, + "num_input_tokens_seen": 13622880, + "odds_ratio_loss": 9.80493450164795, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.10162810981273651, + "rewards/margins": 2.005833387374878, + "rewards/rejected": -2.107461452484131, + "sft_loss": 0.04224376007914543, + "step": 10390 + }, + { + "epoch": 0.809023726176585, + "grad_norm": 3.158832550048828, + "learning_rate": 4.3661813392566865e-06, + "logits/chosen": -0.19001172482967377, + "logits/rejected": -1.2967747449874878, + "logps/chosen": -0.9470418691635132, + "logps/rejected": -18.34066390991211, + "loss": 0.9472, + "num_input_tokens_seen": 13636208, + "odds_ratio_loss": 9.365311622619629, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09470418840646744, + "rewards/margins": 1.7393624782562256, + "rewards/rejected": -1.83406662940979, + "sft_loss": 0.010707495734095573, + "step": 10400 + }, + { + "epoch": 0.809801633605601, + "grad_norm": 4.036028861999512, + "learning_rate": 4.331746725667132e-06, + "logits/chosen": -0.4446299970149994, + "logits/rejected": -1.4362730979919434, + "logps/chosen": -0.8088845014572144, + "logps/rejected": -14.285024642944336, + "loss": 0.8132, + "num_input_tokens_seen": 13648832, + "odds_ratio_loss": 7.9063849449157715, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.08088846504688263, + "rewards/margins": 1.347614049911499, + "rewards/rejected": -1.4285024404525757, + "sft_loss": 0.022565852850675583, + "step": 10410 + }, + { + "epoch": 0.8105795410346169, + "grad_norm": 11.664623260498047, + "learning_rate": 4.297435552995391e-06, + "logits/chosen": -0.5774062871932983, + "logits/rejected": -1.1489648818969727, + "logps/chosen": -0.9516938924789429, + "logps/rejected": -19.207168579101562, + "loss": 0.9724, + "num_input_tokens_seen": 13659936, + "odds_ratio_loss": 9.546696662902832, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.09516939520835876, + "rewards/margins": 1.825547456741333, + "rewards/rejected": -1.9207168817520142, + "sft_loss": 0.017754215747117996, + "step": 10420 + }, + { + "epoch": 0.8113574484636328, + "grad_norm": 3.810201406478882, + "learning_rate": 4.2632480261645705e-06, + "logits/chosen": -0.31553104519844055, + "logits/rejected": -1.482110857963562, + "logps/chosen": -0.833909809589386, + "logps/rejected": -22.53671646118164, + "loss": 0.8341, + "num_input_tokens_seen": 13672304, + "odds_ratio_loss": 8.198694229125977, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.083390973508358, + "rewards/margins": 2.1702804565429688, + "rewards/rejected": -2.253671407699585, + "sft_loss": 0.01419934444129467, + "step": 10430 + }, + { + "epoch": 0.8121353558926487, + "grad_norm": 2.328805685043335, + "learning_rate": 4.229184349359294e-06, + "logits/chosen": -0.43258970975875854, + "logits/rejected": -1.606311559677124, + "logps/chosen": -0.7520237565040588, + "logps/rejected": -16.893381118774414, + "loss": 0.756, + "num_input_tokens_seen": 13686560, + "odds_ratio_loss": 7.460411071777344, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.07520237565040588, + "rewards/margins": 1.6141357421875, + "rewards/rejected": -1.6893380880355835, + "sft_loss": 0.009956094436347485, + "step": 10440 + }, + { + "epoch": 0.8129132633216647, + "grad_norm": 5.632044315338135, + "learning_rate": 4.1952447260244995e-06, + "logits/chosen": -0.2724373936653137, + "logits/rejected": -1.5779287815093994, + "logps/chosen": -0.842615008354187, + "logps/rejected": -16.831958770751953, + "loss": 0.8569, + "num_input_tokens_seen": 13697088, + "odds_ratio_loss": 8.503708839416504, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.08426151424646378, + "rewards/margins": 1.5989340543746948, + "rewards/rejected": -1.68319571018219, + "sft_loss": 0.006496152840554714, + "step": 10450 + }, + { + "epoch": 0.8136911707506806, + "grad_norm": 2.986135482788086, + "learning_rate": 4.1614293588642324e-06, + "logits/chosen": -0.21738651394844055, + "logits/rejected": -1.6314799785614014, + "logps/chosen": -0.982624888420105, + "logps/rejected": -10.346254348754883, + "loss": 1.0095, + "num_input_tokens_seen": 13707952, + "odds_ratio_loss": 10.010721206665039, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.0982624962925911, + "rewards/margins": 0.9363630414009094, + "rewards/rejected": -1.0346254110336304, + "sft_loss": 0.008467676118016243, + "step": 10460 + }, + { + "epoch": 0.8144690781796966, + "grad_norm": 3.291046142578125, + "learning_rate": 4.127738449840396e-06, + "logits/chosen": -0.13250042498111725, + "logits/rejected": -1.6850128173828125, + "logps/chosen": -0.9671382904052734, + "logps/rejected": -14.425585746765137, + "loss": 0.9935, + "num_input_tokens_seen": 13720752, + "odds_ratio_loss": 9.776313781738281, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.0967138260602951, + "rewards/margins": 1.3458447456359863, + "rewards/rejected": -1.4425586462020874, + "sft_loss": 0.015822656452655792, + "step": 10470 + }, + { + "epoch": 0.8152469856087126, + "grad_norm": 4.166664123535156, + "learning_rate": 4.094172200171578e-06, + "logits/chosen": -0.3846088945865631, + "logits/rejected": -1.1095737218856812, + "logps/chosen": -0.7261198163032532, + "logps/rejected": -12.331188201904297, + "loss": 0.7456, + "num_input_tokens_seen": 13732592, + "odds_ratio_loss": 7.3647050857543945, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.0726119726896286, + "rewards/margins": 1.1605069637298584, + "rewards/rejected": -1.2331188917160034, + "sft_loss": 0.009162858128547668, + "step": 10480 + }, + { + "epoch": 0.8160248930377285, + "grad_norm": 1.8134665489196777, + "learning_rate": 4.060730810331839e-06, + "logits/chosen": -0.12342437356710434, + "logits/rejected": -1.7950830459594727, + "logps/chosen": -0.8293974995613098, + "logps/rejected": -15.393221855163574, + "loss": 0.8456, + "num_input_tokens_seen": 13745536, + "odds_ratio_loss": 8.391043663024902, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.0829397514462471, + "rewards/margins": 1.4563825130462646, + "rewards/rejected": -1.5393221378326416, + "sft_loss": 0.006471716798841953, + "step": 10490 + }, + { + "epoch": 0.8168028004667445, + "grad_norm": 15.088645935058594, + "learning_rate": 4.0274144800495216e-06, + "logits/chosen": -0.47999221086502075, + "logits/rejected": -1.3440254926681519, + "logps/chosen": -0.7757308483123779, + "logps/rejected": -13.22809886932373, + "loss": 0.7924, + "num_input_tokens_seen": 13759344, + "odds_ratio_loss": 7.608541965484619, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.07757309079170227, + "rewards/margins": 1.2452367544174194, + "rewards/rejected": -1.3228099346160889, + "sft_loss": 0.03156624361872673, + "step": 10500 + }, + { + "epoch": 0.8175807078957604, + "grad_norm": 7.884941101074219, + "learning_rate": 3.994223408306039e-06, + "logits/chosen": -0.41769304871559143, + "logits/rejected": -1.6686872243881226, + "logps/chosen": -0.8849675059318542, + "logps/rejected": -22.058025360107422, + "loss": 0.8891, + "num_input_tokens_seen": 13772224, + "odds_ratio_loss": 8.78945255279541, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.08849674463272095, + "rewards/margins": 2.1173057556152344, + "rewards/rejected": -2.2058024406433105, + "sft_loss": 0.010153439827263355, + "step": 10510 + }, + { + "epoch": 0.8183586153247764, + "grad_norm": 2.005485773086548, + "learning_rate": 3.961157793334708e-06, + "logits/chosen": -0.07831288129091263, + "logits/rejected": -1.9417102336883545, + "logps/chosen": -0.7648784518241882, + "logps/rejected": -22.269865036010742, + "loss": 0.7657, + "num_input_tokens_seen": 13785824, + "odds_ratio_loss": 7.601174831390381, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.07648786157369614, + "rewards/margins": 2.150498867034912, + "rewards/rejected": -2.2269864082336426, + "sft_loss": 0.005598908755928278, + "step": 10520 + }, + { + "epoch": 0.8191365227537923, + "grad_norm": 3.384782075881958, + "learning_rate": 3.928217832619555e-06, + "logits/chosen": -0.14389245212078094, + "logits/rejected": -1.483295202255249, + "logps/chosen": -0.7602868676185608, + "logps/rejected": -17.16800880432129, + "loss": 0.7805, + "num_input_tokens_seen": 13799584, + "odds_ratio_loss": 7.6541008949279785, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.07602868229150772, + "rewards/margins": 1.6407722234725952, + "rewards/rejected": -1.7168009281158447, + "sft_loss": 0.015076467767357826, + "step": 10530 + }, + { + "epoch": 0.8199144301828083, + "grad_norm": 3.2515947818756104, + "learning_rate": 3.89540372289414e-06, + "logits/chosen": -0.5360939502716064, + "logits/rejected": -1.637587547302246, + "logps/chosen": -0.8743669390678406, + "logps/rejected": -22.532119750976562, + "loss": 0.8786, + "num_input_tokens_seen": 13815536, + "odds_ratio_loss": 8.461068153381348, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.08743669837713242, + "rewards/margins": 2.1657752990722656, + "rewards/rejected": -2.2532119750976562, + "sft_loss": 0.03245777636766434, + "step": 10540 + }, + { + "epoch": 0.8206923376118241, + "grad_norm": 7.553914546966553, + "learning_rate": 3.862715660140376e-06, + "logits/chosen": -0.23243942856788635, + "logits/rejected": -1.265459418296814, + "logps/chosen": -0.8319419622421265, + "logps/rejected": -6.5457587242126465, + "loss": 0.8717, + "num_input_tokens_seen": 13831408, + "odds_ratio_loss": 8.624181747436523, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.08319420367479324, + "rewards/margins": 0.571381688117981, + "rewards/rejected": -0.6545758843421936, + "sft_loss": 0.00926089659333229, + "step": 10550 + }, + { + "epoch": 0.8214702450408401, + "grad_norm": 1.2170356512069702, + "learning_rate": 3.830153839587352e-06, + "logits/chosen": -0.12456146627664566, + "logits/rejected": -1.6984800100326538, + "logps/chosen": -0.8001117706298828, + "logps/rejected": -22.34385871887207, + "loss": 0.824, + "num_input_tokens_seen": 13842800, + "odds_ratio_loss": 8.16563606262207, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.08001118153333664, + "rewards/margins": 2.154374599456787, + "rewards/rejected": -2.2343852519989014, + "sft_loss": 0.007441601250320673, + "step": 10560 + }, + { + "epoch": 0.822248152469856, + "grad_norm": 10.002525329589844, + "learning_rate": 3.7977184557101975e-06, + "logits/chosen": -0.6857665777206421, + "logits/rejected": -1.205599069595337, + "logps/chosen": -0.9053228497505188, + "logps/rejected": -17.684093475341797, + "loss": 0.9086, + "num_input_tokens_seen": 13858096, + "odds_ratio_loss": 8.848712921142578, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09053228795528412, + "rewards/margins": 1.6778770685195923, + "rewards/rejected": -1.7684093713760376, + "sft_loss": 0.023730546236038208, + "step": 10570 + }, + { + "epoch": 0.823026059898872, + "grad_norm": 18.812427520751953, + "learning_rate": 3.765409702228881e-06, + "logits/chosen": -0.17264044284820557, + "logits/rejected": -1.8149480819702148, + "logps/chosen": -0.9068514108657837, + "logps/rejected": -22.119543075561523, + "loss": 0.9093, + "num_input_tokens_seen": 13872416, + "odds_ratio_loss": 8.850945472717285, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09068514406681061, + "rewards/margins": 2.1212692260742188, + "rewards/rejected": -2.211954355239868, + "sft_loss": 0.024195067584514618, + "step": 10580 + }, + { + "epoch": 0.823803967327888, + "grad_norm": 5.036831378936768, + "learning_rate": 3.7332277721070925e-06, + "logits/chosen": 0.01380936335772276, + "logits/rejected": -1.7724119424819946, + "logps/chosen": -0.8250659108161926, + "logps/rejected": -20.20041275024414, + "loss": 0.8318, + "num_input_tokens_seen": 13883168, + "odds_ratio_loss": 8.276803016662598, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.08250658214092255, + "rewards/margins": 1.9375345706939697, + "rewards/rejected": -2.0200412273406982, + "sft_loss": 0.0041191792115569115, + "step": 10590 + }, + { + "epoch": 0.8245818747569039, + "grad_norm": 2.521698474884033, + "learning_rate": 3.7011728575510525e-06, + "logits/chosen": -0.10515572875738144, + "logits/rejected": -1.746950387954712, + "logps/chosen": -0.7915820479393005, + "logps/rejected": -24.0881404876709, + "loss": 0.7929, + "num_input_tokens_seen": 13897536, + "odds_ratio_loss": 7.8814826011657715, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.07915820926427841, + "rewards/margins": 2.329655885696411, + "rewards/rejected": -2.408813953399658, + "sft_loss": 0.00473423209041357, + "step": 10600 + }, + { + "epoch": 0.8253597821859199, + "grad_norm": 1.3302189111709595, + "learning_rate": 3.6692451500083908e-06, + "logits/chosen": -0.2942681908607483, + "logits/rejected": -1.1378456354141235, + "logps/chosen": -0.9133954048156738, + "logps/rejected": -15.458047866821289, + "loss": 0.936, + "num_input_tokens_seen": 13918352, + "odds_ratio_loss": 9.19874095916748, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.09133954346179962, + "rewards/margins": 1.4544650316238403, + "rewards/rejected": -1.545804738998413, + "sft_loss": 0.016142334789037704, + "step": 10610 + }, + { + "epoch": 0.8261376896149358, + "grad_norm": 8.7985258102417, + "learning_rate": 3.6374448401669952e-06, + "logits/chosen": -0.34568607807159424, + "logits/rejected": -1.3344576358795166, + "logps/chosen": -0.8753089904785156, + "logps/rejected": -14.121788024902344, + "loss": 0.8878, + "num_input_tokens_seen": 13934352, + "odds_ratio_loss": 8.807962417602539, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.08753089606761932, + "rewards/margins": 1.324648141860962, + "rewards/rejected": -1.412178874015808, + "sft_loss": 0.006974914111196995, + "step": 10620 + }, + { + "epoch": 0.8269155970439518, + "grad_norm": 6.550417423248291, + "learning_rate": 3.6057721179538678e-06, + "logits/chosen": 0.013199204578995705, + "logits/rejected": -1.4703713655471802, + "logps/chosen": -0.8771390914916992, + "logps/rejected": -9.699310302734375, + "loss": 0.9111, + "num_input_tokens_seen": 13947840, + "odds_ratio_loss": 9.066411972045898, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.08771391212940216, + "rewards/margins": 0.8822172284126282, + "rewards/rejected": -0.9699310064315796, + "sft_loss": 0.004470839165151119, + "step": 10630 + }, + { + "epoch": 0.8276935044729677, + "grad_norm": 1.9927451610565186, + "learning_rate": 3.574227172533995e-06, + "logits/chosen": -0.4474928379058838, + "logits/rejected": -1.1933181285858154, + "logps/chosen": -3.157609701156616, + "logps/rejected": -17.399599075317383, + "loss": 3.1668, + "num_input_tokens_seen": 13960624, + "odds_ratio_loss": 31.481128692626953, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.3157609701156616, + "rewards/margins": 1.424198865890503, + "rewards/rejected": -1.739959955215454, + "sft_loss": 0.018684662878513336, + "step": 10640 + }, + { + "epoch": 0.8284714119019837, + "grad_norm": 2.234353542327881, + "learning_rate": 3.5428101923092262e-06, + "logits/chosen": -0.2922361493110657, + "logits/rejected": -1.6244525909423828, + "logps/chosen": -1.1844645738601685, + "logps/rejected": -16.810644149780273, + "loss": 1.2053, + "num_input_tokens_seen": 13977696, + "odds_ratio_loss": 12.007806777954102, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.11844644695520401, + "rewards/margins": 1.5626181364059448, + "rewards/rejected": -1.6810643672943115, + "sft_loss": 0.004485656972974539, + "step": 10650 + }, + { + "epoch": 0.8292493193309997, + "grad_norm": 1.6060020923614502, + "learning_rate": 3.5115213649171306e-06, + "logits/chosen": -0.4599802494049072, + "logits/rejected": -1.4395368099212646, + "logps/chosen": -0.9598096609115601, + "logps/rejected": -4.537492752075195, + "loss": 0.9772, + "num_input_tokens_seen": 13987552, + "odds_ratio_loss": 8.78867244720459, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.09598096460103989, + "rewards/margins": 0.3577682673931122, + "rewards/rejected": -0.45374923944473267, + "sft_loss": 0.09836249053478241, + "step": 10660 + }, + { + "epoch": 0.8300272267600156, + "grad_norm": 5.407586097717285, + "learning_rate": 3.4803608772298907e-06, + "logits/chosen": -0.6109030842781067, + "logits/rejected": -1.033789873123169, + "logps/chosen": -0.6999441981315613, + "logps/rejected": -14.462682723999023, + "loss": 0.7224, + "num_input_tokens_seen": 13994992, + "odds_ratio_loss": 7.111445426940918, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.06999442726373672, + "rewards/margins": 1.3762738704681396, + "rewards/rejected": -1.4462683200836182, + "sft_loss": 0.0112073989585042, + "step": 10670 + }, + { + "epoch": 0.8308051341890315, + "grad_norm": 14.045202255249023, + "learning_rate": 3.4493289153531836e-06, + "logits/chosen": -0.4810110926628113, + "logits/rejected": -1.4211429357528687, + "logps/chosen": -0.8086579442024231, + "logps/rejected": -12.509838104248047, + "loss": 0.8346, + "num_input_tokens_seen": 14009648, + "odds_ratio_loss": 8.250049591064453, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.08086579293012619, + "rewards/margins": 1.170117974281311, + "rewards/rejected": -1.250983715057373, + "sft_loss": 0.009561174549162388, + "step": 10680 + }, + { + "epoch": 0.8315830416180474, + "grad_norm": 13.156830787658691, + "learning_rate": 3.4184256646250524e-06, + "logits/chosen": -0.25289249420166016, + "logits/rejected": -1.280796766281128, + "logps/chosen": -0.8776252865791321, + "logps/rejected": -16.361913681030273, + "loss": 0.9122, + "num_input_tokens_seen": 14022320, + "odds_ratio_loss": 8.958372116088867, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.08776251971721649, + "rewards/margins": 1.5484288930892944, + "rewards/rejected": -1.6361916065216064, + "sft_loss": 0.016396891325712204, + "step": 10690 + }, + { + "epoch": 0.8323609490470634, + "grad_norm": 4.0661540031433105, + "learning_rate": 3.3876513096148338e-06, + "logits/chosen": -0.14225341379642487, + "logits/rejected": -1.3277254104614258, + "logps/chosen": -0.9199932217597961, + "logps/rejected": -11.546847343444824, + "loss": 0.9276, + "num_input_tokens_seen": 14031024, + "odds_ratio_loss": 9.208245277404785, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09199932217597961, + "rewards/margins": 1.062685489654541, + "rewards/rejected": -1.1546847820281982, + "sft_loss": 0.006757513619959354, + "step": 10700 + }, + { + "epoch": 0.8331388564760793, + "grad_norm": 1.2136611938476562, + "learning_rate": 3.3570060341220234e-06, + "logits/chosen": -0.15510103106498718, + "logits/rejected": -1.907846212387085, + "logps/chosen": -0.9624649882316589, + "logps/rejected": -20.84872055053711, + "loss": 0.967, + "num_input_tokens_seen": 14046720, + "odds_ratio_loss": 9.59048080444336, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09624650329351425, + "rewards/margins": 1.9886255264282227, + "rewards/rejected": -2.084871768951416, + "sft_loss": 0.007957112044095993, + "step": 10710 + }, + { + "epoch": 0.8339167639050953, + "grad_norm": 6.186117172241211, + "learning_rate": 3.326490021175205e-06, + "logits/chosen": -0.11530347168445587, + "logits/rejected": -1.3156671524047852, + "logps/chosen": -0.6982190608978271, + "logps/rejected": -6.572912693023682, + "loss": 0.7118, + "num_input_tokens_seen": 14060992, + "odds_ratio_loss": 7.040630340576172, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.06982190907001495, + "rewards/margins": 0.5874693393707275, + "rewards/rejected": -0.6572912931442261, + "sft_loss": 0.007760833017528057, + "step": 10720 + }, + { + "epoch": 0.8346946713341112, + "grad_norm": 9.213081359863281, + "learning_rate": 3.296103453030927e-06, + "logits/chosen": -0.46214666962623596, + "logits/rejected": -1.044995665550232, + "logps/chosen": -0.7721800804138184, + "logps/rejected": -7.03396463394165, + "loss": 0.7835, + "num_input_tokens_seen": 14072560, + "odds_ratio_loss": 7.719517707824707, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.07721801102161407, + "rewards/margins": 0.626178503036499, + "rewards/rejected": -0.7033964395523071, + "sft_loss": 0.011503453366458416, + "step": 10730 + }, + { + "epoch": 0.8354725787631272, + "grad_norm": 3.0902130603790283, + "learning_rate": 3.2658465111726405e-06, + "logits/chosen": -0.3015756905078888, + "logits/rejected": -0.9444737434387207, + "logps/chosen": -1.0036855936050415, + "logps/rejected": -8.566162109375, + "loss": 1.0248, + "num_input_tokens_seen": 14085216, + "odds_ratio_loss": 10.090906143188477, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.10036854445934296, + "rewards/margins": 0.7562476396560669, + "rewards/rejected": -0.856616199016571, + "sft_loss": 0.015708167105913162, + "step": 10740 + }, + { + "epoch": 0.8362504861921431, + "grad_norm": 1.6802676916122437, + "learning_rate": 3.2357193763095984e-06, + "logits/chosen": -0.3023141026496887, + "logits/rejected": -1.2213256359100342, + "logps/chosen": -0.8354895710945129, + "logps/rejected": -8.458917617797852, + "loss": 0.8609, + "num_input_tokens_seen": 14102064, + "odds_ratio_loss": 8.399511337280273, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.08354896306991577, + "rewards/margins": 0.7623429298400879, + "rewards/rejected": -0.8458918333053589, + "sft_loss": 0.020914433524012566, + "step": 10750 + }, + { + "epoch": 0.8370283936211591, + "grad_norm": 2.0074286460876465, + "learning_rate": 3.2057222283757877e-06, + "logits/chosen": -0.1700136959552765, + "logits/rejected": -1.258719801902771, + "logps/chosen": -0.9247137904167175, + "logps/rejected": -15.314018249511719, + "loss": 0.9311, + "num_input_tokens_seen": 14117840, + "odds_ratio_loss": 8.977886199951172, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09247139096260071, + "rewards/margins": 1.438930630683899, + "rewards/rejected": -1.5314018726348877, + "sft_loss": 0.03332524746656418, + "step": 10760 + }, + { + "epoch": 0.8378063010501751, + "grad_norm": 22.7186336517334, + "learning_rate": 3.175855246528847e-06, + "logits/chosen": -0.2710705101490021, + "logits/rejected": -1.2464778423309326, + "logps/chosen": -1.6098449230194092, + "logps/rejected": -8.80943775177002, + "loss": 1.7031, + "num_input_tokens_seen": 14128832, + "odds_ratio_loss": 16.840373992919922, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.16098450124263763, + "rewards/margins": 0.7199593782424927, + "rewards/rejected": -0.880943775177002, + "sft_loss": 0.01902218721807003, + "step": 10770 + }, + { + "epoch": 0.838584208479191, + "grad_norm": 27.165802001953125, + "learning_rate": 3.1461186091490015e-06, + "logits/chosen": -0.4816823899745941, + "logits/rejected": -1.344750165939331, + "logps/chosen": -0.9202563166618347, + "logps/rejected": -9.979837417602539, + "loss": 0.9301, + "num_input_tokens_seen": 14143392, + "odds_ratio_loss": 9.173930168151855, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.09202565252780914, + "rewards/margins": 0.9059581756591797, + "rewards/rejected": -0.9979838132858276, + "sft_loss": 0.012739296071231365, + "step": 10780 + }, + { + "epoch": 0.839362115908207, + "grad_norm": 3.174644947052002, + "learning_rate": 3.1165124938379947e-06, + "logits/chosen": -0.2760254740715027, + "logits/rejected": -1.6259291172027588, + "logps/chosen": -0.772201418876648, + "logps/rejected": -14.874751091003418, + "loss": 0.7818, + "num_input_tokens_seen": 14153776, + "odds_ratio_loss": 7.721850395202637, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.07722015678882599, + "rewards/margins": 1.4102550745010376, + "rewards/rejected": -1.4874751567840576, + "sft_loss": 0.009580541402101517, + "step": 10790 + }, + { + "epoch": 0.8401400233372228, + "grad_norm": 1.4849799871444702, + "learning_rate": 3.0870370774180315e-06, + "logits/chosen": -0.20634207129478455, + "logits/rejected": -1.5450636148452759, + "logps/chosen": -0.9883825182914734, + "logps/rejected": -11.121337890625, + "loss": 1.0033, + "num_input_tokens_seen": 14169824, + "odds_ratio_loss": 9.974977493286133, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09883825480937958, + "rewards/margins": 1.0132956504821777, + "rewards/rejected": -1.1121338605880737, + "sft_loss": 0.005842863116413355, + "step": 10800 + }, + { + "epoch": 0.8409179307662388, + "grad_norm": 3.3578526973724365, + "learning_rate": 3.0576925359307063e-06, + "logits/chosen": -0.178543820977211, + "logits/rejected": -1.4022276401519775, + "logps/chosen": -0.7552523612976074, + "logps/rejected": -9.426691055297852, + "loss": 0.7738, + "num_input_tokens_seen": 14181936, + "odds_ratio_loss": 7.660872459411621, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.07552523910999298, + "rewards/margins": 0.8671439290046692, + "rewards/rejected": -0.9426690936088562, + "sft_loss": 0.007728863507509232, + "step": 10810 + }, + { + "epoch": 0.8416958381952547, + "grad_norm": 1.7538191080093384, + "learning_rate": 3.0284790446359817e-06, + "logits/chosen": -0.6491594314575195, + "logits/rejected": -1.37003755569458, + "logps/chosen": -1.3433282375335693, + "logps/rejected": -15.055395126342773, + "loss": 1.3577, + "num_input_tokens_seen": 14194352, + "odds_ratio_loss": 12.862637519836426, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1343328058719635, + "rewards/margins": 1.3712069988250732, + "rewards/rejected": -1.5055397748947144, + "sft_loss": 0.0714530199766159, + "step": 10820 + }, + { + "epoch": 0.8424737456242707, + "grad_norm": 1.6737394332885742, + "learning_rate": 2.9993967780111126e-06, + "logits/chosen": -0.3436136543750763, + "logits/rejected": -1.3898818492889404, + "logps/chosen": -0.7258359789848328, + "logps/rejected": -13.831029891967773, + "loss": 0.7328, + "num_input_tokens_seen": 14213520, + "odds_ratio_loss": 7.284417152404785, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.07258360087871552, + "rewards/margins": 1.3105193376541138, + "rewards/rejected": -1.3831030130386353, + "sft_loss": 0.004367014858871698, + "step": 10830 + }, + { + "epoch": 0.8432516530532866, + "grad_norm": 4.193950176239014, + "learning_rate": 2.970445909749614e-06, + "logits/chosen": -0.2003314197063446, + "logits/rejected": -1.7939174175262451, + "logps/chosen": -0.8816348314285278, + "logps/rejected": -16.744592666625977, + "loss": 0.9138, + "num_input_tokens_seen": 14228208, + "odds_ratio_loss": 9.093521118164062, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.08816348016262054, + "rewards/margins": 1.5862958431243896, + "rewards/rejected": -1.6744590997695923, + "sft_loss": 0.004440158139914274, + "step": 10840 + }, + { + "epoch": 0.8440295604823026, + "grad_norm": 1.9024956226348877, + "learning_rate": 2.9416266127602484e-06, + "logits/chosen": -0.10373874008655548, + "logits/rejected": -1.681274652481079, + "logps/chosen": -0.7674292325973511, + "logps/rejected": -14.403085708618164, + "loss": 0.7905, + "num_input_tokens_seen": 14243744, + "odds_ratio_loss": 7.8598175048828125, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.07674293220043182, + "rewards/margins": 1.3635656833648682, + "rewards/rejected": -1.4403088092803955, + "sft_loss": 0.004566981457173824, + "step": 10850 + }, + { + "epoch": 0.8448074679113186, + "grad_norm": 3.021470308303833, + "learning_rate": 2.9129390591659422e-06, + "logits/chosen": -0.182247593998909, + "logits/rejected": -1.098589539527893, + "logps/chosen": -0.9050281643867493, + "logps/rejected": -12.335296630859375, + "loss": 0.9084, + "num_input_tokens_seen": 14253184, + "odds_ratio_loss": 8.209359169006348, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09050282090902328, + "rewards/margins": 1.1430268287658691, + "rewards/rejected": -1.2335296869277954, + "sft_loss": 0.08741842210292816, + "step": 10860 + }, + { + "epoch": 0.8455853753403345, + "grad_norm": 1.78042733669281, + "learning_rate": 2.8843834203027997e-06, + "logits/chosen": -0.12538185715675354, + "logits/rejected": -2.3496804237365723, + "logps/chosen": -0.771729588508606, + "logps/rejected": -11.53658390045166, + "loss": 0.786, + "num_input_tokens_seen": 14267296, + "odds_ratio_loss": 7.821356773376465, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.07717296481132507, + "rewards/margins": 1.0764853954315186, + "rewards/rejected": -1.153658390045166, + "sft_loss": 0.003909476101398468, + "step": 10870 + }, + { + "epoch": 0.8463632827693505, + "grad_norm": 3.1438002586364746, + "learning_rate": 2.855959866719063e-06, + "logits/chosen": -0.27122312784194946, + "logits/rejected": -1.5149050951004028, + "logps/chosen": -0.8944989442825317, + "logps/rejected": -16.33709716796875, + "loss": 0.8987, + "num_input_tokens_seen": 14276864, + "odds_ratio_loss": 8.923892974853516, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.08944989740848541, + "rewards/margins": 1.544259786605835, + "rewards/rejected": -1.6337096691131592, + "sft_loss": 0.006331826094537973, + "step": 10880 + }, + { + "epoch": 0.8471411901983664, + "grad_norm": 3.468661069869995, + "learning_rate": 2.8276685681740966e-06, + "logits/chosen": -0.2570245862007141, + "logits/rejected": -1.0485957860946655, + "logps/chosen": -0.8954674005508423, + "logps/rejected": -10.502070426940918, + "loss": 0.9112, + "num_input_tokens_seen": 14289088, + "odds_ratio_loss": 8.95766544342041, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.08954674750566483, + "rewards/margins": 0.9606602787971497, + "rewards/rejected": -1.0502071380615234, + "sft_loss": 0.015441042371094227, + "step": 10890 + }, + { + "epoch": 0.8479190976273824, + "grad_norm": 2.709669828414917, + "learning_rate": 2.7995096936373728e-06, + "logits/chosen": -0.47932925820350647, + "logits/rejected": -1.279759168624878, + "logps/chosen": -0.8552338480949402, + "logps/rejected": -11.21165943145752, + "loss": 0.8632, + "num_input_tokens_seen": 14303184, + "odds_ratio_loss": 8.557912826538086, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.08552340418100357, + "rewards/margins": 1.035642385482788, + "rewards/rejected": -1.1211658716201782, + "sft_loss": 0.007410106714814901, + "step": 10900 + }, + { + "epoch": 0.8486970050563983, + "grad_norm": 2.0692508220672607, + "learning_rate": 2.771483411287462e-06, + "logits/chosen": -0.22692546248435974, + "logits/rejected": -1.7416139841079712, + "logps/chosen": -1.1929271221160889, + "logps/rejected": -11.187047958374023, + "loss": 1.1989, + "num_input_tokens_seen": 14314880, + "odds_ratio_loss": 11.927255630493164, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1192927211523056, + "rewards/margins": 0.9994121789932251, + "rewards/rejected": -1.1187047958374023, + "sft_loss": 0.006167359184473753, + "step": 10910 + }, + { + "epoch": 0.8494749124854143, + "grad_norm": 4.281651020050049, + "learning_rate": 2.7435898885110332e-06, + "logits/chosen": -0.37550920248031616, + "logits/rejected": -1.643833875656128, + "logps/chosen": -0.7633379101753235, + "logps/rejected": -4.50496244430542, + "loss": 0.7916, + "num_input_tokens_seen": 14325008, + "odds_ratio_loss": 7.83477783203125, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.07633379846811295, + "rewards/margins": 0.374162495136261, + "rewards/rejected": -0.45049628615379333, + "sft_loss": 0.008120447397232056, + "step": 10920 + }, + { + "epoch": 0.8502528199144301, + "grad_norm": 3.8543097972869873, + "learning_rate": 2.715829291901853e-06, + "logits/chosen": -0.18652768433094025, + "logits/rejected": -1.3088423013687134, + "logps/chosen": -1.8462274074554443, + "logps/rejected": -13.132868766784668, + "loss": 1.8577, + "num_input_tokens_seen": 14345152, + "odds_ratio_loss": 18.501949310302734, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.18462274968624115, + "rewards/margins": 1.1286640167236328, + "rewards/rejected": -1.3132869005203247, + "sft_loss": 0.007510988973081112, + "step": 10930 + }, + { + "epoch": 0.8510307273434461, + "grad_norm": 3.9623801708221436, + "learning_rate": 2.6882017872597725e-06, + "logits/chosen": -0.2059568464756012, + "logits/rejected": -1.487349510192871, + "logps/chosen": -0.9327132105827332, + "logps/rejected": -15.16093921661377, + "loss": 0.952, + "num_input_tokens_seen": 14355328, + "odds_ratio_loss": 9.465570449829102, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.09327132254838943, + "rewards/margins": 1.4228225946426392, + "rewards/rejected": -1.5160939693450928, + "sft_loss": 0.005433371756225824, + "step": 10940 + }, + { + "epoch": 0.851808634772462, + "grad_norm": 4.204638481140137, + "learning_rate": 2.660707539589766e-06, + "logits/chosen": -0.46590957045555115, + "logits/rejected": -1.0994277000427246, + "logps/chosen": -0.847273051738739, + "logps/rejected": -9.813700675964355, + "loss": 0.8647, + "num_input_tokens_seen": 14369184, + "odds_ratio_loss": 8.497247695922852, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.08472730964422226, + "rewards/margins": 0.8966428637504578, + "rewards/rejected": -0.9813701510429382, + "sft_loss": 0.014988129027187824, + "step": 10950 + }, + { + "epoch": 0.852586542201478, + "grad_norm": 2.0978615283966064, + "learning_rate": 2.633346713100929e-06, + "logits/chosen": -0.3497718870639801, + "logits/rejected": -1.447076678276062, + "logps/chosen": -1.2323142290115356, + "logps/rejected": -22.83798599243164, + "loss": 1.2342, + "num_input_tokens_seen": 14382736, + "odds_ratio_loss": 11.533686637878418, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1232314333319664, + "rewards/margins": 2.160567283630371, + "rewards/rejected": -2.2837986946105957, + "sft_loss": 0.08079709857702255, + "step": 10960 + }, + { + "epoch": 0.853364449630494, + "grad_norm": 2.069700241088867, + "learning_rate": 2.6061194712054927e-06, + "logits/chosen": -0.10964655876159668, + "logits/rejected": -1.6845324039459229, + "logps/chosen": -0.8405076861381531, + "logps/rejected": -19.095523834228516, + "loss": 0.8513, + "num_input_tokens_seen": 14395296, + "odds_ratio_loss": 8.465188026428223, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.08405075967311859, + "rewards/margins": 1.825501799583435, + "rewards/rejected": -1.9095525741577148, + "sft_loss": 0.004737283103168011, + "step": 10970 + }, + { + "epoch": 0.8541423570595099, + "grad_norm": 34.36608123779297, + "learning_rate": 2.5790259765178764e-06, + "logits/chosen": -0.23506894707679749, + "logits/rejected": -1.7545793056488037, + "logps/chosen": -0.8853933215141296, + "logps/rejected": -20.537443161010742, + "loss": 0.8863, + "num_input_tokens_seen": 14409472, + "odds_ratio_loss": 8.764898300170898, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.08853934705257416, + "rewards/margins": 1.9652049541473389, + "rewards/rejected": -2.053744316101074, + "sft_loss": 0.009778345935046673, + "step": 10980 + }, + { + "epoch": 0.8549202644885259, + "grad_norm": 6.184960842132568, + "learning_rate": 2.55206639085368e-06, + "logits/chosen": -0.20178565382957458, + "logits/rejected": -1.3336738348007202, + "logps/chosen": -1.004481554031372, + "logps/rejected": -21.75019073486328, + "loss": 1.0076, + "num_input_tokens_seen": 14421104, + "odds_ratio_loss": 9.94190788269043, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.10044815391302109, + "rewards/margins": 2.074570894241333, + "rewards/rejected": -2.175018787384033, + "sft_loss": 0.013437485322356224, + "step": 10990 + }, + { + "epoch": 0.8556981719175418, + "grad_norm": 174.71400451660156, + "learning_rate": 2.5252408752287295e-06, + "logits/chosen": -0.24337315559387207, + "logits/rejected": -1.8969571590423584, + "logps/chosen": -1.0682896375656128, + "logps/rejected": -11.880017280578613, + "loss": 1.072, + "num_input_tokens_seen": 14435216, + "odds_ratio_loss": 10.644633293151855, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1068289503455162, + "rewards/margins": 1.0811727046966553, + "rewards/rejected": -1.1880018711090088, + "sft_loss": 0.007560241036117077, + "step": 11000 + }, + { + "epoch": 0.8564760793465578, + "grad_norm": 2.8752541542053223, + "learning_rate": 2.498549589858126e-06, + "logits/chosen": -0.2807459533214569, + "logits/rejected": -1.4375717639923096, + "logps/chosen": -1.1921361684799194, + "logps/rejected": -10.120828628540039, + "loss": 1.2045, + "num_input_tokens_seen": 14445648, + "odds_ratio_loss": 11.83732795715332, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.11921361833810806, + "rewards/margins": 0.8928692936897278, + "rewards/rejected": -1.012082815170288, + "sft_loss": 0.02073931321501732, + "step": 11010 + }, + { + "epoch": 0.8572539867755737, + "grad_norm": 6.071030139923096, + "learning_rate": 2.471992694155284e-06, + "logits/chosen": -0.06983019411563873, + "logits/rejected": -1.918835997581482, + "logps/chosen": -0.9633820652961731, + "logps/rejected": -21.31793785095215, + "loss": 0.9651, + "num_input_tokens_seen": 14461808, + "odds_ratio_loss": 9.575490951538086, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.0963381975889206, + "rewards/margins": 2.0354557037353516, + "rewards/rejected": -2.131793737411499, + "sft_loss": 0.0075905052945017815, + "step": 11020 + }, + { + "epoch": 0.8580318942045897, + "grad_norm": 9.991567611694336, + "learning_rate": 2.4455703467309743e-06, + "logits/chosen": -0.17449812591075897, + "logits/rejected": -1.6064729690551758, + "logps/chosen": -0.9717493057250977, + "logps/rejected": -8.299715042114258, + "loss": 1.0341, + "num_input_tokens_seen": 14475792, + "odds_ratio_loss": 9.821267127990723, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.09717492759227753, + "rewards/margins": 0.7327965497970581, + "rewards/rejected": -0.8299714922904968, + "sft_loss": 0.052019111812114716, + "step": 11030 + }, + { + "epoch": 0.8588098016336057, + "grad_norm": 19.450063705444336, + "learning_rate": 2.4192827053923754e-06, + "logits/chosen": -0.2539360225200653, + "logits/rejected": -1.26289963722229, + "logps/chosen": -1.0389549732208252, + "logps/rejected": -15.10387897491455, + "loss": 1.0406, + "num_input_tokens_seen": 14493312, + "odds_ratio_loss": 10.181611061096191, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.10389550775289536, + "rewards/margins": 1.4064924716949463, + "rewards/rejected": -1.5103880167007446, + "sft_loss": 0.022460829466581345, + "step": 11040 + }, + { + "epoch": 0.8595877090626215, + "grad_norm": 10.944372177124023, + "learning_rate": 2.3931299271421485e-06, + "logits/chosen": -0.042080096900463104, + "logits/rejected": -1.5874948501586914, + "logps/chosen": -0.7615327835083008, + "logps/rejected": -7.649228096008301, + "loss": 0.7799, + "num_input_tokens_seen": 14506896, + "odds_ratio_loss": 7.745556831359863, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.07615328580141068, + "rewards/margins": 0.6887694597244263, + "rewards/rejected": -0.7649228572845459, + "sft_loss": 0.005318674724549055, + "step": 11050 + }, + { + "epoch": 0.8603656164916375, + "grad_norm": 3.6714460849761963, + "learning_rate": 2.3671121681774655e-06, + "logits/chosen": -0.15332692861557007, + "logits/rejected": -1.4491770267486572, + "logps/chosen": -0.9941250681877136, + "logps/rejected": -13.707873344421387, + "loss": 1.0027, + "num_input_tokens_seen": 14519024, + "odds_ratio_loss": 9.954381942749023, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09941250830888748, + "rewards/margins": 1.2713749408721924, + "rewards/rejected": -1.3707873821258545, + "sft_loss": 0.007261206395924091, + "step": 11060 + }, + { + "epoch": 0.8611435239206534, + "grad_norm": 1.2879174947738647, + "learning_rate": 2.3412295838891123e-06, + "logits/chosen": -0.004231536295264959, + "logits/rejected": -1.5393919944763184, + "logps/chosen": -0.8462308049201965, + "logps/rejected": -18.241506576538086, + "loss": 0.8474, + "num_input_tokens_seen": 14537424, + "odds_ratio_loss": 8.435163497924805, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.08462308347225189, + "rewards/margins": 1.7395274639129639, + "rewards/rejected": -1.8241506814956665, + "sft_loss": 0.003909712191671133, + "step": 11070 + }, + { + "epoch": 0.8619214313496694, + "grad_norm": 8.174221992492676, + "learning_rate": 2.315482328860544e-06, + "logits/chosen": -0.4593052268028259, + "logits/rejected": -1.0363997220993042, + "logps/chosen": -1.0086044073104858, + "logps/rejected": -9.086363792419434, + "loss": 1.0211, + "num_input_tokens_seen": 14548432, + "odds_ratio_loss": 10.112740516662598, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.10086043924093246, + "rewards/margins": 0.8077758550643921, + "rewards/rejected": -0.9086363911628723, + "sft_loss": 0.009857363067567348, + "step": 11080 + }, + { + "epoch": 0.8626993387786853, + "grad_norm": 4.442285060882568, + "learning_rate": 2.2898705568669605e-06, + "logits/chosen": -0.3338615298271179, + "logits/rejected": -1.3318408727645874, + "logps/chosen": -0.9805607795715332, + "logps/rejected": -15.963170051574707, + "loss": 0.9877, + "num_input_tokens_seen": 14558768, + "odds_ratio_loss": 9.789030075073242, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09805609285831451, + "rewards/margins": 1.4982608556747437, + "rewards/rejected": -1.5963170528411865, + "sft_loss": 0.008807901293039322, + "step": 11090 + }, + { + "epoch": 0.8634772462077013, + "grad_norm": 2.3018581867218018, + "learning_rate": 2.2643944208743912e-06, + "logits/chosen": -0.09829515963792801, + "logits/rejected": -1.4474740028381348, + "logps/chosen": -0.8492511510848999, + "logps/rejected": -22.195430755615234, + "loss": 0.8501, + "num_input_tokens_seen": 14576400, + "odds_ratio_loss": 8.346790313720703, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.08492513000965118, + "rewards/margins": 2.134618043899536, + "rewards/rejected": -2.219543218612671, + "sft_loss": 0.015408432111144066, + "step": 11100 + }, + { + "epoch": 0.8642551536367172, + "grad_norm": 1.9978951215744019, + "learning_rate": 2.2390540730387768e-06, + "logits/chosen": -0.23037929832935333, + "logits/rejected": -1.7510086297988892, + "logps/chosen": -0.8015023469924927, + "logps/rejected": -8.597331047058105, + "loss": 0.8109, + "num_input_tokens_seen": 14587648, + "odds_ratio_loss": 7.873522758483887, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.08015023171901703, + "rewards/margins": 0.7795829772949219, + "rewards/rejected": -0.8597332239151001, + "sft_loss": 0.023554079234600067, + "step": 11110 + }, + { + "epoch": 0.8650330610657332, + "grad_norm": 1.5626524686813354, + "learning_rate": 2.2138496647050783e-06, + "logits/chosen": -0.2506280243396759, + "logits/rejected": -1.3620119094848633, + "logps/chosen": -0.7624492049217224, + "logps/rejected": -13.497182846069336, + "loss": 0.7796, + "num_input_tokens_seen": 14596320, + "odds_ratio_loss": 7.666055202484131, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.07624492794275284, + "rewards/margins": 1.2734732627868652, + "rewards/rejected": -1.3497183322906494, + "sft_loss": 0.013004831969738007, + "step": 11120 + }, + { + "epoch": 0.8658109684947491, + "grad_norm": 1.4940675497055054, + "learning_rate": 2.1887813464063433e-06, + "logits/chosen": -0.17947162687778473, + "logits/rejected": -1.5230398178100586, + "logps/chosen": -0.8179603815078735, + "logps/rejected": -15.498629570007324, + "loss": 0.8225, + "num_input_tokens_seen": 14609632, + "odds_ratio_loss": 8.163220405578613, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.08179602771997452, + "rewards/margins": 1.4680670499801636, + "rewards/rejected": -1.5498631000518799, + "sft_loss": 0.0061700670048594475, + "step": 11130 + }, + { + "epoch": 0.8665888759237651, + "grad_norm": 2.0545926094055176, + "learning_rate": 2.163849267862833e-06, + "logits/chosen": -0.2326630800962448, + "logits/rejected": -1.1611685752868652, + "logps/chosen": -0.8332999348640442, + "logps/rejected": -12.10827922821045, + "loss": 0.8531, + "num_input_tokens_seen": 14622960, + "odds_ratio_loss": 8.454458236694336, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.08332999050617218, + "rewards/margins": 1.127497911453247, + "rewards/rejected": -1.2108280658721924, + "sft_loss": 0.007665277924388647, + "step": 11140 + }, + { + "epoch": 0.8673667833527811, + "grad_norm": 3.2545974254608154, + "learning_rate": 2.139053577981115e-06, + "logits/chosen": -0.12574820220470428, + "logits/rejected": -1.5405362844467163, + "logps/chosen": -0.94415682554245, + "logps/rejected": -11.875570297241211, + "loss": 0.9558, + "num_input_tokens_seen": 14635344, + "odds_ratio_loss": 9.519662857055664, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09441567957401276, + "rewards/margins": 1.0931413173675537, + "rewards/rejected": -1.1875572204589844, + "sft_loss": 0.003794616786763072, + "step": 11150 + }, + { + "epoch": 0.868144690781797, + "grad_norm": 3.8555874824523926, + "learning_rate": 2.114394424853178e-06, + "logits/chosen": -0.030388841405510902, + "logits/rejected": -1.6810109615325928, + "logps/chosen": -0.8759109377861023, + "logps/rejected": -16.781967163085938, + "loss": 0.8762, + "num_input_tokens_seen": 14648704, + "odds_ratio_loss": 8.72721004486084, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.08759109675884247, + "rewards/margins": 1.5906057357788086, + "rewards/rejected": -1.678196668624878, + "sft_loss": 0.003505890490487218, + "step": 11160 + }, + { + "epoch": 0.868922598210813, + "grad_norm": 2.347398281097412, + "learning_rate": 2.089871955755543e-06, + "logits/chosen": -0.04993351176381111, + "logits/rejected": -1.4341214895248413, + "logps/chosen": -0.8593047261238098, + "logps/rejected": -7.1145734786987305, + "loss": 0.8976, + "num_input_tokens_seen": 14659184, + "odds_ratio_loss": 8.902770042419434, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.0859304741024971, + "rewards/margins": 0.6255267858505249, + "rewards/rejected": -0.7114574313163757, + "sft_loss": 0.007294154725968838, + "step": 11170 + }, + { + "epoch": 0.8697005056398288, + "grad_norm": 2.703765869140625, + "learning_rate": 2.065486317148396e-06, + "logits/chosen": -0.11260922253131866, + "logits/rejected": -1.4905784130096436, + "logps/chosen": -0.7861498594284058, + "logps/rejected": -7.284165382385254, + "loss": 0.8291, + "num_input_tokens_seen": 14671552, + "odds_ratio_loss": 8.243640899658203, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.0786149874329567, + "rewards/margins": 0.6498016119003296, + "rewards/rejected": -0.7284165620803833, + "sft_loss": 0.004718432668596506, + "step": 11180 + }, + { + "epoch": 0.8704784130688448, + "grad_norm": 1.5052175521850586, + "learning_rate": 2.041237654674691e-06, + "logits/chosen": -0.058899033814668655, + "logits/rejected": -1.6328794956207275, + "logps/chosen": -0.9910447001457214, + "logps/rejected": -19.82402801513672, + "loss": 0.9947, + "num_input_tokens_seen": 14686624, + "odds_ratio_loss": 9.896458625793457, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09910447150468826, + "rewards/margins": 1.8832982778549194, + "rewards/rejected": -1.9824028015136719, + "sft_loss": 0.005023724865168333, + "step": 11190 + }, + { + "epoch": 0.8712563204978607, + "grad_norm": 3.2280795574188232, + "learning_rate": 2.017126113159301e-06, + "logits/chosen": -0.1030881404876709, + "logits/rejected": -1.545559048652649, + "logps/chosen": -0.9980408549308777, + "logps/rejected": -17.469547271728516, + "loss": 1.012, + "num_input_tokens_seen": 14702144, + "odds_ratio_loss": 10.044103622436523, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.09980408847332001, + "rewards/margins": 1.6471507549285889, + "rewards/rejected": -1.7469545602798462, + "sft_loss": 0.0076124765910208225, + "step": 11200 + }, + { + "epoch": 0.8720342279268767, + "grad_norm": 7.864035129547119, + "learning_rate": 1.993151836608151e-06, + "logits/chosen": -0.17491404712200165, + "logits/rejected": -1.3855005502700806, + "logps/chosen": -0.5778329372406006, + "logps/rejected": -13.704730033874512, + "loss": 0.5797, + "num_input_tokens_seen": 14715008, + "odds_ratio_loss": 5.734278202056885, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.05778329446911812, + "rewards/margins": 1.3126897811889648, + "rewards/rejected": -1.3704731464385986, + "sft_loss": 0.0062981559894979, + "step": 11210 + }, + { + "epoch": 0.8728121353558926, + "grad_norm": 161.39295959472656, + "learning_rate": 1.9693149682073393e-06, + "logits/chosen": -0.49991482496261597, + "logits/rejected": -1.12697434425354, + "logps/chosen": -1.1875393390655518, + "logps/rejected": -12.513293266296387, + "loss": 1.1946, + "num_input_tokens_seen": 14726608, + "odds_ratio_loss": 11.83639144897461, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.11875393241643906, + "rewards/margins": 1.132575511932373, + "rewards/rejected": -1.2513294219970703, + "sft_loss": 0.01099352352321148, + "step": 11220 + }, + { + "epoch": 0.8735900427849086, + "grad_norm": 29.137767791748047, + "learning_rate": 1.9456156503223107e-06, + "logits/chosen": -0.20925652980804443, + "logits/rejected": -1.4698998928070068, + "logps/chosen": -0.962933361530304, + "logps/rejected": -16.86948013305664, + "loss": 0.9768, + "num_input_tokens_seen": 14739920, + "odds_ratio_loss": 9.696186065673828, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.09629333019256592, + "rewards/margins": 1.5906546115875244, + "rewards/rejected": -1.6869480609893799, + "sft_loss": 0.0071391938254237175, + "step": 11230 + }, + { + "epoch": 0.8743679502139245, + "grad_norm": 3.734955072402954, + "learning_rate": 1.922054024496975e-06, + "logits/chosen": -0.39152416586875916, + "logits/rejected": -1.984805703163147, + "logps/chosen": -1.262678861618042, + "logps/rejected": -21.21131134033203, + "loss": 1.2682, + "num_input_tokens_seen": 14750096, + "odds_ratio_loss": 12.499911308288574, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.12626788020133972, + "rewards/margins": 1.9948632717132568, + "rewards/rejected": -2.121131420135498, + "sft_loss": 0.01821861043572426, + "step": 11240 + }, + { + "epoch": 0.8751458576429405, + "grad_norm": 3.2851293087005615, + "learning_rate": 1.898630231452897e-06, + "logits/chosen": -0.4599381983280182, + "logits/rejected": -1.0844638347625732, + "logps/chosen": -0.9390494227409363, + "logps/rejected": -15.288253784179688, + "loss": 0.9424, + "num_input_tokens_seen": 14762112, + "odds_ratio_loss": 9.057352066040039, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09390494227409363, + "rewards/margins": 1.4349205493927002, + "rewards/rejected": -1.5288255214691162, + "sft_loss": 0.03662865236401558, + "step": 11250 + }, + { + "epoch": 0.8759237650719565, + "grad_norm": 1.8594413995742798, + "learning_rate": 1.8753444110884172e-06, + "logits/chosen": -0.2478855401277542, + "logits/rejected": -1.322733998298645, + "logps/chosen": -1.0074350833892822, + "logps/rejected": -10.416272163391113, + "loss": 1.0263, + "num_input_tokens_seen": 14772640, + "odds_ratio_loss": 10.212084770202637, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.10074350982904434, + "rewards/margins": 0.9408837556838989, + "rewards/rejected": -1.041627287864685, + "sft_loss": 0.005066190846264362, + "step": 11260 + }, + { + "epoch": 0.8767016725009724, + "grad_norm": 5.2853827476501465, + "learning_rate": 1.8521967024778487e-06, + "logits/chosen": -0.49381309747695923, + "logits/rejected": -1.2733218669891357, + "logps/chosen": -0.8687035441398621, + "logps/rejected": -16.657901763916016, + "loss": 0.8807, + "num_input_tokens_seen": 14783424, + "odds_ratio_loss": 8.629514694213867, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.08687035739421844, + "rewards/margins": 1.5789198875427246, + "rewards/rejected": -1.6657902002334595, + "sft_loss": 0.017769496887922287, + "step": 11270 + }, + { + "epoch": 0.8774795799299884, + "grad_norm": 3.3902578353881836, + "learning_rate": 1.829187243870628e-06, + "logits/chosen": -0.3714262843132019, + "logits/rejected": -1.1370850801467896, + "logps/chosen": -0.8502224683761597, + "logps/rejected": -13.264406204223633, + "loss": 0.8601, + "num_input_tokens_seen": 14798224, + "odds_ratio_loss": 8.506078720092773, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.08502225577831268, + "rewards/margins": 1.2414183616638184, + "rewards/rejected": -1.3264405727386475, + "sft_loss": 0.009510818868875504, + "step": 11280 + }, + { + "epoch": 0.8782574873590043, + "grad_norm": 3.657287120819092, + "learning_rate": 1.8063161726904976e-06, + "logits/chosen": -0.20781084895133972, + "logits/rejected": -1.3679994344711304, + "logps/chosen": -0.8147785067558289, + "logps/rejected": -9.873483657836914, + "loss": 0.8365, + "num_input_tokens_seen": 14809200, + "odds_ratio_loss": 8.311952590942383, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.08147785812616348, + "rewards/margins": 0.9058707356452942, + "rewards/rejected": -0.9873485565185547, + "sft_loss": 0.005294256377965212, + "step": 11290 + }, + { + "epoch": 0.8790353947880202, + "grad_norm": 2.625351667404175, + "learning_rate": 1.783583625534682e-06, + "logits/chosen": 0.0041309236548841, + "logits/rejected": -1.4668128490447998, + "logps/chosen": -0.8829860687255859, + "logps/rejected": -15.129183769226074, + "loss": 0.9016, + "num_input_tokens_seen": 14828032, + "odds_ratio_loss": 8.983991622924805, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.08829861134290695, + "rewards/margins": 1.4246199131011963, + "rewards/rejected": -1.512918472290039, + "sft_loss": 0.003154673147946596, + "step": 11300 + }, + { + "epoch": 0.8798133022170361, + "grad_norm": 8.057369232177734, + "learning_rate": 1.7609897381730662e-06, + "logits/chosen": -0.4841797351837158, + "logits/rejected": -1.6735862493515015, + "logps/chosen": -0.9034742116928101, + "logps/rejected": -19.795330047607422, + "loss": 0.9042, + "num_input_tokens_seen": 14840864, + "odds_ratio_loss": 8.911484718322754, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09034740924835205, + "rewards/margins": 1.8891855478286743, + "rewards/rejected": -1.9795329570770264, + "sft_loss": 0.013028261251747608, + "step": 11310 + }, + { + "epoch": 0.8805912096460521, + "grad_norm": 3.4109108448028564, + "learning_rate": 1.7385346455474016e-06, + "logits/chosen": -0.47292718291282654, + "logits/rejected": -1.4690214395523071, + "logps/chosen": -0.978543758392334, + "logps/rejected": -16.113407135009766, + "loss": 0.9976, + "num_input_tokens_seen": 14855616, + "odds_ratio_loss": 9.500051498413086, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.097854383289814, + "rewards/margins": 1.5134862661361694, + "rewards/rejected": -1.611340880393982, + "sft_loss": 0.04762210696935654, + "step": 11320 + }, + { + "epoch": 0.881369117075068, + "grad_norm": 3.220376968383789, + "learning_rate": 1.716218481770482e-06, + "logits/chosen": 0.024894937872886658, + "logits/rejected": -1.4938503503799438, + "logps/chosen": -0.8936010599136353, + "logps/rejected": -11.886364936828613, + "loss": 0.8987, + "num_input_tokens_seen": 14864624, + "odds_ratio_loss": 8.929697036743164, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.08936010301113129, + "rewards/margins": 1.0992765426635742, + "rewards/rejected": -1.1886365413665771, + "sft_loss": 0.00577815156430006, + "step": 11330 + }, + { + "epoch": 0.882147024504084, + "grad_norm": 1.3350600004196167, + "learning_rate": 1.6940413801253524e-06, + "logits/chosen": -0.13089770078659058, + "logits/rejected": -1.4039313793182373, + "logps/chosen": -0.8123601078987122, + "logps/rejected": -11.132844924926758, + "loss": 0.8339, + "num_input_tokens_seen": 14880496, + "odds_ratio_loss": 8.236737251281738, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.08123601227998734, + "rewards/margins": 1.0320484638214111, + "rewards/rejected": -1.1132845878601074, + "sft_loss": 0.010246511548757553, + "step": 11340 + }, + { + "epoch": 0.8829249319331, + "grad_norm": 3.229665994644165, + "learning_rate": 1.6720034730645152e-06, + "logits/chosen": -0.26986435055732727, + "logits/rejected": -1.1807479858398438, + "logps/chosen": -1.585399866104126, + "logps/rejected": -12.02979564666748, + "loss": 1.5936, + "num_input_tokens_seen": 14891360, + "odds_ratio_loss": 15.809844970703125, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.15853998064994812, + "rewards/margins": 1.044439673423767, + "rewards/rejected": -1.2029794454574585, + "sft_loss": 0.01263892650604248, + "step": 11350 + }, + { + "epoch": 0.8837028393621159, + "grad_norm": 10.792923927307129, + "learning_rate": 1.6501048922091255e-06, + "logits/chosen": -0.7156990766525269, + "logits/rejected": -1.0069489479064941, + "logps/chosen": -1.0320378541946411, + "logps/rejected": -12.793333053588867, + "loss": 1.0346, + "num_input_tokens_seen": 14902688, + "odds_ratio_loss": 9.464272499084473, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.10320379585027695, + "rewards/margins": 1.1761295795440674, + "rewards/rejected": -1.2793333530426025, + "sft_loss": 0.08816216140985489, + "step": 11360 + }, + { + "epoch": 0.8844807467911319, + "grad_norm": 24.13997459411621, + "learning_rate": 1.628345768348219e-06, + "logits/chosen": -0.17151141166687012, + "logits/rejected": -1.7036211490631104, + "logps/chosen": -0.9553998112678528, + "logps/rejected": -15.676922798156738, + "loss": 0.9604, + "num_input_tokens_seen": 14915024, + "odds_ratio_loss": 8.78715705871582, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09553997963666916, + "rewards/margins": 1.4721522331237793, + "rewards/rejected": -1.5676923990249634, + "sft_loss": 0.08164925128221512, + "step": 11370 + }, + { + "epoch": 0.8852586542201478, + "grad_norm": 3.026034355163574, + "learning_rate": 1.6067262314379327e-06, + "logits/chosen": -0.3190793991088867, + "logits/rejected": -1.447247862815857, + "logps/chosen": -0.9018405079841614, + "logps/rejected": -13.972577095031738, + "loss": 0.9175, + "num_input_tokens_seen": 14924096, + "odds_ratio_loss": 9.034456253051758, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.0901840552687645, + "rewards/margins": 1.3070735931396484, + "rewards/rejected": -1.397257685661316, + "sft_loss": 0.014050662517547607, + "step": 11380 + }, + { + "epoch": 0.8860365616491638, + "grad_norm": 1.9133740663528442, + "learning_rate": 1.585246410600713e-06, + "logits/chosen": -0.0015375495422631502, + "logits/rejected": -1.5664477348327637, + "logps/chosen": -0.9613587260246277, + "logps/rejected": -9.113849639892578, + "loss": 0.9809, + "num_input_tokens_seen": 14937840, + "odds_ratio_loss": 9.752358436584473, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09613586962223053, + "rewards/margins": 0.8152491450309753, + "rewards/rejected": -0.9113848805427551, + "sft_loss": 0.005694333929568529, + "step": 11390 + }, + { + "epoch": 0.8868144690781797, + "grad_norm": 23.818763732910156, + "learning_rate": 1.563906434124554e-06, + "logits/chosen": -0.5499352216720581, + "logits/rejected": -1.3355505466461182, + "logps/chosen": -0.7920863032341003, + "logps/rejected": -23.026779174804688, + "loss": 0.7929, + "num_input_tokens_seen": 14949600, + "odds_ratio_loss": 7.54303503036499, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.0792086273431778, + "rewards/margins": 2.2234692573547363, + "rewards/rejected": -2.302677869796753, + "sft_loss": 0.0385659821331501, + "step": 11400 + }, + { + "epoch": 0.8875923765071957, + "grad_norm": 3.3071045875549316, + "learning_rate": 1.5427064294622318e-06, + "logits/chosen": -0.22468701004981995, + "logits/rejected": -1.2597036361694336, + "logps/chosen": -0.7308349609375, + "logps/rejected": -8.260499954223633, + "loss": 0.7525, + "num_input_tokens_seen": 14962416, + "odds_ratio_loss": 7.281064510345459, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.07308349758386612, + "rewards/margins": 0.7529664635658264, + "rewards/rejected": -0.8260499238967896, + "sft_loss": 0.02442113682627678, + "step": 11410 + }, + { + "epoch": 0.8883702839362115, + "grad_norm": 2.004885673522949, + "learning_rate": 1.521646523230552e-06, + "logits/chosen": -0.45084089040756226, + "logits/rejected": -1.5717991590499878, + "logps/chosen": -0.6790512204170227, + "logps/rejected": -18.60866355895996, + "loss": 0.6949, + "num_input_tokens_seen": 14974592, + "odds_ratio_loss": 6.8787360191345215, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.06790511310100555, + "rewards/margins": 1.7929614782333374, + "rewards/rejected": -1.8608665466308594, + "sft_loss": 0.007033783011138439, + "step": 11420 + }, + { + "epoch": 0.8891481913652275, + "grad_norm": 1.7734142541885376, + "learning_rate": 1.5007268412095654e-06, + "logits/chosen": -0.3790365755558014, + "logits/rejected": -1.9984188079833984, + "logps/chosen": -0.9335896372795105, + "logps/rejected": -18.616718292236328, + "loss": 0.9439, + "num_input_tokens_seen": 14983280, + "odds_ratio_loss": 8.583988189697266, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.09335896372795105, + "rewards/margins": 1.7683128118515015, + "rewards/rejected": -1.861671805381775, + "sft_loss": 0.08547282963991165, + "step": 11430 + }, + { + "epoch": 0.8899260987942434, + "grad_norm": 3.9545562267303467, + "learning_rate": 1.4799475083418474e-06, + "logits/chosen": -0.3433544635772705, + "logits/rejected": -1.3527787923812866, + "logps/chosen": -0.8042358160018921, + "logps/rejected": -15.672798156738281, + "loss": 0.8139, + "num_input_tokens_seen": 14995552, + "odds_ratio_loss": 8.090936660766602, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.08042358607053757, + "rewards/margins": 1.4868563413619995, + "rewards/rejected": -1.5672800540924072, + "sft_loss": 0.0047922274097800255, + "step": 11440 + }, + { + "epoch": 0.8907040062232594, + "grad_norm": 11.453007698059082, + "learning_rate": 1.4593086487317386e-06, + "logits/chosen": -0.2827529311180115, + "logits/rejected": -1.4444782733917236, + "logps/chosen": -0.8965161442756653, + "logps/rejected": -13.879638671875, + "loss": 0.9004, + "num_input_tokens_seen": 15005920, + "odds_ratio_loss": 8.895330429077148, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.08965159952640533, + "rewards/margins": 1.2983124256134033, + "rewards/rejected": -1.3879640102386475, + "sft_loss": 0.010893709026277065, + "step": 11450 + }, + { + "epoch": 0.8914819136522754, + "grad_norm": 2.555229902267456, + "learning_rate": 1.4388103856446067e-06, + "logits/chosen": -0.34105560183525085, + "logits/rejected": -1.4867103099822998, + "logps/chosen": -0.9608953595161438, + "logps/rejected": -15.809675216674805, + "loss": 0.972, + "num_input_tokens_seen": 15018672, + "odds_ratio_loss": 9.660136222839355, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.09608952701091766, + "rewards/margins": 1.4848780632019043, + "rewards/rejected": -1.580967664718628, + "sft_loss": 0.0059747775085270405, + "step": 11460 + }, + { + "epoch": 0.8922598210812913, + "grad_norm": 2.2812118530273438, + "learning_rate": 1.4184528415060988e-06, + "logits/chosen": -0.2780112624168396, + "logits/rejected": -1.9391981363296509, + "logps/chosen": -0.9633452296257019, + "logps/rejected": -26.054580688476562, + "loss": 0.9642, + "num_input_tokens_seen": 15030528, + "odds_ratio_loss": 9.586477279663086, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09633452445268631, + "rewards/margins": 2.5091235637664795, + "rewards/rejected": -2.6054580211639404, + "sft_loss": 0.005508183501660824, + "step": 11470 + }, + { + "epoch": 0.8930377285103073, + "grad_norm": 1.5275903940200806, + "learning_rate": 1.3982361379014319e-06, + "logits/chosen": -0.13214851915836334, + "logits/rejected": -1.8039649724960327, + "logps/chosen": -1.1114555597305298, + "logps/rejected": -19.613750457763672, + "loss": 1.1139, + "num_input_tokens_seen": 15042128, + "odds_ratio_loss": 10.567789077758789, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.11114554107189178, + "rewards/margins": 1.8502298593521118, + "rewards/rejected": -1.9613752365112305, + "sft_loss": 0.057110778987407684, + "step": 11480 + }, + { + "epoch": 0.8938156359393232, + "grad_norm": 3.1028542518615723, + "learning_rate": 1.378160395574643e-06, + "logits/chosen": -0.22435922920703888, + "logits/rejected": -1.4496055841445923, + "logps/chosen": -0.6374467611312866, + "logps/rejected": -20.233562469482422, + "loss": 0.6479, + "num_input_tokens_seen": 15053104, + "odds_ratio_loss": 6.419812202453613, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.06374466419219971, + "rewards/margins": 1.9596116542816162, + "rewards/rejected": -2.0233561992645264, + "sft_loss": 0.005906237289309502, + "step": 11490 + }, + { + "epoch": 0.8945935433683392, + "grad_norm": 1.0684871673583984, + "learning_rate": 1.358225734427887e-06, + "logits/chosen": -0.29106268286705017, + "logits/rejected": -1.0870394706726074, + "logps/chosen": -1.0519826412200928, + "logps/rejected": -11.098809242248535, + "loss": 1.0813, + "num_input_tokens_seen": 15066512, + "odds_ratio_loss": 9.962423324584961, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.10519824922084808, + "rewards/margins": 1.0046826601028442, + "rewards/rejected": -1.109881043434143, + "sft_loss": 0.08502072095870972, + "step": 11500 + }, + { + "epoch": 0.8953714507973551, + "grad_norm": 6.9853010177612305, + "learning_rate": 1.3384322735207123e-06, + "logits/chosen": -0.0020437941420823336, + "logits/rejected": -1.803490400314331, + "logps/chosen": -0.9121143221855164, + "logps/rejected": -15.750417709350586, + "loss": 0.9158, + "num_input_tokens_seen": 15079392, + "odds_ratio_loss": 9.098687171936035, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09121143072843552, + "rewards/margins": 1.483830451965332, + "rewards/rejected": -1.5750417709350586, + "sft_loss": 0.005918119102716446, + "step": 11510 + }, + { + "epoch": 0.8961493582263711, + "grad_norm": 5.0061211585998535, + "learning_rate": 1.318780131069347e-06, + "logits/chosen": -0.30716389417648315, + "logits/rejected": -1.30451238155365, + "logps/chosen": -0.6082115769386292, + "logps/rejected": -13.56993579864502, + "loss": 0.6161, + "num_input_tokens_seen": 15092336, + "odds_ratio_loss": 6.121800422668457, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.060821156948804855, + "rewards/margins": 1.2961723804473877, + "rewards/rejected": -1.356993556022644, + "sft_loss": 0.003893594490364194, + "step": 11520 + }, + { + "epoch": 0.896927265655387, + "grad_norm": 7.328099250793457, + "learning_rate": 1.2992694244460008e-06, + "logits/chosen": -0.4314120411872864, + "logits/rejected": -1.3663753271102905, + "logps/chosen": -0.7535328269004822, + "logps/rejected": -16.93317413330078, + "loss": 0.7547, + "num_input_tokens_seen": 15107072, + "odds_ratio_loss": 7.248821258544922, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.07535327970981598, + "rewards/margins": 1.6179643869400024, + "rewards/rejected": -1.6933176517486572, + "sft_loss": 0.029783323407173157, + "step": 11530 + }, + { + "epoch": 0.897705173084403, + "grad_norm": 2.2374439239501953, + "learning_rate": 1.2799002701781526e-06, + "logits/chosen": -0.08255377411842346, + "logits/rejected": -1.3694899082183838, + "logps/chosen": -1.6034971475601196, + "logps/rejected": -10.680310249328613, + "loss": 1.6142, + "num_input_tokens_seen": 15122032, + "odds_ratio_loss": 15.007932662963867, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.16034969687461853, + "rewards/margins": 0.9076812863349915, + "rewards/rejected": -1.0680310726165771, + "sft_loss": 0.11340135335922241, + "step": 11540 + }, + { + "epoch": 0.8984830805134189, + "grad_norm": 2.645416021347046, + "learning_rate": 1.2606727839478721e-06, + "logits/chosen": -0.17675833404064178, + "logits/rejected": -1.2538154125213623, + "logps/chosen": -0.646391749382019, + "logps/rejected": -19.915109634399414, + "loss": 0.6518, + "num_input_tokens_seen": 15135648, + "odds_ratio_loss": 6.432554721832275, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.06463916599750519, + "rewards/margins": 1.9268718957901, + "rewards/rejected": -1.9915109872817993, + "sft_loss": 0.00850028544664383, + "step": 11550 + }, + { + "epoch": 0.8992609879424348, + "grad_norm": 5.000986576080322, + "learning_rate": 1.2415870805911056e-06, + "logits/chosen": -0.20390114188194275, + "logits/rejected": -1.4622828960418701, + "logps/chosen": -0.9593456983566284, + "logps/rejected": -12.142216682434082, + "loss": 0.9793, + "num_input_tokens_seen": 15146576, + "odds_ratio_loss": 9.574671745300293, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.09593456983566284, + "rewards/margins": 1.1182870864868164, + "rewards/rejected": -1.2142215967178345, + "sft_loss": 0.021797997877001762, + "step": 11560 + }, + { + "epoch": 0.9000388953714508, + "grad_norm": 5.9946088790893555, + "learning_rate": 1.2226432740970073e-06, + "logits/chosen": -0.18088433146476746, + "logits/rejected": -1.5347539186477661, + "logps/chosen": -0.8887677192687988, + "logps/rejected": -14.090968132019043, + "loss": 0.9127, + "num_input_tokens_seen": 15160624, + "odds_ratio_loss": 9.078805923461914, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.08887676894664764, + "rewards/margins": 1.3202202320098877, + "rewards/rejected": -1.4090967178344727, + "sft_loss": 0.004832427017390728, + "step": 11570 + }, + { + "epoch": 0.9008168028004667, + "grad_norm": 5.074446678161621, + "learning_rate": 1.2038414776072593e-06, + "logits/chosen": -0.04299873486161232, + "logits/rejected": -1.6535265445709229, + "logps/chosen": -0.9011691808700562, + "logps/rejected": -11.200329780578613, + "loss": 0.9546, + "num_input_tokens_seen": 15175184, + "odds_ratio_loss": 9.490025520324707, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.09011692553758621, + "rewards/margins": 1.0299160480499268, + "rewards/rejected": -1.1200329065322876, + "sft_loss": 0.005555834621191025, + "step": 11580 + }, + { + "epoch": 0.9015947102294827, + "grad_norm": 4.478003978729248, + "learning_rate": 1.1851818034153833e-06, + "logits/chosen": -0.0886651873588562, + "logits/rejected": -1.5455336570739746, + "logps/chosen": -1.1897647380828857, + "logps/rejected": -10.544865608215332, + "loss": 1.2083, + "num_input_tokens_seen": 15186752, + "odds_ratio_loss": 11.775666236877441, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.11897647380828857, + "rewards/margins": 0.9355101585388184, + "rewards/rejected": -1.054486632347107, + "sft_loss": 0.030766809359192848, + "step": 11590 + }, + { + "epoch": 0.9023726176584986, + "grad_norm": 2.3363897800445557, + "learning_rate": 1.1666643629660856e-06, + "logits/chosen": -0.8766778707504272, + "logits/rejected": -0.7064003944396973, + "logps/chosen": -1.0922223329544067, + "logps/rejected": -8.02529239654541, + "loss": 1.095, + "num_input_tokens_seen": 15196896, + "odds_ratio_loss": 10.510167121887207, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.10922224819660187, + "rewards/margins": 0.6933069229125977, + "rewards/rejected": -0.8025292158126831, + "sft_loss": 0.04393957182765007, + "step": 11600 + }, + { + "epoch": 0.9031505250875146, + "grad_norm": 2.4352638721466064, + "learning_rate": 1.1482892668545742e-06, + "logits/chosen": -0.14885909855365753, + "logits/rejected": -1.753119707107544, + "logps/chosen": -0.7887487411499023, + "logps/rejected": -13.43248176574707, + "loss": 0.7962, + "num_input_tokens_seen": 15210464, + "odds_ratio_loss": 7.846124172210693, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.078874871134758, + "rewards/margins": 1.264373540878296, + "rewards/rejected": -1.3432481288909912, + "sft_loss": 0.011604775674641132, + "step": 11610 + }, + { + "epoch": 0.9039284325165305, + "grad_norm": 2.8514859676361084, + "learning_rate": 1.1300566248259092e-06, + "logits/chosen": -0.23481757938861847, + "logits/rejected": -1.2238056659698486, + "logps/chosen": -0.7990361452102661, + "logps/rejected": -9.189723014831543, + "loss": 0.8106, + "num_input_tokens_seen": 15219536, + "odds_ratio_loss": 8.021627426147461, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.07990361750125885, + "rewards/margins": 0.8390687108039856, + "rewards/rejected": -0.9189723134040833, + "sft_loss": 0.008481072261929512, + "step": 11620 + }, + { + "epoch": 0.9047063399455465, + "grad_norm": 1.4698771238327026, + "learning_rate": 1.1119665457743484e-06, + "logits/chosen": -0.05531783029437065, + "logits/rejected": -1.8035624027252197, + "logps/chosen": -1.8927888870239258, + "logps/rejected": -19.5323429107666, + "loss": 1.8981, + "num_input_tokens_seen": 15231792, + "odds_ratio_loss": 18.91992950439453, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.18927890062332153, + "rewards/margins": 1.7639554738998413, + "rewards/rejected": -1.953234314918518, + "sft_loss": 0.006080970633774996, + "step": 11630 + }, + { + "epoch": 0.9054842473745625, + "grad_norm": 8.166078567504883, + "learning_rate": 1.0940191377426938e-06, + "logits/chosen": -0.18938185274600983, + "logits/rejected": -1.0948208570480347, + "logps/chosen": -0.7792876958847046, + "logps/rejected": -10.034112930297852, + "loss": 0.7905, + "num_input_tokens_seen": 15242496, + "odds_ratio_loss": 7.8034515380859375, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.07792877405881882, + "rewards/margins": 0.9254825711250305, + "rewards/rejected": -1.0034112930297852, + "sft_loss": 0.0101962611079216, + "step": 11640 + }, + { + "epoch": 0.9062621548035784, + "grad_norm": 2.3272478580474854, + "learning_rate": 1.076214507921644e-06, + "logits/chosen": -0.2795869708061218, + "logits/rejected": -1.4620563983917236, + "logps/chosen": -0.9061124920845032, + "logps/rejected": -14.130447387695312, + "loss": 0.917, + "num_input_tokens_seen": 15256656, + "odds_ratio_loss": 9.063624382019043, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09061124175786972, + "rewards/margins": 1.3224334716796875, + "rewards/rejected": -1.4130446910858154, + "sft_loss": 0.010650251060724258, + "step": 11650 + }, + { + "epoch": 0.9070400622325944, + "grad_norm": 3.9123356342315674, + "learning_rate": 1.0585527626491564e-06, + "logits/chosen": -0.16889598965644836, + "logits/rejected": -1.0222868919372559, + "logps/chosen": -0.9573249816894531, + "logps/rejected": -10.36623477935791, + "loss": 0.9742, + "num_input_tokens_seen": 15273632, + "odds_ratio_loss": 9.666048049926758, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09573251754045486, + "rewards/margins": 0.9408909678459167, + "rewards/rejected": -1.036623477935791, + "sft_loss": 0.007627577520906925, + "step": 11660 + }, + { + "epoch": 0.9078179696616102, + "grad_norm": 2.5804312229156494, + "learning_rate": 1.0410340074098162e-06, + "logits/chosen": -0.29126492142677307, + "logits/rejected": -1.21029794216156, + "logps/chosen": -0.8057632446289062, + "logps/rejected": -7.643909454345703, + "loss": 0.8336, + "num_input_tokens_seen": 15285872, + "odds_ratio_loss": 8.258122444152832, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.0805763304233551, + "rewards/margins": 0.6838146448135376, + "rewards/rejected": -0.7643910050392151, + "sft_loss": 0.007780120708048344, + "step": 11670 + }, + { + "epoch": 0.9085958770906262, + "grad_norm": 1.9556622505187988, + "learning_rate": 1.0236583468341914e-06, + "logits/chosen": -0.3107249438762665, + "logits/rejected": -1.642421007156372, + "logps/chosen": -0.9437317848205566, + "logps/rejected": -11.675893783569336, + "loss": 0.9522, + "num_input_tokens_seen": 15298096, + "odds_ratio_loss": 9.248234748840332, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09437316656112671, + "rewards/margins": 1.073216199874878, + "rewards/rejected": -1.1675894260406494, + "sft_loss": 0.027393311262130737, + "step": 11680 + }, + { + "epoch": 0.9093737845196421, + "grad_norm": 1.9555792808532715, + "learning_rate": 1.0064258846982282e-06, + "logits/chosen": -0.3013989329338074, + "logits/rejected": -1.2657264471054077, + "logps/chosen": -1.101585865020752, + "logps/rejected": -16.60849952697754, + "loss": 1.1176, + "num_input_tokens_seen": 15307776, + "odds_ratio_loss": 10.710332870483398, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.11015859991312027, + "rewards/margins": 1.5506913661956787, + "rewards/rejected": -1.660850167274475, + "sft_loss": 0.04653622955083847, + "step": 11690 + }, + { + "epoch": 0.9101516919486581, + "grad_norm": 3.1522815227508545, + "learning_rate": 9.893367239226208e-07, + "logits/chosen": -0.09980596601963043, + "logits/rejected": -1.88507878780365, + "logps/chosen": -0.9663987159729004, + "logps/rejected": -11.776286125183105, + "loss": 0.9805, + "num_input_tokens_seen": 15320480, + "odds_ratio_loss": 9.767982482910156, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.09663987159729004, + "rewards/margins": 1.0809887647628784, + "rewards/rejected": -1.1776286363601685, + "sft_loss": 0.0037159952335059643, + "step": 11700 + }, + { + "epoch": 0.910929599377674, + "grad_norm": 2.2265751361846924, + "learning_rate": 9.723909665721954e-07, + "logits/chosen": -0.07370741665363312, + "logits/rejected": -1.345320701599121, + "logps/chosen": -0.670822024345398, + "logps/rejected": -15.495183944702148, + "loss": 0.6806, + "num_input_tokens_seen": 15329120, + "odds_ratio_loss": 6.744556427001953, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.06708221137523651, + "rewards/margins": 1.4824364185333252, + "rewards/rejected": -1.5495185852050781, + "sft_loss": 0.006134716793894768, + "step": 11710 + }, + { + "epoch": 0.91170750680669, + "grad_norm": 5.613677978515625, + "learning_rate": 9.555887138553015e-07, + "logits/chosen": -0.037133391946554184, + "logits/rejected": -1.6061843633651733, + "logps/chosen": -0.9063383340835571, + "logps/rejected": -6.80635929107666, + "loss": 0.919, + "num_input_tokens_seen": 15341536, + "odds_ratio_loss": 9.143983840942383, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.0906338319182396, + "rewards/margins": 0.5900020599365234, + "rewards/rejected": -0.6806358098983765, + "sft_loss": 0.004566813353449106, + "step": 11720 + }, + { + "epoch": 0.912485414235706, + "grad_norm": 1.902655005455017, + "learning_rate": 9.389300661232142e-07, + "logits/chosen": -0.09280941635370255, + "logits/rejected": -1.4851089715957642, + "logps/chosen": -0.75700843334198, + "logps/rejected": -11.415763854980469, + "loss": 0.7633, + "num_input_tokens_seen": 15351520, + "odds_ratio_loss": 7.47744607925415, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.07570084929466248, + "rewards/margins": 1.06587553024292, + "rewards/rejected": -1.1415762901306152, + "sft_loss": 0.015602183528244495, + "step": 11730 + }, + { + "epoch": 0.9132633216647219, + "grad_norm": 1.1491291522979736, + "learning_rate": 9.224151228695216e-07, + "logits/chosen": -0.21760201454162598, + "logits/rejected": -1.598064661026001, + "logps/chosen": -0.8968531489372253, + "logps/rejected": -14.969205856323242, + "loss": 0.9022, + "num_input_tokens_seen": 15370992, + "odds_ratio_loss": 8.735642433166504, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.08968530595302582, + "rewards/margins": 1.4072353839874268, + "rewards/rejected": -1.4969207048416138, + "sft_loss": 0.028603632003068924, + "step": 11740 + }, + { + "epoch": 0.9140412290937379, + "grad_norm": 3.618295431137085, + "learning_rate": 9.060439827295486e-07, + "logits/chosen": -0.34190061688423157, + "logits/rejected": -1.1805375814437866, + "logps/chosen": -0.8074049949645996, + "logps/rejected": -21.932445526123047, + "loss": 0.8075, + "num_input_tokens_seen": 15386080, + "odds_ratio_loss": 7.9582200050354, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.08074049651622772, + "rewards/margins": 2.112504005432129, + "rewards/rejected": -2.193244457244873, + "sft_loss": 0.011641708202660084, + "step": 11750 + }, + { + "epoch": 0.9148191365227538, + "grad_norm": 6.058084964752197, + "learning_rate": 8.898167434797433e-07, + "logits/chosen": -0.13754260540008545, + "logits/rejected": -1.4365981817245483, + "logps/chosen": -1.115795373916626, + "logps/rejected": -14.768132209777832, + "loss": 1.1229, + "num_input_tokens_seen": 15399072, + "odds_ratio_loss": 10.982545852661133, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.11157955229282379, + "rewards/margins": 1.3652336597442627, + "rewards/rejected": -1.4768131971359253, + "sft_loss": 0.024679964408278465, + "step": 11760 + }, + { + "epoch": 0.9155970439517698, + "grad_norm": 2.0407490730285645, + "learning_rate": 8.737335020371218e-07, + "logits/chosen": -0.06862975656986237, + "logits/rejected": -1.5248997211456299, + "logps/chosen": -0.7661813497543335, + "logps/rejected": -13.9795560836792, + "loss": 0.7756, + "num_input_tokens_seen": 15412064, + "odds_ratio_loss": 7.698896884918213, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.07661812752485275, + "rewards/margins": 1.3213374614715576, + "rewards/rejected": -1.3979556560516357, + "sft_loss": 0.005741656757891178, + "step": 11770 + }, + { + "epoch": 0.9163749513807857, + "grad_norm": 4.3902974128723145, + "learning_rate": 8.577943544586686e-07, + "logits/chosen": -0.1258087009191513, + "logits/rejected": -1.358526587486267, + "logps/chosen": -0.9527864456176758, + "logps/rejected": -10.854793548583984, + "loss": 0.9682, + "num_input_tokens_seen": 15420832, + "odds_ratio_loss": 9.32841968536377, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.09527863562107086, + "rewards/margins": 0.9902006983757019, + "rewards/rejected": -1.0854793787002563, + "sft_loss": 0.03540115803480148, + "step": 11780 + }, + { + "epoch": 0.9171528588098017, + "grad_norm": 3.496689558029175, + "learning_rate": 8.419993959407652e-07, + "logits/chosen": -0.13915188610553741, + "logits/rejected": -1.347084641456604, + "logps/chosen": -0.7917124032974243, + "logps/rejected": -9.26572036743164, + "loss": 0.7964, + "num_input_tokens_seen": 15434000, + "odds_ratio_loss": 7.921217918395996, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.07917124032974243, + "rewards/margins": 0.8474008440971375, + "rewards/rejected": -0.9265719652175903, + "sft_loss": 0.004300933331251144, + "step": 11790 + }, + { + "epoch": 0.9179307662388175, + "grad_norm": 1.8610042333602905, + "learning_rate": 8.263487208186372e-07, + "logits/chosen": -0.3398294746875763, + "logits/rejected": -1.3658941984176636, + "logps/chosen": -0.7725250124931335, + "logps/rejected": -18.21072006225586, + "loss": 0.7813, + "num_input_tokens_seen": 15447920, + "odds_ratio_loss": 7.75238561630249, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.07725249975919724, + "rewards/margins": 1.7438195943832397, + "rewards/rejected": -1.821071982383728, + "sft_loss": 0.0060457391664385796, + "step": 11800 + }, + { + "epoch": 0.9187086736678335, + "grad_norm": 3.241182327270508, + "learning_rate": 8.108424225657635e-07, + "logits/chosen": -0.3613438308238983, + "logits/rejected": -1.475386619567871, + "logps/chosen": -0.6234281659126282, + "logps/rejected": -14.582191467285156, + "loss": 0.6235, + "num_input_tokens_seen": 15458496, + "odds_ratio_loss": 6.197271823883057, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.0623428151011467, + "rewards/margins": 1.3958762884140015, + "rewards/rejected": -1.458219051361084, + "sft_loss": 0.003796472679823637, + "step": 11810 + }, + { + "epoch": 0.9194865810968494, + "grad_norm": 2.7631192207336426, + "learning_rate": 7.954805937933435e-07, + "logits/chosen": -0.07728441059589386, + "logits/rejected": -1.3429712057113647, + "logps/chosen": -0.9366790652275085, + "logps/rejected": -7.251186370849609, + "loss": 0.9534, + "num_input_tokens_seen": 15471504, + "odds_ratio_loss": 9.483495712280273, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.0936679020524025, + "rewards/margins": 0.6314507722854614, + "rewards/rejected": -0.7251186966896057, + "sft_loss": 0.00500117614865303, + "step": 11820 + }, + { + "epoch": 0.9202644885258654, + "grad_norm": 3.3543310165405273, + "learning_rate": 7.802633262497333e-07, + "logits/chosen": -0.2646792531013489, + "logits/rejected": -1.4928174018859863, + "logps/chosen": -0.8666411638259888, + "logps/rejected": -13.072135925292969, + "loss": 0.8795, + "num_input_tokens_seen": 15483184, + "odds_ratio_loss": 8.742317199707031, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.0866641029715538, + "rewards/margins": 1.2205493450164795, + "rewards/rejected": -1.3072136640548706, + "sft_loss": 0.005289946682751179, + "step": 11830 + }, + { + "epoch": 0.9210423959548814, + "grad_norm": 3.0707082748413086, + "learning_rate": 7.651907108198992e-07, + "logits/chosen": -0.19627130031585693, + "logits/rejected": -1.3972231149673462, + "logps/chosen": -1.4005322456359863, + "logps/rejected": -15.51996898651123, + "loss": 1.4278, + "num_input_tokens_seen": 15495376, + "odds_ratio_loss": 14.157544136047363, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.14005321264266968, + "rewards/margins": 1.4119436740875244, + "rewards/rejected": -1.5519969463348389, + "sft_loss": 0.01207679696381092, + "step": 11840 + }, + { + "epoch": 0.9218203033838973, + "grad_norm": 5.255340099334717, + "learning_rate": 7.502628375248704e-07, + "logits/chosen": -0.3192160725593567, + "logits/rejected": -1.200587511062622, + "logps/chosen": -0.9118558168411255, + "logps/rejected": -15.272189140319824, + "loss": 0.9122, + "num_input_tokens_seen": 15510192, + "odds_ratio_loss": 9.04362678527832, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09118557721376419, + "rewards/margins": 1.4360332489013672, + "rewards/rejected": -1.5272188186645508, + "sft_loss": 0.007811239454895258, + "step": 11850 + }, + { + "epoch": 0.9225982108129133, + "grad_norm": 39.124534606933594, + "learning_rate": 7.354797955212073e-07, + "logits/chosen": -0.28398269414901733, + "logits/rejected": -1.553201675415039, + "logps/chosen": -0.9308874011039734, + "logps/rejected": -14.798632621765137, + "loss": 0.937, + "num_input_tokens_seen": 15522240, + "odds_ratio_loss": 9.252286911010742, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09308873116970062, + "rewards/margins": 1.3867747783660889, + "rewards/rejected": -1.4798635244369507, + "sft_loss": 0.011812415905296803, + "step": 11860 + }, + { + "epoch": 0.9233761182419292, + "grad_norm": 2.2880563735961914, + "learning_rate": 7.208416731004724e-07, + "logits/chosen": -0.31729552149772644, + "logits/rejected": -1.6201820373535156, + "logps/chosen": -0.9650070071220398, + "logps/rejected": -13.590810775756836, + "loss": 0.9866, + "num_input_tokens_seen": 15537056, + "odds_ratio_loss": 9.812111854553223, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.0965007022023201, + "rewards/margins": 1.2625805139541626, + "rewards/rejected": -1.3590812683105469, + "sft_loss": 0.005394432228058577, + "step": 11870 + }, + { + "epoch": 0.9241540256709452, + "grad_norm": 4.735197067260742, + "learning_rate": 7.063485576886852e-07, + "logits/chosen": -0.34278565645217896, + "logits/rejected": -1.0348883867263794, + "logps/chosen": -0.8091912269592285, + "logps/rejected": -11.004549980163574, + "loss": 0.8228, + "num_input_tokens_seen": 15548384, + "odds_ratio_loss": 8.163427352905273, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.08091913163661957, + "rewards/margins": 1.0195358991622925, + "rewards/rejected": -1.1004550457000732, + "sft_loss": 0.0064763561822474, + "step": 11880 + }, + { + "epoch": 0.9249319330999611, + "grad_norm": 1.6394617557525635, + "learning_rate": 6.920005358458243e-07, + "logits/chosen": -0.3871787190437317, + "logits/rejected": -1.1279314756393433, + "logps/chosen": -1.104557991027832, + "logps/rejected": -11.183547019958496, + "loss": 1.116, + "num_input_tokens_seen": 15562736, + "odds_ratio_loss": 11.037538528442383, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.11045581102371216, + "rewards/margins": 1.0078990459442139, + "rewards/rejected": -1.1183546781539917, + "sft_loss": 0.012201225385069847, + "step": 11890 + }, + { + "epoch": 0.9257098405289771, + "grad_norm": 1.6821295022964478, + "learning_rate": 6.777976932652946e-07, + "logits/chosen": -0.038701239973306656, + "logits/rejected": -2.0137062072753906, + "logps/chosen": -1.0222748517990112, + "logps/rejected": -14.266027450561523, + "loss": 1.0268, + "num_input_tokens_seen": 15578576, + "odds_ratio_loss": 10.136931419372559, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.10222749412059784, + "rewards/margins": 1.3243753910064697, + "rewards/rejected": -1.4266027212142944, + "sft_loss": 0.013138200156390667, + "step": 11900 + }, + { + "epoch": 0.926487747957993, + "grad_norm": 1.7491499185562134, + "learning_rate": 6.637401147734173e-07, + "logits/chosen": -0.10276250541210175, + "logits/rejected": -1.3762527704238892, + "logps/chosen": -0.9235138893127441, + "logps/rejected": -15.89281940460205, + "loss": 0.9288, + "num_input_tokens_seen": 15592032, + "odds_ratio_loss": 9.059270858764648, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09235139191150665, + "rewards/margins": 1.4969305992126465, + "rewards/rejected": -1.5892821550369263, + "sft_loss": 0.02283237874507904, + "step": 11910 + }, + { + "epoch": 0.9272656553870089, + "grad_norm": 12.425597190856934, + "learning_rate": 6.498278843289324e-07, + "logits/chosen": -0.2992832064628601, + "logits/rejected": -1.7471001148223877, + "logps/chosen": -1.0092207193374634, + "logps/rejected": -16.291868209838867, + "loss": 1.0104, + "num_input_tokens_seen": 15604512, + "odds_ratio_loss": 9.880388259887695, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.10092207044363022, + "rewards/margins": 1.5282647609710693, + "rewards/rejected": -1.629186987876892, + "sft_loss": 0.022376861423254013, + "step": 11920 + }, + { + "epoch": 0.9280435628160248, + "grad_norm": 1.985065221786499, + "learning_rate": 6.360610850224741e-07, + "logits/chosen": -0.15450415015220642, + "logits/rejected": -1.873984694480896, + "logps/chosen": -1.154370903968811, + "logps/rejected": -19.407909393310547, + "loss": 1.1551, + "num_input_tokens_seen": 15620544, + "odds_ratio_loss": 11.513033866882324, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1154371052980423, + "rewards/margins": 1.825353980064392, + "rewards/rejected": -1.9407908916473389, + "sft_loss": 0.0037880162708461285, + "step": 11930 + }, + { + "epoch": 0.9288214702450408, + "grad_norm": 1.0911420583724976, + "learning_rate": 6.224397990761105e-07, + "logits/chosen": -0.08118706941604614, + "logits/rejected": -1.5405961275100708, + "logps/chosen": -0.827751636505127, + "logps/rejected": -11.274827003479004, + "loss": 0.8369, + "num_input_tokens_seen": 15638288, + "odds_ratio_loss": 8.341136932373047, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.08277516067028046, + "rewards/margins": 1.0447075366973877, + "rewards/rejected": -1.1274826526641846, + "sft_loss": 0.002821421716362238, + "step": 11940 + }, + { + "epoch": 0.9295993776740568, + "grad_norm": 1.6755629777908325, + "learning_rate": 6.089641078428188e-07, + "logits/chosen": -0.1963869035243988, + "logits/rejected": -1.457040548324585, + "logps/chosen": -0.9162726402282715, + "logps/rejected": -11.366378784179688, + "loss": 0.935, + "num_input_tokens_seen": 15650016, + "odds_ratio_loss": 9.271358489990234, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.09162726998329163, + "rewards/margins": 1.0450108051300049, + "rewards/rejected": -1.1366379261016846, + "sft_loss": 0.007887369021773338, + "step": 11950 + }, + { + "epoch": 0.9303772851030727, + "grad_norm": 4.658792972564697, + "learning_rate": 5.956340918060188e-07, + "logits/chosen": -0.6142601370811462, + "logits/rejected": -1.1708184480667114, + "logps/chosen": -0.6964209079742432, + "logps/rejected": -22.007936477661133, + "loss": 0.7011, + "num_input_tokens_seen": 15664480, + "odds_ratio_loss": 6.959555625915527, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.0696420893073082, + "rewards/margins": 2.1311516761779785, + "rewards/rejected": -2.200793743133545, + "sft_loss": 0.005143972113728523, + "step": 11960 + }, + { + "epoch": 0.9311551925320887, + "grad_norm": 2.936549425125122, + "learning_rate": 5.824498305790904e-07, + "logits/chosen": -0.06384022533893585, + "logits/rejected": -1.625780463218689, + "logps/chosen": -0.9296265840530396, + "logps/rejected": -14.322857856750488, + "loss": 0.9359, + "num_input_tokens_seen": 15678096, + "odds_ratio_loss": 9.310887336730957, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09296265989542007, + "rewards/margins": 1.3393231630325317, + "rewards/rejected": -1.4322856664657593, + "sft_loss": 0.004858965985476971, + "step": 11970 + }, + { + "epoch": 0.9319330999611046, + "grad_norm": 2.577427387237549, + "learning_rate": 5.694114029048847e-07, + "logits/chosen": -0.2968783378601074, + "logits/rejected": -1.3519415855407715, + "logps/chosen": -0.7401311993598938, + "logps/rejected": -12.539484977722168, + "loss": 0.7642, + "num_input_tokens_seen": 15687184, + "odds_ratio_loss": 7.474023342132568, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.0740131139755249, + "rewards/margins": 1.1799354553222656, + "rewards/rejected": -1.253948450088501, + "sft_loss": 0.016818318516016006, + "step": 11980 + }, + { + "epoch": 0.9327110073901206, + "grad_norm": 1.4747344255447388, + "learning_rate": 5.565188866552718e-07, + "logits/chosen": -0.20611505210399628, + "logits/rejected": -1.4000139236450195, + "logps/chosen": -1.1651408672332764, + "logps/rejected": -16.422460556030273, + "loss": 1.1732, + "num_input_tokens_seen": 15703504, + "odds_ratio_loss": 10.402894973754883, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.11651410162448883, + "rewards/margins": 1.5257320404052734, + "rewards/rejected": -1.6422462463378906, + "sft_loss": 0.13292676210403442, + "step": 11990 + }, + { + "epoch": 0.9334889148191365, + "grad_norm": 3.6748807430267334, + "learning_rate": 5.437723588306659e-07, + "logits/chosen": -0.31006914377212524, + "logits/rejected": -1.5102678537368774, + "logps/chosen": -1.2047278881072998, + "logps/rejected": -19.45448875427246, + "loss": 1.2573, + "num_input_tokens_seen": 15720608, + "odds_ratio_loss": 10.378247261047363, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.12047278881072998, + "rewards/margins": 1.8249757289886475, + "rewards/rejected": -1.945448875427246, + "sft_loss": 0.2194538116455078, + "step": 12000 + }, + { + "epoch": 0.9342668222481525, + "grad_norm": 2.386253833770752, + "learning_rate": 5.311718955595651e-07, + "logits/chosen": -0.2048005610704422, + "logits/rejected": -1.4392439126968384, + "logps/chosen": -0.904831051826477, + "logps/rejected": -10.53231143951416, + "loss": 0.9168, + "num_input_tokens_seen": 15734576, + "odds_ratio_loss": 9.017570495605469, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.09048312157392502, + "rewards/margins": 0.9627480506896973, + "rewards/rejected": -1.053231120109558, + "sft_loss": 0.015087531879544258, + "step": 12010 + }, + { + "epoch": 0.9350447296771685, + "grad_norm": 1.5931123495101929, + "learning_rate": 5.187175720980957e-07, + "logits/chosen": -0.21956145763397217, + "logits/rejected": -1.9383878707885742, + "logps/chosen": -0.6608736515045166, + "logps/rejected": -16.625524520874023, + "loss": 0.6628, + "num_input_tokens_seen": 15750064, + "odds_ratio_loss": 6.601613521575928, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.06608735024929047, + "rewards/margins": 1.5964651107788086, + "rewards/rejected": -1.6625524759292603, + "sft_loss": 0.0026680524460971355, + "step": 12020 + }, + { + "epoch": 0.9358226371061844, + "grad_norm": 2.137845754623413, + "learning_rate": 5.064094628295741e-07, + "logits/chosen": -0.4625570774078369, + "logits/rejected": -1.1971545219421387, + "logps/chosen": -1.0364329814910889, + "logps/rejected": -17.639053344726562, + "loss": 1.0443, + "num_input_tokens_seen": 15765904, + "odds_ratio_loss": 10.371828079223633, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.10364329814910889, + "rewards/margins": 1.660262107849121, + "rewards/rejected": -1.7639052867889404, + "sft_loss": 0.007115009240806103, + "step": 12030 + }, + { + "epoch": 0.9366005445352004, + "grad_norm": 4.474743843078613, + "learning_rate": 4.942476412640479e-07, + "logits/chosen": -0.4454718232154846, + "logits/rejected": -1.316896915435791, + "logps/chosen": -0.9748123288154602, + "logps/rejected": -11.002370834350586, + "loss": 0.9946, + "num_input_tokens_seen": 15780736, + "odds_ratio_loss": 9.50248908996582, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.09748123586177826, + "rewards/margins": 1.002755880355835, + "rewards/rejected": -1.1002370119094849, + "sft_loss": 0.04437413066625595, + "step": 12040 + }, + { + "epoch": 0.9373784519642162, + "grad_norm": 1.7154042720794678, + "learning_rate": 4.822321800378643e-07, + "logits/chosen": -0.22523906826972961, + "logits/rejected": -2.0052857398986816, + "logps/chosen": -0.9130438566207886, + "logps/rejected": -11.436153411865234, + "loss": 0.9204, + "num_input_tokens_seen": 15794608, + "odds_ratio_loss": 9.132055282592773, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09130439162254333, + "rewards/margins": 1.052310824394226, + "rewards/rejected": -1.1436152458190918, + "sft_loss": 0.007221040781587362, + "step": 12050 + }, + { + "epoch": 0.9381563593932322, + "grad_norm": 2.577000617980957, + "learning_rate": 4.7036315091323337e-07, + "logits/chosen": -0.4465862810611725, + "logits/rejected": -1.540804386138916, + "logps/chosen": -1.045684814453125, + "logps/rejected": -16.930381774902344, + "loss": 1.0583, + "num_input_tokens_seen": 15806912, + "odds_ratio_loss": 9.347288131713867, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.1045684814453125, + "rewards/margins": 1.5884698629379272, + "rewards/rejected": -1.6930382251739502, + "sft_loss": 0.12357378005981445, + "step": 12060 + }, + { + "epoch": 0.9389342668222481, + "grad_norm": 311.5629577636719, + "learning_rate": 4.586406247778008e-07, + "logits/chosen": -0.02665615640580654, + "logits/rejected": -1.868891716003418, + "logps/chosen": -1.5359704494476318, + "logps/rejected": -16.84640121459961, + "loss": 1.5365, + "num_input_tokens_seen": 15822048, + "odds_ratio_loss": 15.245460510253906, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.15359702706336975, + "rewards/margins": 1.5310429334640503, + "rewards/rejected": -1.6846399307250977, + "sft_loss": 0.011904889717698097, + "step": 12070 + }, + { + "epoch": 0.9397121742512641, + "grad_norm": 1.5435036420822144, + "learning_rate": 4.4706467164422894e-07, + "logits/chosen": -0.10313475131988525, + "logits/rejected": -1.3765543699264526, + "logps/chosen": -0.7199575901031494, + "logps/rejected": -14.565961837768555, + "loss": 0.7322, + "num_input_tokens_seen": 15838416, + "odds_ratio_loss": 7.288201332092285, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.07199576497077942, + "rewards/margins": 1.3846005201339722, + "rewards/rejected": -1.4565962553024292, + "sft_loss": 0.0034093856811523438, + "step": 12080 + }, + { + "epoch": 0.94049008168028, + "grad_norm": 2.868589401245117, + "learning_rate": 4.356353606497693e-07, + "logits/chosen": -0.037973783910274506, + "logits/rejected": -1.7042020559310913, + "logps/chosen": -0.9549576640129089, + "logps/rejected": -8.380606651306152, + "loss": 0.9799, + "num_input_tokens_seen": 15854096, + "odds_ratio_loss": 9.75303840637207, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.09549576044082642, + "rewards/margins": 0.7425647974014282, + "rewards/rejected": -0.8380606770515442, + "sft_loss": 0.00460343761369586, + "step": 12090 + }, + { + "epoch": 0.941267989109296, + "grad_norm": 4.285772323608398, + "learning_rate": 4.2435276005586e-07, + "logits/chosen": -0.04987701028585434, + "logits/rejected": -1.9801149368286133, + "logps/chosen": -0.9888293147087097, + "logps/rejected": -16.238651275634766, + "loss": 0.991, + "num_input_tokens_seen": 15867280, + "odds_ratio_loss": 9.881705284118652, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09888292849063873, + "rewards/margins": 1.524982213973999, + "rewards/rejected": -1.6238653659820557, + "sft_loss": 0.0028091927524656057, + "step": 12100 + }, + { + "epoch": 0.9420458965383119, + "grad_norm": 12.25771427154541, + "learning_rate": 4.132169372477124e-07, + "logits/chosen": -0.25258541107177734, + "logits/rejected": -0.953656792640686, + "logps/chosen": -0.962501049041748, + "logps/rejected": -15.844717025756836, + "loss": 0.9771, + "num_input_tokens_seen": 15879712, + "odds_ratio_loss": 9.643468856811523, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.09625010937452316, + "rewards/margins": 1.4882216453552246, + "rewards/rejected": -1.5844718217849731, + "sft_loss": 0.012737992219626904, + "step": 12110 + }, + { + "epoch": 0.9428238039673279, + "grad_norm": 3.2377405166625977, + "learning_rate": 4.0222795873390283e-07, + "logits/chosen": -0.19470559060573578, + "logits/rejected": -1.4375954866409302, + "logps/chosen": -0.7517939805984497, + "logps/rejected": -12.408689498901367, + "loss": 0.7652, + "num_input_tokens_seen": 15891616, + "odds_ratio_loss": 7.581107139587402, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.07517940551042557, + "rewards/margins": 1.1656897068023682, + "rewards/rejected": -1.240869164466858, + "sft_loss": 0.0070818522945046425, + "step": 12120 + }, + { + "epoch": 0.9436017113963439, + "grad_norm": 2.6857593059539795, + "learning_rate": 3.9138589014598717e-07, + "logits/chosen": -0.07612824440002441, + "logits/rejected": -1.4703692197799683, + "logps/chosen": -0.9121390581130981, + "logps/rejected": -5.44936990737915, + "loss": 0.9324, + "num_input_tokens_seen": 15904240, + "odds_ratio_loss": 9.259724617004395, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.0912138968706131, + "rewards/margins": 0.45372313261032104, + "rewards/rejected": -0.544937014579773, + "sft_loss": 0.006381546147167683, + "step": 12130 + }, + { + "epoch": 0.9443796188253598, + "grad_norm": 2.4428467750549316, + "learning_rate": 3.806907962381007e-07, + "logits/chosen": -0.19738474488258362, + "logits/rejected": -1.2523488998413086, + "logps/chosen": -0.9463440775871277, + "logps/rejected": -10.515398025512695, + "loss": 0.9589, + "num_input_tokens_seen": 15915312, + "odds_ratio_loss": 9.487909317016602, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09463441371917725, + "rewards/margins": 0.9569055438041687, + "rewards/rejected": -1.0515398979187012, + "sft_loss": 0.010065768845379353, + "step": 12140 + }, + { + "epoch": 0.9451575262543758, + "grad_norm": 2.169193983078003, + "learning_rate": 3.701427408865782e-07, + "logits/chosen": -0.24170875549316406, + "logits/rejected": -1.6161463260650635, + "logps/chosen": -0.7222901582717896, + "logps/rejected": -17.738393783569336, + "loss": 0.7232, + "num_input_tokens_seen": 15929984, + "odds_ratio_loss": 7.122027397155762, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.07222901284694672, + "rewards/margins": 1.7016105651855469, + "rewards/rejected": -1.7738393545150757, + "sft_loss": 0.010986115783452988, + "step": 12150 + }, + { + "epoch": 0.9459354336833917, + "grad_norm": 1.722773790359497, + "learning_rate": 3.597417870895653e-07, + "logits/chosen": -0.008847051300108433, + "logits/rejected": -1.943717360496521, + "logps/chosen": -0.959466814994812, + "logps/rejected": -19.27611541748047, + "loss": 0.9622, + "num_input_tokens_seen": 15945424, + "odds_ratio_loss": 9.529878616333008, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09594668447971344, + "rewards/margins": 1.831664800643921, + "rewards/rejected": -1.9276115894317627, + "sft_loss": 0.009181035682559013, + "step": 12160 + }, + { + "epoch": 0.9467133411124076, + "grad_norm": 1.7755272388458252, + "learning_rate": 3.494879969666437e-07, + "logits/chosen": -0.3166256844997406, + "logits/rejected": -1.0808227062225342, + "logps/chosen": -0.9705327749252319, + "logps/rejected": -7.702474117279053, + "loss": 0.9803, + "num_input_tokens_seen": 15956768, + "odds_ratio_loss": 9.704435348510742, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.09705328941345215, + "rewards/margins": 0.6731940507888794, + "rewards/rejected": -0.7702473998069763, + "sft_loss": 0.009903199970722198, + "step": 12170 + }, + { + "epoch": 0.9474912485414235, + "grad_norm": 2.0175669193267822, + "learning_rate": 3.393814317584648e-07, + "logits/chosen": -0.19483661651611328, + "logits/rejected": -1.8273557424545288, + "logps/chosen": -1.0359901189804077, + "logps/rejected": -13.904330253601074, + "loss": 1.0365, + "num_input_tokens_seen": 15969616, + "odds_ratio_loss": 10.292900085449219, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.10359902679920197, + "rewards/margins": 1.2868340015411377, + "rewards/rejected": -1.3904330730438232, + "sft_loss": 0.00725086173042655, + "step": 12180 + }, + { + "epoch": 0.9482691559704395, + "grad_norm": 3.474292039871216, + "learning_rate": 3.294221518263779e-07, + "logits/chosen": -0.24808935821056366, + "logits/rejected": -1.5131680965423584, + "logps/chosen": -0.8394571542739868, + "logps/rejected": -24.768207550048828, + "loss": 0.841, + "num_input_tokens_seen": 15983360, + "odds_ratio_loss": 8.330411911010742, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.08394570648670197, + "rewards/margins": 2.3928751945495605, + "rewards/rejected": -2.476820945739746, + "sft_loss": 0.007986358366906643, + "step": 12190 + }, + { + "epoch": 0.9490470633994554, + "grad_norm": 2.0075721740722656, + "learning_rate": 3.196102166520776e-07, + "logits/chosen": -0.4564247131347656, + "logits/rejected": -1.2702101469039917, + "logps/chosen": -1.1077969074249268, + "logps/rejected": -15.049591064453125, + "loss": 1.1184, + "num_input_tokens_seen": 15995120, + "odds_ratio_loss": 10.883405685424805, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.11077970266342163, + "rewards/margins": 1.394179344177246, + "rewards/rejected": -1.5049591064453125, + "sft_loss": 0.030080148950219154, + "step": 12200 + }, + { + "epoch": 0.9498249708284714, + "grad_norm": 4.91524076461792, + "learning_rate": 3.0994568483723453e-07, + "logits/chosen": -0.18206897377967834, + "logits/rejected": -1.224856972694397, + "logps/chosen": -0.7585890889167786, + "logps/rejected": -9.87165641784668, + "loss": 0.7673, + "num_input_tokens_seen": 16005920, + "odds_ratio_loss": 7.561065673828125, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.07585891336202621, + "rewards/margins": 0.9113066792488098, + "rewards/rejected": -0.987165629863739, + "sft_loss": 0.011143623851239681, + "step": 12210 + }, + { + "epoch": 0.9506028782574873, + "grad_norm": 156.9337615966797, + "learning_rate": 3.004286141031626e-07, + "logits/chosen": -0.17164663970470428, + "logits/rejected": -1.5703425407409668, + "logps/chosen": -0.9372615814208984, + "logps/rejected": -18.172954559326172, + "loss": 0.9459, + "num_input_tokens_seen": 16019360, + "odds_ratio_loss": 9.393759727478027, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.09372617304325104, + "rewards/margins": 1.7235692739486694, + "rewards/rejected": -1.8172953128814697, + "sft_loss": 0.006557709537446499, + "step": 12220 + }, + { + "epoch": 0.9513807856865033, + "grad_norm": 6.203927993774414, + "learning_rate": 2.910590612904635e-07, + "logits/chosen": -0.4827880859375, + "logits/rejected": -0.7654441595077515, + "logps/chosen": -0.8061830401420593, + "logps/rejected": -6.185842037200928, + "loss": 0.834, + "num_input_tokens_seen": 16029104, + "odds_ratio_loss": 8.147764205932617, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.08061830699443817, + "rewards/margins": 0.5379658937454224, + "rewards/rejected": -0.6185842156410217, + "sft_loss": 0.01920284889638424, + "step": 12230 + }, + { + "epoch": 0.9521586931155193, + "grad_norm": 3.568031072616577, + "learning_rate": 2.81837082358688e-07, + "logits/chosen": -0.18624334037303925, + "logits/rejected": -1.3989797830581665, + "logps/chosen": -0.9238718748092651, + "logps/rejected": -12.488553047180176, + "loss": 0.9304, + "num_input_tokens_seen": 16043568, + "odds_ratio_loss": 9.237310409545898, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09238718450069427, + "rewards/margins": 1.156468152999878, + "rewards/rejected": -1.2488553524017334, + "sft_loss": 0.0066881850361824036, + "step": 12240 + }, + { + "epoch": 0.9529366005445352, + "grad_norm": 1.8548458814620972, + "learning_rate": 2.72762732386006e-07, + "logits/chosen": -0.2077573537826538, + "logits/rejected": -1.9637689590454102, + "logps/chosen": -0.9647920727729797, + "logps/rejected": -13.010641098022461, + "loss": 0.9775, + "num_input_tokens_seen": 16057312, + "odds_ratio_loss": 9.729558944702148, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.09647920727729797, + "rewards/margins": 1.2045848369598389, + "rewards/rejected": -1.3010642528533936, + "sft_loss": 0.004576006438583136, + "step": 12250 + }, + { + "epoch": 0.9537145079735512, + "grad_norm": 4.205636501312256, + "learning_rate": 2.63836065568876e-07, + "logits/chosen": -0.6094192266464233, + "logits/rejected": -1.2007038593292236, + "logps/chosen": -0.7853139042854309, + "logps/rejected": -14.346330642700195, + "loss": 0.8093, + "num_input_tokens_seen": 16068064, + "odds_ratio_loss": 7.98935079574585, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.07853139191865921, + "rewards/margins": 1.356101632118225, + "rewards/rejected": -1.4346331357955933, + "sft_loss": 0.01034087035804987, + "step": 12260 + }, + { + "epoch": 0.9544924154025671, + "grad_norm": 3.066114902496338, + "learning_rate": 2.5505713522171457e-07, + "logits/chosen": -0.52752685546875, + "logits/rejected": -1.322624921798706, + "logps/chosen": -1.0002782344818115, + "logps/rejected": -16.155460357666016, + "loss": 1.0088, + "num_input_tokens_seen": 16080032, + "odds_ratio_loss": 9.885408401489258, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.10002782195806503, + "rewards/margins": 1.515518307685852, + "rewards/rejected": -1.6155462265014648, + "sft_loss": 0.02022591605782509, + "step": 12270 + }, + { + "epoch": 0.9552703228315831, + "grad_norm": 9.273058891296387, + "learning_rate": 2.464259937765889e-07, + "logits/chosen": -0.6523706316947937, + "logits/rejected": -1.2843421697616577, + "logps/chosen": -0.9229806065559387, + "logps/rejected": -22.3568172454834, + "loss": 0.9232, + "num_input_tokens_seen": 16090400, + "odds_ratio_loss": 9.077737808227539, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09229806065559387, + "rewards/margins": 2.143383502960205, + "rewards/rejected": -2.2356815338134766, + "sft_loss": 0.015461449511349201, + "step": 12280 + }, + { + "epoch": 0.956048230260599, + "grad_norm": 17.13327980041504, + "learning_rate": 2.3794269278289982e-07, + "logits/chosen": -0.6235595345497131, + "logits/rejected": -0.8743928670883179, + "logps/chosen": -0.841866135597229, + "logps/rejected": -14.01153564453125, + "loss": 0.8555, + "num_input_tokens_seen": 16101152, + "odds_ratio_loss": 8.061131477355957, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.0841866135597229, + "rewards/margins": 1.3169668912887573, + "rewards/rejected": -1.4011536836624146, + "sft_loss": 0.04942449554800987, + "step": 12290 + }, + { + "epoch": 0.9568261376896149, + "grad_norm": 23.418697357177734, + "learning_rate": 2.2960728290706836e-07, + "logits/chosen": -0.31731849908828735, + "logits/rejected": -1.3659123182296753, + "logps/chosen": -1.1973960399627686, + "logps/rejected": -12.80005168914795, + "loss": 1.2045, + "num_input_tokens_seen": 16114400, + "odds_ratio_loss": 11.009976387023926, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.11973961442708969, + "rewards/margins": 1.160265564918518, + "rewards/rejected": -1.2800052165985107, + "sft_loss": 0.10345391929149628, + "step": 12300 + }, + { + "epoch": 0.9576040451186308, + "grad_norm": 8.194009780883789, + "learning_rate": 2.2141981393223876e-07, + "logits/chosen": -0.21181325614452362, + "logits/rejected": -1.5768486261367798, + "logps/chosen": -1.3449286222457886, + "logps/rejected": -16.770870208740234, + "loss": 1.3834, + "num_input_tokens_seen": 16124784, + "odds_ratio_loss": 11.494306564331055, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.1344928741455078, + "rewards/margins": 1.5425941944122314, + "rewards/rejected": -1.6770871877670288, + "sft_loss": 0.23394536972045898, + "step": 12310 + }, + { + "epoch": 0.9583819525476468, + "grad_norm": 4.417098522186279, + "learning_rate": 2.1338033475798158e-07, + "logits/chosen": -0.12259181588888168, + "logits/rejected": -1.6844972372055054, + "logps/chosen": -1.0650262832641602, + "logps/rejected": -13.395116806030273, + "loss": 1.0797, + "num_input_tokens_seen": 16137648, + "odds_ratio_loss": 10.725841522216797, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.10650262981653214, + "rewards/margins": 1.2330090999603271, + "rewards/rejected": -1.3395118713378906, + "sft_loss": 0.007141365669667721, + "step": 12320 + }, + { + "epoch": 0.9591598599766628, + "grad_norm": 1.7791497707366943, + "learning_rate": 2.0548889339999654e-07, + "logits/chosen": -0.3604108393192291, + "logits/rejected": -1.5325709581375122, + "logps/chosen": -0.8818775415420532, + "logps/rejected": -20.89768409729004, + "loss": 0.8836, + "num_input_tokens_seen": 16150128, + "odds_ratio_loss": 7.5479888916015625, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.08818776160478592, + "rewards/margins": 2.0015807151794434, + "rewards/rejected": -2.089768171310425, + "sft_loss": 0.12881183624267578, + "step": 12330 + }, + { + "epoch": 0.9599377674056787, + "grad_norm": 6.587460517883301, + "learning_rate": 1.9774553698983512e-07, + "logits/chosen": -0.38189247250556946, + "logits/rejected": -1.1007959842681885, + "logps/chosen": -0.7522581219673157, + "logps/rejected": -6.890204429626465, + "loss": 0.7582, + "num_input_tokens_seen": 16160560, + "odds_ratio_loss": 7.452940464019775, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.0752258226275444, + "rewards/margins": 0.6137946844100952, + "rewards/rejected": -0.6890205144882202, + "sft_loss": 0.01288747787475586, + "step": 12340 + }, + { + "epoch": 0.9607156748346947, + "grad_norm": 5.235318183898926, + "learning_rate": 1.9015031177460619e-07, + "logits/chosen": -0.1968858242034912, + "logits/rejected": -1.5090630054473877, + "logps/chosen": -0.9505993127822876, + "logps/rejected": -14.273775100708008, + "loss": 0.9609, + "num_input_tokens_seen": 16175728, + "odds_ratio_loss": 9.44192886352539, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09505993127822876, + "rewards/margins": 1.3323174715042114, + "rewards/rejected": -1.4273775815963745, + "sft_loss": 0.01673353649675846, + "step": 12350 + }, + { + "epoch": 0.9614935822637106, + "grad_norm": 6.162037372589111, + "learning_rate": 1.827032631167097e-07, + "logits/chosen": -0.5855585336685181, + "logits/rejected": -1.5549615621566772, + "logps/chosen": -0.9294301867485046, + "logps/rejected": -9.373213768005371, + "loss": 0.9568, + "num_input_tokens_seen": 16187920, + "odds_ratio_loss": 8.704675674438477, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.09294302016496658, + "rewards/margins": 0.8443782925605774, + "rewards/rejected": -0.9373214840888977, + "sft_loss": 0.08634259551763535, + "step": 12360 + }, + { + "epoch": 0.9622714896927266, + "grad_norm": 5.875050067901611, + "learning_rate": 1.7540443549356188e-07, + "logits/chosen": -0.6409750580787659, + "logits/rejected": -0.7209451794624329, + "logps/chosen": -0.8512948751449585, + "logps/rejected": -20.480974197387695, + "loss": 0.859, + "num_input_tokens_seen": 16201888, + "odds_ratio_loss": 8.250844955444336, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.0851295068860054, + "rewards/margins": 1.962967872619629, + "rewards/rejected": -2.0480971336364746, + "sft_loss": 0.03389885649085045, + "step": 12370 + }, + { + "epoch": 0.9630493971217425, + "grad_norm": 7.472117900848389, + "learning_rate": 1.6825387249733704e-07, + "logits/chosen": -0.42525428533554077, + "logits/rejected": -1.3136355876922607, + "logps/chosen": -0.6914771795272827, + "logps/rejected": -8.531336784362793, + "loss": 0.7037, + "num_input_tokens_seen": 16214576, + "odds_ratio_loss": 6.910098075866699, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.06914772093296051, + "rewards/margins": 0.7839860320091248, + "rewards/rejected": -0.8531337976455688, + "sft_loss": 0.012650936841964722, + "step": 12380 + }, + { + "epoch": 0.9638273045507585, + "grad_norm": 3.758302927017212, + "learning_rate": 1.6125161683468726e-07, + "logits/chosen": -0.42221713066101074, + "logits/rejected": -1.5183813571929932, + "logps/chosen": -1.3994024991989136, + "logps/rejected": -12.888708114624023, + "loss": 1.4037, + "num_input_tokens_seen": 16225616, + "odds_ratio_loss": 13.958964347839355, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.13994024693965912, + "rewards/margins": 1.148930549621582, + "rewards/rejected": -1.288870930671692, + "sft_loss": 0.007841033861041069, + "step": 12390 + }, + { + "epoch": 0.9646052119797744, + "grad_norm": 13.455365180969238, + "learning_rate": 1.5439771032651485e-07, + "logits/chosen": -0.23251929879188538, + "logits/rejected": -1.6611559391021729, + "logps/chosen": -0.786699652671814, + "logps/rejected": -14.888818740844727, + "loss": 0.8089, + "num_input_tokens_seen": 16238272, + "odds_ratio_loss": 7.606795310974121, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.07866998016834259, + "rewards/margins": 1.4102119207382202, + "rewards/rejected": -1.4888819456100464, + "sft_loss": 0.048230789601802826, + "step": 12400 + }, + { + "epoch": 0.9653831194087904, + "grad_norm": 8.305849075317383, + "learning_rate": 1.476921939076975e-07, + "logits/chosen": -0.09227391332387924, + "logits/rejected": -1.1034928560256958, + "logps/chosen": -0.9128230214118958, + "logps/rejected": -7.003422737121582, + "loss": 0.9533, + "num_input_tokens_seen": 16249104, + "odds_ratio_loss": 8.971957206726074, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.09128230065107346, + "rewards/margins": 0.6090599298477173, + "rewards/rejected": -0.7003422975540161, + "sft_loss": 0.0561177060008049, + "step": 12410 + }, + { + "epoch": 0.9661610268378062, + "grad_norm": 0.31519854068756104, + "learning_rate": 1.41135107626858e-07, + "logits/chosen": -0.36162978410720825, + "logits/rejected": -0.9066141843795776, + "logps/chosen": -0.8489063382148743, + "logps/rejected": -8.324593544006348, + "loss": 0.8506, + "num_input_tokens_seen": 16259392, + "odds_ratio_loss": 8.225701332092285, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.08489063382148743, + "rewards/margins": 0.7475687265396118, + "rewards/rejected": -0.8324593305587769, + "sft_loss": 0.028013313189148903, + "step": 12420 + }, + { + "epoch": 0.9669389342668222, + "grad_norm": 21.114967346191406, + "learning_rate": 1.3472649064612818e-07, + "logits/chosen": -0.39116179943084717, + "logits/rejected": -1.32956862449646, + "logps/chosen": -0.8328605890274048, + "logps/rejected": -11.167863845825195, + "loss": 0.8505, + "num_input_tokens_seen": 16270256, + "odds_ratio_loss": 8.265515327453613, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.08328606188297272, + "rewards/margins": 1.03350031375885, + "rewards/rejected": -1.116786241531372, + "sft_loss": 0.02390402927994728, + "step": 12430 + }, + { + "epoch": 0.9677168416958382, + "grad_norm": 2.498950481414795, + "learning_rate": 1.2846638124089928e-07, + "logits/chosen": -0.25835880637168884, + "logits/rejected": -1.3009707927703857, + "logps/chosen": -0.7393811345100403, + "logps/rejected": -14.841459274291992, + "loss": 0.7486, + "num_input_tokens_seen": 16280016, + "odds_ratio_loss": 7.436209678649902, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.07393811643123627, + "rewards/margins": 1.410207986831665, + "rewards/rejected": -1.484145998954773, + "sft_loss": 0.00495152547955513, + "step": 12440 + }, + { + "epoch": 0.9684947491248541, + "grad_norm": 43.50111770629883, + "learning_rate": 1.2235481679960803e-07, + "logits/chosen": -0.0818486213684082, + "logits/rejected": -1.5897998809814453, + "logps/chosen": -0.9484689831733704, + "logps/rejected": -16.3314208984375, + "loss": 0.9589, + "num_input_tokens_seen": 16295184, + "odds_ratio_loss": 9.431092262268066, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09484690427780151, + "rewards/margins": 1.538295030593872, + "rewards/rejected": -1.633142113685608, + "sft_loss": 0.015782522037625313, + "step": 12450 + }, + { + "epoch": 0.9692726565538701, + "grad_norm": 4.413215637207031, + "learning_rate": 1.1639183382350649e-07, + "logits/chosen": -0.6921803951263428, + "logits/rejected": -0.9575518369674683, + "logps/chosen": -1.122647762298584, + "logps/rejected": -12.000131607055664, + "loss": 1.1484, + "num_input_tokens_seen": 16307840, + "odds_ratio_loss": 10.92697811126709, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.11226477473974228, + "rewards/margins": 1.0877482891082764, + "rewards/rejected": -1.2000130414962769, + "sft_loss": 0.05570362135767937, + "step": 12460 + }, + { + "epoch": 0.970050563982886, + "grad_norm": 9.900873184204102, + "learning_rate": 1.1057746792644541e-07, + "logits/chosen": -0.11293037235736847, + "logits/rejected": -1.5653810501098633, + "logps/chosen": -0.8573525547981262, + "logps/rejected": -10.341009140014648, + "loss": 0.8724, + "num_input_tokens_seen": 16319312, + "odds_ratio_loss": 8.643301963806152, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.0857352465391159, + "rewards/margins": 0.9483655691146851, + "rewards/rejected": -1.0341007709503174, + "sft_loss": 0.0080681461840868, + "step": 12470 + }, + { + "epoch": 0.970828471411902, + "grad_norm": 2.6572036743164062, + "learning_rate": 1.0491175383466056e-07, + "logits/chosen": -0.30273956060409546, + "logits/rejected": -1.4701600074768066, + "logps/chosen": -0.9312847852706909, + "logps/rejected": -18.610532760620117, + "loss": 0.9361, + "num_input_tokens_seen": 16333312, + "odds_ratio_loss": 9.294637680053711, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.0931284949183464, + "rewards/margins": 1.7679250240325928, + "rewards/rejected": -1.861053228378296, + "sft_loss": 0.006683858577162027, + "step": 12480 + }, + { + "epoch": 0.9716063788409179, + "grad_norm": 1.9054896831512451, + "learning_rate": 9.939472538657013e-08, + "logits/chosen": -0.9018571972846985, + "logits/rejected": -0.8955874443054199, + "logps/chosen": -0.6079337000846863, + "logps/rejected": -12.151650428771973, + "loss": 0.6204, + "num_input_tokens_seen": 16346000, + "odds_ratio_loss": 6.025877952575684, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.060793377459049225, + "rewards/margins": 1.1543715000152588, + "rewards/rejected": -1.2151648998260498, + "sft_loss": 0.017820732668042183, + "step": 12490 + }, + { + "epoch": 0.9723842862699339, + "grad_norm": 5.731510639190674, + "learning_rate": 9.402641553256375e-08, + "logits/chosen": -0.10272973775863647, + "logits/rejected": -1.1582386493682861, + "logps/chosen": -0.7395635843276978, + "logps/rejected": -10.743902206420898, + "loss": 0.7431, + "num_input_tokens_seen": 16354496, + "odds_ratio_loss": 7.344037055969238, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.07395635545253754, + "rewards/margins": 1.0004339218139648, + "rewards/rejected": -1.0743902921676636, + "sft_loss": 0.008720135316252708, + "step": 12500 + }, + { + "epoch": 0.9731621936989499, + "grad_norm": 32.50389099121094, + "learning_rate": 8.880685633481656e-08, + "logits/chosen": -0.18928098678588867, + "logits/rejected": -1.7569280862808228, + "logps/chosen": -0.8919623494148254, + "logps/rejected": -20.336193084716797, + "loss": 0.895, + "num_input_tokens_seen": 16367104, + "odds_ratio_loss": 8.890974044799805, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.08919622749090195, + "rewards/margins": 1.9444230794906616, + "rewards/rejected": -2.0336194038391113, + "sft_loss": 0.0058658611960709095, + "step": 12510 + }, + { + "epoch": 0.9739401011279658, + "grad_norm": 2.3467681407928467, + "learning_rate": 8.373607896708657e-08, + "logits/chosen": -0.04171920567750931, + "logits/rejected": -1.8098809719085693, + "logps/chosen": -0.8330823183059692, + "logps/rejected": -14.04285717010498, + "loss": 0.8349, + "num_input_tokens_seen": 16378816, + "odds_ratio_loss": 8.255022048950195, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.08330823481082916, + "rewards/margins": 1.3209774494171143, + "rewards/rejected": -1.4042856693267822, + "sft_loss": 0.009358550421893597, + "step": 12520 + }, + { + "epoch": 0.9747180085569818, + "grad_norm": 1.3713639974594116, + "learning_rate": 7.881411371453707e-08, + "logits/chosen": -0.28759709000587463, + "logits/rejected": -1.3889890909194946, + "logps/chosen": -0.9286860227584839, + "logps/rejected": -7.614617347717285, + "loss": 0.9356, + "num_input_tokens_seen": 16390256, + "odds_ratio_loss": 9.244423866271973, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.0928686186671257, + "rewards/margins": 0.668593168258667, + "rewards/rejected": -0.7614617347717285, + "sft_loss": 0.01112511195242405, + "step": 12530 + }, + { + "epoch": 0.9754959159859977, + "grad_norm": 4.426767826080322, + "learning_rate": 7.404098997355335e-08, + "logits/chosen": -0.35824424028396606, + "logits/rejected": -1.33821439743042, + "logps/chosen": -0.8309882879257202, + "logps/rejected": -18.37192153930664, + "loss": 0.8335, + "num_input_tokens_seen": 16403888, + "odds_ratio_loss": 8.10359001159668, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.08309882879257202, + "rewards/margins": 1.7540934085845947, + "rewards/rejected": -1.8371922969818115, + "sft_loss": 0.023182952776551247, + "step": 12540 + }, + { + "epoch": 0.9762738234150136, + "grad_norm": 313.7215576171875, + "learning_rate": 6.94167362515652e-08, + "logits/chosen": -0.6350809335708618, + "logits/rejected": -1.0927356481552124, + "logps/chosen": -1.0203081369400024, + "logps/rejected": -13.360295295715332, + "loss": 1.0235, + "num_input_tokens_seen": 16419856, + "odds_ratio_loss": 9.871319770812988, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.10203079879283905, + "rewards/margins": 1.2339987754821777, + "rewards/rejected": -1.3360296487808228, + "sft_loss": 0.03635597229003906, + "step": 12550 + }, + { + "epoch": 0.9770517308440295, + "grad_norm": 1.7516701221466064, + "learning_rate": 6.494138016687468e-08, + "logits/chosen": 0.02967851236462593, + "logits/rejected": -1.5164343118667603, + "logps/chosen": -0.6879326105117798, + "logps/rejected": -13.3673677444458, + "loss": 0.6919, + "num_input_tokens_seen": 16438128, + "odds_ratio_loss": 6.892519474029541, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.06879327446222305, + "rewards/margins": 1.2679435014724731, + "rewards/rejected": -1.3367364406585693, + "sft_loss": 0.00268021272495389, + "step": 12560 + }, + { + "epoch": 0.9778296382730455, + "grad_norm": 10.79580307006836, + "learning_rate": 6.06149484485008e-08, + "logits/chosen": -0.2581590712070465, + "logits/rejected": -1.425004482269287, + "logps/chosen": -0.7572137117385864, + "logps/rejected": -18.632070541381836, + "loss": 0.7878, + "num_input_tokens_seen": 16449392, + "odds_ratio_loss": 7.764173984527588, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.0757213681936264, + "rewards/margins": 1.7874854803085327, + "rewards/rejected": -1.8632068634033203, + "sft_loss": 0.011387160047888756, + "step": 12570 + }, + { + "epoch": 0.9786075457020614, + "grad_norm": 1.9176056385040283, + "learning_rate": 5.643746693600738e-08, + "logits/chosen": -0.3222039043903351, + "logits/rejected": -1.4580246210098267, + "logps/chosen": -1.1318254470825195, + "logps/rejected": -13.271893501281738, + "loss": 1.1372, + "num_input_tokens_seen": 16460176, + "odds_ratio_loss": 11.175423622131348, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.11318254470825195, + "rewards/margins": 1.214006781578064, + "rewards/rejected": -1.327189326286316, + "sft_loss": 0.019621601328253746, + "step": 12580 + }, + { + "epoch": 0.9793854531310774, + "grad_norm": 3.187330961227417, + "learning_rate": 5.240896057935596e-08, + "logits/chosen": -0.09735438972711563, + "logits/rejected": -1.3586671352386475, + "logps/chosen": -0.9365214109420776, + "logps/rejected": -11.869119644165039, + "loss": 0.9705, + "num_input_tokens_seen": 16474832, + "odds_ratio_loss": 9.631131172180176, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.09365214407444, + "rewards/margins": 1.0932599306106567, + "rewards/rejected": -1.186911940574646, + "sft_loss": 0.007435616105794907, + "step": 12590 + }, + { + "epoch": 0.9801633605600933, + "grad_norm": 1.5250585079193115, + "learning_rate": 4.852945343876147e-08, + "logits/chosen": -0.2481372356414795, + "logits/rejected": -1.2218652963638306, + "logps/chosen": -1.1172990798950195, + "logps/rejected": -14.136247634887695, + "loss": 1.1277, + "num_input_tokens_seen": 16487808, + "odds_ratio_loss": 10.643720626831055, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.11172991991043091, + "rewards/margins": 1.3018949031829834, + "rewards/rejected": -1.413624882698059, + "sft_loss": 0.06337771564722061, + "step": 12600 + }, + { + "epoch": 0.9809412679891093, + "grad_norm": 5.009439945220947, + "learning_rate": 4.4798968684536816e-08, + "logits/chosen": -0.12432391941547394, + "logits/rejected": -1.9330546855926514, + "logps/chosen": -1.0320632457733154, + "logps/rejected": -15.102289199829102, + "loss": 1.0389, + "num_input_tokens_seen": 16500688, + "odds_ratio_loss": 10.34919261932373, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1032063215970993, + "rewards/margins": 1.407022476196289, + "rewards/rejected": -1.5102288722991943, + "sft_loss": 0.003973107319325209, + "step": 12610 + }, + { + "epoch": 0.9817191754181253, + "grad_norm": 1.481100082397461, + "learning_rate": 4.12175285969596e-08, + "logits/chosen": -0.18140652775764465, + "logits/rejected": -1.433404564857483, + "logps/chosen": -0.9879193305969238, + "logps/rejected": -12.899325370788574, + "loss": 1.0095, + "num_input_tokens_seen": 16513120, + "odds_ratio_loss": 9.938295364379883, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.0987919345498085, + "rewards/margins": 1.1911407709121704, + "rewards/rejected": -1.2899327278137207, + "sft_loss": 0.015678998082876205, + "step": 12620 + }, + { + "epoch": 0.9824970828471412, + "grad_norm": 3.2705838680267334, + "learning_rate": 3.7785154566141734e-08, + "logits/chosen": -0.16677150130271912, + "logits/rejected": -2.082714796066284, + "logps/chosen": -0.9549390077590942, + "logps/rejected": -22.509668350219727, + "loss": 0.955, + "num_input_tokens_seen": 16529152, + "odds_ratio_loss": 9.085559844970703, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09549389779567719, + "rewards/margins": 2.155473232269287, + "rewards/rejected": -2.250967025756836, + "sft_loss": 0.046428509056568146, + "step": 12630 + }, + { + "epoch": 0.9832749902761572, + "grad_norm": 1.5827522277832031, + "learning_rate": 3.450186709190173e-08, + "logits/chosen": 0.0006435498362407088, + "logits/rejected": -1.7825992107391357, + "logps/chosen": -1.3783544301986694, + "logps/rejected": -12.360357284545898, + "loss": 1.4183, + "num_input_tokens_seen": 16545968, + "odds_ratio_loss": 14.124194145202637, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.13783542811870575, + "rewards/margins": 1.0982002019882202, + "rewards/rejected": -1.236035704612732, + "sft_loss": 0.005850933492183685, + "step": 12640 + }, + { + "epoch": 0.9840528977051731, + "grad_norm": 2.583888053894043, + "learning_rate": 3.1367685783637027e-08, + "logits/chosen": -0.08986677974462509, + "logits/rejected": -1.441861629486084, + "logps/chosen": -0.8815041780471802, + "logps/rejected": -8.078205108642578, + "loss": 0.8927, + "num_input_tokens_seen": 16559760, + "odds_ratio_loss": 8.863988876342773, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.08815042674541473, + "rewards/margins": 0.7196700572967529, + "rewards/rejected": -0.8078204393386841, + "sft_loss": 0.006350231356918812, + "step": 12650 + }, + { + "epoch": 0.9848308051341891, + "grad_norm": 3.0049779415130615, + "learning_rate": 2.8382629360212964e-08, + "logits/chosen": -0.47108444571495056, + "logits/rejected": -1.0828173160552979, + "logps/chosen": -0.7092539072036743, + "logps/rejected": -14.784589767456055, + "loss": 0.7103, + "num_input_tokens_seen": 16573264, + "odds_ratio_loss": 6.922545433044434, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.07092537730932236, + "rewards/margins": 1.4075334072113037, + "rewards/rejected": -1.4784588813781738, + "sft_loss": 0.01802605763077736, + "step": 12660 + }, + { + "epoch": 0.9856087125632049, + "grad_norm": 3.0464465618133545, + "learning_rate": 2.5546715649843454e-08, + "logits/chosen": -0.06019148975610733, + "logits/rejected": -1.9300884008407593, + "logps/chosen": -0.8260527849197388, + "logps/rejected": -6.8024725914001465, + "loss": 0.8865, + "num_input_tokens_seen": 16584240, + "odds_ratio_loss": 8.806262016296387, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.0826052725315094, + "rewards/margins": 0.5976420044898987, + "rewards/rejected": -0.6802473664283752, + "sft_loss": 0.00587079580873251, + "step": 12670 + }, + { + "epoch": 0.9863866199922209, + "grad_norm": 2.0222275257110596, + "learning_rate": 2.2859961589996592e-08, + "logits/chosen": -0.03779103606939316, + "logits/rejected": -1.8276106119155884, + "logps/chosen": -0.968870997428894, + "logps/rejected": -21.92473030090332, + "loss": 0.9782, + "num_input_tokens_seen": 16599776, + "odds_ratio_loss": 9.445577621459961, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.09688709676265717, + "rewards/margins": 2.095586061477661, + "rewards/rejected": -2.1924729347229004, + "sft_loss": 0.03368552029132843, + "step": 12680 + }, + { + "epoch": 0.9871645274212368, + "grad_norm": 2.786588430404663, + "learning_rate": 2.0322383227278085e-08, + "logits/chosen": -0.07717380672693253, + "logits/rejected": -1.7762495279312134, + "logps/chosen": -0.8848506808280945, + "logps/rejected": -15.007204055786133, + "loss": 0.8853, + "num_input_tokens_seen": 16613616, + "odds_ratio_loss": 8.807829856872559, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.08848506212234497, + "rewards/margins": 1.4122353792190552, + "rewards/rejected": -1.5007203817367554, + "sft_loss": 0.004489428363740444, + "step": 12690 + }, + { + "epoch": 0.9879424348502528, + "grad_norm": 1.800603985786438, + "learning_rate": 1.7933995717347996e-08, + "logits/chosen": -0.2031760960817337, + "logits/rejected": -1.3554677963256836, + "logps/chosen": -0.7762188911437988, + "logps/rejected": -14.93109130859375, + "loss": 0.7783, + "num_input_tokens_seen": 16624656, + "odds_ratio_loss": 7.481186866760254, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.07762189209461212, + "rewards/margins": 1.4154871702194214, + "rewards/rejected": -1.4931089878082275, + "sft_loss": 0.03014567494392395, + "step": 12700 + }, + { + "epoch": 0.9887203422792687, + "grad_norm": 3.7643752098083496, + "learning_rate": 1.569481332482636e-08, + "logits/chosen": 0.05465736240148544, + "logits/rejected": -1.5168482065200806, + "logps/chosen": -0.9140115976333618, + "logps/rejected": -10.084342956542969, + "loss": 0.9228, + "num_input_tokens_seen": 16641136, + "odds_ratio_loss": 9.198801040649414, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09140116721391678, + "rewards/margins": 0.917033314704895, + "rewards/rejected": -1.0084344148635864, + "sft_loss": 0.002949314657598734, + "step": 12710 + }, + { + "epoch": 0.9894982497082847, + "grad_norm": 3.4115965366363525, + "learning_rate": 1.3604849423201616e-08, + "logits/chosen": -0.2617233991622925, + "logits/rejected": -1.460564136505127, + "logps/chosen": -0.8608863949775696, + "logps/rejected": -11.34762191772461, + "loss": 0.8719, + "num_input_tokens_seen": 16655472, + "odds_ratio_loss": 8.64155387878418, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.0860886350274086, + "rewards/margins": 1.0486735105514526, + "rewards/rejected": -1.1347620487213135, + "sft_loss": 0.007763334549963474, + "step": 12720 + }, + { + "epoch": 0.9902761571373007, + "grad_norm": 2.1883881092071533, + "learning_rate": 1.166411649476673e-08, + "logits/chosen": -0.4011611044406891, + "logits/rejected": -0.961927592754364, + "logps/chosen": -0.7738547921180725, + "logps/rejected": -7.4190521240234375, + "loss": 0.8047, + "num_input_tokens_seen": 16666992, + "odds_ratio_loss": 7.953970432281494, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.07738548517227173, + "rewards/margins": 0.664519727230072, + "rewards/rejected": -0.7419052124023438, + "sft_loss": 0.009307285770773888, + "step": 12730 + }, + { + "epoch": 0.9910540645663166, + "grad_norm": 4.376555919647217, + "learning_rate": 9.87262613052209e-09, + "logits/chosen": -0.19814147055149078, + "logits/rejected": -1.5912666320800781, + "logps/chosen": -0.7829992175102234, + "logps/rejected": -14.966654777526855, + "loss": 0.785, + "num_input_tokens_seen": 16684736, + "odds_ratio_loss": 7.817573547363281, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.07829991728067398, + "rewards/margins": 1.4183655977249146, + "rewards/rejected": -1.496665596961975, + "sft_loss": 0.003252812195569277, + "step": 12740 + }, + { + "epoch": 0.9918319719953326, + "grad_norm": 2.65453839302063, + "learning_rate": 8.230389030128293e-09, + "logits/chosen": -0.28969889879226685, + "logits/rejected": -1.453749179840088, + "logps/chosen": -0.9020107388496399, + "logps/rejected": -15.950787544250488, + "loss": 0.9037, + "num_input_tokens_seen": 16701296, + "odds_ratio_loss": 8.81907844543457, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09020107239484787, + "rewards/margins": 1.5048778057098389, + "rewards/rejected": -1.595078706741333, + "sft_loss": 0.02180342748761177, + "step": 12750 + }, + { + "epoch": 0.9926098794243485, + "grad_norm": 2.612891435623169, + "learning_rate": 6.7374150018256754e-09, + "logits/chosen": -0.2980986535549164, + "logits/rejected": -1.6294724941253662, + "logps/chosen": -0.9495716094970703, + "logps/rejected": -17.207256317138672, + "loss": 0.9526, + "num_input_tokens_seen": 16713664, + "odds_ratio_loss": 9.388467788696289, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09495715796947479, + "rewards/margins": 1.6257686614990234, + "rewards/rejected": -1.7207257747650146, + "sft_loss": 0.013775520026683807, + "step": 12760 + }, + { + "epoch": 0.9933877868533645, + "grad_norm": 2.1296916007995605, + "learning_rate": 5.39371296238711e-09, + "logits/chosen": -0.03332703188061714, + "logits/rejected": -1.5227935314178467, + "logps/chosen": -0.6612142324447632, + "logps/rejected": -12.962564468383789, + "loss": 0.6679, + "num_input_tokens_seen": 16727856, + "odds_ratio_loss": 6.621134281158447, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.0661214217543602, + "rewards/margins": 1.230135202407837, + "rewards/rejected": -1.296256422996521, + "sft_loss": 0.005786126479506493, + "step": 12770 + }, + { + "epoch": 0.9941656942823804, + "grad_norm": 1.1724915504455566, + "learning_rate": 4.199290937056955e-09, + "logits/chosen": -0.24994008243083954, + "logits/rejected": -1.2555500268936157, + "logps/chosen": -1.1840388774871826, + "logps/rejected": -13.798118591308594, + "loss": 1.2418, + "num_input_tokens_seen": 16740848, + "odds_ratio_loss": 11.353900909423828, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.11840388923883438, + "rewards/margins": 1.2614080905914307, + "rewards/rejected": -1.3798120021820068, + "sft_loss": 0.10643362998962402, + "step": 12780 + }, + { + "epoch": 0.9949436017113964, + "grad_norm": 1.7221916913986206, + "learning_rate": 3.154156059512192e-09, + "logits/chosen": -0.31520482897758484, + "logits/rejected": -1.4942389726638794, + "logps/chosen": -0.9685817956924438, + "logps/rejected": -15.389326095581055, + "loss": 0.9696, + "num_input_tokens_seen": 16754048, + "odds_ratio_loss": 9.584997177124023, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.0968581885099411, + "rewards/margins": 1.4420745372772217, + "rewards/rejected": -1.5389328002929688, + "sft_loss": 0.011118141934275627, + "step": 12790 + }, + { + "epoch": 0.9957215091404122, + "grad_norm": 1.8254626989364624, + "learning_rate": 2.258314571804143e-09, + "logits/chosen": -0.04389285668730736, + "logits/rejected": -2.0369677543640137, + "logps/chosen": -0.956657886505127, + "logps/rejected": -13.710368156433105, + "loss": 0.98, + "num_input_tokens_seen": 16769024, + "odds_ratio_loss": 9.739141464233398, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.09566578269004822, + "rewards/margins": 1.2753711938858032, + "rewards/rejected": -1.3710368871688843, + "sft_loss": 0.006069186609238386, + "step": 12800 + }, + { + "epoch": 0.9964994165694282, + "grad_norm": 1.4335428476333618, + "learning_rate": 1.511771824336261e-09, + "logits/chosen": -0.19139555096626282, + "logits/rejected": -1.8160762786865234, + "logps/chosen": -0.9348017573356628, + "logps/rejected": -13.172691345214844, + "loss": 0.9398, + "num_input_tokens_seen": 16784912, + "odds_ratio_loss": 9.348949432373047, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.093480184674263, + "rewards/margins": 1.2237889766693115, + "rewards/rejected": -1.3172690868377686, + "sft_loss": 0.00491794990375638, + "step": 12810 + }, + { + "epoch": 0.9972773239984442, + "grad_norm": 2.756385326385498, + "learning_rate": 9.145322758308261e-10, + "logits/chosen": -0.37790343165397644, + "logits/rejected": -1.0679850578308105, + "logps/chosen": -1.01203191280365, + "logps/rejected": -18.016756057739258, + "loss": 1.0175, + "num_input_tokens_seen": 16799360, + "odds_ratio_loss": 8.870290756225586, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.10120320320129395, + "rewards/margins": 1.7004724740982056, + "rewards/rejected": -1.80167555809021, + "sft_loss": 0.13049033284187317, + "step": 12820 + }, + { + "epoch": 0.9980552314274601, + "grad_norm": 4.3858795166015625, + "learning_rate": 4.665994932956385e-10, + "logits/chosen": -0.1261058747768402, + "logits/rejected": -1.3934625387191772, + "logps/chosen": -0.8405144810676575, + "logps/rejected": -18.738025665283203, + "loss": 0.8406, + "num_input_tokens_seen": 16817520, + "odds_ratio_loss": 8.351066589355469, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.08405144512653351, + "rewards/margins": 1.7897508144378662, + "rewards/rejected": -1.8738024234771729, + "sft_loss": 0.005541075021028519, + "step": 12830 + }, + { + "epoch": 0.9988331388564761, + "grad_norm": 107.50608825683594, + "learning_rate": 1.679761519990386e-10, + "logits/chosen": -0.2797563970088959, + "logits/rejected": -1.545263648033142, + "logps/chosen": -1.2055482864379883, + "logps/rejected": -11.273192405700684, + "loss": 1.2627, + "num_input_tokens_seen": 16830640, + "odds_ratio_loss": 12.5514554977417, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.12055482715368271, + "rewards/margins": 1.00676429271698, + "rewards/rejected": -1.1273192167282104, + "sft_loss": 0.007532855961471796, + "step": 12840 + }, + { + "epoch": 0.999611046285492, + "grad_norm": 1.6087356805801392, + "learning_rate": 1.8664035467130802e-11, + "logits/chosen": -0.23476950824260712, + "logits/rejected": -1.456777811050415, + "logps/chosen": -0.7568179965019226, + "logps/rejected": -14.984893798828125, + "loss": 0.758, + "num_input_tokens_seen": 16844576, + "odds_ratio_loss": 7.390267848968506, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.07568180561065674, + "rewards/margins": 1.4228076934814453, + "rewards/rejected": -1.4984893798828125, + "sft_loss": 0.018998144194483757, + "step": 12850 + }, + { + "epoch": 1.0, + "num_input_tokens_seen": 16848032, + "step": 12855, + "total_flos": 7.671391583498404e+17, + "train_loss": 0.9753038262820161, + "train_runtime": 20926.3402, + "train_samples_per_second": 0.614, + "train_steps_per_second": 0.614 + } + ], + "logging_steps": 10, + "max_steps": 12855, + "num_input_tokens_seen": 16848032, + "num_train_epochs": 1, + "save_steps": 1000, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": true + }, + "attributes": {} + } + }, + "total_flos": 7.671391583498404e+17, + "train_batch_size": 1, + "trial_name": null, + "trial_params": null +}