{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 1.0,
  "eval_steps": 500,
  "global_step": 12855,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.0007779074290159471,
      "grad_norm": 5.248451232910156,
      "learning_rate": 4.9999925343886e-05,
      "logits/chosen": -0.08551748096942902,
      "logits/rejected": -1.3160173892974854,
      "logps/chosen": -0.9243558645248413,
      "logps/rejected": -0.9934996366500854,
      "loss": 0.9946,
      "num_input_tokens_seen": 12288,
      "odds_ratio_loss": 9.867249488830566,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.09243559837341309,
      "rewards/margins": 0.00691437441855669,
      "rewards/rejected": -0.0993499681353569,
      "sft_loss": 0.007888625375926495,
      "step": 10
    },
    {
      "epoch": 0.0015558148580318942,
      "grad_norm": 21.039575576782227,
      "learning_rate": 4.999970137598988e-05,
      "logits/chosen": -0.35449445247650146,
      "logits/rejected": -0.5382658839225769,
      "logps/chosen": -1.1569985151290894,
      "logps/rejected": -1.1496927738189697,
      "loss": 1.2427,
      "num_input_tokens_seen": 26128,
      "odds_ratio_loss": 11.52686882019043,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.11569985002279282,
      "rewards/margins": -0.000730576750356704,
      "rewards/rejected": -0.11496929079294205,
      "sft_loss": 0.08998985588550568,
      "step": 20
    },
    {
      "epoch": 0.002333722287047841,
      "grad_norm": 9.672579765319824,
      "learning_rate": 4.999932809764929e-05,
      "logits/chosen": -0.048610180616378784,
      "logits/rejected": -1.061124563217163,
      "logps/chosen": -1.2201629877090454,
      "logps/rejected": -1.2427245378494263,
      "loss": 1.2893,
      "num_input_tokens_seen": 41488,
      "odds_ratio_loss": 11.61978530883789,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.1220162883400917,
      "rewards/margins": 0.0022561654914170504,
      "rewards/rejected": -0.12427246570587158,
      "sft_loss": 0.12736229598522186,
      "step": 30
    },
    {
      "epoch": 0.0031116297160637884,
      "grad_norm": 3.708526372909546,
      "learning_rate": 4.999880551109363e-05,
      "logits/chosen": -0.2156565636396408,
      "logits/rejected": -1.1819674968719482,
      "logps/chosen": -1.1763250827789307,
      "logps/rejected": -1.1008272171020508,
      "loss": 1.2549,
      "num_input_tokens_seen": 53664,
      "odds_ratio_loss": 12.200116157531738,
      "rewards/accuracies": 0.20000000298023224,
      "rewards/chosen": -0.11763250827789307,
      "rewards/margins": -0.007549784146249294,
      "rewards/rejected": -0.1100827306509018,
      "sft_loss": 0.03486320376396179,
      "step": 40
    },
    {
      "epoch": 0.0038895371450797353,
      "grad_norm": 4.0965495109558105,
      "learning_rate": 4.999813361944403e-05,
      "logits/chosen": -0.061433859169483185,
      "logits/rejected": -0.9057363271713257,
      "logps/chosen": -0.7127934694290161,
      "logps/rejected": -2.0076801776885986,
      "loss": 0.7365,
      "num_input_tokens_seen": 62688,
      "odds_ratio_loss": 7.168131351470947,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.07127933204174042,
      "rewards/margins": 0.12948869168758392,
      "rewards/rejected": -0.20076803863048553,
      "sft_loss": 0.01973068341612816,
      "step": 50
    },
    {
      "epoch": 0.004667444574095682,
      "grad_norm": 2.570751190185547,
      "learning_rate": 4.9997312426713375e-05,
      "logits/chosen": -0.030793923884630203,
      "logits/rejected": -0.8716487884521484,
      "logps/chosen": -0.8243891596794128,
      "logps/rejected": -1.0122798681259155,
      "loss": 0.8873,
      "num_input_tokens_seen": 76224,
      "odds_ratio_loss": 8.698861122131348,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.08243891596794128,
      "rewards/margins": 0.018789071589708328,
      "rewards/rejected": -0.10122797638177872,
      "sft_loss": 0.017441704869270325,
      "step": 60
    },
    {
      "epoch": 0.0054453520031116295,
      "grad_norm": 4.000652313232422,
      "learning_rate": 4.9996341937806215e-05,
      "logits/chosen": -0.3607181906700134,
      "logits/rejected": -0.8948682546615601,
      "logps/chosen": -1.1096030473709106,
      "logps/rejected": -1.1755973100662231,
      "loss": 1.1815,
      "num_input_tokens_seen": 85600,
      "odds_ratio_loss": 9.79625415802002,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.11096030473709106,
      "rewards/margins": 0.006599433720111847,
      "rewards/rejected": -0.11755974590778351,
      "sft_loss": 0.2019163817167282,
      "step": 70
    },
    {
      "epoch": 0.006223259432127577,
      "grad_norm": 48.832618713378906,
      "learning_rate": 4.999522215851879e-05,
      "logits/chosen": 0.17318834364414215,
      "logits/rejected": -1.169816493988037,
      "logps/chosen": -1.2467583417892456,
      "logps/rejected": -1.9135162830352783,
      "loss": 1.3283,
      "num_input_tokens_seen": 98944,
      "odds_ratio_loss": 11.729827880859375,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.12467584758996964,
      "rewards/margins": 0.06667578965425491,
      "rewards/rejected": -0.19135163724422455,
      "sft_loss": 0.15530584752559662,
      "step": 80
    },
    {
      "epoch": 0.007001166861143524,
      "grad_norm": 2.759312629699707,
      "learning_rate": 4.999395309553898e-05,
      "logits/chosen": 0.00044447611435316503,
      "logits/rejected": -1.0682010650634766,
      "logps/chosen": -0.9102838635444641,
      "logps/rejected": -1.688788652420044,
      "loss": 0.9855,
      "num_input_tokens_seen": 112032,
      "odds_ratio_loss": 9.6449613571167,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.09102839231491089,
      "rewards/margins": 0.07785046845674515,
      "rewards/rejected": -0.16887886822223663,
      "sft_loss": 0.02097008004784584,
      "step": 90
    },
    {
      "epoch": 0.007779074290159471,
      "grad_norm": 2.1516456604003906,
      "learning_rate": 4.999253475644623e-05,
      "logits/chosen": -0.36004287004470825,
      "logits/rejected": -0.6734232902526855,
      "logps/chosen": -0.8154226541519165,
      "logps/rejected": -1.0999035835266113,
      "loss": 0.8797,
      "num_input_tokens_seen": 123424,
      "odds_ratio_loss": 8.56216049194336,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.08154226839542389,
      "rewards/margins": 0.028448080644011497,
      "rewards/rejected": -0.10999034345149994,
      "sft_loss": 0.02346923016011715,
      "step": 100
    },
    {
      "epoch": 0.008556981719175419,
      "grad_norm": 1.4805128574371338,
      "learning_rate": 4.999096714971156e-05,
      "logits/chosen": -0.04372026026248932,
      "logits/rejected": -0.735072910785675,
      "logps/chosen": -1.1332687139511108,
      "logps/rejected": -1.4848557710647583,
      "loss": 1.2075,
      "num_input_tokens_seen": 138400,
      "odds_ratio_loss": 11.999357223510742,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": -0.11332686990499496,
      "rewards/margins": 0.03515869751572609,
      "rewards/rejected": -0.14848557114601135,
      "sft_loss": 0.0076140896417200565,
      "step": 110
    },
    {
      "epoch": 0.009334889148191364,
      "grad_norm": 3.1642253398895264,
      "learning_rate": 4.99892502846975e-05,
      "logits/chosen": 0.1289406716823578,
      "logits/rejected": -0.9220048785209656,
      "logps/chosen": -0.7731643319129944,
      "logps/rejected": -1.410657525062561,
      "loss": 0.8176,
      "num_input_tokens_seen": 153360,
      "odds_ratio_loss": 8.046560287475586,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.07731644809246063,
      "rewards/margins": 0.06374931335449219,
      "rewards/rejected": -0.14106574654579163,
      "sft_loss": 0.01298043318092823,
      "step": 120
    },
    {
      "epoch": 0.010112796577207312,
      "grad_norm": 2.5500879287719727,
      "learning_rate": 4.9987384171657994e-05,
      "logits/chosen": 0.19079621136188507,
      "logits/rejected": -1.2353627681732178,
      "logps/chosen": -0.9447005391120911,
      "logps/rejected": -1.266804575920105,
      "loss": 1.0001,
      "num_input_tokens_seen": 165072,
      "odds_ratio_loss": 9.937886238098145,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.0944700613617897,
      "rewards/margins": 0.032210398465394974,
      "rewards/rejected": -0.12668046355247498,
      "sft_loss": 0.006324116140604019,
      "step": 130
    },
    {
      "epoch": 0.010890704006223259,
      "grad_norm": 7.7927422523498535,
      "learning_rate": 4.9985368821738385e-05,
      "logits/chosen": -0.07595299184322357,
      "logits/rejected": -0.8760563731193542,
      "logps/chosen": -0.8572665452957153,
      "logps/rejected": -2.1462793350219727,
      "loss": 0.8903,
      "num_input_tokens_seen": 175824,
      "odds_ratio_loss": 8.764748573303223,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.08572666347026825,
      "rewards/margins": 0.12890127301216125,
      "rewards/rejected": -0.2146279513835907,
      "sft_loss": 0.013779623433947563,
      "step": 140
    },
    {
      "epoch": 0.011668611435239206,
      "grad_norm": 3.3617570400238037,
      "learning_rate": 4.998320424697532e-05,
      "logits/chosen": 0.15023799240589142,
      "logits/rejected": -0.7425843477249146,
      "logps/chosen": -0.9067550897598267,
      "logps/rejected": -1.3671596050262451,
      "loss": 0.9588,
      "num_input_tokens_seen": 186320,
      "odds_ratio_loss": 9.49233341217041,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.09067549556493759,
      "rewards/margins": 0.046040456742048264,
      "rewards/rejected": -0.13671596348285675,
      "sft_loss": 0.009604133665561676,
      "step": 150
    },
    {
      "epoch": 0.012446518864255154,
      "grad_norm": 2.341899871826172,
      "learning_rate": 4.998089046029671e-05,
      "logits/chosen": 0.06832513958215714,
      "logits/rejected": -0.9852465391159058,
      "logps/chosen": -0.9017775654792786,
      "logps/rejected": -0.9677242040634155,
      "loss": 0.9697,
      "num_input_tokens_seen": 203072,
      "odds_ratio_loss": 9.57034969329834,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.0901777595281601,
      "rewards/margins": 0.006594663951545954,
      "rewards/rejected": -0.09677241742610931,
      "sft_loss": 0.012652057223021984,
      "step": 160
    },
    {
      "epoch": 0.013224426293271101,
      "grad_norm": 7.437110900878906,
      "learning_rate": 4.9978427475521616e-05,
      "logits/chosen": -0.009473335929214954,
      "logits/rejected": -0.7211953401565552,
      "logps/chosen": -0.9088319540023804,
      "logps/rejected": -1.050347089767456,
      "loss": 0.9788,
      "num_input_tokens_seen": 214336,
      "odds_ratio_loss": 9.662178993225098,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.09088320285081863,
      "rewards/margins": 0.014151516370475292,
      "rewards/rejected": -0.1050347238779068,
      "sft_loss": 0.012536203488707542,
      "step": 170
    },
    {
      "epoch": 0.014002333722287048,
      "grad_norm": 3.429651975631714,
      "learning_rate": 4.9975815307360194e-05,
      "logits/chosen": -0.16578452289104462,
      "logits/rejected": -0.4752795100212097,
      "logps/chosen": -0.8099583387374878,
      "logps/rejected": -0.9804008603096008,
      "loss": 0.8951,
      "num_input_tokens_seen": 232368,
      "odds_ratio_loss": 8.755950927734375,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.0809958428144455,
      "rewards/margins": 0.017044242471456528,
      "rewards/rejected": -0.09804007411003113,
      "sft_loss": 0.019553907215595245,
      "step": 180
    },
    {
      "epoch": 0.014780241151302996,
      "grad_norm": 5.283464431762695,
      "learning_rate": 4.997305397141357e-05,
      "logits/chosen": -0.18071290850639343,
      "logits/rejected": -0.5703662037849426,
      "logps/chosen": -0.9150312542915344,
      "logps/rejected": -1.393818974494934,
      "loss": 0.9661,
      "num_input_tokens_seen": 242960,
      "odds_ratio_loss": 9.338902473449707,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.09150312840938568,
      "rewards/margins": 0.04787876456975937,
      "rewards/rejected": -0.13938190042972565,
      "sft_loss": 0.03223598375916481,
      "step": 190
    },
    {
      "epoch": 0.015558148580318941,
      "grad_norm": 2.1387197971343994,
      "learning_rate": 4.997014348417382e-05,
      "logits/chosen": 0.12889909744262695,
      "logits/rejected": -1.100764513015747,
      "logps/chosen": -0.8966400027275085,
      "logps/rejected": -1.0923173427581787,
      "loss": 0.9529,
      "num_input_tokens_seen": 258080,
      "odds_ratio_loss": 9.400256156921387,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.08966399729251862,
      "rewards/margins": 0.019567731767892838,
      "rewards/rejected": -0.10923173278570175,
      "sft_loss": 0.012894442304968834,
      "step": 200
    },
    {
      "epoch": 0.01633605600933489,
      "grad_norm": 4.736515998840332,
      "learning_rate": 4.9967083863023776e-05,
      "logits/chosen": -0.09360722452402115,
      "logits/rejected": -1.2860757112503052,
      "logps/chosen": -1.1631531715393066,
      "logps/rejected": -1.368138074874878,
      "loss": 1.2197,
      "num_input_tokens_seen": 271920,
      "odds_ratio_loss": 11.848650932312012,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.1163153275847435,
      "rewards/margins": 0.020498499274253845,
      "rewards/rejected": -0.13681381940841675,
      "sft_loss": 0.034875985234975815,
      "step": 210
    },
    {
      "epoch": 0.017113963438350838,
      "grad_norm": 4.38301944732666,
      "learning_rate": 4.9963875126237016e-05,
      "logits/chosen": -0.29661667346954346,
      "logits/rejected": -0.6016083359718323,
      "logps/chosen": -0.7634471654891968,
      "logps/rejected": -0.9885290265083313,
      "loss": 0.8338,
      "num_input_tokens_seen": 286576,
      "odds_ratio_loss": 8.109630584716797,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.07634471356868744,
      "rewards/margins": 0.022508183494210243,
      "rewards/rejected": -0.09885289520025253,
      "sft_loss": 0.0228743739426136,
      "step": 220
    },
    {
      "epoch": 0.017891870867366783,
      "grad_norm": 2.4642908573150635,
      "learning_rate": 4.9960517292977655e-05,
      "logits/chosen": -0.04434485360980034,
      "logits/rejected": -0.9968097805976868,
      "logps/chosen": -1.2226582765579224,
      "logps/rejected": -2.301300525665283,
      "loss": 1.2993,
      "num_input_tokens_seen": 296896,
      "odds_ratio_loss": 12.393956184387207,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.12226583063602448,
      "rewards/margins": 0.10786418616771698,
      "rewards/rejected": -0.23013003170490265,
      "sft_loss": 0.05994652584195137,
      "step": 230
    },
    {
      "epoch": 0.01866977829638273,
      "grad_norm": 2.916964054107666,
      "learning_rate": 4.995701038330033e-05,
      "logits/chosen": -0.3185642659664154,
      "logits/rejected": -1.0956424474716187,
      "logps/chosen": -1.0076221227645874,
      "logps/rejected": -1.1618595123291016,
      "loss": 1.0844,
      "num_input_tokens_seen": 311296,
      "odds_ratio_loss": 9.93652057647705,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": -0.10076222568750381,
      "rewards/margins": 0.01542372815310955,
      "rewards/rejected": -0.11618594825267792,
      "sft_loss": 0.09077021479606628,
      "step": 240
    },
    {
      "epoch": 0.019447685725398678,
      "grad_norm": 2.7747602462768555,
      "learning_rate": 4.995335441815003e-05,
      "logits/chosen": 0.045975618064403534,
      "logits/rejected": -1.4385907649993896,
      "logps/chosen": -1.1574515104293823,
      "logps/rejected": -1.4619978666305542,
      "loss": 1.2243,
      "num_input_tokens_seen": 321184,
      "odds_ratio_loss": 12.18043327331543,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.11574514955282211,
      "rewards/margins": 0.030454616993665695,
      "rewards/rejected": -0.1461997926235199,
      "sft_loss": 0.00630641495808959,
      "step": 250
    },
    {
      "epoch": 0.020225593154414623,
      "grad_norm": 16.452539443969727,
      "learning_rate": 4.9949549419361954e-05,
      "logits/chosen": -0.2351895123720169,
      "logits/rejected": -1.024817705154419,
      "logps/chosen": -0.9535134434700012,
      "logps/rejected": -1.9824984073638916,
      "loss": 1.0084,
      "num_input_tokens_seen": 331328,
      "odds_ratio_loss": 9.948028564453125,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.09535135328769684,
      "rewards/margins": 0.1028984934091568,
      "rewards/rejected": -0.19824984669685364,
      "sft_loss": 0.013568353839218616,
      "step": 260
    },
    {
      "epoch": 0.021003500583430573,
      "grad_norm": 3.6468498706817627,
      "learning_rate": 4.994559540966142e-05,
      "logits/chosen": -0.022709887474775314,
      "logits/rejected": -1.4626227617263794,
      "logps/chosen": -0.9807567596435547,
      "logps/rejected": -1.9314972162246704,
      "loss": 1.0466,
      "num_input_tokens_seen": 345456,
      "odds_ratio_loss": 10.377405166625977,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.09807567298412323,
      "rewards/margins": 0.09507404267787933,
      "rewards/rejected": -0.19314971566200256,
      "sft_loss": 0.008827922865748405,
      "step": 270
    },
    {
      "epoch": 0.021781408012446518,
      "grad_norm": 6.5957465171813965,
      "learning_rate": 4.994149241266371e-05,
      "logits/chosen": -0.41178959608078003,
      "logits/rejected": -0.37596777081489563,
      "logps/chosen": -1.0016124248504639,
      "logps/rejected": -0.9876958727836609,
      "loss": 1.0896,
      "num_input_tokens_seen": 357568,
      "odds_ratio_loss": 10.677938461303711,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": -0.10016125440597534,
      "rewards/margins": -0.0013916685711592436,
      "rewards/rejected": -0.09876958280801773,
      "sft_loss": 0.02179970219731331,
      "step": 280
    },
    {
      "epoch": 0.022559315441462467,
      "grad_norm": 1.5292377471923828,
      "learning_rate": 4.993724045287392e-05,
      "logits/chosen": 0.10405155271291733,
      "logits/rejected": -1.1914265155792236,
      "logps/chosen": -0.9256628751754761,
      "logps/rejected": -1.3289026021957397,
      "loss": 0.9888,
      "num_input_tokens_seen": 373472,
      "odds_ratio_loss": 9.648874282836914,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.09256630390882492,
      "rewards/margins": 0.040323980152606964,
      "rewards/rejected": -0.13289028406143188,
      "sft_loss": 0.023866329342126846,
      "step": 290
    },
    {
      "epoch": 0.023337222870478413,
      "grad_norm": 2.4046130180358887,
      "learning_rate": 4.993283955568685e-05,
      "logits/chosen": 0.010790085420012474,
      "logits/rejected": -1.7810585498809814,
      "logps/chosen": -0.8726987838745117,
      "logps/rejected": -0.8844345211982727,
      "loss": 0.9519,
      "num_input_tokens_seen": 386080,
      "odds_ratio_loss": 9.43667984008789,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.08726988732814789,
      "rewards/margins": 0.001173576689325273,
      "rewards/rejected": -0.08844345062971115,
      "sft_loss": 0.008213376626372337,
      "step": 300
    },
    {
      "epoch": 0.024115130299494362,
      "grad_norm": 3.012260913848877,
      "learning_rate": 4.99282897473868e-05,
      "logits/chosen": -0.04694366827607155,
      "logits/rejected": -1.3969290256500244,
      "logps/chosen": -0.9051028490066528,
      "logps/rejected": -1.2001574039459229,
      "loss": 0.961,
      "num_input_tokens_seen": 396368,
      "odds_ratio_loss": 9.4262056350708,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.0905102863907814,
      "rewards/margins": 0.029505455866456032,
      "rewards/rejected": -0.12001574039459229,
      "sft_loss": 0.018341612070798874,
      "step": 310
    },
    {
      "epoch": 0.024893037728510307,
      "grad_norm": 1.637371301651001,
      "learning_rate": 4.9923591055147455e-05,
      "logits/chosen": 0.16533169150352478,
      "logits/rejected": -1.5700806379318237,
      "logps/chosen": -1.0949220657348633,
      "logps/rejected": -1.2998539209365845,
      "loss": 1.1594,
      "num_input_tokens_seen": 414368,
      "odds_ratio_loss": 11.56415843963623,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.10949220508337021,
      "rewards/margins": 0.020493196323513985,
      "rewards/rejected": -0.12998540699481964,
      "sft_loss": 0.0029837344773113728,
      "step": 320
    },
    {
      "epoch": 0.025670945157526253,
      "grad_norm": 3.8987412452697754,
      "learning_rate": 4.99187435070317e-05,
      "logits/chosen": -0.1262105256319046,
      "logits/rejected": -1.0445555448532104,
      "logps/chosen": -0.9046597480773926,
      "logps/rejected": -1.073553442955017,
      "loss": 0.9619,
      "num_input_tokens_seen": 424800,
      "odds_ratio_loss": 9.364073753356934,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.0904659777879715,
      "rewards/margins": 0.016889363527297974,
      "rewards/rejected": -0.10735534131526947,
      "sft_loss": 0.025530826300382614,
      "step": 330
    },
    {
      "epoch": 0.026448852586542202,
      "grad_norm": 1.8967704772949219,
      "learning_rate": 4.991374713199145e-05,
      "logits/chosen": -0.1018231064081192,
      "logits/rejected": -1.3850098848342896,
      "logps/chosen": -1.0545568466186523,
      "logps/rejected": -1.2990403175354004,
      "loss": 1.1314,
      "num_input_tokens_seen": 435984,
      "odds_ratio_loss": 11.010573387145996,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.105455681681633,
      "rewards/margins": 0.024448346346616745,
      "rewards/rejected": -0.12990403175354004,
      "sft_loss": 0.0303510669618845,
      "step": 340
    },
    {
      "epoch": 0.027226760015558148,
      "grad_norm": 5.541660785675049,
      "learning_rate": 4.990860195986753e-05,
      "logits/chosen": -0.24207575619220734,
      "logits/rejected": -1.1700140237808228,
      "logps/chosen": -1.1388294696807861,
      "logps/rejected": -2.223168134689331,
      "loss": 1.2046,
      "num_input_tokens_seen": 449136,
      "odds_ratio_loss": 10.3714599609375,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.11388297379016876,
      "rewards/margins": 0.10843388736248016,
      "rewards/rejected": -0.22231683135032654,
      "sft_loss": 0.16744010150432587,
      "step": 350
    },
    {
      "epoch": 0.028004667444574097,
      "grad_norm": 1.861402153968811,
      "learning_rate": 4.990330802138941e-05,
      "logits/chosen": -0.31768426299095154,
      "logits/rejected": -1.2700172662734985,
      "logps/chosen": -0.8037441372871399,
      "logps/rejected": -1.0384769439697266,
      "loss": 0.8601,
      "num_input_tokens_seen": 461712,
      "odds_ratio_loss": 8.533435821533203,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.08037441223859787,
      "rewards/margins": 0.023473283275961876,
      "rewards/rejected": -0.10384770482778549,
      "sft_loss": 0.006723404861986637,
      "step": 360
    },
    {
      "epoch": 0.028782574873590042,
      "grad_norm": 8.835140228271484,
      "learning_rate": 4.989786534817507e-05,
      "logits/chosen": -0.3392910659313202,
      "logits/rejected": -1.3058607578277588,
      "logps/chosen": -0.8710435628890991,
      "logps/rejected": -1.9576623439788818,
      "loss": 0.9149,
      "num_input_tokens_seen": 471328,
      "odds_ratio_loss": 8.99726390838623,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.08710435032844543,
      "rewards/margins": 0.10866190493106842,
      "rewards/rejected": -0.19576624035835266,
      "sft_loss": 0.015141059644520283,
      "step": 370
    },
    {
      "epoch": 0.02956048230260599,
      "grad_norm": 2.149712562561035,
      "learning_rate": 4.989227397273082e-05,
      "logits/chosen": -0.45678621530532837,
      "logits/rejected": -0.8773413896560669,
      "logps/chosen": -0.9752458333969116,
      "logps/rejected": -1.3521517515182495,
      "loss": 1.0315,
      "num_input_tokens_seen": 481728,
      "odds_ratio_loss": 10.136479377746582,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.09752458333969116,
      "rewards/margins": 0.03769059106707573,
      "rewards/rejected": -0.13521519303321838,
      "sft_loss": 0.017892986536026,
      "step": 380
    },
    {
      "epoch": 0.030338389731621937,
      "grad_norm": 4.1422553062438965,
      "learning_rate": 4.9886533928451104e-05,
      "logits/chosen": -0.24568918347358704,
      "logits/rejected": -1.1686650514602661,
      "logps/chosen": -1.0748536586761475,
      "logps/rejected": -1.366524577140808,
      "loss": 1.1379,
      "num_input_tokens_seen": 491856,
      "odds_ratio_loss": 11.042202949523926,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.10748536884784698,
      "rewards/margins": 0.029167091473937035,
      "rewards/rejected": -0.13665245473384857,
      "sft_loss": 0.03370191156864166,
      "step": 390
    },
    {
      "epoch": 0.031116297160637883,
      "grad_norm": 3.0260448455810547,
      "learning_rate": 4.9880645249618266e-05,
      "logits/chosen": 0.010119600221514702,
      "logits/rejected": -1.2736787796020508,
      "logps/chosen": -0.9264433979988098,
      "logps/rejected": -1.1216003894805908,
      "loss": 0.9889,
      "num_input_tokens_seen": 504080,
      "odds_ratio_loss": 9.685811042785645,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.0926443412899971,
      "rewards/margins": 0.01951570436358452,
      "rewards/rejected": -0.11216004192829132,
      "sft_loss": 0.020315980538725853,
      "step": 400
    },
    {
      "epoch": 0.03189420458965383,
      "grad_norm": 2.714024782180786,
      "learning_rate": 4.9874607971402366e-05,
      "logits/chosen": -0.21029357612133026,
      "logits/rejected": -1.3044781684875488,
      "logps/chosen": -1.0438721179962158,
      "logps/rejected": -1.292240858078003,
      "loss": 1.1025,
      "num_input_tokens_seen": 518688,
      "odds_ratio_loss": 10.874516487121582,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.10438720881938934,
      "rewards/margins": 0.024836886674165726,
      "rewards/rejected": -0.12922410666942596,
      "sft_loss": 0.015052947215735912,
      "step": 410
    },
    {
      "epoch": 0.03267211201866978,
      "grad_norm": 3.6389522552490234,
      "learning_rate": 4.986842212986099e-05,
      "logits/chosen": -0.32333192229270935,
      "logits/rejected": -1.2866684198379517,
      "logps/chosen": -1.022554636001587,
      "logps/rejected": -1.1091924905776978,
      "loss": 1.0915,
      "num_input_tokens_seen": 531904,
      "odds_ratio_loss": 10.820093154907227,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.10225546360015869,
      "rewards/margins": 0.008663791231811047,
      "rewards/rejected": -0.11091925948858261,
      "sft_loss": 0.009445266798138618,
      "step": 420
    },
    {
      "epoch": 0.033450019447685726,
      "grad_norm": 5.962457656860352,
      "learning_rate": 4.986208776193902e-05,
      "logits/chosen": -0.07435014098882675,
      "logits/rejected": -1.4142100811004639,
      "logps/chosen": -1.0092417001724243,
      "logps/rejected": -1.314469575881958,
      "loss": 1.0653,
      "num_input_tokens_seen": 544448,
      "odds_ratio_loss": 10.6132173538208,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.10092417895793915,
      "rewards/margins": 0.030522778630256653,
      "rewards/rejected": -0.1314469575881958,
      "sft_loss": 0.003954497165977955,
      "step": 430
    },
    {
      "epoch": 0.034227926876701675,
      "grad_norm": 4.502202987670898,
      "learning_rate": 4.985560490546839e-05,
      "logits/chosen": -0.1421446055173874,
      "logits/rejected": -1.3914377689361572,
      "logps/chosen": -0.8869674801826477,
      "logps/rejected": -2.607506275177002,
      "loss": 0.9403,
      "num_input_tokens_seen": 558576,
      "odds_ratio_loss": 9.332474708557129,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.08869674056768417,
      "rewards/margins": 0.17205388844013214,
      "rewards/rejected": -0.2607506215572357,
      "sft_loss": 0.007023976184427738,
      "step": 440
    },
    {
      "epoch": 0.03500583430571762,
      "grad_norm": 2.3669939041137695,
      "learning_rate": 4.984897359916788e-05,
      "logits/chosen": -0.030847545713186264,
      "logits/rejected": -0.9774616360664368,
      "logps/chosen": -0.8356963992118835,
      "logps/rejected": -0.9540478587150574,
      "loss": 0.8978,
      "num_input_tokens_seen": 567568,
      "odds_ratio_loss": 8.91874885559082,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.08356963843107224,
      "rewards/margins": 0.011835141107439995,
      "rewards/rejected": -0.09540478140115738,
      "sft_loss": 0.00597239937633276,
      "step": 450
    },
    {
      "epoch": 0.035783741734733567,
      "grad_norm": 21.677845001220703,
      "learning_rate": 4.9842193882642906e-05,
      "logits/chosen": -0.2935951352119446,
      "logits/rejected": -1.1201730966567993,
      "logps/chosen": -1.3038840293884277,
      "logps/rejected": -1.2353122234344482,
      "loss": 1.3919,
      "num_input_tokens_seen": 581632,
      "odds_ratio_loss": 13.499517440795898,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": -0.13038839399814606,
      "rewards/margins": -0.006857173051685095,
      "rewards/rejected": -0.12353122234344482,
      "sft_loss": 0.041925784200429916,
      "step": 460
    },
    {
      "epoch": 0.036561649163749516,
      "grad_norm": 2.8555731773376465,
      "learning_rate": 4.9835265796385254e-05,
      "logits/chosen": -0.1229843869805336,
      "logits/rejected": -1.5147597789764404,
      "logps/chosen": -1.1090964078903198,
      "logps/rejected": -2.1519970893859863,
      "loss": 1.163,
      "num_input_tokens_seen": 594912,
      "odds_ratio_loss": 11.57403564453125,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.11090965569019318,
      "rewards/margins": 0.10429003089666367,
      "rewards/rejected": -0.21519970893859863,
      "sft_loss": 0.005632361862808466,
      "step": 470
    },
    {
      "epoch": 0.03733955659276546,
      "grad_norm": 1.7752621173858643,
      "learning_rate": 4.982818938177284e-05,
      "logits/chosen": 0.023658571764826775,
      "logits/rejected": -1.7298905849456787,
      "logps/chosen": -1.1194090843200684,
      "logps/rejected": -1.389075756072998,
      "loss": 1.1806,
      "num_input_tokens_seen": 607808,
      "odds_ratio_loss": 11.7571439743042,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.11194092035293579,
      "rewards/margins": 0.026966670528054237,
      "rewards/rejected": -0.13890758156776428,
      "sft_loss": 0.004934356547892094,
      "step": 480
    },
    {
      "epoch": 0.03811746402178141,
      "grad_norm": 5.799675464630127,
      "learning_rate": 4.982096468106947e-05,
      "logits/chosen": -0.17389316856861115,
      "logits/rejected": -1.2688932418823242,
      "logps/chosen": -1.0437113046646118,
      "logps/rejected": -1.5706325769424438,
      "loss": 1.1041,
      "num_input_tokens_seen": 625984,
      "odds_ratio_loss": 10.988919258117676,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.10437111556529999,
      "rewards/margins": 0.05269213393330574,
      "rewards/rejected": -0.15706324577331543,
      "sft_loss": 0.005206743720918894,
      "step": 490
    },
    {
      "epoch": 0.038895371450797356,
      "grad_norm": 5.300241470336914,
      "learning_rate": 4.9813591737424596e-05,
      "logits/chosen": -0.4578474164009094,
      "logits/rejected": -1.0031487941741943,
      "logps/chosen": -0.8955472707748413,
      "logps/rejected": -1.3793914318084717,
      "loss": 0.9547,
      "num_input_tokens_seen": 639856,
      "odds_ratio_loss": 9.262350082397461,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.08955473452806473,
      "rewards/margins": 0.0483844131231308,
      "rewards/rejected": -0.13793914020061493,
      "sft_loss": 0.02842315472662449,
      "step": 500
    },
    {
      "epoch": 0.039673278879813305,
      "grad_norm": 20.943037033081055,
      "learning_rate": 4.9806070594873037e-05,
      "logits/chosen": -0.3508077561855316,
      "logits/rejected": -1.051695466041565,
      "logps/chosen": -0.9110622406005859,
      "logps/rejected": -1.484948754310608,
      "loss": 0.9541,
      "num_input_tokens_seen": 650656,
      "odds_ratio_loss": 9.375656127929688,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09110622853040695,
      "rewards/margins": 0.05738866329193115,
      "rewards/rejected": -0.1484948843717575,
      "sft_loss": 0.016496889293193817,
      "step": 510
    },
    {
      "epoch": 0.04045118630882925,
      "grad_norm": 3.846301555633545,
      "learning_rate": 4.979840129833474e-05,
      "logits/chosen": -0.4234844148159027,
      "logits/rejected": -0.675470232963562,
      "logps/chosen": -1.0217151641845703,
      "logps/rejected": -1.2225643396377563,
      "loss": 1.1046,
      "num_input_tokens_seen": 661552,
      "odds_ratio_loss": 10.809840202331543,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.10217152535915375,
      "rewards/margins": 0.020084915682673454,
      "rewards/rejected": -0.12225644290447235,
      "sft_loss": 0.023616602644324303,
      "step": 520
    },
    {
      "epoch": 0.041229093737845196,
      "grad_norm": 6.235986709594727,
      "learning_rate": 4.97905838936145e-05,
      "logits/chosen": 0.06146078556776047,
      "logits/rejected": -1.7578405141830444,
      "logps/chosen": -0.9877039194107056,
      "logps/rejected": -1.5740721225738525,
      "loss": 1.043,
      "num_input_tokens_seen": 675936,
      "odds_ratio_loss": 10.361382484436035,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.0987703949213028,
      "rewards/margins": 0.05863680690526962,
      "rewards/rejected": -0.15740719437599182,
      "sft_loss": 0.006865262053906918,
      "step": 530
    },
    {
      "epoch": 0.042007001166861145,
      "grad_norm": 1.6892807483673096,
      "learning_rate": 4.978261842740166e-05,
      "logits/chosen": -0.05991193652153015,
      "logits/rejected": -1.4414308071136475,
      "logps/chosen": -1.058107852935791,
      "logps/rejected": -1.4011973142623901,
      "loss": 1.1117,
      "num_input_tokens_seen": 690112,
      "odds_ratio_loss": 11.0405912399292,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.10581078380346298,
      "rewards/margins": 0.03430896997451782,
      "rewards/rejected": -0.1401197612285614,
      "sft_loss": 0.007605980150401592,
      "step": 540
    },
    {
      "epoch": 0.04278490859587709,
      "grad_norm": 2.5099849700927734,
      "learning_rate": 4.9774504947269905e-05,
      "logits/chosen": 0.03550032898783684,
      "logits/rejected": -1.306727647781372,
      "logps/chosen": -0.9737154245376587,
      "logps/rejected": -1.42258882522583,
      "loss": 1.0439,
      "num_input_tokens_seen": 704480,
      "odds_ratio_loss": 9.935417175292969,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.09737154096364975,
      "rewards/margins": 0.04488734155893326,
      "rewards/rejected": -0.14225886762142181,
      "sft_loss": 0.05035661906003952,
      "step": 550
    },
    {
      "epoch": 0.043562816024893036,
      "grad_norm": 2.5092949867248535,
      "learning_rate": 4.97662435016769e-05,
      "logits/chosen": 0.20719823241233826,
      "logits/rejected": -1.6484529972076416,
      "logps/chosen": -0.8146620988845825,
      "logps/rejected": -1.2458853721618652,
      "loss": 0.8628,
      "num_input_tokens_seen": 720512,
      "odds_ratio_loss": 8.611961364746094,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.08146621286869049,
      "rewards/margins": 0.04312232881784439,
      "rewards/rejected": -0.12458853423595428,
      "sft_loss": 0.0016197937075048685,
      "step": 560
    },
    {
      "epoch": 0.044340723453908985,
      "grad_norm": 7.840872764587402,
      "learning_rate": 4.9757834139964035e-05,
      "logits/chosen": -0.6304762363433838,
      "logits/rejected": -0.601016640663147,
      "logps/chosen": -0.9585424661636353,
      "logps/rejected": -1.712816596031189,
      "loss": 1.0203,
      "num_input_tokens_seen": 731840,
      "odds_ratio_loss": 9.28771686553955,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.0958542451262474,
      "rewards/margins": 0.07542743533849716,
      "rewards/rejected": -0.17128166556358337,
      "sft_loss": 0.09156389534473419,
      "step": 570
    },
    {
      "epoch": 0.045118630882924934,
      "grad_norm": 3.6173086166381836,
      "learning_rate": 4.974927691235614e-05,
      "logits/chosen": -0.04471359774470329,
      "logits/rejected": -1.2187974452972412,
      "logps/chosen": -0.8615912199020386,
      "logps/rejected": -1.62959885597229,
      "loss": 0.9147,
      "num_input_tokens_seen": 742800,
      "odds_ratio_loss": 9.08159065246582,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.0861591249704361,
      "rewards/margins": 0.07680076360702515,
      "rewards/rejected": -0.16295988857746124,
      "sft_loss": 0.006538393907248974,
      "step": 580
    },
    {
      "epoch": 0.045896538311940877,
      "grad_norm": 3.3760595321655273,
      "learning_rate": 4.9740571869961137e-05,
      "logits/chosen": -0.3412732481956482,
      "logits/rejected": -1.3786067962646484,
      "logps/chosen": -1.2360124588012695,
      "logps/rejected": -1.6927540302276611,
      "loss": 1.293,
      "num_input_tokens_seen": 752016,
      "odds_ratio_loss": 12.306550979614258,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.12360125780105591,
      "rewards/margins": 0.04567413777112961,
      "rewards/rejected": -0.16927538812160492,
      "sft_loss": 0.06239398568868637,
      "step": 590
    },
    {
      "epoch": 0.046674445740956826,
      "grad_norm": 3.992867946624756,
      "learning_rate": 4.9731719064769833e-05,
      "logits/chosen": -0.07947269827127457,
      "logits/rejected": -1.2102258205413818,
      "logps/chosen": -0.8327411413192749,
      "logps/rejected": -1.5643784999847412,
      "loss": 0.8797,
      "num_input_tokens_seen": 763376,
      "odds_ratio_loss": 8.484746932983398,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.08327411860227585,
      "rewards/margins": 0.07316374778747559,
      "rewards/rejected": -0.15643787384033203,
      "sft_loss": 0.03125034272670746,
      "step": 600
    },
    {
      "epoch": 0.047452353169972775,
      "grad_norm": 10.707353591918945,
      "learning_rate": 4.972271854965549e-05,
      "logits/chosen": 0.01606535166501999,
      "logits/rejected": -1.0644062757492065,
      "logps/chosen": -0.9305408596992493,
      "logps/rejected": -1.4815337657928467,
      "loss": 0.9802,
      "num_input_tokens_seen": 775424,
      "odds_ratio_loss": 9.663485527038574,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.09305407851934433,
      "rewards/margins": 0.05509928986430168,
      "rewards/rejected": -0.1481533646583557,
      "sft_loss": 0.01387445442378521,
      "step": 610
    },
    {
      "epoch": 0.048230260598988724,
      "grad_norm": 2.879865884780884,
      "learning_rate": 4.9713570378373584e-05,
      "logits/chosen": 0.09017481654882431,
      "logits/rejected": -1.2260386943817139,
      "logps/chosen": -0.9439705610275269,
      "logps/rejected": -1.1371946334838867,
      "loss": 1.0053,
      "num_input_tokens_seen": 792192,
      "odds_ratio_loss": 9.941380500793457,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.09439705312252045,
      "rewards/margins": 0.01932242140173912,
      "rewards/rejected": -0.11371947824954987,
      "sft_loss": 0.011146670207381248,
      "step": 620
    },
    {
      "epoch": 0.049008168028004666,
      "grad_norm": 2.4512083530426025,
      "learning_rate": 4.9704274605561486e-05,
      "logits/chosen": -0.06639071553945541,
      "logits/rejected": -1.1143431663513184,
      "logps/chosen": -0.8876552581787109,
      "logps/rejected": -1.388165831565857,
      "loss": 0.9485,
      "num_input_tokens_seen": 803920,
      "odds_ratio_loss": 9.404537200927734,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.08876553177833557,
      "rewards/margins": 0.05005106329917908,
      "rewards/rejected": -0.13881659507751465,
      "sft_loss": 0.008023647591471672,
      "step": 630
    },
    {
      "epoch": 0.049786075457020615,
      "grad_norm": 13.352177619934082,
      "learning_rate": 4.969483128673808e-05,
      "logits/chosen": -0.09846919775009155,
      "logits/rejected": -0.6065551042556763,
      "logps/chosen": -0.8927276730537415,
      "logps/rejected": -1.3452032804489136,
      "loss": 0.9619,
      "num_input_tokens_seen": 817856,
      "odds_ratio_loss": 9.278109550476074,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.08927276730537415,
      "rewards/margins": 0.04524756222963333,
      "rewards/rejected": -0.13452033698558807,
      "sft_loss": 0.034084875136613846,
      "step": 640
    },
    {
      "epoch": 0.050563982886036564,
      "grad_norm": 5.00535249710083,
      "learning_rate": 4.96852404783035e-05,
      "logits/chosen": 0.00019475221051834524,
      "logits/rejected": -1.6706771850585938,
      "logps/chosen": -1.0463913679122925,
      "logps/rejected": -1.2002830505371094,
      "loss": 1.1217,
      "num_input_tokens_seen": 835488,
      "odds_ratio_loss": 11.174688339233398,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.10463915020227432,
      "rewards/margins": 0.015389163978397846,
      "rewards/rejected": -0.12002831697463989,
      "sft_loss": 0.004188282880932093,
      "step": 650
    },
    {
      "epoch": 0.051341890315052506,
      "grad_norm": 2.9358901977539062,
      "learning_rate": 4.967550223753873e-05,
      "logits/chosen": -0.2236834019422531,
      "logits/rejected": -0.8710101246833801,
      "logps/chosen": -1.2662832736968994,
      "logps/rejected": -2.0422096252441406,
      "loss": 1.3282,
      "num_input_tokens_seen": 852288,
      "odds_ratio_loss": 12.909830093383789,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.1266283094882965,
      "rewards/margins": 0.0775926411151886,
      "rewards/rejected": -0.2042209655046463,
      "sft_loss": 0.037256959825754166,
      "step": 660
    },
    {
      "epoch": 0.052119797744068455,
      "grad_norm": 28.316844940185547,
      "learning_rate": 4.966561662260532e-05,
      "logits/chosen": -0.16411355137825012,
      "logits/rejected": -0.9293942451477051,
      "logps/chosen": -0.9301309585571289,
      "logps/rejected": -1.9711850881576538,
      "loss": 0.9846,
      "num_input_tokens_seen": 861984,
      "odds_ratio_loss": 9.354632377624512,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.09301309287548065,
      "rewards/margins": 0.10410543531179428,
      "rewards/rejected": -0.19711852073669434,
      "sft_loss": 0.04918523132801056,
      "step": 670
    },
    {
      "epoch": 0.052897705173084404,
      "grad_norm": 2.13519287109375,
      "learning_rate": 4.9655583692544986e-05,
      "logits/chosen": -0.29984229803085327,
      "logits/rejected": -1.138826847076416,
      "logps/chosen": -0.8817435503005981,
      "logps/rejected": -1.1321008205413818,
      "loss": 0.9382,
      "num_input_tokens_seen": 875440,
      "odds_ratio_loss": 9.27894115447998,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.08817435801029205,
      "rewards/margins": 0.025035735219717026,
      "rewards/rejected": -0.11321009695529938,
      "sft_loss": 0.010337069630622864,
      "step": 680
    },
    {
      "epoch": 0.05367561260210035,
      "grad_norm": 8.119619369506836,
      "learning_rate": 4.96454035072793e-05,
      "logits/chosen": -0.019457275047898293,
      "logits/rejected": -1.780207633972168,
      "logps/chosen": -1.1262990236282349,
      "logps/rejected": -1.7122234106063843,
      "loss": 1.1782,
      "num_input_tokens_seen": 890912,
      "odds_ratio_loss": 11.341121673583984,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.11262989044189453,
      "rewards/margins": 0.05859243869781494,
      "rewards/rejected": -0.17122234404087067,
      "sft_loss": 0.044103674590587616,
      "step": 690
    },
    {
      "epoch": 0.054453520031116295,
      "grad_norm": 2.790867805480957,
      "learning_rate": 4.963507612760932e-05,
      "logits/chosen": 0.0026435554027557373,
      "logits/rejected": -1.221129298210144,
      "logps/chosen": -1.1090561151504517,
      "logps/rejected": -1.7434148788452148,
      "loss": 1.1765,
      "num_input_tokens_seen": 906464,
      "odds_ratio_loss": 10.66307258605957,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.11090561002492905,
      "rewards/margins": 0.06343585252761841,
      "rewards/rejected": -0.17434147000312805,
      "sft_loss": 0.11023900657892227,
      "step": 700
    },
    {
      "epoch": 0.055231427460132244,
      "grad_norm": 8.295060157775879,
      "learning_rate": 4.962460161521518e-05,
      "logits/chosen": -0.43848055601119995,
      "logits/rejected": -1.1596039533615112,
      "logps/chosen": -1.5424635410308838,
      "logps/rejected": -1.3884707689285278,
      "loss": 1.659,
      "num_input_tokens_seen": 919648,
      "odds_ratio_loss": 15.883735656738281,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.15424636006355286,
      "rewards/margins": -0.015399277210235596,
      "rewards/rejected": -0.13884706795215607,
      "sft_loss": 0.07059884071350098,
      "step": 710
    },
    {
      "epoch": 0.056009334889148193,
      "grad_norm": 2.7219836711883545,
      "learning_rate": 4.961398003265582e-05,
      "logits/chosen": -0.0401337593793869,
      "logits/rejected": -0.8950508832931519,
      "logps/chosen": -0.9055600166320801,
      "logps/rejected": -1.2787041664123535,
      "loss": 0.9826,
      "num_input_tokens_seen": 932144,
      "odds_ratio_loss": 9.71101188659668,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": -0.09055600315332413,
      "rewards/margins": 0.03731442242860794,
      "rewards/rejected": -0.12787041068077087,
      "sft_loss": 0.011469913646578789,
      "step": 720
    },
    {
      "epoch": 0.056787242318164136,
      "grad_norm": 9.607157707214355,
      "learning_rate": 4.96032114433685e-05,
      "logits/chosen": -0.20233073830604553,
      "logits/rejected": -1.0893396139144897,
      "logps/chosen": -1.086705207824707,
      "logps/rejected": -1.1163735389709473,
      "loss": 1.1655,
      "num_input_tokens_seen": 944384,
      "odds_ratio_loss": 11.479543685913086,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.10867051780223846,
      "rewards/margins": 0.002966833533719182,
      "rewards/rejected": -0.11163735389709473,
      "sft_loss": 0.017573310062289238,
      "step": 730
    },
    {
      "epoch": 0.057565149747180085,
      "grad_norm": 6.647035598754883,
      "learning_rate": 4.9592295911668516e-05,
      "logits/chosen": -0.1728004515171051,
      "logits/rejected": -1.3159668445587158,
      "logps/chosen": -0.8700982928276062,
      "logps/rejected": -2.0540032386779785,
      "loss": 0.9084,
      "num_input_tokens_seen": 956528,
      "odds_ratio_loss": 8.972583770751953,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.08700983226299286,
      "rewards/margins": 0.1183905154466629,
      "rewards/rejected": -0.20540031790733337,
      "sft_loss": 0.011142651550471783,
      "step": 740
    },
    {
      "epoch": 0.058343057176196034,
      "grad_norm": 1.6915847063064575,
      "learning_rate": 4.958123350274878e-05,
      "logits/chosen": 0.09090377390384674,
      "logits/rejected": -1.1098421812057495,
      "logps/chosen": -0.828323483467102,
      "logps/rejected": -1.0652092695236206,
      "loss": 0.8905,
      "num_input_tokens_seen": 973072,
      "odds_ratio_loss": 8.869556427001953,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.08283234387636185,
      "rewards/margins": 0.02368859015405178,
      "rewards/rejected": -0.10652093589305878,
      "sft_loss": 0.003506676061078906,
      "step": 750
    },
    {
      "epoch": 0.05912096460521198,
      "grad_norm": 4.410206317901611,
      "learning_rate": 4.957002428267937e-05,
      "logits/chosen": -0.12386639416217804,
      "logits/rejected": -1.4178106784820557,
      "logps/chosen": -1.006008505821228,
      "logps/rejected": -1.5750558376312256,
      "loss": 1.0587,
      "num_input_tokens_seen": 990416,
      "odds_ratio_loss": 10.437189102172852,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.10060086101293564,
      "rewards/margins": 0.05690472200512886,
      "rewards/rejected": -0.157505601644516,
      "sft_loss": 0.01501976978033781,
      "step": 760
    },
    {
      "epoch": 0.059898872034227925,
      "grad_norm": 4.2691192626953125,
      "learning_rate": 4.9558668318407264e-05,
      "logits/chosen": -0.0732850581407547,
      "logits/rejected": -1.1064198017120361,
      "logps/chosen": -0.8314032554626465,
      "logps/rejected": -1.7381566762924194,
      "loss": 0.8686,
      "num_input_tokens_seen": 1000784,
      "odds_ratio_loss": 8.609987258911133,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.08314032852649689,
      "rewards/margins": 0.09067533910274506,
      "rewards/rejected": -0.17381566762924194,
      "sft_loss": 0.0076231942512094975,
      "step": 770
    },
    {
      "epoch": 0.060676779463243874,
      "grad_norm": 2.8287811279296875,
      "learning_rate": 4.954716567775582e-05,
      "logits/chosen": -0.02032371237874031,
      "logits/rejected": -1.1463935375213623,
      "logps/chosen": -0.9587979316711426,
      "logps/rejected": -1.1363391876220703,
      "loss": 1.0232,
      "num_input_tokens_seen": 1016992,
      "odds_ratio_loss": 10.191497802734375,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.09587979316711426,
      "rewards/margins": 0.017754126340150833,
      "rewards/rejected": -0.11363391578197479,
      "sft_loss": 0.00401442963629961,
      "step": 780
    },
    {
      "epoch": 0.06145468689225982,
      "grad_norm": 3.20035719871521,
      "learning_rate": 4.9535516429424444e-05,
      "logits/chosen": -0.4328770041465759,
      "logits/rejected": -0.9181963801383972,
      "logps/chosen": -0.858111560344696,
      "logps/rejected": -1.4384349584579468,
      "loss": 0.8984,
      "num_input_tokens_seen": 1027120,
      "odds_ratio_loss": 8.835628509521484,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.08581115305423737,
      "rewards/margins": 0.05803234130144119,
      "rewards/rejected": -0.14384350180625916,
      "sft_loss": 0.014793934300541878,
      "step": 790
    },
    {
      "epoch": 0.062232594321275765,
      "grad_norm": 8.685051918029785,
      "learning_rate": 4.952372064298813e-05,
      "logits/chosen": -0.3011876940727234,
      "logits/rejected": -0.7921189069747925,
      "logps/chosen": -0.9806588888168335,
      "logps/rejected": -1.0401127338409424,
      "loss": 1.0546,
      "num_input_tokens_seen": 1040192,
      "odds_ratio_loss": 10.229253768920898,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": -0.09806589782238007,
      "rewards/margins": 0.005945396143943071,
      "rewards/rejected": -0.10401128232479095,
      "sft_loss": 0.03166633099317551,
      "step": 800
    },
    {
      "epoch": 0.06301050175029171,
      "grad_norm": 13.73974323272705,
      "learning_rate": 4.9511778388897094e-05,
      "logits/chosen": -0.46943092346191406,
      "logits/rejected": -1.2852362394332886,
      "logps/chosen": -0.8816627264022827,
      "logps/rejected": -1.0180727243423462,
      "loss": 0.9553,
      "num_input_tokens_seen": 1054160,
      "odds_ratio_loss": 9.431961059570312,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.08816628158092499,
      "rewards/margins": 0.013641009107232094,
      "rewards/rejected": -0.10180728137493134,
      "sft_loss": 0.012087122537195683,
      "step": 810
    },
    {
      "epoch": 0.06378840917930766,
      "grad_norm": 2.1410396099090576,
      "learning_rate": 4.949968973847632e-05,
      "logits/chosen": -0.11463584750890732,
      "logits/rejected": -1.3929070234298706,
      "logps/chosen": -1.0623462200164795,
      "logps/rejected": -1.6423686742782593,
      "loss": 1.124,
      "num_input_tokens_seen": 1067616,
      "odds_ratio_loss": 10.206562995910645,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.10623463243246078,
      "rewards/margins": 0.058002255856990814,
      "rewards/rejected": -0.1642368733882904,
      "sft_loss": 0.10329482704401016,
      "step": 820
    },
    {
      "epoch": 0.06456631660832361,
      "grad_norm": 2.5342459678649902,
      "learning_rate": 4.9487454763925135e-05,
      "logits/chosen": 0.09601770341396332,
      "logits/rejected": -1.3720271587371826,
      "logps/chosen": -0.9235900044441223,
      "logps/rejected": -1.3671214580535889,
      "loss": 0.9705,
      "num_input_tokens_seen": 1083808,
      "odds_ratio_loss": 9.668932914733887,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.09235899150371552,
      "rewards/margins": 0.044353142380714417,
      "rewards/rejected": -0.13671213388442993,
      "sft_loss": 0.0036094163078814745,
      "step": 830
    },
    {
      "epoch": 0.06534422403733955,
      "grad_norm": 2.265723705291748,
      "learning_rate": 4.94750735383168e-05,
      "logits/chosen": -0.2784004509449005,
      "logits/rejected": -1.390732765197754,
      "logps/chosen": -1.2053892612457275,
      "logps/rejected": -1.8831628561019897,
      "loss": 1.2595,
      "num_input_tokens_seen": 1095568,
      "odds_ratio_loss": 12.26213264465332,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.12053891271352768,
      "rewards/margins": 0.06777738034725189,
      "rewards/rejected": -0.18831630051136017,
      "sft_loss": 0.0333065465092659,
      "step": 840
    },
    {
      "epoch": 0.06612213146635551,
      "grad_norm": 4.156375885009766,
      "learning_rate": 4.946254613559803e-05,
      "logits/chosen": -0.0888671875,
      "logits/rejected": -1.1945364475250244,
      "logps/chosen": -0.7892787456512451,
      "logps/rejected": -2.2669517993927,
      "loss": 0.8426,
      "num_input_tokens_seen": 1110960,
      "odds_ratio_loss": 8.253103256225586,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.07892787456512451,
      "rewards/margins": 0.1477673053741455,
      "rewards/rejected": -0.22669517993927002,
      "sft_loss": 0.01730215921998024,
      "step": 850
    },
    {
      "epoch": 0.06690003889537145,
      "grad_norm": 1.8222236633300781,
      "learning_rate": 4.944987263058863e-05,
      "logits/chosen": -0.27693435549736023,
      "logits/rejected": -1.2136726379394531,
      "logps/chosen": -0.9566059112548828,
      "logps/rejected": -1.0269935131072998,
      "loss": 1.0273,
      "num_input_tokens_seen": 1121584,
      "odds_ratio_loss": 10.1735200881958,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.09566058963537216,
      "rewards/margins": 0.007038764655590057,
      "rewards/rejected": -0.10269935429096222,
      "sft_loss": 0.009919771924614906,
      "step": 860
    },
    {
      "epoch": 0.0676779463243874,
      "grad_norm": 1.8523871898651123,
      "learning_rate": 4.9437053098980956e-05,
      "logits/chosen": -0.2266802042722702,
      "logits/rejected": -1.2647974491119385,
      "logps/chosen": -1.1253491640090942,
      "logps/rejected": -1.2813799381256104,
      "loss": 1.1889,
      "num_input_tokens_seen": 1137296,
      "odds_ratio_loss": 11.128896713256836,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.11253491789102554,
      "rewards/margins": 0.01560310460627079,
      "rewards/rejected": -0.12813802063465118,
      "sft_loss": 0.0759621262550354,
      "step": 870
    },
    {
      "epoch": 0.06845585375340335,
      "grad_norm": 8.056722640991211,
      "learning_rate": 4.942408761733952e-05,
      "logits/chosen": -0.10846817493438721,
      "logits/rejected": -1.035222053527832,
      "logps/chosen": -1.1279523372650146,
      "logps/rejected": -1.899043321609497,
      "loss": 1.1801,
      "num_input_tokens_seen": 1152848,
      "odds_ratio_loss": 10.552392959594727,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.11279523372650146,
      "rewards/margins": 0.07710909843444824,
      "rewards/rejected": -0.1899043172597885,
      "sft_loss": 0.12487705796957016,
      "step": 880
    },
    {
      "epoch": 0.06923376118241929,
      "grad_norm": 10.801746368408203,
      "learning_rate": 4.9410976263100515e-05,
      "logits/chosen": -0.4096245765686035,
      "logits/rejected": -0.9306976199150085,
      "logps/chosen": -0.8880082964897156,
      "logps/rejected": -1.1635149717330933,
      "loss": 0.9443,
      "num_input_tokens_seen": 1164000,
      "odds_ratio_loss": 9.348076820373535,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.0888008326292038,
      "rewards/margins": 0.02755066193640232,
      "rewards/rejected": -0.11635150760412216,
      "sft_loss": 0.00947362557053566,
      "step": 890
    },
    {
      "epoch": 0.07001166861143523,
      "grad_norm": 1.9704222679138184,
      "learning_rate": 4.9397719114571386e-05,
      "logits/chosen": 0.06543467938899994,
      "logits/rejected": -1.1813740730285645,
      "logps/chosen": -1.018379807472229,
      "logps/rejected": -1.2177107334136963,
      "loss": 1.0886,
      "num_input_tokens_seen": 1174592,
      "odds_ratio_loss": 10.819978713989258,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.10183799266815186,
      "rewards/margins": 0.019933100789785385,
      "rewards/rejected": -0.12177108228206635,
      "sft_loss": 0.006566903553903103,
      "step": 900
    },
    {
      "epoch": 0.07078957604045119,
      "grad_norm": 1.8523725271224976,
      "learning_rate": 4.9384316250930276e-05,
      "logits/chosen": -0.06241592764854431,
      "logits/rejected": -1.6512749195098877,
      "logps/chosen": -0.9898072481155396,
      "logps/rejected": -1.2340342998504639,
      "loss": 1.0652,
      "num_input_tokens_seen": 1187040,
      "odds_ratio_loss": 10.588815689086914,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.09898072481155396,
      "rewards/margins": 0.02442272938787937,
      "rewards/rejected": -0.12340345233678818,
      "sft_loss": 0.006280974484980106,
      "step": 910
    },
    {
      "epoch": 0.07156748346946713,
      "grad_norm": 5.821229934692383,
      "learning_rate": 4.937076775222566e-05,
      "logits/chosen": -0.2428424060344696,
      "logits/rejected": -1.3016266822814941,
      "logps/chosen": -0.9424579739570618,
      "logps/rejected": -1.9924252033233643,
      "loss": 0.9786,
      "num_input_tokens_seen": 1195472,
      "odds_ratio_loss": 9.530224800109863,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.0942457988858223,
      "rewards/margins": 0.10499672591686249,
      "rewards/rejected": -0.1992425173521042,
      "sft_loss": 0.025574322789907455,
      "step": 920
    },
    {
      "epoch": 0.07234539089848308,
      "grad_norm": 7.764154434204102,
      "learning_rate": 4.93570736993758e-05,
      "logits/chosen": -0.2199537456035614,
      "logits/rejected": -0.9988893270492554,
      "logps/chosen": -0.7839959263801575,
      "logps/rejected": -1.4383164644241333,
      "loss": 0.8377,
      "num_input_tokens_seen": 1208128,
      "odds_ratio_loss": 8.278602600097656,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.07839959859848022,
      "rewards/margins": 0.06543205678462982,
      "rewards/rejected": -0.14383165538311005,
      "sft_loss": 0.009836836718022823,
      "step": 930
    },
    {
      "epoch": 0.07312329832749903,
      "grad_norm": 3.802403211593628,
      "learning_rate": 4.934323417416827e-05,
      "logits/chosen": -0.15197499096393585,
      "logits/rejected": -1.2443273067474365,
      "logps/chosen": -0.9255944490432739,
      "logps/rejected": -1.183050513267517,
      "loss": 0.9847,
      "num_input_tokens_seen": 1216976,
      "odds_ratio_loss": 9.783621788024902,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.09255944192409515,
      "rewards/margins": 0.025745611637830734,
      "rewards/rejected": -0.11830506473779678,
      "sft_loss": 0.0063563063740730286,
      "step": 940
    },
    {
      "epoch": 0.07390120575651497,
      "grad_norm": 1.7451285123825073,
      "learning_rate": 4.932924925925949e-05,
      "logits/chosen": -0.1452958881855011,
      "logits/rejected": -1.4032037258148193,
      "logps/chosen": -1.0042587518692017,
      "logps/rejected": -1.2171082496643066,
      "loss": 1.0711,
      "num_input_tokens_seen": 1230656,
      "odds_ratio_loss": 10.356279373168945,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.10042587667703629,
      "rewards/margins": 0.021284956485033035,
      "rewards/rejected": -0.12171083688735962,
      "sft_loss": 0.03550771623849869,
      "step": 950
    },
    {
      "epoch": 0.07467911318553092,
      "grad_norm": 5.48258638381958,
      "learning_rate": 4.9315119038174206e-05,
      "logits/chosen": 0.07203695178031921,
      "logits/rejected": -1.419390082359314,
      "logps/chosen": -0.8708435893058777,
      "logps/rejected": -2.2798571586608887,
      "loss": 0.9308,
      "num_input_tokens_seen": 1245024,
      "odds_ratio_loss": 8.826228141784668,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.08708436042070389,
      "rewards/margins": 0.1409013420343399,
      "rewards/rejected": -0.2279857099056244,
      "sft_loss": 0.04818565398454666,
      "step": 960
    },
    {
      "epoch": 0.07545702061454687,
      "grad_norm": 5.357232570648193,
      "learning_rate": 4.930084359530502e-05,
      "logits/chosen": -0.06561549007892609,
      "logits/rejected": -1.3615022897720337,
      "logps/chosen": -0.9727898836135864,
      "logps/rejected": -1.3841956853866577,
      "loss": 1.0159,
      "num_input_tokens_seen": 1260400,
      "odds_ratio_loss": 9.965991973876953,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09727897495031357,
      "rewards/margins": 0.04114057868719101,
      "rewards/rejected": -0.13841956853866577,
      "sft_loss": 0.01931620016694069,
      "step": 970
    },
    {
      "epoch": 0.07623492804356281,
      "grad_norm": 6.6753435134887695,
      "learning_rate": 4.9286423015911845e-05,
      "logits/chosen": -0.3039540946483612,
      "logits/rejected": -0.6542695760726929,
      "logps/chosen": -0.9842222929000854,
      "logps/rejected": -1.1113169193267822,
      "loss": 1.0535,
      "num_input_tokens_seen": 1272128,
      "odds_ratio_loss": 10.407747268676758,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.09842222929000854,
      "rewards/margins": 0.012709471397101879,
      "rewards/rejected": -0.1111316904425621,
      "sft_loss": 0.012736847624182701,
      "step": 980
    },
    {
      "epoch": 0.07701283547257877,
      "grad_norm": 1.6576621532440186,
      "learning_rate": 4.927185738612145e-05,
      "logits/chosen": -0.2928544282913208,
      "logits/rejected": -1.067185878753662,
      "logps/chosen": -0.835498034954071,
      "logps/rejected": -1.6010229587554932,
      "loss": 0.8751,
      "num_input_tokens_seen": 1285616,
      "odds_ratio_loss": 8.540815353393555,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.08354981243610382,
      "rewards/margins": 0.07655248790979385,
      "rewards/rejected": -0.16010229289531708,
      "sft_loss": 0.021057048812508583,
      "step": 990
    },
    {
      "epoch": 0.07779074290159471,
      "grad_norm": 2.4466707706451416,
      "learning_rate": 4.925714679292689e-05,
      "logits/chosen": -0.13626167178153992,
      "logits/rejected": -0.7506991624832153,
      "logps/chosen": -1.0066719055175781,
      "logps/rejected": -1.3263317346572876,
      "loss": 1.0701,
      "num_input_tokens_seen": 1304160,
      "odds_ratio_loss": 10.451330184936523,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.10066720098257065,
      "rewards/margins": 0.03196597844362259,
      "rewards/rejected": -0.13263316452503204,
      "sft_loss": 0.024980902671813965,
      "step": 1000
    },
    {
      "epoch": 0.07856865033061065,
      "grad_norm": 8.611452102661133,
      "learning_rate": 4.9242291324187024e-05,
      "logits/chosen": -0.006370346061885357,
      "logits/rejected": -1.4868528842926025,
      "logps/chosen": -0.871174693107605,
      "logps/rejected": -1.2081753015518188,
      "loss": 0.9235,
      "num_input_tokens_seen": 1317664,
      "odds_ratio_loss": 9.131412506103516,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.08711747825145721,
      "rewards/margins": 0.03370004519820213,
      "rewards/rejected": -0.12081752717494965,
      "sft_loss": 0.01033018808811903,
      "step": 1010
    },
    {
      "epoch": 0.07934655775962661,
      "grad_norm": 9.985763549804688,
      "learning_rate": 4.9227291068625983e-05,
      "logits/chosen": -0.009476041421294212,
      "logits/rejected": -1.3522567749023438,
      "logps/chosen": -1.0734918117523193,
      "logps/rejected": -1.6543095111846924,
      "loss": 1.1205,
      "num_input_tokens_seen": 1335264,
      "odds_ratio_loss": 11.110357284545898,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.10734919458627701,
      "rewards/margins": 0.05808175355195999,
      "rewards/rejected": -0.1654309332370758,
      "sft_loss": 0.00942428968846798,
      "step": 1020
    },
    {
      "epoch": 0.08012446518864255,
      "grad_norm": 2.383880138397217,
      "learning_rate": 4.921214611583262e-05,
      "logits/chosen": -0.02636711858212948,
      "logits/rejected": -1.0757944583892822,
      "logps/chosen": -0.8740970492362976,
      "logps/rejected": -1.1859877109527588,
      "loss": 0.9276,
      "num_input_tokens_seen": 1350464,
      "odds_ratio_loss": 9.17290210723877,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.08740971237421036,
      "rewards/margins": 0.031189054250717163,
      "rewards/rejected": -0.11859877407550812,
      "sft_loss": 0.010294022038578987,
      "step": 1030
    },
    {
      "epoch": 0.0809023726176585,
      "grad_norm": 1.8636724948883057,
      "learning_rate": 4.9196856556260004e-05,
      "logits/chosen": 0.16306543350219727,
      "logits/rejected": -1.4347612857818604,
      "logps/chosen": -0.8971742391586304,
      "logps/rejected": -0.8668204545974731,
      "loss": 0.9785,
      "num_input_tokens_seen": 1368112,
      "odds_ratio_loss": 9.759934425354004,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": -0.08971743285655975,
      "rewards/margins": -0.0030353828333318233,
      "rewards/rejected": -0.0866820439696312,
      "sft_loss": 0.002534131519496441,
      "step": 1040
    },
    {
      "epoch": 0.08168028004667445,
      "grad_norm": 17.53750228881836,
      "learning_rate": 4.918142248122488e-05,
      "logits/chosen": -0.10349367558956146,
      "logits/rejected": -1.0907751321792603,
      "logps/chosen": -0.8256258964538574,
      "logps/rejected": -1.1939427852630615,
      "loss": 0.8723,
      "num_input_tokens_seen": 1380816,
      "odds_ratio_loss": 8.530784606933594,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.08256258070468903,
      "rewards/margins": 0.03683171421289444,
      "rewards/rejected": -0.11939430236816406,
      "sft_loss": 0.019271310418844223,
      "step": 1050
    },
    {
      "epoch": 0.08245818747569039,
      "grad_norm": 3.393805742263794,
      "learning_rate": 4.916584398290707e-05,
      "logits/chosen": 0.09704780578613281,
      "logits/rejected": -1.274310827255249,
      "logps/chosen": -0.9823578000068665,
      "logps/rejected": -2.0368709564208984,
      "loss": 1.0289,
      "num_input_tokens_seen": 1394336,
      "odds_ratio_loss": 10.22205924987793,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.09823577105998993,
      "rewards/margins": 0.1054513230919838,
      "rewards/rejected": -0.20368710160255432,
      "sft_loss": 0.006736454553902149,
      "step": 1060
    },
    {
      "epoch": 0.08323609490470633,
      "grad_norm": 1.6751441955566406,
      "learning_rate": 4.915012115434899e-05,
      "logits/chosen": 0.005007547326385975,
      "logits/rejected": -1.2583626508712769,
      "logps/chosen": -0.8267304301261902,
      "logps/rejected": -1.207726240158081,
      "loss": 0.8746,
      "num_input_tokens_seen": 1408224,
      "odds_ratio_loss": 8.649084091186523,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.08267304301261902,
      "rewards/margins": 0.03809957951307297,
      "rewards/rejected": -0.12077263742685318,
      "sft_loss": 0.009731943719089031,
      "step": 1070
    },
    {
      "epoch": 0.08401400233372229,
      "grad_norm": 33.09058380126953,
      "learning_rate": 4.913425408945508e-05,
      "logits/chosen": -0.19444052875041962,
      "logits/rejected": -0.9926478266716003,
      "logps/chosen": -1.0728189945220947,
      "logps/rejected": -1.1655198335647583,
      "loss": 1.15,
      "num_input_tokens_seen": 1418224,
      "odds_ratio_loss": 10.621275901794434,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.10728190094232559,
      "rewards/margins": 0.009270080365240574,
      "rewards/rejected": -0.11655198037624359,
      "sft_loss": 0.08791467547416687,
      "step": 1080
    },
    {
      "epoch": 0.08479190976273823,
      "grad_norm": 2.9355344772338867,
      "learning_rate": 4.911824288299119e-05,
      "logits/chosen": -0.01051928848028183,
      "logits/rejected": -1.152430534362793,
      "logps/chosen": -1.1754026412963867,
      "logps/rejected": -1.0413397550582886,
      "loss": 1.2581,
      "num_input_tokens_seen": 1430928,
      "odds_ratio_loss": 12.01326847076416,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.11754026263952255,
      "rewards/margins": -0.01340627670288086,
      "rewards/rejected": -0.1041339859366417,
      "sft_loss": 0.056798458099365234,
      "step": 1090
    },
    {
      "epoch": 0.08556981719175417,
      "grad_norm": 3.590717077255249,
      "learning_rate": 4.9102087630584096e-05,
      "logits/chosen": -0.20410041511058807,
      "logits/rejected": -0.8998339772224426,
      "logps/chosen": -0.8227648735046387,
      "logps/rejected": -1.7237087488174438,
      "loss": 0.878,
      "num_input_tokens_seen": 1440912,
      "odds_ratio_loss": 8.626799583435059,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.08227649331092834,
      "rewards/margins": 0.09009438753128052,
      "rewards/rejected": -0.17237088084220886,
      "sft_loss": 0.015276836231350899,
      "step": 1100
    },
    {
      "epoch": 0.08634772462077013,
      "grad_norm": 2.8673641681671143,
      "learning_rate": 4.9085788428720845e-05,
      "logits/chosen": -0.22915875911712646,
      "logits/rejected": -0.9836863279342651,
      "logps/chosen": -1.005216360092163,
      "logps/rejected": -1.4263298511505127,
      "loss": 1.06,
      "num_input_tokens_seen": 1450960,
      "odds_ratio_loss": 10.289142608642578,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.10052163898944855,
      "rewards/margins": 0.0421113483607769,
      "rewards/rejected": -0.14263299107551575,
      "sft_loss": 0.03111717477440834,
      "step": 1110
    },
    {
      "epoch": 0.08712563204978607,
      "grad_norm": 3.8207547664642334,
      "learning_rate": 4.9069345374748265e-05,
      "logits/chosen": -0.14207658171653748,
      "logits/rejected": -1.0518529415130615,
      "logps/chosen": -0.7499372959136963,
      "logps/rejected": -1.2938522100448608,
      "loss": 0.7924,
      "num_input_tokens_seen": 1463232,
      "odds_ratio_loss": 7.8319525718688965,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.07499372959136963,
      "rewards/margins": 0.05439149588346481,
      "rewards/rejected": -0.12938521802425385,
      "sft_loss": 0.009246365167200565,
      "step": 1120
    },
    {
      "epoch": 0.08790353947880203,
      "grad_norm": 2.379305839538574,
      "learning_rate": 4.9052758566872305e-05,
      "logits/chosen": -0.0017435208428651094,
      "logits/rejected": -1.5067884922027588,
      "logps/chosen": -0.9974363446235657,
      "logps/rejected": -1.6765762567520142,
      "loss": 1.0578,
      "num_input_tokens_seen": 1478576,
      "odds_ratio_loss": 10.503931045532227,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.09974364191293716,
      "rewards/margins": 0.0679139792919159,
      "rewards/rejected": -0.16765762865543365,
      "sft_loss": 0.007378022186458111,
      "step": 1130
    },
    {
      "epoch": 0.08868144690781797,
      "grad_norm": 2.0183286666870117,
      "learning_rate": 4.903602810415749e-05,
      "logits/chosen": -0.038505952805280685,
      "logits/rejected": -1.4772956371307373,
      "logps/chosen": -1.0257680416107178,
      "logps/rejected": -1.224268913269043,
      "loss": 1.0967,
      "num_input_tokens_seen": 1489696,
      "odds_ratio_loss": 10.111272811889648,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.10257680714130402,
      "rewards/margins": 0.01985008828341961,
      "rewards/rejected": -0.12242688983678818,
      "sft_loss": 0.08557184785604477,
      "step": 1140
    },
    {
      "epoch": 0.08945935433683391,
      "grad_norm": 15.378406524658203,
      "learning_rate": 4.901915408652635e-05,
      "logits/chosen": -0.26834017038345337,
      "logits/rejected": -0.5739148855209351,
      "logps/chosen": -0.7833901643753052,
      "logps/rejected": -1.7032454013824463,
      "loss": 0.8442,
      "num_input_tokens_seen": 1503280,
      "odds_ratio_loss": 8.320596694946289,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.07833901792764664,
      "rewards/margins": 0.09198550879955292,
      "rewards/rejected": -0.17032451927661896,
      "sft_loss": 0.01215345785021782,
      "step": 1150
    },
    {
      "epoch": 0.09023726176584987,
      "grad_norm": 3.9435245990753174,
      "learning_rate": 4.900213661475874e-05,
      "logits/chosen": -0.365293949842453,
      "logits/rejected": -0.7508081197738647,
      "logps/chosen": -1.0041760206222534,
      "logps/rejected": -1.228610873222351,
      "loss": 1.0602,
      "num_input_tokens_seen": 1518464,
      "odds_ratio_loss": 10.431593894958496,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.1004176139831543,
      "rewards/margins": 0.022443488240242004,
      "rewards/rejected": -0.1228611022233963,
      "sft_loss": 0.01701296493411064,
      "step": 1160
    },
    {
      "epoch": 0.09101516919486581,
      "grad_norm": 4.047309398651123,
      "learning_rate": 4.898497579049135e-05,
      "logits/chosen": -0.20835068821907043,
      "logits/rejected": -1.1838566064834595,
      "logps/chosen": -1.052578091621399,
      "logps/rejected": -1.1051318645477295,
      "loss": 1.1269,
      "num_input_tokens_seen": 1534032,
      "odds_ratio_loss": 10.748819351196289,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": -0.10525782406330109,
      "rewards/margins": 0.005255373660475016,
      "rewards/rejected": -0.11051319539546967,
      "sft_loss": 0.052055876702070236,
      "step": 1170
    },
    {
      "epoch": 0.09179307662388175,
      "grad_norm": 5.090392589569092,
      "learning_rate": 4.8967671716217e-05,
      "logits/chosen": -0.4639959931373596,
      "logits/rejected": -0.8973940014839172,
      "logps/chosen": -0.7618675231933594,
      "logps/rejected": -1.0851995944976807,
      "loss": 0.8111,
      "num_input_tokens_seen": 1549760,
      "odds_ratio_loss": 8.026649475097656,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.07618675380945206,
      "rewards/margins": 0.03233320266008377,
      "rewards/rejected": -0.10851994901895523,
      "sft_loss": 0.008396802470088005,
      "step": 1180
    },
    {
      "epoch": 0.09257098405289771,
      "grad_norm": 2.946673631668091,
      "learning_rate": 4.8950224495284095e-05,
      "logits/chosen": 0.011005115695297718,
      "logits/rejected": -1.8329797983169556,
      "logps/chosen": -1.0231001377105713,
      "logps/rejected": -1.3699067831039429,
      "loss": 1.0743,
      "num_input_tokens_seen": 1568048,
      "odds_ratio_loss": 10.699010848999023,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.10231000185012817,
      "rewards/margins": 0.034680671989917755,
      "rewards/rejected": -0.13699068129062653,
      "sft_loss": 0.004405025392770767,
      "step": 1190
    },
    {
      "epoch": 0.09334889148191365,
      "grad_norm": 4.360876560211182,
      "learning_rate": 4.893263423189597e-05,
      "logits/chosen": -0.48811715841293335,
      "logits/rejected": -0.7015589475631714,
      "logps/chosen": -0.983946681022644,
      "logps/rejected": -1.409174919128418,
      "loss": 1.0317,
      "num_input_tokens_seen": 1580768,
      "odds_ratio_loss": 9.939173698425293,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.09839466959238052,
      "rewards/margins": 0.04252283275127411,
      "rewards/rejected": -0.14091750979423523,
      "sft_loss": 0.03777718171477318,
      "step": 1200
    },
    {
      "epoch": 0.0941267989109296,
      "grad_norm": 3.2938485145568848,
      "learning_rate": 4.891490103111027e-05,
      "logits/chosen": -0.3066580891609192,
      "logits/rejected": -1.1762993335723877,
      "logps/chosen": -0.8579041361808777,
      "logps/rejected": -1.3002510070800781,
      "loss": 0.9136,
      "num_input_tokens_seen": 1592448,
      "odds_ratio_loss": 9.070494651794434,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.08579041808843613,
      "rewards/margins": 0.04423467814922333,
      "rewards/rejected": -0.13002508878707886,
      "sft_loss": 0.006512727588415146,
      "step": 1210
    },
    {
      "epoch": 0.09490470633994555,
      "grad_norm": 25.387502670288086,
      "learning_rate": 4.8897024998838363e-05,
      "logits/chosen": -0.6901842355728149,
      "logits/rejected": -1.5692954063415527,
      "logps/chosen": -0.918140709400177,
      "logps/rejected": -1.9830029010772705,
      "loss": 0.9607,
      "num_input_tokens_seen": 1605392,
      "odds_ratio_loss": 8.890172958374023,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.0918140709400177,
      "rewards/margins": 0.10648620128631592,
      "rewards/rejected": -0.1983002871274948,
      "sft_loss": 0.07163845747709274,
      "step": 1220
    },
    {
      "epoch": 0.09568261376896149,
      "grad_norm": 2.1307079792022705,
      "learning_rate": 4.887900624184465e-05,
      "logits/chosen": -0.3583410382270813,
      "logits/rejected": -1.3679367303848267,
      "logps/chosen": -0.8472444415092468,
      "logps/rejected": -2.5962655544281006,
      "loss": 0.8907,
      "num_input_tokens_seen": 1619680,
      "odds_ratio_loss": 8.66484546661377,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.08472445607185364,
      "rewards/margins": 0.17490212619304657,
      "rewards/rejected": -0.259626567363739,
      "sft_loss": 0.024249475449323654,
      "step": 1230
    },
    {
      "epoch": 0.09646052119797745,
      "grad_norm": 4.3973283767700195,
      "learning_rate": 4.886084486774596e-05,
      "logits/chosen": 0.027990514412522316,
      "logits/rejected": -1.3642051219940186,
      "logps/chosen": -0.9216273427009583,
      "logps/rejected": -1.4038372039794922,
      "loss": 0.9652,
      "num_input_tokens_seen": 1635408,
      "odds_ratio_loss": 9.592658996582031,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.09216274321079254,
      "rewards/margins": 0.04822098836302757,
      "rewards/rejected": -0.1403837502002716,
      "sft_loss": 0.00590053666383028,
      "step": 1240
    },
    {
      "epoch": 0.09723842862699339,
      "grad_norm": 1.9861226081848145,
      "learning_rate": 4.8842540985010896e-05,
      "logits/chosen": -0.05565056949853897,
      "logits/rejected": -1.8242801427841187,
      "logps/chosen": -0.8096386194229126,
      "logps/rejected": -2.4190220832824707,
      "loss": 0.8558,
      "num_input_tokens_seen": 1650128,
      "odds_ratio_loss": 8.537267684936523,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.0809638649225235,
      "rewards/margins": 0.1609383374452591,
      "rewards/rejected": -0.2419021874666214,
      "sft_loss": 0.0020962427370250225,
      "step": 1250
    },
    {
      "epoch": 0.09801633605600933,
      "grad_norm": 4.605903148651123,
      "learning_rate": 4.882409470295921e-05,
      "logits/chosen": -0.8531273007392883,
      "logits/rejected": -0.8438383936882019,
      "logps/chosen": -1.0195088386535645,
      "logps/rejected": -1.2654592990875244,
      "loss": 1.0799,
      "num_input_tokens_seen": 1660384,
      "odds_ratio_loss": 10.63732624053955,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.10195086896419525,
      "rewards/margins": 0.024595048278570175,
      "rewards/rejected": -0.12654592096805573,
      "sft_loss": 0.016118118539452553,
      "step": 1260
    },
    {
      "epoch": 0.09879424348502529,
      "grad_norm": 34.797637939453125,
      "learning_rate": 4.880550613176111e-05,
      "logits/chosen": -0.18267883360385895,
      "logits/rejected": -1.4181299209594727,
      "logps/chosen": -1.1587470769882202,
      "logps/rejected": -2.900524854660034,
      "loss": 1.1986,
      "num_input_tokens_seen": 1679104,
      "odds_ratio_loss": 11.581075668334961,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.11587472259998322,
      "rewards/margins": 0.17417779564857483,
      "rewards/rejected": -0.29005253314971924,
      "sft_loss": 0.04045722261071205,
      "step": 1270
    },
    {
      "epoch": 0.09957215091404123,
      "grad_norm": 2.3860249519348145,
      "learning_rate": 4.8786775382436646e-05,
      "logits/chosen": 0.012228338047862053,
      "logits/rejected": -1.394362449645996,
      "logps/chosen": -1.030656099319458,
      "logps/rejected": -2.4141783714294434,
      "loss": 1.0842,
      "num_input_tokens_seen": 1694560,
      "odds_ratio_loss": 10.41369342803955,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.1030656099319458,
      "rewards/margins": 0.13835224509239197,
      "rewards/rejected": -0.24141785502433777,
      "sft_loss": 0.04287029802799225,
      "step": 1280
    },
    {
      "epoch": 0.10035005834305717,
      "grad_norm": 1.527199625968933,
      "learning_rate": 4.8767902566855004e-05,
      "logits/chosen": -0.12059100717306137,
      "logits/rejected": -1.3332120180130005,
      "logps/chosen": -0.8000995516777039,
      "logps/rejected": -1.1178014278411865,
      "loss": 0.8518,
      "num_input_tokens_seen": 1708288,
      "odds_ratio_loss": 8.446915626525879,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.08000995963811874,
      "rewards/margins": 0.03177017718553543,
      "rewards/rejected": -0.11178014427423477,
      "sft_loss": 0.0071195634081959724,
      "step": 1290
    },
    {
      "epoch": 0.10112796577207313,
      "grad_norm": 4.501274108886719,
      "learning_rate": 4.874888779773388e-05,
      "logits/chosen": -0.2184199094772339,
      "logits/rejected": -1.169959306716919,
      "logps/chosen": -0.9113515615463257,
      "logps/rejected": -1.130590796470642,
      "loss": 0.9644,
      "num_input_tokens_seen": 1723024,
      "odds_ratio_loss": 9.515974044799805,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.09113515913486481,
      "rewards/margins": 0.02192392572760582,
      "rewards/rejected": -0.11305908113718033,
      "sft_loss": 0.012761697173118591,
      "step": 1300
    },
    {
      "epoch": 0.10190587320108907,
      "grad_norm": 2.2603354454040527,
      "learning_rate": 4.872973118863877e-05,
      "logits/chosen": -0.058025557547807693,
      "logits/rejected": -0.956487774848938,
      "logps/chosen": -0.8199183344841003,
      "logps/rejected": -1.225602388381958,
      "loss": 0.8717,
      "num_input_tokens_seen": 1731728,
      "odds_ratio_loss": 8.62109375,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.08199183642864227,
      "rewards/margins": 0.04056840017437935,
      "rewards/rejected": -0.12256024032831192,
      "sft_loss": 0.00954120047390461,
      "step": 1310
    },
    {
      "epoch": 0.10268378063010501,
      "grad_norm": 27.04770278930664,
      "learning_rate": 4.871043285398231e-05,
      "logits/chosen": -0.09762338548898697,
      "logits/rejected": -1.3713834285736084,
      "logps/chosen": -0.8707898855209351,
      "logps/rejected": -1.5118682384490967,
      "loss": 0.9145,
      "num_input_tokens_seen": 1743568,
      "odds_ratio_loss": 8.92141056060791,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.0870789885520935,
      "rewards/margins": 0.06410785019397736,
      "rewards/rejected": -0.15118683874607086,
      "sft_loss": 0.02232534810900688,
      "step": 1320
    },
    {
      "epoch": 0.10346168805912097,
      "grad_norm": 2.578408718109131,
      "learning_rate": 4.8690992909023606e-05,
      "logits/chosen": -0.3252541124820709,
      "logits/rejected": -1.0389803647994995,
      "logps/chosen": -0.9129737019538879,
      "logps/rejected": -1.4556406736373901,
      "loss": 0.954,
      "num_input_tokens_seen": 1756272,
      "odds_ratio_loss": 8.928561210632324,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.09129737317562103,
      "rewards/margins": 0.05426669865846634,
      "rewards/rejected": -0.14556407928466797,
      "sft_loss": 0.0611644871532917,
      "step": 1330
    },
    {
      "epoch": 0.10423959548813691,
      "grad_norm": 2.151125907897949,
      "learning_rate": 4.86714114698675e-05,
      "logits/chosen": -0.3139194846153259,
      "logits/rejected": -1.2926034927368164,
      "logps/chosen": -0.7979762554168701,
      "logps/rejected": -1.579063057899475,
      "loss": 0.8471,
      "num_input_tokens_seen": 1769328,
      "odds_ratio_loss": 8.359407424926758,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.0797976404428482,
      "rewards/margins": 0.07810865342617035,
      "rewards/rejected": -0.15790629386901855,
      "sft_loss": 0.011173962615430355,
      "step": 1340
    },
    {
      "epoch": 0.10501750291715285,
      "grad_norm": 3.7997851371765137,
      "learning_rate": 4.865168865346393e-05,
      "logits/chosen": -0.1871212124824524,
      "logits/rejected": -1.3270195722579956,
      "logps/chosen": -0.9265115857124329,
      "logps/rejected": -1.0720789432525635,
      "loss": 0.9921,
      "num_input_tokens_seen": 1783504,
      "odds_ratio_loss": 9.808382034301758,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.09265115857124329,
      "rewards/margins": 0.014556738547980785,
      "rewards/rejected": -0.10720790922641754,
      "sft_loss": 0.011273539625108242,
      "step": 1350
    },
    {
      "epoch": 0.10579541034616881,
      "grad_norm": 9.577467918395996,
      "learning_rate": 4.863182457760722e-05,
      "logits/chosen": -0.16290147602558136,
      "logits/rejected": -1.5737485885620117,
      "logps/chosen": -1.00246000289917,
      "logps/rejected": -1.5742182731628418,
      "loss": 1.0608,
      "num_input_tokens_seen": 1804496,
      "odds_ratio_loss": 10.485246658325195,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.10024599730968475,
      "rewards/margins": 0.057175833731889725,
      "rewards/rejected": -0.15742184221744537,
      "sft_loss": 0.012254795990884304,
      "step": 1360
    },
    {
      "epoch": 0.10657331777518475,
      "grad_norm": 3.4173192977905273,
      "learning_rate": 4.861181936093533e-05,
      "logits/chosen": 0.01892375573515892,
      "logits/rejected": -1.1257383823394775,
      "logps/chosen": -0.9024341702461243,
      "logps/rejected": -1.1515446901321411,
      "loss": 0.9693,
      "num_input_tokens_seen": 1817024,
      "odds_ratio_loss": 9.511759757995605,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.09024341404438019,
      "rewards/margins": 0.02491104230284691,
      "rewards/rejected": -0.11515446752309799,
      "sft_loss": 0.018130768090486526,
      "step": 1370
    },
    {
      "epoch": 0.1073512252042007,
      "grad_norm": 62.1481819152832,
      "learning_rate": 4.859167312292919e-05,
      "logits/chosen": -0.0794883444905281,
      "logits/rejected": -1.1580173969268799,
      "logps/chosen": -1.090137243270874,
      "logps/rejected": -1.5018608570098877,
      "loss": 1.1456,
      "num_input_tokens_seen": 1824992,
      "odds_ratio_loss": 10.897436141967773,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.10901372134685516,
      "rewards/margins": 0.04117236286401749,
      "rewards/rejected": -0.15018606185913086,
      "sft_loss": 0.055823612958192825,
      "step": 1380
    },
    {
      "epoch": 0.10812913263321665,
      "grad_norm": 6.845315456390381,
      "learning_rate": 4.857138598391201e-05,
      "logits/chosen": -0.37189120054244995,
      "logits/rejected": -0.7980047464370728,
      "logps/chosen": -0.8744696378707886,
      "logps/rejected": -1.1153911352157593,
      "loss": 0.9373,
      "num_input_tokens_seen": 1836608,
      "odds_ratio_loss": 9.201163291931152,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.08744696527719498,
      "rewards/margins": 0.02409215085208416,
      "rewards/rejected": -0.11153911054134369,
      "sft_loss": 0.017146050930023193,
      "step": 1390
    },
    {
      "epoch": 0.10890704006223259,
      "grad_norm": 3.4855692386627197,
      "learning_rate": 4.855095806504849e-05,
      "logits/chosen": -0.17076408863067627,
      "logits/rejected": -1.1082465648651123,
      "logps/chosen": -0.8685711026191711,
      "logps/rejected": -1.8029320240020752,
      "loss": 0.9069,
      "num_input_tokens_seen": 1850608,
      "odds_ratio_loss": 8.906305313110352,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.08685711771249771,
      "rewards/margins": 0.0934360921382904,
      "rewards/rejected": -0.1802932173013687,
      "sft_loss": 0.016220781952142715,
      "step": 1400
    },
    {
      "epoch": 0.10968494749124855,
      "grad_norm": 2.627373695373535,
      "learning_rate": 4.8530389488344176e-05,
      "logits/chosen": -0.3422343134880066,
      "logits/rejected": -0.7871395945549011,
      "logps/chosen": -1.2754361629486084,
      "logps/rejected": -2.4510159492492676,
      "loss": 1.3462,
      "num_input_tokens_seen": 1863232,
      "odds_ratio_loss": 12.735507011413574,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.1275436282157898,
      "rewards/margins": 0.11755795776844025,
      "rewards/rejected": -0.24510157108306885,
      "sft_loss": 0.07264798879623413,
      "step": 1410
    },
    {
      "epoch": 0.11046285492026449,
      "grad_norm": 6.909982681274414,
      "learning_rate": 4.850968037664463e-05,
      "logits/chosen": 0.03238892927765846,
      "logits/rejected": -1.570744514465332,
      "logps/chosen": -0.822226881980896,
      "logps/rejected": -2.0880284309387207,
      "loss": 0.8495,
      "num_input_tokens_seen": 1874480,
      "odds_ratio_loss": 8.436205863952637,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08222270011901855,
      "rewards/margins": 0.1265801340341568,
      "rewards/rejected": -0.20880281925201416,
      "sft_loss": 0.005918358452618122,
      "step": 1420
    },
    {
      "epoch": 0.11124076234928043,
      "grad_norm": 2.540519952774048,
      "learning_rate": 4.8488830853634835e-05,
      "logits/chosen": -0.6312833428382874,
      "logits/rejected": -0.735874354839325,
      "logps/chosen": -0.9134576916694641,
      "logps/rejected": -2.435875177383423,
      "loss": 0.9673,
      "num_input_tokens_seen": 1884768,
      "odds_ratio_loss": 9.412214279174805,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.09134577214717865,
      "rewards/margins": 0.1522417664527893,
      "rewards/rejected": -0.24358752369880676,
      "sft_loss": 0.026076728478074074,
      "step": 1430
    },
    {
      "epoch": 0.11201866977829639,
      "grad_norm": 2.2013466358184814,
      "learning_rate": 4.8467841043838315e-05,
      "logits/chosen": -0.2646324932575226,
      "logits/rejected": -1.3403586149215698,
      "logps/chosen": -1.020293116569519,
      "logps/rejected": -2.1002237796783447,
      "loss": 1.0608,
      "num_input_tokens_seen": 1897216,
      "odds_ratio_loss": 10.047431945800781,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.10202930867671967,
      "rewards/margins": 0.10799306631088257,
      "rewards/rejected": -0.21002237498760223,
      "sft_loss": 0.05609869956970215,
      "step": 1440
    },
    {
      "epoch": 0.11279657720731233,
      "grad_norm": 6.7772908210754395,
      "learning_rate": 4.844671107261649e-05,
      "logits/chosen": -0.565276026725769,
      "logits/rejected": -1.2158313989639282,
      "logps/chosen": -1.1465237140655518,
      "logps/rejected": -2.0157454013824463,
      "loss": 1.1853,
      "num_input_tokens_seen": 1908384,
      "odds_ratio_loss": 11.634048461914062,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.1146523728966713,
      "rewards/margins": 0.08692215383052826,
      "rewards/rejected": -0.20157453417778015,
      "sft_loss": 0.021862532943487167,
      "step": 1450
    },
    {
      "epoch": 0.11357448463632827,
      "grad_norm": 1.6834242343902588,
      "learning_rate": 4.842544106616789e-05,
      "logits/chosen": 0.08155706524848938,
      "logits/rejected": -1.4333988428115845,
      "logps/chosen": -0.9613820910453796,
      "logps/rejected": -1.2888075113296509,
      "loss": 1.0162,
      "num_input_tokens_seen": 1923600,
      "odds_ratio_loss": 10.128491401672363,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.09613820910453796,
      "rewards/margins": 0.032742537558078766,
      "rewards/rejected": -0.12888075411319733,
      "sft_loss": 0.0033340167719870806,
      "step": 1460
    },
    {
      "epoch": 0.11435239206534423,
      "grad_norm": 1.6985890865325928,
      "learning_rate": 4.8404031151527375e-05,
      "logits/chosen": 0.07832728326320648,
      "logits/rejected": -1.7995054721832275,
      "logps/chosen": -0.9585608243942261,
      "logps/rejected": -2.9924867153167725,
      "loss": 0.9941,
      "num_input_tokens_seen": 1939600,
      "odds_ratio_loss": 9.870729446411133,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.09585610032081604,
      "rewards/margins": 0.20339255034923553,
      "rewards/rejected": -0.2992486357688904,
      "sft_loss": 0.007033943198621273,
      "step": 1470
    },
    {
      "epoch": 0.11513029949436017,
      "grad_norm": 3.885937452316284,
      "learning_rate": 4.838248145656545e-05,
      "logits/chosen": -0.3160240650177002,
      "logits/rejected": -1.2407915592193604,
      "logps/chosen": -1.0432136058807373,
      "logps/rejected": -1.7586313486099243,
      "loss": 1.0903,
      "num_input_tokens_seen": 1950864,
      "odds_ratio_loss": 10.701848983764648,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.10432137548923492,
      "rewards/margins": 0.07154176384210587,
      "rewards/rejected": -0.175863116979599,
      "sft_loss": 0.020145347341895103,
      "step": 1480
    },
    {
      "epoch": 0.11590820692337611,
      "grad_norm": 6.555927753448486,
      "learning_rate": 4.836079210998743e-05,
      "logits/chosen": -0.38438206911087036,
      "logits/rejected": -1.0794670581817627,
      "logps/chosen": -1.1527174711227417,
      "logps/rejected": -1.513710618019104,
      "loss": 1.2121,
      "num_input_tokens_seen": 1960960,
      "odds_ratio_loss": 10.904621124267578,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.11527176201343536,
      "rewards/margins": 0.036099307239055634,
      "rewards/rejected": -0.1513710618019104,
      "sft_loss": 0.12164745479822159,
      "step": 1490
    },
    {
      "epoch": 0.11668611435239207,
      "grad_norm": 4.298940181732178,
      "learning_rate": 4.833896324133269e-05,
      "logits/chosen": -0.027753740549087524,
      "logits/rejected": -1.0681359767913818,
      "logps/chosen": -0.6166560649871826,
      "logps/rejected": -1.9046653509140015,
      "loss": 0.661,
      "num_input_tokens_seen": 1974304,
      "odds_ratio_loss": 6.545144081115723,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.0616656132042408,
      "rewards/margins": 0.1288009136915207,
      "rewards/rejected": -0.1904665231704712,
      "sft_loss": 0.006462976336479187,
      "step": 1500
    },
    {
      "epoch": 0.11746402178140801,
      "grad_norm": 3.952348470687866,
      "learning_rate": 4.8316994980973924e-05,
      "logits/chosen": -0.04972705617547035,
      "logits/rejected": -1.4225316047668457,
      "logps/chosen": -0.8579155206680298,
      "logps/rejected": -0.9653614163398743,
      "loss": 0.9237,
      "num_input_tokens_seen": 1987552,
      "odds_ratio_loss": 9.174779891967773,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.08579154312610626,
      "rewards/margins": 0.010744589380919933,
      "rewards/rejected": -0.09653614461421967,
      "sft_loss": 0.006242315750569105,
      "step": 1510
    },
    {
      "epoch": 0.11824192921042397,
      "grad_norm": 2.8112475872039795,
      "learning_rate": 4.8294887460116315e-05,
      "logits/chosen": -0.1582581251859665,
      "logits/rejected": -1.0091254711151123,
      "logps/chosen": -0.8943713903427124,
      "logps/rejected": -1.2091583013534546,
      "loss": 0.9495,
      "num_input_tokens_seen": 2004064,
      "odds_ratio_loss": 9.371780395507812,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.08943714946508408,
      "rewards/margins": 0.03147869184613228,
      "rewards/rejected": -0.12091584503650665,
      "sft_loss": 0.012294212356209755,
      "step": 1520
    },
    {
      "epoch": 0.11901983663943991,
      "grad_norm": 2.3471946716308594,
      "learning_rate": 4.82726408107968e-05,
      "logits/chosen": -0.16725249588489532,
      "logits/rejected": -1.5413681268692017,
      "logps/chosen": -1.0477161407470703,
      "logps/rejected": -2.4742424488067627,
      "loss": 1.0902,
      "num_input_tokens_seen": 2016864,
      "odds_ratio_loss": 10.714027404785156,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.10477161407470703,
      "rewards/margins": 0.14265263080596924,
      "rewards/rejected": -0.24742427468299866,
      "sft_loss": 0.01874869130551815,
      "step": 1530
    },
    {
      "epoch": 0.11979774406845585,
      "grad_norm": 3.0610897541046143,
      "learning_rate": 4.825025516588325e-05,
      "logits/chosen": -0.3162890076637268,
      "logits/rejected": -1.1895010471343994,
      "logps/chosen": -1.181998610496521,
      "logps/rejected": -1.7462644577026367,
      "loss": 1.2516,
      "num_input_tokens_seen": 2034576,
      "odds_ratio_loss": 11.94798469543457,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.11819986253976822,
      "rewards/margins": 0.056426577270030975,
      "rewards/rejected": -0.1746264398097992,
      "sft_loss": 0.05677152797579765,
      "step": 1540
    },
    {
      "epoch": 0.1205756514974718,
      "grad_norm": 3.618778944015503,
      "learning_rate": 4.822773065907368e-05,
      "logits/chosen": -0.20199429988861084,
      "logits/rejected": -1.1273068189620972,
      "logps/chosen": -0.8808591961860657,
      "logps/rejected": -2.315542221069336,
      "loss": 0.9078,
      "num_input_tokens_seen": 2048816,
      "odds_ratio_loss": 8.197305679321289,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08808592706918716,
      "rewards/margins": 0.14346830546855927,
      "rewards/rejected": -0.23155423998832703,
      "sft_loss": 0.08805806934833527,
      "step": 1550
    },
    {
      "epoch": 0.12135355892648775,
      "grad_norm": 2.7620632648468018,
      "learning_rate": 4.8205067424895464e-05,
      "logits/chosen": -0.1513085812330246,
      "logits/rejected": -1.329477071762085,
      "logps/chosen": -1.0271084308624268,
      "logps/rejected": -1.7614002227783203,
      "loss": 1.0768,
      "num_input_tokens_seen": 2059440,
      "odds_ratio_loss": 10.682302474975586,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.10271084308624268,
      "rewards/margins": 0.0734291821718216,
      "rewards/rejected": -0.17614004015922546,
      "sft_loss": 0.008544063195586205,
      "step": 1560
    },
    {
      "epoch": 0.12213146635550369,
      "grad_norm": 3.6041388511657715,
      "learning_rate": 4.8182265598704536e-05,
      "logits/chosen": 0.1286245584487915,
      "logits/rejected": -1.131695032119751,
      "logps/chosen": -1.0837154388427734,
      "logps/rejected": -1.2388603687286377,
      "loss": 1.1554,
      "num_input_tokens_seen": 2076736,
      "odds_ratio_loss": 11.479883193969727,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.1083715409040451,
      "rewards/margins": 0.015514492988586426,
      "rewards/rejected": -0.12388603389263153,
      "sft_loss": 0.007456196006387472,
      "step": 1570
    },
    {
      "epoch": 0.12290937378451965,
      "grad_norm": 3.0888729095458984,
      "learning_rate": 4.8159325316684524e-05,
      "logits/chosen": 0.005407279822975397,
      "logits/rejected": -1.2279163599014282,
      "logps/chosen": -0.6726976633071899,
      "logps/rejected": -1.9678910970687866,
      "loss": 0.7018,
      "num_input_tokens_seen": 2087664,
      "odds_ratio_loss": 6.969337463378906,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.06726975739002228,
      "rewards/margins": 0.12951937317848206,
      "rewards/rejected": -0.19678911566734314,
      "sft_loss": 0.0048288991674780846,
      "step": 1580
    },
    {
      "epoch": 0.12368728121353559,
      "grad_norm": 1.3733124732971191,
      "learning_rate": 4.813624671584604e-05,
      "logits/chosen": -0.06323923170566559,
      "logits/rejected": -1.087624430656433,
      "logps/chosen": -0.7786732912063599,
      "logps/rejected": -1.5691801309585571,
      "loss": 0.8094,
      "num_input_tokens_seen": 2101888,
      "odds_ratio_loss": 7.827001094818115,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.07786732167005539,
      "rewards/margins": 0.0790506973862648,
      "rewards/rejected": -0.1569180190563202,
      "sft_loss": 0.026730451732873917,
      "step": 1590
    },
    {
      "epoch": 0.12446518864255153,
      "grad_norm": 1.6684848070144653,
      "learning_rate": 4.811302993402576e-05,
      "logits/chosen": -0.01859951578080654,
      "logits/rejected": -1.343358039855957,
      "logps/chosen": -0.8631114959716797,
      "logps/rejected": -1.6435956954956055,
      "loss": 0.9019,
      "num_input_tokens_seen": 2113792,
      "odds_ratio_loss": 8.9646577835083,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.08631114661693573,
      "rewards/margins": 0.07804843038320541,
      "rewards/rejected": -0.16435958445072174,
      "sft_loss": 0.005461776163429022,
      "step": 1600
    },
    {
      "epoch": 0.12524309607156747,
      "grad_norm": 2.922821283340454,
      "learning_rate": 4.808967510988567e-05,
      "logits/chosen": -0.4168400764465332,
      "logits/rejected": -1.010283350944519,
      "logps/chosen": -0.817193865776062,
      "logps/rejected": -1.413362741470337,
      "loss": 0.8562,
      "num_input_tokens_seen": 2128176,
      "odds_ratio_loss": 8.470296859741211,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.08171939849853516,
      "rewards/margins": 0.059616874903440475,
      "rewards/rejected": -0.14133629202842712,
      "sft_loss": 0.009124183095991611,
      "step": 1610
    },
    {
      "epoch": 0.12602100350058343,
      "grad_norm": 2.7644245624542236,
      "learning_rate": 4.806618238291221e-05,
      "logits/chosen": -0.06522510945796967,
      "logits/rejected": -1.4439966678619385,
      "logps/chosen": -0.9762029647827148,
      "logps/rejected": -1.2714521884918213,
      "loss": 1.0377,
      "num_input_tokens_seen": 2137808,
      "odds_ratio_loss": 10.251848220825195,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.09762029349803925,
      "rewards/margins": 0.029524922370910645,
      "rewards/rejected": -0.1271452009677887,
      "sft_loss": 0.012493262067437172,
      "step": 1620
    },
    {
      "epoch": 0.12679891092959938,
      "grad_norm": 9.554707527160645,
      "learning_rate": 4.8042551893415413e-05,
      "logits/chosen": -0.07397669553756714,
      "logits/rejected": -1.4646739959716797,
      "logps/chosen": -0.9141004681587219,
      "logps/rejected": -1.7428028583526611,
      "loss": 0.9564,
      "num_input_tokens_seen": 2149376,
      "odds_ratio_loss": 9.50117015838623,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.09141004830598831,
      "rewards/margins": 0.0828702375292778,
      "rewards/rejected": -0.17428025603294373,
      "sft_loss": 0.006242091301828623,
      "step": 1630
    },
    {
      "epoch": 0.1275768183586153,
      "grad_norm": 11.70290756225586,
      "learning_rate": 4.801878378252814e-05,
      "logits/chosen": -0.33434975147247314,
      "logits/rejected": -1.3799288272857666,
      "logps/chosen": -0.9573726654052734,
      "logps/rejected": -1.7814598083496094,
      "loss": 0.9976,
      "num_input_tokens_seen": 2162304,
      "odds_ratio_loss": 9.793394088745117,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.0957372635602951,
      "rewards/margins": 0.08240871876478195,
      "rewards/rejected": -0.17814597487449646,
      "sft_loss": 0.018307171761989594,
      "step": 1640
    },
    {
      "epoch": 0.12835472578763127,
      "grad_norm": 2.2834181785583496,
      "learning_rate": 4.799487819220516e-05,
      "logits/chosen": -0.09276885539293289,
      "logits/rejected": -1.5006898641586304,
      "logps/chosen": -0.8897638320922852,
      "logps/rejected": -1.636853575706482,
      "loss": 0.9256,
      "num_input_tokens_seen": 2174896,
      "odds_ratio_loss": 9.220222473144531,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.08897639065980911,
      "rewards/margins": 0.0747089758515358,
      "rewards/rejected": -0.16368535161018372,
      "sft_loss": 0.0035331719554960728,
      "step": 1650
    },
    {
      "epoch": 0.12913263321664722,
      "grad_norm": 2.08956241607666,
      "learning_rate": 4.7970835265222374e-05,
      "logits/chosen": -0.12462923675775528,
      "logits/rejected": -1.3280928134918213,
      "logps/chosen": -0.8080765008926392,
      "logps/rejected": -1.0823023319244385,
      "loss": 0.8614,
      "num_input_tokens_seen": 2189168,
      "odds_ratio_loss": 8.544610023498535,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.0808076411485672,
      "rewards/margins": 0.0274225864559412,
      "rewards/rejected": -0.10823021829128265,
      "sft_loss": 0.006928306072950363,
      "step": 1660
    },
    {
      "epoch": 0.12991054064566315,
      "grad_norm": 2.577354907989502,
      "learning_rate": 4.794665514517588e-05,
      "logits/chosen": -0.08472862094640732,
      "logits/rejected": -1.416602373123169,
      "logps/chosen": -0.7574108242988586,
      "logps/rejected": -1.9338436126708984,
      "loss": 0.7921,
      "num_input_tokens_seen": 2200080,
      "odds_ratio_loss": 7.881539821624756,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.07574108242988586,
      "rewards/margins": 0.1176433190703392,
      "rewards/rejected": -0.19338437914848328,
      "sft_loss": 0.003906466998159885,
      "step": 1670
    },
    {
      "epoch": 0.1306884480746791,
      "grad_norm": 2.179121255874634,
      "learning_rate": 4.792233797648119e-05,
      "logits/chosen": -0.22027191519737244,
      "logits/rejected": -1.1554373502731323,
      "logps/chosen": -0.9135554432868958,
      "logps/rejected": -1.2012169361114502,
      "loss": 0.9724,
      "num_input_tokens_seen": 2210304,
      "odds_ratio_loss": 9.670771598815918,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.09135554730892181,
      "rewards/margins": 0.028766151517629623,
      "rewards/rejected": -0.12012170255184174,
      "sft_loss": 0.005340161733329296,
      "step": 1680
    },
    {
      "epoch": 0.13146635550369506,
      "grad_norm": 2.312490940093994,
      "learning_rate": 4.789788390437233e-05,
      "logits/chosen": -0.25304338335990906,
      "logits/rejected": -0.7925938367843628,
      "logps/chosen": -0.8756221532821655,
      "logps/rejected": -1.050024390220642,
      "loss": 0.9342,
      "num_input_tokens_seen": 2222928,
      "odds_ratio_loss": 8.787582397460938,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.0875622034072876,
      "rewards/margins": 0.017440233379602432,
      "rewards/rejected": -0.10500244051218033,
      "sft_loss": 0.05540759488940239,
      "step": 1690
    },
    {
      "epoch": 0.13224426293271102,
      "grad_norm": 1.9695780277252197,
      "learning_rate": 4.7873293074900984e-05,
      "logits/chosen": 0.15910422801971436,
      "logits/rejected": -1.55144464969635,
      "logps/chosen": -0.951107382774353,
      "logps/rejected": -1.2606693506240845,
      "loss": 1.0041,
      "num_input_tokens_seen": 2235984,
      "odds_ratio_loss": 9.915519714355469,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.09511072933673859,
      "rewards/margins": 0.030956193804740906,
      "rewards/rejected": -0.12606693804264069,
      "sft_loss": 0.012589936144649982,
      "step": 1700
    },
    {
      "epoch": 0.13302217036172695,
      "grad_norm": 2.928964853286743,
      "learning_rate": 4.7848565634935605e-05,
      "logits/chosen": -0.24512776732444763,
      "logits/rejected": -1.1192513704299927,
      "logps/chosen": -1.0003267526626587,
      "logps/rejected": -1.929778814315796,
      "loss": 1.0628,
      "num_input_tokens_seen": 2245280,
      "odds_ratio_loss": 9.978555679321289,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.10003267228603363,
      "rewards/margins": 0.09294520318508148,
      "rewards/rejected": -0.1929778754711151,
      "sft_loss": 0.06496123969554901,
      "step": 1710
    },
    {
      "epoch": 0.1338000777907429,
      "grad_norm": 2.1121087074279785,
      "learning_rate": 4.782370173216056e-05,
      "logits/chosen": -0.08122909069061279,
      "logits/rejected": -1.1934936046600342,
      "logps/chosen": -0.8961094617843628,
      "logps/rejected": -1.861797571182251,
      "loss": 0.9457,
      "num_input_tokens_seen": 2259712,
      "odds_ratio_loss": 9.394018173217773,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.08961095660924911,
      "rewards/margins": 0.09656880795955658,
      "rewards/rejected": -0.1861797571182251,
      "sft_loss": 0.006347438786178827,
      "step": 1720
    },
    {
      "epoch": 0.13457798521975886,
      "grad_norm": 1.7788792848587036,
      "learning_rate": 4.7798701515075236e-05,
      "logits/chosen": 0.067621149122715,
      "logits/rejected": -1.4974925518035889,
      "logps/chosen": -0.9736189842224121,
      "logps/rejected": -1.7845993041992188,
      "loss": 1.0156,
      "num_input_tokens_seen": 2275328,
      "odds_ratio_loss": 10.119224548339844,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.09736189991235733,
      "rewards/margins": 0.0810980275273323,
      "rewards/rejected": -0.17845991253852844,
      "sft_loss": 0.003652214305475354,
      "step": 1730
    },
    {
      "epoch": 0.1353558926487748,
      "grad_norm": 6.806885242462158,
      "learning_rate": 4.777356513299317e-05,
      "logits/chosen": -0.2556004524230957,
      "logits/rejected": -1.1190215349197388,
      "logps/chosen": -1.2167608737945557,
      "logps/rejected": -1.4703128337860107,
      "loss": 1.2947,
      "num_input_tokens_seen": 2285040,
      "odds_ratio_loss": 12.760774612426758,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.12167608737945557,
      "rewards/margins": 0.025355201214551926,
      "rewards/rejected": -0.1470312774181366,
      "sft_loss": 0.01857621595263481,
      "step": 1740
    },
    {
      "epoch": 0.13613380007779075,
      "grad_norm": 3.1653451919555664,
      "learning_rate": 4.7748292736041114e-05,
      "logits/chosen": -0.3644051253795624,
      "logits/rejected": -1.1646645069122314,
      "logps/chosen": -1.006195068359375,
      "logps/rejected": -1.2280415296554565,
      "loss": 1.0739,
      "num_input_tokens_seen": 2296960,
      "odds_ratio_loss": 10.570137023925781,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.10061951726675034,
      "rewards/margins": 0.022184643894433975,
      "rewards/rejected": -0.12280416488647461,
      "sft_loss": 0.016888169571757317,
      "step": 1750
    },
    {
      "epoch": 0.1369117075068067,
      "grad_norm": 7.864021301269531,
      "learning_rate": 4.77228844751582e-05,
      "logits/chosen": -0.11170890182256699,
      "logits/rejected": -0.6443509459495544,
      "logps/chosen": -0.7633668780326843,
      "logps/rejected": -1.3804105520248413,
      "loss": 0.811,
      "num_input_tokens_seen": 2307584,
      "odds_ratio_loss": 7.9055681228637695,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.07633669674396515,
      "rewards/margins": 0.0617043562233448,
      "rewards/rejected": -0.13804103434085846,
      "sft_loss": 0.020423702895641327,
      "step": 1760
    },
    {
      "epoch": 0.13768961493582263,
      "grad_norm": 22.770339965820312,
      "learning_rate": 4.769734050209498e-05,
      "logits/chosen": 0.022691845893859863,
      "logits/rejected": -1.144308090209961,
      "logps/chosen": -0.9744008183479309,
      "logps/rejected": -2.4538748264312744,
      "loss": 1.0189,
      "num_input_tokens_seen": 2322576,
      "odds_ratio_loss": 9.946294784545898,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.09744006395339966,
      "rewards/margins": 0.14794743061065674,
      "rewards/rejected": -0.2453875094652176,
      "sft_loss": 0.024235859513282776,
      "step": 1770
    },
    {
      "epoch": 0.13846752236483859,
      "grad_norm": 1.8606911897659302,
      "learning_rate": 4.7671660969412556e-05,
      "logits/chosen": -0.13664504885673523,
      "logits/rejected": -0.7414021492004395,
      "logps/chosen": -0.9381047487258911,
      "logps/rejected": -1.4822548627853394,
      "loss": 0.9899,
      "num_input_tokens_seen": 2332432,
      "odds_ratio_loss": 9.61351203918457,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.09381048381328583,
      "rewards/margins": 0.0544150173664093,
      "rewards/rejected": -0.14822550117969513,
      "sft_loss": 0.028502438217401505,
      "step": 1780
    },
    {
      "epoch": 0.13924542979385454,
      "grad_norm": 2.0753164291381836,
      "learning_rate": 4.7645846030481656e-05,
      "logits/chosen": -0.0015734911430627108,
      "logits/rejected": -1.2405633926391602,
      "logps/chosen": -0.9424698948860168,
      "logps/rejected": -1.425398349761963,
      "loss": 0.9868,
      "num_input_tokens_seen": 2345360,
      "odds_ratio_loss": 9.795587539672852,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.09424698352813721,
      "rewards/margins": 0.0482928566634655,
      "rewards/rejected": -0.1425398290157318,
      "sft_loss": 0.0072590867057442665,
      "step": 1790
    },
    {
      "epoch": 0.14002333722287047,
      "grad_norm": 1.9002023935317993,
      "learning_rate": 4.761989583948173e-05,
      "logits/chosen": 0.12105624377727509,
      "logits/rejected": -1.2195909023284912,
      "logps/chosen": -1.0945359468460083,
      "logps/rejected": -1.4879124164581299,
      "loss": 1.2197,
      "num_input_tokens_seen": 2356688,
      "odds_ratio_loss": 10.811766624450684,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.10945358127355576,
      "rewards/margins": 0.0393376424908638,
      "rewards/rejected": -0.14879122376441956,
      "sft_loss": 0.1385701596736908,
      "step": 1800
    },
    {
      "epoch": 0.14080124465188643,
      "grad_norm": 4.353161811828613,
      "learning_rate": 4.7593810551399996e-05,
      "logits/chosen": -0.2872072756290436,
      "logits/rejected": -1.187946081161499,
      "logps/chosen": -1.1624119281768799,
      "logps/rejected": -2.9942376613616943,
      "loss": 1.2146,
      "num_input_tokens_seen": 2368128,
      "odds_ratio_loss": 11.870499610900879,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.11624119430780411,
      "rewards/margins": 0.18318259716033936,
      "rewards/rejected": -0.29942378401756287,
      "sft_loss": 0.027569111436605453,
      "step": 1810
    },
    {
      "epoch": 0.14157915208090238,
      "grad_norm": 2.369858741760254,
      "learning_rate": 4.756759032203058e-05,
      "logits/chosen": -0.4844452440738678,
      "logits/rejected": -1.1404305696487427,
      "logps/chosen": -0.870577335357666,
      "logps/rejected": -2.190336227416992,
      "loss": 0.8944,
      "num_input_tokens_seen": 2379456,
      "odds_ratio_loss": 8.819035530090332,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.08705773204565048,
      "rewards/margins": 0.13197587430477142,
      "rewards/rejected": -0.2190335988998413,
      "sft_loss": 0.012462143786251545,
      "step": 1820
    },
    {
      "epoch": 0.1423570595099183,
      "grad_norm": 2.999417304992676,
      "learning_rate": 4.754123530797349e-05,
      "logits/chosen": -0.22438016533851624,
      "logits/rejected": -1.3170385360717773,
      "logps/chosen": -0.9477856755256653,
      "logps/rejected": -1.3005917072296143,
      "loss": 1.0135,
      "num_input_tokens_seen": 2390576,
      "odds_ratio_loss": 10.019603729248047,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.09477856755256653,
      "rewards/margins": 0.03528060019016266,
      "rewards/rejected": -0.1300591677427292,
      "sft_loss": 0.011541618965566158,
      "step": 1830
    },
    {
      "epoch": 0.14313496693893427,
      "grad_norm": 9.854547500610352,
      "learning_rate": 4.7514745666633765e-05,
      "logits/chosen": -0.17480389773845673,
      "logits/rejected": -1.8269084692001343,
      "logps/chosen": -0.9835509061813354,
      "logps/rejected": -1.5236918926239014,
      "loss": 1.0258,
      "num_input_tokens_seen": 2403664,
      "odds_ratio_loss": 10.115317344665527,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.09835509955883026,
      "rewards/margins": 0.05401410534977913,
      "rewards/rejected": -0.1523691713809967,
      "sft_loss": 0.01426604948937893,
      "step": 1840
    },
    {
      "epoch": 0.14391287436795022,
      "grad_norm": 1.7128846645355225,
      "learning_rate": 4.748812155622051e-05,
      "logits/chosen": -0.6130805015563965,
      "logits/rejected": -1.215207815170288,
      "logps/chosen": -1.1691943407058716,
      "logps/rejected": -2.4545435905456543,
      "loss": 1.2283,
      "num_input_tokens_seen": 2412992,
      "odds_ratio_loss": 11.835399627685547,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.11691941320896149,
      "rewards/margins": 0.1285349428653717,
      "rewards/rejected": -0.245454341173172,
      "sft_loss": 0.04480103775858879,
      "step": 1850
    },
    {
      "epoch": 0.14469078179696615,
      "grad_norm": 2.1353652477264404,
      "learning_rate": 4.746136313574593e-05,
      "logits/chosen": -0.2938610017299652,
      "logits/rejected": -1.362804651260376,
      "logps/chosen": -0.9929152727127075,
      "logps/rejected": -2.309945583343506,
      "loss": 1.0447,
      "num_input_tokens_seen": 2425600,
      "odds_ratio_loss": 10.25415325164795,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.09929152578115463,
      "rewards/margins": 0.13170306384563446,
      "rewards/rejected": -0.23099461197853088,
      "sft_loss": 0.019318608567118645,
      "step": 1860
    },
    {
      "epoch": 0.1454686892259821,
      "grad_norm": 5.580964088439941,
      "learning_rate": 4.74344705650244e-05,
      "logits/chosen": -0.2848891615867615,
      "logits/rejected": -1.2315860986709595,
      "logps/chosen": -0.9934043884277344,
      "logps/rejected": -1.4588521718978882,
      "loss": 1.0503,
      "num_input_tokens_seen": 2434992,
      "odds_ratio_loss": 10.418456077575684,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.09934045374393463,
      "rewards/margins": 0.046544767916202545,
      "rewards/rejected": -0.14588521420955658,
      "sft_loss": 0.008504530414938927,
      "step": 1870
    },
    {
      "epoch": 0.14624659665499806,
      "grad_norm": 3.9102535247802734,
      "learning_rate": 4.74074440046715e-05,
      "logits/chosen": -0.15559545159339905,
      "logits/rejected": -1.0929044485092163,
      "logps/chosen": -0.9797903299331665,
      "logps/rejected": -1.7228037118911743,
      "loss": 1.0229,
      "num_input_tokens_seen": 2449712,
      "odds_ratio_loss": 10.148366928100586,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.09797903150320053,
      "rewards/margins": 0.0743013322353363,
      "rewards/rejected": -0.17228034138679504,
      "sft_loss": 0.008058695122599602,
      "step": 1880
    },
    {
      "epoch": 0.147024504084014,
      "grad_norm": 2.150550603866577,
      "learning_rate": 4.738028361610308e-05,
      "logits/chosen": -0.06973959505558014,
      "logits/rejected": -1.7949050664901733,
      "logps/chosen": -0.9789167642593384,
      "logps/rejected": -1.9717823266983032,
      "loss": 1.0202,
      "num_input_tokens_seen": 2466224,
      "odds_ratio_loss": 9.753425598144531,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.0978916734457016,
      "rewards/margins": 0.09928654879331589,
      "rewards/rejected": -0.19717822968959808,
      "sft_loss": 0.04484931379556656,
      "step": 1890
    },
    {
      "epoch": 0.14780241151302995,
      "grad_norm": 1.9614287614822388,
      "learning_rate": 4.7352989561534254e-05,
      "logits/chosen": -0.08428236842155457,
      "logits/rejected": -1.1122157573699951,
      "logps/chosen": -0.8200074434280396,
      "logps/rejected": -1.4336493015289307,
      "loss": 0.8636,
      "num_input_tokens_seen": 2481952,
      "odds_ratio_loss": 8.588362693786621,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.0820007398724556,
      "rewards/margins": 0.06136419251561165,
      "rewards/rejected": -0.14336493611335754,
      "sft_loss": 0.0047540245577692986,
      "step": 1900
    },
    {
      "epoch": 0.1485803189420459,
      "grad_norm": 2.2471156120300293,
      "learning_rate": 4.732556200397848e-05,
      "logits/chosen": 0.047021109610795975,
      "logits/rejected": -1.0816609859466553,
      "logps/chosen": -0.7682046294212341,
      "logps/rejected": -0.7830635905265808,
      "loss": 0.8538,
      "num_input_tokens_seen": 2495664,
      "odds_ratio_loss": 8.502764701843262,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.07682047039270401,
      "rewards/margins": 0.0014858979266136885,
      "rewards/rejected": -0.07830636203289032,
      "sft_loss": 0.0034969509579241276,
      "step": 1910
    },
    {
      "epoch": 0.14935822637106183,
      "grad_norm": 1.4071708917617798,
      "learning_rate": 4.729800110724654e-05,
      "logits/chosen": -0.16252145171165466,
      "logits/rejected": -1.0662734508514404,
      "logps/chosen": -0.9402019381523132,
      "logps/rejected": -1.3474619388580322,
      "loss": 1.0023,
      "num_input_tokens_seen": 2512992,
      "odds_ratio_loss": 9.984285354614258,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.09402020275592804,
      "rewards/margins": 0.040725983679294586,
      "rewards/rejected": -0.13474617898464203,
      "sft_loss": 0.0038742502219974995,
      "step": 1920
    },
    {
      "epoch": 0.1501361338000778,
      "grad_norm": 2.530600070953369,
      "learning_rate": 4.727030703594557e-05,
      "logits/chosen": -0.022717636078596115,
      "logits/rejected": -1.2906973361968994,
      "logps/chosen": -0.9707010388374329,
      "logps/rejected": -1.3414376974105835,
      "loss": 1.0211,
      "num_input_tokens_seen": 2529264,
      "odds_ratio_loss": 10.094282150268555,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.09707011282444,
      "rewards/margins": 0.037073664367198944,
      "rewards/rejected": -0.13414378464221954,
      "sft_loss": 0.011715163476765156,
      "step": 1930
    },
    {
      "epoch": 0.15091404122909374,
      "grad_norm": 6.173549652099609,
      "learning_rate": 4.7242479955478145e-05,
      "logits/chosen": -0.04072573408484459,
      "logits/rejected": -0.8758264780044556,
      "logps/chosen": -0.844284176826477,
      "logps/rejected": -1.7218849658966064,
      "loss": 0.8893,
      "num_input_tokens_seen": 2537824,
      "odds_ratio_loss": 8.815264701843262,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.08442841470241547,
      "rewards/margins": 0.0877600908279419,
      "rewards/rejected": -0.17218850553035736,
      "sft_loss": 0.007810283452272415,
      "step": 1940
    },
    {
      "epoch": 0.1516919486581097,
      "grad_norm": 3.6885008811950684,
      "learning_rate": 4.721452003204118e-05,
      "logits/chosen": -0.03177540749311447,
      "logits/rejected": -1.4413988590240479,
      "logps/chosen": -0.8689084053039551,
      "logps/rejected": -2.869048595428467,
      "loss": 0.8987,
      "num_input_tokens_seen": 2551184,
      "odds_ratio_loss": 8.938272476196289,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08689083904027939,
      "rewards/margins": 0.20001402497291565,
      "rewards/rejected": -0.28690487146377563,
      "sft_loss": 0.004882193636149168,
      "step": 1950
    },
    {
      "epoch": 0.15246985608712563,
      "grad_norm": 10.387799263000488,
      "learning_rate": 4.7186427432625005e-05,
      "logits/chosen": -0.015120279975235462,
      "logits/rejected": -0.9145073890686035,
      "logps/chosen": -0.6851322054862976,
      "logps/rejected": -1.577131986618042,
      "loss": 0.7254,
      "num_input_tokens_seen": 2568528,
      "odds_ratio_loss": 7.129334449768066,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.06851321458816528,
      "rewards/margins": 0.0891999751329422,
      "rewards/rejected": -0.15771318972110748,
      "sft_loss": 0.012512214481830597,
      "step": 1960
    },
    {
      "epoch": 0.15324776351614158,
      "grad_norm": 12.012364387512207,
      "learning_rate": 4.715820232501239e-05,
      "logits/chosen": -0.10855318605899811,
      "logits/rejected": -1.3677846193313599,
      "logps/chosen": -0.998216450214386,
      "logps/rejected": -1.1140133142471313,
      "loss": 1.064,
      "num_input_tokens_seen": 2582928,
      "odds_ratio_loss": 10.501612663269043,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.09982164204120636,
      "rewards/margins": 0.01157968956977129,
      "rewards/rejected": -0.11140134185552597,
      "sft_loss": 0.013802697882056236,
      "step": 1970
    },
    {
      "epoch": 0.15402567094515754,
      "grad_norm": 2.588481903076172,
      "learning_rate": 4.7129844877777456e-05,
      "logits/chosen": -0.28296536207199097,
      "logits/rejected": -1.1763520240783691,
      "logps/chosen": -0.9495035409927368,
      "logps/rejected": -1.0533324480056763,
      "loss": 1.0163,
      "num_input_tokens_seen": 2595600,
      "odds_ratio_loss": 10.042470932006836,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.0949503630399704,
      "rewards/margins": 0.010382888838648796,
      "rewards/rejected": -0.10533325374126434,
      "sft_loss": 0.01205428782850504,
      "step": 1980
    },
    {
      "epoch": 0.15480357837417347,
      "grad_norm": 5.5801897048950195,
      "learning_rate": 4.710135526028477e-05,
      "logits/chosen": 0.04890453815460205,
      "logits/rejected": -1.548763632774353,
      "logps/chosen": -0.9930458068847656,
      "logps/rejected": -1.474796175956726,
      "loss": 1.0394,
      "num_input_tokens_seen": 2616560,
      "odds_ratio_loss": 10.355806350708008,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.09930458664894104,
      "rewards/margins": 0.048175033181905746,
      "rewards/rejected": -0.14747962355613708,
      "sft_loss": 0.0038034829776734114,
      "step": 1990
    },
    {
      "epoch": 0.15558148580318942,
      "grad_norm": 1.9893940687179565,
      "learning_rate": 4.707273364268825e-05,
      "logits/chosen": -0.15720073878765106,
      "logits/rejected": -0.8983931541442871,
      "logps/chosen": -1.2000892162322998,
      "logps/rejected": -1.751647710800171,
      "loss": 1.2625,
      "num_input_tokens_seen": 2629616,
      "odds_ratio_loss": 11.613740921020508,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.1200089082121849,
      "rewards/margins": 0.055155880749225616,
      "rewards/rejected": -0.17516478896141052,
      "sft_loss": 0.10116754472255707,
      "step": 2000
    },
    {
      "epoch": 0.15635939323220538,
      "grad_norm": 1.1382113695144653,
      "learning_rate": 4.70439801959302e-05,
      "logits/chosen": -0.15353144705295563,
      "logits/rejected": -1.406969428062439,
      "logps/chosen": -1.0247169733047485,
      "logps/rejected": -1.8631995916366577,
      "loss": 1.0967,
      "num_input_tokens_seen": 2643328,
      "odds_ratio_loss": 10.818486213684082,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.10247169435024261,
      "rewards/margins": 0.08384827524423599,
      "rewards/rejected": -0.1863199770450592,
      "sft_loss": 0.01486129779368639,
      "step": 2010
    },
    {
      "epoch": 0.1571373006612213,
      "grad_norm": 9.748759269714355,
      "learning_rate": 4.701509509174027e-05,
      "logits/chosen": 0.012250833213329315,
      "logits/rejected": -1.0942684412002563,
      "logps/chosen": -0.9024358987808228,
      "logps/rejected": -1.7884238958358765,
      "loss": 0.9446,
      "num_input_tokens_seen": 2653136,
      "odds_ratio_loss": 9.02326488494873,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.09024359285831451,
      "rewards/margins": 0.08859880268573761,
      "rewards/rejected": -0.17884239554405212,
      "sft_loss": 0.042305268347263336,
      "step": 2020
    },
    {
      "epoch": 0.15791520809023726,
      "grad_norm": 2.3677189350128174,
      "learning_rate": 4.698607850263442e-05,
      "logits/chosen": -0.1969408094882965,
      "logits/rejected": -1.3645201921463013,
      "logps/chosen": -0.9819061160087585,
      "logps/rejected": -1.4384701251983643,
      "loss": 1.036,
      "num_input_tokens_seen": 2662192,
      "odds_ratio_loss": 10.110983848571777,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.09819060564041138,
      "rewards/margins": 0.045656394213438034,
      "rewards/rejected": -0.1438470035791397,
      "sft_loss": 0.024949556216597557,
      "step": 2030
    },
    {
      "epoch": 0.15869311551925322,
      "grad_norm": 1.7309666872024536,
      "learning_rate": 4.6956930601913924e-05,
      "logits/chosen": 0.028863852843642235,
      "logits/rejected": -1.6371591091156006,
      "logps/chosen": -1.029279112815857,
      "logps/rejected": -2.1302683353424072,
      "loss": 1.069,
      "num_input_tokens_seen": 2677456,
      "odds_ratio_loss": 10.366458892822266,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.10292792320251465,
      "rewards/margins": 0.11009888350963593,
      "rewards/rejected": -0.21302680671215057,
      "sft_loss": 0.032364509999752045,
      "step": 2040
    },
    {
      "epoch": 0.15947102294826915,
      "grad_norm": 2.105114698410034,
      "learning_rate": 4.6927651563664296e-05,
      "logits/chosen": -0.2948930859565735,
      "logits/rejected": -0.6920993328094482,
      "logps/chosen": -0.723526656627655,
      "logps/rejected": -1.1117141246795654,
      "loss": 0.7734,
      "num_input_tokens_seen": 2692944,
      "odds_ratio_loss": 7.633302211761475,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.07235266268253326,
      "rewards/margins": 0.03881874680519104,
      "rewards/rejected": -0.1111714243888855,
      "sft_loss": 0.010118861682713032,
      "step": 2050
    },
    {
      "epoch": 0.1602489303772851,
      "grad_norm": 8.95576000213623,
      "learning_rate": 4.689824156275428e-05,
      "logits/chosen": -0.4269348680973053,
      "logits/rejected": -0.9338502883911133,
      "logps/chosen": -0.8120001554489136,
      "logps/rejected": -1.077885389328003,
      "loss": 0.8686,
      "num_input_tokens_seen": 2703904,
      "odds_ratio_loss": 8.558115005493164,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.0812000185251236,
      "rewards/margins": 0.02658851072192192,
      "rewards/rejected": -0.10778852552175522,
      "sft_loss": 0.01276195328682661,
      "step": 2060
    },
    {
      "epoch": 0.16102683780630106,
      "grad_norm": 7.518883228302002,
      "learning_rate": 4.686870077483478e-05,
      "logits/chosen": -0.5116538405418396,
      "logits/rejected": -0.652612566947937,
      "logps/chosen": -1.0511877536773682,
      "logps/rejected": -1.480696439743042,
      "loss": 1.1021,
      "num_input_tokens_seen": 2714112,
      "odds_ratio_loss": 10.629408836364746,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.1051187738776207,
      "rewards/margins": 0.04295086860656738,
      "rewards/rejected": -0.14806964993476868,
      "sft_loss": 0.03919699788093567,
      "step": 2070
    },
    {
      "epoch": 0.161804745235317,
      "grad_norm": 2.5984585285186768,
      "learning_rate": 4.6839029376337825e-05,
      "logits/chosen": -0.25364458560943604,
      "logits/rejected": -1.1314189434051514,
      "logps/chosen": -0.9039451479911804,
      "logps/rejected": -3.1899101734161377,
      "loss": 0.9435,
      "num_input_tokens_seen": 2728544,
      "odds_ratio_loss": 9.305583000183105,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.0903945118188858,
      "rewards/margins": 0.22859647870063782,
      "rewards/rejected": -0.3189910054206848,
      "sft_loss": 0.012947039678692818,
      "step": 2080
    },
    {
      "epoch": 0.16258265266433294,
      "grad_norm": 2.2118406295776367,
      "learning_rate": 4.680922754447554e-05,
      "logits/chosen": -0.28953272104263306,
      "logits/rejected": -0.8960913419723511,
      "logps/chosen": -1.0112407207489014,
      "logps/rejected": -1.2518157958984375,
      "loss": 1.0744,
      "num_input_tokens_seen": 2742592,
      "odds_ratio_loss": 9.534306526184082,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.10112407058477402,
      "rewards/margins": 0.02405751682817936,
      "rewards/rejected": -0.12518158555030823,
      "sft_loss": 0.12095620483160019,
      "step": 2090
    },
    {
      "epoch": 0.1633605600933489,
      "grad_norm": 1.8755844831466675,
      "learning_rate": 4.6779295457239025e-05,
      "logits/chosen": -0.12629246711730957,
      "logits/rejected": -1.1058032512664795,
      "logps/chosen": -0.8406488299369812,
      "logps/rejected": -1.691583275794983,
      "loss": 0.8857,
      "num_input_tokens_seen": 2754208,
      "odds_ratio_loss": 8.595414161682129,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.08406487852334976,
      "rewards/margins": 0.0850934311747551,
      "rewards/rejected": -0.16915829479694366,
      "sft_loss": 0.026196395978331566,
      "step": 2100
    },
    {
      "epoch": 0.16413846752236483,
      "grad_norm": 5.349771022796631,
      "learning_rate": 4.6749233293397346e-05,
      "logits/chosen": -0.8121733665466309,
      "logits/rejected": -0.8647876977920532,
      "logps/chosen": -0.8223133087158203,
      "logps/rejected": -1.5897068977355957,
      "loss": 0.8577,
      "num_input_tokens_seen": 2763648,
      "odds_ratio_loss": 8.505762100219727,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.08223132789134979,
      "rewards/margins": 0.0767393708229065,
      "rewards/rejected": -0.1589706987142563,
      "sft_loss": 0.007078877184540033,
      "step": 2110
    },
    {
      "epoch": 0.16491637495138078,
      "grad_norm": 0.6865819692611694,
      "learning_rate": 4.671904123249645e-05,
      "logits/chosen": -0.2070043534040451,
      "logits/rejected": -1.5442055463790894,
      "logps/chosen": -0.7468840479850769,
      "logps/rejected": -1.994861364364624,
      "loss": 0.7813,
      "num_input_tokens_seen": 2778096,
      "odds_ratio_loss": 7.69118595123291,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.07468841224908829,
      "rewards/margins": 0.12479771673679352,
      "rewards/rejected": -0.1994861364364624,
      "sft_loss": 0.012225426733493805,
      "step": 2120
    },
    {
      "epoch": 0.16569428238039674,
      "grad_norm": 5.290821552276611,
      "learning_rate": 4.66887194548581e-05,
      "logits/chosen": 0.052118025720119476,
      "logits/rejected": -1.3986949920654297,
      "logps/chosen": -0.9030756950378418,
      "logps/rejected": -1.7299425601959229,
      "loss": 0.9546,
      "num_input_tokens_seen": 2789712,
      "odds_ratio_loss": 9.505033493041992,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.0903075709939003,
      "rewards/margins": 0.08268668502569199,
      "rewards/rejected": -0.17299427092075348,
      "sft_loss": 0.00411962578073144,
      "step": 2130
    },
    {
      "epoch": 0.16647218980941267,
      "grad_norm": 4.0711798667907715,
      "learning_rate": 4.665826814157878e-05,
      "logits/chosen": -0.04582834988832474,
      "logits/rejected": -1.3113749027252197,
      "logps/chosen": -1.0264580249786377,
      "logps/rejected": -1.1542977094650269,
      "loss": 1.0925,
      "num_input_tokens_seen": 2805600,
      "odds_ratio_loss": 10.873018264770508,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.10264579206705093,
      "rewards/margins": 0.01278398372232914,
      "rewards/rejected": -0.11542978137731552,
      "sft_loss": 0.005238743498921394,
      "step": 2140
    },
    {
      "epoch": 0.16725009723842862,
      "grad_norm": 1.4560567140579224,
      "learning_rate": 4.662768747452862e-05,
      "logits/chosen": -0.21474024653434753,
      "logits/rejected": -1.1197646856307983,
      "logps/chosen": -0.805280864238739,
      "logps/rejected": -1.3285717964172363,
      "loss": 0.86,
      "num_input_tokens_seen": 2816848,
      "odds_ratio_loss": 7.857496738433838,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.08052808791399002,
      "rewards/margins": 0.05232909321784973,
      "rewards/rejected": -0.13285718858242035,
      "sft_loss": 0.07425809651613235,
      "step": 2150
    },
    {
      "epoch": 0.16802800466744458,
      "grad_norm": 1.6971755027770996,
      "learning_rate": 4.6596977636350325e-05,
      "logits/chosen": -0.24752207100391388,
      "logits/rejected": -1.3036912679672241,
      "logps/chosen": -1.1429294347763062,
      "logps/rejected": -3.008378267288208,
      "loss": 1.2139,
      "num_input_tokens_seen": 2827472,
      "odds_ratio_loss": 11.748734474182129,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.1142929419875145,
      "rewards/margins": 0.18654491007328033,
      "rewards/rejected": -0.30083784461021423,
      "sft_loss": 0.03905164450407028,
      "step": 2160
    },
    {
      "epoch": 0.1688059120964605,
      "grad_norm": 3.530958652496338,
      "learning_rate": 4.656613881045808e-05,
      "logits/chosen": -0.20504558086395264,
      "logits/rejected": -1.0912846326828003,
      "logps/chosen": -0.8128896951675415,
      "logps/rejected": -1.1443490982055664,
      "loss": 0.8659,
      "num_input_tokens_seen": 2839984,
      "odds_ratio_loss": 8.59929084777832,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.08128897100687027,
      "rewards/margins": 0.03314594551920891,
      "rewards/rejected": -0.11443491280078888,
      "sft_loss": 0.005995646119117737,
      "step": 2170
    },
    {
      "epoch": 0.16958381952547646,
      "grad_norm": 2.934725761413574,
      "learning_rate": 4.653517118103642e-05,
      "logits/chosen": -0.06856352090835571,
      "logits/rejected": -1.1658178567886353,
      "logps/chosen": -0.7763811349868774,
      "logps/rejected": -3.0190224647521973,
      "loss": 0.8124,
      "num_input_tokens_seen": 2851120,
      "odds_ratio_loss": 7.866454124450684,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.07763811200857162,
      "rewards/margins": 0.22426410019397736,
      "rewards/rejected": -0.30190223455429077,
      "sft_loss": 0.025785360485315323,
      "step": 2180
    },
    {
      "epoch": 0.17036172695449242,
      "grad_norm": 31.268569946289062,
      "learning_rate": 4.6504074933039185e-05,
      "logits/chosen": -0.23514564335346222,
      "logits/rejected": -0.8977664709091187,
      "logps/chosen": -0.7419293522834778,
      "logps/rejected": -1.1732724905014038,
      "loss": 0.7893,
      "num_input_tokens_seen": 2859984,
      "odds_ratio_loss": 7.76514196395874,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.07419292628765106,
      "rewards/margins": 0.04313432425260544,
      "rewards/rejected": -0.1173272579908371,
      "sft_loss": 0.012793630361557007,
      "step": 2190
    },
    {
      "epoch": 0.17113963438350835,
      "grad_norm": 3.4862051010131836,
      "learning_rate": 4.647285025218837e-05,
      "logits/chosen": 0.15971137583255768,
      "logits/rejected": -1.107314109802246,
      "logps/chosen": -1.1586195230484009,
      "logps/rejected": -1.0516465902328491,
      "loss": 1.2555,
      "num_input_tokens_seen": 2876672,
      "odds_ratio_loss": 11.734086036682129,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.11586196720600128,
      "rewards/margins": -0.010697288438677788,
      "rewards/rejected": -0.10516466945409775,
      "sft_loss": 0.08209719508886337,
      "step": 2200
    },
    {
      "epoch": 0.1719175418125243,
      "grad_norm": 2.9709460735321045,
      "learning_rate": 4.644149732497305e-05,
      "logits/chosen": -0.35991913080215454,
      "logits/rejected": -0.9069361686706543,
      "logps/chosen": -0.9952637553215027,
      "logps/rejected": -1.203416109085083,
      "loss": 1.0537,
      "num_input_tokens_seen": 2887568,
      "odds_ratio_loss": 9.413312911987305,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.09952638298273087,
      "rewards/margins": 0.020815229043364525,
      "rewards/rejected": -0.12034161388874054,
      "sft_loss": 0.11237955093383789,
      "step": 2210
    },
    {
      "epoch": 0.17269544924154026,
      "grad_norm": 2.8426527976989746,
      "learning_rate": 4.6410016338648246e-05,
      "logits/chosen": -0.5125675201416016,
      "logits/rejected": -0.7049168348312378,
      "logps/chosen": -0.9495420455932617,
      "logps/rejected": -1.2189608812332153,
      "loss": 1.0015,
      "num_input_tokens_seen": 2901824,
      "odds_ratio_loss": 9.808638572692871,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.09495420008897781,
      "rewards/margins": 0.026941895484924316,
      "rewards/rejected": -0.12189608812332153,
      "sft_loss": 0.02062970958650112,
      "step": 2220
    },
    {
      "epoch": 0.17347335667055622,
      "grad_norm": 3.1397180557250977,
      "learning_rate": 4.637840748123379e-05,
      "logits/chosen": -0.31803905963897705,
      "logits/rejected": -1.4113385677337646,
      "logps/chosen": -1.1904270648956299,
      "logps/rejected": -1.7130476236343384,
      "loss": 1.2448,
      "num_input_tokens_seen": 2914160,
      "odds_ratio_loss": 12.293588638305664,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.11904270946979523,
      "rewards/margins": 0.052262045443058014,
      "rewards/rejected": -0.17130474746227264,
      "sft_loss": 0.015424099750816822,
      "step": 2230
    },
    {
      "epoch": 0.17425126409957215,
      "grad_norm": 1.6466132402420044,
      "learning_rate": 4.634667094151325e-05,
      "logits/chosen": -0.2235032320022583,
      "logits/rejected": -0.9990476369857788,
      "logps/chosen": -0.9258815050125122,
      "logps/rejected": -1.9906742572784424,
      "loss": 0.9599,
      "num_input_tokens_seen": 2925072,
      "odds_ratio_loss": 9.475238800048828,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.0925881564617157,
      "rewards/margins": 0.10647926479578018,
      "rewards/rejected": -0.19906741380691528,
      "sft_loss": 0.012406408786773682,
      "step": 2240
    },
    {
      "epoch": 0.1750291715285881,
      "grad_norm": 8.311766624450684,
      "learning_rate": 4.6314806909032766e-05,
      "logits/chosen": 0.008072186261415482,
      "logits/rejected": -1.0863254070281982,
      "logps/chosen": -0.8214001655578613,
      "logps/rejected": -1.5726441144943237,
      "loss": 0.8628,
      "num_input_tokens_seen": 2940336,
      "odds_ratio_loss": 8.559614181518555,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.08214002847671509,
      "rewards/margins": 0.07512439787387848,
      "rewards/rejected": -0.15726442635059357,
      "sft_loss": 0.006882502697408199,
      "step": 2250
    },
    {
      "epoch": 0.17580707895760406,
      "grad_norm": 3.6163852214813232,
      "learning_rate": 4.6282815574099915e-05,
      "logits/chosen": -0.03153163567185402,
      "logits/rejected": -1.0020339488983154,
      "logps/chosen": -0.9039045572280884,
      "logps/rejected": -1.3647565841674805,
      "loss": 0.9506,
      "num_input_tokens_seen": 2955120,
      "odds_ratio_loss": 9.3183012008667,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.09039046615362167,
      "rewards/margins": 0.04608519747853279,
      "rewards/rejected": -0.13647565245628357,
      "sft_loss": 0.018744414672255516,
      "step": 2260
    },
    {
      "epoch": 0.17658498638661999,
      "grad_norm": 4.02025032043457,
      "learning_rate": 4.625069712778262e-05,
      "logits/chosen": 0.02350408211350441,
      "logits/rejected": -1.1788727045059204,
      "logps/chosen": -1.0226409435272217,
      "logps/rejected": -1.3775674104690552,
      "loss": 1.0807,
      "num_input_tokens_seen": 2971120,
      "odds_ratio_loss": 10.762960433959961,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.10226409137248993,
      "rewards/margins": 0.03549265116453171,
      "rewards/rejected": -0.13775673508644104,
      "sft_loss": 0.004435698967427015,
      "step": 2270
    },
    {
      "epoch": 0.17736289381563594,
      "grad_norm": 3.7961761951446533,
      "learning_rate": 4.6218451761907916e-05,
      "logits/chosen": -0.44552960991859436,
      "logits/rejected": -1.1484557390213013,
      "logps/chosen": -0.9121963381767273,
      "logps/rejected": -1.7838424444198608,
      "loss": 0.948,
      "num_input_tokens_seen": 2983664,
      "odds_ratio_loss": 9.294628143310547,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.09121964126825333,
      "rewards/margins": 0.08716461062431335,
      "rewards/rejected": -0.17838425934314728,
      "sft_loss": 0.018568426370620728,
      "step": 2280
    },
    {
      "epoch": 0.1781408012446519,
      "grad_norm": 2.345093250274658,
      "learning_rate": 4.618607966906093e-05,
      "logits/chosen": 0.13724735379219055,
      "logits/rejected": -1.7830671072006226,
      "logps/chosen": -1.0102989673614502,
      "logps/rejected": -1.6919376850128174,
      "loss": 1.072,
      "num_input_tokens_seen": 2994544,
      "odds_ratio_loss": 10.666772842407227,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.1010298952460289,
      "rewards/margins": 0.06816387176513672,
      "rewards/rejected": -0.16919377446174622,
      "sft_loss": 0.005309919826686382,
      "step": 2290
    },
    {
      "epoch": 0.17891870867366783,
      "grad_norm": 46.41415786743164,
      "learning_rate": 4.615358104258362e-05,
      "logits/chosen": -0.37161797285079956,
      "logits/rejected": -1.3046677112579346,
      "logps/chosen": -0.9496458768844604,
      "logps/rejected": -1.2153384685516357,
      "loss": 1.0113,
      "num_input_tokens_seen": 3007296,
      "odds_ratio_loss": 9.624669075012207,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.09496460109949112,
      "rewards/margins": 0.02656925842165947,
      "rewards/rejected": -0.12153385579586029,
      "sft_loss": 0.04888255149126053,
      "step": 2300
    },
    {
      "epoch": 0.17969661610268378,
      "grad_norm": 3.0734081268310547,
      "learning_rate": 4.612095607657368e-05,
      "logits/chosen": -0.11081395298242569,
      "logits/rejected": -1.347336769104004,
      "logps/chosen": -0.8333578109741211,
      "logps/rejected": -1.7155996561050415,
      "loss": 0.8714,
      "num_input_tokens_seen": 3019632,
      "odds_ratio_loss": 8.64091682434082,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08333578705787659,
      "rewards/margins": 0.08822415769100189,
      "rewards/rejected": -0.17155995965003967,
      "sft_loss": 0.007296589203178883,
      "step": 2310
    },
    {
      "epoch": 0.18047452353169974,
      "grad_norm": 6.051811218261719,
      "learning_rate": 4.6088204965883374e-05,
      "logits/chosen": 0.131056547164917,
      "logits/rejected": -0.9874318242073059,
      "logps/chosen": -0.9227386713027954,
      "logps/rejected": -1.371084451675415,
      "loss": 0.9708,
      "num_input_tokens_seen": 3035568,
      "odds_ratio_loss": 9.314332962036133,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.09227386862039566,
      "rewards/margins": 0.044834576547145844,
      "rewards/rejected": -0.1371084451675415,
      "sft_loss": 0.03936336562037468,
      "step": 2320
    },
    {
      "epoch": 0.18125243096071567,
      "grad_norm": 12.885236740112305,
      "learning_rate": 4.605532790611834e-05,
      "logits/chosen": -0.13883432745933533,
      "logits/rejected": -1.3708261251449585,
      "logps/chosen": -1.0801494121551514,
      "logps/rejected": -1.9117374420166016,
      "loss": 1.128,
      "num_input_tokens_seen": 3050976,
      "odds_ratio_loss": 11.087190628051758,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.10801491886377335,
      "rewards/margins": 0.08315880596637726,
      "rewards/rejected": -0.1911737322807312,
      "sft_loss": 0.019236447289586067,
      "step": 2330
    },
    {
      "epoch": 0.18203033838973162,
      "grad_norm": 1.722999930381775,
      "learning_rate": 4.602232509363647e-05,
      "logits/chosen": -0.15571023523807526,
      "logits/rejected": -1.4897704124450684,
      "logps/chosen": -0.9368125796318054,
      "logps/rejected": -1.0839000940322876,
      "loss": 1.0032,
      "num_input_tokens_seen": 3068576,
      "odds_ratio_loss": 9.988349914550781,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.09368126094341278,
      "rewards/margins": 0.014708745293319225,
      "rewards/rejected": -0.10839001089334488,
      "sft_loss": 0.004401323851197958,
      "step": 2340
    },
    {
      "epoch": 0.18280824581874758,
      "grad_norm": 2.3113698959350586,
      "learning_rate": 4.5989196725546704e-05,
      "logits/chosen": -0.5129882097244263,
      "logits/rejected": -0.7843137979507446,
      "logps/chosen": -0.8895527124404907,
      "logps/rejected": -1.5355644226074219,
      "loss": 0.9253,
      "num_input_tokens_seen": 3080592,
      "odds_ratio_loss": 8.968894958496094,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.08895527571439743,
      "rewards/margins": 0.06460118293762207,
      "rewards/rejected": -0.1535564512014389,
      "sft_loss": 0.028372284024953842,
      "step": 2350
    },
    {
      "epoch": 0.1835861532477635,
      "grad_norm": 2.3606832027435303,
      "learning_rate": 4.595594299970786e-05,
      "logits/chosen": -0.12805475294589996,
      "logits/rejected": -1.2973390817642212,
      "logps/chosen": -0.7868680953979492,
      "logps/rejected": -2.545942783355713,
      "loss": 0.842,
      "num_input_tokens_seen": 3096016,
      "odds_ratio_loss": 8.354960441589355,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.07868681102991104,
      "rewards/margins": 0.17590750753879547,
      "rewards/rejected": -0.2545942962169647,
      "sft_loss": 0.006476639304310083,
      "step": 2360
    },
    {
      "epoch": 0.18436406067677946,
      "grad_norm": 2.4345719814300537,
      "learning_rate": 4.592256411472744e-05,
      "logits/chosen": 0.039739884436130524,
      "logits/rejected": -1.3224941492080688,
      "logps/chosen": -0.7756458520889282,
      "logps/rejected": -2.3631529808044434,
      "loss": 0.7966,
      "num_input_tokens_seen": 3107456,
      "odds_ratio_loss": 7.849300384521484,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.07756458222866058,
      "rewards/margins": 0.1587507277727127,
      "rewards/rejected": -0.23631532490253448,
      "sft_loss": 0.011640993878245354,
      "step": 2370
    },
    {
      "epoch": 0.18514196810579542,
      "grad_norm": 4.071404457092285,
      "learning_rate": 4.5889060269960484e-05,
      "logits/chosen": 0.11023125797510147,
      "logits/rejected": -1.3383567333221436,
      "logps/chosen": -0.9944393038749695,
      "logps/rejected": -1.7868140935897827,
      "loss": 1.0393,
      "num_input_tokens_seen": 3118288,
      "odds_ratio_loss": 10.298423767089844,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.09944392740726471,
      "rewards/margins": 0.07923749089241028,
      "rewards/rejected": -0.17868143320083618,
      "sft_loss": 0.009473122656345367,
      "step": 2380
    },
    {
      "epoch": 0.18591987553481135,
      "grad_norm": 3.297738552093506,
      "learning_rate": 4.5855431665508345e-05,
      "logits/chosen": 0.050270576030015945,
      "logits/rejected": -1.6520401239395142,
      "logps/chosen": -1.0888586044311523,
      "logps/rejected": -1.9133188724517822,
      "loss": 1.1342,
      "num_input_tokens_seen": 3134016,
      "odds_ratio_loss": 11.285490036010742,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.10888586193323135,
      "rewards/margins": 0.08244602382183075,
      "rewards/rejected": -0.1913318932056427,
      "sft_loss": 0.0056497156620025635,
      "step": 2390
    },
    {
      "epoch": 0.1866977829638273,
      "grad_norm": 9.202101707458496,
      "learning_rate": 4.582167850221748e-05,
      "logits/chosen": 0.002509523881599307,
      "logits/rejected": -1.233533263206482,
      "logps/chosen": -0.9214789271354675,
      "logps/rejected": -0.9710084795951843,
      "loss": 1.0009,
      "num_input_tokens_seen": 3146544,
      "odds_ratio_loss": 9.797624588012695,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.09214789420366287,
      "rewards/margins": 0.0049529531970620155,
      "rewards/rejected": -0.09710085391998291,
      "sft_loss": 0.021165817975997925,
      "step": 2400
    },
    {
      "epoch": 0.18747569039284326,
      "grad_norm": 3.42102313041687,
      "learning_rate": 4.5787800981678296e-05,
      "logits/chosen": 0.04520236700773239,
      "logits/rejected": -1.4154117107391357,
      "logps/chosen": -0.6949530839920044,
      "logps/rejected": -1.560598611831665,
      "loss": 0.7382,
      "num_input_tokens_seen": 3164224,
      "odds_ratio_loss": 7.338306427001953,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.0694953054189682,
      "rewards/margins": 0.0865645632147789,
      "rewards/rejected": -0.1560598909854889,
      "sft_loss": 0.0043824478052556515,
      "step": 2410
    },
    {
      "epoch": 0.1882535978218592,
      "grad_norm": 4.071820259094238,
      "learning_rate": 4.5753799306223925e-05,
      "logits/chosen": -0.16165246069431305,
      "logits/rejected": -1.2863690853118896,
      "logps/chosen": -1.1060092449188232,
      "logps/rejected": -1.3174792528152466,
      "loss": 1.1651,
      "num_input_tokens_seen": 3179568,
      "odds_ratio_loss": 11.241264343261719,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.11060092598199844,
      "rewards/margins": 0.021147005259990692,
      "rewards/rejected": -0.13174793124198914,
      "sft_loss": 0.04098730534315109,
      "step": 2420
    },
    {
      "epoch": 0.18903150525087514,
      "grad_norm": 2.3402061462402344,
      "learning_rate": 4.5719673678929e-05,
      "logits/chosen": -0.33619269728660583,
      "logits/rejected": -0.7193960547447205,
      "logps/chosen": -0.7185854911804199,
      "logps/rejected": -1.4699690341949463,
      "loss": 0.7549,
      "num_input_tokens_seen": 3192832,
      "odds_ratio_loss": 7.497372627258301,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.07185854017734528,
      "rewards/margins": 0.07513834536075592,
      "rewards/rejected": -0.1469969004392624,
      "sft_loss": 0.005116223357617855,
      "step": 2430
    },
    {
      "epoch": 0.1898094126798911,
      "grad_norm": 3.9390127658843994,
      "learning_rate": 4.568542430360846e-05,
      "logits/chosen": -0.3282054662704468,
      "logits/rejected": -1.3228813409805298,
      "logps/chosen": -1.1381219625473022,
      "logps/rejected": -1.7825075387954712,
      "loss": 1.1856,
      "num_input_tokens_seen": 3204080,
      "odds_ratio_loss": 11.355585098266602,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.11381220817565918,
      "rewards/margins": 0.06443854421377182,
      "rewards/rejected": -0.1782507598400116,
      "sft_loss": 0.05002690106630325,
      "step": 2440
    },
    {
      "epoch": 0.19058732010890703,
      "grad_norm": 3.2009530067443848,
      "learning_rate": 4.565105138481631e-05,
      "logits/chosen": -0.47313547134399414,
      "logits/rejected": -0.8150053024291992,
      "logps/chosen": -1.141099214553833,
      "logps/rejected": -1.3412197828292847,
      "loss": 1.2029,
      "num_input_tokens_seen": 3218432,
      "odds_ratio_loss": 11.356019973754883,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.11410991847515106,
      "rewards/margins": 0.02001207135617733,
      "rewards/rejected": -0.13412198424339294,
      "sft_loss": 0.06727258861064911,
      "step": 2450
    },
    {
      "epoch": 0.19136522753792298,
      "grad_norm": 2.887192726135254,
      "learning_rate": 4.5616555127844455e-05,
      "logits/chosen": -0.710000216960907,
      "logits/rejected": -0.689724326133728,
      "logps/chosen": -0.7518829107284546,
      "logps/rejected": -1.4371519088745117,
      "loss": 0.7949,
      "num_input_tokens_seen": 3231408,
      "odds_ratio_loss": 7.861302852630615,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.0751882940530777,
      "rewards/margins": 0.06852690875530243,
      "rewards/rejected": -0.14371520280838013,
      "sft_loss": 0.00877359975129366,
      "step": 2460
    },
    {
      "epoch": 0.19214313496693894,
      "grad_norm": 5.989782333374023,
      "learning_rate": 4.55819357387214e-05,
      "logits/chosen": -0.17640532553195953,
      "logits/rejected": -1.0639609098434448,
      "logps/chosen": -1.0197169780731201,
      "logps/rejected": -1.1484568119049072,
      "loss": 1.091,
      "num_input_tokens_seen": 3243872,
      "odds_ratio_loss": 10.208763122558594,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.10197170823812485,
      "rewards/margins": 0.012873975560069084,
      "rewards/rejected": -0.11484567821025848,
      "sft_loss": 0.07008983194828033,
      "step": 2470
    },
    {
      "epoch": 0.1929210423959549,
      "grad_norm": 2.637138605117798,
      "learning_rate": 4.554719342421108e-05,
      "logits/chosen": -0.2950691282749176,
      "logits/rejected": -1.0944535732269287,
      "logps/chosen": -0.8581943511962891,
      "logps/rejected": -0.9821914434432983,
      "loss": 0.9194,
      "num_input_tokens_seen": 3255296,
      "odds_ratio_loss": 9.095025062561035,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.08581943809986115,
      "rewards/margins": 0.012399706058204174,
      "rewards/rejected": -0.098219133913517,
      "sft_loss": 0.009927446953952312,
      "step": 2480
    },
    {
      "epoch": 0.19369894982497082,
      "grad_norm": 3.5724494457244873,
      "learning_rate": 4.551232839181159e-05,
      "logits/chosen": -0.1675427407026291,
      "logits/rejected": -1.0505664348602295,
      "logps/chosen": -1.0215306282043457,
      "logps/rejected": -1.6657826900482178,
      "loss": 1.0779,
      "num_input_tokens_seen": 3273408,
      "odds_ratio_loss": 10.157814025878906,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.10215306282043457,
      "rewards/margins": 0.06442520767450333,
      "rewards/rejected": -0.1665782630443573,
      "sft_loss": 0.06208919361233711,
      "step": 2490
    },
    {
      "epoch": 0.19447685725398678,
      "grad_norm": 9.58640193939209,
      "learning_rate": 4.5477340849753944e-05,
      "logits/chosen": -0.2685242295265198,
      "logits/rejected": -0.9891644716262817,
      "logps/chosen": -0.9026187062263489,
      "logps/rejected": -1.8491036891937256,
      "loss": 0.9503,
      "num_input_tokens_seen": 3284784,
      "odds_ratio_loss": 8.730802536010742,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.09026187658309937,
      "rewards/margins": 0.09464851021766663,
      "rewards/rejected": -0.184910386800766,
      "sft_loss": 0.07725141942501068,
      "step": 2500
    },
    {
      "epoch": 0.19525476468300274,
      "grad_norm": 0.9771826863288879,
      "learning_rate": 4.544223100700087e-05,
      "logits/chosen": -0.17309458553791046,
      "logits/rejected": -1.0318530797958374,
      "logps/chosen": -0.9268582463264465,
      "logps/rejected": -1.784379243850708,
      "loss": 0.9709,
      "num_input_tokens_seen": 3298928,
      "odds_ratio_loss": 9.390172004699707,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.09268582612276077,
      "rewards/margins": 0.08575211465358734,
      "rewards/rejected": -0.17843793332576752,
      "sft_loss": 0.03189669921994209,
      "step": 2510
    },
    {
      "epoch": 0.19603267211201866,
      "grad_norm": 3.369408130645752,
      "learning_rate": 4.540699907324552e-05,
      "logits/chosen": 0.031320180743932724,
      "logits/rejected": -1.3592791557312012,
      "logps/chosen": -0.9459033012390137,
      "logps/rejected": -1.9444615840911865,
      "loss": 1.0037,
      "num_input_tokens_seen": 3308880,
      "odds_ratio_loss": 9.935675621032715,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.09459032863378525,
      "rewards/margins": 0.09985584020614624,
      "rewards/rejected": -0.19444617629051208,
      "sft_loss": 0.01017682533711195,
      "step": 2520
    },
    {
      "epoch": 0.19681057954103462,
      "grad_norm": 5.482847690582275,
      "learning_rate": 4.537164525891023e-05,
      "logits/chosen": -0.1788056343793869,
      "logits/rejected": -1.1290700435638428,
      "logps/chosen": -0.8016532063484192,
      "logps/rejected": -2.0308566093444824,
      "loss": 0.8472,
      "num_input_tokens_seen": 3324896,
      "odds_ratio_loss": 8.375109672546387,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.0801653191447258,
      "rewards/margins": 0.12292035669088364,
      "rewards/rejected": -0.20308569073677063,
      "sft_loss": 0.009639225900173187,
      "step": 2530
    },
    {
      "epoch": 0.19758848697005058,
      "grad_norm": 10.093277931213379,
      "learning_rate": 4.533616977514527e-05,
      "logits/chosen": -0.10968391597270966,
      "logits/rejected": -1.4015908241271973,
      "logps/chosen": -0.8579193353652954,
      "logps/rejected": -1.047619104385376,
      "loss": 0.9156,
      "num_input_tokens_seen": 3341072,
      "odds_ratio_loss": 8.98912239074707,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.0857919305562973,
      "rewards/margins": 0.01896999403834343,
      "rewards/rejected": -0.10476192086935043,
      "sft_loss": 0.0166663508862257,
      "step": 2540
    },
    {
      "epoch": 0.1983663943990665,
      "grad_norm": 1.3628586530685425,
      "learning_rate": 4.530057283382759e-05,
      "logits/chosen": -0.13702496886253357,
      "logits/rejected": -1.4749879837036133,
      "logps/chosen": -0.6388250589370728,
      "logps/rejected": -1.442430853843689,
      "loss": 0.6655,
      "num_input_tokens_seen": 3354048,
      "odds_ratio_loss": 6.584555149078369,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.06388251483440399,
      "rewards/margins": 0.08036056905984879,
      "rewards/rejected": -0.14424307644367218,
      "sft_loss": 0.007017000578343868,
      "step": 2550
    },
    {
      "epoch": 0.19914430182808246,
      "grad_norm": 2.2133822441101074,
      "learning_rate": 4.5264854647559516e-05,
      "logits/chosen": -0.0047277300618588924,
      "logits/rejected": -2.1251773834228516,
      "logps/chosen": -0.9321849942207336,
      "logps/rejected": -1.8874273300170898,
      "loss": 0.9926,
      "num_input_tokens_seen": 3367584,
      "odds_ratio_loss": 9.882081031799316,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.09321849048137665,
      "rewards/margins": 0.09552423655986786,
      "rewards/rejected": -0.1887427270412445,
      "sft_loss": 0.004392318893224001,
      "step": 2560
    },
    {
      "epoch": 0.19992220925709842,
      "grad_norm": 3.6601076126098633,
      "learning_rate": 4.522901542966756e-05,
      "logits/chosen": -0.11102037131786346,
      "logits/rejected": -0.8737829923629761,
      "logps/chosen": -0.8281256556510925,
      "logps/rejected": -1.7336536645889282,
      "loss": 0.8642,
      "num_input_tokens_seen": 3375024,
      "odds_ratio_loss": 8.55268383026123,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.08281257003545761,
      "rewards/margins": 0.09055278450250626,
      "rewards/rejected": -0.17336536943912506,
      "sft_loss": 0.008942141197621822,
      "step": 2570
    },
    {
      "epoch": 0.20070011668611434,
      "grad_norm": 4.413498401641846,
      "learning_rate": 4.5193055394201025e-05,
      "logits/chosen": -0.2287522554397583,
      "logits/rejected": -1.363667607307434,
      "logps/chosen": -1.1148064136505127,
      "logps/rejected": -1.7411060333251953,
      "loss": 1.1658,
      "num_input_tokens_seen": 3387376,
      "odds_ratio_loss": 11.551223754882812,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.11148063838481903,
      "rewards/margins": 0.06262998282909393,
      "rewards/rejected": -0.17411060631275177,
      "sft_loss": 0.01069558784365654,
      "step": 2580
    },
    {
      "epoch": 0.2014780241151303,
      "grad_norm": 4.33030366897583,
      "learning_rate": 4.515697475593085e-05,
      "logits/chosen": -0.03919697552919388,
      "logits/rejected": -1.1795130968093872,
      "logps/chosen": -0.8254842758178711,
      "logps/rejected": -0.8753021955490112,
      "loss": 0.909,
      "num_input_tokens_seen": 3403952,
      "odds_ratio_loss": 9.031672477722168,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.08254842460155487,
      "rewards/margins": 0.0049817985855042934,
      "rewards/rejected": -0.0875302106142044,
      "sft_loss": 0.0058304606936872005,
      "step": 2590
    },
    {
      "epoch": 0.20225593154414626,
      "grad_norm": 6.164618968963623,
      "learning_rate": 4.512077373034825e-05,
      "logits/chosen": -0.1275373250246048,
      "logits/rejected": -1.0006495714187622,
      "logps/chosen": -0.8229962587356567,
      "logps/rejected": -2.5332837104797363,
      "loss": 0.8639,
      "num_input_tokens_seen": 3415264,
      "odds_ratio_loss": 8.567689895629883,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.0822996199131012,
      "rewards/margins": 0.1710287630558014,
      "rewards/rejected": -0.2533283829689026,
      "sft_loss": 0.007158086635172367,
      "step": 2600
    },
    {
      "epoch": 0.20303383897316218,
      "grad_norm": 20.658157348632812,
      "learning_rate": 4.5084452533663466e-05,
      "logits/chosen": -0.0677039623260498,
      "logits/rejected": -1.151421308517456,
      "logps/chosen": -1.050412654876709,
      "logps/rejected": -2.1510255336761475,
      "loss": 1.1177,
      "num_input_tokens_seen": 3427392,
      "odds_ratio_loss": 10.269535064697266,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.1050412654876709,
      "rewards/margins": 0.11006130278110504,
      "rewards/rejected": -0.21510258316993713,
      "sft_loss": 0.09070286899805069,
      "step": 2610
    },
    {
      "epoch": 0.20381174640217814,
      "grad_norm": 6.2879533767700195,
      "learning_rate": 4.504801138280445e-05,
      "logits/chosen": -0.12882912158966064,
      "logits/rejected": -1.148313283920288,
      "logps/chosen": -1.0179685354232788,
      "logps/rejected": -1.5564950704574585,
      "loss": 1.0858,
      "num_input_tokens_seen": 3440992,
      "odds_ratio_loss": 10.7903470993042,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.10179685056209564,
      "rewards/margins": 0.053852636367082596,
      "rewards/rejected": -0.15564949810504913,
      "sft_loss": 0.006785678677260876,
      "step": 2620
    },
    {
      "epoch": 0.2045896538311941,
      "grad_norm": 47.05371856689453,
      "learning_rate": 4.5011450495415555e-05,
      "logits/chosen": 0.04895355924963951,
      "logits/rejected": -1.5381346940994263,
      "logps/chosen": -0.958077073097229,
      "logps/rejected": -2.2323126792907715,
      "loss": 1.008,
      "num_input_tokens_seen": 3455888,
      "odds_ratio_loss": 9.713939666748047,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.09580769389867783,
      "rewards/margins": 0.12742355465888977,
      "rewards/rejected": -0.223231241106987,
      "sft_loss": 0.03663123771548271,
      "step": 2630
    },
    {
      "epoch": 0.20536756126021002,
      "grad_norm": 2.227385997772217,
      "learning_rate": 4.49747700898563e-05,
      "logits/chosen": -0.019439058378338814,
      "logits/rejected": -1.1940211057662964,
      "logps/chosen": -0.8006144762039185,
      "logps/rejected": -1.171351671218872,
      "loss": 0.8501,
      "num_input_tokens_seen": 3472864,
      "odds_ratio_loss": 8.339761734008789,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.08006144315004349,
      "rewards/margins": 0.03707372397184372,
      "rewards/rejected": -0.11713516712188721,
      "sft_loss": 0.016088882461190224,
      "step": 2640
    },
    {
      "epoch": 0.20614546868922598,
      "grad_norm": 9.593066215515137,
      "learning_rate": 4.493797038520001e-05,
      "logits/chosen": -0.08355199545621872,
      "logits/rejected": -1.4018335342407227,
      "logps/chosen": -0.9090895652770996,
      "logps/rejected": -2.0214409828186035,
      "loss": 0.9523,
      "num_input_tokens_seen": 3487200,
      "odds_ratio_loss": 9.115594863891602,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.0909089595079422,
      "rewards/margins": 0.11123514175415039,
      "rewards/rejected": -0.2021441012620926,
      "sft_loss": 0.04074696823954582,
      "step": 2650
    },
    {
      "epoch": 0.20692337611824194,
      "grad_norm": 5.4159932136535645,
      "learning_rate": 4.4901051601232526e-05,
      "logits/chosen": -0.22638556361198425,
      "logits/rejected": -1.0507686138153076,
      "logps/chosen": -1.0690250396728516,
      "logps/rejected": -1.880934476852417,
      "loss": 1.1143,
      "num_input_tokens_seen": 3502288,
      "odds_ratio_loss": 10.672109603881836,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.10690250247716904,
      "rewards/margins": 0.08119096606969833,
      "rewards/rejected": -0.18809348344802856,
      "sft_loss": 0.047121696174144745,
      "step": 2660
    },
    {
      "epoch": 0.20770128354725786,
      "grad_norm": 4.875866413116455,
      "learning_rate": 4.4864013958450867e-05,
      "logits/chosen": -0.07939404249191284,
      "logits/rejected": -1.7361027002334595,
      "logps/chosen": -1.1061766147613525,
      "logps/rejected": -1.8076751232147217,
      "loss": 1.1485,
      "num_input_tokens_seen": 3518000,
      "odds_ratio_loss": 11.435917854309082,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.11061765998601913,
      "rewards/margins": 0.07014986127614975,
      "rewards/rejected": -0.18076752126216888,
      "sft_loss": 0.004879302810877562,
      "step": 2670
    },
    {
      "epoch": 0.20847919097627382,
      "grad_norm": 1.931626319885254,
      "learning_rate": 4.482685767806196e-05,
      "logits/chosen": -0.18810833990573883,
      "logits/rejected": -1.2008379697799683,
      "logps/chosen": -1.0920370817184448,
      "logps/rejected": -1.5815891027450562,
      "loss": 1.1486,
      "num_input_tokens_seen": 3528432,
      "odds_ratio_loss": 11.227903366088867,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.10920371860265732,
      "rewards/margins": 0.04895520210266113,
      "rewards/rejected": -0.15815891325473785,
      "sft_loss": 0.025777261704206467,
      "step": 2680
    },
    {
      "epoch": 0.20925709840528978,
      "grad_norm": 7.632688045501709,
      "learning_rate": 4.4789582981981294e-05,
      "logits/chosen": -0.1754075586795807,
      "logits/rejected": -1.7076215744018555,
      "logps/chosen": -0.9906984567642212,
      "logps/rejected": -2.3729805946350098,
      "loss": 1.0319,
      "num_input_tokens_seen": 3539520,
      "odds_ratio_loss": 10.129727363586426,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.09906984865665436,
      "rewards/margins": 0.13822820782661438,
      "rewards/rejected": -0.23729804158210754,
      "sft_loss": 0.018918436020612717,
      "step": 2690
    },
    {
      "epoch": 0.2100350058343057,
      "grad_norm": 2.336216688156128,
      "learning_rate": 4.475219009283157e-05,
      "logits/chosen": -0.4126604199409485,
      "logits/rejected": -1.1251232624053955,
      "logps/chosen": -0.8320616483688354,
      "logps/rejected": -1.44077730178833,
      "loss": 0.8774,
      "num_input_tokens_seen": 3553856,
      "odds_ratio_loss": 8.672832489013672,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.0832061693072319,
      "rewards/margins": 0.06087157130241394,
      "rewards/rejected": -0.14407773315906525,
      "sft_loss": 0.01009486522525549,
      "step": 2700
    },
    {
      "epoch": 0.21081291326332166,
      "grad_norm": 10.049501419067383,
      "learning_rate": 4.471467923394142e-05,
      "logits/chosen": -0.47965916991233826,
      "logits/rejected": -0.8203874826431274,
      "logps/chosen": -0.8291003108024597,
      "logps/rejected": -1.9053233861923218,
      "loss": 0.8601,
      "num_input_tokens_seen": 3566720,
      "odds_ratio_loss": 7.984526634216309,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.08291003853082657,
      "rewards/margins": 0.10762230306863785,
      "rewards/rejected": -0.19053234159946442,
      "sft_loss": 0.061681754887104034,
      "step": 2710
    },
    {
      "epoch": 0.21159082069233762,
      "grad_norm": 1.9754427671432495,
      "learning_rate": 4.467705062934405e-05,
      "logits/chosen": -0.33071309328079224,
      "logits/rejected": -1.6625646352767944,
      "logps/chosen": -0.7561646699905396,
      "logps/rejected": -2.0466246604919434,
      "loss": 0.7865,
      "num_input_tokens_seen": 3579088,
      "odds_ratio_loss": 7.769416809082031,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.07561646401882172,
      "rewards/margins": 0.1290459930896759,
      "rewards/rejected": -0.20466244220733643,
      "sft_loss": 0.00955834798514843,
      "step": 2720
    },
    {
      "epoch": 0.21236872812135357,
      "grad_norm": 8.636311531066895,
      "learning_rate": 4.463930450377587e-05,
      "logits/chosen": -0.21290338039398193,
      "logits/rejected": -1.0868850946426392,
      "logps/chosen": -0.8203240633010864,
      "logps/rejected": -1.304052710533142,
      "loss": 0.8716,
      "num_input_tokens_seen": 3591184,
      "odds_ratio_loss": 8.614351272583008,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.08203241229057312,
      "rewards/margins": 0.04837285727262497,
      "rewards/rejected": -0.1304052770137787,
      "sft_loss": 0.010142195969820023,
      "step": 2730
    },
    {
      "epoch": 0.2131466355503695,
      "grad_norm": 2.121227979660034,
      "learning_rate": 4.460144108267522e-05,
      "logits/chosen": -0.5269980430603027,
      "logits/rejected": -1.0767323970794678,
      "logps/chosen": -0.8903768658638,
      "logps/rejected": -2.5388858318328857,
      "loss": 0.9273,
      "num_input_tokens_seen": 3600656,
      "odds_ratio_loss": 9.161144256591797,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.08903767913579941,
      "rewards/margins": 0.16485092043876648,
      "rewards/rejected": -0.2538886070251465,
      "sft_loss": 0.011165516451001167,
      "step": 2740
    },
    {
      "epoch": 0.21392454297938546,
      "grad_norm": 3.538132667541504,
      "learning_rate": 4.4563460592180966e-05,
      "logits/chosen": -0.1817234754562378,
      "logits/rejected": -0.8993843793869019,
      "logps/chosen": -0.9267904162406921,
      "logps/rejected": -2.0416767597198486,
      "loss": 0.9811,
      "num_input_tokens_seen": 3611760,
      "odds_ratio_loss": 9.679262161254883,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.09267903864383698,
      "rewards/margins": 0.11148865520954132,
      "rewards/rejected": -0.2041676789522171,
      "sft_loss": 0.013199281878769398,
      "step": 2750
    },
    {
      "epoch": 0.2147024504084014,
      "grad_norm": 1.918135166168213,
      "learning_rate": 4.4525363259131175e-05,
      "logits/chosen": -0.20659813284873962,
      "logits/rejected": -1.5829854011535645,
      "logps/chosen": -1.042778730392456,
      "logps/rejected": -2.7298874855041504,
      "loss": 1.0853,
      "num_input_tokens_seen": 3624352,
      "odds_ratio_loss": 10.755561828613281,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.10427787154912949,
      "rewards/margins": 0.1687108874320984,
      "rewards/rejected": -0.2729887366294861,
      "sft_loss": 0.009703483432531357,
      "step": 2760
    },
    {
      "epoch": 0.21548035783741734,
      "grad_norm": 1.848811149597168,
      "learning_rate": 4.4487149311061754e-05,
      "logits/chosen": -0.48155227303504944,
      "logits/rejected": -0.7977389097213745,
      "logps/chosen": -0.7917124032974243,
      "logps/rejected": -1.8758751153945923,
      "loss": 0.8297,
      "num_input_tokens_seen": 3636736,
      "odds_ratio_loss": 7.977292537689209,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.07917123287916183,
      "rewards/margins": 0.10841628164052963,
      "rewards/rejected": -0.18758751451969147,
      "sft_loss": 0.0319630466401577,
      "step": 2770
    },
    {
      "epoch": 0.2162582652664333,
      "grad_norm": 9.27741527557373,
      "learning_rate": 4.444881897620509e-05,
      "logits/chosen": -0.4793930649757385,
      "logits/rejected": -0.9707077145576477,
      "logps/chosen": -0.9283552169799805,
      "logps/rejected": -1.2709600925445557,
      "loss": 0.9828,
      "num_input_tokens_seen": 3649920,
      "odds_ratio_loss": 9.733741760253906,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.09283552318811417,
      "rewards/margins": 0.034260500222444534,
      "rewards/rejected": -0.127096027135849,
      "sft_loss": 0.009438985027372837,
      "step": 2780
    },
    {
      "epoch": 0.21703617269544925,
      "grad_norm": 5.2187066078186035,
      "learning_rate": 4.441037248348869e-05,
      "logits/chosen": -0.03495123237371445,
      "logits/rejected": -1.3832604885101318,
      "logps/chosen": -0.9088050723075867,
      "logps/rejected": -1.8502451181411743,
      "loss": 0.9442,
      "num_input_tokens_seen": 3662992,
      "odds_ratio_loss": 9.403382301330566,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.09088051319122314,
      "rewards/margins": 0.09414400160312653,
      "rewards/rejected": -0.18502449989318848,
      "sft_loss": 0.0038636140525341034,
      "step": 2790
    },
    {
      "epoch": 0.21781408012446518,
      "grad_norm": 1.8032280206680298,
      "learning_rate": 4.437181006253382e-05,
      "logits/chosen": -0.29241687059402466,
      "logits/rejected": -1.4174654483795166,
      "logps/chosen": -0.9323759078979492,
      "logps/rejected": -2.653144121170044,
      "loss": 0.9668,
      "num_input_tokens_seen": 3674288,
      "odds_ratio_loss": 9.554191589355469,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09323760122060776,
      "rewards/margins": 0.17207682132720947,
      "rewards/rejected": -0.26531440019607544,
      "sft_loss": 0.011398935690522194,
      "step": 2800
    },
    {
      "epoch": 0.21859198755348114,
      "grad_norm": 6.935772895812988,
      "learning_rate": 4.433313194365411e-05,
      "logits/chosen": -0.11382085084915161,
      "logits/rejected": -1.1239991188049316,
      "logps/chosen": -1.0345927476882935,
      "logps/rejected": -2.4821836948394775,
      "loss": 1.0816,
      "num_input_tokens_seen": 3683472,
      "odds_ratio_loss": 10.74571418762207,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.10345926135778427,
      "rewards/margins": 0.14475905895233154,
      "rewards/rejected": -0.24821837246418,
      "sft_loss": 0.00707204919308424,
      "step": 2810
    },
    {
      "epoch": 0.2193698949824971,
      "grad_norm": 1.740234375,
      "learning_rate": 4.429433835785422e-05,
      "logits/chosen": -0.09553475677967072,
      "logits/rejected": -1.2517316341400146,
      "logps/chosen": -1.050550937652588,
      "logps/rejected": -1.3104311227798462,
      "loss": 1.1106,
      "num_input_tokens_seen": 3695840,
      "odds_ratio_loss": 10.7747220993042,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.10505510866641998,
      "rewards/margins": 0.025988012552261353,
      "rewards/rejected": -0.13104310631752014,
      "sft_loss": 0.033088210970163345,
      "step": 2820
    },
    {
      "epoch": 0.22014780241151302,
      "grad_norm": 5.372836112976074,
      "learning_rate": 4.425542953682841e-05,
      "logits/chosen": -0.15488819777965546,
      "logits/rejected": -1.968838095664978,
      "logps/chosen": -0.8588301539421082,
      "logps/rejected": -1.6684284210205078,
      "loss": 0.9,
      "num_input_tokens_seen": 3709792,
      "odds_ratio_loss": 8.847394943237305,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.08588302135467529,
      "rewards/margins": 0.08095981180667877,
      "rewards/rejected": -0.16684284806251526,
      "sft_loss": 0.01527967769652605,
      "step": 2830
    },
    {
      "epoch": 0.22092570984052898,
      "grad_norm": 2.368696689605713,
      "learning_rate": 4.421640571295918e-05,
      "logits/chosen": -0.1854361593723297,
      "logits/rejected": -0.9668446779251099,
      "logps/chosen": -1.0571156740188599,
      "logps/rejected": -2.4429931640625,
      "loss": 1.1238,
      "num_input_tokens_seen": 3721664,
      "odds_ratio_loss": 10.837104797363281,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.10571156442165375,
      "rewards/margins": 0.13858774304389954,
      "rewards/rejected": -0.24429932236671448,
      "sft_loss": 0.04005955904722214,
      "step": 2840
    },
    {
      "epoch": 0.22170361726954493,
      "grad_norm": 3.42175555229187,
      "learning_rate": 4.4177267119315905e-05,
      "logits/chosen": -0.17983666062355042,
      "logits/rejected": -0.6118065118789673,
      "logps/chosen": -0.9091208577156067,
      "logps/rejected": -1.3174176216125488,
      "loss": 0.9641,
      "num_input_tokens_seen": 3729968,
      "odds_ratio_loss": 9.469156265258789,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.09091208875179291,
      "rewards/margins": 0.040829677134752274,
      "rewards/rejected": -0.13174176216125488,
      "sft_loss": 0.017219817265868187,
      "step": 2850
    },
    {
      "epoch": 0.22248152469856086,
      "grad_norm": 5.314608573913574,
      "learning_rate": 4.4138013989653406e-05,
      "logits/chosen": -0.027306431904435158,
      "logits/rejected": -0.819806694984436,
      "logps/chosen": -0.8254560232162476,
      "logps/rejected": -2.61415433883667,
      "loss": 0.886,
      "num_input_tokens_seen": 3744544,
      "odds_ratio_loss": 8.529707908630371,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.08254560083150864,
      "rewards/margins": 0.1788698434829712,
      "rewards/rejected": -0.26141542196273804,
      "sft_loss": 0.03305988386273384,
      "step": 2860
    },
    {
      "epoch": 0.22325943212757682,
      "grad_norm": 2.7968525886535645,
      "learning_rate": 4.409864655841058e-05,
      "logits/chosen": 0.05771436542272568,
      "logits/rejected": -1.489516019821167,
      "logps/chosen": -1.114302396774292,
      "logps/rejected": -2.0234618186950684,
      "loss": 1.1547,
      "num_input_tokens_seen": 3760112,
      "odds_ratio_loss": 11.147990226745605,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.11143024265766144,
      "rewards/margins": 0.09091595560312271,
      "rewards/rejected": -0.20234617590904236,
      "sft_loss": 0.039897941052913666,
      "step": 2870
    },
    {
      "epoch": 0.22403733955659277,
      "grad_norm": 2.459502696990967,
      "learning_rate": 4.405916506070897e-05,
      "logits/chosen": -0.17634642124176025,
      "logits/rejected": -1.053168535232544,
      "logps/chosen": -0.7258324027061462,
      "logps/rejected": -1.2297321557998657,
      "loss": 0.7769,
      "num_input_tokens_seen": 3774064,
      "odds_ratio_loss": 7.716550350189209,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.07258324325084686,
      "rewards/margins": 0.050389982759952545,
      "rewards/rejected": -0.12297321856021881,
      "sft_loss": 0.005258309654891491,
      "step": 2880
    },
    {
      "epoch": 0.2248152469856087,
      "grad_norm": 2.5929694175720215,
      "learning_rate": 4.40195697323514e-05,
      "logits/chosen": -0.31808415055274963,
      "logits/rejected": -1.3807660341262817,
      "logps/chosen": -0.8598939776420593,
      "logps/rejected": -1.0913549661636353,
      "loss": 0.9157,
      "num_input_tokens_seen": 3787120,
      "odds_ratio_loss": 9.119691848754883,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.08598940074443817,
      "rewards/margins": 0.023146091029047966,
      "rewards/rejected": -0.10913548618555069,
      "sft_loss": 0.003712741192430258,
      "step": 2890
    },
    {
      "epoch": 0.22559315441462466,
      "grad_norm": 1.6673979759216309,
      "learning_rate": 4.397986080982053e-05,
      "logits/chosen": -0.18936365842819214,
      "logits/rejected": -0.9067906141281128,
      "logps/chosen": -0.811468780040741,
      "logps/rejected": -1.1548564434051514,
      "loss": 0.8606,
      "num_input_tokens_seen": 3797968,
      "odds_ratio_loss": 8.536993026733398,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.08114688098430634,
      "rewards/margins": 0.034338753670454025,
      "rewards/rejected": -0.11548563092947006,
      "sft_loss": 0.006941842380911112,
      "step": 2900
    },
    {
      "epoch": 0.22637106184364061,
      "grad_norm": 2.810910224914551,
      "learning_rate": 4.394003853027748e-05,
      "logits/chosen": -0.534250020980835,
      "logits/rejected": -0.8749955296516418,
      "logps/chosen": -0.7439308762550354,
      "logps/rejected": -1.425173282623291,
      "loss": 0.7953,
      "num_input_tokens_seen": 3812112,
      "odds_ratio_loss": 7.768838405609131,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.07439308613538742,
      "rewards/margins": 0.06812424957752228,
      "rewards/rejected": -0.1425173431634903,
      "sft_loss": 0.018367454409599304,
      "step": 2910
    },
    {
      "epoch": 0.22714896927265654,
      "grad_norm": 1.7785518169403076,
      "learning_rate": 4.3900103131560376e-05,
      "logits/chosen": -0.4530204236507416,
      "logits/rejected": -1.1013813018798828,
      "logps/chosen": -0.804338276386261,
      "logps/rejected": -1.8796924352645874,
      "loss": 0.8429,
      "num_input_tokens_seen": 3824768,
      "odds_ratio_loss": 8.357107162475586,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.08043383061885834,
      "rewards/margins": 0.10753540694713593,
      "rewards/rejected": -0.18796923756599426,
      "sft_loss": 0.0071527608670294285,
      "step": 2920
    },
    {
      "epoch": 0.2279268767016725,
      "grad_norm": 2.90664005279541,
      "learning_rate": 4.386005485218294e-05,
      "logits/chosen": -0.10354918241500854,
      "logits/rejected": -1.3656952381134033,
      "logps/chosen": -0.8719033002853394,
      "logps/rejected": -5.1970534324646,
      "loss": 0.9075,
      "num_input_tokens_seen": 3836960,
      "odds_ratio_loss": 8.931937217712402,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.08719031512737274,
      "rewards/margins": 0.43251505494117737,
      "rewards/rejected": -0.5197054147720337,
      "sft_loss": 0.014338955283164978,
      "step": 2930
    },
    {
      "epoch": 0.22870478413068845,
      "grad_norm": 2.312278985977173,
      "learning_rate": 4.3819893931333106e-05,
      "logits/chosen": -0.1507093757390976,
      "logits/rejected": -1.4808213710784912,
      "logps/chosen": -0.8520559072494507,
      "logps/rejected": -2.1594061851501465,
      "loss": 0.8842,
      "num_input_tokens_seen": 3852752,
      "odds_ratio_loss": 8.767353057861328,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.08520559221506119,
      "rewards/margins": 0.13073505461215973,
      "rewards/rejected": -0.21594062447547913,
      "sft_loss": 0.007421643938869238,
      "step": 2940
    },
    {
      "epoch": 0.22948269155970438,
      "grad_norm": 2.6451447010040283,
      "learning_rate": 4.3779620608871526e-05,
      "logits/chosen": -0.20507876574993134,
      "logits/rejected": -0.8062494397163391,
      "logps/chosen": -1.0439388751983643,
      "logps/rejected": -1.4790273904800415,
      "loss": 1.0918,
      "num_input_tokens_seen": 3863120,
      "odds_ratio_loss": 10.622690200805664,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.10439388453960419,
      "rewards/margins": 0.043508853763341904,
      "rewards/rejected": -0.1479027271270752,
      "sft_loss": 0.029571745544672012,
      "step": 2950
    },
    {
      "epoch": 0.23026059898872034,
      "grad_norm": 2.9421136379241943,
      "learning_rate": 4.373923512533018e-05,
      "logits/chosen": -0.11698918044567108,
      "logits/rejected": -1.4051399230957031,
      "logps/chosen": -0.987946629524231,
      "logps/rejected": -1.9638170003890991,
      "loss": 1.0268,
      "num_input_tokens_seen": 3876384,
      "odds_ratio_loss": 9.981897354125977,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09879465401172638,
      "rewards/margins": 0.09758704155683517,
      "rewards/rejected": -0.19638168811798096,
      "sft_loss": 0.028579894453287125,
      "step": 2960
    },
    {
      "epoch": 0.2310385064177363,
      "grad_norm": 4.920382976531982,
      "learning_rate": 4.369873772191092e-05,
      "logits/chosen": -0.18340758979320526,
      "logits/rejected": -1.0555959939956665,
      "logps/chosen": -0.994560718536377,
      "logps/rejected": -3.392359495162964,
      "loss": 1.0257,
      "num_input_tokens_seen": 3887008,
      "odds_ratio_loss": 9.342180252075195,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.09945607930421829,
      "rewards/margins": 0.23977987468242645,
      "rewards/rejected": -0.33923593163490295,
      "sft_loss": 0.09144171327352524,
      "step": 2970
    },
    {
      "epoch": 0.23181641384675222,
      "grad_norm": 2.079035997390747,
      "learning_rate": 4.365812864048407e-05,
      "logits/chosen": 0.05672953277826309,
      "logits/rejected": -1.5294697284698486,
      "logps/chosen": -0.996350884437561,
      "logps/rejected": -1.1568883657455444,
      "loss": 1.0754,
      "num_input_tokens_seen": 3902128,
      "odds_ratio_loss": 10.721492767333984,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.09963510185480118,
      "rewards/margins": 0.016053764149546623,
      "rewards/rejected": -0.11568886041641235,
      "sft_loss": 0.0032391403801739216,
      "step": 2980
    },
    {
      "epoch": 0.23259432127576818,
      "grad_norm": 2.5082170963287354,
      "learning_rate": 4.3617408123586913e-05,
      "logits/chosen": 0.05996738746762276,
      "logits/rejected": -1.517051100730896,
      "logps/chosen": -1.1436173915863037,
      "logps/rejected": -2.44610595703125,
      "loss": 1.1817,
      "num_input_tokens_seen": 3917712,
      "odds_ratio_loss": 10.85283088684082,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.11436174064874649,
      "rewards/margins": 0.13024888932704926,
      "rewards/rejected": -0.24461062252521515,
      "sft_loss": 0.09644796699285507,
      "step": 2990
    },
    {
      "epoch": 0.23337222870478413,
      "grad_norm": 2.336026668548584,
      "learning_rate": 4.357657641442229e-05,
      "logits/chosen": -0.17219378054141998,
      "logits/rejected": -1.2242201566696167,
      "logps/chosen": -0.9584628939628601,
      "logps/rejected": -1.6986106634140015,
      "loss": 1.0064,
      "num_input_tokens_seen": 3930480,
      "odds_ratio_loss": 9.946453094482422,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.09584628790616989,
      "rewards/margins": 0.07401478290557861,
      "rewards/rejected": -0.1698610782623291,
      "sft_loss": 0.011790582910180092,
      "step": 3000
    },
    {
      "epoch": 0.2341501361338001,
      "grad_norm": 3.9625983238220215,
      "learning_rate": 4.3535633756857136e-05,
      "logits/chosen": -0.14156615734100342,
      "logits/rejected": -0.9244439005851746,
      "logps/chosen": -0.9330800771713257,
      "logps/rejected": -1.97588312625885,
      "loss": 0.9786,
      "num_input_tokens_seen": 3939456,
      "odds_ratio_loss": 9.251434326171875,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.09330800920724869,
      "rewards/margins": 0.10428030788898468,
      "rewards/rejected": -0.19758832454681396,
      "sft_loss": 0.053438376635313034,
      "step": 3010
    },
    {
      "epoch": 0.23492804356281602,
      "grad_norm": 4.78399658203125,
      "learning_rate": 4.349458039542105e-05,
      "logits/chosen": -0.0739276185631752,
      "logits/rejected": -1.4249768257141113,
      "logps/chosen": -1.1414638757705688,
      "logps/rejected": -2.1657207012176514,
      "loss": 1.1815,
      "num_input_tokens_seen": 3948896,
      "odds_ratio_loss": 11.685748100280762,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.114146389067173,
      "rewards/margins": 0.10242567211389542,
      "rewards/rejected": -0.21657204627990723,
      "sft_loss": 0.012904743663966656,
      "step": 3020
    },
    {
      "epoch": 0.23570595099183198,
      "grad_norm": 4.978982925415039,
      "learning_rate": 4.345341657530476e-05,
      "logits/chosen": -0.0407116636633873,
      "logits/rejected": -1.3743739128112793,
      "logps/chosen": -0.8405038714408875,
      "logps/rejected": -2.2632803916931152,
      "loss": 0.8875,
      "num_input_tokens_seen": 3963056,
      "odds_ratio_loss": 8.778888702392578,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.08405039459466934,
      "rewards/margins": 0.14227768778800964,
      "rewards/rejected": -0.2263280600309372,
      "sft_loss": 0.009616486728191376,
      "step": 3030
    },
    {
      "epoch": 0.23648385842084793,
      "grad_norm": 2.736375570297241,
      "learning_rate": 4.3412142542358746e-05,
      "logits/chosen": -0.09749384224414825,
      "logits/rejected": -1.2443464994430542,
      "logps/chosen": -0.9987101554870605,
      "logps/rejected": -3.874497652053833,
      "loss": 1.0336,
      "num_input_tokens_seen": 3973632,
      "odds_ratio_loss": 10.03593635559082,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.09987101703882217,
      "rewards/margins": 0.287578821182251,
      "rewards/rejected": -0.38744983077049255,
      "sft_loss": 0.030052388086915016,
      "step": 3040
    },
    {
      "epoch": 0.23726176584986386,
      "grad_norm": 1.4619851112365723,
      "learning_rate": 4.3370758543091735e-05,
      "logits/chosen": -0.030552173033356667,
      "logits/rejected": -1.4240920543670654,
      "logps/chosen": -0.9915860295295715,
      "logps/rejected": -1.7709944248199463,
      "loss": 1.0508,
      "num_input_tokens_seen": 3983488,
      "odds_ratio_loss": 10.395299911499023,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.0991586297750473,
      "rewards/margins": 0.07794082909822464,
      "rewards/rejected": -0.17709943652153015,
      "sft_loss": 0.011277599260210991,
      "step": 3050
    },
    {
      "epoch": 0.23803967327887982,
      "grad_norm": 4.666402816772461,
      "learning_rate": 4.332926482466919e-05,
      "logits/chosen": -0.033102355897426605,
      "logits/rejected": -0.9652382731437683,
      "logps/chosen": -0.8036381006240845,
      "logps/rejected": -1.126878023147583,
      "loss": 0.8586,
      "num_input_tokens_seen": 3993264,
      "odds_ratio_loss": 8.400969505310059,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.08036380261182785,
      "rewards/margins": 0.03232399746775627,
      "rewards/rejected": -0.11268781125545502,
      "sft_loss": 0.018524659797549248,
      "step": 3060
    },
    {
      "epoch": 0.23881758070789577,
      "grad_norm": 7.239573955535889,
      "learning_rate": 4.32876616349119e-05,
      "logits/chosen": -0.0428139865398407,
      "logits/rejected": -1.262446641921997,
      "logps/chosen": -1.015485405921936,
      "logps/rejected": -1.9241611957550049,
      "loss": 1.0565,
      "num_input_tokens_seen": 4006208,
      "odds_ratio_loss": 10.502401351928711,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.10154855251312256,
      "rewards/margins": 0.09086756408214569,
      "rewards/rejected": -0.19241611659526825,
      "sft_loss": 0.006284940056502819,
      "step": 3070
    },
    {
      "epoch": 0.2395954881369117,
      "grad_norm": 2.8917922973632812,
      "learning_rate": 4.3245949222294466e-05,
      "logits/chosen": -0.06223181635141373,
      "logits/rejected": -1.25765061378479,
      "logps/chosen": -0.9540655016899109,
      "logps/rejected": -2.0520453453063965,
      "loss": 1.0042,
      "num_input_tokens_seen": 4019776,
      "odds_ratio_loss": 9.981868743896484,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.09540656208992004,
      "rewards/margins": 0.10979799926280975,
      "rewards/rejected": -0.2052045613527298,
      "sft_loss": 0.006059139966964722,
      "step": 3080
    },
    {
      "epoch": 0.24037339556592766,
      "grad_norm": 3.6924211978912354,
      "learning_rate": 4.320412783594381e-05,
      "logits/chosen": 0.03607861325144768,
      "logits/rejected": -1.6893059015274048,
      "logps/chosen": -0.9528789520263672,
      "logps/rejected": -1.5469565391540527,
      "loss": 0.9954,
      "num_input_tokens_seen": 4031968,
      "odds_ratio_loss": 9.90943717956543,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09528790414333344,
      "rewards/margins": 0.059407759457826614,
      "rewards/rejected": -0.15469565987586975,
      "sft_loss": 0.004419064149260521,
      "step": 3090
    },
    {
      "epoch": 0.2411513029949436,
      "grad_norm": 2.1363964080810547,
      "learning_rate": 4.316219772563772e-05,
      "logits/chosen": -0.01864052750170231,
      "logits/rejected": -1.3548976182937622,
      "logps/chosen": -0.8682738542556763,
      "logps/rejected": -1.7034642696380615,
      "loss": 0.9118,
      "num_input_tokens_seen": 4044592,
      "odds_ratio_loss": 8.747833251953125,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.08682739734649658,
      "rewards/margins": 0.08351902663707733,
      "rewards/rejected": -0.1703464239835739,
      "sft_loss": 0.037029556930065155,
      "step": 3100
    },
    {
      "epoch": 0.24192921042395954,
      "grad_norm": 6.060778617858887,
      "learning_rate": 4.312015914180332e-05,
      "logits/chosen": -0.16518588364124298,
      "logits/rejected": -0.9924762845039368,
      "logps/chosen": -0.9183260798454285,
      "logps/rejected": -3.4362235069274902,
      "loss": 0.9622,
      "num_input_tokens_seen": 4056096,
      "odds_ratio_loss": 9.315229415893555,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.09183261543512344,
      "rewards/margins": 0.25178977847099304,
      "rewards/rejected": -0.3436223864555359,
      "sft_loss": 0.030677739530801773,
      "step": 3110
    },
    {
      "epoch": 0.2427071178529755,
      "grad_norm": 3.24570631980896,
      "learning_rate": 4.307801233551558e-05,
      "logits/chosen": -0.4022963047027588,
      "logits/rejected": -0.7498866319656372,
      "logps/chosen": -0.9533295631408691,
      "logps/rejected": -2.0013861656188965,
      "loss": 1.0044,
      "num_input_tokens_seen": 4068272,
      "odds_ratio_loss": 9.946795463562012,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.09533295035362244,
      "rewards/margins": 0.10480568557977676,
      "rewards/rejected": -0.2001386433839798,
      "sft_loss": 0.009699802845716476,
      "step": 3120
    },
    {
      "epoch": 0.24348502528199145,
      "grad_norm": 2.2562637329101562,
      "learning_rate": 4.303575755849587e-05,
      "logits/chosen": -0.20986226201057434,
      "logits/rejected": -0.7615283727645874,
      "logps/chosen": -1.099443793296814,
      "logps/rejected": -2.7838354110717773,
      "loss": 1.1389,
      "num_input_tokens_seen": 4079664,
      "odds_ratio_loss": 11.104766845703125,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.10994438081979752,
      "rewards/margins": 0.16843917965888977,
      "rewards/rejected": -0.2783835530281067,
      "sft_loss": 0.028436914086341858,
      "step": 3130
    },
    {
      "epoch": 0.24426293271100738,
      "grad_norm": 7.0951409339904785,
      "learning_rate": 4.2993395063110365e-05,
      "logits/chosen": 0.050548214465379715,
      "logits/rejected": -1.5015652179718018,
      "logps/chosen": -0.9225507974624634,
      "logps/rejected": -2.1379103660583496,
      "loss": 0.9769,
      "num_input_tokens_seen": 4094448,
      "odds_ratio_loss": 9.712604522705078,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.09225507825613022,
      "rewards/margins": 0.12153597921133041,
      "rewards/rejected": -0.21379104256629944,
      "sft_loss": 0.005675177555531263,
      "step": 3140
    },
    {
      "epoch": 0.24504084014002334,
      "grad_norm": 2.7226779460906982,
      "learning_rate": 4.2950925102368605e-05,
      "logits/chosen": -0.22441892325878143,
      "logits/rejected": -0.9779049754142761,
      "logps/chosen": -1.048175573348999,
      "logps/rejected": -1.4846611022949219,
      "loss": 1.1078,
      "num_input_tokens_seen": 4108192,
      "odds_ratio_loss": 10.27183723449707,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.10481754690408707,
      "rewards/margins": 0.043648574501276016,
      "rewards/rejected": -0.14846612513065338,
      "sft_loss": 0.08061198890209198,
      "step": 3150
    },
    {
      "epoch": 0.2458187475690393,
      "grad_norm": 1.8000941276550293,
      "learning_rate": 4.290834792992198e-05,
      "logits/chosen": 0.08007173985242844,
      "logits/rejected": -1.388127326965332,
      "logps/chosen": -0.8269979357719421,
      "logps/rejected": -1.2204464673995972,
      "loss": 0.8765,
      "num_input_tokens_seen": 4127088,
      "odds_ratio_loss": 8.728238105773926,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.08269979804754257,
      "rewards/margins": 0.03934483975172043,
      "rewards/rejected": -0.122044637799263,
      "sft_loss": 0.0036289836280047894,
      "step": 3160
    },
    {
      "epoch": 0.24659665499805522,
      "grad_norm": 1.884130835533142,
      "learning_rate": 4.286566380006218e-05,
      "logits/chosen": -0.21565194427967072,
      "logits/rejected": -1.293933391571045,
      "logps/chosen": -0.9503879547119141,
      "logps/rejected": -1.7476370334625244,
      "loss": 0.9898,
      "num_input_tokens_seen": 4140960,
      "odds_ratio_loss": 9.796585083007812,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.09503878653049469,
      "rewards/margins": 0.07972491532564163,
      "rewards/rejected": -0.17476370930671692,
      "sft_loss": 0.010092451237142086,
      "step": 3170
    },
    {
      "epoch": 0.24737456242707118,
      "grad_norm": 2.0614566802978516,
      "learning_rate": 4.282287296771972e-05,
      "logits/chosen": 0.061348915100097656,
      "logits/rejected": -1.3977020978927612,
      "logps/chosen": -0.9699851870536804,
      "logps/rejected": -1.573901891708374,
      "loss": 1.0286,
      "num_input_tokens_seen": 4155904,
      "odds_ratio_loss": 10.165769577026367,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.09699852019548416,
      "rewards/margins": 0.06039166450500488,
      "rewards/rejected": -0.15739016234874725,
      "sft_loss": 0.012014877051115036,
      "step": 3180
    },
    {
      "epoch": 0.24815246985608713,
      "grad_norm": 4.029314994812012,
      "learning_rate": 4.277997568846237e-05,
      "logits/chosen": 0.060758672654628754,
      "logits/rejected": -1.4077775478363037,
      "logps/chosen": -0.9179569482803345,
      "logps/rejected": -1.417022466659546,
      "loss": 0.9598,
      "num_input_tokens_seen": 4174832,
      "odds_ratio_loss": 9.570338249206543,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.09179569035768509,
      "rewards/margins": 0.04990655183792114,
      "rewards/rejected": -0.14170224964618683,
      "sft_loss": 0.0027877301909029484,
      "step": 3190
    },
    {
      "epoch": 0.24893037728510306,
      "grad_norm": 1.9069956541061401,
      "learning_rate": 4.2736972218493664e-05,
      "logits/chosen": -0.34009242057800293,
      "logits/rejected": -0.9457812309265137,
      "logps/chosen": -0.9835821390151978,
      "logps/rejected": -3.5627548694610596,
      "loss": 1.0203,
      "num_input_tokens_seen": 4191232,
      "odds_ratio_loss": 10.105257987976074,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.09835819900035858,
      "rewards/margins": 0.25791728496551514,
      "rewards/rejected": -0.3562754988670349,
      "sft_loss": 0.009779800660908222,
      "step": 3200
    },
    {
      "epoch": 0.24970828471411902,
      "grad_norm": 11.421212196350098,
      "learning_rate": 4.269386281465136e-05,
      "logits/chosen": -0.2749151587486267,
      "logits/rejected": -0.9223438501358032,
      "logps/chosen": -1.0342686176300049,
      "logps/rejected": -1.3730922937393188,
      "loss": 1.0954,
      "num_input_tokens_seen": 4204640,
      "odds_ratio_loss": 10.71926212310791,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.10342685878276825,
      "rewards/margins": 0.033882349729537964,
      "rewards/rejected": -0.1373092234134674,
      "sft_loss": 0.023464282974600792,
      "step": 3210
    },
    {
      "epoch": 0.25048619214313494,
      "grad_norm": 2.6410937309265137,
      "learning_rate": 4.2650647734405914e-05,
      "logits/chosen": 0.08165550976991653,
      "logits/rejected": -1.680293083190918,
      "logps/chosen": -0.9836196899414062,
      "logps/rejected": -2.458129644393921,
      "loss": 1.0289,
      "num_input_tokens_seen": 4220752,
      "odds_ratio_loss": 10.125057220458984,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.09836197644472122,
      "rewards/margins": 0.1474509984254837,
      "rewards/rejected": -0.24581298232078552,
      "sft_loss": 0.016405515372753143,
      "step": 3220
    },
    {
      "epoch": 0.2512640995721509,
      "grad_norm": 1.7691044807434082,
      "learning_rate": 4.2607327235858894e-05,
      "logits/chosen": -0.3390534520149231,
      "logits/rejected": -1.4492483139038086,
      "logps/chosen": -0.9570733308792114,
      "logps/rejected": -2.46934175491333,
      "loss": 0.9947,
      "num_input_tokens_seen": 4233920,
      "odds_ratio_loss": 9.843822479248047,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.09570731967687607,
      "rewards/margins": 0.15122681856155396,
      "rewards/rejected": -0.24693414568901062,
      "sft_loss": 0.010312440805137157,
      "step": 3230
    },
    {
      "epoch": 0.25204200700116686,
      "grad_norm": 4.76840353012085,
      "learning_rate": 4.256390157774154e-05,
      "logits/chosen": 0.003717652056366205,
      "logits/rejected": -1.1586898565292358,
      "logps/chosen": -1.0405880212783813,
      "logps/rejected": -2.021322250366211,
      "loss": 1.0714,
      "num_input_tokens_seen": 4243920,
      "odds_ratio_loss": 10.504170417785645,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.10405881702899933,
      "rewards/margins": 0.09807340800762177,
      "rewards/rejected": -0.2021322250366211,
      "sft_loss": 0.02094918303191662,
      "step": 3240
    },
    {
      "epoch": 0.2528199144301828,
      "grad_norm": 3.3458433151245117,
      "learning_rate": 4.2520371019413095e-05,
      "logits/chosen": 0.016684969887137413,
      "logits/rejected": -1.4637718200683594,
      "logps/chosen": -0.9979241490364075,
      "logps/rejected": -1.426872730255127,
      "loss": 1.0548,
      "num_input_tokens_seen": 4256112,
      "odds_ratio_loss": 10.42685317993164,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.09979242086410522,
      "rewards/margins": 0.04289485886693001,
      "rewards/rejected": -0.14268726110458374,
      "sft_loss": 0.012084713205695152,
      "step": 3250
    },
    {
      "epoch": 0.25359782185919877,
      "grad_norm": 3.482081174850464,
      "learning_rate": 4.2476735820859374e-05,
      "logits/chosen": 0.01486960332840681,
      "logits/rejected": -1.3319334983825684,
      "logps/chosen": -1.0822274684906006,
      "logps/rejected": -1.4299288988113403,
      "loss": 1.1476,
      "num_input_tokens_seen": 4270368,
      "odds_ratio_loss": 11.378705978393555,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.10822276026010513,
      "rewards/margins": 0.034770142287015915,
      "rewards/rejected": -0.14299288392066956,
      "sft_loss": 0.00969963613897562,
      "step": 3260
    },
    {
      "epoch": 0.2543757292882147,
      "grad_norm": 5.3337721824646,
      "learning_rate": 4.2432996242691095e-05,
      "logits/chosen": -0.19184128940105438,
      "logits/rejected": -1.325873851776123,
      "logps/chosen": -0.8774449229240417,
      "logps/rejected": -1.518665075302124,
      "loss": 0.9217,
      "num_input_tokens_seen": 4283568,
      "odds_ratio_loss": 9.148040771484375,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.08774449676275253,
      "rewards/margins": 0.06412201374769211,
      "rewards/rejected": -0.15186652541160583,
      "sft_loss": 0.006918758153915405,
      "step": 3270
    },
    {
      "epoch": 0.2551536367172306,
      "grad_norm": 6.184454441070557,
      "learning_rate": 4.238915254614243e-05,
      "logits/chosen": -0.2082027643918991,
      "logits/rejected": -1.2313646078109741,
      "logps/chosen": -1.2307305335998535,
      "logps/rejected": -1.9771459102630615,
      "loss": 1.2814,
      "num_input_tokens_seen": 4293056,
      "odds_ratio_loss": 12.01191520690918,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.1230730414390564,
      "rewards/margins": 0.07464151829481125,
      "rewards/rejected": -0.19771459698677063,
      "sft_loss": 0.0802154690027237,
      "step": 3280
    },
    {
      "epoch": 0.2559315441462466,
      "grad_norm": 1.5774378776550293,
      "learning_rate": 4.2345204993069376e-05,
      "logits/chosen": -0.09383687376976013,
      "logits/rejected": -1.0173933506011963,
      "logps/chosen": -0.8213062286376953,
      "logps/rejected": -1.9807208776474,
      "loss": 0.8583,
      "num_input_tokens_seen": 4307728,
      "odds_ratio_loss": 8.337804794311523,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.08213062584400177,
      "rewards/margins": 0.11594147980213165,
      "rewards/rejected": -0.19807210564613342,
      "sft_loss": 0.02448856830596924,
      "step": 3290
    },
    {
      "epoch": 0.25670945157526254,
      "grad_norm": 2.8531248569488525,
      "learning_rate": 4.230115384594821e-05,
      "logits/chosen": 0.13179947435855865,
      "logits/rejected": -1.6411634683609009,
      "logps/chosen": -0.7820327281951904,
      "logps/rejected": -2.090890884399414,
      "loss": 0.8122,
      "num_input_tokens_seen": 4320736,
      "odds_ratio_loss": 8.024853706359863,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.07820327579975128,
      "rewards/margins": 0.13088582456111908,
      "rewards/rejected": -0.20908911526203156,
      "sft_loss": 0.009706547483801842,
      "step": 3300
    },
    {
      "epoch": 0.2574873590042785,
      "grad_norm": 4.10869026184082,
      "learning_rate": 4.225699936787394e-05,
      "logits/chosen": -0.10895247757434845,
      "logits/rejected": -1.4337221384048462,
      "logps/chosen": -0.8080259561538696,
      "logps/rejected": -2.6543774604797363,
      "loss": 0.8513,
      "num_input_tokens_seen": 4332432,
      "odds_ratio_loss": 8.121633529663086,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.08080259710550308,
      "rewards/margins": 0.18463513255119324,
      "rewards/rejected": -0.2654377222061157,
      "sft_loss": 0.039158955216407776,
      "step": 3310
    },
    {
      "epoch": 0.25826526643329445,
      "grad_norm": 2.5322458744049072,
      "learning_rate": 4.22127418225587e-05,
      "logits/chosen": 0.08826179057359695,
      "logits/rejected": -1.37474524974823,
      "logps/chosen": -0.9683723449707031,
      "logps/rejected": -1.6476446390151978,
      "loss": 1.0142,
      "num_input_tokens_seen": 4348576,
      "odds_ratio_loss": 10.062582015991211,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.09683723747730255,
      "rewards/margins": 0.06792722642421722,
      "rewards/rejected": -0.16476444900035858,
      "sft_loss": 0.007978400215506554,
      "step": 3320
    },
    {
      "epoch": 0.2590431738623104,
      "grad_norm": 6.394929885864258,
      "learning_rate": 4.21683814743302e-05,
      "logits/chosen": -0.4122388958930969,
      "logits/rejected": -1.254586935043335,
      "logps/chosen": -0.7898470759391785,
      "logps/rejected": -1.3862065076828003,
      "loss": 0.8378,
      "num_input_tokens_seen": 4361584,
      "odds_ratio_loss": 8.037008285522461,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.07898470759391785,
      "rewards/margins": 0.0596359483897686,
      "rewards/rejected": -0.13862064480781555,
      "sft_loss": 0.03409110754728317,
      "step": 3330
    },
    {
      "epoch": 0.2598210812913263,
      "grad_norm": 3.825871706008911,
      "learning_rate": 4.212391858813013e-05,
      "logits/chosen": -0.12090665102005005,
      "logits/rejected": -1.2372742891311646,
      "logps/chosen": -0.8906979560852051,
      "logps/rejected": -3.0938544273376465,
      "loss": 0.9238,
      "num_input_tokens_seen": 4372672,
      "odds_ratio_loss": 9.202934265136719,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08906979858875275,
      "rewards/margins": 0.22031569480895996,
      "rewards/rejected": -0.3093855082988739,
      "sft_loss": 0.0035380299668759108,
      "step": 3340
    },
    {
      "epoch": 0.26059898872034226,
      "grad_norm": 2.6060073375701904,
      "learning_rate": 4.207935342951261e-05,
      "logits/chosen": -0.35914385318756104,
      "logits/rejected": -1.3731544017791748,
      "logps/chosen": -1.066075325012207,
      "logps/rejected": -1.582295536994934,
      "loss": 1.11,
      "num_input_tokens_seen": 4382704,
      "odds_ratio_loss": 11.008880615234375,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.10660751909017563,
      "rewards/margins": 0.05162200331687927,
      "rewards/rejected": -0.15822955965995789,
      "sft_loss": 0.009122246876358986,
      "step": 3350
    },
    {
      "epoch": 0.2613768961493582,
      "grad_norm": 4.739326477050781,
      "learning_rate": 4.203468626464254e-05,
      "logits/chosen": -0.03921103477478027,
      "logits/rejected": -1.5998603105545044,
      "logps/chosen": -0.8556955456733704,
      "logps/rejected": -1.8652175664901733,
      "loss": 0.8993,
      "num_input_tokens_seen": 4394320,
      "odds_ratio_loss": 8.893582344055176,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.08556955307722092,
      "rewards/margins": 0.10095222294330597,
      "rewards/rejected": -0.18652178347110748,
      "sft_loss": 0.009982654824852943,
      "step": 3360
    },
    {
      "epoch": 0.2621548035783742,
      "grad_norm": 6.477309226989746,
      "learning_rate": 4.1989917360294106e-05,
      "logits/chosen": -0.5154345035552979,
      "logits/rejected": -0.8112955093383789,
      "logps/chosen": -1.1205177307128906,
      "logps/rejected": -2.7822554111480713,
      "loss": 1.1625,
      "num_input_tokens_seen": 4401664,
      "odds_ratio_loss": 11.260440826416016,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.11205177009105682,
      "rewards/margins": 0.1661737859249115,
      "rewards/rejected": -0.27822554111480713,
      "sft_loss": 0.03643777221441269,
      "step": 3370
    },
    {
      "epoch": 0.26293271100739013,
      "grad_norm": 4.272067546844482,
      "learning_rate": 4.194504698384908e-05,
      "logits/chosen": -0.19038191437721252,
      "logits/rejected": -1.9764163494110107,
      "logps/chosen": -1.1597676277160645,
      "logps/rejected": -3.2375998497009277,
      "loss": 1.184,
      "num_input_tokens_seen": 4413920,
      "odds_ratio_loss": 11.342437744140625,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.11597676575183868,
      "rewards/margins": 0.20778322219848633,
      "rewards/rejected": -0.3237599730491638,
      "sft_loss": 0.04977894946932793,
      "step": 3380
    },
    {
      "epoch": 0.2637106184364061,
      "grad_norm": 7.2915191650390625,
      "learning_rate": 4.1900075403295304e-05,
      "logits/chosen": -0.2795480191707611,
      "logits/rejected": -0.964443027973175,
      "logps/chosen": -0.9262790679931641,
      "logps/rejected": -1.7858781814575195,
      "loss": 0.9663,
      "num_input_tokens_seen": 4424784,
      "odds_ratio_loss": 9.245964050292969,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.09262792021036148,
      "rewards/margins": 0.08595990389585495,
      "rewards/rejected": -0.17858782410621643,
      "sft_loss": 0.041728947311639786,
      "step": 3390
    },
    {
      "epoch": 0.26448852586542204,
      "grad_norm": 0.9598113894462585,
      "learning_rate": 4.185500288722505e-05,
      "logits/chosen": -0.24663646519184113,
      "logits/rejected": -1.0148065090179443,
      "logps/chosen": -0.7349613904953003,
      "logps/rejected": -1.3731105327606201,
      "loss": 0.7776,
      "num_input_tokens_seen": 4433632,
      "odds_ratio_loss": 7.688754081726074,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.07349614799022675,
      "rewards/margins": 0.0638149231672287,
      "rewards/rejected": -0.13731107115745544,
      "sft_loss": 0.00868178904056549,
      "step": 3400
    },
    {
      "epoch": 0.26526643329443794,
      "grad_norm": 1.427416443824768,
      "learning_rate": 4.180982970483344e-05,
      "logits/chosen": -0.0015018790727481246,
      "logits/rejected": -1.599158763885498,
      "logps/chosen": -0.9013495445251465,
      "logps/rejected": -2.0213136672973633,
      "loss": 0.9434,
      "num_input_tokens_seen": 4446192,
      "odds_ratio_loss": 9.38914680480957,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.09013496339321136,
      "rewards/margins": 0.11199643462896347,
      "rewards/rejected": -0.20213139057159424,
      "sft_loss": 0.004515963606536388,
      "step": 3410
    },
    {
      "epoch": 0.2660443407234539,
      "grad_norm": 5.408295154571533,
      "learning_rate": 4.176455612591681e-05,
      "logits/chosen": -0.4608125686645508,
      "logits/rejected": -0.97273188829422,
      "logps/chosen": -0.7134547233581543,
      "logps/rejected": -1.6708390712738037,
      "loss": 0.7579,
      "num_input_tokens_seen": 4456784,
      "odds_ratio_loss": 7.530245304107666,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.07134547084569931,
      "rewards/margins": 0.09573844820261002,
      "rewards/rejected": -0.16708391904830933,
      "sft_loss": 0.0048301443457603455,
      "step": 3420
    },
    {
      "epoch": 0.26682224815246985,
      "grad_norm": 5.093200206756592,
      "learning_rate": 4.1719182420871104e-05,
      "logits/chosen": -0.10298941284418106,
      "logits/rejected": -1.2602466344833374,
      "logps/chosen": -0.9793552160263062,
      "logps/rejected": -1.680318832397461,
      "loss": 1.0241,
      "num_input_tokens_seen": 4468704,
      "odds_ratio_loss": 10.133760452270508,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.09793553501367569,
      "rewards/margins": 0.07009634375572205,
      "rewards/rejected": -0.16803188621997833,
      "sft_loss": 0.010695156641304493,
      "step": 3430
    },
    {
      "epoch": 0.2676001555814858,
      "grad_norm": 4.929412364959717,
      "learning_rate": 4.16737088606903e-05,
      "logits/chosen": -0.06423575431108475,
      "logits/rejected": -1.2866432666778564,
      "logps/chosen": -0.8811386227607727,
      "logps/rejected": -1.5141222476959229,
      "loss": 0.9317,
      "num_input_tokens_seen": 4480144,
      "odds_ratio_loss": 8.985457420349121,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.08811385929584503,
      "rewards/margins": 0.06329836696386337,
      "rewards/rejected": -0.151412233710289,
      "sft_loss": 0.03313466161489487,
      "step": 3440
    },
    {
      "epoch": 0.26837806301050177,
      "grad_norm": 2.390888214111328,
      "learning_rate": 4.1628135716964734e-05,
      "logits/chosen": -0.019254203885793686,
      "logits/rejected": -1.3756763935089111,
      "logps/chosen": -0.9164663553237915,
      "logps/rejected": -2.7706151008605957,
      "loss": 0.9583,
      "num_input_tokens_seen": 4493504,
      "odds_ratio_loss": 8.82702350616455,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.09164663404226303,
      "rewards/margins": 0.1854148954153061,
      "rewards/rejected": -0.2770615518093109,
      "sft_loss": 0.07561619579792023,
      "step": 3450
    },
    {
      "epoch": 0.2691559704395177,
      "grad_norm": 2.5836713314056396,
      "learning_rate": 4.158246326187951e-05,
      "logits/chosen": -0.09574378281831741,
      "logits/rejected": -1.3053169250488281,
      "logps/chosen": -0.9305857419967651,
      "logps/rejected": -2.1507930755615234,
      "loss": 0.9748,
      "num_input_tokens_seen": 4507200,
      "odds_ratio_loss": 9.583921432495117,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.09305857867002487,
      "rewards/margins": 0.12202074378728867,
      "rewards/rejected": -0.21507930755615234,
      "sft_loss": 0.01643938571214676,
      "step": 3460
    },
    {
      "epoch": 0.2699338778685336,
      "grad_norm": 2.783560276031494,
      "learning_rate": 4.153669176821288e-05,
      "logits/chosen": -0.2140583097934723,
      "logits/rejected": -0.7443190813064575,
      "logps/chosen": -1.1720969676971436,
      "logps/rejected": -2.1571438312530518,
      "loss": 1.2102,
      "num_input_tokens_seen": 4519392,
      "odds_ratio_loss": 11.70689868927002,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.11720969527959824,
      "rewards/margins": 0.0985046997666359,
      "rewards/rejected": -0.21571438014507294,
      "sft_loss": 0.03948521986603737,
      "step": 3470
    },
    {
      "epoch": 0.2707117852975496,
      "grad_norm": 12.581844329833984,
      "learning_rate": 4.149082150933456e-05,
      "logits/chosen": -0.15463215112686157,
      "logits/rejected": -1.3547630310058594,
      "logps/chosen": -1.0013000965118408,
      "logps/rejected": -2.429577589035034,
      "loss": 1.0537,
      "num_input_tokens_seen": 4531984,
      "odds_ratio_loss": 10.430364608764648,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.10013000667095184,
      "rewards/margins": 0.14282776415348053,
      "rewards/rejected": -0.24295778572559357,
      "sft_loss": 0.01061457023024559,
      "step": 3480
    },
    {
      "epoch": 0.27148969272656553,
      "grad_norm": 4.4241533279418945,
      "learning_rate": 4.144485275920422e-05,
      "logits/chosen": -0.17204315960407257,
      "logits/rejected": -0.9298747777938843,
      "logps/chosen": -1.1662946939468384,
      "logps/rejected": -1.887099027633667,
      "loss": 1.2139,
      "num_input_tokens_seen": 4547792,
      "odds_ratio_loss": 11.961484909057617,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.11662948131561279,
      "rewards/margins": 0.07208044826984406,
      "rewards/rejected": -0.18870989978313446,
      "sft_loss": 0.017767272889614105,
      "step": 3490
    },
    {
      "epoch": 0.2722676001555815,
      "grad_norm": 3.9844746589660645,
      "learning_rate": 4.1398785792369676e-05,
      "logits/chosen": -0.10230766236782074,
      "logits/rejected": -1.183638334274292,
      "logps/chosen": -1.080854892730713,
      "logps/rejected": -3.9099273681640625,
      "loss": 1.1075,
      "num_input_tokens_seen": 4560016,
      "odds_ratio_loss": 10.963944435119629,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.108085498213768,
      "rewards/margins": 0.28290724754333496,
      "rewards/rejected": -0.39099276065826416,
      "sft_loss": 0.011096212081611156,
      "step": 3500
    },
    {
      "epoch": 0.27304550758459745,
      "grad_norm": 3.405622959136963,
      "learning_rate": 4.135262088396542e-05,
      "logits/chosen": -0.11642368882894516,
      "logits/rejected": -1.573893427848816,
      "logps/chosen": -1.002684235572815,
      "logps/rejected": -2.7077748775482178,
      "loss": 1.0274,
      "num_input_tokens_seen": 4573536,
      "odds_ratio_loss": 9.916679382324219,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1002684235572815,
      "rewards/margins": 0.17050907015800476,
      "rewards/rejected": -0.27077749371528625,
      "sft_loss": 0.035697899758815765,
      "step": 3510
    },
    {
      "epoch": 0.2738234150136134,
      "grad_norm": 1.425681710243225,
      "learning_rate": 4.130635830971084e-05,
      "logits/chosen": -0.13541904091835022,
      "logits/rejected": -1.7754377126693726,
      "logps/chosen": -0.8972693681716919,
      "logps/rejected": -2.1450066566467285,
      "loss": 0.931,
      "num_input_tokens_seen": 4592768,
      "odds_ratio_loss": 9.266926765441895,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.08972693979740143,
      "rewards/margins": 0.12477374076843262,
      "rewards/rejected": -0.21450069546699524,
      "sft_loss": 0.004262088797986507,
      "step": 3520
    },
    {
      "epoch": 0.2746013224426293,
      "grad_norm": 2.133974075317383,
      "learning_rate": 4.125999834590867e-05,
      "logits/chosen": -0.5876361131668091,
      "logits/rejected": -1.141478180885315,
      "logps/chosen": -1.6601444482803345,
      "logps/rejected": -1.2758491039276123,
      "loss": 1.7911,
      "num_input_tokens_seen": 4602480,
      "odds_ratio_loss": 16.37868881225586,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": -0.16601444780826569,
      "rewards/margins": -0.03842953220009804,
      "rewards/rejected": -0.12758490443229675,
      "sft_loss": 0.15324907004833221,
      "step": 3530
    },
    {
      "epoch": 0.27537922987164526,
      "grad_norm": 4.674381256103516,
      "learning_rate": 4.121354126944329e-05,
      "logits/chosen": -0.5263653993606567,
      "logits/rejected": -1.006824254989624,
      "logps/chosen": -1.0107089281082153,
      "logps/rejected": -1.957911729812622,
      "loss": 1.0487,
      "num_input_tokens_seen": 4614832,
      "odds_ratio_loss": 10.238679885864258,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.10107089579105377,
      "rewards/margins": 0.0947202816605568,
      "rewards/rejected": -0.19579115509986877,
      "sft_loss": 0.024857979267835617,
      "step": 3540
    },
    {
      "epoch": 0.2761571373006612,
      "grad_norm": 8.11523723602295,
      "learning_rate": 4.116698735777907e-05,
      "logits/chosen": -0.3436376452445984,
      "logits/rejected": -1.0312341451644897,
      "logps/chosen": -0.7612732648849487,
      "logps/rejected": -2.2395224571228027,
      "loss": 0.8017,
      "num_input_tokens_seen": 4626304,
      "odds_ratio_loss": 7.937858581542969,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.07612732797861099,
      "rewards/margins": 0.14782488346099854,
      "rewards/rejected": -0.2239522486925125,
      "sft_loss": 0.007900206372141838,
      "step": 3550
    },
    {
      "epoch": 0.27693504472967717,
      "grad_norm": 2.475008010864258,
      "learning_rate": 4.1120336888958766e-05,
      "logits/chosen": -0.2083013951778412,
      "logits/rejected": -1.0854121446609497,
      "logps/chosen": -0.9013233184814453,
      "logps/rejected": -2.847856044769287,
      "loss": 0.9514,
      "num_input_tokens_seen": 4638624,
      "odds_ratio_loss": 9.257806777954102,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.09013234078884125,
      "rewards/margins": 0.19465331733226776,
      "rewards/rejected": -0.284785658121109,
      "sft_loss": 0.025621671229600906,
      "step": 3560
    },
    {
      "epoch": 0.2777129521586931,
      "grad_norm": 8.027059555053711,
      "learning_rate": 4.107359014160177e-05,
      "logits/chosen": -0.2505590319633484,
      "logits/rejected": -1.458212971687317,
      "logps/chosen": -1.0533485412597656,
      "logps/rejected": -2.902940511703491,
      "loss": 1.0792,
      "num_input_tokens_seen": 4647168,
      "odds_ratio_loss": 10.279277801513672,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.10533484071493149,
      "rewards/margins": 0.18495921790599823,
      "rewards/rejected": -0.2902940809726715,
      "sft_loss": 0.05122938007116318,
      "step": 3570
    },
    {
      "epoch": 0.2784908595877091,
      "grad_norm": 1.9160383939743042,
      "learning_rate": 4.102674739490254e-05,
      "logits/chosen": -0.05519244819879532,
      "logits/rejected": -1.2347227334976196,
      "logps/chosen": -0.9562760591506958,
      "logps/rejected": -2.191115379333496,
      "loss": 1.0222,
      "num_input_tokens_seen": 4659312,
      "odds_ratio_loss": 9.945977210998535,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.09562759846448898,
      "rewards/margins": 0.12348394095897675,
      "rewards/rejected": -0.21911153197288513,
      "sft_loss": 0.027616357430815697,
      "step": 3580
    },
    {
      "epoch": 0.279268767016725,
      "grad_norm": 2.969160556793213,
      "learning_rate": 4.097980892862886e-05,
      "logits/chosen": -0.13975301384925842,
      "logits/rejected": -0.8550666570663452,
      "logps/chosen": -0.8890474438667297,
      "logps/rejected": -1.838004469871521,
      "loss": 0.923,
      "num_input_tokens_seen": 4674992,
      "odds_ratio_loss": 9.136037826538086,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.0889047384262085,
      "rewards/margins": 0.09489572048187256,
      "rewards/rejected": -0.18380047380924225,
      "sft_loss": 0.009357091039419174,
      "step": 3590
    },
    {
      "epoch": 0.28004667444574094,
      "grad_norm": 2.5685956478118896,
      "learning_rate": 4.093277502312022e-05,
      "logits/chosen": -0.6892591118812561,
      "logits/rejected": -0.5192093849182129,
      "logps/chosen": -1.035064458847046,
      "logps/rejected": -2.2198333740234375,
      "loss": 1.0698,
      "num_input_tokens_seen": 4684448,
      "odds_ratio_loss": 9.483429908752441,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.10350646078586578,
      "rewards/margins": 0.11847686767578125,
      "rewards/rejected": -0.22198334336280823,
      "sft_loss": 0.12144489586353302,
      "step": 3600
    },
    {
      "epoch": 0.2808245818747569,
      "grad_norm": 2.3054659366607666,
      "learning_rate": 4.088564595928609e-05,
      "logits/chosen": -0.18362614512443542,
      "logits/rejected": -0.8535488247871399,
      "logps/chosen": -1.262632131576538,
      "logps/rejected": -2.090200424194336,
      "loss": 1.3427,
      "num_input_tokens_seen": 4695472,
      "odds_ratio_loss": 12.58495044708252,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.12626323103904724,
      "rewards/margins": 0.08275681734085083,
      "rewards/rejected": -0.20902001857757568,
      "sft_loss": 0.0841931700706482,
      "step": 3610
    },
    {
      "epoch": 0.28160248930377285,
      "grad_norm": 3.6128058433532715,
      "learning_rate": 4.083842201860431e-05,
      "logits/chosen": -0.11260410398244858,
      "logits/rejected": -0.9792426228523254,
      "logps/chosen": -1.070449948310852,
      "logps/rejected": -2.4821507930755615,
      "loss": 1.1047,
      "num_input_tokens_seen": 4708896,
      "odds_ratio_loss": 10.800509452819824,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.1070449948310852,
      "rewards/margins": 0.14117011427879333,
      "rewards/rejected": -0.24821512401103973,
      "sft_loss": 0.02461850829422474,
      "step": 3620
    },
    {
      "epoch": 0.2823803967327888,
      "grad_norm": 2.64975643157959,
      "learning_rate": 4.079110348311934e-05,
      "logits/chosen": 0.1377231627702713,
      "logits/rejected": -1.7623564004898071,
      "logps/chosen": -0.8980050086975098,
      "logps/rejected": -1.417351484298706,
      "loss": 0.9384,
      "num_input_tokens_seen": 4723376,
      "odds_ratio_loss": 9.318737030029297,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.08980049937963486,
      "rewards/margins": 0.05193463712930679,
      "rewards/rejected": -0.14173513650894165,
      "sft_loss": 0.006572979036718607,
      "step": 3630
    },
    {
      "epoch": 0.28315830416180476,
      "grad_norm": 6.5229411125183105,
      "learning_rate": 4.0743690635440634e-05,
      "logits/chosen": 0.1733946055173874,
      "logits/rejected": -1.2729114294052124,
      "logps/chosen": -0.9009236097335815,
      "logps/rejected": -2.0939249992370605,
      "loss": 0.9396,
      "num_input_tokens_seen": 4734944,
      "odds_ratio_loss": 9.328462600708008,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.09009236097335815,
      "rewards/margins": 0.11930011212825775,
      "rewards/rejected": -0.2093924731016159,
      "sft_loss": 0.0067521268501877785,
      "step": 3640
    },
    {
      "epoch": 0.2839362115908207,
      "grad_norm": 2.036282539367676,
      "learning_rate": 4.0696183758740894e-05,
      "logits/chosen": 0.10636024177074432,
      "logits/rejected": -1.588081955909729,
      "logps/chosen": -0.9661546945571899,
      "logps/rejected": -2.7496097087860107,
      "loss": 0.9965,
      "num_input_tokens_seen": 4748816,
      "odds_ratio_loss": 9.889032363891602,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09661546349525452,
      "rewards/margins": 0.17834553122520447,
      "rewards/rejected": -0.274960994720459,
      "sft_loss": 0.0075736879371106625,
      "step": 3650
    },
    {
      "epoch": 0.2847141190198366,
      "grad_norm": 4.932629585266113,
      "learning_rate": 4.064858313675443e-05,
      "logits/chosen": -0.26715725660324097,
      "logits/rejected": -0.7281715273857117,
      "logps/chosen": -1.0199708938598633,
      "logps/rejected": -2.7163097858428955,
      "loss": 1.0673,
      "num_input_tokens_seen": 4762544,
      "odds_ratio_loss": 10.461103439331055,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.10199709981679916,
      "rewards/margins": 0.16963385045528412,
      "rewards/rejected": -0.2716309726238251,
      "sft_loss": 0.02118172124028206,
      "step": 3660
    },
    {
      "epoch": 0.2854920264488526,
      "grad_norm": 7.332515239715576,
      "learning_rate": 4.060088905377543e-05,
      "logits/chosen": -0.023127097636461258,
      "logits/rejected": -1.021946668624878,
      "logps/chosen": -1.0226608514785767,
      "logps/rejected": -2.9361438751220703,
      "loss": 1.0629,
      "num_input_tokens_seen": 4778464,
      "odds_ratio_loss": 10.557701110839844,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.10226607322692871,
      "rewards/margins": 0.19134829938411713,
      "rewards/rejected": -0.29361438751220703,
      "sft_loss": 0.0071670361794531345,
      "step": 3670
    },
    {
      "epoch": 0.28626993387786853,
      "grad_norm": 9.548507690429688,
      "learning_rate": 4.05531017946563e-05,
      "logits/chosen": -0.19037041068077087,
      "logits/rejected": -0.8707745671272278,
      "logps/chosen": -0.9563606381416321,
      "logps/rejected": -3.273907423019409,
      "loss": 0.9816,
      "num_input_tokens_seen": 4790656,
      "odds_ratio_loss": 9.66557502746582,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.09563605487346649,
      "rewards/margins": 0.23175469040870667,
      "rewards/rejected": -0.32739073038101196,
      "sft_loss": 0.0150833148509264,
      "step": 3680
    },
    {
      "epoch": 0.2870478413068845,
      "grad_norm": 2.4525821208953857,
      "learning_rate": 4.0505221644805915e-05,
      "logits/chosen": -0.003124893642961979,
      "logits/rejected": -1.4493802785873413,
      "logps/chosen": -0.9687921404838562,
      "logps/rejected": -4.383993148803711,
      "loss": 1.011,
      "num_input_tokens_seen": 4803440,
      "odds_ratio_loss": 9.884014129638672,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.09687922894954681,
      "rewards/margins": 0.3415201008319855,
      "rewards/rejected": -0.43839937448501587,
      "sft_loss": 0.022603165358304977,
      "step": 3690
    },
    {
      "epoch": 0.28782574873590044,
      "grad_norm": 5.859448432922363,
      "learning_rate": 4.045724889018795e-05,
      "logits/chosen": -0.5072391629219055,
      "logits/rejected": -0.9983925819396973,
      "logps/chosen": -0.9392580986022949,
      "logps/rejected": -1.865422010421753,
      "loss": 0.9823,
      "num_input_tokens_seen": 4817680,
      "odds_ratio_loss": 9.259554862976074,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.09392580389976501,
      "rewards/margins": 0.09261641651391983,
      "rewards/rejected": -0.18654224276542664,
      "sft_loss": 0.05638272687792778,
      "step": 3700
    },
    {
      "epoch": 0.2886036561649164,
      "grad_norm": 2.2526004314422607,
      "learning_rate": 4.040918381731916e-05,
      "logits/chosen": -0.21405701339244843,
      "logits/rejected": -1.030743956565857,
      "logps/chosen": -1.2350473403930664,
      "logps/rejected": -1.3367514610290527,
      "loss": 1.3073,
      "num_input_tokens_seen": 4830976,
      "odds_ratio_loss": 12.2762451171875,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.12350473552942276,
      "rewards/margins": 0.010170431807637215,
      "rewards/rejected": -0.13367517292499542,
      "sft_loss": 0.07963992655277252,
      "step": 3710
    },
    {
      "epoch": 0.2893815635939323,
      "grad_norm": 10.45372486114502,
      "learning_rate": 4.036102671326768e-05,
      "logits/chosen": -0.540435791015625,
      "logits/rejected": -0.8520223498344421,
      "logps/chosen": -0.8511611819267273,
      "logps/rejected": -1.6462376117706299,
      "loss": 0.8937,
      "num_input_tokens_seen": 4839504,
      "odds_ratio_loss": 8.561546325683594,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.08511612564325333,
      "rewards/margins": 0.07950763404369354,
      "rewards/rejected": -0.16462375223636627,
      "sft_loss": 0.03758342191576958,
      "step": 3720
    },
    {
      "epoch": 0.29015947102294826,
      "grad_norm": 28.474945068359375,
      "learning_rate": 4.031277786565128e-05,
      "logits/chosen": 0.027608368545770645,
      "logits/rejected": -1.2352272272109985,
      "logps/chosen": -0.9236048460006714,
      "logps/rejected": -1.9401235580444336,
      "loss": 0.971,
      "num_input_tokens_seen": 4851552,
      "odds_ratio_loss": 9.641461372375488,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.0923604816198349,
      "rewards/margins": 0.1016518846154213,
      "rewards/rejected": -0.1940123587846756,
      "sft_loss": 0.006871473044157028,
      "step": 3730
    },
    {
      "epoch": 0.2909373784519642,
      "grad_norm": 5.415572166442871,
      "learning_rate": 4.0264437562635674e-05,
      "logits/chosen": -0.164834126830101,
      "logits/rejected": -1.1962740421295166,
      "logps/chosen": -0.8205951452255249,
      "logps/rejected": -2.995657444000244,
      "loss": 0.8648,
      "num_input_tokens_seen": 4861456,
      "odds_ratio_loss": 8.583596229553223,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.08205951750278473,
      "rewards/margins": 0.21750621497631073,
      "rewards/rejected": -0.29956573247909546,
      "sft_loss": 0.006485482212156057,
      "step": 3740
    },
    {
      "epoch": 0.29171528588098017,
      "grad_norm": 3.854557514190674,
      "learning_rate": 4.0216006092932815e-05,
      "logits/chosen": -0.38443735241889954,
      "logits/rejected": -0.8124167323112488,
      "logps/chosen": -0.7476634979248047,
      "logps/rejected": -1.6348378658294678,
      "loss": 0.7778,
      "num_input_tokens_seen": 4872080,
      "odds_ratio_loss": 7.580167293548584,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.07476634532213211,
      "rewards/margins": 0.08871743828058243,
      "rewards/rejected": -0.16348378360271454,
      "sft_loss": 0.019787630066275597,
      "step": 3750
    },
    {
      "epoch": 0.2924931933099961,
      "grad_norm": 3.8585402965545654,
      "learning_rate": 4.01674837457991e-05,
      "logits/chosen": -0.4993743896484375,
      "logits/rejected": -1.3653934001922607,
      "logps/chosen": -0.9110993146896362,
      "logps/rejected": -1.640768051147461,
      "loss": 0.9672,
      "num_input_tokens_seen": 4881072,
      "odds_ratio_loss": 9.55540943145752,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.09110994637012482,
      "rewards/margins": 0.07296689599752426,
      "rewards/rejected": -0.1640768200159073,
      "sft_loss": 0.01162740308791399,
      "step": 3760
    },
    {
      "epoch": 0.2932711007390121,
      "grad_norm": 2.339550256729126,
      "learning_rate": 4.011887081103374e-05,
      "logits/chosen": 0.10720139741897583,
      "logits/rejected": -1.6887003183364868,
      "logps/chosen": -0.9534517526626587,
      "logps/rejected": -2.0855872631073,
      "loss": 0.9834,
      "num_input_tokens_seen": 4891296,
      "odds_ratio_loss": 9.790858268737793,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.0953451618552208,
      "rewards/margins": 0.11321356147527695,
      "rewards/rejected": -0.20855875313282013,
      "sft_loss": 0.004304885398596525,
      "step": 3770
    },
    {
      "epoch": 0.294049008168028,
      "grad_norm": 2.4522814750671387,
      "learning_rate": 4.007016757897696e-05,
      "logits/chosen": -0.20804552733898163,
      "logits/rejected": -0.932209849357605,
      "logps/chosen": -0.971185564994812,
      "logps/rejected": -1.918980598449707,
      "loss": 1.0289,
      "num_input_tokens_seen": 4904416,
      "odds_ratio_loss": 10.114686965942383,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.0971185564994812,
      "rewards/margins": 0.09477949887514114,
      "rewards/rejected": -0.19189807772636414,
      "sft_loss": 0.01742658205330372,
      "step": 3780
    },
    {
      "epoch": 0.29482691559704394,
      "grad_norm": 2.2531633377075195,
      "learning_rate": 4.0021374340508266e-05,
      "logits/chosen": -0.1509544998407364,
      "logits/rejected": -1.0268183946609497,
      "logps/chosen": -1.0117778778076172,
      "logps/rejected": -2.7240467071533203,
      "loss": 1.0588,
      "num_input_tokens_seen": 4915760,
      "odds_ratio_loss": 10.479829788208008,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.10117778927087784,
      "rewards/margins": 0.1712268888950348,
      "rewards/rejected": -0.27240467071533203,
      "sft_loss": 0.010849922895431519,
      "step": 3790
    },
    {
      "epoch": 0.2956048230260599,
      "grad_norm": 4.629971981048584,
      "learning_rate": 3.997249138704476e-05,
      "logits/chosen": -0.4955291152000427,
      "logits/rejected": -0.7846877574920654,
      "logps/chosen": -0.9722871780395508,
      "logps/rejected": -2.4349236488342285,
      "loss": 1.0018,
      "num_input_tokens_seen": 4928688,
      "odds_ratio_loss": 9.719146728515625,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.09722872078418732,
      "rewards/margins": 0.14626367390155792,
      "rewards/rejected": -0.24349236488342285,
      "sft_loss": 0.029885223135352135,
      "step": 3800
    },
    {
      "epoch": 0.29638273045507585,
      "grad_norm": 2.5899016857147217,
      "learning_rate": 3.992351901053934e-05,
      "logits/chosen": -0.21175506711006165,
      "logits/rejected": -1.361531138420105,
      "logps/chosen": -0.9431182146072388,
      "logps/rejected": -1.7130239009857178,
      "loss": 0.9906,
      "num_input_tokens_seen": 4942880,
      "odds_ratio_loss": 9.751505851745605,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.09431181848049164,
      "rewards/margins": 0.07699058949947357,
      "rewards/rejected": -0.1713024079799652,
      "sft_loss": 0.015456100925803185,
      "step": 3810
    },
    {
      "epoch": 0.2971606378840918,
      "grad_norm": 2.118806838989258,
      "learning_rate": 3.9874457503479e-05,
      "logits/chosen": 0.025537211447954178,
      "logits/rejected": -1.928781509399414,
      "logps/chosen": -1.0842783451080322,
      "logps/rejected": -2.6051692962646484,
      "loss": 1.1504,
      "num_input_tokens_seen": 4954816,
      "odds_ratio_loss": 11.466569900512695,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.10842782258987427,
      "rewards/margins": 0.15208911895751953,
      "rewards/rejected": -0.2605169415473938,
      "sft_loss": 0.0037107430398464203,
      "step": 3820
    },
    {
      "epoch": 0.29793854531310776,
      "grad_norm": 10.088512420654297,
      "learning_rate": 3.982530715888306e-05,
      "logits/chosen": -0.16875715553760529,
      "logits/rejected": -1.5386735200881958,
      "logps/chosen": -1.2174850702285767,
      "logps/rejected": -2.4324543476104736,
      "loss": 1.2595,
      "num_input_tokens_seen": 4965600,
      "odds_ratio_loss": 12.139209747314453,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.12174850702285767,
      "rewards/margins": 0.12149693816900253,
      "rewards/rejected": -0.2432454377412796,
      "sft_loss": 0.045557282865047455,
      "step": 3830
    },
    {
      "epoch": 0.29871645274212366,
      "grad_norm": 3.1706979274749756,
      "learning_rate": 3.9776068270301406e-05,
      "logits/chosen": -0.016481909900903702,
      "logits/rejected": -0.8835228085517883,
      "logps/chosen": -0.6874436140060425,
      "logps/rejected": -2.4763224124908447,
      "loss": 0.7151,
      "num_input_tokens_seen": 4976512,
      "odds_ratio_loss": 7.089423179626465,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.06874434649944305,
      "rewards/margins": 0.17888788878917694,
      "rewards/rejected": -0.24763223528862,
      "sft_loss": 0.006194235756993294,
      "step": 3840
    },
    {
      "epoch": 0.2994943601711396,
      "grad_norm": 2.881826877593994,
      "learning_rate": 3.972674113181277e-05,
      "logits/chosen": -0.12652811408042908,
      "logits/rejected": -1.2228888273239136,
      "logps/chosen": -0.8278632164001465,
      "logps/rejected": -1.1442887783050537,
      "loss": 0.8862,
      "num_input_tokens_seen": 4992464,
      "odds_ratio_loss": 8.791101455688477,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.08278631418943405,
      "rewards/margins": 0.03164254501461983,
      "rewards/rejected": -0.11442887783050537,
      "sft_loss": 0.007116052322089672,
      "step": 3850
    },
    {
      "epoch": 0.3002722676001556,
      "grad_norm": 1.8632662296295166,
      "learning_rate": 3.9677326038022955e-05,
      "logits/chosen": -0.12285123020410538,
      "logits/rejected": -1.1610023975372314,
      "logps/chosen": -0.922530472278595,
      "logps/rejected": -1.739963173866272,
      "loss": 0.9586,
      "num_input_tokens_seen": 5006896,
      "odds_ratio_loss": 9.477170944213867,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09225304424762726,
      "rewards/margins": 0.0817432776093483,
      "rewards/rejected": -0.17399632930755615,
      "sft_loss": 0.01084755826741457,
      "step": 3860
    },
    {
      "epoch": 0.30105017502917153,
      "grad_norm": 9.572855949401855,
      "learning_rate": 3.962782328406306e-05,
      "logits/chosen": -0.06145212799310684,
      "logits/rejected": -1.5028049945831299,
      "logps/chosen": -0.9595915675163269,
      "logps/rejected": -2.8785479068756104,
      "loss": 0.9902,
      "num_input_tokens_seen": 5016528,
      "odds_ratio_loss": 9.786698341369629,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.09595916420221329,
      "rewards/margins": 0.19189564883708954,
      "rewards/rejected": -0.28785479068756104,
      "sft_loss": 0.011535370722413063,
      "step": 3870
    },
    {
      "epoch": 0.3018280824581875,
      "grad_norm": 3.2435624599456787,
      "learning_rate": 3.9578233165587755e-05,
      "logits/chosen": -0.2727770209312439,
      "logits/rejected": -1.1245701313018799,
      "logps/chosen": -1.0421762466430664,
      "logps/rejected": -1.6434085369110107,
      "loss": 1.0913,
      "num_input_tokens_seen": 5032528,
      "odds_ratio_loss": 10.329371452331543,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.10421763360500336,
      "rewards/margins": 0.060123227536678314,
      "rewards/rejected": -0.16434086859226227,
      "sft_loss": 0.05831795930862427,
      "step": 3880
    },
    {
      "epoch": 0.30260598988720344,
      "grad_norm": 2.970083236694336,
      "learning_rate": 3.9528555978773476e-05,
      "logits/chosen": -0.0969950333237648,
      "logits/rejected": -1.2138078212738037,
      "logps/chosen": -0.8614737391471863,
      "logps/rejected": -2.951115846633911,
      "loss": 0.8901,
      "num_input_tokens_seen": 5043472,
      "odds_ratio_loss": 8.799966812133789,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08614736795425415,
      "rewards/margins": 0.20896419882774353,
      "rewards/rejected": -0.29511159658432007,
      "sft_loss": 0.010101880878210068,
      "step": 3890
    },
    {
      "epoch": 0.3033838973162194,
      "grad_norm": 3.662118434906006,
      "learning_rate": 3.9478792020316685e-05,
      "logits/chosen": -0.12759213149547577,
      "logits/rejected": -1.413557767868042,
      "logps/chosen": -0.9169372320175171,
      "logps/rejected": -1.3593989610671997,
      "loss": 0.9658,
      "num_input_tokens_seen": 5060352,
      "odds_ratio_loss": 9.604103088378906,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.09169372916221619,
      "rewards/margins": 0.044246163219213486,
      "rewards/rejected": -0.13593989610671997,
      "sft_loss": 0.005390730686485767,
      "step": 3900
    },
    {
      "epoch": 0.3041618047452353,
      "grad_norm": 5.8356614112854,
      "learning_rate": 3.9428941587432084e-05,
      "logits/chosen": -0.12662261724472046,
      "logits/rejected": -1.0600976943969727,
      "logps/chosen": -0.8249911069869995,
      "logps/rejected": -2.696192502975464,
      "loss": 0.8534,
      "num_input_tokens_seen": 5069808,
      "odds_ratio_loss": 8.45701789855957,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08249910920858383,
      "rewards/margins": 0.18712015450000763,
      "rewards/rejected": -0.26961925625801086,
      "sft_loss": 0.007720357738435268,
      "step": 3910
    },
    {
      "epoch": 0.30493971217425125,
      "grad_norm": 2.135950803756714,
      "learning_rate": 3.937900497785083e-05,
      "logits/chosen": 0.018372492864727974,
      "logits/rejected": -1.4753367900848389,
      "logps/chosen": -0.8515621423721313,
      "logps/rejected": -2.814161777496338,
      "loss": 0.8767,
      "num_input_tokens_seen": 5084512,
      "odds_ratio_loss": 8.717264175415039,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08515620976686478,
      "rewards/margins": 0.19625994563102722,
      "rewards/rejected": -0.2814161777496338,
      "sft_loss": 0.004960400518029928,
      "step": 3920
    },
    {
      "epoch": 0.3057176196032672,
      "grad_norm": 3.209294557571411,
      "learning_rate": 3.93289824898188e-05,
      "logits/chosen": 0.11812561750411987,
      "logits/rejected": -1.5049880743026733,
      "logps/chosen": -0.8617255091667175,
      "logps/rejected": -2.3110384941101074,
      "loss": 0.9163,
      "num_input_tokens_seen": 5104352,
      "odds_ratio_loss": 9.127669334411621,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.08617255091667175,
      "rewards/margins": 0.14493130147457123,
      "rewards/rejected": -0.23110385239124298,
      "sft_loss": 0.003509619738906622,
      "step": 3930
    },
    {
      "epoch": 0.30649552703228317,
      "grad_norm": 3.6836912631988525,
      "learning_rate": 3.9278874422094735e-05,
      "logits/chosen": -0.19296979904174805,
      "logits/rejected": -1.0401495695114136,
      "logps/chosen": -0.9293330907821655,
      "logps/rejected": -2.3793680667877197,
      "loss": 0.9711,
      "num_input_tokens_seen": 5115648,
      "odds_ratio_loss": 9.585695266723633,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.09293331205844879,
      "rewards/margins": 0.14500349760055542,
      "rewards/rejected": -0.23793677985668182,
      "sft_loss": 0.01250266470015049,
      "step": 3940
    },
    {
      "epoch": 0.3072734344612991,
      "grad_norm": 10.170770645141602,
      "learning_rate": 3.9228681073948545e-05,
      "logits/chosen": -0.37889277935028076,
      "logits/rejected": -1.431139349937439,
      "logps/chosen": -0.9039124250411987,
      "logps/rejected": -2.598503589630127,
      "loss": 0.9282,
      "num_input_tokens_seen": 5138352,
      "odds_ratio_loss": 9.163290023803711,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09039124101400375,
      "rewards/margins": 0.16945910453796387,
      "rewards/rejected": -0.2598503530025482,
      "sft_loss": 0.011916982010006905,
      "step": 3950
    },
    {
      "epoch": 0.3080513418903151,
      "grad_norm": 1.9521749019622803,
      "learning_rate": 3.917840274515945e-05,
      "logits/chosen": -0.17992599308490753,
      "logits/rejected": -1.6150553226470947,
      "logps/chosen": -1.086472749710083,
      "logps/rejected": -1.8796952962875366,
      "loss": 1.1391,
      "num_input_tokens_seen": 5156576,
      "odds_ratio_loss": 11.34900951385498,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.10864728689193726,
      "rewards/margins": 0.07932224869728088,
      "rewards/rejected": -0.18796953558921814,
      "sft_loss": 0.004219838418066502,
      "step": 3960
    },
    {
      "epoch": 0.308829249319331,
      "grad_norm": 3.9300570487976074,
      "learning_rate": 3.9128039736014225e-05,
      "logits/chosen": -0.24541409313678741,
      "logits/rejected": -1.1223763227462769,
      "logps/chosen": -0.98028165102005,
      "logps/rejected": -1.4245949983596802,
      "loss": 1.0275,
      "num_input_tokens_seen": 5167008,
      "odds_ratio_loss": 10.201057434082031,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.09802816808223724,
      "rewards/margins": 0.04443133622407913,
      "rewards/rejected": -0.14245949685573578,
      "sft_loss": 0.007406421005725861,
      "step": 3970
    },
    {
      "epoch": 0.30960715674834693,
      "grad_norm": 1.8432157039642334,
      "learning_rate": 3.907759234730537e-05,
      "logits/chosen": -0.21457970142364502,
      "logits/rejected": -1.5651886463165283,
      "logps/chosen": -0.992315948009491,
      "logps/rejected": -1.2637860774993896,
      "loss": 1.0453,
      "num_input_tokens_seen": 5179360,
      "odds_ratio_loss": 10.372383117675781,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.09923159331083298,
      "rewards/margins": 0.02714700624346733,
      "rewards/rejected": -0.12637859582901,
      "sft_loss": 0.008084376342594624,
      "step": 3980
    },
    {
      "epoch": 0.3103850641773629,
      "grad_norm": 2.259423017501831,
      "learning_rate": 3.9027060880329394e-05,
      "logits/chosen": -0.20980079472064972,
      "logits/rejected": -1.3999924659729004,
      "logps/chosen": -1.0523958206176758,
      "logps/rejected": -1.4480522871017456,
      "loss": 1.1037,
      "num_input_tokens_seen": 5193008,
      "odds_ratio_loss": 10.699328422546387,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.10523958504199982,
      "rewards/margins": 0.039565641433000565,
      "rewards/rejected": -0.14480522274971008,
      "sft_loss": 0.033770475536584854,
      "step": 3990
    },
    {
      "epoch": 0.31116297160637885,
      "grad_norm": 2.654867649078369,
      "learning_rate": 3.897644563688493e-05,
      "logits/chosen": -0.07522716373205185,
      "logits/rejected": -1.35951566696167,
      "logps/chosen": -0.9276152849197388,
      "logps/rejected": -1.5378810167312622,
      "loss": 0.9933,
      "num_input_tokens_seen": 5205440,
      "odds_ratio_loss": 9.885408401489258,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.09276153147220612,
      "rewards/margins": 0.06102656200528145,
      "rewards/rejected": -0.15378808975219727,
      "sft_loss": 0.004794456530362368,
      "step": 4000
    },
    {
      "epoch": 0.3119408790353948,
      "grad_norm": 3.115788698196411,
      "learning_rate": 3.892574691927096e-05,
      "logits/chosen": 0.01536107063293457,
      "logits/rejected": -1.748814582824707,
      "logps/chosen": -1.3016383647918701,
      "logps/rejected": -1.8964124917984009,
      "loss": 1.3451,
      "num_input_tokens_seen": 5218096,
      "odds_ratio_loss": 13.239151000976562,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.13016386330127716,
      "rewards/margins": 0.05947741121053696,
      "rewards/rejected": -0.18964125216007233,
      "sft_loss": 0.021146830171346664,
      "step": 4010
    },
    {
      "epoch": 0.31271878646441076,
      "grad_norm": 3.643676996231079,
      "learning_rate": 3.887496503028501e-05,
      "logits/chosen": -0.5208491683006287,
      "logits/rejected": -1.0714832544326782,
      "logps/chosen": -1.2488865852355957,
      "logps/rejected": -1.78982412815094,
      "loss": 1.3005,
      "num_input_tokens_seen": 5230224,
      "odds_ratio_loss": 12.145222663879395,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.12488865852355957,
      "rewards/margins": 0.054093748331069946,
      "rewards/rejected": -0.17898240685462952,
      "sft_loss": 0.08594609797000885,
      "step": 4020
    },
    {
      "epoch": 0.31349669389342666,
      "grad_norm": 1.7634806632995605,
      "learning_rate": 3.88241002732214e-05,
      "logits/chosen": -0.26208943128585815,
      "logits/rejected": -1.0980716943740845,
      "logps/chosen": -1.0256167650222778,
      "logps/rejected": -1.8397510051727295,
      "loss": 1.0805,
      "num_input_tokens_seen": 5244752,
      "odds_ratio_loss": 10.735319137573242,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.10256167501211166,
      "rewards/margins": 0.08141344785690308,
      "rewards/rejected": -0.18397513031959534,
      "sft_loss": 0.006956270430237055,
      "step": 4030
    },
    {
      "epoch": 0.3142746013224426,
      "grad_norm": 2.526923179626465,
      "learning_rate": 3.87731529518693e-05,
      "logits/chosen": -0.37667712569236755,
      "logits/rejected": -1.2309520244598389,
      "logps/chosen": -1.100372076034546,
      "logps/rejected": -1.6383819580078125,
      "loss": 1.1504,
      "num_input_tokens_seen": 5258256,
      "odds_ratio_loss": 11.36614990234375,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.11003722995519638,
      "rewards/margins": 0.05380095914006233,
      "rewards/rejected": -0.16383817791938782,
      "sft_loss": 0.013796374201774597,
      "step": 4040
    },
    {
      "epoch": 0.31505250875145857,
      "grad_norm": 2.473900079727173,
      "learning_rate": 3.8722123370511045e-05,
      "logits/chosen": -0.17751950025558472,
      "logits/rejected": -1.7557871341705322,
      "logps/chosen": -0.9826386570930481,
      "logps/rejected": -1.5881037712097168,
      "loss": 1.0416,
      "num_input_tokens_seen": 5270784,
      "odds_ratio_loss": 10.001214981079102,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.09826386719942093,
      "rewards/margins": 0.06054651737213135,
      "rewards/rejected": -0.15881037712097168,
      "sft_loss": 0.04149065166711807,
      "step": 4050
    },
    {
      "epoch": 0.3158304161804745,
      "grad_norm": 2.0946033000946045,
      "learning_rate": 3.8671011833920253e-05,
      "logits/chosen": -0.45757579803466797,
      "logits/rejected": -1.309895634651184,
      "logps/chosen": -0.8470166325569153,
      "logps/rejected": -1.6577746868133545,
      "loss": 0.8825,
      "num_input_tokens_seen": 5283760,
      "odds_ratio_loss": 8.305177688598633,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08470165729522705,
      "rewards/margins": 0.08107583224773407,
      "rewards/rejected": -0.16577747464179993,
      "sft_loss": 0.051960818469524384,
      "step": 4060
    },
    {
      "epoch": 0.3166083236094905,
      "grad_norm": 5.183399677276611,
      "learning_rate": 3.8619818647360036e-05,
      "logits/chosen": -0.13480818271636963,
      "logits/rejected": -0.695132851600647,
      "logps/chosen": -0.6039600968360901,
      "logps/rejected": -2.416942596435547,
      "loss": 0.656,
      "num_input_tokens_seen": 5294784,
      "odds_ratio_loss": 6.408038139343262,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.060396015644073486,
      "rewards/margins": 0.18129824101924896,
      "rewards/rejected": -0.24169425666332245,
      "sft_loss": 0.015180411748588085,
      "step": 4070
    },
    {
      "epoch": 0.31738623103850644,
      "grad_norm": 3.9851224422454834,
      "learning_rate": 3.856854411658112e-05,
      "logits/chosen": -0.4109697937965393,
      "logits/rejected": -0.8812214136123657,
      "logps/chosen": -0.8279398083686829,
      "logps/rejected": -1.6647628545761108,
      "loss": 0.8613,
      "num_input_tokens_seen": 5305744,
      "odds_ratio_loss": 8.519083976745605,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08279397338628769,
      "rewards/margins": 0.08368229866027832,
      "rewards/rejected": -0.1664762794971466,
      "sft_loss": 0.00942680612206459,
      "step": 4080
    },
    {
      "epoch": 0.31816413846752234,
      "grad_norm": 1.8268203735351562,
      "learning_rate": 3.85171885478201e-05,
      "logits/chosen": -0.12919794023036957,
      "logits/rejected": -1.543035626411438,
      "logps/chosen": -0.8919490575790405,
      "logps/rejected": -2.169306993484497,
      "loss": 0.9306,
      "num_input_tokens_seen": 5319440,
      "odds_ratio_loss": 8.916237831115723,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.08919490873813629,
      "rewards/margins": 0.12773580849170685,
      "rewards/rejected": -0.21693070232868195,
      "sft_loss": 0.03893839567899704,
      "step": 4090
    },
    {
      "epoch": 0.3189420458965383,
      "grad_norm": 55.99069595336914,
      "learning_rate": 3.846575224779754e-05,
      "logits/chosen": -0.34843164682388306,
      "logits/rejected": -0.8747657537460327,
      "logps/chosen": -1.342206358909607,
      "logps/rejected": -2.3212666511535645,
      "loss": 1.4213,
      "num_input_tokens_seen": 5332400,
      "odds_ratio_loss": 13.317713737487793,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.13422062993049622,
      "rewards/margins": 0.09790606051683426,
      "rewards/rejected": -0.23212668299674988,
      "sft_loss": 0.08950953185558319,
      "step": 4100
    },
    {
      "epoch": 0.31971995332555425,
      "grad_norm": 3.501796245574951,
      "learning_rate": 3.841423552371619e-05,
      "logits/chosen": -0.2077409327030182,
      "logits/rejected": -1.3427925109863281,
      "logps/chosen": -0.8884612917900085,
      "logps/rejected": -1.882891058921814,
      "loss": 0.9302,
      "num_input_tokens_seen": 5345552,
      "odds_ratio_loss": 9.219976425170898,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.0888461321592331,
      "rewards/margins": 0.0994429737329483,
      "rewards/rejected": -0.1882891207933426,
      "sft_loss": 0.0081623625010252,
      "step": 4110
    },
    {
      "epoch": 0.3204978607545702,
      "grad_norm": 15.101680755615234,
      "learning_rate": 3.836263868325911e-05,
      "logits/chosen": -0.25319308042526245,
      "logits/rejected": -0.8677579164505005,
      "logps/chosen": -0.7892023324966431,
      "logps/rejected": -3.2379326820373535,
      "loss": 0.8212,
      "num_input_tokens_seen": 5359504,
      "odds_ratio_loss": 8.134763717651367,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.07892023026943207,
      "rewards/margins": 0.244873046875,
      "rewards/rejected": -0.32379329204559326,
      "sft_loss": 0.007730512414127588,
      "step": 4120
    },
    {
      "epoch": 0.32127576818358616,
      "grad_norm": 2.807835817337036,
      "learning_rate": 3.831096203458788e-05,
      "logits/chosen": -0.41573959589004517,
      "logits/rejected": -1.315179705619812,
      "logps/chosen": -0.9759588241577148,
      "logps/rejected": -1.5809203386306763,
      "loss": 1.0161,
      "num_input_tokens_seen": 5373568,
      "odds_ratio_loss": 9.983091354370117,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.09759588539600372,
      "rewards/margins": 0.06049615144729614,
      "rewards/rejected": -0.15809203684329987,
      "sft_loss": 0.01783646084368229,
      "step": 4130
    },
    {
      "epoch": 0.3220536756126021,
      "grad_norm": 3.5603480339050293,
      "learning_rate": 3.825920588634073e-05,
      "logits/chosen": -0.29883021116256714,
      "logits/rejected": -0.9933277368545532,
      "logps/chosen": -0.990680992603302,
      "logps/rejected": -2.809013843536377,
      "loss": 1.0337,
      "num_input_tokens_seen": 5383136,
      "odds_ratio_loss": 10.23255729675293,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.09906809777021408,
      "rewards/margins": 0.18183325231075287,
      "rewards/rejected": -0.28090134263038635,
      "sft_loss": 0.01048289518803358,
      "step": 4140
    },
    {
      "epoch": 0.322831583041618,
      "grad_norm": 3.0425772666931152,
      "learning_rate": 3.820737054763068e-05,
      "logits/chosen": -0.035363975912332535,
      "logits/rejected": -1.1433265209197998,
      "logps/chosen": -0.9977181553840637,
      "logps/rejected": -2.1278040409088135,
      "loss": 1.0341,
      "num_input_tokens_seen": 5397440,
      "odds_ratio_loss": 10.24191665649414,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.09977181255817413,
      "rewards/margins": 0.11300859600305557,
      "rewards/rejected": -0.2127804309129715,
      "sft_loss": 0.009899460710585117,
      "step": 4150
    },
    {
      "epoch": 0.323609490470634,
      "grad_norm": 3.0186002254486084,
      "learning_rate": 3.8155456328043735e-05,
      "logits/chosen": -0.27061909437179565,
      "logits/rejected": -1.1650453805923462,
      "logps/chosen": -0.9972823262214661,
      "logps/rejected": -2.490751266479492,
      "loss": 1.0286,
      "num_input_tokens_seen": 5408160,
      "odds_ratio_loss": 10.072731018066406,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09972824156284332,
      "rewards/margins": 0.14934687316417694,
      "rewards/rejected": -0.24907509982585907,
      "sft_loss": 0.02131205052137375,
      "step": 4160
    },
    {
      "epoch": 0.32438739789964993,
      "grad_norm": 1.8467899560928345,
      "learning_rate": 3.8103463537636996e-05,
      "logits/chosen": -0.05058874562382698,
      "logits/rejected": -1.356675624847412,
      "logps/chosen": -0.7572735548019409,
      "logps/rejected": -1.5102696418762207,
      "loss": 0.8184,
      "num_input_tokens_seen": 5418992,
      "odds_ratio_loss": 7.95639181137085,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.07572735100984573,
      "rewards/margins": 0.07529960572719574,
      "rewards/rejected": -0.15102696418762207,
      "sft_loss": 0.02272329106926918,
      "step": 4170
    },
    {
      "epoch": 0.3251653053286659,
      "grad_norm": 2.185946464538574,
      "learning_rate": 3.8051392486936844e-05,
      "logits/chosen": -0.08555354177951813,
      "logits/rejected": -0.8780487179756165,
      "logps/chosen": -1.0520031452178955,
      "logps/rejected": -2.304363489151001,
      "loss": 1.1085,
      "num_input_tokens_seen": 5435808,
      "odds_ratio_loss": 11.030229568481445,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.10520030558109283,
      "rewards/margins": 0.12523601949214935,
      "rewards/rejected": -0.2304363250732422,
      "sft_loss": 0.005523464176803827,
      "step": 4180
    },
    {
      "epoch": 0.32594321275768184,
      "grad_norm": 1.7623631954193115,
      "learning_rate": 3.799924348693706e-05,
      "logits/chosen": -0.41470474004745483,
      "logits/rejected": -0.833422839641571,
      "logps/chosen": -0.9697521328926086,
      "logps/rejected": -2.2706055641174316,
      "loss": 1.0086,
      "num_input_tokens_seen": 5449056,
      "odds_ratio_loss": 9.848416328430176,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.09697521477937698,
      "rewards/margins": 0.1300853192806244,
      "rewards/rejected": -0.22706055641174316,
      "sft_loss": 0.023781007155776024,
      "step": 4190
    },
    {
      "epoch": 0.3267211201866978,
      "grad_norm": 15.622406005859375,
      "learning_rate": 3.794701684909698e-05,
      "logits/chosen": -0.2555413544178009,
      "logits/rejected": -1.5905277729034424,
      "logps/chosen": -1.1539592742919922,
      "logps/rejected": -3.2055859565734863,
      "loss": 1.2011,
      "num_input_tokens_seen": 5460400,
      "odds_ratio_loss": 11.80296802520752,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.1153959184885025,
      "rewards/margins": 0.2051626741886139,
      "rewards/rejected": -0.3205585777759552,
      "sft_loss": 0.02078629657626152,
      "step": 4200
    },
    {
      "epoch": 0.32749902761571376,
      "grad_norm": 5.139983654022217,
      "learning_rate": 3.7894712885339634e-05,
      "logits/chosen": -0.08099023252725601,
      "logits/rejected": -1.3205289840698242,
      "logps/chosen": -0.9128491282463074,
      "logps/rejected": -4.242014408111572,
      "loss": 0.9318,
      "num_input_tokens_seen": 5478928,
      "odds_ratio_loss": 9.188984870910645,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.09128491580486298,
      "rewards/margins": 0.3329164683818817,
      "rewards/rejected": -0.4242013990879059,
      "sft_loss": 0.012938452884554863,
      "step": 4210
    },
    {
      "epoch": 0.32827693504472966,
      "grad_norm": 4.9469099044799805,
      "learning_rate": 3.784233190804988e-05,
      "logits/chosen": -0.1827361285686493,
      "logits/rejected": -1.0644224882125854,
      "logps/chosen": -0.8118671178817749,
      "logps/rejected": -1.7130250930786133,
      "loss": 0.8587,
      "num_input_tokens_seen": 5497600,
      "odds_ratio_loss": 8.533797264099121,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.08118671178817749,
      "rewards/margins": 0.09011580049991608,
      "rewards/rejected": -0.17130251228809357,
      "sft_loss": 0.005304677411913872,
      "step": 4220
    },
    {
      "epoch": 0.3290548424737456,
      "grad_norm": 3.007436752319336,
      "learning_rate": 3.778987423007252e-05,
      "logits/chosen": -0.6660521626472473,
      "logits/rejected": -0.7182964086532593,
      "logps/chosen": -0.8447635769844055,
      "logps/rejected": -1.9354795217514038,
      "loss": 0.8841,
      "num_input_tokens_seen": 5510512,
      "odds_ratio_loss": 8.513199806213379,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.08447636663913727,
      "rewards/margins": 0.10907159745693207,
      "rewards/rejected": -0.19354796409606934,
      "sft_loss": 0.032828912138938904,
      "step": 4230
    },
    {
      "epoch": 0.32983274990276157,
      "grad_norm": 3.3129470348358154,
      "learning_rate": 3.7737340164710474e-05,
      "logits/chosen": -0.3203393816947937,
      "logits/rejected": -0.8482300639152527,
      "logps/chosen": -0.9250307083129883,
      "logps/rejected": -2.723344326019287,
      "loss": 0.9538,
      "num_input_tokens_seen": 5522992,
      "odds_ratio_loss": 9.43076229095459,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.09250307828187943,
      "rewards/margins": 0.1798313558101654,
      "rewards/rejected": -0.2723344564437866,
      "sft_loss": 0.010698824189603329,
      "step": 4240
    },
    {
      "epoch": 0.3306106573317775,
      "grad_norm": 9.28976821899414,
      "learning_rate": 3.7684730025722884e-05,
      "logits/chosen": -0.1900298297405243,
      "logits/rejected": -0.8809641003608704,
      "logps/chosen": -0.9689387083053589,
      "logps/rejected": -1.8893556594848633,
      "loss": 1.0076,
      "num_input_tokens_seen": 5539344,
      "odds_ratio_loss": 10.023103713989258,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.09689386934041977,
      "rewards/margins": 0.09204168617725372,
      "rewards/rejected": -0.1889355629682541,
      "sft_loss": 0.005292147863656282,
      "step": 4250
    },
    {
      "epoch": 0.3313885647607935,
      "grad_norm": 1.712719440460205,
      "learning_rate": 3.7632044127323216e-05,
      "logits/chosen": 0.1145520806312561,
      "logits/rejected": -1.800127625465393,
      "logps/chosen": -0.9322274923324585,
      "logps/rejected": -1.7594856023788452,
      "loss": 0.9729,
      "num_input_tokens_seen": 5555152,
      "odds_ratio_loss": 9.706247329711914,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.09322275966405869,
      "rewards/margins": 0.08272580802440643,
      "rewards/rejected": -0.17594856023788452,
      "sft_loss": 0.0022511687129735947,
      "step": 4260
    },
    {
      "epoch": 0.33216647218980944,
      "grad_norm": 1.8186136484146118,
      "learning_rate": 3.7579282784177436e-05,
      "logits/chosen": -0.21599099040031433,
      "logits/rejected": -1.5185308456420898,
      "logps/chosen": -0.8938544988632202,
      "logps/rejected": -1.7810747623443604,
      "loss": 0.9353,
      "num_input_tokens_seen": 5566192,
      "odds_ratio_loss": 9.18561840057373,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.08938544988632202,
      "rewards/margins": 0.08872202783823013,
      "rewards/rejected": -0.17810747027397156,
      "sft_loss": 0.016736816614866257,
      "step": 4270
    },
    {
      "epoch": 0.33294437961882534,
      "grad_norm": 2.4397411346435547,
      "learning_rate": 3.7526446311402095e-05,
      "logits/chosen": -0.014413388445973396,
      "logits/rejected": -1.4831351041793823,
      "logps/chosen": -0.7523231506347656,
      "logps/rejected": -2.23205828666687,
      "loss": 0.7786,
      "num_input_tokens_seen": 5580640,
      "odds_ratio_loss": 7.747849464416504,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.07523230463266373,
      "rewards/margins": 0.14797353744506836,
      "rewards/rejected": -0.2232058346271515,
      "sft_loss": 0.0037966384552419186,
      "step": 4280
    },
    {
      "epoch": 0.3337222870478413,
      "grad_norm": 1.911331295967102,
      "learning_rate": 3.7473535024562446e-05,
      "logits/chosen": -0.1065264493227005,
      "logits/rejected": -1.228137731552124,
      "logps/chosen": -0.8722216486930847,
      "logps/rejected": -1.2977032661437988,
      "loss": 0.9205,
      "num_input_tokens_seen": 5591376,
      "odds_ratio_loss": 9.155069351196289,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.08722217381000519,
      "rewards/margins": 0.042548153549432755,
      "rewards/rejected": -0.12977032363414764,
      "sft_loss": 0.005033262073993683,
      "step": 4290
    },
    {
      "epoch": 0.33450019447685725,
      "grad_norm": 1.7152159214019775,
      "learning_rate": 3.742054923967057e-05,
      "logits/chosen": 0.07246137410402298,
      "logits/rejected": -1.9047914743423462,
      "logps/chosen": -0.8156659007072449,
      "logps/rejected": -3.5807089805603027,
      "loss": 0.8458,
      "num_input_tokens_seen": 5606752,
      "odds_ratio_loss": 8.434159278869629,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.08156659454107285,
      "rewards/margins": 0.2765043377876282,
      "rewards/rejected": -0.35807090997695923,
      "sft_loss": 0.002411172492429614,
      "step": 4300
    },
    {
      "epoch": 0.3352781019058732,
      "grad_norm": 1.6043893098831177,
      "learning_rate": 3.7367489273183495e-05,
      "logits/chosen": -0.13491670787334442,
      "logits/rejected": -1.4279476404190063,
      "logps/chosen": -0.675383448600769,
      "logps/rejected": -2.471374034881592,
      "loss": 0.7059,
      "num_input_tokens_seen": 5618800,
      "odds_ratio_loss": 6.939095497131348,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.06753835082054138,
      "rewards/margins": 0.1795990765094757,
      "rewards/rejected": -0.2471374273300171,
      "sft_loss": 0.01202771533280611,
      "step": 4310
    },
    {
      "epoch": 0.33605600933488916,
      "grad_norm": 6.193589687347412,
      "learning_rate": 3.7314355442001294e-05,
      "logits/chosen": -0.4281120300292969,
      "logits/rejected": -0.8102872967720032,
      "logps/chosen": -1.042340636253357,
      "logps/rejected": -2.1481173038482666,
      "loss": 1.079,
      "num_input_tokens_seen": 5630448,
      "odds_ratio_loss": 10.683089256286621,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.10423406213521957,
      "rewards/margins": 0.11057764291763306,
      "rewards/rejected": -0.21481171250343323,
      "sft_loss": 0.010718410834670067,
      "step": 4320
    },
    {
      "epoch": 0.3368339167639051,
      "grad_norm": 3.1446692943573,
      "learning_rate": 3.726114806346518e-05,
      "logits/chosen": -0.5779768228530884,
      "logits/rejected": -1.1927872896194458,
      "logps/chosen": -0.9345456957817078,
      "logps/rejected": -2.156569004058838,
      "loss": 0.9621,
      "num_input_tokens_seen": 5643584,
      "odds_ratio_loss": 9.511880874633789,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.09345457702875137,
      "rewards/margins": 0.12220235913991928,
      "rewards/rejected": -0.21565695106983185,
      "sft_loss": 0.010866018943488598,
      "step": 4330
    },
    {
      "epoch": 0.337611824192921,
      "grad_norm": 2.2310080528259277,
      "learning_rate": 3.720786745535568e-05,
      "logits/chosen": 0.017054501920938492,
      "logits/rejected": -1.5218160152435303,
      "logps/chosen": -0.9744178652763367,
      "logps/rejected": -2.552828311920166,
      "loss": 1.0142,
      "num_input_tokens_seen": 5656624,
      "odds_ratio_loss": 10.09164810180664,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.09744178503751755,
      "rewards/margins": 0.1578410416841507,
      "rewards/rejected": -0.25528278946876526,
      "sft_loss": 0.005024941172450781,
      "step": 4340
    },
    {
      "epoch": 0.338389731621937,
      "grad_norm": 2.9606218338012695,
      "learning_rate": 3.7154513935890614e-05,
      "logits/chosen": -0.18985190987586975,
      "logits/rejected": -1.7492328882217407,
      "logps/chosen": -0.8841496706008911,
      "logps/rejected": -2.6282975673675537,
      "loss": 0.9231,
      "num_input_tokens_seen": 5666960,
      "odds_ratio_loss": 9.061014175415039,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.08841496706008911,
      "rewards/margins": 0.17441478371620178,
      "rewards/rejected": -0.2628297507762909,
      "sft_loss": 0.0170004703104496,
      "step": 4350
    },
    {
      "epoch": 0.33916763905095293,
      "grad_norm": 12.98637866973877,
      "learning_rate": 3.710108782372331e-05,
      "logits/chosen": -0.5041710138320923,
      "logits/rejected": -0.9508134722709656,
      "logps/chosen": -1.128469705581665,
      "logps/rejected": -1.6678993701934814,
      "loss": 1.18,
      "num_input_tokens_seen": 5679232,
      "odds_ratio_loss": 11.528806686401367,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.11284694820642471,
      "rewards/margins": 0.053942978382110596,
      "rewards/rejected": -0.1667899340391159,
      "sft_loss": 0.027155181393027306,
      "step": 4360
    },
    {
      "epoch": 0.3399455464799689,
      "grad_norm": 31.380678176879883,
      "learning_rate": 3.7047589437940636e-05,
      "logits/chosen": -0.7777343988418579,
      "logits/rejected": -1.0380951166152954,
      "logps/chosen": -0.94134920835495,
      "logps/rejected": -2.7589385509490967,
      "loss": 0.9895,
      "num_input_tokens_seen": 5690656,
      "odds_ratio_loss": 9.376608848571777,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.09413491189479828,
      "rewards/margins": 0.18175895512104034,
      "rewards/rejected": -0.27589383721351624,
      "sft_loss": 0.05186940357089043,
      "step": 4370
    },
    {
      "epoch": 0.34072345390898484,
      "grad_norm": 5.7405595779418945,
      "learning_rate": 3.6994019098061123e-05,
      "logits/chosen": -0.3143104910850525,
      "logits/rejected": -0.915662407875061,
      "logps/chosen": -0.9450228810310364,
      "logps/rejected": -1.476238489151001,
      "loss": 0.9972,
      "num_input_tokens_seen": 5702304,
      "odds_ratio_loss": 9.882279396057129,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.0945022851228714,
      "rewards/margins": 0.05312156677246094,
      "rewards/rejected": -0.14762386679649353,
      "sft_loss": 0.008927664719522,
      "step": 4380
    },
    {
      "epoch": 0.3415013613380008,
      "grad_norm": 2.7874977588653564,
      "learning_rate": 3.6940377124033033e-05,
      "logits/chosen": -0.540741503238678,
      "logits/rejected": -1.1405988931655884,
      "logps/chosen": -0.9208908081054688,
      "logps/rejected": -2.2547502517700195,
      "loss": 0.9975,
      "num_input_tokens_seen": 5713360,
      "odds_ratio_loss": 9.85312271118164,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.09208908677101135,
      "rewards/margins": 0.13338592648506165,
      "rewards/rejected": -0.2254750281572342,
      "sft_loss": 0.012210485525429249,
      "step": 4390
    },
    {
      "epoch": 0.3422792687670167,
      "grad_norm": 1.7215228080749512,
      "learning_rate": 3.688666383623249e-05,
      "logits/chosen": -0.3327955901622772,
      "logits/rejected": -1.1232503652572632,
      "logps/chosen": -1.0700501203536987,
      "logps/rejected": -1.328126311302185,
      "loss": 1.1692,
      "num_input_tokens_seen": 5728016,
      "odds_ratio_loss": 11.548150062561035,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": -0.10700502246618271,
      "rewards/margins": 0.025807607918977737,
      "rewards/rejected": -0.13281263411045074,
      "sft_loss": 0.014384521171450615,
      "step": 4400
    },
    {
      "epoch": 0.34305717619603265,
      "grad_norm": 2.786660671234131,
      "learning_rate": 3.6832879555461507e-05,
      "logits/chosen": -0.06084407493472099,
      "logits/rejected": -1.2869935035705566,
      "logps/chosen": -0.970739483833313,
      "logps/rejected": -2.362373113632202,
      "loss": 0.9998,
      "num_input_tokens_seen": 5742624,
      "odds_ratio_loss": 9.313217163085938,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09707394987344742,
      "rewards/margins": 0.13916337490081787,
      "rewards/rejected": -0.2362373322248459,
      "sft_loss": 0.06852734088897705,
      "step": 4410
    },
    {
      "epoch": 0.3438350836250486,
      "grad_norm": 11.903273582458496,
      "learning_rate": 3.677902460294612e-05,
      "logits/chosen": 0.05326567962765694,
      "logits/rejected": -1.6322355270385742,
      "logps/chosen": -0.9381688833236694,
      "logps/rejected": -1.8394731283187866,
      "loss": 0.9744,
      "num_input_tokens_seen": 5756800,
      "odds_ratio_loss": 9.685869216918945,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.09381689876317978,
      "rewards/margins": 0.09013044089078903,
      "rewards/rejected": -0.18394732475280762,
      "sft_loss": 0.005861429963260889,
      "step": 4420
    },
    {
      "epoch": 0.34461299105406457,
      "grad_norm": 4.127033710479736,
      "learning_rate": 3.6725099300334446e-05,
      "logits/chosen": -0.16183969378471375,
      "logits/rejected": -1.6115763187408447,
      "logps/chosen": -0.887466311454773,
      "logps/rejected": -2.323113441467285,
      "loss": 0.9084,
      "num_input_tokens_seen": 5769152,
      "odds_ratio_loss": 9.000121116638184,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08874664455652237,
      "rewards/margins": 0.14356471598148346,
      "rewards/rejected": -0.23231136798858643,
      "sft_loss": 0.008424920961260796,
      "step": 4430
    },
    {
      "epoch": 0.3453908984830805,
      "grad_norm": 2.3415582180023193,
      "learning_rate": 3.667110396969477e-05,
      "logits/chosen": -0.02460600808262825,
      "logits/rejected": -1.9235773086547852,
      "logps/chosen": -0.9934428930282593,
      "logps/rejected": -2.2113564014434814,
      "loss": 1.0324,
      "num_input_tokens_seen": 5786576,
      "odds_ratio_loss": 10.282426834106445,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09934429824352264,
      "rewards/margins": 0.12179134041070938,
      "rewards/rejected": -0.22113564610481262,
      "sft_loss": 0.004182109609246254,
      "step": 4440
    },
    {
      "epoch": 0.3461688059120965,
      "grad_norm": 2.3995091915130615,
      "learning_rate": 3.661703893351361e-05,
      "logits/chosen": 0.07914382219314575,
      "logits/rejected": -1.6564337015151978,
      "logps/chosen": -0.9553931951522827,
      "logps/rejected": -1.6407089233398438,
      "loss": 0.9945,
      "num_input_tokens_seen": 5799632,
      "odds_ratio_loss": 9.90433406829834,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.09553932398557663,
      "rewards/margins": 0.0685315653681755,
      "rewards/rejected": -0.16407087445259094,
      "sft_loss": 0.004019489046186209,
      "step": 4450
    },
    {
      "epoch": 0.34694671334111243,
      "grad_norm": 4.87782621383667,
      "learning_rate": 3.656290451469382e-05,
      "logits/chosen": -0.36600252985954285,
      "logits/rejected": -0.7956069707870483,
      "logps/chosen": -0.8138600587844849,
      "logps/rejected": -2.0378212928771973,
      "loss": 0.8456,
      "num_input_tokens_seen": 5811184,
      "odds_ratio_loss": 8.397814750671387,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08138599991798401,
      "rewards/margins": 0.12239613384008408,
      "rewards/rejected": -0.2037821263074875,
      "sft_loss": 0.005857658572494984,
      "step": 4460
    },
    {
      "epoch": 0.34772462077012833,
      "grad_norm": 4.152865886688232,
      "learning_rate": 3.6508701036552624e-05,
      "logits/chosen": 0.022718677297234535,
      "logits/rejected": -1.404191017150879,
      "logps/chosen": -0.9380876421928406,
      "logps/rejected": -1.9611479043960571,
      "loss": 0.9886,
      "num_input_tokens_seen": 5823296,
      "odds_ratio_loss": 9.800165176391602,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.09380875527858734,
      "rewards/margins": 0.10230604559183121,
      "rewards/rejected": -0.19611480832099915,
      "sft_loss": 0.008533725515007973,
      "step": 4470
    },
    {
      "epoch": 0.3485025281991443,
      "grad_norm": 1.5638813972473145,
      "learning_rate": 3.6454428822819695e-05,
      "logits/chosen": -0.2070436030626297,
      "logits/rejected": -1.384708046913147,
      "logps/chosen": -1.103393793106079,
      "logps/rejected": -2.512498140335083,
      "loss": 1.1549,
      "num_input_tokens_seen": 5836768,
      "odds_ratio_loss": 11.028097152709961,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.11033938825130463,
      "rewards/margins": 0.14091041684150696,
      "rewards/rejected": -0.25124984979629517,
      "sft_loss": 0.05206207558512688,
      "step": 4480
    },
    {
      "epoch": 0.34928043562816025,
      "grad_norm": 5.045617580413818,
      "learning_rate": 3.640008819763525e-05,
      "logits/chosen": -0.35230201482772827,
      "logits/rejected": -1.3696832656860352,
      "logps/chosen": -1.197439432144165,
      "logps/rejected": -2.184636116027832,
      "loss": 1.2687,
      "num_input_tokens_seen": 5847728,
      "odds_ratio_loss": 11.410408973693848,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.11974392831325531,
      "rewards/margins": 0.09871968626976013,
      "rewards/rejected": -0.21846361458301544,
      "sft_loss": 0.12763787806034088,
      "step": 4490
    },
    {
      "epoch": 0.3500583430571762,
      "grad_norm": 1.8348246812820435,
      "learning_rate": 3.634567948554807e-05,
      "logits/chosen": -0.05636955052614212,
      "logits/rejected": -1.405261516571045,
      "logps/chosen": -1.0076950788497925,
      "logps/rejected": -1.3658123016357422,
      "loss": 1.0606,
      "num_input_tokens_seen": 5861776,
      "odds_ratio_loss": 10.531423568725586,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.1007695198059082,
      "rewards/margins": 0.03581172972917557,
      "rewards/rejected": -0.13658125698566437,
      "sft_loss": 0.007421608082950115,
      "step": 4500
    },
    {
      "epoch": 0.35083625048619216,
      "grad_norm": 1.8715535402297974,
      "learning_rate": 3.62912030115136e-05,
      "logits/chosen": -0.3818695545196533,
      "logits/rejected": -1.6756805181503296,
      "logps/chosen": -1.0263088941574097,
      "logps/rejected": -2.775115966796875,
      "loss": 1.0608,
      "num_input_tokens_seen": 5872640,
      "odds_ratio_loss": 10.09156608581543,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.10263089835643768,
      "rewards/margins": 0.174880713224411,
      "rewards/rejected": -0.2775115966796875,
      "sft_loss": 0.05163625627756119,
      "step": 4510
    },
    {
      "epoch": 0.3516141579152081,
      "grad_norm": 1.4873770475387573,
      "learning_rate": 3.6236659100892e-05,
      "logits/chosen": 0.11474569141864777,
      "logits/rejected": -1.988440752029419,
      "logps/chosen": -0.980634868144989,
      "logps/rejected": -1.8994451761245728,
      "loss": 1.0176,
      "num_input_tokens_seen": 5892208,
      "odds_ratio_loss": 10.155592918395996,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09806347638368607,
      "rewards/margins": 0.09188102185726166,
      "rewards/rejected": -0.18994450569152832,
      "sft_loss": 0.0020749946124851704,
      "step": 4520
    },
    {
      "epoch": 0.352392065344224,
      "grad_norm": 3.3406612873077393,
      "learning_rate": 3.618204807944617e-05,
      "logits/chosen": 0.010874845087528229,
      "logits/rejected": -1.3346697092056274,
      "logps/chosen": -0.8537413477897644,
      "logps/rejected": -3.3422837257385254,
      "loss": 0.8963,
      "num_input_tokens_seen": 5903216,
      "odds_ratio_loss": 8.909745216369629,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.0853741317987442,
      "rewards/margins": 0.24885420501232147,
      "rewards/rejected": -0.3342283368110657,
      "sft_loss": 0.005339351482689381,
      "step": 4530
    },
    {
      "epoch": 0.35316997277323997,
      "grad_norm": 1.2649593353271484,
      "learning_rate": 3.612737027333984e-05,
      "logits/chosen": -0.04491196200251579,
      "logits/rejected": -1.3434958457946777,
      "logps/chosen": -0.8880999684333801,
      "logps/rejected": -1.592533826828003,
      "loss": 0.921,
      "num_input_tokens_seen": 5918576,
      "odds_ratio_loss": 9.174939155578613,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08880998939275742,
      "rewards/margins": 0.07044339925050735,
      "rewards/rejected": -0.15925338864326477,
      "sft_loss": 0.0034705442376434803,
      "step": 4540
    },
    {
      "epoch": 0.3539478802022559,
      "grad_norm": 2.581486225128174,
      "learning_rate": 3.607262600913563e-05,
      "logits/chosen": -0.402034193277359,
      "logits/rejected": -0.3176518380641937,
      "logps/chosen": -0.7697569727897644,
      "logps/rejected": -1.591722011566162,
      "loss": 0.8021,
      "num_input_tokens_seen": 5931248,
      "odds_ratio_loss": 7.6087541580200195,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.07697569578886032,
      "rewards/margins": 0.08219649642705917,
      "rewards/rejected": -0.1591722071170807,
      "sft_loss": 0.041197795420885086,
      "step": 4550
    },
    {
      "epoch": 0.3547257876312719,
      "grad_norm": 3.187448263168335,
      "learning_rate": 3.601781561379304e-05,
      "logits/chosen": -0.0909782350063324,
      "logits/rejected": -1.6023838520050049,
      "logps/chosen": -1.0848517417907715,
      "logps/rejected": -1.7551847696304321,
      "loss": 1.1528,
      "num_input_tokens_seen": 5945392,
      "odds_ratio_loss": 11.288145065307617,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.10848517715930939,
      "rewards/margins": 0.0670332983136177,
      "rewards/rejected": -0.1755184680223465,
      "sft_loss": 0.02397894114255905,
      "step": 4560
    },
    {
      "epoch": 0.35550369506028784,
      "grad_norm": 6.8400373458862305,
      "learning_rate": 3.596293941466658e-05,
      "logits/chosen": -0.3084474205970764,
      "logits/rejected": -1.0309423208236694,
      "logps/chosen": -0.8194421529769897,
      "logps/rejected": -2.439091205596924,
      "loss": 0.8622,
      "num_input_tokens_seen": 5956272,
      "odds_ratio_loss": 8.532068252563477,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.08194421231746674,
      "rewards/margins": 0.16196492314338684,
      "rewards/rejected": -0.24390912055969238,
      "sft_loss": 0.008951345458626747,
      "step": 4570
    },
    {
      "epoch": 0.3562816024893038,
      "grad_norm": 5.4224772453308105,
      "learning_rate": 3.5907997739503735e-05,
      "logits/chosen": 0.02636636421084404,
      "logits/rejected": -1.5030062198638916,
      "logps/chosen": -0.7921457290649414,
      "logps/rejected": -4.366400718688965,
      "loss": 0.811,
      "num_input_tokens_seen": 5968640,
      "odds_ratio_loss": 8.064387321472168,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.07921457290649414,
      "rewards/margins": 0.3574255406856537,
      "rewards/rejected": -0.4366401135921478,
      "sft_loss": 0.0045320941135287285,
      "step": 4580
    },
    {
      "epoch": 0.3570595099183197,
      "grad_norm": 30.135780334472656,
      "learning_rate": 3.585299091644307e-05,
      "logits/chosen": -0.31513237953186035,
      "logits/rejected": -1.236054539680481,
      "logps/chosen": -0.8894597887992859,
      "logps/rejected": -4.156570911407471,
      "loss": 0.9329,
      "num_input_tokens_seen": 5977824,
      "odds_ratio_loss": 8.80221176147461,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.08894598484039307,
      "rewards/margins": 0.32671108841896057,
      "rewards/rejected": -0.41565704345703125,
      "sft_loss": 0.05266100913286209,
      "step": 4590
    },
    {
      "epoch": 0.35783741734733565,
      "grad_norm": 3.0575132369995117,
      "learning_rate": 3.5797919274012235e-05,
      "logits/chosen": -0.0275888592004776,
      "logits/rejected": -1.386785864830017,
      "logps/chosen": -1.0150604248046875,
      "logps/rejected": -2.2971482276916504,
      "loss": 1.056,
      "num_input_tokens_seen": 5985584,
      "odds_ratio_loss": 10.500213623046875,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.1015060544013977,
      "rewards/margins": 0.12820878624916077,
      "rewards/rejected": -0.22971484065055847,
      "sft_loss": 0.005982295144349337,
      "step": 4600
    },
    {
      "epoch": 0.3586153247763516,
      "grad_norm": 6.465326309204102,
      "learning_rate": 3.5742783141126016e-05,
      "logits/chosen": -0.48850011825561523,
      "logits/rejected": -0.8825419545173645,
      "logps/chosen": -0.9011627435684204,
      "logps/rejected": -1.7979437112808228,
      "loss": 0.9462,
      "num_input_tokens_seen": 5999360,
      "odds_ratio_loss": 9.355947494506836,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.09011626988649368,
      "rewards/margins": 0.08967811614274979,
      "rewards/rejected": -0.17979438602924347,
      "sft_loss": 0.010608791373670101,
      "step": 4610
    },
    {
      "epoch": 0.35939323220536756,
      "grad_norm": 2.4869587421417236,
      "learning_rate": 3.568758284708437e-05,
      "logits/chosen": -0.01620178297162056,
      "logits/rejected": -1.0039026737213135,
      "logps/chosen": -0.8032919764518738,
      "logps/rejected": -2.2006430625915527,
      "loss": 0.8354,
      "num_input_tokens_seen": 6010208,
      "odds_ratio_loss": 8.280613899230957,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.08032919466495514,
      "rewards/margins": 0.13973508775234222,
      "rewards/rejected": -0.22006428241729736,
      "sft_loss": 0.0073692104779183865,
      "step": 4620
    },
    {
      "epoch": 0.3601711396343835,
      "grad_norm": 33.07954025268555,
      "learning_rate": 3.563231872157045e-05,
      "logits/chosen": -0.32904794812202454,
      "logits/rejected": -1.1162834167480469,
      "logps/chosen": -1.023409128189087,
      "logps/rejected": -2.7538928985595703,
      "loss": 1.0469,
      "num_input_tokens_seen": 6021760,
      "odds_ratio_loss": 10.05885124206543,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.10234091430902481,
      "rewards/margins": 0.17304839193820953,
      "rewards/rejected": -0.27538928389549255,
      "sft_loss": 0.04098767414689064,
      "step": 4630
    },
    {
      "epoch": 0.3609490470633995,
      "grad_norm": 5.191165447235107,
      "learning_rate": 3.557699109464865e-05,
      "logits/chosen": -0.34737399220466614,
      "logits/rejected": -0.7770891189575195,
      "logps/chosen": -0.8472837209701538,
      "logps/rejected": -2.7982988357543945,
      "loss": 0.8721,
      "num_input_tokens_seen": 6036992,
      "odds_ratio_loss": 8.06086254119873,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08472837507724762,
      "rewards/margins": 0.19510149955749512,
      "rewards/rejected": -0.27982988953590393,
      "sft_loss": 0.06597577035427094,
      "step": 4640
    },
    {
      "epoch": 0.3617269544924154,
      "grad_norm": 4.145506858825684,
      "learning_rate": 3.55216002967626e-05,
      "logits/chosen": -0.5288735628128052,
      "logits/rejected": -0.6832670569419861,
      "logps/chosen": -0.841774582862854,
      "logps/rejected": -1.3679779767990112,
      "loss": 0.9025,
      "num_input_tokens_seen": 6048112,
      "odds_ratio_loss": 8.584033012390137,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.08417744934558868,
      "rewards/margins": 0.05262034386396408,
      "rewards/rejected": -0.13679781556129456,
      "sft_loss": 0.04414389654994011,
      "step": 4650
    },
    {
      "epoch": 0.36250486192143133,
      "grad_norm": 1.8345520496368408,
      "learning_rate": 3.546614665873326e-05,
      "logits/chosen": -0.19236251711845398,
      "logits/rejected": -1.0131992101669312,
      "logps/chosen": -0.9838958978652954,
      "logps/rejected": -1.544071912765503,
      "loss": 1.0348,
      "num_input_tokens_seen": 6064512,
      "odds_ratio_loss": 10.184259414672852,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.09838961064815521,
      "rewards/margins": 0.05601758882403374,
      "rewards/rejected": -0.15440720319747925,
      "sft_loss": 0.016333328559994698,
      "step": 4660
    },
    {
      "epoch": 0.3632827693504473,
      "grad_norm": 1.775390863418579,
      "learning_rate": 3.5410630511756874e-05,
      "logits/chosen": 0.026402678340673447,
      "logits/rejected": -1.7419201135635376,
      "logps/chosen": -0.9833968877792358,
      "logps/rejected": -3.8465347290039062,
      "loss": 1.0076,
      "num_input_tokens_seen": 6083280,
      "odds_ratio_loss": 9.9813814163208,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.09833969175815582,
      "rewards/margins": 0.2863138020038605,
      "rewards/rejected": -0.3846535086631775,
      "sft_loss": 0.009477846324443817,
      "step": 4670
    },
    {
      "epoch": 0.36406067677946324,
      "grad_norm": 9.401809692382812,
      "learning_rate": 3.535505218740303e-05,
      "logits/chosen": -0.07243140041828156,
      "logits/rejected": -1.3489840030670166,
      "logps/chosen": -0.9240199327468872,
      "logps/rejected": -1.2963130474090576,
      "loss": 0.9727,
      "num_input_tokens_seen": 6098496,
      "odds_ratio_loss": 9.667537689208984,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.09240199625492096,
      "rewards/margins": 0.037229303270578384,
      "rewards/rejected": -0.12963131070137024,
      "sft_loss": 0.005986756179481745,
      "step": 4680
    },
    {
      "epoch": 0.3648385842084792,
      "grad_norm": 5.5551581382751465,
      "learning_rate": 3.529941201761264e-05,
      "logits/chosen": -0.06656485050916672,
      "logits/rejected": -1.1562308073043823,
      "logps/chosen": -0.8639378547668457,
      "logps/rejected": -2.3257360458374023,
      "loss": 0.8889,
      "num_input_tokens_seen": 6111008,
      "odds_ratio_loss": 8.844882011413574,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08639378100633621,
      "rewards/margins": 0.14617982506752014,
      "rewards/rejected": -0.23257359862327576,
      "sft_loss": 0.004383668769150972,
      "step": 4690
    },
    {
      "epoch": 0.36561649163749516,
      "grad_norm": 12.059032440185547,
      "learning_rate": 3.5243710334696047e-05,
      "logits/chosen": -0.08709411323070526,
      "logits/rejected": -1.2754627466201782,
      "logps/chosen": -0.921644389629364,
      "logps/rejected": -2.91294002532959,
      "loss": 0.9603,
      "num_input_tokens_seen": 6123424,
      "odds_ratio_loss": 9.507317543029785,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.09216442704200745,
      "rewards/margins": 0.19912958145141602,
      "rewards/rejected": -0.29129400849342346,
      "sft_loss": 0.009597374126315117,
      "step": 4700
    },
    {
      "epoch": 0.3663943990665111,
      "grad_norm": 1.602284550666809,
      "learning_rate": 3.5187947471330925e-05,
      "logits/chosen": -0.22274041175842285,
      "logits/rejected": -1.458547830581665,
      "logps/chosen": -0.9226606488227844,
      "logps/rejected": -2.3675448894500732,
      "loss": 0.9565,
      "num_input_tokens_seen": 6138416,
      "odds_ratio_loss": 9.507163047790527,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.09226606041193008,
      "rewards/margins": 0.14448843896389008,
      "rewards/rejected": -0.23675450682640076,
      "sft_loss": 0.005814017727971077,
      "step": 4710
    },
    {
      "epoch": 0.367172306495527,
      "grad_norm": 9.358242988586426,
      "learning_rate": 3.5132123760560375e-05,
      "logits/chosen": -0.4783569872379303,
      "logits/rejected": -0.9221259355545044,
      "logps/chosen": -0.9382320642471313,
      "logps/rejected": -3.0921168327331543,
      "loss": 0.9693,
      "num_input_tokens_seen": 6153056,
      "odds_ratio_loss": 8.790300369262695,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.09382319450378418,
      "rewards/margins": 0.2153884917497635,
      "rewards/rejected": -0.30921173095703125,
      "sft_loss": 0.09023318439722061,
      "step": 4720
    },
    {
      "epoch": 0.36795021392454297,
      "grad_norm": 6.939538478851318,
      "learning_rate": 3.507623953579091e-05,
      "logits/chosen": -0.3538041114807129,
      "logits/rejected": -0.9801716804504395,
      "logps/chosen": -1.0121508836746216,
      "logps/rejected": -1.9558436870574951,
      "loss": 1.049,
      "num_input_tokens_seen": 6161680,
      "odds_ratio_loss": 10.354455947875977,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.10121508687734604,
      "rewards/margins": 0.09436929225921631,
      "rewards/rejected": -0.19558438658714294,
      "sft_loss": 0.013524286448955536,
      "step": 4730
    },
    {
      "epoch": 0.3687281213535589,
      "grad_norm": 4.77536678314209,
      "learning_rate": 3.502029513079043e-05,
      "logits/chosen": -0.03763290494680405,
      "logits/rejected": -1.1161601543426514,
      "logps/chosen": -0.8724561929702759,
      "logps/rejected": -1.8304494619369507,
      "loss": 0.9243,
      "num_input_tokens_seen": 6180208,
      "odds_ratio_loss": 9.193095207214355,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.08724562078714371,
      "rewards/margins": 0.09579931199550629,
      "rewards/rejected": -0.18304495513439178,
      "sft_loss": 0.004998179618269205,
      "step": 4740
    },
    {
      "epoch": 0.3695060287825749,
      "grad_norm": 3.624525308609009,
      "learning_rate": 3.496429087968631e-05,
      "logits/chosen": -0.18473979830741882,
      "logits/rejected": -0.7089244723320007,
      "logps/chosen": -0.718346118927002,
      "logps/rejected": -2.575134038925171,
      "loss": 0.7439,
      "num_input_tokens_seen": 6191808,
      "odds_ratio_loss": 7.302772521972656,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.07183460891246796,
      "rewards/margins": 0.18567879498004913,
      "rewards/rejected": -0.2575134336948395,
      "sft_loss": 0.013608465902507305,
      "step": 4750
    },
    {
      "epoch": 0.37028393621159084,
      "grad_norm": 1.931633710861206,
      "learning_rate": 3.490822711696333e-05,
      "logits/chosen": 0.20464691519737244,
      "logits/rejected": -1.3728229999542236,
      "logps/chosen": -0.8623288869857788,
      "logps/rejected": -1.52870774269104,
      "loss": 0.9078,
      "num_input_tokens_seen": 6206112,
      "odds_ratio_loss": 9.046833038330078,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.08623289316892624,
      "rewards/margins": 0.06663788855075836,
      "rewards/rejected": -0.152870774269104,
      "sft_loss": 0.003099628258496523,
      "step": 4760
    },
    {
      "epoch": 0.3710618436406068,
      "grad_norm": 16.25254249572754,
      "learning_rate": 3.4852104177461685e-05,
      "logits/chosen": -0.030558357015252113,
      "logits/rejected": -1.134142279624939,
      "logps/chosen": -0.9946354627609253,
      "logps/rejected": -3.872499942779541,
      "loss": 1.0323,
      "num_input_tokens_seen": 6223760,
      "odds_ratio_loss": 10.13349437713623,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.099463552236557,
      "rewards/margins": 0.28778642416000366,
      "rewards/rejected": -0.38725000619888306,
      "sft_loss": 0.018958527594804764,
      "step": 4770
    },
    {
      "epoch": 0.3718397510696227,
      "grad_norm": 6.025865077972412,
      "learning_rate": 3.4795922396375034e-05,
      "logits/chosen": -0.03270745649933815,
      "logits/rejected": -1.0162115097045898,
      "logps/chosen": -0.7854177951812744,
      "logps/rejected": -2.39085054397583,
      "loss": 0.8294,
      "num_input_tokens_seen": 6232512,
      "odds_ratio_loss": 8.224783897399902,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.07854177802801132,
      "rewards/margins": 0.1605432778596878,
      "rewards/rejected": -0.23908504843711853,
      "sft_loss": 0.0069030774757266045,
      "step": 4780
    },
    {
      "epoch": 0.37261765849863865,
      "grad_norm": 1.815285563468933,
      "learning_rate": 3.473968210924845e-05,
      "logits/chosen": 0.16790811717510223,
      "logits/rejected": -1.867071509361267,
      "logps/chosen": -0.8914972543716431,
      "logps/rejected": -2.30869197845459,
      "loss": 0.9189,
      "num_input_tokens_seen": 6250160,
      "odds_ratio_loss": 9.165290832519531,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08914972096681595,
      "rewards/margins": 0.14171946048736572,
      "rewards/rejected": -0.23086917400360107,
      "sft_loss": 0.002407553605735302,
      "step": 4790
    },
    {
      "epoch": 0.3733955659276546,
      "grad_norm": 5.934152603149414,
      "learning_rate": 3.468338365197643e-05,
      "logits/chosen": -0.04665079340338707,
      "logits/rejected": -1.3595021963119507,
      "logps/chosen": -0.8833147883415222,
      "logps/rejected": -2.2141613960266113,
      "loss": 0.9124,
      "num_input_tokens_seen": 6262560,
      "odds_ratio_loss": 9.068648338317871,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.08833148330450058,
      "rewards/margins": 0.13308466970920563,
      "rewards/rejected": -0.2214161604642868,
      "sft_loss": 0.005574858281761408,
      "step": 4800
    },
    {
      "epoch": 0.37417347335667056,
      "grad_norm": 1.7159769535064697,
      "learning_rate": 3.4627027360800914e-05,
      "logits/chosen": -0.22343473136425018,
      "logits/rejected": -1.1393592357635498,
      "logps/chosen": -0.8822799921035767,
      "logps/rejected": -3.1561760902404785,
      "loss": 0.9097,
      "num_input_tokens_seen": 6276928,
      "odds_ratio_loss": 8.91008186340332,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.08822799474000931,
      "rewards/margins": 0.2273896187543869,
      "rewards/rejected": -0.3156176209449768,
      "sft_loss": 0.018716171383857727,
      "step": 4810
    },
    {
      "epoch": 0.3749513807856865,
      "grad_norm": 2.7350761890411377,
      "learning_rate": 3.457061357230922e-05,
      "logits/chosen": 0.11034146696329117,
      "logits/rejected": -1.421015739440918,
      "logps/chosen": -0.9038509130477905,
      "logps/rejected": -1.2649097442626953,
      "loss": 0.9624,
      "num_input_tokens_seen": 6293984,
      "odds_ratio_loss": 9.54714298248291,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.09038509428501129,
      "rewards/margins": 0.03610587492585182,
      "rewards/rejected": -0.12649096548557281,
      "sft_loss": 0.0076478696428239346,
      "step": 4820
    },
    {
      "epoch": 0.3757292882147025,
      "grad_norm": 4.121143817901611,
      "learning_rate": 3.451414262343209e-05,
      "logits/chosen": -0.07758097350597382,
      "logits/rejected": -1.2049821615219116,
      "logps/chosen": -0.9723950624465942,
      "logps/rejected": -2.1746673583984375,
      "loss": 1.0058,
      "num_input_tokens_seen": 6309888,
      "odds_ratio_loss": 10.01242733001709,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.09723950177431107,
      "rewards/margins": 0.12022723257541656,
      "rewards/rejected": -0.21746671199798584,
      "sft_loss": 0.004573773592710495,
      "step": 4830
    },
    {
      "epoch": 0.3765071956437184,
      "grad_norm": 3.1243743896484375,
      "learning_rate": 3.4457614851441664e-05,
      "logits/chosen": -0.06046917289495468,
      "logits/rejected": -1.2953927516937256,
      "logps/chosen": -0.972058892250061,
      "logps/rejected": -2.9306788444519043,
      "loss": 1.0167,
      "num_input_tokens_seen": 6319312,
      "odds_ratio_loss": 10.093836784362793,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.09720588475465775,
      "rewards/margins": 0.19586201012134552,
      "rewards/rejected": -0.2930678725242615,
      "sft_loss": 0.0073084523901343346,
      "step": 4840
    },
    {
      "epoch": 0.37728510307273433,
      "grad_norm": 21.9497127532959,
      "learning_rate": 3.440103059394942e-05,
      "logits/chosen": -0.3451583981513977,
      "logits/rejected": -1.1908886432647705,
      "logps/chosen": -1.0274622440338135,
      "logps/rejected": -2.5549333095550537,
      "loss": 1.0703,
      "num_input_tokens_seen": 6331040,
      "odds_ratio_loss": 10.037371635437012,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.10274622589349747,
      "rewards/margins": 0.15274709463119507,
      "rewards/rejected": -0.25549331307411194,
      "sft_loss": 0.06652641296386719,
      "step": 4850
    },
    {
      "epoch": 0.3780630105017503,
      "grad_norm": 2.5502912998199463,
      "learning_rate": 3.434439018890425e-05,
      "logits/chosen": -0.2128363847732544,
      "logits/rejected": -1.292725920677185,
      "logps/chosen": -1.1600662469863892,
      "logps/rejected": -2.37194561958313,
      "loss": 1.1907,
      "num_input_tokens_seen": 6345360,
      "odds_ratio_loss": 11.305562973022461,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.11600663512945175,
      "rewards/margins": 0.12118792533874512,
      "rewards/rejected": -0.23719453811645508,
      "sft_loss": 0.060100257396698,
      "step": 4860
    },
    {
      "epoch": 0.37884091793076624,
      "grad_norm": 2.542813777923584,
      "learning_rate": 3.428769397459034e-05,
      "logits/chosen": -0.11289496719837189,
      "logits/rejected": -1.2843087911605835,
      "logps/chosen": -1.1906862258911133,
      "logps/rejected": -4.575067043304443,
      "loss": 1.2152,
      "num_input_tokens_seen": 6358656,
      "odds_ratio_loss": 11.52924633026123,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.11906862258911133,
      "rewards/margins": 0.3384380340576172,
      "rewards/rejected": -0.4575066566467285,
      "sft_loss": 0.062293488532304764,
      "step": 4870
    },
    {
      "epoch": 0.3796188253597822,
      "grad_norm": 4.05035924911499,
      "learning_rate": 3.423094228962522e-05,
      "logits/chosen": -0.1508137285709381,
      "logits/rejected": -1.2871469259262085,
      "logps/chosen": -1.0064743757247925,
      "logps/rejected": -2.946434497833252,
      "loss": 1.0262,
      "num_input_tokens_seen": 6373184,
      "odds_ratio_loss": 10.089973449707031,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1006474494934082,
      "rewards/margins": 0.19399599730968475,
      "rewards/rejected": -0.29464346170425415,
      "sft_loss": 0.017155107110738754,
      "step": 4880
    },
    {
      "epoch": 0.38039673278879815,
      "grad_norm": 0.47668442130088806,
      "learning_rate": 3.41741354729577e-05,
      "logits/chosen": -0.6911668181419373,
      "logits/rejected": -0.9342254400253296,
      "logps/chosen": -0.6154254078865051,
      "logps/rejected": -2.7076973915100098,
      "loss": 0.6428,
      "num_input_tokens_seen": 6388032,
      "odds_ratio_loss": 6.27427339553833,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.061542533338069916,
      "rewards/margins": 0.20922720432281494,
      "rewards/rejected": -0.27076974511146545,
      "sft_loss": 0.01537760067731142,
      "step": 4890
    },
    {
      "epoch": 0.38117464021781405,
      "grad_norm": 2.141674280166626,
      "learning_rate": 3.411727386386589e-05,
      "logits/chosen": -0.0400090292096138,
      "logits/rejected": -1.7212409973144531,
      "logps/chosen": -0.8987911343574524,
      "logps/rejected": -1.8310871124267578,
      "loss": 0.9343,
      "num_input_tokens_seen": 6406128,
      "odds_ratio_loss": 9.297147750854492,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.089879110455513,
      "rewards/margins": 0.09322960674762726,
      "rewards/rejected": -0.18310873210430145,
      "sft_loss": 0.004615682177245617,
      "step": 4900
    },
    {
      "epoch": 0.38195254764683,
      "grad_norm": 7.255565166473389,
      "learning_rate": 3.406035780195512e-05,
      "logits/chosen": -0.299398273229599,
      "logits/rejected": -1.0692157745361328,
      "logps/chosen": -0.8373295664787292,
      "logps/rejected": -1.964667558670044,
      "loss": 0.8708,
      "num_input_tokens_seen": 6417088,
      "odds_ratio_loss": 8.478549003601074,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.0837329626083374,
      "rewards/margins": 0.11273376643657684,
      "rewards/rejected": -0.19646672904491425,
      "sft_loss": 0.02297946810722351,
      "step": 4910
    },
    {
      "epoch": 0.38273045507584597,
      "grad_norm": 1.2640591859817505,
      "learning_rate": 3.400338762715595e-05,
      "logits/chosen": -0.13642269372940063,
      "logits/rejected": -1.1650359630584717,
      "logps/chosen": -1.0347317457199097,
      "logps/rejected": -3.073350429534912,
      "loss": 1.0928,
      "num_input_tokens_seen": 6432048,
      "odds_ratio_loss": 10.22927474975586,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.10347318649291992,
      "rewards/margins": 0.20386183261871338,
      "rewards/rejected": -0.3073350489139557,
      "sft_loss": 0.06988820433616638,
      "step": 4920
    },
    {
      "epoch": 0.3835083625048619,
      "grad_norm": 2.732785940170288,
      "learning_rate": 3.394636367972214e-05,
      "logits/chosen": -0.21746814250946045,
      "logits/rejected": -0.9812092781066895,
      "logps/chosen": -0.8451567888259888,
      "logps/rejected": -5.4030632972717285,
      "loss": 0.8965,
      "num_input_tokens_seen": 6444384,
      "odds_ratio_loss": 8.777750015258789,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.08451567590236664,
      "rewards/margins": 0.4557906687259674,
      "rewards/rejected": -0.5403063297271729,
      "sft_loss": 0.018716566264629364,
      "step": 4930
    },
    {
      "epoch": 0.3842862699338779,
      "grad_norm": 1.8364019393920898,
      "learning_rate": 3.388928630022858e-05,
      "logits/chosen": -0.25838810205459595,
      "logits/rejected": -1.060112476348877,
      "logps/chosen": -0.9007120132446289,
      "logps/rejected": -2.080401659011841,
      "loss": 0.9343,
      "num_input_tokens_seen": 6458848,
      "odds_ratio_loss": 9.283602714538574,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.09007120132446289,
      "rewards/margins": 0.11796896159648895,
      "rewards/rejected": -0.20804016292095184,
      "sft_loss": 0.005900096148252487,
      "step": 4940
    },
    {
      "epoch": 0.38506417736289383,
      "grad_norm": 4.771830081939697,
      "learning_rate": 3.383215582956931e-05,
      "logits/chosen": -0.3711922764778137,
      "logits/rejected": -0.9806662797927856,
      "logps/chosen": -0.7544740438461304,
      "logps/rejected": -3.114260196685791,
      "loss": 0.7843,
      "num_input_tokens_seen": 6472848,
      "odds_ratio_loss": 7.793927192687988,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.07544741779565811,
      "rewards/margins": 0.2359786480665207,
      "rewards/rejected": -0.311426043510437,
      "sft_loss": 0.004883417394012213,
      "step": 4950
    },
    {
      "epoch": 0.3858420847919098,
      "grad_norm": 2.1738204956054688,
      "learning_rate": 3.377497260895545e-05,
      "logits/chosen": -0.3498469293117523,
      "logits/rejected": -1.3903651237487793,
      "logps/chosen": -0.7919679880142212,
      "logps/rejected": -3.7652854919433594,
      "loss": 0.817,
      "num_input_tokens_seen": 6489472,
      "odds_ratio_loss": 8.089635848999023,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.07919679582118988,
      "rewards/margins": 0.2973317503929138,
      "rewards/rejected": -0.3765285611152649,
      "sft_loss": 0.008001464419066906,
      "step": 4960
    },
    {
      "epoch": 0.3866199922209257,
      "grad_norm": 3.357419013977051,
      "learning_rate": 3.371773697991315e-05,
      "logits/chosen": -0.10142382234334946,
      "logits/rejected": -1.1423218250274658,
      "logps/chosen": -0.9958864450454712,
      "logps/rejected": -2.3799166679382324,
      "loss": 1.0363,
      "num_input_tokens_seen": 6504976,
      "odds_ratio_loss": 10.3013277053833,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.09958864003419876,
      "rewards/margins": 0.1384030282497406,
      "rewards/rejected": -0.23799166083335876,
      "sft_loss": 0.006184398662298918,
      "step": 4970
    },
    {
      "epoch": 0.38739789964994165,
      "grad_norm": 2.800766944885254,
      "learning_rate": 3.366044928428158e-05,
      "logits/chosen": -0.24856014549732208,
      "logits/rejected": -1.6147916316986084,
      "logps/chosen": -1.0334672927856445,
      "logps/rejected": -2.046121835708618,
      "loss": 1.0743,
      "num_input_tokens_seen": 6516480,
      "odds_ratio_loss": 10.621127128601074,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.10334674268960953,
      "rewards/margins": 0.10126541554927826,
      "rewards/rejected": -0.20461216568946838,
      "sft_loss": 0.012207184918224812,
      "step": 4980
    },
    {
      "epoch": 0.3881758070789576,
      "grad_norm": 3.6391632556915283,
      "learning_rate": 3.3603109864210885e-05,
      "logits/chosen": -0.26488277316093445,
      "logits/rejected": -1.168951392173767,
      "logps/chosen": -0.7505983114242554,
      "logps/rejected": -3.6049633026123047,
      "loss": 0.7821,
      "num_input_tokens_seen": 6527696,
      "odds_ratio_loss": 7.635157585144043,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.07505983859300613,
      "rewards/margins": 0.2854365110397339,
      "rewards/rejected": -0.3604963719844818,
      "sft_loss": 0.018557529896497726,
      "step": 4990
    },
    {
      "epoch": 0.38895371450797356,
      "grad_norm": 10.413762092590332,
      "learning_rate": 3.354571906216013e-05,
      "logits/chosen": -0.7141574621200562,
      "logits/rejected": -0.7490317821502686,
      "logps/chosen": -0.8036062121391296,
      "logps/rejected": -2.085695266723633,
      "loss": 0.8372,
      "num_input_tokens_seen": 6539104,
      "odds_ratio_loss": 7.791023254394531,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.08036062121391296,
      "rewards/margins": 0.12820890545845032,
      "rewards/rejected": -0.20856952667236328,
      "sft_loss": 0.05809778720140457,
      "step": 5000
    },
    {
      "epoch": 0.3897316219369895,
      "grad_norm": 13.636780738830566,
      "learning_rate": 3.3488277220895255e-05,
      "logits/chosen": -0.1624249964952469,
      "logits/rejected": -1.1657661199569702,
      "logps/chosen": -0.842096209526062,
      "logps/rejected": -3.2395873069763184,
      "loss": 0.8656,
      "num_input_tokens_seen": 6551712,
      "odds_ratio_loss": 8.55160903930664,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.0842096209526062,
      "rewards/margins": 0.23974916338920593,
      "rewards/rejected": -0.32395878434181213,
      "sft_loss": 0.010446015745401382,
      "step": 5010
    },
    {
      "epoch": 0.39050952936600547,
      "grad_norm": 1.952941656112671,
      "learning_rate": 3.343078468348703e-05,
      "logits/chosen": -0.17302784323692322,
      "logits/rejected": -1.580682635307312,
      "logps/chosen": -1.2223979234695435,
      "logps/rejected": -2.8610918521881104,
      "loss": 1.2445,
      "num_input_tokens_seen": 6568784,
      "odds_ratio_loss": 11.82412052154541,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.12223976850509644,
      "rewards/margins": 0.16386938095092773,
      "rewards/rejected": -0.28610917925834656,
      "sft_loss": 0.06208343058824539,
      "step": 5020
    },
    {
      "epoch": 0.39128743679502137,
      "grad_norm": 2.657878875732422,
      "learning_rate": 3.337324179330901e-05,
      "logits/chosen": -0.16119320690631866,
      "logits/rejected": -1.2853561639785767,
      "logps/chosen": -0.680398166179657,
      "logps/rejected": -2.567169666290283,
      "loss": 0.7005,
      "num_input_tokens_seen": 6583136,
      "odds_ratio_loss": 6.976306915283203,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.06803981959819794,
      "rewards/margins": 0.18867716193199158,
      "rewards/rejected": -0.2567169964313507,
      "sft_loss": 0.0028994691092520952,
      "step": 5030
    },
    {
      "epoch": 0.3920653442240373,
      "grad_norm": 1.8662550449371338,
      "learning_rate": 3.331564889403547e-05,
      "logits/chosen": -0.13829556107521057,
      "logits/rejected": -1.6017764806747437,
      "logps/chosen": -1.0351202487945557,
      "logps/rejected": -3.8283722400665283,
      "loss": 1.0596,
      "num_input_tokens_seen": 6596496,
      "odds_ratio_loss": 10.485677719116211,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.10351201146841049,
      "rewards/margins": 0.2793252468109131,
      "rewards/rejected": -0.3828372657299042,
      "sft_loss": 0.010996273718774319,
      "step": 5040
    },
    {
      "epoch": 0.3928432516530533,
      "grad_norm": 6.15413236618042,
      "learning_rate": 3.3258006329639394e-05,
      "logits/chosen": -0.13947170972824097,
      "logits/rejected": -1.517034888267517,
      "logps/chosen": -0.8672447204589844,
      "logps/rejected": -1.9955562353134155,
      "loss": 0.9209,
      "num_input_tokens_seen": 6610672,
      "odds_ratio_loss": 9.139081954956055,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.08672447502613068,
      "rewards/margins": 0.11283116042613983,
      "rewards/rejected": -0.1995556354522705,
      "sft_loss": 0.006971079856157303,
      "step": 5050
    },
    {
      "epoch": 0.39362115908206924,
      "grad_norm": 10.247533798217773,
      "learning_rate": 3.320031444439036e-05,
      "logits/chosen": -0.1946418732404709,
      "logits/rejected": -1.4651105403900146,
      "logps/chosen": -0.9956845045089722,
      "logps/rejected": -3.3512864112854004,
      "loss": 1.0244,
      "num_input_tokens_seen": 6626704,
      "odds_ratio_loss": 10.028584480285645,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.0995684489607811,
      "rewards/margins": 0.23556020855903625,
      "rewards/rejected": -0.33512866497039795,
      "sft_loss": 0.021575702354311943,
      "step": 5060
    },
    {
      "epoch": 0.3943990665110852,
      "grad_norm": 4.458131790161133,
      "learning_rate": 3.314257358285253e-05,
      "logits/chosen": -0.11686799675226212,
      "logits/rejected": -1.1872190237045288,
      "logps/chosen": -0.8236813545227051,
      "logps/rejected": -4.407199382781982,
      "loss": 0.8377,
      "num_input_tokens_seen": 6644080,
      "odds_ratio_loss": 8.333956718444824,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.0823681429028511,
      "rewards/margins": 0.35835182666778564,
      "rewards/rejected": -0.44071999192237854,
      "sft_loss": 0.004319559782743454,
      "step": 5070
    },
    {
      "epoch": 0.39517697394010115,
      "grad_norm": 4.913539886474609,
      "learning_rate": 3.308478408988256e-05,
      "logits/chosen": 0.11457274109125137,
      "logits/rejected": -1.8358453512191772,
      "logps/chosen": -0.9558965563774109,
      "logps/rejected": -2.9340109825134277,
      "loss": 0.9787,
      "num_input_tokens_seen": 6656352,
      "odds_ratio_loss": 9.736452102661133,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09558966010808945,
      "rewards/margins": 0.19781143963336945,
      "rewards/rejected": -0.2934010624885559,
      "sft_loss": 0.0050883106887340546,
      "step": 5080
    },
    {
      "epoch": 0.39595488136911705,
      "grad_norm": 2.4029388427734375,
      "learning_rate": 3.3026946310627575e-05,
      "logits/chosen": -0.24575765430927277,
      "logits/rejected": -1.149924397468567,
      "logps/chosen": -0.8266646265983582,
      "logps/rejected": -2.8694467544555664,
      "loss": 0.8558,
      "num_input_tokens_seen": 6671216,
      "odds_ratio_loss": 8.494571685791016,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.08266647160053253,
      "rewards/margins": 0.20427823066711426,
      "rewards/rejected": -0.2869446873664856,
      "sft_loss": 0.006306462921202183,
      "step": 5090
    },
    {
      "epoch": 0.396732788798133,
      "grad_norm": 2.4077677726745605,
      "learning_rate": 3.296906059052308e-05,
      "logits/chosen": 0.02295600436627865,
      "logits/rejected": -1.56650710105896,
      "logps/chosen": -0.9330233335494995,
      "logps/rejected": -3.9007136821746826,
      "loss": 0.9509,
      "num_input_tokens_seen": 6686496,
      "odds_ratio_loss": 9.481006622314453,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.09330235421657562,
      "rewards/margins": 0.29676908254623413,
      "rewards/rejected": -0.39007142186164856,
      "sft_loss": 0.002811084035784006,
      "step": 5100
    },
    {
      "epoch": 0.39751069622714896,
      "grad_norm": 6.374454498291016,
      "learning_rate": 3.2911127275290923e-05,
      "logits/chosen": -0.3088083863258362,
      "logits/rejected": -1.3737026453018188,
      "logps/chosen": -0.9243408441543579,
      "logps/rejected": -4.967630386352539,
      "loss": 0.9321,
      "num_input_tokens_seen": 6696800,
      "odds_ratio_loss": 9.218812942504883,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09243406355381012,
      "rewards/margins": 0.40432897210121155,
      "rewards/rejected": -0.49676305055618286,
      "sft_loss": 0.010226202197372913,
      "step": 5110
    },
    {
      "epoch": 0.3982886036561649,
      "grad_norm": 2.183419704437256,
      "learning_rate": 3.285314671093717e-05,
      "logits/chosen": -0.22676067054271698,
      "logits/rejected": -0.9388538599014282,
      "logps/chosen": -0.8574481010437012,
      "logps/rejected": -2.431346893310547,
      "loss": 0.8923,
      "num_input_tokens_seen": 6711424,
      "odds_ratio_loss": 8.803481101989746,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.08574481308460236,
      "rewards/margins": 0.15738989412784576,
      "rewards/rejected": -0.24313469231128693,
      "sft_loss": 0.011917488649487495,
      "step": 5120
    },
    {
      "epoch": 0.3990665110851809,
      "grad_norm": 3.2639870643615723,
      "learning_rate": 3.279511924375014e-05,
      "logits/chosen": -0.2976762056350708,
      "logits/rejected": -0.8617954254150391,
      "logps/chosen": -0.9393598437309265,
      "logps/rejected": -2.5974528789520264,
      "loss": 0.9994,
      "num_input_tokens_seen": 6721808,
      "odds_ratio_loss": 9.516226768493652,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.09393598884344101,
      "rewards/margins": 0.16580930352210999,
      "rewards/rejected": -0.2597452998161316,
      "sft_loss": 0.04779388755559921,
      "step": 5130
    },
    {
      "epoch": 0.39984441851419683,
      "grad_norm": 3.7727150917053223,
      "learning_rate": 3.273704522029824e-05,
      "logits/chosen": -0.0710880383849144,
      "logits/rejected": -1.1556459665298462,
      "logps/chosen": -0.9698354601860046,
      "logps/rejected": -2.05775785446167,
      "loss": 0.9936,
      "num_input_tokens_seen": 6738656,
      "odds_ratio_loss": 9.846353530883789,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09698355197906494,
      "rewards/margins": 0.1087922677397728,
      "rewards/rejected": -0.20577578246593475,
      "sft_loss": 0.009006194770336151,
      "step": 5140
    },
    {
      "epoch": 0.40062232594321273,
      "grad_norm": 2.765150547027588,
      "learning_rate": 3.2678924987427935e-05,
      "logits/chosen": -0.22569546103477478,
      "logits/rejected": -1.0096803903579712,
      "logps/chosen": -1.295796513557434,
      "logps/rejected": -4.616843223571777,
      "loss": 1.35,
      "num_input_tokens_seen": 6748608,
      "odds_ratio_loss": 11.873702049255371,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.12957964837551117,
      "rewards/margins": 0.3321046829223633,
      "rewards/rejected": -0.46168431639671326,
      "sft_loss": 0.1626376509666443,
      "step": 5150
    },
    {
      "epoch": 0.4014002333722287,
      "grad_norm": 3.987420082092285,
      "learning_rate": 3.2620758892261684e-05,
      "logits/chosen": -0.13436201214790344,
      "logits/rejected": -1.4836794137954712,
      "logps/chosen": -1.3089988231658936,
      "logps/rejected": -6.551393985748291,
      "loss": 1.3293,
      "num_input_tokens_seen": 6759200,
      "odds_ratio_loss": 13.168505668640137,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.13089989125728607,
      "rewards/margins": 0.5242395401000977,
      "rewards/rejected": -0.6551393866539001,
      "sft_loss": 0.012489492073655128,
      "step": 5160
    },
    {
      "epoch": 0.40217814080124464,
      "grad_norm": 1.4179848432540894,
      "learning_rate": 3.2562547282195864e-05,
      "logits/chosen": -0.013448494486510754,
      "logits/rejected": -1.2190428972244263,
      "logps/chosen": -0.8156849145889282,
      "logps/rejected": -3.563967227935791,
      "loss": 0.8285,
      "num_input_tokens_seen": 6774192,
      "odds_ratio_loss": 8.089681625366211,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08156848698854446,
      "rewards/margins": 0.2748281955718994,
      "rewards/rejected": -0.35639673471450806,
      "sft_loss": 0.019520580768585205,
      "step": 5170
    },
    {
      "epoch": 0.4029560482302606,
      "grad_norm": 3.3582563400268555,
      "learning_rate": 3.250429050489868e-05,
      "logits/chosen": -0.08996443450450897,
      "logits/rejected": -1.5054981708526611,
      "logps/chosen": -0.9165959358215332,
      "logps/rejected": -2.1653242111206055,
      "loss": 0.9387,
      "num_input_tokens_seen": 6785632,
      "odds_ratio_loss": 9.320246696472168,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09165959060192108,
      "rewards/margins": 0.12487280368804932,
      "rewards/rejected": -0.2165323942899704,
      "sft_loss": 0.006640143692493439,
      "step": 5180
    },
    {
      "epoch": 0.40373395565927656,
      "grad_norm": 2.3919262886047363,
      "learning_rate": 3.244598890830811e-05,
      "logits/chosen": 0.04672724008560181,
      "logits/rejected": -1.244020700454712,
      "logps/chosen": -1.1241716146469116,
      "logps/rejected": -3.711254835128784,
      "loss": 1.1392,
      "num_input_tokens_seen": 6796816,
      "odds_ratio_loss": 11.346402168273926,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.11241716146469116,
      "rewards/margins": 0.25870823860168457,
      "rewards/rejected": -0.3711254298686981,
      "sft_loss": 0.004553156904876232,
      "step": 5190
    },
    {
      "epoch": 0.4045118630882925,
      "grad_norm": 3.7833569049835205,
      "learning_rate": 3.2387642840629795e-05,
      "logits/chosen": -0.17067503929138184,
      "logits/rejected": -1.752317190170288,
      "logps/chosen": -0.9981927871704102,
      "logps/rejected": -4.065258502960205,
      "loss": 1.0181,
      "num_input_tokens_seen": 6807600,
      "odds_ratio_loss": 10.092321395874023,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.09981928765773773,
      "rewards/margins": 0.3067065477371216,
      "rewards/rejected": -0.40652579069137573,
      "sft_loss": 0.008861953392624855,
      "step": 5200
    },
    {
      "epoch": 0.40528977051730847,
      "grad_norm": 2.497227191925049,
      "learning_rate": 3.232925265033499e-05,
      "logits/chosen": -0.17980130016803741,
      "logits/rejected": -1.2274338006973267,
      "logps/chosen": -1.2723392248153687,
      "logps/rejected": -2.9400558471679688,
      "loss": 1.2985,
      "num_input_tokens_seen": 6818256,
      "odds_ratio_loss": 12.26331901550293,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.12723393738269806,
      "rewards/margins": 0.16677165031433105,
      "rewards/rejected": -0.2940055727958679,
      "sft_loss": 0.07213227450847626,
      "step": 5210
    },
    {
      "epoch": 0.40606767794632437,
      "grad_norm": 4.260927200317383,
      "learning_rate": 3.227081868615847e-05,
      "logits/chosen": -0.17838871479034424,
      "logits/rejected": -1.1781151294708252,
      "logps/chosen": -0.8753217458724976,
      "logps/rejected": -3.9808316230773926,
      "loss": 0.9,
      "num_input_tokens_seen": 6827504,
      "odds_ratio_loss": 8.838985443115234,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.0875321626663208,
      "rewards/margins": 0.3105510175228119,
      "rewards/rejected": -0.3980831503868103,
      "sft_loss": 0.016073519363999367,
      "step": 5220
    },
    {
      "epoch": 0.4068455853753403,
      "grad_norm": 7.187779903411865,
      "learning_rate": 3.221234129709645e-05,
      "logits/chosen": -0.019155466929078102,
      "logits/rejected": -1.0761711597442627,
      "logps/chosen": -0.9712806940078735,
      "logps/rejected": -3.239480495452881,
      "loss": 0.9842,
      "num_input_tokens_seen": 6838800,
      "odds_ratio_loss": 9.74677848815918,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09712807089090347,
      "rewards/margins": 0.2268199920654297,
      "rewards/rejected": -0.32394805550575256,
      "sft_loss": 0.00948383565992117,
      "step": 5230
    },
    {
      "epoch": 0.4076234928043563,
      "grad_norm": 7.241087436676025,
      "learning_rate": 3.215382083240451e-05,
      "logits/chosen": -0.1222321018576622,
      "logits/rejected": -1.066963791847229,
      "logps/chosen": -1.1054328680038452,
      "logps/rejected": -7.408546447753906,
      "loss": 1.1193,
      "num_input_tokens_seen": 6850048,
      "odds_ratio_loss": 11.099000930786133,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.11054328829050064,
      "rewards/margins": 0.6303113102912903,
      "rewards/rejected": -0.7408546209335327,
      "sft_loss": 0.009413559921085835,
      "step": 5240
    },
    {
      "epoch": 0.40840140023337224,
      "grad_norm": 2.376088857650757,
      "learning_rate": 3.209525764159548e-05,
      "logits/chosen": -0.11934573948383331,
      "logits/rejected": -1.205902338027954,
      "logps/chosen": -0.6773741245269775,
      "logps/rejected": -2.1595306396484375,
      "loss": 0.702,
      "num_input_tokens_seen": 6863520,
      "odds_ratio_loss": 6.923120021820068,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.06773741543292999,
      "rewards/margins": 0.148215651512146,
      "rewards/rejected": -0.215953066945076,
      "sft_loss": 0.009694124571979046,
      "step": 5250
    },
    {
      "epoch": 0.4091793076623882,
      "grad_norm": 2.091048240661621,
      "learning_rate": 3.203665207443738e-05,
      "logits/chosen": -0.08640505373477936,
      "logits/rejected": -1.4066154956817627,
      "logps/chosen": -0.7765898704528809,
      "logps/rejected": -5.977321624755859,
      "loss": 0.788,
      "num_input_tokens_seen": 6873776,
      "odds_ratio_loss": 7.773428916931152,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.0776589885354042,
      "rewards/margins": 0.5200731754302979,
      "rewards/rejected": -0.5977321267127991,
      "sft_loss": 0.010621096938848495,
      "step": 5260
    },
    {
      "epoch": 0.40995721509140415,
      "grad_norm": 3.409365177154541,
      "learning_rate": 3.197800448095133e-05,
      "logits/chosen": -0.14782679080963135,
      "logits/rejected": -1.2552828788757324,
      "logps/chosen": -0.9665555953979492,
      "logps/rejected": -6.867901802062988,
      "loss": 0.9847,
      "num_input_tokens_seen": 6890400,
      "odds_ratio_loss": 9.714597702026367,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09665556252002716,
      "rewards/margins": 0.5901346206665039,
      "rewards/rejected": -0.6867901682853699,
      "sft_loss": 0.013249441981315613,
      "step": 5270
    },
    {
      "epoch": 0.41073512252042005,
      "grad_norm": 2.304940938949585,
      "learning_rate": 3.191931521140943e-05,
      "logits/chosen": -0.18876931071281433,
      "logits/rejected": -1.5803499221801758,
      "logps/chosen": -1.3756680488586426,
      "logps/rejected": -6.449460029602051,
      "loss": 1.4162,
      "num_input_tokens_seen": 6906960,
      "odds_ratio_loss": 14.041711807250977,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.13756680488586426,
      "rewards/margins": 0.5073791742324829,
      "rewards/rejected": -0.6449459791183472,
      "sft_loss": 0.012073583900928497,
      "step": 5280
    },
    {
      "epoch": 0.411513029949436,
      "grad_norm": 10.533405303955078,
      "learning_rate": 3.186058461633271e-05,
      "logits/chosen": -0.13900762796401978,
      "logits/rejected": -1.0330493450164795,
      "logps/chosen": -0.9527713656425476,
      "logps/rejected": -3.1085875034332275,
      "loss": 0.9869,
      "num_input_tokens_seen": 6917440,
      "odds_ratio_loss": 9.745183944702148,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.09527714550495148,
      "rewards/margins": 0.21558161079883575,
      "rewards/rejected": -0.31085875630378723,
      "sft_loss": 0.012343376874923706,
      "step": 5290
    },
    {
      "epoch": 0.41229093737845196,
      "grad_norm": 11.712982177734375,
      "learning_rate": 3.180181304648902e-05,
      "logits/chosen": -0.20885948836803436,
      "logits/rejected": -0.5603643655776978,
      "logps/chosen": -0.7855948805809021,
      "logps/rejected": -1.346688985824585,
      "loss": 0.8738,
      "num_input_tokens_seen": 6925824,
      "odds_ratio_loss": 8.597783088684082,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.07855948805809021,
      "rewards/margins": 0.056109391152858734,
      "rewards/rejected": -0.13466887176036835,
      "sft_loss": 0.01399481762200594,
      "step": 5300
    },
    {
      "epoch": 0.4130688448074679,
      "grad_norm": 2.2742912769317627,
      "learning_rate": 3.17430008528909e-05,
      "logits/chosen": -0.27529218792915344,
      "logits/rejected": -1.3195812702178955,
      "logps/chosen": -0.8954521417617798,
      "logps/rejected": -2.502065896987915,
      "loss": 0.9261,
      "num_input_tokens_seen": 6937024,
      "odds_ratio_loss": 9.151135444641113,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.08954521268606186,
      "rewards/margins": 0.16066136956214905,
      "rewards/rejected": -0.2502065598964691,
      "sft_loss": 0.011016755364835262,
      "step": 5310
    },
    {
      "epoch": 0.4138467522364839,
      "grad_norm": 3.655043125152588,
      "learning_rate": 3.168414838679356e-05,
      "logits/chosen": -0.16698357462882996,
      "logits/rejected": -1.3797894716262817,
      "logps/chosen": -0.9265694618225098,
      "logps/rejected": -3.8845913410186768,
      "loss": 0.962,
      "num_input_tokens_seen": 6949456,
      "odds_ratio_loss": 9.402085304260254,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.0926569476723671,
      "rewards/margins": 0.29580220580101013,
      "rewards/rejected": -0.388459175825119,
      "sft_loss": 0.02179957926273346,
      "step": 5320
    },
    {
      "epoch": 0.41462465966549983,
      "grad_norm": 4.217288017272949,
      "learning_rate": 3.1625255999692707e-05,
      "logits/chosen": -0.21513691544532776,
      "logits/rejected": -1.0598417520523071,
      "logps/chosen": -1.184991478919983,
      "logps/rejected": -3.253505229949951,
      "loss": 1.2351,
      "num_input_tokens_seen": 6965536,
      "odds_ratio_loss": 12.20081615447998,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.11849913746118546,
      "rewards/margins": 0.20685140788555145,
      "rewards/rejected": -0.32535049319267273,
      "sft_loss": 0.014983227476477623,
      "step": 5330
    },
    {
      "epoch": 0.41540256709451573,
      "grad_norm": 115.32563018798828,
      "learning_rate": 3.156632404332247e-05,
      "logits/chosen": -0.0027507960330694914,
      "logits/rejected": -1.1567466259002686,
      "logps/chosen": -1.2281917333602905,
      "logps/rejected": -4.739614009857178,
      "loss": 1.2494,
      "num_input_tokens_seen": 6982160,
      "odds_ratio_loss": 11.43757152557373,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.12281918525695801,
      "rewards/margins": 0.3511422574520111,
      "rewards/rejected": -0.4739614427089691,
      "sft_loss": 0.10563137382268906,
      "step": 5340
    },
    {
      "epoch": 0.4161804745235317,
      "grad_norm": 5.8340163230896,
      "learning_rate": 3.1507352869653326e-05,
      "logits/chosen": -0.37204450368881226,
      "logits/rejected": -0.7582011222839355,
      "logps/chosen": -0.8406723737716675,
      "logps/rejected": -2.9044442176818848,
      "loss": 0.8583,
      "num_input_tokens_seen": 7000208,
      "odds_ratio_loss": 8.48363971710205,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08406723290681839,
      "rewards/margins": 0.20637719333171844,
      "rewards/rejected": -0.29044443368911743,
      "sft_loss": 0.009952530264854431,
      "step": 5350
    },
    {
      "epoch": 0.41695838195254764,
      "grad_norm": 6.446136951446533,
      "learning_rate": 3.144834283088997e-05,
      "logits/chosen": -0.38958272337913513,
      "logits/rejected": -0.5921080708503723,
      "logps/chosen": -0.6633752584457397,
      "logps/rejected": -2.818526268005371,
      "loss": 0.6832,
      "num_input_tokens_seen": 7015120,
      "odds_ratio_loss": 6.703489780426025,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.06633754074573517,
      "rewards/margins": 0.21551513671875,
      "rewards/rejected": -0.2818526327610016,
      "sft_loss": 0.012843889184296131,
      "step": 5360
    },
    {
      "epoch": 0.4177362893815636,
      "grad_norm": 2.075299024581909,
      "learning_rate": 3.1389294279469215e-05,
      "logits/chosen": -0.20397326350212097,
      "logits/rejected": -1.4514131546020508,
      "logps/chosen": -0.9150875806808472,
      "logps/rejected": -4.986923694610596,
      "loss": 0.9423,
      "num_input_tokens_seen": 7028880,
      "odds_ratio_loss": 9.248687744140625,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.09150876104831696,
      "rewards/margins": 0.4071836471557617,
      "rewards/rejected": -0.49869242310523987,
      "sft_loss": 0.017469432204961777,
      "step": 5370
    },
    {
      "epoch": 0.41851419681057955,
      "grad_norm": 7.3845295906066895,
      "learning_rate": 3.133020756805789e-05,
      "logits/chosen": -0.1727880984544754,
      "logits/rejected": -0.7701871395111084,
      "logps/chosen": -0.9352876543998718,
      "logps/rejected": -1.9459960460662842,
      "loss": 0.984,
      "num_input_tokens_seen": 7038720,
      "odds_ratio_loss": 9.56579303741455,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.09352876245975494,
      "rewards/margins": 0.1010708436369896,
      "rewards/rejected": -0.19459959864616394,
      "sft_loss": 0.027424853295087814,
      "step": 5380
    },
    {
      "epoch": 0.4192921042395955,
      "grad_norm": 1.9074008464813232,
      "learning_rate": 3.1271083049550736e-05,
      "logits/chosen": 0.10264843702316284,
      "logits/rejected": -1.5509620904922485,
      "logps/chosen": -0.8807462453842163,
      "logps/rejected": -4.2532196044921875,
      "loss": 0.9056,
      "num_input_tokens_seen": 7052992,
      "odds_ratio_loss": 9.034521102905273,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.08807463198900223,
      "rewards/margins": 0.337247371673584,
      "rewards/rejected": -0.4253219962120056,
      "sft_loss": 0.0021968758665025234,
      "step": 5390
    },
    {
      "epoch": 0.4200700116686114,
      "grad_norm": 1.974965214729309,
      "learning_rate": 3.12119210770683e-05,
      "logits/chosen": -0.13084354996681213,
      "logits/rejected": -1.0727672576904297,
      "logps/chosen": -0.8137876391410828,
      "logps/rejected": -3.66515851020813,
      "loss": 0.837,
      "num_input_tokens_seen": 7071088,
      "odds_ratio_loss": 8.31655216217041,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.08137877285480499,
      "rewards/margins": 0.2851371169090271,
      "rewards/rejected": -0.3665158450603485,
      "sft_loss": 0.005319107323884964,
      "step": 5400
    },
    {
      "epoch": 0.42084791909762737,
      "grad_norm": 3.967224359512329,
      "learning_rate": 3.1152722003954815e-05,
      "logits/chosen": -0.14671170711517334,
      "logits/rejected": -0.9043130874633789,
      "logps/chosen": -0.9685951471328735,
      "logps/rejected": -2.1605186462402344,
      "loss": 1.0268,
      "num_input_tokens_seen": 7081392,
      "odds_ratio_loss": 10.207342147827148,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.09685951471328735,
      "rewards/margins": 0.11919237673282623,
      "rewards/rejected": -0.2160518616437912,
      "sft_loss": 0.006040588952600956,
      "step": 5410
    },
    {
      "epoch": 0.4216258265266433,
      "grad_norm": 4.425365447998047,
      "learning_rate": 3.10934861837761e-05,
      "logits/chosen": -0.38159123063087463,
      "logits/rejected": -1.5197417736053467,
      "logps/chosen": -0.9468908309936523,
      "logps/rejected": -5.209157466888428,
      "loss": 0.9555,
      "num_input_tokens_seen": 7093248,
      "odds_ratio_loss": 9.486285209655762,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09468908607959747,
      "rewards/margins": 0.4262266159057617,
      "rewards/rejected": -0.520915687084198,
      "sft_loss": 0.006851466838270426,
      "step": 5420
    },
    {
      "epoch": 0.4224037339556593,
      "grad_norm": 30.725133895874023,
      "learning_rate": 3.103421397031745e-05,
      "logits/chosen": -0.34320956468582153,
      "logits/rejected": -1.181138038635254,
      "logps/chosen": -1.0038588047027588,
      "logps/rejected": -3.035968065261841,
      "loss": 1.0209,
      "num_input_tokens_seen": 7112240,
      "odds_ratio_loss": 9.553422927856445,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.1003858894109726,
      "rewards/margins": 0.20321090519428253,
      "rewards/rejected": -0.3035968244075775,
      "sft_loss": 0.06559192389249802,
      "step": 5430
    },
    {
      "epoch": 0.42318164138467523,
      "grad_norm": 2.8233392238616943,
      "learning_rate": 3.097490571758151e-05,
      "logits/chosen": -0.029369115829467773,
      "logits/rejected": -1.5265640020370483,
      "logps/chosen": -0.7838415503501892,
      "logps/rejected": -4.505312919616699,
      "loss": 0.7893,
      "num_input_tokens_seen": 7131280,
      "odds_ratio_loss": 7.869359016418457,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.0783841460943222,
      "rewards/margins": 0.37214717268943787,
      "rewards/rejected": -0.45053133368492126,
      "sft_loss": 0.0024120414163917303,
      "step": 5440
    },
    {
      "epoch": 0.4239595488136912,
      "grad_norm": 10.336220741271973,
      "learning_rate": 3.091556177978619e-05,
      "logits/chosen": -0.2264358550310135,
      "logits/rejected": -1.415867805480957,
      "logps/chosen": -0.71709144115448,
      "logps/rejected": -7.780030727386475,
      "loss": 0.7227,
      "num_input_tokens_seen": 7145920,
      "odds_ratio_loss": 7.129467010498047,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.07170914113521576,
      "rewards/margins": 0.706294059753418,
      "rewards/rejected": -0.7780030965805054,
      "sft_loss": 0.009785325266420841,
      "step": 5450
    },
    {
      "epoch": 0.42473745624270715,
      "grad_norm": 1.6062244176864624,
      "learning_rate": 3.08561825113625e-05,
      "logits/chosen": -0.037660397589206696,
      "logits/rejected": -1.3419077396392822,
      "logps/chosen": -0.9172475934028625,
      "logps/rejected": -2.8567147254943848,
      "loss": 0.9378,
      "num_input_tokens_seen": 7161216,
      "odds_ratio_loss": 9.303727149963379,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09172476828098297,
      "rewards/margins": 0.1939467191696167,
      "rewards/rejected": -0.28567150235176086,
      "sft_loss": 0.007443229667842388,
      "step": 5460
    },
    {
      "epoch": 0.42551536367172305,
      "grad_norm": 6.955587387084961,
      "learning_rate": 3.079676826695248e-05,
      "logits/chosen": -0.2916269302368164,
      "logits/rejected": -0.9078566431999207,
      "logps/chosen": -0.8469215631484985,
      "logps/rejected": -4.218659400939941,
      "loss": 0.8656,
      "num_input_tokens_seen": 7172848,
      "odds_ratio_loss": 8.542197227478027,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08469214290380478,
      "rewards/margins": 0.3371737599372864,
      "rewards/rejected": -0.42186588048934937,
      "sft_loss": 0.011388765648007393,
      "step": 5470
    },
    {
      "epoch": 0.426293271100739,
      "grad_norm": 6.72831392288208,
      "learning_rate": 3.0737319401407037e-05,
      "logits/chosen": -0.1381830871105194,
      "logits/rejected": -0.9215238690376282,
      "logps/chosen": -0.926182746887207,
      "logps/rejected": -2.7554633617401123,
      "loss": 0.9704,
      "num_input_tokens_seen": 7187648,
      "odds_ratio_loss": 9.581047058105469,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.09261827915906906,
      "rewards/margins": 0.18292807042598724,
      "rewards/rejected": -0.2755463719367981,
      "sft_loss": 0.01227156538516283,
      "step": 5480
    },
    {
      "epoch": 0.42707117852975496,
      "grad_norm": 1.813859224319458,
      "learning_rate": 3.0677836269783895e-05,
      "logits/chosen": -0.4149433672428131,
      "logits/rejected": -1.3414149284362793,
      "logps/chosen": -1.15546715259552,
      "logps/rejected": -4.314212799072266,
      "loss": 1.1696,
      "num_input_tokens_seen": 7200368,
      "odds_ratio_loss": 11.48894214630127,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.11554670333862305,
      "rewards/margins": 0.3158745765686035,
      "rewards/rejected": -0.4314212203025818,
      "sft_loss": 0.020742299035191536,
      "step": 5490
    },
    {
      "epoch": 0.4278490859587709,
      "grad_norm": 2.206411361694336,
      "learning_rate": 3.061831922734541e-05,
      "logits/chosen": -0.4122841954231262,
      "logits/rejected": -1.3862360715866089,
      "logps/chosen": -0.9372841715812683,
      "logps/rejected": -3.5864288806915283,
      "loss": 0.9635,
      "num_input_tokens_seen": 7211472,
      "odds_ratio_loss": 9.549125671386719,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.09372840821743011,
      "rewards/margins": 0.26491445302963257,
      "rewards/rejected": -0.3586428761482239,
      "sft_loss": 0.008612281642854214,
      "step": 5500
    },
    {
      "epoch": 0.42862699338778687,
      "grad_norm": 7.141944885253906,
      "learning_rate": 3.055876862955647e-05,
      "logits/chosen": 0.02164841815829277,
      "logits/rejected": -1.631679892539978,
      "logps/chosen": -0.8621139526367188,
      "logps/rejected": -6.096848487854004,
      "loss": 0.8666,
      "num_input_tokens_seen": 7222832,
      "odds_ratio_loss": 8.421870231628418,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08621139079332352,
      "rewards/margins": 0.5234734416007996,
      "rewards/rejected": -0.6096848249435425,
      "sft_loss": 0.024400796741247177,
      "step": 5510
    },
    {
      "epoch": 0.4294049008168028,
      "grad_norm": 2.589207410812378,
      "learning_rate": 3.0499184832082362e-05,
      "logits/chosen": -0.3766138255596161,
      "logits/rejected": -1.7519257068634033,
      "logps/chosen": -0.8969801664352417,
      "logps/rejected": -3.5163254737854004,
      "loss": 0.9139,
      "num_input_tokens_seen": 7232992,
      "odds_ratio_loss": 8.830495834350586,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08969802409410477,
      "rewards/margins": 0.2619345486164093,
      "rewards/rejected": -0.3516325354576111,
      "sft_loss": 0.030821654945611954,
      "step": 5520
    },
    {
      "epoch": 0.4301828082458187,
      "grad_norm": 2.4507274627685547,
      "learning_rate": 3.0439568190786682e-05,
      "logits/chosen": -0.02348879538476467,
      "logits/rejected": -1.166263461112976,
      "logps/chosen": -0.8205503225326538,
      "logps/rejected": -5.710233688354492,
      "loss": 0.8556,
      "num_input_tokens_seen": 7241760,
      "odds_ratio_loss": 8.516347885131836,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.08205503225326538,
      "rewards/margins": 0.4889683723449707,
      "rewards/rejected": -0.5710233449935913,
      "sft_loss": 0.003920835442841053,
      "step": 5530
    },
    {
      "epoch": 0.4309607156748347,
      "grad_norm": 2.328852891921997,
      "learning_rate": 3.0379919061729168e-05,
      "logits/chosen": 0.01200108602643013,
      "logits/rejected": -1.959764838218689,
      "logps/chosen": -0.8829509019851685,
      "logps/rejected": -3.345860004425049,
      "loss": 0.8981,
      "num_input_tokens_seen": 7254960,
      "odds_ratio_loss": 8.910298347473145,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.08829507976770401,
      "rewards/margins": 0.2462908923625946,
      "rewards/rejected": -0.3345859944820404,
      "sft_loss": 0.007096708752214909,
      "step": 5540
    },
    {
      "epoch": 0.43173862310385064,
      "grad_norm": 9.930980682373047,
      "learning_rate": 3.032023780116358e-05,
      "logits/chosen": -0.35067591071128845,
      "logits/rejected": -0.4748639166355133,
      "logps/chosen": -0.743221640586853,
      "logps/rejected": -5.461892127990723,
      "loss": 0.7485,
      "num_input_tokens_seen": 7273088,
      "odds_ratio_loss": 7.39886999130249,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.0743221715092659,
      "rewards/margins": 0.47186699509620667,
      "rewards/rejected": -0.5461891889572144,
      "sft_loss": 0.008651467971503735,
      "step": 5550
    },
    {
      "epoch": 0.4325165305328666,
      "grad_norm": 1.6604907512664795,
      "learning_rate": 3.026052476553561e-05,
      "logits/chosen": 0.056398797780275345,
      "logits/rejected": -1.9943794012069702,
      "logps/chosen": -0.8136641383171082,
      "logps/rejected": -6.4685492515563965,
      "loss": 0.8194,
      "num_input_tokens_seen": 7286160,
      "odds_ratio_loss": 8.159880638122559,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08136641979217529,
      "rewards/margins": 0.5654885172843933,
      "rewards/rejected": -0.6468549966812134,
      "sft_loss": 0.0034152872394770384,
      "step": 5560
    },
    {
      "epoch": 0.43329443796188255,
      "grad_norm": 17.088098526000977,
      "learning_rate": 3.0200780311480716e-05,
      "logits/chosen": -0.6167064309120178,
      "logits/rejected": -1.160718321800232,
      "logps/chosen": -1.155081868171692,
      "logps/rejected": -6.337732315063477,
      "loss": 1.1695,
      "num_input_tokens_seen": 7297920,
      "odds_ratio_loss": 11.395452499389648,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.11550818383693695,
      "rewards/margins": 0.5182650685310364,
      "rewards/rejected": -0.6337732076644897,
      "sft_loss": 0.029999535530805588,
      "step": 5570
    },
    {
      "epoch": 0.4340723453908985,
      "grad_norm": 3.5572893619537354,
      "learning_rate": 3.0141004795821992e-05,
      "logits/chosen": -0.5635565519332886,
      "logits/rejected": -0.6509861350059509,
      "logps/chosen": -1.1619627475738525,
      "logps/rejected": -8.93120002746582,
      "loss": 1.1741,
      "num_input_tokens_seen": 7311472,
      "odds_ratio_loss": 11.183347702026367,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.11619627475738525,
      "rewards/margins": 0.7769237756729126,
      "rewards/rejected": -0.8931199312210083,
      "sft_loss": 0.05573863908648491,
      "step": 5580
    },
    {
      "epoch": 0.4348502528199144,
      "grad_norm": 2.9649763107299805,
      "learning_rate": 3.008119857556807e-05,
      "logits/chosen": -0.1862354576587677,
      "logits/rejected": -1.009856939315796,
      "logps/chosen": -0.8901270627975464,
      "logps/rejected": -2.387753963470459,
      "loss": 0.9199,
      "num_input_tokens_seen": 7322080,
      "odds_ratio_loss": 8.999714851379395,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.08901270478963852,
      "rewards/margins": 0.14976267516613007,
      "rewards/rejected": -0.23877540230751038,
      "sft_loss": 0.019966933876276016,
      "step": 5590
    },
    {
      "epoch": 0.43562816024893036,
      "grad_norm": 6.203644752502441,
      "learning_rate": 3.002136200791093e-05,
      "logits/chosen": -0.3672073185443878,
      "logits/rejected": -1.2089344263076782,
      "logps/chosen": -1.1483100652694702,
      "logps/rejected": -6.217892646789551,
      "loss": 1.1625,
      "num_input_tokens_seen": 7337232,
      "odds_ratio_loss": 10.810770034790039,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.11483101546764374,
      "rewards/margins": 0.5069582462310791,
      "rewards/rejected": -0.6217892169952393,
      "sft_loss": 0.08141560107469559,
      "step": 5600
    },
    {
      "epoch": 0.4364060676779463,
      "grad_norm": 1.8932578563690186,
      "learning_rate": 2.996149545022383e-05,
      "logits/chosen": -0.06057189032435417,
      "logits/rejected": -1.631291389465332,
      "logps/chosen": -0.9629873037338257,
      "logps/rejected": -7.250810146331787,
      "loss": 0.9769,
      "num_input_tokens_seen": 7347408,
      "odds_ratio_loss": 9.381027221679688,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.09629872441291809,
      "rewards/margins": 0.628782331943512,
      "rewards/rejected": -0.7250810265541077,
      "sft_loss": 0.03883747756481171,
      "step": 5610
    },
    {
      "epoch": 0.4371839751069623,
      "grad_norm": 25.1219425201416,
      "learning_rate": 2.990159926005913e-05,
      "logits/chosen": -0.252627432346344,
      "logits/rejected": -1.3857918977737427,
      "logps/chosen": -1.0234429836273193,
      "logps/rejected": -6.393345832824707,
      "loss": 1.0359,
      "num_input_tokens_seen": 7362448,
      "odds_ratio_loss": 9.738905906677246,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.10234431177377701,
      "rewards/margins": 0.5369902849197388,
      "rewards/rejected": -0.6393346190452576,
      "sft_loss": 0.06196347624063492,
      "step": 5620
    },
    {
      "epoch": 0.43796188253597823,
      "grad_norm": 9.728668212890625,
      "learning_rate": 2.984167379514618e-05,
      "logits/chosen": -0.07025805860757828,
      "logits/rejected": -1.309217929840088,
      "logps/chosen": -1.034991979598999,
      "logps/rejected": -6.303004741668701,
      "loss": 1.0562,
      "num_input_tokens_seen": 7377168,
      "odds_ratio_loss": 10.495607376098633,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.10349918901920319,
      "rewards/margins": 0.5268012881278992,
      "rewards/rejected": -0.6303004622459412,
      "sft_loss": 0.0066757239401340485,
      "step": 5630
    },
    {
      "epoch": 0.4387397899649942,
      "grad_norm": 2.4033548831939697,
      "learning_rate": 2.978171941338917e-05,
      "logits/chosen": 0.06687622517347336,
      "logits/rejected": -1.3250255584716797,
      "logps/chosen": -0.8779948949813843,
      "logps/rejected": -5.355679512023926,
      "loss": 0.9189,
      "num_input_tokens_seen": 7391184,
      "odds_ratio_loss": 9.139822006225586,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.08779950439929962,
      "rewards/margins": 0.4477684497833252,
      "rewards/rejected": -0.5355679392814636,
      "sft_loss": 0.004900778643786907,
      "step": 5640
    },
    {
      "epoch": 0.4395176973940101,
      "grad_norm": 4.902503490447998,
      "learning_rate": 2.972173647286499e-05,
      "logits/chosen": -0.16414262354373932,
      "logits/rejected": -1.3518102169036865,
      "logps/chosen": -1.7802966833114624,
      "logps/rejected": -11.64123249053955,
      "loss": 1.8715,
      "num_input_tokens_seen": 7405376,
      "odds_ratio_loss": 18.06220245361328,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.17802968621253967,
      "rewards/margins": 0.9860936403274536,
      "rewards/rejected": -1.1641231775283813,
      "sft_loss": 0.0653175562620163,
      "step": 5650
    },
    {
      "epoch": 0.44029560482302604,
      "grad_norm": 7.490935325622559,
      "learning_rate": 2.96617253318211e-05,
      "logits/chosen": -0.11388234794139862,
      "logits/rejected": -1.323212742805481,
      "logps/chosen": -1.8963305950164795,
      "logps/rejected": -6.981576442718506,
      "loss": 1.9488,
      "num_input_tokens_seen": 7419312,
      "odds_ratio_loss": 18.875633239746094,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.1896330714225769,
      "rewards/margins": 0.5085245966911316,
      "rewards/rejected": -0.6981576681137085,
      "sft_loss": 0.061241038143634796,
      "step": 5660
    },
    {
      "epoch": 0.441073512252042,
      "grad_norm": 3.349161148071289,
      "learning_rate": 2.960168634867338e-05,
      "logits/chosen": -0.3609488606452942,
      "logits/rejected": -0.9301692247390747,
      "logps/chosen": -1.1135082244873047,
      "logps/rejected": -6.0065412521362305,
      "loss": 1.1235,
      "num_input_tokens_seen": 7432704,
      "odds_ratio_loss": 10.837892532348633,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.11135081946849823,
      "rewards/margins": 0.48930326104164124,
      "rewards/rejected": -0.600654125213623,
      "sft_loss": 0.039745159447193146,
      "step": 5670
    },
    {
      "epoch": 0.44185141968105796,
      "grad_norm": 7.245255947113037,
      "learning_rate": 2.9541619882004007e-05,
      "logits/chosen": -0.09249483048915863,
      "logits/rejected": -1.1441720724105835,
      "logps/chosen": -0.8567533493041992,
      "logps/rejected": -4.396549701690674,
      "loss": 0.8774,
      "num_input_tokens_seen": 7447376,
      "odds_ratio_loss": 8.738390922546387,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.08567533642053604,
      "rewards/margins": 0.3539796471595764,
      "rewards/rejected": -0.43965497612953186,
      "sft_loss": 0.003584805177524686,
      "step": 5680
    },
    {
      "epoch": 0.4426293271100739,
      "grad_norm": 1.8555481433868408,
      "learning_rate": 2.9481526290559298e-05,
      "logits/chosen": -0.10176386684179306,
      "logits/rejected": -1.3324403762817383,
      "logps/chosen": -1.0124648809432983,
      "logps/rejected": -4.5107245445251465,
      "loss": 1.026,
      "num_input_tokens_seen": 7457856,
      "odds_ratio_loss": 9.876537322998047,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.10124649107456207,
      "rewards/margins": 0.34982600808143616,
      "rewards/rejected": -0.4510725140571594,
      "sft_loss": 0.038344431668519974,
      "step": 5690
    },
    {
      "epoch": 0.44340723453908987,
      "grad_norm": 1.75498366355896,
      "learning_rate": 2.9421405933247582e-05,
      "logits/chosen": -0.010723483748733997,
      "logits/rejected": -1.041904091835022,
      "logps/chosen": -0.8967369794845581,
      "logps/rejected": -5.024321556091309,
      "loss": 0.9163,
      "num_input_tokens_seen": 7469360,
      "odds_ratio_loss": 9.101985931396484,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.089673712849617,
      "rewards/margins": 0.4127584397792816,
      "rewards/rejected": -0.502432107925415,
      "sft_loss": 0.006076131947338581,
      "step": 5700
    },
    {
      "epoch": 0.44418514196810577,
      "grad_norm": 3.833069086074829,
      "learning_rate": 2.936125916913704e-05,
      "logits/chosen": -0.18692442774772644,
      "logits/rejected": -1.2350839376449585,
      "logps/chosen": -1.1202170848846436,
      "logps/rejected": -4.992281913757324,
      "loss": 1.139,
      "num_input_tokens_seen": 7478880,
      "odds_ratio_loss": 11.292125701904297,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.11202170699834824,
      "rewards/margins": 0.38720643520355225,
      "rewards/rejected": -0.49922817945480347,
      "sft_loss": 0.00977136380970478,
      "step": 5710
    },
    {
      "epoch": 0.4449630493971217,
      "grad_norm": 1.6511999368667603,
      "learning_rate": 2.9301086357453557e-05,
      "logits/chosen": -0.22126416862010956,
      "logits/rejected": -0.9571665525436401,
      "logps/chosen": -1.0049254894256592,
      "logps/rejected": -3.3358731269836426,
      "loss": 1.0214,
      "num_input_tokens_seen": 7495696,
      "odds_ratio_loss": 10.157580375671387,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.10049253702163696,
      "rewards/margins": 0.23309478163719177,
      "rewards/rejected": -0.33358731865882874,
      "sft_loss": 0.005630288738757372,
      "step": 5720
    },
    {
      "epoch": 0.4457409568261377,
      "grad_norm": 2.4155290126800537,
      "learning_rate": 2.9240887857578597e-05,
      "logits/chosen": -0.13161590695381165,
      "logits/rejected": -0.9564615488052368,
      "logps/chosen": -0.7802937626838684,
      "logps/rejected": -6.504634857177734,
      "loss": 0.7909,
      "num_input_tokens_seen": 7507056,
      "odds_ratio_loss": 7.859260559082031,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.07802937924861908,
      "rewards/margins": 0.57243412733078,
      "rewards/rejected": -0.6504634022712708,
      "sft_loss": 0.004980637691915035,
      "step": 5730
    },
    {
      "epoch": 0.44651886425515364,
      "grad_norm": 3.0114312171936035,
      "learning_rate": 2.918066402904705e-05,
      "logits/chosen": 0.14526775479316711,
      "logits/rejected": -1.194178581237793,
      "logps/chosen": -0.7590327858924866,
      "logps/rejected": -11.647214889526367,
      "loss": 0.7787,
      "num_input_tokens_seen": 7517136,
      "odds_ratio_loss": 7.698354244232178,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.0759032815694809,
      "rewards/margins": 1.0888183116912842,
      "rewards/rejected": -1.1647217273712158,
      "sft_loss": 0.008911113254725933,
      "step": 5740
    },
    {
      "epoch": 0.4472967716841696,
      "grad_norm": 2.222205638885498,
      "learning_rate": 2.9120415231545066e-05,
      "logits/chosen": 0.02403981052339077,
      "logits/rejected": -1.470847487449646,
      "logps/chosen": -0.9761005640029907,
      "logps/rejected": -8.883180618286133,
      "loss": 0.9802,
      "num_input_tokens_seen": 7530208,
      "odds_ratio_loss": 9.752192497253418,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09761004894971848,
      "rewards/margins": 0.7907080054283142,
      "rewards/rejected": -0.8883180618286133,
      "sft_loss": 0.00500735267996788,
      "step": 5750
    },
    {
      "epoch": 0.44807467911318555,
      "grad_norm": 2.7538716793060303,
      "learning_rate": 2.906014182490795e-05,
      "logits/chosen": -0.13734382390975952,
      "logits/rejected": -1.3238089084625244,
      "logps/chosen": -0.9427657127380371,
      "logps/rejected": -5.268524169921875,
      "loss": 0.9545,
      "num_input_tokens_seen": 7544272,
      "odds_ratio_loss": 9.417902946472168,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.094276562333107,
      "rewards/margins": 0.43257585167884827,
      "rewards/rejected": -0.5268524289131165,
      "sft_loss": 0.01273438148200512,
      "step": 5760
    },
    {
      "epoch": 0.4488525865422015,
      "grad_norm": 2.635197877883911,
      "learning_rate": 2.8999844169117958e-05,
      "logits/chosen": -0.20401224493980408,
      "logits/rejected": -0.8030599355697632,
      "logps/chosen": -0.8307873606681824,
      "logps/rejected": -4.622495651245117,
      "loss": 0.8435,
      "num_input_tokens_seen": 7558480,
      "odds_ratio_loss": 8.378512382507324,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08307874202728271,
      "rewards/margins": 0.37917080521583557,
      "rewards/rejected": -0.4622495770454407,
      "sft_loss": 0.0056395018473267555,
      "step": 5770
    },
    {
      "epoch": 0.4496304939712174,
      "grad_norm": 10.95548152923584,
      "learning_rate": 2.893952262430219e-05,
      "logits/chosen": -0.2868979871273041,
      "logits/rejected": -1.515904426574707,
      "logps/chosen": -1.033301830291748,
      "logps/rejected": -6.378268718719482,
      "loss": 1.0583,
      "num_input_tokens_seen": 7572112,
      "odds_ratio_loss": 9.888237953186035,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.10333018004894257,
      "rewards/margins": 0.5344967246055603,
      "rewards/rejected": -0.6378269195556641,
      "sft_loss": 0.06948982179164886,
      "step": 5780
    },
    {
      "epoch": 0.45040840140023336,
      "grad_norm": 2.2623848915100098,
      "learning_rate": 2.8879177550730407e-05,
      "logits/chosen": -0.10103366523981094,
      "logits/rejected": -1.231601595878601,
      "logps/chosen": -0.8214209675788879,
      "logps/rejected": -4.604743480682373,
      "loss": 0.847,
      "num_input_tokens_seen": 7585360,
      "odds_ratio_loss": 8.389901161193848,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.08214209973812103,
      "rewards/margins": 0.3783322274684906,
      "rewards/rejected": -0.4604743421077728,
      "sft_loss": 0.007993288338184357,
      "step": 5790
    },
    {
      "epoch": 0.4511863088292493,
      "grad_norm": 2.8834474086761475,
      "learning_rate": 2.8818809308812906e-05,
      "logits/chosen": -0.21622204780578613,
      "logits/rejected": -1.0297644138336182,
      "logps/chosen": -0.9987231492996216,
      "logps/rejected": -5.402844429016113,
      "loss": 1.0204,
      "num_input_tokens_seen": 7599648,
      "odds_ratio_loss": 10.143254280090332,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.09987231343984604,
      "rewards/margins": 0.44041213393211365,
      "rewards/rejected": -0.5402845144271851,
      "sft_loss": 0.006070182658731937,
      "step": 5800
    },
    {
      "epoch": 0.45196421625826527,
      "grad_norm": 7.1191864013671875,
      "learning_rate": 2.875841825909835e-05,
      "logits/chosen": -0.04676201939582825,
      "logits/rejected": -1.37875235080719,
      "logps/chosen": -0.7726394534111023,
      "logps/rejected": -5.68399715423584,
      "loss": 0.7841,
      "num_input_tokens_seen": 7607328,
      "odds_ratio_loss": 7.746439456939697,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.07726395130157471,
      "rewards/margins": 0.49113577604293823,
      "rewards/rejected": -0.5683996677398682,
      "sft_loss": 0.009432977065443993,
      "step": 5810
    },
    {
      "epoch": 0.45274212368728123,
      "grad_norm": 2.4421451091766357,
      "learning_rate": 2.8698004762271634e-05,
      "logits/chosen": -0.13264720141887665,
      "logits/rejected": -1.1827833652496338,
      "logps/chosen": -1.1583963632583618,
      "logps/rejected": -6.693757057189941,
      "loss": 1.161,
      "num_input_tokens_seen": 7619840,
      "odds_ratio_loss": 11.536027908325195,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1158396452665329,
      "rewards/margins": 0.553536057472229,
      "rewards/rejected": -0.6693757176399231,
      "sft_loss": 0.007445620838552713,
      "step": 5820
    },
    {
      "epoch": 0.4535200311162972,
      "grad_norm": 2.106879234313965,
      "learning_rate": 2.8637569179151714e-05,
      "logits/chosen": -0.2481449842453003,
      "logits/rejected": -0.9333834648132324,
      "logps/chosen": -0.8990824818611145,
      "logps/rejected": -7.192356109619141,
      "loss": 0.9073,
      "num_input_tokens_seen": 7632912,
      "odds_ratio_loss": 8.996527671813965,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08990823477506638,
      "rewards/margins": 0.6293273568153381,
      "rewards/rejected": -0.7192355990409851,
      "sft_loss": 0.007650734391063452,
      "step": 5830
    },
    {
      "epoch": 0.4542979385453131,
      "grad_norm": 2.554652214050293,
      "learning_rate": 2.8577111870689454e-05,
      "logits/chosen": -0.3907225728034973,
      "logits/rejected": -1.3036072254180908,
      "logps/chosen": -0.8883325457572937,
      "logps/rejected": -4.797537803649902,
      "loss": 0.9031,
      "num_input_tokens_seen": 7645040,
      "odds_ratio_loss": 8.559049606323242,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.0888332650065422,
      "rewards/margins": 0.39092057943344116,
      "rewards/rejected": -0.4797538220882416,
      "sft_loss": 0.04718649387359619,
      "step": 5840
    },
    {
      "epoch": 0.45507584597432904,
      "grad_norm": 2.30387020111084,
      "learning_rate": 2.8516633197965458e-05,
      "logits/chosen": -0.20695964992046356,
      "logits/rejected": -0.8214203715324402,
      "logps/chosen": -1.097795844078064,
      "logps/rejected": -4.349630355834961,
      "loss": 1.116,
      "num_input_tokens_seen": 7659872,
      "odds_ratio_loss": 11.061562538146973,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.10977959632873535,
      "rewards/margins": 0.3251834511756897,
      "rewards/rejected": -0.43496307730674744,
      "sft_loss": 0.009882526472210884,
      "step": 5850
    },
    {
      "epoch": 0.455853753403345,
      "grad_norm": 2.312739372253418,
      "learning_rate": 2.845613352218795e-05,
      "logits/chosen": 0.01900560036301613,
      "logits/rejected": -1.2111179828643799,
      "logps/chosen": -0.8945795893669128,
      "logps/rejected": -6.274838447570801,
      "loss": 0.904,
      "num_input_tokens_seen": 7675120,
      "odds_ratio_loss": 8.90841293334961,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08945796638727188,
      "rewards/margins": 0.5380259156227112,
      "rewards/rejected": -0.6274839639663696,
      "sft_loss": 0.01311540324240923,
      "step": 5860
    },
    {
      "epoch": 0.45663166083236095,
      "grad_norm": 1.5392405986785889,
      "learning_rate": 2.839561320469058e-05,
      "logits/chosen": -0.062470972537994385,
      "logits/rejected": -1.4860166311264038,
      "logps/chosen": -0.806922435760498,
      "logps/rejected": -5.0439581871032715,
      "loss": 0.8221,
      "num_input_tokens_seen": 7690256,
      "odds_ratio_loss": 8.170222282409668,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.08069224655628204,
      "rewards/margins": 0.4237035810947418,
      "rewards/rejected": -0.5043958425521851,
      "sft_loss": 0.005118037573993206,
      "step": 5870
    },
    {
      "epoch": 0.4574095682613769,
      "grad_norm": 2.0916812419891357,
      "learning_rate": 2.8335072606930287e-05,
      "logits/chosen": -0.2488502562046051,
      "logits/rejected": -1.1231826543807983,
      "logps/chosen": -1.1016508340835571,
      "logps/rejected": -6.482033729553223,
      "loss": 1.1136,
      "num_input_tokens_seen": 7702640,
      "odds_ratio_loss": 10.864334106445312,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.11016509681940079,
      "rewards/margins": 0.5380383133888245,
      "rewards/rejected": -0.6482033729553223,
      "sft_loss": 0.027160849422216415,
      "step": 5880
    },
    {
      "epoch": 0.45818747569039286,
      "grad_norm": 2.859052896499634,
      "learning_rate": 2.827451209048514e-05,
      "logits/chosen": -0.2412732094526291,
      "logits/rejected": -1.3144216537475586,
      "logps/chosen": -0.961597740650177,
      "logps/rejected": -2.7586493492126465,
      "loss": 0.996,
      "num_input_tokens_seen": 7713392,
      "odds_ratio_loss": 9.801130294799805,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.09615977108478546,
      "rewards/margins": 0.1797051727771759,
      "rewards/rejected": -0.27586492896080017,
      "sft_loss": 0.0158519484102726,
      "step": 5890
    },
    {
      "epoch": 0.45896538311940877,
      "grad_norm": 6.613534450531006,
      "learning_rate": 2.8213932017052164e-05,
      "logits/chosen": -0.15394410490989685,
      "logits/rejected": -1.5015206336975098,
      "logps/chosen": -0.8291906118392944,
      "logps/rejected": -4.777556419372559,
      "loss": 0.843,
      "num_input_tokens_seen": 7723536,
      "odds_ratio_loss": 8.361349105834961,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08291905373334885,
      "rewards/margins": 0.3948366343975067,
      "rewards/rejected": -0.4777556359767914,
      "sft_loss": 0.006913746241480112,
      "step": 5900
    },
    {
      "epoch": 0.4597432905484247,
      "grad_norm": 2.7631146907806396,
      "learning_rate": 2.8153332748445178e-05,
      "logits/chosen": -0.13939619064331055,
      "logits/rejected": -1.1255842447280884,
      "logps/chosen": -0.849032998085022,
      "logps/rejected": -3.494724988937378,
      "loss": 0.8648,
      "num_input_tokens_seen": 7733392,
      "odds_ratio_loss": 8.560256004333496,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.08490331470966339,
      "rewards/margins": 0.2645691931247711,
      "rewards/rejected": -0.3494724929332733,
      "sft_loss": 0.008783399127423763,
      "step": 5910
    },
    {
      "epoch": 0.4605211979774407,
      "grad_norm": 7.09406852722168,
      "learning_rate": 2.8092714646592654e-05,
      "logits/chosen": -0.1654086410999298,
      "logits/rejected": -1.4517545700073242,
      "logps/chosen": -0.8999756574630737,
      "logps/rejected": -4.131761074066162,
      "loss": 0.9214,
      "num_input_tokens_seen": 7745856,
      "odds_ratio_loss": 9.104066848754883,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.08999757468700409,
      "rewards/margins": 0.32317858934402466,
      "rewards/rejected": -0.41317611932754517,
      "sft_loss": 0.01101828645914793,
      "step": 5920
    },
    {
      "epoch": 0.46129910540645663,
      "grad_norm": 2.690000295639038,
      "learning_rate": 2.803207807353556e-05,
      "logits/chosen": -0.2681902348995209,
      "logits/rejected": -0.8913981318473816,
      "logps/chosen": -0.6731475591659546,
      "logps/rejected": -5.323077201843262,
      "loss": 0.6876,
      "num_input_tokens_seen": 7759568,
      "odds_ratio_loss": 6.8018670082092285,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.0673147663474083,
      "rewards/margins": 0.46499305963516235,
      "rewards/rejected": -0.5323078036308289,
      "sft_loss": 0.007376432418823242,
      "step": 5930
    },
    {
      "epoch": 0.4620770128354726,
      "grad_norm": 3.621859073638916,
      "learning_rate": 2.7971423391425154e-05,
      "logits/chosen": 0.03137628361582756,
      "logits/rejected": -1.338469386100769,
      "logps/chosen": -1.0437556505203247,
      "logps/rejected": -8.069334983825684,
      "loss": 1.0659,
      "num_input_tokens_seen": 7774960,
      "odds_ratio_loss": 9.991592407226562,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.10437557846307755,
      "rewards/margins": 0.7025579214096069,
      "rewards/rejected": -0.806933581829071,
      "sft_loss": 0.06670835614204407,
      "step": 5940
    },
    {
      "epoch": 0.46285492026448855,
      "grad_norm": 2.3327841758728027,
      "learning_rate": 2.7910750962520872e-05,
      "logits/chosen": 0.03029645048081875,
      "logits/rejected": -1.226078748703003,
      "logps/chosen": -0.6837007403373718,
      "logps/rejected": -5.380637168884277,
      "loss": 0.6976,
      "num_input_tokens_seen": 7786784,
      "odds_ratio_loss": 6.941661834716797,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.06837008148431778,
      "rewards/margins": 0.469693660736084,
      "rewards/rejected": -0.5380637645721436,
      "sft_loss": 0.003387672360986471,
      "step": 5950
    },
    {
      "epoch": 0.46363282769350445,
      "grad_norm": 90.8195571899414,
      "learning_rate": 2.7850061149188146e-05,
      "logits/chosen": -0.4019021987915039,
      "logits/rejected": -1.172957181930542,
      "logps/chosen": -1.0341510772705078,
      "logps/rejected": -9.447542190551758,
      "loss": 1.0446,
      "num_input_tokens_seen": 7799360,
      "odds_ratio_loss": 10.295927047729492,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.1034151166677475,
      "rewards/margins": 0.841339111328125,
      "rewards/rejected": -0.9447541236877441,
      "sft_loss": 0.014985822141170502,
      "step": 5960
    },
    {
      "epoch": 0.4644107351225204,
      "grad_norm": 5.015258312225342,
      "learning_rate": 2.7789354313896203e-05,
      "logits/chosen": -0.1693064272403717,
      "logits/rejected": -1.2654229402542114,
      "logps/chosen": -0.8729956746101379,
      "logps/rejected": -7.783166408538818,
      "loss": 0.8972,
      "num_input_tokens_seen": 7815408,
      "odds_ratio_loss": 8.893301010131836,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.08729957044124603,
      "rewards/margins": 0.691017210483551,
      "rewards/rejected": -0.7783167362213135,
      "sft_loss": 0.007835522294044495,
      "step": 5970
    },
    {
      "epoch": 0.46518864255153636,
      "grad_norm": 8.604205131530762,
      "learning_rate": 2.7728630819215978e-05,
      "logits/chosen": -0.21987919509410858,
      "logits/rejected": -1.4255279302597046,
      "logps/chosen": -1.0022478103637695,
      "logps/rejected": -6.178910255432129,
      "loss": 1.0319,
      "num_input_tokens_seen": 7826352,
      "odds_ratio_loss": 10.210367202758789,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.10022477805614471,
      "rewards/margins": 0.517666220664978,
      "rewards/rejected": -0.6178910136222839,
      "sft_loss": 0.010909122414886951,
      "step": 5980
    },
    {
      "epoch": 0.4659665499805523,
      "grad_norm": 5.230869770050049,
      "learning_rate": 2.7667891027817865e-05,
      "logits/chosen": -0.3464639186859131,
      "logits/rejected": -1.0649701356887817,
      "logps/chosen": -0.7585294842720032,
      "logps/rejected": -5.745346546173096,
      "loss": 0.7815,
      "num_input_tokens_seen": 7833904,
      "odds_ratio_loss": 7.651752471923828,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.07585294544696808,
      "rewards/margins": 0.4986816942691803,
      "rewards/rejected": -0.5745345950126648,
      "sft_loss": 0.016333168372511864,
      "step": 5990
    },
    {
      "epoch": 0.46674445740956827,
      "grad_norm": 507.97247314453125,
      "learning_rate": 2.7607135302469615e-05,
      "logits/chosen": -0.22762298583984375,
      "logits/rejected": -1.140218734741211,
      "logps/chosen": -1.5735101699829102,
      "logps/rejected": -3.470374345779419,
      "loss": 1.6485,
      "num_input_tokens_seen": 7850688,
      "odds_ratio_loss": 15.951501846313477,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.1573510318994522,
      "rewards/margins": 0.1896863877773285,
      "rewards/rejected": -0.3470374047756195,
      "sft_loss": 0.053358934819698334,
      "step": 6000
    },
    {
      "epoch": 0.4675223648385842,
      "grad_norm": 2.535000801086426,
      "learning_rate": 2.7546364006034137e-05,
      "logits/chosen": -0.06629699468612671,
      "logits/rejected": -1.4996873140335083,
      "logps/chosen": -0.8568462133407593,
      "logps/rejected": -9.733963012695312,
      "loss": 0.8854,
      "num_input_tokens_seen": 7862704,
      "odds_ratio_loss": 8.79551887512207,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.085684634745121,
      "rewards/margins": 0.8877116441726685,
      "rewards/rejected": -0.9733963012695312,
      "sft_loss": 0.005818786099553108,
      "step": 6010
    },
    {
      "epoch": 0.4683002722676002,
      "grad_norm": 2.4148144721984863,
      "learning_rate": 2.7485577501467337e-05,
      "logits/chosen": -0.3303348124027252,
      "logits/rejected": -0.8454314470291138,
      "logps/chosen": -0.819682240486145,
      "logps/rejected": -3.0566112995147705,
      "loss": 0.849,
      "num_input_tokens_seen": 7876240,
      "odds_ratio_loss": 8.1669921875,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.08196821808815002,
      "rewards/margins": 0.22369292378425598,
      "rewards/rejected": -0.305661141872406,
      "sft_loss": 0.03231917321681976,
      "step": 6020
    },
    {
      "epoch": 0.4690781796966161,
      "grad_norm": 2.606917381286621,
      "learning_rate": 2.742477615181596e-05,
      "logits/chosen": 0.09807735681533813,
      "logits/rejected": -1.4461350440979004,
      "logps/chosen": -1.050130844116211,
      "logps/rejected": -6.8379645347595215,
      "loss": 1.0526,
      "num_input_tokens_seen": 7885536,
      "odds_ratio_loss": 10.428014755249023,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.10501308739185333,
      "rewards/margins": 0.578783392906189,
      "rewards/rejected": -0.6837964653968811,
      "sft_loss": 0.009837312623858452,
      "step": 6030
    },
    {
      "epoch": 0.46985608712563204,
      "grad_norm": 1.6283973455429077,
      "learning_rate": 2.7363960320215393e-05,
      "logits/chosen": -0.24384596943855286,
      "logits/rejected": -1.2933290004730225,
      "logps/chosen": -0.591637134552002,
      "logps/rejected": -6.946697235107422,
      "loss": 0.6009,
      "num_input_tokens_seen": 7898160,
      "odds_ratio_loss": 5.6380205154418945,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.059163711965084076,
      "rewards/margins": 0.6355060338973999,
      "rewards/rejected": -0.694669783115387,
      "sft_loss": 0.0371074303984642,
      "step": 6040
    },
    {
      "epoch": 0.470633994554648,
      "grad_norm": 5.1402177810668945,
      "learning_rate": 2.730313036988753e-05,
      "logits/chosen": -0.12846294045448303,
      "logits/rejected": -1.385025143623352,
      "logps/chosen": -0.8894141912460327,
      "logps/rejected": -3.4371979236602783,
      "loss": 0.9212,
      "num_input_tokens_seen": 7910608,
      "odds_ratio_loss": 8.882787704467773,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.08894142508506775,
      "rewards/margins": 0.2547783851623535,
      "rewards/rejected": -0.34371981024742126,
      "sft_loss": 0.03289348632097244,
      "step": 6050
    },
    {
      "epoch": 0.47141190198366395,
      "grad_norm": 2.494810104370117,
      "learning_rate": 2.7242286664138595e-05,
      "logits/chosen": 0.08889412134885788,
      "logits/rejected": -1.808688759803772,
      "logps/chosen": -0.7223685383796692,
      "logps/rejected": -4.377908229827881,
      "loss": 0.7449,
      "num_input_tokens_seen": 7923776,
      "odds_ratio_loss": 7.426810264587402,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.07223685085773468,
      "rewards/margins": 0.36555397510528564,
      "rewards/rejected": -0.43779081106185913,
      "sft_loss": 0.0022290514316409826,
      "step": 6060
    },
    {
      "epoch": 0.4721898094126799,
      "grad_norm": 0.6981421113014221,
      "learning_rate": 2.7181429566356952e-05,
      "logits/chosen": -0.44528698921203613,
      "logits/rejected": -1.303655743598938,
      "logps/chosen": -0.7890397310256958,
      "logps/rejected": -4.5952653884887695,
      "loss": 0.8009,
      "num_input_tokens_seen": 7934848,
      "odds_ratio_loss": 7.944195747375488,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.07890397310256958,
      "rewards/margins": 0.38062259554862976,
      "rewards/rejected": -0.45952659845352173,
      "sft_loss": 0.006529558449983597,
      "step": 6070
    },
    {
      "epoch": 0.47296771684169586,
      "grad_norm": 2.9179821014404297,
      "learning_rate": 2.712055944001095e-05,
      "logits/chosen": -0.22472205758094788,
      "logits/rejected": -1.4862409830093384,
      "logps/chosen": -0.9329856038093567,
      "logps/rejected": -12.334473609924316,
      "loss": 0.9441,
      "num_input_tokens_seen": 7946752,
      "odds_ratio_loss": 9.35265064239502,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.0932985469698906,
      "rewards/margins": 1.1401488780975342,
      "rewards/rejected": -1.233447551727295,
      "sft_loss": 0.008851487189531326,
      "step": 6080
    },
    {
      "epoch": 0.47374562427071176,
      "grad_norm": 3.321930408477783,
      "learning_rate": 2.705967664864678e-05,
      "logits/chosen": -0.2719866633415222,
      "logits/rejected": -1.5798949003219604,
      "logps/chosen": -0.9579636454582214,
      "logps/rejected": -11.035733222961426,
      "loss": 0.9719,
      "num_input_tokens_seen": 7959040,
      "odds_ratio_loss": 9.649393081665039,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.0957963690161705,
      "rewards/margins": 1.0077769756317139,
      "rewards/rejected": -1.1035733222961426,
      "sft_loss": 0.006914190948009491,
      "step": 6090
    },
    {
      "epoch": 0.4745235316997277,
      "grad_norm": 7.015444278717041,
      "learning_rate": 2.6998781555886227e-05,
      "logits/chosen": -0.2518048584461212,
      "logits/rejected": -1.1595853567123413,
      "logps/chosen": -0.810549259185791,
      "logps/rejected": -17.622051239013672,
      "loss": 0.8161,
      "num_input_tokens_seen": 7972240,
      "odds_ratio_loss": 8.099400520324707,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.0810549259185791,
      "rewards/margins": 1.6811498403549194,
      "rewards/rejected": -1.762204885482788,
      "sft_loss": 0.006167970597743988,
      "step": 6100
    },
    {
      "epoch": 0.4753014391287437,
      "grad_norm": 1.4676387310028076,
      "learning_rate": 2.6937874525424577e-05,
      "logits/chosen": -0.2762657403945923,
      "logits/rejected": -1.0840905904769897,
      "logps/chosen": -1.2074869871139526,
      "logps/rejected": -11.498342514038086,
      "loss": 1.2286,
      "num_input_tokens_seen": 7987488,
      "odds_ratio_loss": 10.559377670288086,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.12074869871139526,
      "rewards/margins": 1.029085636138916,
      "rewards/rejected": -1.149834394454956,
      "sft_loss": 0.1726953685283661,
      "step": 6110
    },
    {
      "epoch": 0.47607934655775963,
      "grad_norm": 1.908224105834961,
      "learning_rate": 2.6876955921028408e-05,
      "logits/chosen": -0.20587265491485596,
      "logits/rejected": -1.3624060153961182,
      "logps/chosen": -0.7148422002792358,
      "logps/rejected": -7.384983062744141,
      "loss": 0.7285,
      "num_input_tokens_seen": 7998720,
      "odds_ratio_loss": 7.2302398681640625,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.07148422300815582,
      "rewards/margins": 0.6670140027999878,
      "rewards/rejected": -0.7384982705116272,
      "sft_loss": 0.005456112790852785,
      "step": 6120
    },
    {
      "epoch": 0.4768572539867756,
      "grad_norm": 1.6788196563720703,
      "learning_rate": 2.6816026106533428e-05,
      "logits/chosen": -0.14552515745162964,
      "logits/rejected": -1.3555206060409546,
      "logps/chosen": -0.8666656613349915,
      "logps/rejected": -16.5738582611084,
      "loss": 0.878,
      "num_input_tokens_seen": 8016608,
      "odds_ratio_loss": 8.666595458984375,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08666656166315079,
      "rewards/margins": 1.5707193613052368,
      "rewards/rejected": -1.6573858261108398,
      "sft_loss": 0.01131691224873066,
      "step": 6130
    },
    {
      "epoch": 0.47763516141579154,
      "grad_norm": 2.0778162479400635,
      "learning_rate": 2.6755085445842276e-05,
      "logits/chosen": -0.06541848182678223,
      "logits/rejected": -1.4086366891860962,
      "logps/chosen": -0.8241838216781616,
      "logps/rejected": -12.803787231445312,
      "loss": 0.8433,
      "num_input_tokens_seen": 8029872,
      "odds_ratio_loss": 8.388916969299316,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.08241839706897736,
      "rewards/margins": 1.1979604959487915,
      "rewards/rejected": -1.2803786993026733,
      "sft_loss": 0.0044305888004601,
      "step": 6140
    },
    {
      "epoch": 0.47841306884480744,
      "grad_norm": 7.2569074630737305,
      "learning_rate": 2.6694134302922396e-05,
      "logits/chosen": -0.15363359451293945,
      "logits/rejected": -1.0825797319412231,
      "logps/chosen": -0.9479478001594543,
      "logps/rejected": -9.695534706115723,
      "loss": 0.9548,
      "num_input_tokens_seen": 8045200,
      "odds_ratio_loss": 9.402900695800781,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09479478001594543,
      "rewards/margins": 0.8747587203979492,
      "rewards/rejected": -0.9695534706115723,
      "sft_loss": 0.014496704563498497,
      "step": 6150
    },
    {
      "epoch": 0.4791909762738234,
      "grad_norm": 1.3309544324874878,
      "learning_rate": 2.6633173041803828e-05,
      "logits/chosen": -0.1747881919145584,
      "logits/rejected": -1.319799542427063,
      "logps/chosen": -0.8536643981933594,
      "logps/rejected": -5.712998867034912,
      "loss": 0.8676,
      "num_input_tokens_seen": 8060896,
      "odds_ratio_loss": 8.596722602844238,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08536644279956818,
      "rewards/margins": 0.48593348264694214,
      "rewards/rejected": -0.5712999105453491,
      "sft_loss": 0.00788833387196064,
      "step": 6160
    },
    {
      "epoch": 0.47996888370283936,
      "grad_norm": 4.741382122039795,
      "learning_rate": 2.6572202026577036e-05,
      "logits/chosen": -0.07354072481393814,
      "logits/rejected": -1.1875712871551514,
      "logps/chosen": -0.7627614736557007,
      "logps/rejected": -10.352548599243164,
      "loss": 0.7677,
      "num_input_tokens_seen": 8074224,
      "odds_ratio_loss": 7.301914215087891,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.07627616077661514,
      "rewards/margins": 0.9589789509773254,
      "rewards/rejected": -1.035254955291748,
      "sft_loss": 0.037500251084566116,
      "step": 6170
    },
    {
      "epoch": 0.4807467911318553,
      "grad_norm": 22.816436767578125,
      "learning_rate": 2.651122162139074e-05,
      "logits/chosen": -0.5151816606521606,
      "logits/rejected": -1.1198275089263916,
      "logps/chosen": -0.9731024503707886,
      "logps/rejected": -12.562901496887207,
      "loss": 0.996,
      "num_input_tokens_seen": 8088144,
      "odds_ratio_loss": 9.769756317138672,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.09731025993824005,
      "rewards/margins": 1.1589797735214233,
      "rewards/rejected": -1.256290078163147,
      "sft_loss": 0.019033921882510185,
      "step": 6180
    },
    {
      "epoch": 0.48152469856087127,
      "grad_norm": 3.49373722076416,
      "learning_rate": 2.6450232190449758e-05,
      "logits/chosen": -0.05174265056848526,
      "logits/rejected": -1.6913894414901733,
      "logps/chosen": -0.8371627926826477,
      "logps/rejected": -12.714179039001465,
      "loss": 0.8474,
      "num_input_tokens_seen": 8101824,
      "odds_ratio_loss": 8.428500175476074,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08371627330780029,
      "rewards/margins": 1.18770170211792,
      "rewards/rejected": -1.2714178562164307,
      "sft_loss": 0.0045602647587656975,
      "step": 6190
    },
    {
      "epoch": 0.4823026059898872,
      "grad_norm": 3.0217878818511963,
      "learning_rate": 2.6389234098012792e-05,
      "logits/chosen": -0.1306189000606537,
      "logits/rejected": -1.3144865036010742,
      "logps/chosen": -1.0501823425292969,
      "logps/rejected": -17.49735450744629,
      "loss": 1.0605,
      "num_input_tokens_seen": 8119472,
      "odds_ratio_loss": 10.536087989807129,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.1050182357430458,
      "rewards/margins": 1.6447172164916992,
      "rewards/rejected": -1.7497354745864868,
      "sft_loss": 0.006926544010639191,
      "step": 6200
    },
    {
      "epoch": 0.4830805134189031,
      "grad_norm": 2.083096981048584,
      "learning_rate": 2.6328227708390297e-05,
      "logits/chosen": -0.15976101160049438,
      "logits/rejected": -1.337317705154419,
      "logps/chosen": -0.7609018087387085,
      "logps/rejected": -17.045391082763672,
      "loss": 0.7782,
      "num_input_tokens_seen": 8133248,
      "odds_ratio_loss": 7.667045593261719,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.07609017193317413,
      "rewards/margins": 1.6284490823745728,
      "rewards/rejected": -1.7045392990112305,
      "sft_loss": 0.0114522036164999,
      "step": 6210
    },
    {
      "epoch": 0.4838584208479191,
      "grad_norm": 72.3661117553711,
      "learning_rate": 2.6267213385942262e-05,
      "logits/chosen": -0.5410548448562622,
      "logits/rejected": -1.1711819171905518,
      "logps/chosen": -1.275607943534851,
      "logps/rejected": -17.424577713012695,
      "loss": 1.2817,
      "num_input_tokens_seen": 8149296,
      "odds_ratio_loss": 11.32263469696045,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1275607943534851,
      "rewards/margins": 1.6148971319198608,
      "rewards/rejected": -1.7424579858779907,
      "sft_loss": 0.14939887821674347,
      "step": 6220
    },
    {
      "epoch": 0.48463632827693504,
      "grad_norm": 2.0818560123443604,
      "learning_rate": 2.6206191495076065e-05,
      "logits/chosen": -0.006120137870311737,
      "logits/rejected": -1.480681300163269,
      "logps/chosen": -0.987036406993866,
      "logps/rejected": -8.894469261169434,
      "loss": 0.9965,
      "num_input_tokens_seen": 8165152,
      "odds_ratio_loss": 9.909078598022461,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09870363026857376,
      "rewards/margins": 0.7907432913780212,
      "rewards/rejected": -0.8894469141960144,
      "sft_loss": 0.0056028286926448345,
      "step": 6230
    },
    {
      "epoch": 0.485414235705951,
      "grad_norm": 2.703519582748413,
      "learning_rate": 2.614516240024429e-05,
      "logits/chosen": -0.06827685236930847,
      "logits/rejected": -1.7971550226211548,
      "logps/chosen": -0.9212660789489746,
      "logps/rejected": -12.907064437866211,
      "loss": 0.9536,
      "num_input_tokens_seen": 8177776,
      "odds_ratio_loss": 9.493310928344727,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.09212660044431686,
      "rewards/margins": 1.1985799074172974,
      "rewards/rejected": -1.2907065153121948,
      "sft_loss": 0.00423536216840148,
      "step": 6240
    },
    {
      "epoch": 0.48619214313496695,
      "grad_norm": 1.8637549877166748,
      "learning_rate": 2.608412646594254e-05,
      "logits/chosen": 0.13315926492214203,
      "logits/rejected": -1.7044929265975952,
      "logps/chosen": -0.945042610168457,
      "logps/rejected": -4.411226272583008,
      "loss": 0.9641,
      "num_input_tokens_seen": 8192080,
      "odds_ratio_loss": 9.610013008117676,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.09450425952672958,
      "rewards/margins": 0.3466184139251709,
      "rewards/rejected": -0.4411226212978363,
      "sft_loss": 0.0030971444211900234,
      "step": 6250
    },
    {
      "epoch": 0.4869700505639829,
      "grad_norm": 1.8692630529403687,
      "learning_rate": 2.6023084056707258e-05,
      "logits/chosen": -0.03700752183794975,
      "logits/rejected": -1.5287142992019653,
      "logps/chosen": -0.7440109848976135,
      "logps/rejected": -6.019461154937744,
      "loss": 0.7637,
      "num_input_tokens_seen": 8204656,
      "odds_ratio_loss": 7.601499080657959,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.07440110296010971,
      "rewards/margins": 0.5275450944900513,
      "rewards/rejected": -0.6019461750984192,
      "sft_loss": 0.003576572285965085,
      "step": 6260
    },
    {
      "epoch": 0.48774795799299886,
      "grad_norm": 2.0554590225219727,
      "learning_rate": 2.596203553711359e-05,
      "logits/chosen": -0.07751911878585815,
      "logits/rejected": -1.502213478088379,
      "logps/chosen": -0.9608389735221863,
      "logps/rejected": -7.2761054039001465,
      "loss": 0.9785,
      "num_input_tokens_seen": 8216896,
      "odds_ratio_loss": 9.725781440734863,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.09608390182256699,
      "rewards/margins": 0.63152676820755,
      "rewards/rejected": -0.7276106476783752,
      "sft_loss": 0.0058743832632899284,
      "step": 6270
    },
    {
      "epoch": 0.48852586542201476,
      "grad_norm": 13.380386352539062,
      "learning_rate": 2.5900981271773133e-05,
      "logits/chosen": -0.20113714039325714,
      "logits/rejected": -1.1153767108917236,
      "logps/chosen": -0.8374971151351929,
      "logps/rejected": -4.752079010009766,
      "loss": 0.8551,
      "num_input_tokens_seen": 8232096,
      "odds_ratio_loss": 8.423782348632812,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.08374972641468048,
      "rewards/margins": 0.3914581835269928,
      "rewards/rejected": -0.4752078950405121,
      "sft_loss": 0.012685893103480339,
      "step": 6280
    },
    {
      "epoch": 0.4893037728510307,
      "grad_norm": 11.337404251098633,
      "learning_rate": 2.5839921625331842e-05,
      "logits/chosen": -0.49898165464401245,
      "logits/rejected": -0.7197166085243225,
      "logps/chosen": -2.081268310546875,
      "logps/rejected": -3.8042755126953125,
      "loss": 2.2166,
      "num_input_tokens_seen": 8241072,
      "odds_ratio_loss": 21.73521614074707,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.20812682807445526,
      "rewards/margins": 0.17230072617530823,
      "rewards/rejected": -0.3804275393486023,
      "sft_loss": 0.04303508251905441,
      "step": 6290
    },
    {
      "epoch": 0.49008168028004667,
      "grad_norm": 3.3102755546569824,
      "learning_rate": 2.5778856962467777e-05,
      "logits/chosen": -0.13400593400001526,
      "logits/rejected": -1.450063705444336,
      "logps/chosen": -0.9296320080757141,
      "logps/rejected": -4.581965446472168,
      "loss": 0.9603,
      "num_input_tokens_seen": 8249264,
      "odds_ratio_loss": 9.388015747070312,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.09296319633722305,
      "rewards/margins": 0.3652333617210388,
      "rewards/rejected": -0.4581965506076813,
      "sft_loss": 0.021517466753721237,
      "step": 6300
    },
    {
      "epoch": 0.49085958770906263,
      "grad_norm": 2.8110644817352295,
      "learning_rate": 2.571778764788898e-05,
      "logits/chosen": -0.0194260124117136,
      "logits/rejected": -1.2334833145141602,
      "logps/chosen": -0.8915126919746399,
      "logps/rejected": -6.6634955406188965,
      "loss": 0.9124,
      "num_input_tokens_seen": 8262368,
      "odds_ratio_loss": 9.059114456176758,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.08915125578641891,
      "rewards/margins": 0.5771982669830322,
      "rewards/rejected": -0.6663495302200317,
      "sft_loss": 0.00646891538053751,
      "step": 6310
    },
    {
      "epoch": 0.4916374951380786,
      "grad_norm": 4.480525493621826,
      "learning_rate": 2.5656714046331265e-05,
      "logits/chosen": -0.4957144856452942,
      "logits/rejected": -0.8260617256164551,
      "logps/chosen": -0.9069435000419617,
      "logps/rejected": -3.975454807281494,
      "loss": 0.9271,
      "num_input_tokens_seen": 8273760,
      "odds_ratio_loss": 8.90278434753418,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.0906943529844284,
      "rewards/margins": 0.3068511486053467,
      "rewards/rejected": -0.3975455164909363,
      "sft_loss": 0.03677966445684433,
      "step": 6320
    },
    {
      "epoch": 0.49241540256709454,
      "grad_norm": 1.9025019407272339,
      "learning_rate": 2.559563652255605e-05,
      "logits/chosen": -0.33826085925102234,
      "logits/rejected": -1.4704886674880981,
      "logps/chosen": -0.8480048179626465,
      "logps/rejected": -6.5116119384765625,
      "loss": 0.8607,
      "num_input_tokens_seen": 8285680,
      "odds_ratio_loss": 8.557178497314453,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.08480048179626465,
      "rewards/margins": 0.5663608312606812,
      "rewards/rejected": -0.6511613130569458,
      "sft_loss": 0.0049712504260241985,
      "step": 6330
    },
    {
      "epoch": 0.49319330999611044,
      "grad_norm": 2.8888540267944336,
      "learning_rate": 2.553455544134819e-05,
      "logits/chosen": -0.09928639233112335,
      "logits/rejected": -0.8209127187728882,
      "logps/chosen": -0.9104410409927368,
      "logps/rejected": -4.182798385620117,
      "loss": 0.9345,
      "num_input_tokens_seen": 8297488,
      "odds_ratio_loss": 9.230264663696289,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.0910440981388092,
      "rewards/margins": 0.32723572850227356,
      "rewards/rejected": -0.41827982664108276,
      "sft_loss": 0.011449308134615421,
      "step": 6340
    },
    {
      "epoch": 0.4939712174251264,
      "grad_norm": 2.2666444778442383,
      "learning_rate": 2.547347116751377e-05,
      "logits/chosen": -0.2674545645713806,
      "logits/rejected": -1.1496336460113525,
      "logps/chosen": -0.921932578086853,
      "logps/rejected": -8.103863716125488,
      "loss": 0.9277,
      "num_input_tokens_seen": 8308768,
      "odds_ratio_loss": 9.053686141967773,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09219325333833694,
      "rewards/margins": 0.7181931138038635,
      "rewards/rejected": -0.8103863596916199,
      "sft_loss": 0.022327039390802383,
      "step": 6350
    },
    {
      "epoch": 0.49474912485414235,
      "grad_norm": 7.034699440002441,
      "learning_rate": 2.5412384065877954e-05,
      "logits/chosen": -0.3206288516521454,
      "logits/rejected": -1.109317421913147,
      "logps/chosen": -0.9621008634567261,
      "logps/rejected": -7.422767639160156,
      "loss": 0.9766,
      "num_input_tokens_seen": 8321168,
      "odds_ratio_loss": 9.610101699829102,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.09621009230613708,
      "rewards/margins": 0.6460667848587036,
      "rewards/rejected": -0.7422767877578735,
      "sft_loss": 0.015624640509486198,
      "step": 6360
    },
    {
      "epoch": 0.4955270322831583,
      "grad_norm": 7.01314640045166,
      "learning_rate": 2.5351294501282785e-05,
      "logits/chosen": -0.2326393872499466,
      "logits/rejected": -0.9409756660461426,
      "logps/chosen": -1.0383962392807007,
      "logps/rejected": -15.200935363769531,
      "loss": 1.0673,
      "num_input_tokens_seen": 8337456,
      "odds_ratio_loss": 10.551141738891602,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.10383962094783783,
      "rewards/margins": 1.4162540435791016,
      "rewards/rejected": -1.5200936794281006,
      "sft_loss": 0.012215681374073029,
      "step": 6370
    },
    {
      "epoch": 0.49630493971217426,
      "grad_norm": 4.612608432769775,
      "learning_rate": 2.5290202838585036e-05,
      "logits/chosen": -0.2782391309738159,
      "logits/rejected": -0.8692216873168945,
      "logps/chosen": -1.1118863821029663,
      "logps/rejected": -4.572246551513672,
      "loss": 1.1561,
      "num_input_tokens_seen": 8347088,
      "odds_ratio_loss": 11.084630966186523,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.11118865013122559,
      "rewards/margins": 0.34603601694107056,
      "rewards/rejected": -0.45722466707229614,
      "sft_loss": 0.04765518382191658,
      "step": 6380
    },
    {
      "epoch": 0.4970828471411902,
      "grad_norm": 2.985591411590576,
      "learning_rate": 2.522910944265399e-05,
      "logits/chosen": -0.1539156287908554,
      "logits/rejected": -1.1583564281463623,
      "logps/chosen": -0.8345049619674683,
      "logps/rejected": -7.536215305328369,
      "loss": 0.8428,
      "num_input_tokens_seen": 8361040,
      "odds_ratio_loss": 8.301528930664062,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08345050364732742,
      "rewards/margins": 0.6701710820198059,
      "rewards/rejected": -0.7536215782165527,
      "sft_loss": 0.012616393156349659,
      "step": 6390
    },
    {
      "epoch": 0.4978607545702061,
      "grad_norm": 3.878408670425415,
      "learning_rate": 2.516801467836929e-05,
      "logits/chosen": -0.3584039807319641,
      "logits/rejected": -1.179455041885376,
      "logps/chosen": -0.9978312253952026,
      "logps/rejected": -10.517958641052246,
      "loss": 1.0255,
      "num_input_tokens_seen": 8369088,
      "odds_ratio_loss": 9.225652694702148,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.09978312253952026,
      "rewards/margins": 0.9520127177238464,
      "rewards/rejected": -1.0517958402633667,
      "sft_loss": 0.10292162746191025,
      "step": 6400
    },
    {
      "epoch": 0.4986386619992221,
      "grad_norm": 2.9756100177764893,
      "learning_rate": 2.510691891061875e-05,
      "logits/chosen": 0.055911578238010406,
      "logits/rejected": -1.6097557544708252,
      "logps/chosen": -1.0666486024856567,
      "logps/rejected": -9.044986724853516,
      "loss": 1.0829,
      "num_input_tokens_seen": 8378512,
      "odds_ratio_loss": 10.744181632995605,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.10666485875844955,
      "rewards/margins": 0.7978337407112122,
      "rewards/rejected": -0.9044985771179199,
      "sft_loss": 0.008470003493130207,
      "step": 6410
    },
    {
      "epoch": 0.49941656942823803,
      "grad_norm": 2.4883270263671875,
      "learning_rate": 2.504582250429618e-05,
      "logits/chosen": -0.19540779292583466,
      "logits/rejected": -0.9103869199752808,
      "logps/chosen": -0.8133610486984253,
      "logps/rejected": -8.337348937988281,
      "loss": 0.8362,
      "num_input_tokens_seen": 8393488,
      "odds_ratio_loss": 8.27368450164795,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.08133610337972641,
      "rewards/margins": 0.7523988485336304,
      "rewards/rejected": -0.8337349891662598,
      "sft_loss": 0.008857587352395058,
      "step": 6420
    },
    {
      "epoch": 0.5001944768572539,
      "grad_norm": 2.813175678253174,
      "learning_rate": 2.498472582429921e-05,
      "logits/chosen": -0.30721643567085266,
      "logits/rejected": -1.406435251235962,
      "logps/chosen": -0.8761197924613953,
      "logps/rejected": -6.059558868408203,
      "loss": 0.899,
      "num_input_tokens_seen": 8407136,
      "odds_ratio_loss": 8.8612699508667,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.08761197328567505,
      "rewards/margins": 0.5183438658714294,
      "rewards/rejected": -0.6059558391571045,
      "sft_loss": 0.012886581011116505,
      "step": 6430
    },
    {
      "epoch": 0.5009723842862699,
      "grad_norm": 1.4266654253005981,
      "learning_rate": 2.4923629235527078e-05,
      "logits/chosen": -0.1768713891506195,
      "logits/rejected": -1.1466429233551025,
      "logps/chosen": -0.9270246624946594,
      "logps/rejected": -6.692111968994141,
      "loss": 0.9596,
      "num_input_tokens_seen": 8425696,
      "odds_ratio_loss": 9.194589614868164,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.0927024707198143,
      "rewards/margins": 0.5765087008476257,
      "rewards/rejected": -0.669211208820343,
      "sft_loss": 0.0400998480618,
      "step": 6440
    },
    {
      "epoch": 0.5017502917152858,
      "grad_norm": 2.516162395477295,
      "learning_rate": 2.486253310287851e-05,
      "logits/chosen": -0.2516472041606903,
      "logits/rejected": -1.1252797842025757,
      "logps/chosen": -1.5495781898498535,
      "logps/rejected": -10.633633613586426,
      "loss": 1.5676,
      "num_input_tokens_seen": 8439280,
      "odds_ratio_loss": 15.6168212890625,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.1549578160047531,
      "rewards/margins": 0.908405601978302,
      "rewards/rejected": -1.0633634328842163,
      "sft_loss": 0.005875526927411556,
      "step": 6450
    },
    {
      "epoch": 0.5025281991443018,
      "grad_norm": 2.324152946472168,
      "learning_rate": 2.4801437791249482e-05,
      "logits/chosen": -0.0015714972978457808,
      "logits/rejected": -1.7600247859954834,
      "logps/chosen": -0.7816251516342163,
      "logps/rejected": -7.274757385253906,
      "loss": 0.7902,
      "num_input_tokens_seen": 8456080,
      "odds_ratio_loss": 7.856690883636475,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.07816250622272491,
      "rewards/margins": 0.64931321144104,
      "rewards/rejected": -0.7274757027626038,
      "sft_loss": 0.004567933268845081,
      "step": 6460
    },
    {
      "epoch": 0.5033061065733178,
      "grad_norm": 3.4712696075439453,
      "learning_rate": 2.4740343665531092e-05,
      "logits/chosen": -0.2848728597164154,
      "logits/rejected": -1.1505284309387207,
      "logps/chosen": -0.8179367184638977,
      "logps/rejected": -6.20058012008667,
      "loss": 0.8267,
      "num_input_tokens_seen": 8468992,
      "odds_ratio_loss": 8.157306671142578,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08179367333650589,
      "rewards/margins": 0.5382643342018127,
      "rewards/rejected": -0.6200579404830933,
      "sft_loss": 0.0109955919906497,
      "step": 6470
    },
    {
      "epoch": 0.5040840140023337,
      "grad_norm": 3.0974578857421875,
      "learning_rate": 2.4679251090607333e-05,
      "logits/chosen": -0.22295601665973663,
      "logits/rejected": -1.3508847951889038,
      "logps/chosen": -0.932137668132782,
      "logps/rejected": -8.735374450683594,
      "loss": 0.9399,
      "num_input_tokens_seen": 8481024,
      "odds_ratio_loss": 9.306537628173828,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.093213751912117,
      "rewards/margins": 0.7803236246109009,
      "rewards/rejected": -0.8735373616218567,
      "sft_loss": 0.009225866757333279,
      "step": 6480
    },
    {
      "epoch": 0.5048619214313497,
      "grad_norm": 5.221313953399658,
      "learning_rate": 2.4618160431352946e-05,
      "logits/chosen": 0.10161104053258896,
      "logits/rejected": -1.5568485260009766,
      "logps/chosen": -0.8982173204421997,
      "logps/rejected": -7.640667915344238,
      "loss": 0.9011,
      "num_input_tokens_seen": 8492816,
      "odds_ratio_loss": 8.938986778259277,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.0898217260837555,
      "rewards/margins": 0.6742449998855591,
      "rewards/rejected": -0.764066755771637,
      "sft_loss": 0.007185804657638073,
      "step": 6490
    },
    {
      "epoch": 0.5056398288603656,
      "grad_norm": 1.990058183670044,
      "learning_rate": 2.455707205263123e-05,
      "logits/chosen": -0.3942679166793823,
      "logits/rejected": -0.9469472765922546,
      "logps/chosen": -1.034597635269165,
      "logps/rejected": -9.053838729858398,
      "loss": 1.0465,
      "num_input_tokens_seen": 8509136,
      "odds_ratio_loss": 10.092089653015137,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.10345976054668427,
      "rewards/margins": 0.8019241094589233,
      "rewards/rejected": -0.9053838849067688,
      "sft_loss": 0.037330999970436096,
      "step": 6500
    },
    {
      "epoch": 0.5064177362893816,
      "grad_norm": 2.9650418758392334,
      "learning_rate": 2.4495986319291857e-05,
      "logits/chosen": -0.54425448179245,
      "logits/rejected": -0.8309041261672974,
      "logps/chosen": -0.9747613668441772,
      "logps/rejected": -16.209012985229492,
      "loss": 0.9967,
      "num_input_tokens_seen": 8522096,
      "odds_ratio_loss": 9.86829948425293,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.09747613966464996,
      "rewards/margins": 1.5234253406524658,
      "rewards/rejected": -1.6209014654159546,
      "sft_loss": 0.009864835068583488,
      "step": 6510
    },
    {
      "epoch": 0.5071956437183975,
      "grad_norm": 2.1551918983459473,
      "learning_rate": 2.443490359616871e-05,
      "logits/chosen": -0.22963781654834747,
      "logits/rejected": -0.9503774642944336,
      "logps/chosen": -0.8328534364700317,
      "logps/rejected": -11.546224594116211,
      "loss": 0.838,
      "num_input_tokens_seen": 8538672,
      "odds_ratio_loss": 7.670839786529541,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08328534662723541,
      "rewards/margins": 1.071337103843689,
      "rewards/rejected": -1.1546225547790527,
      "sft_loss": 0.07093828916549683,
      "step": 6520
    },
    {
      "epoch": 0.5079735511474135,
      "grad_norm": 3.307870864868164,
      "learning_rate": 2.437382424807769e-05,
      "logits/chosen": -0.43049168586730957,
      "logits/rejected": -1.3492554426193237,
      "logps/chosen": -1.0302252769470215,
      "logps/rejected": -15.203390121459961,
      "loss": 1.0419,
      "num_input_tokens_seen": 8550192,
      "odds_ratio_loss": 10.329671859741211,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.10302253067493439,
      "rewards/margins": 1.4173165559768677,
      "rewards/rejected": -1.520339012145996,
      "sft_loss": 0.00895792618393898,
      "step": 6530
    },
    {
      "epoch": 0.5087514585764294,
      "grad_norm": 4.686474323272705,
      "learning_rate": 2.431274863981453e-05,
      "logits/chosen": -0.265191912651062,
      "logits/rejected": -0.9243543744087219,
      "logps/chosen": -0.8716975450515747,
      "logps/rejected": -12.998971939086914,
      "loss": 0.8795,
      "num_input_tokens_seen": 8561584,
      "odds_ratio_loss": 8.608287811279297,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08716975152492523,
      "rewards/margins": 1.212727427482605,
      "rewards/rejected": -1.2998971939086914,
      "sft_loss": 0.0186309851706028,
      "step": 6540
    },
    {
      "epoch": 0.5095293660054454,
      "grad_norm": 4.554167747497559,
      "learning_rate": 2.4251677136152642e-05,
      "logits/chosen": -0.5132981538772583,
      "logits/rejected": -1.1222872734069824,
      "logps/chosen": -0.8228030204772949,
      "logps/rejected": -17.707448959350586,
      "loss": 0.823,
      "num_input_tokens_seen": 8570304,
      "odds_ratio_loss": 8.138057708740234,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08228030055761337,
      "rewards/margins": 1.688464879989624,
      "rewards/rejected": -1.770745038986206,
      "sft_loss": 0.009233796037733555,
      "step": 6550
    },
    {
      "epoch": 0.5103072734344613,
      "grad_norm": 2.7331156730651855,
      "learning_rate": 2.419061010184093e-05,
      "logits/chosen": -0.09426519274711609,
      "logits/rejected": -1.7221357822418213,
      "logps/chosen": -0.9415035247802734,
      "logps/rejected": -3.5511512756347656,
      "loss": 0.976,
      "num_input_tokens_seen": 8580016,
      "odds_ratio_loss": 9.654105186462402,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.0941503494977951,
      "rewards/margins": 0.2609647810459137,
      "rewards/rejected": -0.3551151156425476,
      "sft_loss": 0.010551346465945244,
      "step": 6560
    },
    {
      "epoch": 0.5110851808634772,
      "grad_norm": 4.815445423126221,
      "learning_rate": 2.412954790160157e-05,
      "logits/chosen": -0.09299831092357635,
      "logits/rejected": -1.0223968029022217,
      "logps/chosen": -0.6498953104019165,
      "logps/rejected": -7.632620334625244,
      "loss": 0.6732,
      "num_input_tokens_seen": 8592208,
      "odds_ratio_loss": 6.660180568695068,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.06498952209949493,
      "rewards/margins": 0.6982723474502563,
      "rewards/rejected": -0.7632620334625244,
      "sft_loss": 0.007177567575126886,
      "step": 6570
    },
    {
      "epoch": 0.5118630882924932,
      "grad_norm": 29.947223663330078,
      "learning_rate": 2.4068490900127894e-05,
      "logits/chosen": -0.11313676834106445,
      "logits/rejected": -1.2990573644638062,
      "logps/chosen": -1.1999033689498901,
      "logps/rejected": -9.722343444824219,
      "loss": 1.2255,
      "num_input_tokens_seen": 8603552,
      "odds_ratio_loss": 11.905380249023438,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.11999032646417618,
      "rewards/margins": 0.8522440195083618,
      "rewards/rejected": -0.9722343683242798,
      "sft_loss": 0.03497845306992531,
      "step": 6580
    },
    {
      "epoch": 0.5126409957215091,
      "grad_norm": 3.5857746601104736,
      "learning_rate": 2.4007439462082175e-05,
      "logits/chosen": -0.06995495408773422,
      "logits/rejected": -1.8384273052215576,
      "logps/chosen": -0.9535351991653442,
      "logps/rejected": -23.40558624267578,
      "loss": 0.9537,
      "num_input_tokens_seen": 8620960,
      "odds_ratio_loss": 9.443254470825195,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09535352140665054,
      "rewards/margins": 2.2452054023742676,
      "rewards/rejected": -2.3405587673187256,
      "sft_loss": 0.00940064899623394,
      "step": 6590
    },
    {
      "epoch": 0.5134189031505251,
      "grad_norm": 2.141690731048584,
      "learning_rate": 2.394639395209348e-05,
      "logits/chosen": -0.028879564255475998,
      "logits/rejected": -1.3578704595565796,
      "logps/chosen": -0.9493895769119263,
      "logps/rejected": -11.105527877807617,
      "loss": 0.9641,
      "num_input_tokens_seen": 8630704,
      "odds_ratio_loss": 9.563076972961426,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09493895620107651,
      "rewards/margins": 1.0156137943267822,
      "rewards/rejected": -1.1105527877807617,
      "sft_loss": 0.007841681130230427,
      "step": 6600
    },
    {
      "epoch": 0.514196810579541,
      "grad_norm": 2.8887338638305664,
      "learning_rate": 2.388535473475544e-05,
      "logits/chosen": 0.045301564037799835,
      "logits/rejected": -1.2772947549819946,
      "logps/chosen": -0.8695799708366394,
      "logps/rejected": -12.75346565246582,
      "loss": 0.9,
      "num_input_tokens_seen": 8645168,
      "odds_ratio_loss": 8.117189407348633,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.08695799112319946,
      "rewards/margins": 1.188388466835022,
      "rewards/rejected": -1.2753465175628662,
      "sft_loss": 0.08828867971897125,
      "step": 6610
    },
    {
      "epoch": 0.514974718008557,
      "grad_norm": 3.8750803470611572,
      "learning_rate": 2.382432217462412e-05,
      "logits/chosen": -0.0706484317779541,
      "logits/rejected": -1.0864431858062744,
      "logps/chosen": -0.6691504716873169,
      "logps/rejected": -10.930901527404785,
      "loss": 0.679,
      "num_input_tokens_seen": 8659424,
      "odds_ratio_loss": 6.66359806060791,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.06691503524780273,
      "rewards/margins": 1.0261751413345337,
      "rewards/rejected": -1.0930901765823364,
      "sft_loss": 0.012643078342080116,
      "step": 6620
    },
    {
      "epoch": 0.5157526254375729,
      "grad_norm": 1.7773513793945312,
      "learning_rate": 2.3763296636215824e-05,
      "logits/chosen": -0.02193978801369667,
      "logits/rejected": -1.34123957157135,
      "logps/chosen": -0.9752591252326965,
      "logps/rejected": -7.491532325744629,
      "loss": 0.9884,
      "num_input_tokens_seen": 8671056,
      "odds_ratio_loss": 9.739108085632324,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09752590954303741,
      "rewards/margins": 0.6516273021697998,
      "rewards/rejected": -0.749153196811676,
      "sft_loss": 0.014478230848908424,
      "step": 6630
    },
    {
      "epoch": 0.5165305328665889,
      "grad_norm": 5.180094242095947,
      "learning_rate": 2.3702278484004907e-05,
      "logits/chosen": -0.1238371953368187,
      "logits/rejected": -1.0661619901657104,
      "logps/chosen": -2.1213927268981934,
      "logps/rejected": -16.54400062561035,
      "loss": 2.1394,
      "num_input_tokens_seen": 8683952,
      "odds_ratio_loss": 21.279338836669922,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.212139293551445,
      "rewards/margins": 1.442260980606079,
      "rewards/rejected": -1.6544002294540405,
      "sft_loss": 0.011500345543026924,
      "step": 6640
    },
    {
      "epoch": 0.5173084402956049,
      "grad_norm": 4.295303821563721,
      "learning_rate": 2.364126808242163e-05,
      "logits/chosen": -0.8342688679695129,
      "logits/rejected": -0.7375487089157104,
      "logps/chosen": -0.8082100749015808,
      "logps/rejected": -7.813617706298828,
      "loss": 0.8247,
      "num_input_tokens_seen": 8696416,
      "odds_ratio_loss": 8.126741409301758,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.08082100749015808,
      "rewards/margins": 0.7005407214164734,
      "rewards/rejected": -0.7813617587089539,
      "sft_loss": 0.012039626948535442,
      "step": 6650
    },
    {
      "epoch": 0.5180863477246208,
      "grad_norm": 3.939998149871826,
      "learning_rate": 2.3580265795849944e-05,
      "logits/chosen": -0.10337956249713898,
      "logits/rejected": -1.3135967254638672,
      "logps/chosen": -1.0097706317901611,
      "logps/rejected": -9.967623710632324,
      "loss": 1.0107,
      "num_input_tokens_seen": 8707424,
      "odds_ratio_loss": 10.029942512512207,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.10097707808017731,
      "rewards/margins": 0.8957854509353638,
      "rewards/rejected": -0.9967623949050903,
      "sft_loss": 0.00767454644665122,
      "step": 6660
    },
    {
      "epoch": 0.5188642551536368,
      "grad_norm": 2.4933345317840576,
      "learning_rate": 2.3519271988625344e-05,
      "logits/chosen": -0.10118494182825089,
      "logits/rejected": -1.1729929447174072,
      "logps/chosen": -0.7244973182678223,
      "logps/rejected": -13.689013481140137,
      "loss": 0.7268,
      "num_input_tokens_seen": 8721984,
      "odds_ratio_loss": 7.2344560623168945,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.07244972884654999,
      "rewards/margins": 1.2964515686035156,
      "rewards/rejected": -1.368901252746582,
      "sft_loss": 0.003336378838866949,
      "step": 6670
    },
    {
      "epoch": 0.5196421625826526,
      "grad_norm": 2.1454243659973145,
      "learning_rate": 2.3458287025032697e-05,
      "logits/chosen": -0.136545330286026,
      "logits/rejected": -0.8210135698318481,
      "logps/chosen": -0.8043525815010071,
      "logps/rejected": -7.613142967224121,
      "loss": 0.8181,
      "num_input_tokens_seen": 8730688,
      "odds_ratio_loss": 8.056388854980469,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.08043525367975235,
      "rewards/margins": 0.6808789968490601,
      "rewards/rejected": -0.7613142132759094,
      "sft_loss": 0.012427089735865593,
      "step": 6680
    },
    {
      "epoch": 0.5204200700116686,
      "grad_norm": 3.140146017074585,
      "learning_rate": 2.3397311269304006e-05,
      "logits/chosen": -0.10605098307132721,
      "logits/rejected": -1.4007622003555298,
      "logps/chosen": -0.9359437823295593,
      "logps/rejected": -8.159541130065918,
      "loss": 0.9587,
      "num_input_tokens_seen": 8744160,
      "odds_ratio_loss": 9.49116325378418,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.09359437227249146,
      "rewards/margins": 0.7223597764968872,
      "rewards/rejected": -0.8159540891647339,
      "sft_loss": 0.009535567834973335,
      "step": 6690
    },
    {
      "epoch": 0.5211979774406845,
      "grad_norm": 3.360002279281616,
      "learning_rate": 2.3336345085616324e-05,
      "logits/chosen": 0.030445415526628494,
      "logits/rejected": -1.2207574844360352,
      "logps/chosen": -0.8218139410018921,
      "logps/rejected": -9.092556953430176,
      "loss": 0.8714,
      "num_input_tokens_seen": 8756560,
      "odds_ratio_loss": 8.25312614440918,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.0821814090013504,
      "rewards/margins": 0.8270743489265442,
      "rewards/rejected": -0.9092556834220886,
      "sft_loss": 0.04608220234513283,
      "step": 6700
    },
    {
      "epoch": 0.5219758848697005,
      "grad_norm": 8.89459228515625,
      "learning_rate": 2.3275388838089517e-05,
      "logits/chosen": -0.21345002949237823,
      "logits/rejected": -1.5567723512649536,
      "logps/chosen": -1.1306450366973877,
      "logps/rejected": -20.125442504882812,
      "loss": 1.1347,
      "num_input_tokens_seen": 8767408,
      "odds_ratio_loss": 11.194887161254883,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.11306451261043549,
      "rewards/margins": 1.899479627609253,
      "rewards/rejected": -2.0125441551208496,
      "sft_loss": 0.015241652727127075,
      "step": 6710
    },
    {
      "epoch": 0.5227537922987164,
      "grad_norm": 2.6667776107788086,
      "learning_rate": 2.3214442890784107e-05,
      "logits/chosen": 0.04326433688402176,
      "logits/rejected": -1.349846363067627,
      "logps/chosen": -0.8453485369682312,
      "logps/rejected": -11.233430862426758,
      "loss": 0.8482,
      "num_input_tokens_seen": 8785664,
      "odds_ratio_loss": 8.441164016723633,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08453484624624252,
      "rewards/margins": 1.0388084650039673,
      "rewards/rejected": -1.1233432292938232,
      "sft_loss": 0.004091930575668812,
      "step": 6720
    },
    {
      "epoch": 0.5235316997277324,
      "grad_norm": 2.6269643306732178,
      "learning_rate": 2.315350760769911e-05,
      "logits/chosen": 0.026680314913392067,
      "logits/rejected": -1.3829847574234009,
      "logps/chosen": -0.7016235589981079,
      "logps/rejected": -21.766069412231445,
      "loss": 0.7021,
      "num_input_tokens_seen": 8795008,
      "odds_ratio_loss": 6.94488000869751,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.07016235589981079,
      "rewards/margins": 2.1064445972442627,
      "rewards/rejected": -2.1766068935394287,
      "sft_loss": 0.007626746781170368,
      "step": 6730
    },
    {
      "epoch": 0.5243096071567483,
      "grad_norm": 3.123906373977661,
      "learning_rate": 2.3092583352769832e-05,
      "logits/chosen": -0.20880265533924103,
      "logits/rejected": -1.134286642074585,
      "logps/chosen": -1.1254082918167114,
      "logps/rejected": -16.223125457763672,
      "loss": 1.1258,
      "num_input_tokens_seen": 8807312,
      "odds_ratio_loss": 11.091835975646973,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1125408411026001,
      "rewards/margins": 1.5097719430923462,
      "rewards/rejected": -1.6223127841949463,
      "sft_loss": 0.016620712354779243,
      "step": 6740
    },
    {
      "epoch": 0.5250875145857643,
      "grad_norm": 2.3707122802734375,
      "learning_rate": 2.303167048986573e-05,
      "logits/chosen": -0.48967304825782776,
      "logits/rejected": -1.3673675060272217,
      "logps/chosen": -1.1683456897735596,
      "logps/rejected": -14.066203117370605,
      "loss": 1.1774,
      "num_input_tokens_seen": 8821056,
      "odds_ratio_loss": 10.931964874267578,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.11683456599712372,
      "rewards/margins": 1.2897858619689941,
      "rewards/rejected": -1.4066202640533447,
      "sft_loss": 0.08424625545740128,
      "step": 6750
    },
    {
      "epoch": 0.5258654220147803,
      "grad_norm": 1.509379506111145,
      "learning_rate": 2.2970769382788215e-05,
      "logits/chosen": -0.026856541633605957,
      "logits/rejected": -1.7084585428237915,
      "logps/chosen": -1.0115071535110474,
      "logps/rejected": -9.703714370727539,
      "loss": 1.0464,
      "num_input_tokens_seen": 8831952,
      "odds_ratio_loss": 10.413983345031738,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.10115071386098862,
      "rewards/margins": 0.8692208528518677,
      "rewards/rejected": -0.970371425151825,
      "sft_loss": 0.004966267850250006,
      "step": 6760
    },
    {
      "epoch": 0.5266433294437962,
      "grad_norm": 6.818657398223877,
      "learning_rate": 2.2909880395268478e-05,
      "logits/chosen": -0.456535279750824,
      "logits/rejected": -0.9776288866996765,
      "logps/chosen": -0.6454017162322998,
      "logps/rejected": -11.684601783752441,
      "loss": 0.6572,
      "num_input_tokens_seen": 8844432,
      "odds_ratio_loss": 6.454319000244141,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.06454017758369446,
      "rewards/margins": 1.1039199829101562,
      "rewards/rejected": -1.1684601306915283,
      "sft_loss": 0.011763582937419415,
      "step": 6770
    },
    {
      "epoch": 0.5274212368728122,
      "grad_norm": 1.824180245399475,
      "learning_rate": 2.2849003890965347e-05,
      "logits/chosen": 0.14084644615650177,
      "logits/rejected": -1.8677068948745728,
      "logps/chosen": -0.9107322692871094,
      "logps/rejected": -9.742560386657715,
      "loss": 0.9214,
      "num_input_tokens_seen": 8855632,
      "odds_ratio_loss": 9.149304389953613,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09107323735952377,
      "rewards/margins": 0.8831828236579895,
      "rewards/rejected": -0.974256157875061,
      "sft_loss": 0.0064893984235823154,
      "step": 6780
    },
    {
      "epoch": 0.5281991443018281,
      "grad_norm": 3.3605880737304688,
      "learning_rate": 2.278814023346307e-05,
      "logits/chosen": -0.04147498309612274,
      "logits/rejected": -1.2981659173965454,
      "logps/chosen": -1.005117654800415,
      "logps/rejected": -12.193399429321289,
      "loss": 1.0254,
      "num_input_tokens_seen": 8867264,
      "odds_ratio_loss": 10.19458293914795,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.10051177442073822,
      "rewards/margins": 1.118828296661377,
      "rewards/rejected": -1.2193399667739868,
      "sft_loss": 0.005910595878958702,
      "step": 6790
    },
    {
      "epoch": 0.5289770517308441,
      "grad_norm": 1.5352081060409546,
      "learning_rate": 2.2727289786269177e-05,
      "logits/chosen": 0.0757211297750473,
      "logits/rejected": -1.747073769569397,
      "logps/chosen": -0.9647790193557739,
      "logps/rejected": -19.082246780395508,
      "loss": 0.9748,
      "num_input_tokens_seen": 8879312,
      "odds_ratio_loss": 9.69461727142334,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09647790342569351,
      "rewards/margins": 1.8117469549179077,
      "rewards/rejected": -1.9082248210906982,
      "sft_loss": 0.005362776108086109,
      "step": 6800
    },
    {
      "epoch": 0.5297549591598599,
      "grad_norm": 35.42118453979492,
      "learning_rate": 2.2666452912812326e-05,
      "logits/chosen": -0.15618260204792023,
      "logits/rejected": -0.8608878254890442,
      "logps/chosen": -0.9781033396720886,
      "logps/rejected": -8.846330642700195,
      "loss": 0.9884,
      "num_input_tokens_seen": 8889840,
      "odds_ratio_loss": 9.386763572692871,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09781033545732498,
      "rewards/margins": 0.7868226766586304,
      "rewards/rejected": -0.8846330642700195,
      "sft_loss": 0.04968716949224472,
      "step": 6810
    },
    {
      "epoch": 0.5305328665888759,
      "grad_norm": 1.5546804666519165,
      "learning_rate": 2.2605629976440054e-05,
      "logits/chosen": -0.008410501293838024,
      "logits/rejected": -1.3464069366455078,
      "logps/chosen": -1.0442028045654297,
      "logps/rejected": -10.094877243041992,
      "loss": 1.0548,
      "num_input_tokens_seen": 8900288,
      "odds_ratio_loss": 10.442464828491211,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.10442028194665909,
      "rewards/margins": 0.9050676226615906,
      "rewards/rejected": -1.0094878673553467,
      "sft_loss": 0.01057764608412981,
      "step": 6820
    },
    {
      "epoch": 0.5313107740178918,
      "grad_norm": 5.027902603149414,
      "learning_rate": 2.2544821340416696e-05,
      "logits/chosen": -0.05764826387166977,
      "logits/rejected": -1.2585020065307617,
      "logps/chosen": -0.7766581773757935,
      "logps/rejected": -2.905539035797119,
      "loss": 0.8124,
      "num_input_tokens_seen": 8913792,
      "odds_ratio_loss": 7.814454078674316,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.07766580581665039,
      "rewards/margins": 0.21288807690143585,
      "rewards/rejected": -0.29055389761924744,
      "sft_loss": 0.030926724895834923,
      "step": 6830
    },
    {
      "epoch": 0.5320886814469078,
      "grad_norm": 1.7245932817459106,
      "learning_rate": 2.2484027367921172e-05,
      "logits/chosen": -0.09873903542757034,
      "logits/rejected": -0.891947865486145,
      "logps/chosen": -0.8979700803756714,
      "logps/rejected": -4.629054069519043,
      "loss": 0.9119,
      "num_input_tokens_seen": 8924320,
      "odds_ratio_loss": 8.985692977905273,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.0897970125079155,
      "rewards/margins": 0.3731083571910858,
      "rewards/rejected": -0.4629054069519043,
      "sft_loss": 0.013306763954460621,
      "step": 6840
    },
    {
      "epoch": 0.5328665888759238,
      "grad_norm": 3.8417139053344727,
      "learning_rate": 2.2423248422044812e-05,
      "logits/chosen": 0.05378826707601547,
      "logits/rejected": -1.0258357524871826,
      "logps/chosen": -0.8549526929855347,
      "logps/rejected": -4.404053211212158,
      "loss": 0.8929,
      "num_input_tokens_seen": 8933520,
      "odds_ratio_loss": 8.802249908447266,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.08549527078866959,
      "rewards/margins": 0.3549100458621979,
      "rewards/rejected": -0.44040530920028687,
      "sft_loss": 0.012637937441468239,
      "step": 6850
    },
    {
      "epoch": 0.5336444963049397,
      "grad_norm": 22.083005905151367,
      "learning_rate": 2.2362484865789222e-05,
      "logits/chosen": -0.08559930324554443,
      "logits/rejected": -1.1315510272979736,
      "logps/chosen": -0.810233473777771,
      "logps/rejected": -7.210709571838379,
      "loss": 0.8283,
      "num_input_tokens_seen": 8947824,
      "odds_ratio_loss": 8.138298034667969,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.08102335035800934,
      "rewards/margins": 0.6400476098060608,
      "rewards/rejected": -0.7210709452629089,
      "sft_loss": 0.0144663006067276,
      "step": 6860
    },
    {
      "epoch": 0.5344224037339557,
      "grad_norm": 1.6675682067871094,
      "learning_rate": 2.230173706206408e-05,
      "logits/chosen": -0.026315787807106972,
      "logits/rejected": -1.3517112731933594,
      "logps/chosen": -0.9361854791641235,
      "logps/rejected": -12.261492729187012,
      "loss": 0.9492,
      "num_input_tokens_seen": 8963552,
      "odds_ratio_loss": 9.391185760498047,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09361854940652847,
      "rewards/margins": 1.132530689239502,
      "rewards/rejected": -1.226149320602417,
      "sft_loss": 0.010050268843770027,
      "step": 6870
    },
    {
      "epoch": 0.5352003111629716,
      "grad_norm": 88.71906280517578,
      "learning_rate": 2.2241005373684973e-05,
      "logits/chosen": 0.005768129136413336,
      "logits/rejected": -1.5856225490570068,
      "logps/chosen": -1.0896073579788208,
      "logps/rejected": -15.783429145812988,
      "loss": 1.0958,
      "num_input_tokens_seen": 8973680,
      "odds_ratio_loss": 10.530699729919434,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.10896072536706924,
      "rewards/margins": 1.4693820476531982,
      "rewards/rejected": -1.5783427953720093,
      "sft_loss": 0.04275606572628021,
      "step": 6880
    },
    {
      "epoch": 0.5359782185919876,
      "grad_norm": 1.807417392730713,
      "learning_rate": 2.2180290163371254e-05,
      "logits/chosen": -0.3520210087299347,
      "logits/rejected": -0.8492413759231567,
      "logps/chosen": -0.7934950590133667,
      "logps/rejected": -12.94706916809082,
      "loss": 0.8035,
      "num_input_tokens_seen": 8986320,
      "odds_ratio_loss": 7.927926540374756,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.07934950292110443,
      "rewards/margins": 1.2153574228286743,
      "rewards/rejected": -1.29470694065094,
      "sft_loss": 0.010721179656684399,
      "step": 6890
    },
    {
      "epoch": 0.5367561260210035,
      "grad_norm": 4.75670051574707,
      "learning_rate": 2.2119591793743858e-05,
      "logits/chosen": -0.14193794131278992,
      "logits/rejected": -0.600124716758728,
      "logps/chosen": -0.9928326606750488,
      "logps/rejected": -6.36657190322876,
      "loss": 1.0727,
      "num_input_tokens_seen": 8997776,
      "odds_ratio_loss": 10.262323379516602,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.09928327053785324,
      "rewards/margins": 0.5373739004135132,
      "rewards/rejected": -0.636657178401947,
      "sft_loss": 0.046507202088832855,
      "step": 6900
    },
    {
      "epoch": 0.5375340334500195,
      "grad_norm": 2.5006494522094727,
      "learning_rate": 2.205891062732313e-05,
      "logits/chosen": -0.08175057172775269,
      "logits/rejected": -1.2389705181121826,
      "logps/chosen": -0.8751962780952454,
      "logps/rejected": -6.338713645935059,
      "loss": 0.9051,
      "num_input_tokens_seen": 9008288,
      "odds_ratio_loss": 8.97519588470459,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.08751963078975677,
      "rewards/margins": 0.5463517904281616,
      "rewards/rejected": -0.6338714361190796,
      "sft_loss": 0.007537661585956812,
      "step": 6910
    },
    {
      "epoch": 0.5383119408790354,
      "grad_norm": 1.7010674476623535,
      "learning_rate": 2.1998247026526687e-05,
      "logits/chosen": -0.10393454879522324,
      "logits/rejected": -1.0789237022399902,
      "logps/chosen": -0.9110797047615051,
      "logps/rejected": -10.882912635803223,
      "loss": 0.9325,
      "num_input_tokens_seen": 9024400,
      "odds_ratio_loss": 9.21080207824707,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.09110797941684723,
      "rewards/margins": 0.9971832036972046,
      "rewards/rejected": -1.0882911682128906,
      "sft_loss": 0.011449300684034824,
      "step": 6920
    },
    {
      "epoch": 0.5390898483080513,
      "grad_norm": 4.3742451667785645,
      "learning_rate": 2.1937601353667224e-05,
      "logits/chosen": -0.06702017784118652,
      "logits/rejected": -1.0037449598312378,
      "logps/chosen": -1.159448266029358,
      "logps/rejected": -8.16427993774414,
      "loss": 1.2041,
      "num_input_tokens_seen": 9034432,
      "odds_ratio_loss": 11.199224472045898,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.11594481766223907,
      "rewards/margins": 0.7004832029342651,
      "rewards/rejected": -0.8164280652999878,
      "sft_loss": 0.08420418202877045,
      "step": 6930
    },
    {
      "epoch": 0.5398677557370672,
      "grad_norm": 5.265347003936768,
      "learning_rate": 2.187697397095035e-05,
      "logits/chosen": -0.058852922171354294,
      "logits/rejected": -1.5332764387130737,
      "logps/chosen": -0.9087274670600891,
      "logps/rejected": -7.820371150970459,
      "loss": 0.9098,
      "num_input_tokens_seen": 9046656,
      "odds_ratio_loss": 9.041991233825684,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09087274968624115,
      "rewards/margins": 0.6911643743515015,
      "rewards/rejected": -0.782037079334259,
      "sft_loss": 0.005645034834742546,
      "step": 6940
    },
    {
      "epoch": 0.5406456631660832,
      "grad_norm": 16.496196746826172,
      "learning_rate": 2.1816365240472458e-05,
      "logits/chosen": -0.05866460129618645,
      "logits/rejected": -1.5211254358291626,
      "logps/chosen": -0.8906657099723816,
      "logps/rejected": -15.501606941223145,
      "loss": 0.9026,
      "num_input_tokens_seen": 9061104,
      "odds_ratio_loss": 8.945243835449219,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.08906657993793488,
      "rewards/margins": 1.4610940217971802,
      "rewards/rejected": -1.5501604080200195,
      "sft_loss": 0.008040891028940678,
      "step": 6950
    },
    {
      "epoch": 0.5414235705950992,
      "grad_norm": 19.269718170166016,
      "learning_rate": 2.175577552421853e-05,
      "logits/chosen": 0.027465347200632095,
      "logits/rejected": -1.0681571960449219,
      "logps/chosen": -1.0010095834732056,
      "logps/rejected": -6.01483154296875,
      "loss": 1.0318,
      "num_input_tokens_seen": 9075488,
      "odds_ratio_loss": 10.191933631896973,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.10010097175836563,
      "rewards/margins": 0.5013822317123413,
      "rewards/rejected": -0.601483166217804,
      "sft_loss": 0.012642273679375648,
      "step": 6960
    },
    {
      "epoch": 0.5422014780241151,
      "grad_norm": 5.147210121154785,
      "learning_rate": 2.1695205184059987e-05,
      "logits/chosen": -0.010642116889357567,
      "logits/rejected": -1.368715763092041,
      "logps/chosen": -0.741737961769104,
      "logps/rejected": -6.325432777404785,
      "loss": 0.7524,
      "num_input_tokens_seen": 9087680,
      "odds_ratio_loss": 7.441653251647949,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.07417380064725876,
      "rewards/margins": 0.558369517326355,
      "rewards/rejected": -0.6325433254241943,
      "sft_loss": 0.008275500498712063,
      "step": 6970
    },
    {
      "epoch": 0.5429793854531311,
      "grad_norm": 1.992269515991211,
      "learning_rate": 2.1634654581752517e-05,
      "logits/chosen": -0.05781219154596329,
      "logits/rejected": -1.7768752574920654,
      "logps/chosen": -1.0271333456039429,
      "logps/rejected": -15.123313903808594,
      "loss": 1.0431,
      "num_input_tokens_seen": 9103792,
      "odds_ratio_loss": 10.401407241821289,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.10271334648132324,
      "rewards/margins": 1.4096179008483887,
      "rewards/rejected": -1.512331247329712,
      "sft_loss": 0.0029968966264277697,
      "step": 6980
    },
    {
      "epoch": 0.543757292882147,
      "grad_norm": 4.649837493896484,
      "learning_rate": 2.1574124078933953e-05,
      "logits/chosen": -0.3127630352973938,
      "logits/rejected": -1.208951711654663,
      "logps/chosen": -0.952308177947998,
      "logps/rejected": -6.729982852935791,
      "loss": 0.9666,
      "num_input_tokens_seen": 9119392,
      "odds_ratio_loss": 9.563682556152344,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.0952308177947998,
      "rewards/margins": 0.5777674913406372,
      "rewards/rejected": -0.672998309135437,
      "sft_loss": 0.010236539877951145,
      "step": 6990
    },
    {
      "epoch": 0.544535200311163,
      "grad_norm": 3.9194202423095703,
      "learning_rate": 2.151361403712205e-05,
      "logits/chosen": -0.03569873049855232,
      "logits/rejected": -1.1331626176834106,
      "logps/chosen": -0.8960397839546204,
      "logps/rejected": -8.979105949401855,
      "loss": 0.9071,
      "num_input_tokens_seen": 9131184,
      "odds_ratio_loss": 9.027724266052246,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.0896039754152298,
      "rewards/margins": 0.8083066940307617,
      "rewards/rejected": -0.8979107141494751,
      "sft_loss": 0.004334039054811001,
      "step": 7000
    },
    {
      "epoch": 0.5453131077401789,
      "grad_norm": 4.821101665496826,
      "learning_rate": 2.1453124817712382e-05,
      "logits/chosen": -0.1305716633796692,
      "logits/rejected": -1.4796316623687744,
      "logps/chosen": -1.0244022607803345,
      "logps/rejected": -13.108938217163086,
      "loss": 1.0268,
      "num_input_tokens_seen": 9145184,
      "odds_ratio_loss": 10.204194068908691,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.10244021564722061,
      "rewards/margins": 1.2084537744522095,
      "rewards/rejected": -1.3108938932418823,
      "sft_loss": 0.0063963839784264565,
      "step": 7010
    },
    {
      "epoch": 0.5460910151691949,
      "grad_norm": 2.2801997661590576,
      "learning_rate": 2.1392656781976147e-05,
      "logits/chosen": 0.051787324249744415,
      "logits/rejected": -1.641805648803711,
      "logps/chosen": -0.9896619915962219,
      "logps/rejected": -13.139961242675781,
      "loss": 1.0347,
      "num_input_tokens_seen": 9161856,
      "odds_ratio_loss": 10.30107593536377,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.09896618872880936,
      "rewards/margins": 1.2150300741195679,
      "rewards/rejected": -1.3139961957931519,
      "sft_loss": 0.004545119125396013,
      "step": 7020
    },
    {
      "epoch": 0.5468689225982108,
      "grad_norm": 5.194355010986328,
      "learning_rate": 2.1332210291058037e-05,
      "logits/chosen": -0.1215142235159874,
      "logits/rejected": -1.6807050704956055,
      "logps/chosen": -1.1550666093826294,
      "logps/rejected": -11.24571418762207,
      "loss": 1.1721,
      "num_input_tokens_seen": 9177152,
      "odds_ratio_loss": 11.612730026245117,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.11550666391849518,
      "rewards/margins": 1.0090649127960205,
      "rewards/rejected": -1.1245715618133545,
      "sft_loss": 0.010867111384868622,
      "step": 7030
    },
    {
      "epoch": 0.5476468300272268,
      "grad_norm": 2.0088090896606445,
      "learning_rate": 2.1271785705974058e-05,
      "logits/chosen": -0.05257201939821243,
      "logits/rejected": -1.5861520767211914,
      "logps/chosen": -0.8975200653076172,
      "logps/rejected": -5.729624271392822,
      "loss": 0.9255,
      "num_input_tokens_seen": 9191776,
      "odds_ratio_loss": 9.136106491088867,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.08975200355052948,
      "rewards/margins": 0.48321041464805603,
      "rewards/rejected": -0.5729624629020691,
      "sft_loss": 0.01193344034254551,
      "step": 7040
    },
    {
      "epoch": 0.5484247374562428,
      "grad_norm": 1.584776520729065,
      "learning_rate": 2.1211383387609383e-05,
      "logits/chosen": 0.05494903400540352,
      "logits/rejected": -1.6136630773544312,
      "logps/chosen": -0.849213719367981,
      "logps/rejected": -6.129418849945068,
      "loss": 0.8748,
      "num_input_tokens_seen": 9208192,
      "odds_ratio_loss": 8.697319030761719,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08492136001586914,
      "rewards/margins": 0.5280205607414246,
      "rewards/rejected": -0.6129419207572937,
      "sft_loss": 0.005068082828074694,
      "step": 7050
    },
    {
      "epoch": 0.5492026448852586,
      "grad_norm": 1.491456389427185,
      "learning_rate": 2.1151003696716217e-05,
      "logits/chosen": -0.013895402662456036,
      "logits/rejected": -1.5359677076339722,
      "logps/chosen": -1.0025429725646973,
      "logps/rejected": -2.2590880393981934,
      "loss": 1.0409,
      "num_input_tokens_seen": 9220816,
      "odds_ratio_loss": 10.346986770629883,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.10025429725646973,
      "rewards/margins": 0.12565451860427856,
      "rewards/rejected": -0.22590883076190948,
      "sft_loss": 0.006227378733456135,
      "step": 7060
    },
    {
      "epoch": 0.5499805523142746,
      "grad_norm": 12.248387336730957,
      "learning_rate": 2.1090646993911588e-05,
      "logits/chosen": -0.38075366616249084,
      "logits/rejected": -0.7645701169967651,
      "logps/chosen": -0.8709882497787476,
      "logps/rejected": -11.054000854492188,
      "loss": 0.8847,
      "num_input_tokens_seen": 9233984,
      "odds_ratio_loss": 8.402530670166016,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08709883689880371,
      "rewards/margins": 1.018301248550415,
      "rewards/rejected": -1.1054000854492188,
      "sft_loss": 0.04444083198904991,
      "step": 7070
    },
    {
      "epoch": 0.5507584597432905,
      "grad_norm": 7.897884845733643,
      "learning_rate": 2.1030313639675253e-05,
      "logits/chosen": -0.16736795008182526,
      "logits/rejected": -1.2365648746490479,
      "logps/chosen": -0.9248377084732056,
      "logps/rejected": -7.286402702331543,
      "loss": 0.9708,
      "num_input_tokens_seen": 9245328,
      "odds_ratio_loss": 9.52192497253418,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.0924837738275528,
      "rewards/margins": 0.6361564993858337,
      "rewards/rejected": -0.7286401987075806,
      "sft_loss": 0.01863746903836727,
      "step": 7080
    },
    {
      "epoch": 0.5515363671723065,
      "grad_norm": 5.933419227600098,
      "learning_rate": 2.0970003994347514e-05,
      "logits/chosen": 0.006171123590320349,
      "logits/rejected": -1.7836297750473022,
      "logps/chosen": -0.9997663497924805,
      "logps/rejected": -8.293216705322266,
      "loss": 1.0218,
      "num_input_tokens_seen": 9259760,
      "odds_ratio_loss": 10.131877899169922,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.09997662901878357,
      "rewards/margins": 0.7293451428413391,
      "rewards/rejected": -0.8293216824531555,
      "sft_loss": 0.008609093725681305,
      "step": 7090
    },
    {
      "epoch": 0.5523142746013224,
      "grad_norm": 1.800391674041748,
      "learning_rate": 2.0909718418127077e-05,
      "logits/chosen": -0.3593955636024475,
      "logits/rejected": -1.4708583354949951,
      "logps/chosen": -1.018897533416748,
      "logps/rejected": -9.909947395324707,
      "loss": 1.0323,
      "num_input_tokens_seen": 9273216,
      "odds_ratio_loss": 9.919496536254883,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.10188976675271988,
      "rewards/margins": 0.8891050219535828,
      "rewards/rejected": -0.9909947514533997,
      "sft_loss": 0.04033954441547394,
      "step": 7100
    },
    {
      "epoch": 0.5530921820303384,
      "grad_norm": 6.053109645843506,
      "learning_rate": 2.084945727106888e-05,
      "logits/chosen": -0.3147508502006531,
      "logits/rejected": -1.4761817455291748,
      "logps/chosen": -0.9336416125297546,
      "logps/rejected": -16.528560638427734,
      "loss": 0.9503,
      "num_input_tokens_seen": 9284144,
      "odds_ratio_loss": 9.027544975280762,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.0933641642332077,
      "rewards/margins": 1.5594919919967651,
      "rewards/rejected": -1.6528562307357788,
      "sft_loss": 0.04757608473300934,
      "step": 7110
    },
    {
      "epoch": 0.5538700894593543,
      "grad_norm": 3.530287742614746,
      "learning_rate": 2.0789220913081973e-05,
      "logits/chosen": -0.07793496549129486,
      "logits/rejected": -1.2224290370941162,
      "logps/chosen": -2.3578381538391113,
      "logps/rejected": -11.467536926269531,
      "loss": 2.4995,
      "num_input_tokens_seen": 9294624,
      "odds_ratio_loss": 24.468048095703125,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.23578378558158875,
      "rewards/margins": 0.9109698534011841,
      "rewards/rejected": -1.1467535495758057,
      "sft_loss": 0.052667904645204544,
      "step": 7120
    },
    {
      "epoch": 0.5546479968883703,
      "grad_norm": 5.699097633361816,
      "learning_rate": 2.0729009703927353e-05,
      "logits/chosen": -0.17937228083610535,
      "logits/rejected": -1.003334403038025,
      "logps/chosen": -0.9862130880355835,
      "logps/rejected": -17.108503341674805,
      "loss": 0.9927,
      "num_input_tokens_seen": 9308192,
      "odds_ratio_loss": 9.582437515258789,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09862131625413895,
      "rewards/margins": 1.6122291088104248,
      "rewards/rejected": -1.7108503580093384,
      "sft_loss": 0.03447087109088898,
      "step": 7130
    },
    {
      "epoch": 0.5554259043173863,
      "grad_norm": 3.0519697666168213,
      "learning_rate": 2.066882400321581e-05,
      "logits/chosen": -0.1276940554380417,
      "logits/rejected": -1.6967575550079346,
      "logps/chosen": -0.9447711706161499,
      "logps/rejected": -10.891839981079102,
      "loss": 0.9522,
      "num_input_tokens_seen": 9323920,
      "odds_ratio_loss": 9.481612205505371,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09447711706161499,
      "rewards/margins": 0.9947069883346558,
      "rewards/rejected": -1.089184045791626,
      "sft_loss": 0.004014966078102589,
      "step": 7140
    },
    {
      "epoch": 0.5562038117464022,
      "grad_norm": 2.3701045513153076,
      "learning_rate": 2.0608664170405783e-05,
      "logits/chosen": -0.0037519708275794983,
      "logits/rejected": -1.6001594066619873,
      "logps/chosen": -0.8713818788528442,
      "logps/rejected": -8.277201652526855,
      "loss": 0.8797,
      "num_input_tokens_seen": 9339744,
      "odds_ratio_loss": 8.76332950592041,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08713819831609726,
      "rewards/margins": 0.7405818700790405,
      "rewards/rejected": -0.827720046043396,
      "sft_loss": 0.003366539254784584,
      "step": 7150
    },
    {
      "epoch": 0.5569817191754182,
      "grad_norm": 4.49266242980957,
      "learning_rate": 2.054853056480122e-05,
      "logits/chosen": -0.19142615795135498,
      "logits/rejected": -1.3163836002349854,
      "logps/chosen": -1.1975313425064087,
      "logps/rejected": -18.15427017211914,
      "loss": 1.199,
      "num_input_tokens_seen": 9356800,
      "odds_ratio_loss": 10.61500358581543,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.11975312232971191,
      "rewards/margins": 1.6956737041473389,
      "rewards/rejected": -1.8154270648956299,
      "sft_loss": 0.1374645233154297,
      "step": 7160
    },
    {
      "epoch": 0.5577596266044341,
      "grad_norm": 4.971721172332764,
      "learning_rate": 2.048842354554943e-05,
      "logits/chosen": -0.2915323078632355,
      "logits/rejected": -0.9623371362686157,
      "logps/chosen": -0.7749730944633484,
      "logps/rejected": -16.453466415405273,
      "loss": 0.7792,
      "num_input_tokens_seen": 9370544,
      "odds_ratio_loss": 7.725368499755859,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.07749731838703156,
      "rewards/margins": 1.5678495168685913,
      "rewards/rejected": -1.6453468799591064,
      "sft_loss": 0.006691871676594019,
      "step": 7170
    },
    {
      "epoch": 0.55853753403345,
      "grad_norm": 1.6422734260559082,
      "learning_rate": 2.0428343471638933e-05,
      "logits/chosen": -0.09886634349822998,
      "logits/rejected": -1.1431877613067627,
      "logps/chosen": -0.7844544649124146,
      "logps/rejected": -10.52696418762207,
      "loss": 0.8138,
      "num_input_tokens_seen": 9380608,
      "odds_ratio_loss": 7.857357978820801,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.0784454494714737,
      "rewards/margins": 0.9742509722709656,
      "rewards/rejected": -1.0526965856552124,
      "sft_loss": 0.028091808781027794,
      "step": 7180
    },
    {
      "epoch": 0.5593154414624659,
      "grad_norm": 4.934447288513184,
      "learning_rate": 2.0368290701897306e-05,
      "logits/chosen": -0.01598060131072998,
      "logits/rejected": -1.2897093296051025,
      "logps/chosen": -0.9649337530136108,
      "logps/rejected": -7.158110618591309,
      "loss": 0.9823,
      "num_input_tokens_seen": 9389424,
      "odds_ratio_loss": 9.752803802490234,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.09649336338043213,
      "rewards/margins": 0.6193177103996277,
      "rewards/rejected": -0.7158111333847046,
      "sft_loss": 0.007014032453298569,
      "step": 7190
    },
    {
      "epoch": 0.5600933488914819,
      "grad_norm": 16.234329223632812,
      "learning_rate": 2.030826559498907e-05,
      "logits/chosen": 0.006356333382427692,
      "logits/rejected": -1.6087524890899658,
      "logps/chosen": -0.8868484497070312,
      "logps/rejected": -19.774511337280273,
      "loss": 0.8881,
      "num_input_tokens_seen": 9405392,
      "odds_ratio_loss": 8.79615592956543,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08868485689163208,
      "rewards/margins": 1.8887665271759033,
      "rewards/rejected": -1.9774510860443115,
      "sft_loss": 0.008442886173725128,
      "step": 7200
    },
    {
      "epoch": 0.5608712563204978,
      "grad_norm": 3.596815347671509,
      "learning_rate": 2.0248268509413522e-05,
      "logits/chosen": -0.15051034092903137,
      "logits/rejected": -1.5102225542068481,
      "logps/chosen": -1.2850151062011719,
      "logps/rejected": -18.5378360748291,
      "loss": 1.3145,
      "num_input_tokens_seen": 9419088,
      "odds_ratio_loss": 12.647809982299805,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.1285015046596527,
      "rewards/margins": 1.7252819538116455,
      "rewards/rejected": -1.8537836074829102,
      "sft_loss": 0.0497058629989624,
      "step": 7210
    },
    {
      "epoch": 0.5616491637495138,
      "grad_norm": 4.4439496994018555,
      "learning_rate": 2.01882998035026e-05,
      "logits/chosen": -0.6091213226318359,
      "logits/rejected": -0.9413172602653503,
      "logps/chosen": -1.053614854812622,
      "logps/rejected": -25.349395751953125,
      "loss": 1.061,
      "num_input_tokens_seen": 9439232,
      "odds_ratio_loss": 10.500904083251953,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.10536149889230728,
      "rewards/margins": 2.4295783042907715,
      "rewards/rejected": -2.5349395275115967,
      "sft_loss": 0.010934138670563698,
      "step": 7220
    },
    {
      "epoch": 0.5624270711785297,
      "grad_norm": 7.545828819274902,
      "learning_rate": 2.012835983541876e-05,
      "logits/chosen": -0.1955385059118271,
      "logits/rejected": -1.245243787765503,
      "logps/chosen": -0.8543983697891235,
      "logps/rejected": -22.91373634338379,
      "loss": 0.8583,
      "num_input_tokens_seen": 9456352,
      "odds_ratio_loss": 8.325008392333984,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08543984591960907,
      "rewards/margins": 2.2059340476989746,
      "rewards/rejected": -2.2913737297058105,
      "sft_loss": 0.02578604780137539,
      "step": 7230
    },
    {
      "epoch": 0.5632049786075457,
      "grad_norm": 2.2859387397766113,
      "learning_rate": 2.0068448963152786e-05,
      "logits/chosen": -0.06819014251232147,
      "logits/rejected": -1.5684789419174194,
      "logps/chosen": -0.9775391817092896,
      "logps/rejected": -9.996573448181152,
      "loss": 0.9786,
      "num_input_tokens_seen": 9467104,
      "odds_ratio_loss": 9.728795051574707,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09775391221046448,
      "rewards/margins": 0.9019034504890442,
      "rewards/rejected": -0.9996573328971863,
      "sft_loss": 0.0056895045563578606,
      "step": 7240
    },
    {
      "epoch": 0.5639828860365617,
      "grad_norm": 4.704026222229004,
      "learning_rate": 2.000856754452173e-05,
      "logits/chosen": -0.1956000179052353,
      "logits/rejected": -0.9999753832817078,
      "logps/chosen": -0.9804566502571106,
      "logps/rejected": -5.157223224639893,
      "loss": 0.989,
      "num_input_tokens_seen": 9480304,
      "odds_ratio_loss": 9.775531768798828,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09804566949605942,
      "rewards/margins": 0.4176766276359558,
      "rewards/rejected": -0.5157222747802734,
      "sft_loss": 0.01142149604856968,
      "step": 7250
    },
    {
      "epoch": 0.5647607934655776,
      "grad_norm": 1.6146222352981567,
      "learning_rate": 1.9948715937166708e-05,
      "logits/chosen": -0.12992647290229797,
      "logits/rejected": -1.6173648834228516,
      "logps/chosen": -0.927302360534668,
      "logps/rejected": -11.353559494018555,
      "loss": 0.9374,
      "num_input_tokens_seen": 9496448,
      "odds_ratio_loss": 9.286779403686523,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.09273023903369904,
      "rewards/margins": 1.0426256656646729,
      "rewards/rejected": -1.1353559494018555,
      "sft_loss": 0.008731330744922161,
      "step": 7260
    },
    {
      "epoch": 0.5655387008945936,
      "grad_norm": 0.37701746821403503,
      "learning_rate": 1.9888894498550798e-05,
      "logits/chosen": -0.46010270714759827,
      "logits/rejected": -1.1463847160339355,
      "logps/chosen": -0.7036060690879822,
      "logps/rejected": -12.497151374816895,
      "loss": 0.7117,
      "num_input_tokens_seen": 9509680,
      "odds_ratio_loss": 6.6658935546875,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.07036060094833374,
      "rewards/margins": 1.1793544292449951,
      "rewards/rejected": -1.2497152090072632,
      "sft_loss": 0.04514772444963455,
      "step": 7270
    },
    {
      "epoch": 0.5663166083236095,
      "grad_norm": 2.3157222270965576,
      "learning_rate": 1.9829103585956882e-05,
      "logits/chosen": -0.15636548399925232,
      "logits/rejected": -1.0944286584854126,
      "logps/chosen": -0.8795326352119446,
      "logps/rejected": -13.466470718383789,
      "loss": 0.8988,
      "num_input_tokens_seen": 9523712,
      "odds_ratio_loss": 8.924982070922852,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.08795326948165894,
      "rewards/margins": 1.258693814277649,
      "rewards/rejected": -1.3466471433639526,
      "sft_loss": 0.0063031939789652824,
      "step": 7280
    },
    {
      "epoch": 0.5670945157526255,
      "grad_norm": 6.568509578704834,
      "learning_rate": 1.9769343556485544e-05,
      "logits/chosen": -0.7389642000198364,
      "logits/rejected": -0.5600911974906921,
      "logps/chosen": -0.8247898817062378,
      "logps/rejected": -9.657159805297852,
      "loss": 0.8419,
      "num_input_tokens_seen": 9533104,
      "odds_ratio_loss": 8.1779146194458,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.08247898519039154,
      "rewards/margins": 0.8832368850708008,
      "rewards/rejected": -0.965715765953064,
      "sft_loss": 0.024064451456069946,
      "step": 7290
    },
    {
      "epoch": 0.5678724231816414,
      "grad_norm": 5.062512397766113,
      "learning_rate": 1.97096147670529e-05,
      "logits/chosen": -0.3354889750480652,
      "logits/rejected": -1.1099668741226196,
      "logps/chosen": -2.140015125274658,
      "logps/rejected": -15.030134201049805,
      "loss": 2.1731,
      "num_input_tokens_seen": 9544176,
      "odds_ratio_loss": 21.28677749633789,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.21400149166584015,
      "rewards/margins": 1.2890119552612305,
      "rewards/rejected": -1.5030133724212646,
      "sft_loss": 0.04440547153353691,
      "step": 7300
    },
    {
      "epoch": 0.5686503306106573,
      "grad_norm": 3.2668755054473877,
      "learning_rate": 1.964991757438851e-05,
      "logits/chosen": -0.19430990517139435,
      "logits/rejected": -1.2478229999542236,
      "logps/chosen": -0.8429762721061707,
      "logps/rejected": -13.245553970336914,
      "loss": 0.8518,
      "num_input_tokens_seen": 9554544,
      "odds_ratio_loss": 8.369302749633789,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08429761976003647,
      "rewards/margins": 1.240257740020752,
      "rewards/rejected": -1.3245553970336914,
      "sft_loss": 0.014858727343380451,
      "step": 7310
    },
    {
      "epoch": 0.5694282380396732,
      "grad_norm": 2.498728036880493,
      "learning_rate": 1.9590252335033193e-05,
      "logits/chosen": -0.21325652301311493,
      "logits/rejected": -1.3000379800796509,
      "logps/chosen": -0.7684622406959534,
      "logps/rejected": -12.937002182006836,
      "loss": 0.7806,
      "num_input_tokens_seen": 9564480,
      "odds_ratio_loss": 7.7429962158203125,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.07684622704982758,
      "rewards/margins": 1.2168540954589844,
      "rewards/rejected": -1.2937002182006836,
      "sft_loss": 0.006269586272537708,
      "step": 7320
    },
    {
      "epoch": 0.5702061454686892,
      "grad_norm": 1.7052494287490845,
      "learning_rate": 1.953061940533695e-05,
      "logits/chosen": -0.06910298019647598,
      "logits/rejected": -1.5475032329559326,
      "logps/chosen": -0.8709535598754883,
      "logps/rejected": -8.465534210205078,
      "loss": 0.875,
      "num_input_tokens_seen": 9574912,
      "odds_ratio_loss": 8.691818237304688,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08709535002708435,
      "rewards/margins": 0.759458065032959,
      "rewards/rejected": -0.8465534448623657,
      "sft_loss": 0.005867579486221075,
      "step": 7330
    },
    {
      "epoch": 0.5709840528977052,
      "grad_norm": 4.655317306518555,
      "learning_rate": 1.94710191414568e-05,
      "logits/chosen": -0.4469028115272522,
      "logits/rejected": -0.8740050196647644,
      "logps/chosen": -0.9266998171806335,
      "logps/rejected": -10.280158996582031,
      "loss": 0.95,
      "num_input_tokens_seen": 9587824,
      "odds_ratio_loss": 9.35964298248291,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.09266997873783112,
      "rewards/margins": 0.9353460073471069,
      "rewards/rejected": -1.0280159711837769,
      "sft_loss": 0.014021886512637138,
      "step": 7340
    },
    {
      "epoch": 0.5717619603267211,
      "grad_norm": 3.7156572341918945,
      "learning_rate": 1.941145189935467e-05,
      "logits/chosen": 0.022521795704960823,
      "logits/rejected": -1.2985093593597412,
      "logps/chosen": -0.9652273058891296,
      "logps/rejected": -13.660550117492676,
      "loss": 0.9723,
      "num_input_tokens_seen": 9599248,
      "odds_ratio_loss": 9.660152435302734,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.0965227335691452,
      "rewards/margins": 1.269532322883606,
      "rewards/rejected": -1.3660550117492676,
      "sft_loss": 0.006298977881669998,
      "step": 7350
    },
    {
      "epoch": 0.5725398677557371,
      "grad_norm": 8.959341049194336,
      "learning_rate": 1.935191803479527e-05,
      "logits/chosen": -0.09870149940252304,
      "logits/rejected": -1.3404889106750488,
      "logps/chosen": -0.7295455932617188,
      "logps/rejected": -3.998936891555786,
      "loss": 0.7599,
      "num_input_tokens_seen": 9609584,
      "odds_ratio_loss": 7.533753871917725,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.07295455038547516,
      "rewards/margins": 0.3269391655921936,
      "rewards/rejected": -0.39989370107650757,
      "sft_loss": 0.006560014095157385,
      "step": 7360
    },
    {
      "epoch": 0.573317775184753,
      "grad_norm": 14.540144920349121,
      "learning_rate": 1.9292417903343953e-05,
      "logits/chosen": -0.09635542333126068,
      "logits/rejected": -1.358878493309021,
      "logps/chosen": -1.100290298461914,
      "logps/rejected": -12.779852867126465,
      "loss": 1.1131,
      "num_input_tokens_seen": 9629072,
      "odds_ratio_loss": 11.04706859588623,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.11002902686595917,
      "rewards/margins": 1.1679562330245972,
      "rewards/rejected": -1.2779853343963623,
      "sft_loss": 0.008365782909095287,
      "step": 7370
    },
    {
      "epoch": 0.574095682613769,
      "grad_norm": 2.4772889614105225,
      "learning_rate": 1.923295186036461e-05,
      "logits/chosen": -0.40384602546691895,
      "logits/rejected": -0.952344536781311,
      "logps/chosen": -0.9728520512580872,
      "logps/rejected": -9.531290054321289,
      "loss": 0.9861,
      "num_input_tokens_seen": 9641760,
      "odds_ratio_loss": 9.743693351745605,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.0972852036356926,
      "rewards/margins": 0.8558439016342163,
      "rewards/rejected": -0.9531289935112,
      "sft_loss": 0.011719195172190666,
      "step": 7380
    },
    {
      "epoch": 0.5748735900427849,
      "grad_norm": 94.43122100830078,
      "learning_rate": 1.917352026101754e-05,
      "logits/chosen": 0.10378698259592056,
      "logits/rejected": -1.9902175664901733,
      "logps/chosen": -3.0623791217803955,
      "logps/rejected": -19.109020233154297,
      "loss": 3.1164,
      "num_input_tokens_seen": 9659728,
      "odds_ratio_loss": 31.07373046875,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.30623793601989746,
      "rewards/margins": 1.6046640872955322,
      "rewards/rejected": -1.9109020233154297,
      "sft_loss": 0.009036138653755188,
      "step": 7390
    },
    {
      "epoch": 0.5756514974718009,
      "grad_norm": 2.3216805458068848,
      "learning_rate": 1.9114123460257317e-05,
      "logits/chosen": -0.5298863053321838,
      "logits/rejected": -1.2425428628921509,
      "logps/chosen": -0.8780226707458496,
      "logps/rejected": -13.617281913757324,
      "loss": 0.8919,
      "num_input_tokens_seen": 9669920,
      "odds_ratio_loss": 8.812639236450195,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.08780227601528168,
      "rewards/margins": 1.27392578125,
      "rewards/rejected": -1.3617281913757324,
      "sft_loss": 0.010591082274913788,
      "step": 7400
    },
    {
      "epoch": 0.5764294049008168,
      "grad_norm": 2.2508273124694824,
      "learning_rate": 1.905476181283069e-05,
      "logits/chosen": -0.08014758676290512,
      "logits/rejected": -1.3329795598983765,
      "logps/chosen": -1.058363676071167,
      "logps/rejected": -8.324371337890625,
      "loss": 1.1206,
      "num_input_tokens_seen": 9679776,
      "odds_ratio_loss": 11.035293579101562,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.10583637654781342,
      "rewards/margins": 0.7266008853912354,
      "rewards/rejected": -0.8324371576309204,
      "sft_loss": 0.01711886003613472,
      "step": 7410
    },
    {
      "epoch": 0.5772073123298328,
      "grad_norm": 8.43524169921875,
      "learning_rate": 1.8995435673274452e-05,
      "logits/chosen": -0.00843315850943327,
      "logits/rejected": -1.1164019107818604,
      "logps/chosen": -0.9222790002822876,
      "logps/rejected": -8.81623363494873,
      "loss": 0.9453,
      "num_input_tokens_seen": 9693408,
      "odds_ratio_loss": 9.378290176391602,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.09222790598869324,
      "rewards/margins": 0.7893955111503601,
      "rewards/rejected": -0.881623387336731,
      "sft_loss": 0.0074365464970469475,
      "step": 7420
    },
    {
      "epoch": 0.5779852197588486,
      "grad_norm": 4.231116771697998,
      "learning_rate": 1.8936145395913336e-05,
      "logits/chosen": 0.08308565616607666,
      "logits/rejected": -1.8230034112930298,
      "logps/chosen": -0.9667071104049683,
      "logps/rejected": -7.038580417633057,
      "loss": 0.9898,
      "num_input_tokens_seen": 9708144,
      "odds_ratio_loss": 9.870607376098633,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.0966707244515419,
      "rewards/margins": 0.6071873307228088,
      "rewards/rejected": -0.7038580179214478,
      "sft_loss": 0.002733974251896143,
      "step": 7430
    },
    {
      "epoch": 0.5787631271878646,
      "grad_norm": 4.452478885650635,
      "learning_rate": 1.8876891334857858e-05,
      "logits/chosen": -0.3030247390270233,
      "logits/rejected": -1.1824462413787842,
      "logps/chosen": -1.0096709728240967,
      "logps/rejected": -5.993814468383789,
      "loss": 1.0364,
      "num_input_tokens_seen": 9717872,
      "odds_ratio_loss": 9.989054679870605,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.10096710920333862,
      "rewards/margins": 0.49841442704200745,
      "rewards/rejected": -0.5993814468383789,
      "sft_loss": 0.037446800619363785,
      "step": 7440
    },
    {
      "epoch": 0.5795410346168806,
      "grad_norm": 1.922385573387146,
      "learning_rate": 1.8817673844002264e-05,
      "logits/chosen": -0.2290133237838745,
      "logits/rejected": -1.22044837474823,
      "logps/chosen": -0.9724160432815552,
      "logps/rejected": -17.186552047729492,
      "loss": 0.984,
      "num_input_tokens_seen": 9727408,
      "odds_ratio_loss": 9.41455078125,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.0972416028380394,
      "rewards/margins": 1.6214138269424438,
      "rewards/rejected": -1.7186553478240967,
      "sft_loss": 0.042552731931209564,
      "step": 7450
    },
    {
      "epoch": 0.5803189420458965,
      "grad_norm": 1.7939796447753906,
      "learning_rate": 1.8758493277022377e-05,
      "logits/chosen": 0.06815723329782486,
      "logits/rejected": -1.399775743484497,
      "logps/chosen": -0.7001609802246094,
      "logps/rejected": -17.41643524169922,
      "loss": 0.7025,
      "num_input_tokens_seen": 9741200,
      "odds_ratio_loss": 6.983880043029785,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.07001610100269318,
      "rewards/margins": 1.671627402305603,
      "rewards/rejected": -1.7416435480117798,
      "sft_loss": 0.004157370887696743,
      "step": 7460
    },
    {
      "epoch": 0.5810968494749125,
      "grad_norm": 24.65456771850586,
      "learning_rate": 1.8699349987373482e-05,
      "logits/chosen": -0.09804271161556244,
      "logits/rejected": -1.4231456518173218,
      "logps/chosen": -1.0049656629562378,
      "logps/rejected": -9.602811813354492,
      "loss": 1.0205,
      "num_input_tokens_seen": 9756272,
      "odds_ratio_loss": 9.92216682434082,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.1004965677857399,
      "rewards/margins": 0.8597846031188965,
      "rewards/rejected": -0.9602810740470886,
      "sft_loss": 0.028289655223488808,
      "step": 7470
    },
    {
      "epoch": 0.5818747569039284,
      "grad_norm": 10.561646461486816,
      "learning_rate": 1.8640244328288237e-05,
      "logits/chosen": -0.5321280360221863,
      "logits/rejected": -0.9137857556343079,
      "logps/chosen": -1.0886926651000977,
      "logps/rejected": -14.622525215148926,
      "loss": 1.0946,
      "num_input_tokens_seen": 9770016,
      "odds_ratio_loss": 10.751686096191406,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.108869269490242,
      "rewards/margins": 1.3533833026885986,
      "rewards/rejected": -1.4622526168823242,
      "sft_loss": 0.019413534551858902,
      "step": 7480
    },
    {
      "epoch": 0.5826526643329444,
      "grad_norm": 2.4361612796783447,
      "learning_rate": 1.8581176652774552e-05,
      "logits/chosen": -0.15811732411384583,
      "logits/rejected": -1.6251471042633057,
      "logps/chosen": -1.0075920820236206,
      "logps/rejected": -9.168794631958008,
      "loss": 1.0262,
      "num_input_tokens_seen": 9785408,
      "odds_ratio_loss": 10.2182035446167,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.10075920820236206,
      "rewards/margins": 0.8161203265190125,
      "rewards/rejected": -0.9168795347213745,
      "sft_loss": 0.004419906530529261,
      "step": 7490
    },
    {
      "epoch": 0.5834305717619603,
      "grad_norm": 2.919311285018921,
      "learning_rate": 1.8522147313613463e-05,
      "logits/chosen": -0.3516610264778137,
      "logits/rejected": -1.0093597173690796,
      "logps/chosen": -0.8289676904678345,
      "logps/rejected": -9.245644569396973,
      "loss": 0.8332,
      "num_input_tokens_seen": 9796592,
      "odds_ratio_loss": 7.4911017417907715,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08289675414562225,
      "rewards/margins": 0.8416677713394165,
      "rewards/rejected": -0.9245645403862,
      "sft_loss": 0.08413822203874588,
      "step": 7500
    },
    {
      "epoch": 0.5842084791909763,
      "grad_norm": 2.8248913288116455,
      "learning_rate": 1.8463156663357067e-05,
      "logits/chosen": -0.018082041293382645,
      "logits/rejected": -1.0669796466827393,
      "logps/chosen": -0.9453287124633789,
      "logps/rejected": -9.931035041809082,
      "loss": 0.9489,
      "num_input_tokens_seen": 9814848,
      "odds_ratio_loss": 8.671426773071289,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09453286975622177,
      "rewards/margins": 0.8985706567764282,
      "rewards/rejected": -0.9931035041809082,
      "sft_loss": 0.081757552921772,
      "step": 7510
    },
    {
      "epoch": 0.5849863866199922,
      "grad_norm": 5.032744407653809,
      "learning_rate": 1.8404205054326385e-05,
      "logits/chosen": -0.20301274955272675,
      "logits/rejected": -1.5665793418884277,
      "logps/chosen": -1.1178547143936157,
      "logps/rejected": -12.492952346801758,
      "loss": 1.1481,
      "num_input_tokens_seen": 9829264,
      "odds_ratio_loss": 11.17640209197998,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.11178547143936157,
      "rewards/margins": 1.137509822845459,
      "rewards/rejected": -1.2492953538894653,
      "sft_loss": 0.030486587435007095,
      "step": 7520
    },
    {
      "epoch": 0.5857642940490082,
      "grad_norm": 7.078161239624023,
      "learning_rate": 1.834529283860925e-05,
      "logits/chosen": -0.34447169303894043,
      "logits/rejected": -1.1488739252090454,
      "logps/chosen": -0.9062315225601196,
      "logps/rejected": -7.6972527503967285,
      "loss": 0.917,
      "num_input_tokens_seen": 9838160,
      "odds_ratio_loss": 8.966435432434082,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.0906231552362442,
      "rewards/margins": 0.6791021823883057,
      "rewards/rejected": -0.7697253227233887,
      "sft_loss": 0.020331600680947304,
      "step": 7530
    },
    {
      "epoch": 0.5865422014780242,
      "grad_norm": 6.74251127243042,
      "learning_rate": 1.828642036805823e-05,
      "logits/chosen": -0.0008529677870683372,
      "logits/rejected": -1.2243870496749878,
      "logps/chosen": -0.7502158284187317,
      "logps/rejected": -7.243124485015869,
      "loss": 0.7729,
      "num_input_tokens_seen": 9848976,
      "odds_ratio_loss": 7.561089992523193,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.07502157986164093,
      "rewards/margins": 0.649290919303894,
      "rewards/rejected": -0.7243124842643738,
      "sft_loss": 0.016804486513137817,
      "step": 7540
    },
    {
      "epoch": 0.5873201089070401,
      "grad_norm": 1.7365477085113525,
      "learning_rate": 1.8227587994288527e-05,
      "logits/chosen": -0.33039507269859314,
      "logits/rejected": -1.2971482276916504,
      "logps/chosen": -0.8243374824523926,
      "logps/rejected": -11.704877853393555,
      "loss": 0.8381,
      "num_input_tokens_seen": 9860736,
      "odds_ratio_loss": 8.30073070526123,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08243375271558762,
      "rewards/margins": 1.0880540609359741,
      "rewards/rejected": -1.1704877614974976,
      "sft_loss": 0.008057581260800362,
      "step": 7550
    },
    {
      "epoch": 0.588098016336056,
      "grad_norm": 4.24249792098999,
      "learning_rate": 1.8168796068675846e-05,
      "logits/chosen": -0.17244823276996613,
      "logits/rejected": -1.3585816621780396,
      "logps/chosen": -0.9519235491752625,
      "logps/rejected": -13.540249824523926,
      "loss": 0.9545,
      "num_input_tokens_seen": 9872784,
      "odds_ratio_loss": 9.490447998046875,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09519235044717789,
      "rewards/margins": 1.2588326930999756,
      "rewards/rejected": -1.35402512550354,
      "sft_loss": 0.005409600213170052,
      "step": 7560
    },
    {
      "epoch": 0.5888759237650719,
      "grad_norm": 4.247993469238281,
      "learning_rate": 1.8110044942354316e-05,
      "logits/chosen": 0.012781450524926186,
      "logits/rejected": -1.249723196029663,
      "logps/chosen": -0.8717182874679565,
      "logps/rejected": -8.16511058807373,
      "loss": 0.8773,
      "num_input_tokens_seen": 9883712,
      "odds_ratio_loss": 8.610315322875977,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08717183768749237,
      "rewards/margins": 0.7293392419815063,
      "rewards/rejected": -0.8165111541748047,
      "sft_loss": 0.01628163829445839,
      "step": 7570
    },
    {
      "epoch": 0.5896538311940879,
      "grad_norm": 10.727156639099121,
      "learning_rate": 1.8051334966214407e-05,
      "logits/chosen": -0.2588588297367096,
      "logits/rejected": -1.214369535446167,
      "logps/chosen": -0.7946221828460693,
      "logps/rejected": -10.378890991210938,
      "loss": 0.8084,
      "num_input_tokens_seen": 9899888,
      "odds_ratio_loss": 7.984610557556152,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.07946222275495529,
      "rewards/margins": 0.9584269523620605,
      "rewards/rejected": -1.0378892421722412,
      "sft_loss": 0.009910033084452152,
      "step": 7580
    },
    {
      "epoch": 0.5904317386231038,
      "grad_norm": 2.202627182006836,
      "learning_rate": 1.799266649090081e-05,
      "logits/chosen": 0.1137768030166626,
      "logits/rejected": -1.546129822731018,
      "logps/chosen": -0.8701799511909485,
      "logps/rejected": -12.834440231323242,
      "loss": 0.875,
      "num_input_tokens_seen": 9910800,
      "odds_ratio_loss": 8.691085815429688,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08701799809932709,
      "rewards/margins": 1.1964260339736938,
      "rewards/rejected": -1.2834439277648926,
      "sft_loss": 0.0058501651510596275,
      "step": 7590
    },
    {
      "epoch": 0.5912096460521198,
      "grad_norm": 17.521636962890625,
      "learning_rate": 1.7934039866810355e-05,
      "logits/chosen": -0.06771423667669296,
      "logits/rejected": -1.020412802696228,
      "logps/chosen": -0.9952315092086792,
      "logps/rejected": -9.883779525756836,
      "loss": 1.0155,
      "num_input_tokens_seen": 9919408,
      "odds_ratio_loss": 9.928845405578613,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.0995231419801712,
      "rewards/margins": 0.8888548016548157,
      "rewards/rejected": -0.9883779287338257,
      "sft_loss": 0.022567380219697952,
      "step": 7600
    },
    {
      "epoch": 0.5919875534811357,
      "grad_norm": 2.3257410526275635,
      "learning_rate": 1.7875455444089924e-05,
      "logits/chosen": -0.028352582827210426,
      "logits/rejected": -1.1464836597442627,
      "logps/chosen": -0.9791489839553833,
      "logps/rejected": -11.992913246154785,
      "loss": 0.9998,
      "num_input_tokens_seen": 9927168,
      "odds_ratio_loss": 9.925604820251465,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.09791488945484161,
      "rewards/margins": 1.1013762950897217,
      "rewards/rejected": -1.1992913484573364,
      "sft_loss": 0.007277652621269226,
      "step": 7610
    },
    {
      "epoch": 0.5927654609101517,
      "grad_norm": 11.161792755126953,
      "learning_rate": 1.781691357263433e-05,
      "logits/chosen": -0.30510300397872925,
      "logits/rejected": -0.8714715838432312,
      "logps/chosen": -0.7402487993240356,
      "logps/rejected": -8.375066757202148,
      "loss": 0.7437,
      "num_input_tokens_seen": 9935056,
      "odds_ratio_loss": 7.290592193603516,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.07402488589286804,
      "rewards/margins": 0.7634817361831665,
      "rewards/rejected": -0.8375066518783569,
      "sft_loss": 0.0146897342056036,
      "step": 7620
    },
    {
      "epoch": 0.5935433683391677,
      "grad_norm": 3.6200902462005615,
      "learning_rate": 1.7758414602084278e-05,
      "logits/chosen": -0.22039294242858887,
      "logits/rejected": -1.0717424154281616,
      "logps/chosen": -0.8429325819015503,
      "logps/rejected": -16.25857925415039,
      "loss": 0.8539,
      "num_input_tokens_seen": 9946608,
      "odds_ratio_loss": 8.18214225769043,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08429326117038727,
      "rewards/margins": 1.541564702987671,
      "rewards/rejected": -1.625857949256897,
      "sft_loss": 0.03569791093468666,
      "step": 7630
    },
    {
      "epoch": 0.5943212757681836,
      "grad_norm": 5.2960615158081055,
      "learning_rate": 1.7699958881824217e-05,
      "logits/chosen": -0.37763914465904236,
      "logits/rejected": -0.9068030118942261,
      "logps/chosen": -0.9599825143814087,
      "logps/rejected": -8.023133277893066,
      "loss": 0.9926,
      "num_input_tokens_seen": 9961696,
      "odds_ratio_loss": 9.783538818359375,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.09599826484918594,
      "rewards/margins": 0.7063151597976685,
      "rewards/rejected": -0.8023134469985962,
      "sft_loss": 0.014255953952670097,
      "step": 7640
    },
    {
      "epoch": 0.5950991831971996,
      "grad_norm": 2.2685546875,
      "learning_rate": 1.7641546760980322e-05,
      "logits/chosen": -0.32570111751556396,
      "logits/rejected": -1.069291114807129,
      "logps/chosen": -0.8563793897628784,
      "logps/rejected": -11.079155921936035,
      "loss": 0.8647,
      "num_input_tokens_seen": 9975360,
      "odds_ratio_loss": 8.590951919555664,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08563794195652008,
      "rewards/margins": 1.022277593612671,
      "rewards/rejected": -1.1079155206680298,
      "sft_loss": 0.005621200893074274,
      "step": 7650
    },
    {
      "epoch": 0.5958770906262155,
      "grad_norm": 3.5751876831054688,
      "learning_rate": 1.7583178588418336e-05,
      "logits/chosen": -0.34155577421188354,
      "logits/rejected": -0.8890758752822876,
      "logps/chosen": -0.8219859004020691,
      "logps/rejected": -6.9678239822387695,
      "loss": 0.8456,
      "num_input_tokens_seen": 9988336,
      "odds_ratio_loss": 8.05842399597168,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.0821986049413681,
      "rewards/margins": 0.6145837903022766,
      "rewards/rejected": -0.6967824101448059,
      "sft_loss": 0.03971128538250923,
      "step": 7660
    },
    {
      "epoch": 0.5966549980552315,
      "grad_norm": 4.907100200653076,
      "learning_rate": 1.7524854712741535e-05,
      "logits/chosen": 0.010171901434659958,
      "logits/rejected": -1.6238276958465576,
      "logps/chosen": -0.9058274030685425,
      "logps/rejected": -14.971672058105469,
      "loss": 0.9245,
      "num_input_tokens_seen": 9998880,
      "odds_ratio_loss": 9.062615394592285,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.09058274328708649,
      "rewards/margins": 1.4065845012664795,
      "rewards/rejected": -1.4971672296524048,
      "sft_loss": 0.018236646428704262,
      "step": 7670
    },
    {
      "epoch": 0.5974329054842473,
      "grad_norm": 1.8566153049468994,
      "learning_rate": 1.7466575482288642e-05,
      "logits/chosen": -0.10399254411458969,
      "logits/rejected": -1.25956130027771,
      "logps/chosen": -1.007875919342041,
      "logps/rejected": -13.468701362609863,
      "loss": 1.0377,
      "num_input_tokens_seen": 10014272,
      "odds_ratio_loss": 9.983131408691406,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.10078759491443634,
      "rewards/margins": 1.2460824251174927,
      "rewards/rejected": -1.3468698263168335,
      "sft_loss": 0.039339579641819,
      "step": 7680
    },
    {
      "epoch": 0.5982108129132633,
      "grad_norm": 3.9419636726379395,
      "learning_rate": 1.7408341245131704e-05,
      "logits/chosen": -0.35398897528648376,
      "logits/rejected": -1.1243394613265991,
      "logps/chosen": -0.9453102946281433,
      "logps/rejected": -10.045394897460938,
      "loss": 0.9588,
      "num_input_tokens_seen": 10026672,
      "odds_ratio_loss": 9.469941139221191,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.09453103691339493,
      "rewards/margins": 0.910008430480957,
      "rewards/rejected": -1.0045394897460938,
      "sft_loss": 0.011818505823612213,
      "step": 7690
    },
    {
      "epoch": 0.5989887203422792,
      "grad_norm": 2.3425137996673584,
      "learning_rate": 1.7350152349074083e-05,
      "logits/chosen": -0.12070386111736298,
      "logits/rejected": -1.457331895828247,
      "logps/chosen": -0.8209397196769714,
      "logps/rejected": -12.298772811889648,
      "loss": 0.8349,
      "num_input_tokens_seen": 10040640,
      "odds_ratio_loss": 8.312410354614258,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.0820939689874649,
      "rewards/margins": 1.1477832794189453,
      "rewards/rejected": -1.229877233505249,
      "sft_loss": 0.003699915949255228,
      "step": 7700
    },
    {
      "epoch": 0.5997666277712952,
      "grad_norm": 2.7194995880126953,
      "learning_rate": 1.7292009141648335e-05,
      "logits/chosen": -0.28672319650650024,
      "logits/rejected": -1.3284246921539307,
      "logps/chosen": -0.7340124845504761,
      "logps/rejected": -9.563413619995117,
      "loss": 0.7401,
      "num_input_tokens_seen": 10051408,
      "odds_ratio_loss": 7.306893825531006,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.07340125739574432,
      "rewards/margins": 0.8829402923583984,
      "rewards/rejected": -0.9563414454460144,
      "sft_loss": 0.009374911896884441,
      "step": 7710
    },
    {
      "epoch": 0.6005445352003111,
      "grad_norm": 2.2125983238220215,
      "learning_rate": 1.723391197011412e-05,
      "logits/chosen": 0.05357031896710396,
      "logits/rejected": -1.37665593624115,
      "logps/chosen": -0.8307100534439087,
      "logps/rejected": -7.978257656097412,
      "loss": 0.8534,
      "num_input_tokens_seen": 10064832,
      "odds_ratio_loss": 8.482166290283203,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.08307100087404251,
      "rewards/margins": 0.7147548794746399,
      "rewards/rejected": -0.7978259325027466,
      "sft_loss": 0.005207848735153675,
      "step": 7720
    },
    {
      "epoch": 0.6013224426293271,
      "grad_norm": 4.190229415893555,
      "learning_rate": 1.717586118145617e-05,
      "logits/chosen": 0.09481069445610046,
      "logits/rejected": -1.8997443914413452,
      "logps/chosen": -1.0379550457000732,
      "logps/rejected": -10.335776329040527,
      "loss": 1.0536,
      "num_input_tokens_seen": 10078064,
      "odds_ratio_loss": 10.46440315246582,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.10379551351070404,
      "rewards/margins": 0.9297822117805481,
      "rewards/rejected": -1.0335776805877686,
      "sft_loss": 0.0071298182010650635,
      "step": 7730
    },
    {
      "epoch": 0.6021003500583431,
      "grad_norm": 1.5671765804290771,
      "learning_rate": 1.711785712238219e-05,
      "logits/chosen": -0.21722228825092316,
      "logits/rejected": -1.2630269527435303,
      "logps/chosen": -0.782172679901123,
      "logps/rejected": -13.395368576049805,
      "loss": 0.7957,
      "num_input_tokens_seen": 10093328,
      "odds_ratio_loss": 7.910400390625,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.07821726053953171,
      "rewards/margins": 1.261319637298584,
      "rewards/rejected": -1.3395369052886963,
      "sft_loss": 0.0046241264790296555,
      "step": 7740
    },
    {
      "epoch": 0.602878257487359,
      "grad_norm": 4.133308410644531,
      "learning_rate": 1.7059900139320788e-05,
      "logits/chosen": 0.09541847556829453,
      "logits/rejected": -1.746908187866211,
      "logps/chosen": -0.7769309282302856,
      "logps/rejected": -16.251314163208008,
      "loss": 0.7803,
      "num_input_tokens_seen": 10104752,
      "odds_ratio_loss": 7.769944667816162,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.07769308984279633,
      "rewards/margins": 1.547438383102417,
      "rewards/rejected": -1.625131607055664,
      "sft_loss": 0.003262136597186327,
      "step": 7750
    },
    {
      "epoch": 0.603656164916375,
      "grad_norm": 2.230847120285034,
      "learning_rate": 1.700199057841942e-05,
      "logits/chosen": -0.17776095867156982,
      "logits/rejected": -1.3732407093048096,
      "logps/chosen": -1.0622332096099854,
      "logps/rejected": -13.337030410766602,
      "loss": 1.0745,
      "num_input_tokens_seen": 10117184,
      "odds_ratio_loss": 10.697388648986816,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.10622330754995346,
      "rewards/margins": 1.2274798154830933,
      "rewards/rejected": -1.3337030410766602,
      "sft_loss": 0.00480131758376956,
      "step": 7760
    },
    {
      "epoch": 0.6044340723453909,
      "grad_norm": 5.956446647644043,
      "learning_rate": 1.694412878554229e-05,
      "logits/chosen": -0.011517253704369068,
      "logits/rejected": -1.2620794773101807,
      "logps/chosen": -1.0131255388259888,
      "logps/rejected": -16.502269744873047,
      "loss": 1.0178,
      "num_input_tokens_seen": 10129328,
      "odds_ratio_loss": 10.132184028625488,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.101312555372715,
      "rewards/margins": 1.5489144325256348,
      "rewards/rejected": -1.6502269506454468,
      "sft_loss": 0.004577781073749065,
      "step": 7770
    },
    {
      "epoch": 0.6052119797744069,
      "grad_norm": 7.566381454467773,
      "learning_rate": 1.6886315106268356e-05,
      "logits/chosen": -0.6538764238357544,
      "logits/rejected": -1.3822630643844604,
      "logps/chosen": -1.214513897895813,
      "logps/rejected": -17.518587112426758,
      "loss": 1.2516,
      "num_input_tokens_seen": 10142144,
      "odds_ratio_loss": 12.307482719421387,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.12145139276981354,
      "rewards/margins": 1.6304075717926025,
      "rewards/rejected": -1.7518587112426758,
      "sft_loss": 0.02084279991686344,
      "step": 7780
    },
    {
      "epoch": 0.6059898872034228,
      "grad_norm": 1.904586672782898,
      "learning_rate": 1.6828549885889168e-05,
      "logits/chosen": -0.040725018829107285,
      "logits/rejected": -1.3858834505081177,
      "logps/chosen": -0.8801521062850952,
      "logps/rejected": -8.406266212463379,
      "loss": 0.8922,
      "num_input_tokens_seen": 10153456,
      "odds_ratio_loss": 8.876083374023438,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08801521360874176,
      "rewards/margins": 0.7526113986968994,
      "rewards/rejected": -0.8406265377998352,
      "sft_loss": 0.004563276655972004,
      "step": 7790
    },
    {
      "epoch": 0.6067677946324388,
      "grad_norm": 1.555030107498169,
      "learning_rate": 1.677083346940688e-05,
      "logits/chosen": -0.042568035423755646,
      "logits/rejected": -1.6122106313705444,
      "logps/chosen": -0.903033435344696,
      "logps/rejected": -12.794513702392578,
      "loss": 0.9184,
      "num_input_tokens_seen": 10167520,
      "odds_ratio_loss": 9.131217956542969,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.09030334651470184,
      "rewards/margins": 1.1891480684280396,
      "rewards/rejected": -1.2794514894485474,
      "sft_loss": 0.005324669647961855,
      "step": 7800
    },
    {
      "epoch": 0.6075457020614546,
      "grad_norm": 6.690324783325195,
      "learning_rate": 1.671316620153218e-05,
      "logits/chosen": -0.06983957439661026,
      "logits/rejected": -1.5677844285964966,
      "logps/chosen": -0.8355168104171753,
      "logps/rejected": -13.89329719543457,
      "loss": 0.8519,
      "num_input_tokens_seen": 10179824,
      "odds_ratio_loss": 8.46717643737793,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.08355168998241425,
      "rewards/margins": 1.3057780265808105,
      "rewards/rejected": -1.3893296718597412,
      "sft_loss": 0.005196014419198036,
      "step": 7810
    },
    {
      "epoch": 0.6083236094904706,
      "grad_norm": 9.257512092590332,
      "learning_rate": 1.665554842668216e-05,
      "logits/chosen": -0.1954742968082428,
      "logits/rejected": -1.2004436254501343,
      "logps/chosen": -0.7859827876091003,
      "logps/rejected": -12.374399185180664,
      "loss": 0.7911,
      "num_input_tokens_seen": 10200016,
      "odds_ratio_loss": 7.818950653076172,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.0785982757806778,
      "rewards/margins": 1.1588417291641235,
      "rewards/rejected": -1.2374398708343506,
      "sft_loss": 0.00916118361055851,
      "step": 7820
    },
    {
      "epoch": 0.6091015169194866,
      "grad_norm": 5.428943634033203,
      "learning_rate": 1.6597980488978383e-05,
      "logits/chosen": -0.22211293876171112,
      "logits/rejected": -1.2993932962417603,
      "logps/chosen": -1.1758582592010498,
      "logps/rejected": -11.725682258605957,
      "loss": 1.1917,
      "num_input_tokens_seen": 10209760,
      "odds_ratio_loss": 10.848090171813965,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.11758583784103394,
      "rewards/margins": 1.054982304573059,
      "rewards/rejected": -1.1725682020187378,
      "sft_loss": 0.1069209948182106,
      "step": 7830
    },
    {
      "epoch": 0.6098794243485025,
      "grad_norm": 40.075374603271484,
      "learning_rate": 1.654046273224472e-05,
      "logits/chosen": -0.19958052039146423,
      "logits/rejected": -0.9851436614990234,
      "logps/chosen": -1.0347871780395508,
      "logps/rejected": -14.101142883300781,
      "loss": 1.0491,
      "num_input_tokens_seen": 10224176,
      "odds_ratio_loss": 9.863958358764648,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.10347872972488403,
      "rewards/margins": 1.3066356182098389,
      "rewards/rejected": -1.4101145267486572,
      "sft_loss": 0.06271268427371979,
      "step": 7840
    },
    {
      "epoch": 0.6106573317775185,
      "grad_norm": 7.990603446960449,
      "learning_rate": 1.648299550000535e-05,
      "logits/chosen": -0.23650245368480682,
      "logits/rejected": -1.7011165618896484,
      "logps/chosen": -1.1272385120391846,
      "logps/rejected": -12.137048721313477,
      "loss": 1.1298,
      "num_input_tokens_seen": 10236544,
      "odds_ratio_loss": 11.069969177246094,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.11272384971380234,
      "rewards/margins": 1.1009809970855713,
      "rewards/rejected": -1.2137049436569214,
      "sft_loss": 0.022801917046308517,
      "step": 7850
    },
    {
      "epoch": 0.6114352392065344,
      "grad_norm": 3.8824405670166016,
      "learning_rate": 1.6425579135482693e-05,
      "logits/chosen": -0.032112546265125275,
      "logits/rejected": -1.6255460977554321,
      "logps/chosen": -0.8585487604141235,
      "logps/rejected": -12.598340034484863,
      "loss": 0.8854,
      "num_input_tokens_seen": 10247648,
      "odds_ratio_loss": 8.798824310302734,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.08585488051176071,
      "rewards/margins": 1.1739790439605713,
      "rewards/rejected": -1.2598340511322021,
      "sft_loss": 0.0054823956452310085,
      "step": 7860
    },
    {
      "epoch": 0.6122131466355504,
      "grad_norm": 15.668688774108887,
      "learning_rate": 1.636821398159536e-05,
      "logits/chosen": -0.41416144371032715,
      "logits/rejected": -1.1996614933013916,
      "logps/chosen": -0.9376810193061829,
      "logps/rejected": -13.07994556427002,
      "loss": 0.9465,
      "num_input_tokens_seen": 10259424,
      "odds_ratio_loss": 9.251453399658203,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.09376810491085052,
      "rewards/margins": 1.2142266035079956,
      "rewards/rejected": -1.3079947233200073,
      "sft_loss": 0.021313045173883438,
      "step": 7870
    },
    {
      "epoch": 0.6129910540645663,
      "grad_norm": 5.761885643005371,
      "learning_rate": 1.6310900380956107e-05,
      "logits/chosen": -0.03201372176408768,
      "logits/rejected": -1.7225326299667358,
      "logps/chosen": -0.733839213848114,
      "logps/rejected": -21.30135726928711,
      "loss": 0.7377,
      "num_input_tokens_seen": 10272656,
      "odds_ratio_loss": 7.32309627532959,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.07338393479585648,
      "rewards/margins": 2.0567519664764404,
      "rewards/rejected": -2.1301357746124268,
      "sft_loss": 0.00536178145557642,
      "step": 7880
    },
    {
      "epoch": 0.6137689614935823,
      "grad_norm": 2.743339776992798,
      "learning_rate": 1.625363867586979e-05,
      "logits/chosen": -0.18800285458564758,
      "logits/rejected": -1.488651990890503,
      "logps/chosen": -1.2328776121139526,
      "logps/rejected": -16.766881942749023,
      "loss": 1.2778,
      "num_input_tokens_seen": 10283520,
      "odds_ratio_loss": 12.68993854522705,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.12328775227069855,
      "rewards/margins": 1.5534005165100098,
      "rewards/rejected": -1.6766881942749023,
      "sft_loss": 0.008772713132202625,
      "step": 7890
    },
    {
      "epoch": 0.6145468689225982,
      "grad_norm": 1.8026765584945679,
      "learning_rate": 1.619642920833132e-05,
      "logits/chosen": 0.11679697036743164,
      "logits/rejected": -1.7344423532485962,
      "logps/chosen": -1.8516442775726318,
      "logps/rejected": -18.924882888793945,
      "loss": 1.8524,
      "num_input_tokens_seen": 10300160,
      "odds_ratio_loss": 18.48210906982422,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1851644217967987,
      "rewards/margins": 1.7073237895965576,
      "rewards/rejected": -1.8924882411956787,
      "sft_loss": 0.004218951798975468,
      "step": 7900
    },
    {
      "epoch": 0.6153247763516142,
      "grad_norm": 3.9778380393981934,
      "learning_rate": 1.6139272320023623e-05,
      "logits/chosen": -0.24164679646492004,
      "logits/rejected": -0.9055463075637817,
      "logps/chosen": -0.7773641347885132,
      "logps/rejected": -5.6745500564575195,
      "loss": 0.7941,
      "num_input_tokens_seen": 10314528,
      "odds_ratio_loss": 7.78795862197876,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.07773640006780624,
      "rewards/margins": 0.48971861600875854,
      "rewards/rejected": -0.567454993724823,
      "sft_loss": 0.015351097099483013,
      "step": 7910
    },
    {
      "epoch": 0.6161026837806302,
      "grad_norm": 1.4372708797454834,
      "learning_rate": 1.6082168352315595e-05,
      "logits/chosen": -0.12904110550880432,
      "logits/rejected": -1.357168436050415,
      "logps/chosen": -1.0113813877105713,
      "logps/rejected": -11.638294219970703,
      "loss": 1.0317,
      "num_input_tokens_seen": 10327776,
      "odds_ratio_loss": 10.201348304748535,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.10113813728094101,
      "rewards/margins": 1.062691330909729,
      "rewards/rejected": -1.1638294458389282,
      "sft_loss": 0.01153038814663887,
      "step": 7920
    },
    {
      "epoch": 0.616880591209646,
      "grad_norm": 62.50389862060547,
      "learning_rate": 1.6025117646260064e-05,
      "logits/chosen": -0.26859474182128906,
      "logits/rejected": -1.2755110263824463,
      "logps/chosen": -1.3069630861282349,
      "logps/rejected": -12.296360969543457,
      "loss": 1.34,
      "num_input_tokens_seen": 10344272,
      "odds_ratio_loss": 11.950390815734863,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.13069631159305573,
      "rewards/margins": 1.0989396572113037,
      "rewards/rejected": -1.2296360731124878,
      "sft_loss": 0.14492058753967285,
      "step": 7930
    },
    {
      "epoch": 0.617658498638662,
      "grad_norm": 1.6316852569580078,
      "learning_rate": 1.5968120542591737e-05,
      "logits/chosen": 0.008070459589362144,
      "logits/rejected": -1.9038267135620117,
      "logps/chosen": -0.8354727625846863,
      "logps/rejected": -16.232791900634766,
      "loss": 0.8567,
      "num_input_tokens_seen": 10357504,
      "odds_ratio_loss": 8.52620792388916,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.08354727923870087,
      "rewards/margins": 1.5397322177886963,
      "rewards/rejected": -1.6232792139053345,
      "sft_loss": 0.004094335250556469,
      "step": 7940
    },
    {
      "epoch": 0.6184364060676779,
      "grad_norm": 3.011404275894165,
      "learning_rate": 1.5911177381725194e-05,
      "logits/chosen": -0.2592969536781311,
      "logits/rejected": -1.011600375175476,
      "logps/chosen": -1.0075538158416748,
      "logps/rejected": -10.504826545715332,
      "loss": 1.011,
      "num_input_tokens_seen": 10370448,
      "odds_ratio_loss": 9.980040550231934,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.10075537860393524,
      "rewards/margins": 0.9497271776199341,
      "rewards/rejected": -1.0504825115203857,
      "sft_loss": 0.01301506906747818,
      "step": 7950
    },
    {
      "epoch": 0.6192143134966939,
      "grad_norm": 2.981290102005005,
      "learning_rate": 1.585428850375286e-05,
      "logits/chosen": -0.11600154638290405,
      "logits/rejected": -1.1014983654022217,
      "logps/chosen": -0.946338951587677,
      "logps/rejected": -15.650550842285156,
      "loss": 0.98,
      "num_input_tokens_seen": 10381040,
      "odds_ratio_loss": 9.58875846862793,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.09463390707969666,
      "rewards/margins": 1.4704211950302124,
      "rewards/rejected": -1.565055012702942,
      "sft_loss": 0.02110918052494526,
      "step": 7960
    },
    {
      "epoch": 0.6199922209257098,
      "grad_norm": 7.606637477874756,
      "learning_rate": 1.5797454248442932e-05,
      "logits/chosen": -0.11081588268280029,
      "logits/rejected": -1.375892162322998,
      "logps/chosen": -0.8813081979751587,
      "logps/rejected": -9.27597427368164,
      "loss": 0.8969,
      "num_input_tokens_seen": 10392496,
      "odds_ratio_loss": 8.776693344116211,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.08813081681728363,
      "rewards/margins": 0.8394666910171509,
      "rewards/rejected": -0.9275975227355957,
      "sft_loss": 0.01920653134584427,
      "step": 7970
    },
    {
      "epoch": 0.6207701283547258,
      "grad_norm": 1.7592698335647583,
      "learning_rate": 1.5740674955237383e-05,
      "logits/chosen": -0.22309553623199463,
      "logits/rejected": -1.9166593551635742,
      "logps/chosen": -0.9842703938484192,
      "logps/rejected": -14.545404434204102,
      "loss": 0.9891,
      "num_input_tokens_seen": 10406624,
      "odds_ratio_loss": 9.730588912963867,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09842703491449356,
      "rewards/margins": 1.3561134338378906,
      "rewards/rejected": -1.4545403718948364,
      "sft_loss": 0.016037700697779655,
      "step": 7980
    },
    {
      "epoch": 0.6215480357837417,
      "grad_norm": 4.15437650680542,
      "learning_rate": 1.568395096324992e-05,
      "logits/chosen": -0.41758909821510315,
      "logits/rejected": -0.8042207956314087,
      "logps/chosen": -0.787390947341919,
      "logps/rejected": -20.126354217529297,
      "loss": 0.7915,
      "num_input_tokens_seen": 10420240,
      "odds_ratio_loss": 7.432351589202881,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.07873909175395966,
      "rewards/margins": 1.9338963031768799,
      "rewards/rejected": -2.0126352310180664,
      "sft_loss": 0.04827570170164108,
      "step": 7990
    },
    {
      "epoch": 0.6223259432127577,
      "grad_norm": 2.4224088191986084,
      "learning_rate": 1.562728261126398e-05,
      "logits/chosen": -0.1612803190946579,
      "logits/rejected": -1.6714760065078735,
      "logps/chosen": -1.6590150594711304,
      "logps/rejected": -12.734283447265625,
      "loss": 1.7128,
      "num_input_tokens_seen": 10434064,
      "odds_ratio_loss": 16.774051666259766,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.16590151190757751,
      "rewards/margins": 1.1075268983840942,
      "rewards/rejected": -1.2734283208847046,
      "sft_loss": 0.03537450358271599,
      "step": 8000
    },
    {
      "epoch": 0.6231038506417736,
      "grad_norm": 3.840738296508789,
      "learning_rate": 1.557067023773066e-05,
      "logits/chosen": -0.06671886146068573,
      "logits/rejected": -1.5114303827285767,
      "logps/chosen": -0.9673206210136414,
      "logps/rejected": -10.580313682556152,
      "loss": 0.9822,
      "num_input_tokens_seen": 10449616,
      "odds_ratio_loss": 9.774526596069336,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.09673206508159637,
      "rewards/margins": 0.9612993001937866,
      "rewards/rejected": -1.0580313205718994,
      "sft_loss": 0.004785637836903334,
      "step": 8010
    },
    {
      "epoch": 0.6238817580707896,
      "grad_norm": 3.5444207191467285,
      "learning_rate": 1.5514114180766758e-05,
      "logits/chosen": -0.044100046157836914,
      "logits/rejected": -1.8607912063598633,
      "logps/chosen": -1.0431252717971802,
      "logps/rejected": -11.461140632629395,
      "loss": 1.0497,
      "num_input_tokens_seen": 10462192,
      "odds_ratio_loss": 10.443166732788086,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.10431252419948578,
      "rewards/margins": 1.0418015718460083,
      "rewards/rejected": -1.1461141109466553,
      "sft_loss": 0.005407185293734074,
      "step": 8020
    },
    {
      "epoch": 0.6246596654998056,
      "grad_norm": 3.0164756774902344,
      "learning_rate": 1.5457614778152696e-05,
      "logits/chosen": -0.2252255231142044,
      "logits/rejected": -1.2585619688034058,
      "logps/chosen": -0.9918392896652222,
      "logps/rejected": -18.422239303588867,
      "loss": 0.994,
      "num_input_tokens_seen": 10475648,
      "odds_ratio_loss": 9.408598899841309,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09918393194675446,
      "rewards/margins": 1.743039846420288,
      "rewards/rejected": -1.842223882675171,
      "sft_loss": 0.05314243957400322,
      "step": 8030
    },
    {
      "epoch": 0.6254375729288215,
      "grad_norm": 3.0151381492614746,
      "learning_rate": 1.5401172367330563e-05,
      "logits/chosen": -0.320361852645874,
      "logits/rejected": -0.9782750010490417,
      "logps/chosen": -0.9511677622795105,
      "logps/rejected": -12.32588005065918,
      "loss": 0.9753,
      "num_input_tokens_seen": 10487008,
      "odds_ratio_loss": 9.589314460754395,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.09511677920818329,
      "rewards/margins": 1.1374711990356445,
      "rewards/rejected": -1.2325880527496338,
      "sft_loss": 0.016387831419706345,
      "step": 8040
    },
    {
      "epoch": 0.6262154803578374,
      "grad_norm": 13.852144241333008,
      "learning_rate": 1.5344787285402036e-05,
      "logits/chosen": 0.040863268077373505,
      "logits/rejected": -1.9360154867172241,
      "logps/chosen": -0.9229300618171692,
      "logps/rejected": -14.831850051879883,
      "loss": 0.9275,
      "num_input_tokens_seen": 10499472,
      "odds_ratio_loss": 9.100309371948242,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09229300916194916,
      "rewards/margins": 1.3908920288085938,
      "rewards/rejected": -1.483185052871704,
      "sft_loss": 0.01749167963862419,
      "step": 8050
    },
    {
      "epoch": 0.6269933877868533,
      "grad_norm": 5.214956760406494,
      "learning_rate": 1.528845986912639e-05,
      "logits/chosen": -0.15219607949256897,
      "logits/rejected": -1.030737280845642,
      "logps/chosen": -0.9987784624099731,
      "logps/rejected": -13.901496887207031,
      "loss": 1.0396,
      "num_input_tokens_seen": 10515952,
      "odds_ratio_loss": 10.083951950073242,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.09987784177064896,
      "rewards/margins": 1.2902718782424927,
      "rewards/rejected": -1.3901498317718506,
      "sft_loss": 0.03117167390882969,
      "step": 8060
    },
    {
      "epoch": 0.6277712952158693,
      "grad_norm": 1.3211863040924072,
      "learning_rate": 1.5232190454918518e-05,
      "logits/chosen": -0.3884350657463074,
      "logits/rejected": -1.1633074283599854,
      "logps/chosen": -0.9224312901496887,
      "logps/rejected": -15.697378158569336,
      "loss": 0.933,
      "num_input_tokens_seen": 10529248,
      "odds_ratio_loss": 8.834152221679688,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.09224313497543335,
      "rewards/margins": 1.4774949550628662,
      "rewards/rejected": -1.5697380304336548,
      "sft_loss": 0.049607302993535995,
      "step": 8070
    },
    {
      "epoch": 0.6285492026448852,
      "grad_norm": 7.5865702629089355,
      "learning_rate": 1.5175979378846874e-05,
      "logits/chosen": -0.14658516645431519,
      "logits/rejected": -1.6767528057098389,
      "logps/chosen": -1.00062894821167,
      "logps/rejected": -8.337983131408691,
      "loss": 1.017,
      "num_input_tokens_seen": 10542224,
      "odds_ratio_loss": 10.107731819152832,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.10006289184093475,
      "rewards/margins": 0.7337354421615601,
      "rewards/rejected": -0.8337982892990112,
      "sft_loss": 0.006207920610904694,
      "step": 8080
    },
    {
      "epoch": 0.6293271100739012,
      "grad_norm": 1.4249829053878784,
      "learning_rate": 1.511982697663151e-05,
      "logits/chosen": -0.07819326221942902,
      "logits/rejected": -1.4027016162872314,
      "logps/chosen": -0.9682968854904175,
      "logps/rejected": -8.290817260742188,
      "loss": 0.9892,
      "num_input_tokens_seen": 10551648,
      "odds_ratio_loss": 9.809918403625488,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.09682969748973846,
      "rewards/margins": 0.7322520017623901,
      "rewards/rejected": -0.8290818333625793,
      "sft_loss": 0.008252739906311035,
      "step": 8090
    },
    {
      "epoch": 0.6301050175029171,
      "grad_norm": 3.2043020725250244,
      "learning_rate": 1.5063733583642031e-05,
      "logits/chosen": -0.21396026015281677,
      "logits/rejected": -1.2566635608673096,
      "logps/chosen": -1.3404967784881592,
      "logps/rejected": -9.582209587097168,
      "loss": 1.3798,
      "num_input_tokens_seen": 10562768,
      "odds_ratio_loss": 13.57678508758545,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.1340496838092804,
      "rewards/margins": 0.8241713643074036,
      "rewards/rejected": -0.9582209587097168,
      "sft_loss": 0.022097880020737648,
      "step": 8100
    },
    {
      "epoch": 0.6308829249319331,
      "grad_norm": 2.394761323928833,
      "learning_rate": 1.500769953489562e-05,
      "logits/chosen": -0.2207452952861786,
      "logits/rejected": -1.4108034372329712,
      "logps/chosen": -0.9100346565246582,
      "logps/rejected": -10.773725509643555,
      "loss": 0.9213,
      "num_input_tokens_seen": 10580176,
      "odds_ratio_loss": 8.980841636657715,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.09100347757339478,
      "rewards/margins": 0.9863690137863159,
      "rewards/rejected": -1.0773725509643555,
      "sft_loss": 0.023260122165083885,
      "step": 8110
    },
    {
      "epoch": 0.631660832360949,
      "grad_norm": 2.3065590858459473,
      "learning_rate": 1.495172516505502e-05,
      "logits/chosen": -0.026962101459503174,
      "logits/rejected": -1.4148756265640259,
      "logps/chosen": -1.1877614259719849,
      "logps/rejected": -13.182968139648438,
      "loss": 1.1974,
      "num_input_tokens_seen": 10595664,
      "odds_ratio_loss": 11.025718688964844,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.11877612769603729,
      "rewards/margins": 1.199520468711853,
      "rewards/rejected": -1.3182967901229858,
      "sft_loss": 0.09481464326381683,
      "step": 8120
    },
    {
      "epoch": 0.632438739789965,
      "grad_norm": 2.421651601791382,
      "learning_rate": 1.4895810808426547e-05,
      "logits/chosen": -0.3440100848674774,
      "logits/rejected": -1.4067785739898682,
      "logps/chosen": -0.742415726184845,
      "logps/rejected": -27.202030181884766,
      "loss": 0.7445,
      "num_input_tokens_seen": 10609008,
      "odds_ratio_loss": 7.3816704750061035,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.07424157112836838,
      "rewards/margins": 2.645961284637451,
      "rewards/rejected": -2.720202922821045,
      "sft_loss": 0.0063473558984696865,
      "step": 8130
    },
    {
      "epoch": 0.633216647218981,
      "grad_norm": 3.3853743076324463,
      "learning_rate": 1.4839956798958088e-05,
      "logits/chosen": -0.15349504351615906,
      "logits/rejected": -1.580263614654541,
      "logps/chosen": -0.9464637637138367,
      "logps/rejected": -19.33546257019043,
      "loss": 0.9567,
      "num_input_tokens_seen": 10622096,
      "odds_ratio_loss": 9.516892433166504,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.0946463793516159,
      "rewards/margins": 1.8388999700546265,
      "rewards/rejected": -1.9335464239120483,
      "sft_loss": 0.005022524856030941,
      "step": 8140
    },
    {
      "epoch": 0.6339945546479969,
      "grad_norm": 3.37499737739563,
      "learning_rate": 1.4784163470237105e-05,
      "logits/chosen": 0.038762547075748444,
      "logits/rejected": -1.8333539962768555,
      "logps/chosen": -0.8853127360343933,
      "logps/rejected": -19.017131805419922,
      "loss": 0.8871,
      "num_input_tokens_seen": 10635136,
      "odds_ratio_loss": 8.821528434753418,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08853127062320709,
      "rewards/margins": 1.8131821155548096,
      "rewards/rejected": -1.9017131328582764,
      "sft_loss": 0.004929094575345516,
      "step": 8150
    },
    {
      "epoch": 0.6347724620770129,
      "grad_norm": 2.917329788208008,
      "learning_rate": 1.4728431155488648e-05,
      "logits/chosen": -0.06749407947063446,
      "logits/rejected": -1.3514297008514404,
      "logps/chosen": -0.7193356156349182,
      "logps/rejected": -10.20820140838623,
      "loss": 0.7239,
      "num_input_tokens_seen": 10645920,
      "odds_ratio_loss": 7.190954685211182,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.0719335600733757,
      "rewards/margins": 0.9488865733146667,
      "rewards/rejected": -1.020820140838623,
      "sft_loss": 0.004786919802427292,
      "step": 8160
    },
    {
      "epoch": 0.6355503695060288,
      "grad_norm": 17.307886123657227,
      "learning_rate": 1.4672760187573361e-05,
      "logits/chosen": 0.02161557599902153,
      "logits/rejected": -1.5112015008926392,
      "logps/chosen": -1.0977082252502441,
      "logps/rejected": -11.884286880493164,
      "loss": 1.1202,
      "num_input_tokens_seen": 10657040,
      "odds_ratio_loss": 11.025915145874023,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.10977083444595337,
      "rewards/margins": 1.078657865524292,
      "rewards/rejected": -1.1884286403656006,
      "sft_loss": 0.01760939136147499,
      "step": 8170
    },
    {
      "epoch": 0.6363282769350447,
      "grad_norm": 2.5105154514312744,
      "learning_rate": 1.4617150898985504e-05,
      "logits/chosen": -0.2205604761838913,
      "logits/rejected": -1.1295816898345947,
      "logps/chosen": -0.6735633611679077,
      "logps/rejected": -12.9937744140625,
      "loss": 0.6829,
      "num_input_tokens_seen": 10673296,
      "odds_ratio_loss": 6.721777439117432,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.06735634058713913,
      "rewards/margins": 1.2320213317871094,
      "rewards/rejected": -1.29937744140625,
      "sft_loss": 0.010709892027080059,
      "step": 8180
    },
    {
      "epoch": 0.6371061843640606,
      "grad_norm": 14.385485649108887,
      "learning_rate": 1.4561603621850933e-05,
      "logits/chosen": -0.20852942764759064,
      "logits/rejected": -1.5024021863937378,
      "logps/chosen": -1.0381171703338623,
      "logps/rejected": -20.33262062072754,
      "loss": 1.039,
      "num_input_tokens_seen": 10686848,
      "odds_ratio_loss": 10.2601900100708,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.10381171852350235,
      "rewards/margins": 1.9294503927230835,
      "rewards/rejected": -2.033262014389038,
      "sft_loss": 0.012959579937160015,
      "step": 8190
    },
    {
      "epoch": 0.6378840917930766,
      "grad_norm": 4.269653797149658,
      "learning_rate": 1.4506118687925158e-05,
      "logits/chosen": -0.19100096821784973,
      "logits/rejected": -1.4904474020004272,
      "logps/chosen": -0.8303168416023254,
      "logps/rejected": -18.543073654174805,
      "loss": 0.8505,
      "num_input_tokens_seen": 10699184,
      "odds_ratio_loss": 8.387426376342773,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.08303168416023254,
      "rewards/margins": 1.7712757587432861,
      "rewards/rejected": -1.8543074131011963,
      "sft_loss": 0.011750212870538235,
      "step": 8200
    },
    {
      "epoch": 0.6386619992220925,
      "grad_norm": 1.870347261428833,
      "learning_rate": 1.4450696428591346e-05,
      "logits/chosen": -0.008823069743812084,
      "logits/rejected": -2.0110087394714355,
      "logps/chosen": -0.8827959895133972,
      "logps/rejected": -27.75350570678711,
      "loss": 0.8932,
      "num_input_tokens_seen": 10714544,
      "odds_ratio_loss": 8.899144172668457,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.088279590010643,
      "rewards/margins": 2.6870710849761963,
      "rewards/rejected": -2.77535080909729,
      "sft_loss": 0.003280414966866374,
      "step": 8210
    },
    {
      "epoch": 0.6394399066511085,
      "grad_norm": 1.4320480823516846,
      "learning_rate": 1.4395337174858336e-05,
      "logits/chosen": -0.2863291800022125,
      "logits/rejected": -1.024493932723999,
      "logps/chosen": -0.9053533673286438,
      "logps/rejected": -8.440168380737305,
      "loss": 0.9208,
      "num_input_tokens_seen": 10729824,
      "odds_ratio_loss": 8.99482536315918,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.09053533524274826,
      "rewards/margins": 0.7534816861152649,
      "rewards/rejected": -0.8440170288085938,
      "sft_loss": 0.021277474239468575,
      "step": 8220
    },
    {
      "epoch": 0.6402178140801245,
      "grad_norm": 14.688983917236328,
      "learning_rate": 1.4340041257358678e-05,
      "logits/chosen": -0.026918869465589523,
      "logits/rejected": -1.681409478187561,
      "logps/chosen": -1.0343601703643799,
      "logps/rejected": -15.40973949432373,
      "loss": 1.0354,
      "num_input_tokens_seen": 10742416,
      "odds_ratio_loss": 10.315384864807129,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.10343600809574127,
      "rewards/margins": 1.4375379085540771,
      "rewards/rejected": -1.5409739017486572,
      "sft_loss": 0.0038158062379807234,
      "step": 8230
    },
    {
      "epoch": 0.6409957215091404,
      "grad_norm": 5.5650153160095215,
      "learning_rate": 1.4284809006346625e-05,
      "logits/chosen": -0.07448375225067139,
      "logits/rejected": -1.6491645574569702,
      "logps/chosen": -0.9802107810974121,
      "logps/rejected": -20.54385757446289,
      "loss": 0.9911,
      "num_input_tokens_seen": 10758448,
      "odds_ratio_loss": 9.796810150146484,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.09802107512950897,
      "rewards/margins": 1.956364631652832,
      "rewards/rejected": -2.0543856620788574,
      "sft_loss": 0.01137588545680046,
      "step": 8240
    },
    {
      "epoch": 0.6417736289381564,
      "grad_norm": 1.2503637075424194,
      "learning_rate": 1.4229640751696194e-05,
      "logits/chosen": -0.11880844831466675,
      "logits/rejected": -1.7745939493179321,
      "logps/chosen": -0.8601085543632507,
      "logps/rejected": -17.31156349182129,
      "loss": 0.8629,
      "num_input_tokens_seen": 10770160,
      "odds_ratio_loss": 8.552189826965332,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08601085841655731,
      "rewards/margins": 1.6451454162597656,
      "rewards/rejected": -1.731156349182129,
      "sft_loss": 0.0076408893801271915,
      "step": 8250
    },
    {
      "epoch": 0.6425515363671723,
      "grad_norm": 2.0524260997772217,
      "learning_rate": 1.4174536822899201e-05,
      "logits/chosen": -0.13288657367229462,
      "logits/rejected": -1.4250755310058594,
      "logps/chosen": -0.7484859228134155,
      "logps/rejected": -5.569182395935059,
      "loss": 0.7704,
      "num_input_tokens_seen": 10778368,
      "odds_ratio_loss": 7.6065993309021,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.07484859228134155,
      "rewards/margins": 0.48206964135169983,
      "rewards/rejected": -0.5569182634353638,
      "sft_loss": 0.009762279689311981,
      "step": 8260
    },
    {
      "epoch": 0.6433294437961883,
      "grad_norm": 3.603144884109497,
      "learning_rate": 1.4119497549063254e-05,
      "logits/chosen": -0.36294257640838623,
      "logits/rejected": -0.45799484848976135,
      "logps/chosen": -0.6412696838378906,
      "logps/rejected": -5.427443981170654,
      "loss": 0.6591,
      "num_input_tokens_seen": 10793760,
      "odds_ratio_loss": 6.5315985679626465,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.06412696838378906,
      "rewards/margins": 0.47861751914024353,
      "rewards/rejected": -0.5427444577217102,
      "sft_loss": 0.005894807167351246,
      "step": 8270
    },
    {
      "epoch": 0.6441073512252042,
      "grad_norm": 39.19443893432617,
      "learning_rate": 1.4064523258909812e-05,
      "logits/chosen": -0.17389366030693054,
      "logits/rejected": -1.5737632513046265,
      "logps/chosen": -1.433529257774353,
      "logps/rejected": -17.10720443725586,
      "loss": 1.4628,
      "num_input_tokens_seen": 10806208,
      "odds_ratio_loss": 13.620051383972168,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.1433529406785965,
      "rewards/margins": 1.5673677921295166,
      "rewards/rejected": -1.7107206583023071,
      "sft_loss": 0.10076556354761124,
      "step": 8280
    },
    {
      "epoch": 0.6448852586542202,
      "grad_norm": 3.161728858947754,
      "learning_rate": 1.4009614280772232e-05,
      "logits/chosen": -0.25053030252456665,
      "logits/rejected": -1.4600354433059692,
      "logps/chosen": -0.9723641276359558,
      "logps/rejected": -8.221105575561523,
      "loss": 0.9929,
      "num_input_tokens_seen": 10821328,
      "odds_ratio_loss": 9.868061065673828,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.09723640978336334,
      "rewards/margins": 0.7248741388320923,
      "rewards/rejected": -0.8221105337142944,
      "sft_loss": 0.006079370621591806,
      "step": 8290
    },
    {
      "epoch": 0.645663166083236,
      "grad_norm": 2.4112048149108887,
      "learning_rate": 1.3954770942593799e-05,
      "logits/chosen": -0.09294062852859497,
      "logits/rejected": -1.1986109018325806,
      "logps/chosen": -0.8862360119819641,
      "logps/rejected": -10.496671676635742,
      "loss": 0.9003,
      "num_input_tokens_seen": 10829936,
      "odds_ratio_loss": 8.87278938293457,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08862359821796417,
      "rewards/margins": 0.961043655872345,
      "rewards/rejected": -1.049667239189148,
      "sft_loss": 0.012980528175830841,
      "step": 8300
    },
    {
      "epoch": 0.646441073512252,
      "grad_norm": 9.586309432983398,
      "learning_rate": 1.3899993571925723e-05,
      "logits/chosen": -0.4729840159416199,
      "logits/rejected": -1.2745640277862549,
      "logps/chosen": -0.9752651453018188,
      "logps/rejected": -20.047935485839844,
      "loss": 0.9982,
      "num_input_tokens_seen": 10841696,
      "odds_ratio_loss": 9.761887550354004,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.09752652049064636,
      "rewards/margins": 1.9072672128677368,
      "rewards/rejected": -2.004793882369995,
      "sft_loss": 0.02201143279671669,
      "step": 8310
    },
    {
      "epoch": 0.647218980941268,
      "grad_norm": 3.2549245357513428,
      "learning_rate": 1.3845282495925277e-05,
      "logits/chosen": 0.06686688214540482,
      "logits/rejected": -1.5578182935714722,
      "logps/chosen": -0.7869054675102234,
      "logps/rejected": -15.545036315917969,
      "loss": 0.7966,
      "num_input_tokens_seen": 10858688,
      "odds_ratio_loss": 7.923579216003418,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.0786905512213707,
      "rewards/margins": 1.4758131504058838,
      "rewards/rejected": -1.5545036792755127,
      "sft_loss": 0.0042589083313941956,
      "step": 8320
    },
    {
      "epoch": 0.6479968883702839,
      "grad_norm": 1.3830910921096802,
      "learning_rate": 1.379063804135376e-05,
      "logits/chosen": 0.07782074809074402,
      "logits/rejected": -1.69546639919281,
      "logps/chosen": -1.0121997594833374,
      "logps/rejected": -18.158201217651367,
      "loss": 1.0353,
      "num_input_tokens_seen": 10873904,
      "odds_ratio_loss": 10.268865585327148,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.10121996700763702,
      "rewards/margins": 1.7145999670028687,
      "rewards/rejected": -1.8158200979232788,
      "sft_loss": 0.008449789136648178,
      "step": 8330
    },
    {
      "epoch": 0.6487747957992999,
      "grad_norm": 5.086806774139404,
      "learning_rate": 1.373606053457458e-05,
      "logits/chosen": -0.14860081672668457,
      "logits/rejected": -0.8564743995666504,
      "logps/chosen": -1.0690841674804688,
      "logps/rejected": -14.578874588012695,
      "loss": 1.0799,
      "num_input_tokens_seen": 10894304,
      "odds_ratio_loss": 10.592825889587402,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.106908418238163,
      "rewards/margins": 1.3509790897369385,
      "rewards/rejected": -1.4578876495361328,
      "sft_loss": 0.020593276247382164,
      "step": 8340
    },
    {
      "epoch": 0.6495527032283158,
      "grad_norm": 2.1482231616973877,
      "learning_rate": 1.368155030155131e-05,
      "logits/chosen": 0.05506908893585205,
      "logits/rejected": -1.4965693950653076,
      "logps/chosen": -0.7515527009963989,
      "logps/rejected": -9.797318458557129,
      "loss": 0.7538,
      "num_input_tokens_seen": 10908752,
      "odds_ratio_loss": 7.45101261138916,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.07515527307987213,
      "rewards/margins": 0.9045765995979309,
      "rewards/rejected": -0.9797319173812866,
      "sft_loss": 0.008677107281982899,
      "step": 8350
    },
    {
      "epoch": 0.6503306106573318,
      "grad_norm": 2.6537070274353027,
      "learning_rate": 1.3627107667845717e-05,
      "logits/chosen": -0.24765677750110626,
      "logits/rejected": -1.157957673072815,
      "logps/chosen": -0.8520873188972473,
      "logps/rejected": -6.802451133728027,
      "loss": 0.8713,
      "num_input_tokens_seen": 10922384,
      "odds_ratio_loss": 8.526128768920898,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.08520873636007309,
      "rewards/margins": 0.595036506652832,
      "rewards/rejected": -0.6802451014518738,
      "sft_loss": 0.01870984025299549,
      "step": 8360
    },
    {
      "epoch": 0.6511085180863477,
      "grad_norm": 4.838310718536377,
      "learning_rate": 1.3572732958615847e-05,
      "logits/chosen": -0.16812340915203094,
      "logits/rejected": -1.4218782186508179,
      "logps/chosen": -0.890440821647644,
      "logps/rejected": -16.090251922607422,
      "loss": 0.8994,
      "num_input_tokens_seen": 10935392,
      "odds_ratio_loss": 8.929390907287598,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.08904408663511276,
      "rewards/margins": 1.519981026649475,
      "rewards/rejected": -1.609025239944458,
      "sft_loss": 0.006434135138988495,
      "step": 8370
    },
    {
      "epoch": 0.6518864255153637,
      "grad_norm": 2.035271406173706,
      "learning_rate": 1.3518426498614039e-05,
      "logits/chosen": -0.36989468336105347,
      "logits/rejected": -1.2575443983078003,
      "logps/chosen": -0.9331470727920532,
      "logps/rejected": -11.751790046691895,
      "loss": 0.9469,
      "num_input_tokens_seen": 10950480,
      "odds_ratio_loss": 9.325362205505371,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09331471472978592,
      "rewards/margins": 1.081864356994629,
      "rewards/rejected": -1.1751790046691895,
      "sft_loss": 0.01432580966502428,
      "step": 8380
    },
    {
      "epoch": 0.6526643329443796,
      "grad_norm": 3.222139835357666,
      "learning_rate": 1.3464188612185058e-05,
      "logits/chosen": -0.20379066467285156,
      "logits/rejected": -1.3362772464752197,
      "logps/chosen": -0.7404529452323914,
      "logps/rejected": -11.38569450378418,
      "loss": 0.7573,
      "num_input_tokens_seen": 10965280,
      "odds_ratio_loss": 7.494593143463135,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.07404529303312302,
      "rewards/margins": 1.0645242929458618,
      "rewards/rejected": -1.1385693550109863,
      "sft_loss": 0.007848866283893585,
      "step": 8390
    },
    {
      "epoch": 0.6534422403733956,
      "grad_norm": 3.6170237064361572,
      "learning_rate": 1.3410019623264085e-05,
      "logits/chosen": -0.2227931022644043,
      "logits/rejected": -1.260690689086914,
      "logps/chosen": -1.0277048349380493,
      "logps/rejected": -14.743139266967773,
      "loss": 1.0475,
      "num_input_tokens_seen": 10977376,
      "odds_ratio_loss": 10.209665298461914,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.10277047008275986,
      "rewards/margins": 1.371543526649475,
      "rewards/rejected": -1.4743139743804932,
      "sft_loss": 0.026581311598420143,
      "step": 8400
    },
    {
      "epoch": 0.6542201478024116,
      "grad_norm": 2.616525650024414,
      "learning_rate": 1.3355919855374816e-05,
      "logits/chosen": -0.297484815120697,
      "logits/rejected": -1.609807014465332,
      "logps/chosen": -0.8975094556808472,
      "logps/rejected": -12.006285667419434,
      "loss": 0.9106,
      "num_input_tokens_seen": 10986112,
      "odds_ratio_loss": 8.958974838256836,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.08975095301866531,
      "rewards/margins": 1.1108777523040771,
      "rewards/rejected": -1.2006287574768066,
      "sft_loss": 0.014737960882484913,
      "step": 8410
    },
    {
      "epoch": 0.6549980552314275,
      "grad_norm": 3.0650668144226074,
      "learning_rate": 1.3301889631627517e-05,
      "logits/chosen": 0.17799173295497894,
      "logits/rejected": -2.1459360122680664,
      "logps/chosen": -0.8587332963943481,
      "logps/rejected": -16.28403663635254,
      "loss": 0.8656,
      "num_input_tokens_seen": 11001168,
      "odds_ratio_loss": 8.636218070983887,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.0858733281493187,
      "rewards/margins": 1.5425302982330322,
      "rewards/rejected": -1.628403902053833,
      "sft_loss": 0.0019893767312169075,
      "step": 8420
    },
    {
      "epoch": 0.6557759626604434,
      "grad_norm": 7.51597785949707,
      "learning_rate": 1.3247929274717134e-05,
      "logits/chosen": -0.13804951310157776,
      "logits/rejected": -1.4197338819503784,
      "logps/chosen": -0.6933324933052063,
      "logps/rejected": -13.103666305541992,
      "loss": 0.7249,
      "num_input_tokens_seen": 11011248,
      "odds_ratio_loss": 7.01900577545166,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.06933324784040451,
      "rewards/margins": 1.2410335540771484,
      "rewards/rejected": -1.3103668689727783,
      "sft_loss": 0.02296510711312294,
      "step": 8430
    },
    {
      "epoch": 0.6565538700894593,
      "grad_norm": 2.676499366760254,
      "learning_rate": 1.3194039106921285e-05,
      "logits/chosen": -0.29988133907318115,
      "logits/rejected": -1.4580739736557007,
      "logps/chosen": -0.8631059527397156,
      "logps/rejected": -20.36659049987793,
      "loss": 0.8746,
      "num_input_tokens_seen": 11028512,
      "odds_ratio_loss": 8.61852741241455,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08631060272455215,
      "rewards/margins": 1.9503482580184937,
      "rewards/rejected": -2.036658763885498,
      "sft_loss": 0.012698397040367126,
      "step": 8440
    },
    {
      "epoch": 0.6573317775184753,
      "grad_norm": 13.87569808959961,
      "learning_rate": 1.3140219450098418e-05,
      "logits/chosen": -0.4483930468559265,
      "logits/rejected": -1.1865017414093018,
      "logps/chosen": -0.8578631281852722,
      "logps/rejected": -11.311476707458496,
      "loss": 0.8759,
      "num_input_tokens_seen": 11040240,
      "odds_ratio_loss": 8.652433395385742,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.08578631281852722,
      "rewards/margins": 1.0453612804412842,
      "rewards/rejected": -1.1311476230621338,
      "sft_loss": 0.010646623559296131,
      "step": 8450
    },
    {
      "epoch": 0.6581096849474912,
      "grad_norm": 3.449667453765869,
      "learning_rate": 1.3086470625685854e-05,
      "logits/chosen": 0.011820548214018345,
      "logits/rejected": -2.015455722808838,
      "logps/chosen": -0.992369532585144,
      "logps/rejected": -19.755306243896484,
      "loss": 1.0005,
      "num_input_tokens_seen": 11054704,
      "odds_ratio_loss": 9.969560623168945,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09923695772886276,
      "rewards/margins": 1.8762937784194946,
      "rewards/rejected": -1.975530982017517,
      "sft_loss": 0.003559226170182228,
      "step": 8460
    },
    {
      "epoch": 0.6588875923765072,
      "grad_norm": 1.5920977592468262,
      "learning_rate": 1.3032792954697862e-05,
      "logits/chosen": 0.04607129842042923,
      "logits/rejected": -1.4695940017700195,
      "logps/chosen": -0.8204959034919739,
      "logps/rejected": -20.647167205810547,
      "loss": 0.8206,
      "num_input_tokens_seen": 11070592,
      "odds_ratio_loss": 8.174798965454102,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08204958587884903,
      "rewards/margins": 1.9826672077178955,
      "rewards/rejected": -2.0647168159484863,
      "sft_loss": 0.0030748662538826466,
      "step": 8470
    },
    {
      "epoch": 0.6596654998055231,
      "grad_norm": 65.4085922241211,
      "learning_rate": 1.2979186757723744e-05,
      "logits/chosen": -0.40389958024024963,
      "logits/rejected": -1.273033857345581,
      "logps/chosen": -0.9728702306747437,
      "logps/rejected": -11.638945579528809,
      "loss": 0.982,
      "num_input_tokens_seen": 11080528,
      "odds_ratio_loss": 9.580873489379883,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09728703647851944,
      "rewards/margins": 1.0666075944900513,
      "rewards/rejected": -1.163894534111023,
      "sft_loss": 0.023938244208693504,
      "step": 8480
    },
    {
      "epoch": 0.6604434072345391,
      "grad_norm": 3.122735023498535,
      "learning_rate": 1.2925652354925927e-05,
      "logits/chosen": -0.05589236691594124,
      "logits/rejected": -1.5410343408584595,
      "logps/chosen": -0.8380798101425171,
      "logps/rejected": -19.911548614501953,
      "loss": 0.8463,
      "num_input_tokens_seen": 11090880,
      "odds_ratio_loss": 8.223835945129395,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08380798250436783,
      "rewards/margins": 1.9073469638824463,
      "rewards/rejected": -1.9911549091339111,
      "sft_loss": 0.023908449336886406,
      "step": 8490
    },
    {
      "epoch": 0.661221314663555,
      "grad_norm": 3.586082935333252,
      "learning_rate": 1.2872190066038056e-05,
      "logits/chosen": -0.19564349949359894,
      "logits/rejected": -1.6615159511566162,
      "logps/chosen": -0.9936773180961609,
      "logps/rejected": -6.101696491241455,
      "loss": 1.0242,
      "num_input_tokens_seen": 11104016,
      "odds_ratio_loss": 10.164363861083984,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.09936773777008057,
      "rewards/margins": 0.5108018517494202,
      "rewards/rejected": -0.6101695895195007,
      "sft_loss": 0.0077539486810564995,
      "step": 8500
    },
    {
      "epoch": 0.661999222092571,
      "grad_norm": 3.438119888305664,
      "learning_rate": 1.2818800210363063e-05,
      "logits/chosen": -0.04792151600122452,
      "logits/rejected": -1.3143954277038574,
      "logps/chosen": -0.8305968046188354,
      "logps/rejected": -12.585901260375977,
      "loss": 0.8629,
      "num_input_tokens_seen": 11117712,
      "odds_ratio_loss": 8.393974304199219,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.08305969089269638,
      "rewards/margins": 1.1755305528640747,
      "rewards/rejected": -1.2585902214050293,
      "sft_loss": 0.023479651659727097,
      "step": 8510
    },
    {
      "epoch": 0.662777129521587,
      "grad_norm": 1.5884816646575928,
      "learning_rate": 1.2765483106771275e-05,
      "logits/chosen": -0.11270128190517426,
      "logits/rejected": -1.8259063959121704,
      "logps/chosen": -0.9288307428359985,
      "logps/rejected": -12.536392211914062,
      "loss": 0.9358,
      "num_input_tokens_seen": 11131488,
      "odds_ratio_loss": 9.29153823852539,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09288307279348373,
      "rewards/margins": 1.1607561111450195,
      "rewards/rejected": -1.2536393404006958,
      "sft_loss": 0.006653589196503162,
      "step": 8520
    },
    {
      "epoch": 0.6635550369506029,
      "grad_norm": 1.9690076112747192,
      "learning_rate": 1.271223907369853e-05,
      "logits/chosen": -0.41172775626182556,
      "logits/rejected": -1.3405262231826782,
      "logps/chosen": -0.9557029604911804,
      "logps/rejected": -12.360254287719727,
      "loss": 0.9689,
      "num_input_tokens_seen": 11141600,
      "odds_ratio_loss": 9.551517486572266,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.09557029604911804,
      "rewards/margins": 1.1404552459716797,
      "rewards/rejected": -1.2360254526138306,
      "sft_loss": 0.013766573742032051,
      "step": 8530
    },
    {
      "epoch": 0.6643329443796189,
      "grad_norm": 2.0265607833862305,
      "learning_rate": 1.2659068429144228e-05,
      "logits/chosen": -0.4004293382167816,
      "logits/rejected": -1.3477731943130493,
      "logps/chosen": -0.8370245099067688,
      "logps/rejected": -14.558506965637207,
      "loss": 0.8566,
      "num_input_tokens_seen": 11155056,
      "odds_ratio_loss": 8.357165336608887,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.0837024673819542,
      "rewards/margins": 1.3721481561660767,
      "rewards/rejected": -1.45585036277771,
      "sft_loss": 0.02086927741765976,
      "step": 8540
    },
    {
      "epoch": 0.6651108518086347,
      "grad_norm": 1.8647454977035522,
      "learning_rate": 1.2605971490669471e-05,
      "logits/chosen": -0.2736256420612335,
      "logits/rejected": -1.54111647605896,
      "logps/chosen": -0.7586938738822937,
      "logps/rejected": -18.400957107543945,
      "loss": 0.7653,
      "num_input_tokens_seen": 11171168,
      "odds_ratio_loss": 7.573142051696777,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.07586938887834549,
      "rewards/margins": 1.7642263174057007,
      "rewards/rejected": -1.8400957584381104,
      "sft_loss": 0.00795892346650362,
      "step": 8550
    },
    {
      "epoch": 0.6658887592376507,
      "grad_norm": 5.958194255828857,
      "learning_rate": 1.2552948575395135e-05,
      "logits/chosen": -0.16555894911289215,
      "logits/rejected": -1.250742793083191,
      "logps/chosen": -0.975231945514679,
      "logps/rejected": -8.279606819152832,
      "loss": 1.0221,
      "num_input_tokens_seen": 11181328,
      "odds_ratio_loss": 10.13719367980957,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.09752318263053894,
      "rewards/margins": 0.7304375171661377,
      "rewards/rejected": -0.8279607892036438,
      "sft_loss": 0.008426772430539131,
      "step": 8560
    },
    {
      "epoch": 0.6666666666666666,
      "grad_norm": 6.090371608734131,
      "learning_rate": 1.2500000000000006e-05,
      "logits/chosen": -0.26315200328826904,
      "logits/rejected": -1.2788677215576172,
      "logps/chosen": -0.8311313390731812,
      "logps/rejected": -16.073301315307617,
      "loss": 0.8437,
      "num_input_tokens_seen": 11191392,
      "odds_ratio_loss": 8.264939308166504,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08311314135789871,
      "rewards/margins": 1.5242170095443726,
      "rewards/rejected": -1.607330322265625,
      "sft_loss": 0.017202703282237053,
      "step": 8570
    },
    {
      "epoch": 0.6674445740956826,
      "grad_norm": 2.4439313411712646,
      "learning_rate": 1.2447126080718876e-05,
      "logits/chosen": -0.11930867284536362,
      "logits/rejected": -1.3498518466949463,
      "logps/chosen": -0.8209142684936523,
      "logps/rejected": -9.882196426391602,
      "loss": 0.8278,
      "num_input_tokens_seen": 11207888,
      "odds_ratio_loss": 8.222280502319336,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08209143579006195,
      "rewards/margins": 0.9061282277107239,
      "rewards/rejected": -0.9882196187973022,
      "sft_loss": 0.005587009247392416,
      "step": 8580
    },
    {
      "epoch": 0.6682224815246985,
      "grad_norm": 13.487736701965332,
      "learning_rate": 1.2394327133340658e-05,
      "logits/chosen": -0.32252174615859985,
      "logits/rejected": -1.2589683532714844,
      "logps/chosen": -0.8944998979568481,
      "logps/rejected": -17.84218978881836,
      "loss": 0.9001,
      "num_input_tokens_seen": 11220624,
      "odds_ratio_loss": 8.879426002502441,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08944999426603317,
      "rewards/margins": 1.6947689056396484,
      "rewards/rejected": -1.7842190265655518,
      "sft_loss": 0.012113329954445362,
      "step": 8590
    },
    {
      "epoch": 0.6690003889537145,
      "grad_norm": 3.8251841068267822,
      "learning_rate": 1.2341603473206486e-05,
      "logits/chosen": -0.5418458580970764,
      "logits/rejected": -1.5960874557495117,
      "logps/chosen": -1.3800010681152344,
      "logps/rejected": -14.487066268920898,
      "loss": 1.4506,
      "num_input_tokens_seen": 11232528,
      "odds_ratio_loss": 14.339879035949707,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.13800010085105896,
      "rewards/margins": 1.310706377029419,
      "rewards/rejected": -1.4487067461013794,
      "sft_loss": 0.016607291996479034,
      "step": 8600
    },
    {
      "epoch": 0.6697782963827305,
      "grad_norm": 12.286779403686523,
      "learning_rate": 1.2288955415207848e-05,
      "logits/chosen": -0.3529645800590515,
      "logits/rejected": -1.4659942388534546,
      "logps/chosen": -0.9061535000801086,
      "logps/rejected": -27.138967514038086,
      "loss": 0.9102,
      "num_input_tokens_seen": 11244800,
      "odds_ratio_loss": 9.013811111450195,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09061536192893982,
      "rewards/margins": 2.623281717300415,
      "rewards/rejected": -2.7138967514038086,
      "sft_loss": 0.008838102221488953,
      "step": 8610
    },
    {
      "epoch": 0.6705562038117464,
      "grad_norm": 3.788783073425293,
      "learning_rate": 1.2236383273784696e-05,
      "logits/chosen": -0.4239223599433899,
      "logits/rejected": -1.192138910293579,
      "logps/chosen": -0.9421186447143555,
      "logps/rejected": -14.43946647644043,
      "loss": 0.9622,
      "num_input_tokens_seen": 11254816,
      "odds_ratio_loss": 9.390454292297363,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.0942118763923645,
      "rewards/margins": 1.3497350215911865,
      "rewards/rejected": -1.4439469575881958,
      "sft_loss": 0.023183036595582962,
      "step": 8620
    },
    {
      "epoch": 0.6713341112407624,
      "grad_norm": 1.819946527481079,
      "learning_rate": 1.2183887362923577e-05,
      "logits/chosen": -0.10835647583007812,
      "logits/rejected": -1.4449514150619507,
      "logps/chosen": -0.9603055715560913,
      "logps/rejected": -14.043817520141602,
      "loss": 0.9647,
      "num_input_tokens_seen": 11264864,
      "odds_ratio_loss": 9.427949905395508,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09603056311607361,
      "rewards/margins": 1.3083511590957642,
      "rewards/rejected": -1.4043816328048706,
      "sft_loss": 0.021934445947408676,
      "step": 8630
    },
    {
      "epoch": 0.6721120186697783,
      "grad_norm": 1.8487447500228882,
      "learning_rate": 1.213146799615574e-05,
      "logits/chosen": 0.02626688778400421,
      "logits/rejected": -1.3874980211257935,
      "logps/chosen": -0.8622997403144836,
      "logps/rejected": -8.232680320739746,
      "loss": 0.88,
      "num_input_tokens_seen": 11275808,
      "odds_ratio_loss": 8.757139205932617,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.08622998744249344,
      "rewards/margins": 0.7370380759239197,
      "rewards/rejected": -0.8232681155204773,
      "sft_loss": 0.004265786614269018,
      "step": 8640
    },
    {
      "epoch": 0.6728899260987943,
      "grad_norm": 1.3646889925003052,
      "learning_rate": 1.2079125486555279e-05,
      "logits/chosen": -0.10303683578968048,
      "logits/rejected": -1.6437324285507202,
      "logps/chosen": -0.8649684190750122,
      "logps/rejected": -11.726491928100586,
      "loss": 0.8771,
      "num_input_tokens_seen": 11289264,
      "odds_ratio_loss": 8.71319580078125,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.08649684488773346,
      "rewards/margins": 1.0861523151397705,
      "rewards/rejected": -1.1726491451263428,
      "sft_loss": 0.0057486421428620815,
      "step": 8650
    },
    {
      "epoch": 0.6736678335278102,
      "grad_norm": 2.016709804534912,
      "learning_rate": 1.2026860146737281e-05,
      "logits/chosen": -0.7974362969398499,
      "logits/rejected": -1.0762536525726318,
      "logps/chosen": -0.8179294466972351,
      "logps/rejected": -10.683052062988281,
      "loss": 0.8308,
      "num_input_tokens_seen": 11301696,
      "odds_ratio_loss": 8.224560737609863,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08179295808076859,
      "rewards/margins": 0.9865123629570007,
      "rewards/rejected": -1.0683053731918335,
      "sft_loss": 0.008364574983716011,
      "step": 8660
    },
    {
      "epoch": 0.6744457409568262,
      "grad_norm": 1.868152379989624,
      "learning_rate": 1.1974672288855912e-05,
      "logits/chosen": -0.24265766143798828,
      "logits/rejected": -2.250298023223877,
      "logps/chosen": -0.9682738184928894,
      "logps/rejected": -21.465303421020508,
      "loss": 0.9782,
      "num_input_tokens_seen": 11315600,
      "odds_ratio_loss": 9.635416030883789,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.09682738035917282,
      "rewards/margins": 2.0497031211853027,
      "rewards/rejected": -2.1465301513671875,
      "sft_loss": 0.014634659513831139,
      "step": 8670
    },
    {
      "epoch": 0.675223648385842,
      "grad_norm": 1.5183236598968506,
      "learning_rate": 1.192256222460259e-05,
      "logits/chosen": -0.0016524322563782334,
      "logits/rejected": -1.906431794166565,
      "logps/chosen": -0.9167082905769348,
      "logps/rejected": -12.803431510925293,
      "loss": 0.9351,
      "num_input_tokens_seen": 11329840,
      "odds_ratio_loss": 9.321020126342773,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.09167083352804184,
      "rewards/margins": 1.1886723041534424,
      "rewards/rejected": -1.2803431749343872,
      "sft_loss": 0.0030473307706415653,
      "step": 8680
    },
    {
      "epoch": 0.676001555814858,
      "grad_norm": 2.5191078186035156,
      "learning_rate": 1.1870530265204091e-05,
      "logits/chosen": -0.3458879590034485,
      "logits/rejected": -1.210057020187378,
      "logps/chosen": -0.9681374430656433,
      "logps/rejected": -17.70801544189453,
      "loss": 0.9687,
      "num_input_tokens_seen": 11342304,
      "odds_ratio_loss": 9.132461547851562,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09681375324726105,
      "rewards/margins": 1.6739877462387085,
      "rewards/rejected": -1.7708015441894531,
      "sft_loss": 0.05550052598118782,
      "step": 8690
    },
    {
      "epoch": 0.676779463243874,
      "grad_norm": 11.486089706420898,
      "learning_rate": 1.1818576721420734e-05,
      "logits/chosen": -0.07480702549219131,
      "logits/rejected": -1.7288659811019897,
      "logps/chosen": -0.9699427485466003,
      "logps/rejected": -11.04030990600586,
      "loss": 0.9763,
      "num_input_tokens_seen": 11360608,
      "odds_ratio_loss": 9.68722152709961,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09699428081512451,
      "rewards/margins": 1.007036566734314,
      "rewards/rejected": -1.104030966758728,
      "sft_loss": 0.007532379124313593,
      "step": 8700
    },
    {
      "epoch": 0.6775573706728899,
      "grad_norm": 3.7408041954040527,
      "learning_rate": 1.1766701903544497e-05,
      "logits/chosen": -0.5660278797149658,
      "logits/rejected": -1.1048133373260498,
      "logps/chosen": -0.7535013556480408,
      "logps/rejected": -13.154385566711426,
      "loss": 0.7904,
      "num_input_tokens_seen": 11374576,
      "odds_ratio_loss": 7.830130100250244,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.07535013556480408,
      "rewards/margins": 1.2400884628295898,
      "rewards/rejected": -1.3154386281967163,
      "sft_loss": 0.007340291980654001,
      "step": 8710
    },
    {
      "epoch": 0.6783352781019059,
      "grad_norm": 1.3837281465530396,
      "learning_rate": 1.1714906121397162e-05,
      "logits/chosen": -0.07832653075456619,
      "logits/rejected": -1.9233297109603882,
      "logps/chosen": -1.125610589981079,
      "logps/rejected": -14.071528434753418,
      "loss": 1.1281,
      "num_input_tokens_seen": 11391488,
      "odds_ratio_loss": 11.223248481750488,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.11256106942892075,
      "rewards/margins": 1.2945916652679443,
      "rewards/rejected": -1.407152771949768,
      "sft_loss": 0.005823501385748386,
      "step": 8720
    },
    {
      "epoch": 0.6791131855309218,
      "grad_norm": 1.702384114265442,
      "learning_rate": 1.166318968432848e-05,
      "logits/chosen": -0.033839546144008636,
      "logits/rejected": -1.8430595397949219,
      "logps/chosen": -0.812944233417511,
      "logps/rejected": -22.31185531616211,
      "loss": 0.8148,
      "num_input_tokens_seen": 11404816,
      "odds_ratio_loss": 8.115874290466309,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08129443228244781,
      "rewards/margins": 2.1498911380767822,
      "rewards/rejected": -2.2311854362487793,
      "sft_loss": 0.0032049964647740126,
      "step": 8730
    },
    {
      "epoch": 0.6798910929599378,
      "grad_norm": 2.200639009475708,
      "learning_rate": 1.16115529012143e-05,
      "logits/chosen": 0.025216102600097656,
      "logits/rejected": -1.8154652118682861,
      "logps/chosen": -1.0452499389648438,
      "logps/rejected": -17.805185317993164,
      "loss": 1.0578,
      "num_input_tokens_seen": 11423280,
      "odds_ratio_loss": 10.54097843170166,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.10452499240636826,
      "rewards/margins": 1.67599356174469,
      "rewards/rejected": -1.7805185317993164,
      "sft_loss": 0.003734355792403221,
      "step": 8740
    },
    {
      "epoch": 0.6806690003889537,
      "grad_norm": 5.125865459442139,
      "learning_rate": 1.1559996080454752e-05,
      "logits/chosen": -0.46686309576034546,
      "logits/rejected": -1.0633283853530884,
      "logps/chosen": -1.0171476602554321,
      "logps/rejected": -12.088407516479492,
      "loss": 1.0336,
      "num_input_tokens_seen": 11435520,
      "odds_ratio_loss": 9.8032865524292,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.10171475261449814,
      "rewards/margins": 1.1071261167526245,
      "rewards/rejected": -1.2088408470153809,
      "sft_loss": 0.053316712379455566,
      "step": 8750
    },
    {
      "epoch": 0.6814469078179697,
      "grad_norm": 2.6860620975494385,
      "learning_rate": 1.150851952997239e-05,
      "logits/chosen": -0.27116113901138306,
      "logits/rejected": -1.396429419517517,
      "logps/chosen": -0.8566263318061829,
      "logps/rejected": -12.005642890930176,
      "loss": 0.8583,
      "num_input_tokens_seen": 11448432,
      "odds_ratio_loss": 8.500247955322266,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08566261827945709,
      "rewards/margins": 1.1149016618728638,
      "rewards/rejected": -1.2005643844604492,
      "sft_loss": 0.00828742515295744,
      "step": 8760
    },
    {
      "epoch": 0.6822248152469856,
      "grad_norm": 3.973215341567993,
      "learning_rate": 1.1457123557210347e-05,
      "logits/chosen": -0.6482058763504028,
      "logits/rejected": -0.9667502641677856,
      "logps/chosen": -0.8053656816482544,
      "logps/rejected": -10.670485496520996,
      "loss": 0.8311,
      "num_input_tokens_seen": 11457328,
      "odds_ratio_loss": 8.189837455749512,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.08053655922412872,
      "rewards/margins": 0.9865120053291321,
      "rewards/rejected": -1.0670485496520996,
      "sft_loss": 0.012087037786841393,
      "step": 8770
    },
    {
      "epoch": 0.6830027226760016,
      "grad_norm": 1.684739351272583,
      "learning_rate": 1.1405808469130504e-05,
      "logits/chosen": -0.32509225606918335,
      "logits/rejected": -1.5012362003326416,
      "logps/chosen": -0.9674826860427856,
      "logps/rejected": -18.22718048095703,
      "loss": 0.9736,
      "num_input_tokens_seen": 11471040,
      "odds_ratio_loss": 9.610559463500977,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09674827009439468,
      "rewards/margins": 1.725969910621643,
      "rewards/rejected": -1.8227182626724243,
      "sft_loss": 0.012532609514892101,
      "step": 8780
    },
    {
      "epoch": 0.6837806301050176,
      "grad_norm": 1.870661973953247,
      "learning_rate": 1.1354574572211685e-05,
      "logits/chosen": -0.03116082027554512,
      "logits/rejected": -1.4862477779388428,
      "logps/chosen": -0.7511264681816101,
      "logps/rejected": -10.093324661254883,
      "loss": 0.7804,
      "num_input_tokens_seen": 11483152,
      "odds_ratio_loss": 7.756175994873047,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.07511265575885773,
      "rewards/margins": 0.9342197179794312,
      "rewards/rejected": -1.0093324184417725,
      "sft_loss": 0.004796931054443121,
      "step": 8790
    },
    {
      "epoch": 0.6845585375340334,
      "grad_norm": 7.217814922332764,
      "learning_rate": 1.1303422172447781e-05,
      "logits/chosen": -0.1332283765077591,
      "logits/rejected": -1.6998363733291626,
      "logps/chosen": -0.7651488780975342,
      "logps/rejected": -13.795843124389648,
      "loss": 0.7664,
      "num_input_tokens_seen": 11494192,
      "odds_ratio_loss": 7.457201957702637,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.07651487737894058,
      "rewards/margins": 1.3030694723129272,
      "rewards/rejected": -1.3795844316482544,
      "sft_loss": 0.02067355439066887,
      "step": 8800
    },
    {
      "epoch": 0.6853364449630494,
      "grad_norm": 4.628015041351318,
      "learning_rate": 1.1252351575345923e-05,
      "logits/chosen": -0.05683094263076782,
      "logits/rejected": -1.7242136001586914,
      "logps/chosen": -0.7889854311943054,
      "logps/rejected": -22.717567443847656,
      "loss": 0.7949,
      "num_input_tokens_seen": 11509200,
      "odds_ratio_loss": 7.907063961029053,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.07889854162931442,
      "rewards/margins": 2.1928582191467285,
      "rewards/rejected": -2.271756649017334,
      "sft_loss": 0.004231655970215797,
      "step": 8810
    },
    {
      "epoch": 0.6861143523920653,
      "grad_norm": 2.469001531600952,
      "learning_rate": 1.1201363085924709e-05,
      "logits/chosen": -0.21477165818214417,
      "logits/rejected": -0.7675528526306152,
      "logps/chosen": -0.9782177805900574,
      "logps/rejected": -21.221256256103516,
      "loss": 1.0021,
      "num_input_tokens_seen": 11522208,
      "odds_ratio_loss": 9.50112247467041,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.09782177209854126,
      "rewards/margins": 2.024303913116455,
      "rewards/rejected": -2.1221256256103516,
      "sft_loss": 0.052003372460603714,
      "step": 8820
    },
    {
      "epoch": 0.6868922598210813,
      "grad_norm": 2.0622456073760986,
      "learning_rate": 1.1150457008712337e-05,
      "logits/chosen": 0.011802531778812408,
      "logits/rejected": -1.9207899570465088,
      "logps/chosen": -0.9674263000488281,
      "logps/rejected": -21.708913803100586,
      "loss": 0.9748,
      "num_input_tokens_seen": 11538160,
      "odds_ratio_loss": 9.71261978149414,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09674263000488281,
      "rewards/margins": 2.074148654937744,
      "rewards/rejected": -2.170891284942627,
      "sft_loss": 0.0035202347207814455,
      "step": 8830
    },
    {
      "epoch": 0.6876701672500972,
      "grad_norm": 2.0015759468078613,
      "learning_rate": 1.1099633647744797e-05,
      "logits/chosen": -0.19650664925575256,
      "logits/rejected": -1.1552398204803467,
      "logps/chosen": -2.600151538848877,
      "logps/rejected": -9.30272102355957,
      "loss": 2.6104,
      "num_input_tokens_seen": 11552928,
      "odds_ratio_loss": 25.9542179107666,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.26001518964767456,
      "rewards/margins": 0.6702569723129272,
      "rewards/rejected": -0.9302722215652466,
      "sft_loss": 0.014955626800656319,
      "step": 8840
    },
    {
      "epoch": 0.6884480746791132,
      "grad_norm": 2.001830577850342,
      "learning_rate": 1.1048893306564062e-05,
      "logits/chosen": -0.30013757944107056,
      "logits/rejected": -1.426927924156189,
      "logps/chosen": -0.8858613967895508,
      "logps/rejected": -11.036964416503906,
      "loss": 0.892,
      "num_input_tokens_seen": 11564832,
      "odds_ratio_loss": 8.833036422729492,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08858613669872284,
      "rewards/margins": 1.0151102542877197,
      "rewards/rejected": -1.103696346282959,
      "sft_loss": 0.008729054592549801,
      "step": 8850
    },
    {
      "epoch": 0.6892259821081291,
      "grad_norm": 1.8071199655532837,
      "learning_rate": 1.0998236288216265e-05,
      "logits/chosen": 0.06675352901220322,
      "logits/rejected": -1.804593801498413,
      "logps/chosen": -0.8722532987594604,
      "logps/rejected": -10.574170112609863,
      "loss": 0.8744,
      "num_input_tokens_seen": 11579696,
      "odds_ratio_loss": 8.713655471801758,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08722533285617828,
      "rewards/margins": 0.9701916575431824,
      "rewards/rejected": -1.0574170351028442,
      "sft_loss": 0.0030626547522842884,
      "step": 8860
    },
    {
      "epoch": 0.6900038895371451,
      "grad_norm": 2.2742514610290527,
      "learning_rate": 1.0947662895249896e-05,
      "logits/chosen": 0.007434540893882513,
      "logits/rejected": -1.6352884769439697,
      "logps/chosen": -0.8696354031562805,
      "logps/rejected": -18.652408599853516,
      "loss": 0.87,
      "num_input_tokens_seen": 11596480,
      "odds_ratio_loss": 8.673111915588379,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08696354180574417,
      "rewards/margins": 1.7782771587371826,
      "rewards/rejected": -1.8652406930923462,
      "sft_loss": 0.002644627820700407,
      "step": 8870
    },
    {
      "epoch": 0.690781796966161,
      "grad_norm": 3.000932455062866,
      "learning_rate": 1.0897173429714e-05,
      "logits/chosen": -0.13386768102645874,
      "logits/rejected": -1.3530477285385132,
      "logps/chosen": -0.9897897839546204,
      "logps/rejected": -15.522348403930664,
      "loss": 0.9965,
      "num_input_tokens_seen": 11615344,
      "odds_ratio_loss": 9.91053581237793,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09897898137569427,
      "rewards/margins": 1.4532560110092163,
      "rewards/rejected": -1.5522348880767822,
      "sft_loss": 0.005432153586298227,
      "step": 8880
    },
    {
      "epoch": 0.691559704395177,
      "grad_norm": 1.9952462911605835,
      "learning_rate": 1.0846768193156357e-05,
      "logits/chosen": -0.7853596210479736,
      "logits/rejected": -1.0207130908966064,
      "logps/chosen": -0.8637072443962097,
      "logps/rejected": -9.85428237915039,
      "loss": 0.8733,
      "num_input_tokens_seen": 11626128,
      "odds_ratio_loss": 8.149516105651855,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08637072145938873,
      "rewards/margins": 0.8990575075149536,
      "rewards/rejected": -0.9854282140731812,
      "sft_loss": 0.058370787650346756,
      "step": 8890
    },
    {
      "epoch": 0.692337611824193,
      "grad_norm": 11.468011856079102,
      "learning_rate": 1.0796447486621694e-05,
      "logits/chosen": -0.0654086098074913,
      "logits/rejected": -1.213006615638733,
      "logps/chosen": -0.8232144117355347,
      "logps/rejected": -9.072195053100586,
      "loss": 0.8323,
      "num_input_tokens_seen": 11635776,
      "odds_ratio_loss": 8.218587875366211,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.08232144266366959,
      "rewards/margins": 0.8248981237411499,
      "rewards/rejected": -0.9072195291519165,
      "sft_loss": 0.010446458123624325,
      "step": 8900
    },
    {
      "epoch": 0.6931155192532089,
      "grad_norm": 1.6070795059204102,
      "learning_rate": 1.0746211610649873e-05,
      "logits/chosen": 0.10436618328094482,
      "logits/rejected": -1.6011590957641602,
      "logps/chosen": -0.7685268521308899,
      "logps/rejected": -2.133186101913452,
      "loss": 0.7968,
      "num_input_tokens_seen": 11648464,
      "odds_ratio_loss": 7.8565216064453125,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.07685267925262451,
      "rewards/margins": 0.13646593689918518,
      "rewards/rejected": -0.2133186161518097,
      "sft_loss": 0.011129858903586864,
      "step": 8910
    },
    {
      "epoch": 0.6938934266822249,
      "grad_norm": 19.818248748779297,
      "learning_rate": 1.0696060865274139e-05,
      "logits/chosen": -0.2180163860321045,
      "logits/rejected": -1.0414024591445923,
      "logps/chosen": -0.8535948991775513,
      "logps/rejected": -6.71603536605835,
      "loss": 0.8646,
      "num_input_tokens_seen": 11663152,
      "odds_ratio_loss": 8.475765228271484,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08535948395729065,
      "rewards/margins": 0.5862440466880798,
      "rewards/rejected": -0.6716035008430481,
      "sft_loss": 0.01703769713640213,
      "step": 8920
    },
    {
      "epoch": 0.6946713341112407,
      "grad_norm": 12.964266777038574,
      "learning_rate": 1.0645995550019266e-05,
      "logits/chosen": -0.14962530136108398,
      "logits/rejected": -1.0459420680999756,
      "logps/chosen": -0.8487846255302429,
      "logps/rejected": -11.240068435668945,
      "loss": 0.8549,
      "num_input_tokens_seen": 11679952,
      "odds_ratio_loss": 8.277114868164062,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08487847447395325,
      "rewards/margins": 1.039128303527832,
      "rewards/rejected": -1.124006748199463,
      "sft_loss": 0.027226870879530907,
      "step": 8930
    },
    {
      "epoch": 0.6954492415402567,
      "grad_norm": 1.5695898532867432,
      "learning_rate": 1.0596015963899792e-05,
      "logits/chosen": -0.05155446380376816,
      "logits/rejected": -1.243017554283142,
      "logps/chosen": -0.868207573890686,
      "logps/rejected": -7.0557861328125,
      "loss": 0.8793,
      "num_input_tokens_seen": 11694208,
      "odds_ratio_loss": 8.7078218460083,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.08682074397802353,
      "rewards/margins": 0.6187578439712524,
      "rewards/rejected": -0.705578625202179,
      "sft_loss": 0.008534679189324379,
      "step": 8940
    },
    {
      "epoch": 0.6962271489692726,
      "grad_norm": 2.6625661849975586,
      "learning_rate": 1.054612240541826e-05,
      "logits/chosen": -0.12724608182907104,
      "logits/rejected": -1.3886604309082031,
      "logps/chosen": -0.8681074976921082,
      "logps/rejected": -17.198060989379883,
      "loss": 0.8759,
      "num_input_tokens_seen": 11705008,
      "odds_ratio_loss": 8.679101943969727,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08681075274944305,
      "rewards/margins": 1.6329952478408813,
      "rewards/rejected": -1.7198059558868408,
      "sft_loss": 0.00800777692347765,
      "step": 8950
    },
    {
      "epoch": 0.6970050563982886,
      "grad_norm": 5.75851583480835,
      "learning_rate": 1.0496315172563404e-05,
      "logits/chosen": 0.010910524055361748,
      "logits/rejected": -1.5142662525177002,
      "logps/chosen": -0.9781953692436218,
      "logps/rejected": -10.615856170654297,
      "loss": 0.9931,
      "num_input_tokens_seen": 11716576,
      "odds_ratio_loss": 9.874059677124023,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09781954437494278,
      "rewards/margins": 0.9637660980224609,
      "rewards/rejected": -1.061585783958435,
      "sft_loss": 0.005718961823731661,
      "step": 8960
    },
    {
      "epoch": 0.6977829638273045,
      "grad_norm": 1.7544348239898682,
      "learning_rate": 1.0446594562808382e-05,
      "logits/chosen": -0.22071878612041473,
      "logits/rejected": -1.1810603141784668,
      "logps/chosen": -0.7846574783325195,
      "logps/rejected": -7.4228668212890625,
      "loss": 0.8171,
      "num_input_tokens_seen": 11732480,
      "odds_ratio_loss": 8.076042175292969,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.07846574485301971,
      "rewards/margins": 0.6638209819793701,
      "rewards/rejected": -0.742286741733551,
      "sft_loss": 0.009545383974909782,
      "step": 8970
    },
    {
      "epoch": 0.6985608712563205,
      "grad_norm": 4.993774890899658,
      "learning_rate": 1.0396960873108992e-05,
      "logits/chosen": -0.11778923124074936,
      "logits/rejected": -1.838526964187622,
      "logps/chosen": -0.943067193031311,
      "logps/rejected": -21.153854370117188,
      "loss": 0.9468,
      "num_input_tokens_seen": 11744784,
      "odds_ratio_loss": 9.421133041381836,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09430670738220215,
      "rewards/margins": 2.0210790634155273,
      "rewards/rejected": -2.1153855323791504,
      "sft_loss": 0.004669210873544216,
      "step": 8980
    },
    {
      "epoch": 0.6993387786853364,
      "grad_norm": 3.575345993041992,
      "learning_rate": 1.0347414399901908e-05,
      "logits/chosen": -0.47023558616638184,
      "logits/rejected": -1.3087602853775024,
      "logps/chosen": -1.0606311559677124,
      "logps/rejected": -14.766632080078125,
      "loss": 1.062,
      "num_input_tokens_seen": 11757712,
      "odds_ratio_loss": 10.034512519836426,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1060631051659584,
      "rewards/margins": 1.3705999851226807,
      "rewards/rejected": -1.4766628742218018,
      "sft_loss": 0.058538712561130524,
      "step": 8990
    },
    {
      "epoch": 0.7001166861143524,
      "grad_norm": 4.2185163497924805,
      "learning_rate": 1.0297955439102899e-05,
      "logits/chosen": -0.19833922386169434,
      "logits/rejected": -1.2449114322662354,
      "logps/chosen": -0.9985941648483276,
      "logps/rejected": -22.485218048095703,
      "loss": 0.999,
      "num_input_tokens_seen": 11767792,
      "odds_ratio_loss": 9.736976623535156,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09985942393541336,
      "rewards/margins": 2.1486623287200928,
      "rewards/rejected": -2.248521566390991,
      "sft_loss": 0.025351542979478836,
      "step": 9000
    },
    {
      "epoch": 0.7008945935433684,
      "grad_norm": 5.648533821105957,
      "learning_rate": 1.0248584286105077e-05,
      "logits/chosen": -0.11871405690908432,
      "logits/rejected": -1.4459521770477295,
      "logps/chosen": -0.7193819880485535,
      "logps/rejected": -9.93423080444336,
      "loss": 0.7289,
      "num_input_tokens_seen": 11778512,
      "odds_ratio_loss": 7.127845764160156,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.07193820178508759,
      "rewards/margins": 0.9214849472045898,
      "rewards/rejected": -0.9934231042861938,
      "sft_loss": 0.01614934578537941,
      "step": 9010
    },
    {
      "epoch": 0.7016725009723843,
      "grad_norm": 1.579898476600647,
      "learning_rate": 1.0199301235777106e-05,
      "logits/chosen": 0.014426630921661854,
      "logits/rejected": -1.645169973373413,
      "logps/chosen": -1.0434606075286865,
      "logps/rejected": -11.167806625366211,
      "loss": 1.0792,
      "num_input_tokens_seen": 11794976,
      "odds_ratio_loss": 9.845039367675781,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.10434605926275253,
      "rewards/margins": 1.012434720993042,
      "rewards/rejected": -1.1167807579040527,
      "sft_loss": 0.09464666247367859,
      "step": 9020
    },
    {
      "epoch": 0.7024504084014003,
      "grad_norm": 19.114221572875977,
      "learning_rate": 1.0150106582461477e-05,
      "logits/chosen": -0.24926729500293732,
      "logits/rejected": -1.216196060180664,
      "logps/chosen": -0.697177529335022,
      "logps/rejected": -14.336259841918945,
      "loss": 0.7016,
      "num_input_tokens_seen": 11806400,
      "odds_ratio_loss": 6.796858310699463,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.06971774995326996,
      "rewards/margins": 1.3639085292816162,
      "rewards/rejected": -1.4336262941360474,
      "sft_loss": 0.021893560886383057,
      "step": 9030
    },
    {
      "epoch": 0.7032283158304162,
      "grad_norm": 4.123728275299072,
      "learning_rate": 1.0101000619972708e-05,
      "logits/chosen": -0.24322524666786194,
      "logits/rejected": -1.1093257665634155,
      "logps/chosen": -0.8399659991264343,
      "logps/rejected": -11.511434555053711,
      "loss": 0.8557,
      "num_input_tokens_seen": 11818976,
      "odds_ratio_loss": 8.476297378540039,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.08399660885334015,
      "rewards/margins": 1.0671470165252686,
      "rewards/rejected": -1.1511435508728027,
      "sft_loss": 0.008043365553021431,
      "step": 9040
    },
    {
      "epoch": 0.7040062232594321,
      "grad_norm": 2.056776523590088,
      "learning_rate": 1.0051983641595658e-05,
      "logits/chosen": -0.26209187507629395,
      "logits/rejected": -1.374551773071289,
      "logps/chosen": -0.8355258107185364,
      "logps/rejected": -15.121932983398438,
      "loss": 0.8437,
      "num_input_tokens_seen": 11830848,
      "odds_ratio_loss": 8.353340148925781,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08355257660150528,
      "rewards/margins": 1.428640604019165,
      "rewards/rejected": -1.512193202972412,
      "sft_loss": 0.008403485640883446,
      "step": 9050
    },
    {
      "epoch": 0.704784130688448,
      "grad_norm": 1.8826544284820557,
      "learning_rate": 1.0003055940083661e-05,
      "logits/chosen": -0.6671730279922485,
      "logits/rejected": -1.399988055229187,
      "logps/chosen": -0.7520816922187805,
      "logps/rejected": -17.23227310180664,
      "loss": 0.7572,
      "num_input_tokens_seen": 11840480,
      "odds_ratio_loss": 7.3093414306640625,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.07520817220211029,
      "rewards/margins": 1.6480194330215454,
      "rewards/rejected": -1.7232272624969482,
      "sft_loss": 0.02628893218934536,
      "step": 9060
    },
    {
      "epoch": 0.705562038117464,
      "grad_norm": 11.5000638961792,
      "learning_rate": 9.954217807656898e-06,
      "logits/chosen": -0.24273736774921417,
      "logits/rejected": -1.2386590242385864,
      "logps/chosen": -3.6087188720703125,
      "logps/rejected": -13.7881441116333,
      "loss": 3.6154,
      "num_input_tokens_seen": 11851088,
      "odds_ratio_loss": 34.90401077270508,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.36087191104888916,
      "rewards/margins": 1.0179424285888672,
      "rewards/rejected": -1.3788143396377563,
      "sft_loss": 0.12496994435787201,
      "step": 9070
    },
    {
      "epoch": 0.7063399455464799,
      "grad_norm": 6.532557964324951,
      "learning_rate": 9.905469536000583e-06,
      "logits/chosen": -0.4153507649898529,
      "logits/rejected": -1.344873070716858,
      "logps/chosen": -0.8688298463821411,
      "logps/rejected": -10.471773147583008,
      "loss": 0.8814,
      "num_input_tokens_seen": 11862384,
      "odds_ratio_loss": 8.589521408081055,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.08688297867774963,
      "rewards/margins": 0.960294246673584,
      "rewards/rejected": -1.0471774339675903,
      "sft_loss": 0.022413309663534164,
      "step": 9080
    },
    {
      "epoch": 0.7071178529754959,
      "grad_norm": 0.696464478969574,
      "learning_rate": 9.856811416263234e-06,
      "logits/chosen": -0.236698180437088,
      "logits/rejected": -1.628491997718811,
      "logps/chosen": -0.6710294485092163,
      "logps/rejected": -18.233524322509766,
      "loss": 0.678,
      "num_input_tokens_seen": 11876304,
      "odds_ratio_loss": 6.749547004699707,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.06710294634103775,
      "rewards/margins": 1.7562497854232788,
      "rewards/rejected": -1.823352575302124,
      "sft_loss": 0.003054919885471463,
      "step": 9090
    },
    {
      "epoch": 0.7078957604045119,
      "grad_norm": 3.7164294719696045,
      "learning_rate": 9.808243739054945e-06,
      "logits/chosen": -0.09606461226940155,
      "logits/rejected": -1.6019067764282227,
      "logps/chosen": -1.0349172353744507,
      "logps/rejected": -13.971864700317383,
      "loss": 1.0452,
      "num_input_tokens_seen": 11889872,
      "odds_ratio_loss": 10.380487442016602,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.10349173843860626,
      "rewards/margins": 1.2936947345733643,
      "rewards/rejected": -1.397186517715454,
      "sft_loss": 0.007199641317129135,
      "step": 9100
    },
    {
      "epoch": 0.7086736678335278,
      "grad_norm": 2.627157211303711,
      "learning_rate": 9.75976679444564e-06,
      "logits/chosen": -0.26083993911743164,
      "logits/rejected": -1.5357798337936401,
      "logps/chosen": -0.9003885388374329,
      "logps/rejected": -10.310569763183594,
      "loss": 0.9186,
      "num_input_tokens_seen": 11898656,
      "odds_ratio_loss": 8.754205703735352,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.09003885090351105,
      "rewards/margins": 0.9410181045532227,
      "rewards/rejected": -1.0310567617416382,
      "sft_loss": 0.04318580403923988,
      "step": 9110
    },
    {
      "epoch": 0.7094515752625438,
      "grad_norm": 1.5432356595993042,
      "learning_rate": 9.711380871963343e-06,
      "logits/chosen": -0.36176571249961853,
      "logits/rejected": -1.313044548034668,
      "logps/chosen": -0.6721317172050476,
      "logps/rejected": -17.497119903564453,
      "loss": 0.6738,
      "num_input_tokens_seen": 11912496,
      "odds_ratio_loss": 6.629150390625,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.06721317768096924,
      "rewards/margins": 1.682498574256897,
      "rewards/rejected": -1.7497117519378662,
      "sft_loss": 0.010848169215023518,
      "step": 9120
    },
    {
      "epoch": 0.7102294826915597,
      "grad_norm": 4.048521995544434,
      "learning_rate": 9.663086260592447e-06,
      "logits/chosen": -0.3890684247016907,
      "logits/rejected": -1.5890153646469116,
      "logps/chosen": -0.7857718467712402,
      "logps/rejected": -24.208412170410156,
      "loss": 0.7861,
      "num_input_tokens_seen": 11925536,
      "odds_ratio_loss": 7.81637716293335,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.0785771906375885,
      "rewards/margins": 2.342264175415039,
      "rewards/rejected": -2.4208412170410156,
      "sft_loss": 0.004494016990065575,
      "step": 9130
    },
    {
      "epoch": 0.7110073901205757,
      "grad_norm": 2.3026251792907715,
      "learning_rate": 9.614883248771996e-06,
      "logits/chosen": -0.17405235767364502,
      "logits/rejected": -1.2665290832519531,
      "logps/chosen": -0.9642848968505859,
      "logps/rejected": -14.18018913269043,
      "loss": 0.9792,
      "num_input_tokens_seen": 11938576,
      "odds_ratio_loss": 9.722308158874512,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.09642849117517471,
      "rewards/margins": 1.321590542793274,
      "rewards/rejected": -1.4180189371109009,
      "sft_loss": 0.006958406418561935,
      "step": 9140
    },
    {
      "epoch": 0.7117852975495916,
      "grad_norm": 1.8911758661270142,
      "learning_rate": 9.566772124393952e-06,
      "logits/chosen": -0.11972236633300781,
      "logits/rejected": -2.1072115898132324,
      "logps/chosen": -1.1833240985870361,
      "logps/rejected": -21.958148956298828,
      "loss": 1.193,
      "num_input_tokens_seen": 11952224,
      "odds_ratio_loss": 11.897111892700195,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.1183324009180069,
      "rewards/margins": 2.0774824619293213,
      "rewards/rejected": -2.195814847946167,
      "sft_loss": 0.003329962957650423,
      "step": 9150
    },
    {
      "epoch": 0.7125632049786076,
      "grad_norm": 2.6128060817718506,
      "learning_rate": 9.518753174801484e-06,
      "logits/chosen": -0.2604648768901825,
      "logits/rejected": -1.4065415859222412,
      "logps/chosen": -0.8946755528450012,
      "logps/rejected": -10.743023872375488,
      "loss": 0.9171,
      "num_input_tokens_seen": 11967408,
      "odds_ratio_loss": 9.056489944458008,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.08946755528450012,
      "rewards/margins": 0.9848347902297974,
      "rewards/rejected": -1.074302315711975,
      "sft_loss": 0.011441009119153023,
      "step": 9160
    },
    {
      "epoch": 0.7133411124076235,
      "grad_norm": 3.6682426929473877,
      "learning_rate": 9.470826686787235e-06,
      "logits/chosen": -0.0443451926112175,
      "logits/rejected": -1.7861120700836182,
      "logps/chosen": -1.0642540454864502,
      "logps/rejected": -9.87019157409668,
      "loss": 1.0997,
      "num_input_tokens_seen": 11980944,
      "odds_ratio_loss": 10.783086776733398,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.10642538964748383,
      "rewards/margins": 0.8805936574935913,
      "rewards/rejected": -0.9870191812515259,
      "sft_loss": 0.021341949701309204,
      "step": 9170
    },
    {
      "epoch": 0.7141190198366394,
      "grad_norm": 2.1391987800598145,
      "learning_rate": 9.42299294659166e-06,
      "logits/chosen": -0.08032462000846863,
      "logits/rejected": -1.516364336013794,
      "logps/chosen": -0.9333599209785461,
      "logps/rejected": -15.166709899902344,
      "loss": 0.9447,
      "num_input_tokens_seen": 11992208,
      "odds_ratio_loss": 9.370416641235352,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09333600103855133,
      "rewards/margins": 1.4233348369598389,
      "rewards/rejected": -1.5166709423065186,
      "sft_loss": 0.0076144421473145485,
      "step": 9180
    },
    {
      "epoch": 0.7148969272656553,
      "grad_norm": 13.14161491394043,
      "learning_rate": 9.375252239901225e-06,
      "logits/chosen": -0.20920245349407196,
      "logits/rejected": -1.2209100723266602,
      "logps/chosen": -0.9390764236450195,
      "logps/rejected": -7.180729866027832,
      "loss": 0.9496,
      "num_input_tokens_seen": 12003008,
      "odds_ratio_loss": 9.222626686096191,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09390763938426971,
      "rewards/margins": 0.6241652369499207,
      "rewards/rejected": -0.7180729508399963,
      "sft_loss": 0.027297839522361755,
      "step": 9190
    },
    {
      "epoch": 0.7156748346946713,
      "grad_norm": 1.4618985652923584,
      "learning_rate": 9.327604851846791e-06,
      "logits/chosen": -0.06988964974880219,
      "logits/rejected": -1.7549928426742554,
      "logps/chosen": -0.7257611751556396,
      "logps/rejected": -17.123815536499023,
      "loss": 0.7286,
      "num_input_tokens_seen": 12016240,
      "odds_ratio_loss": 7.260410308837891,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.0725761204957962,
      "rewards/margins": 1.6398054361343384,
      "rewards/rejected": -1.7123816013336182,
      "sft_loss": 0.002546419855207205,
      "step": 9200
    },
    {
      "epoch": 0.7164527421236873,
      "grad_norm": 2.0492186546325684,
      "learning_rate": 9.280051067001865e-06,
      "logits/chosen": -0.11623696237802505,
      "logits/rejected": -1.9431318044662476,
      "logps/chosen": -0.816973865032196,
      "logps/rejected": -23.292240142822266,
      "loss": 0.8197,
      "num_input_tokens_seen": 12026816,
      "odds_ratio_loss": 7.786641597747803,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08169738948345184,
      "rewards/margins": 2.2475266456604004,
      "rewards/rejected": -2.329224109649658,
      "sft_loss": 0.04104030877351761,
      "step": 9210
    },
    {
      "epoch": 0.7172306495527032,
      "grad_norm": 1.845423698425293,
      "learning_rate": 9.232591169380912e-06,
      "logits/chosen": -0.03926096111536026,
      "logits/rejected": -1.3892351388931274,
      "logps/chosen": -0.8934553861618042,
      "logps/rejected": -15.741470336914062,
      "loss": 0.9061,
      "num_input_tokens_seen": 12040848,
      "odds_ratio_loss": 8.62596321105957,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.0893455296754837,
      "rewards/margins": 1.4848015308380127,
      "rewards/rejected": -1.5741469860076904,
      "sft_loss": 0.04347647354006767,
      "step": 9220
    },
    {
      "epoch": 0.7180085569817192,
      "grad_norm": 3.811688184738159,
      "learning_rate": 9.185225442437648e-06,
      "logits/chosen": -0.5131353735923767,
      "logits/rejected": -1.2000887393951416,
      "logps/chosen": -0.810464084148407,
      "logps/rejected": -12.801618576049805,
      "loss": 0.8145,
      "num_input_tokens_seen": 12052192,
      "odds_ratio_loss": 7.7980546951293945,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08104641735553741,
      "rewards/margins": 1.199115514755249,
      "rewards/rejected": -1.2801618576049805,
      "sft_loss": 0.03466106206178665,
      "step": 9230
    },
    {
      "epoch": 0.7187864644107351,
      "grad_norm": 8.794293403625488,
      "learning_rate": 9.137954169063365e-06,
      "logits/chosen": -0.12427148967981339,
      "logits/rejected": -1.897454857826233,
      "logps/chosen": -0.7215501666069031,
      "logps/rejected": -11.68718147277832,
      "loss": 0.7312,
      "num_input_tokens_seen": 12067040,
      "odds_ratio_loss": 7.176033973693848,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.07215501368045807,
      "rewards/margins": 1.0965631008148193,
      "rewards/rejected": -1.1687180995941162,
      "sft_loss": 0.013598406687378883,
      "step": 9240
    },
    {
      "epoch": 0.7195643718397511,
      "grad_norm": 1.840759038925171,
      "learning_rate": 9.09077763158523e-06,
      "logits/chosen": -0.05737997964024544,
      "logits/rejected": -1.8592857122421265,
      "logps/chosen": -0.8444797396659851,
      "logps/rejected": -23.798778533935547,
      "loss": 0.8557,
      "num_input_tokens_seen": 12083264,
      "odds_ratio_loss": 8.520927429199219,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.08444797247648239,
      "rewards/margins": 2.2954299449920654,
      "rewards/rejected": -2.379878044128418,
      "sft_loss": 0.0036380328238010406,
      "step": 9250
    },
    {
      "epoch": 0.720342279268767,
      "grad_norm": 1.4600502252578735,
      "learning_rate": 9.043696111764598e-06,
      "logits/chosen": -0.12732329964637756,
      "logits/rejected": -1.7196537256240845,
      "logps/chosen": -0.8292292356491089,
      "logps/rejected": -17.078582763671875,
      "loss": 0.8535,
      "num_input_tokens_seen": 12100672,
      "odds_ratio_loss": 8.224623680114746,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.08292293548583984,
      "rewards/margins": 1.6249353885650635,
      "rewards/rejected": -1.7078583240509033,
      "sft_loss": 0.031013712286949158,
      "step": 9260
    },
    {
      "epoch": 0.721120186697783,
      "grad_norm": 47.71871566772461,
      "learning_rate": 8.996709890795338e-06,
      "logits/chosen": -0.3540479242801666,
      "logits/rejected": -1.249452829360962,
      "logps/chosen": -0.9343889355659485,
      "logps/rejected": -9.030014038085938,
      "loss": 0.9404,
      "num_input_tokens_seen": 12111344,
      "odds_ratio_loss": 8.981085777282715,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09343889355659485,
      "rewards/margins": 0.8095625638961792,
      "rewards/rejected": -0.9030014872550964,
      "sft_loss": 0.04224556311964989,
      "step": 9270
    },
    {
      "epoch": 0.721898094126799,
      "grad_norm": 1.8473955392837524,
      "learning_rate": 8.949819249302138e-06,
      "logits/chosen": -0.08845169842243195,
      "logits/rejected": -1.9312286376953125,
      "logps/chosen": -0.8921386003494263,
      "logps/rejected": -14.9306640625,
      "loss": 0.9023,
      "num_input_tokens_seen": 12126928,
      "odds_ratio_loss": 8.729538917541504,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08921386301517487,
      "rewards/margins": 1.4038527011871338,
      "rewards/rejected": -1.4930665493011475,
      "sft_loss": 0.029351364821195602,
      "step": 9280
    },
    {
      "epoch": 0.7226760015558149,
      "grad_norm": 4.839386463165283,
      "learning_rate": 8.903024467338846e-06,
      "logits/chosen": -0.5877536535263062,
      "logits/rejected": -1.3349168300628662,
      "logps/chosen": -1.2354286909103394,
      "logps/rejected": -17.281709671020508,
      "loss": 1.2405,
      "num_input_tokens_seen": 12138704,
      "odds_ratio_loss": 11.595271110534668,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.12354286760091782,
      "rewards/margins": 1.6046279668807983,
      "rewards/rejected": -1.7281709909439087,
      "sft_loss": 0.0809488296508789,
      "step": 9290
    },
    {
      "epoch": 0.7234539089848308,
      "grad_norm": 2.161583185195923,
      "learning_rate": 8.856325824386792e-06,
      "logits/chosen": -0.2935566306114197,
      "logits/rejected": -1.9119758605957031,
      "logps/chosen": -0.9439760446548462,
      "logps/rejected": -15.964810371398926,
      "loss": 0.9524,
      "num_input_tokens_seen": 12147984,
      "odds_ratio_loss": 9.423673629760742,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09439760446548462,
      "rewards/margins": 1.5020835399627686,
      "rewards/rejected": -1.5964810848236084,
      "sft_loss": 0.010013467632234097,
      "step": 9300
    },
    {
      "epoch": 0.7242318164138467,
      "grad_norm": 3.4106974601745605,
      "learning_rate": 8.809723599353106e-06,
      "logits/chosen": -0.6587649583816528,
      "logits/rejected": -0.8816956281661987,
      "logps/chosen": -0.93231201171875,
      "logps/rejected": -10.11970329284668,
      "loss": 0.9529,
      "num_input_tokens_seen": 12161680,
      "odds_ratio_loss": 9.175849914550781,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.0932312160730362,
      "rewards/margins": 0.9187391400337219,
      "rewards/rejected": -1.0119702816009521,
      "sft_loss": 0.0353216677904129,
      "step": 9310
    },
    {
      "epoch": 0.7250097238428627,
      "grad_norm": 2.1136739253997803,
      "learning_rate": 8.763218070569073e-06,
      "logits/chosen": -0.294426292181015,
      "logits/rejected": -1.9087531566619873,
      "logps/chosen": -0.9304768443107605,
      "logps/rejected": -20.47693634033203,
      "loss": 0.9382,
      "num_input_tokens_seen": 12174112,
      "odds_ratio_loss": 9.206003189086914,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09304769337177277,
      "rewards/margins": 1.954646110534668,
      "rewards/rejected": -2.0476937294006348,
      "sft_loss": 0.017637435346841812,
      "step": 9320
    },
    {
      "epoch": 0.7257876312718786,
      "grad_norm": 1.225160002708435,
      "learning_rate": 8.71680951578846e-06,
      "logits/chosen": -0.24428872764110565,
      "logits/rejected": -1.6217687129974365,
      "logps/chosen": -0.9168788194656372,
      "logps/rejected": -13.614445686340332,
      "loss": 0.9378,
      "num_input_tokens_seen": 12188144,
      "odds_ratio_loss": 9.270367622375488,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.091687873005867,
      "rewards/margins": 1.2697566747665405,
      "rewards/rejected": -1.3614444732666016,
      "sft_loss": 0.010737092234194279,
      "step": 9330
    },
    {
      "epoch": 0.7265655387008946,
      "grad_norm": 14.630156517028809,
      "learning_rate": 8.670498212185854e-06,
      "logits/chosen": -0.17345094680786133,
      "logits/rejected": -1.7669017314910889,
      "logps/chosen": -0.8515311479568481,
      "logps/rejected": -16.09687042236328,
      "loss": 0.8719,
      "num_input_tokens_seen": 12202320,
      "odds_ratio_loss": 8.373773574829102,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.08515311032533646,
      "rewards/margins": 1.524533748626709,
      "rewards/rejected": -1.609687089920044,
      "sft_loss": 0.03454557806253433,
      "step": 9340
    },
    {
      "epoch": 0.7273434461299105,
      "grad_norm": 1.8369165658950806,
      "learning_rate": 8.624284436355012e-06,
      "logits/chosen": -0.3347812294960022,
      "logits/rejected": -1.5777273178100586,
      "logps/chosen": -0.7993783950805664,
      "logps/rejected": -20.731544494628906,
      "loss": 0.8026,
      "num_input_tokens_seen": 12218336,
      "odds_ratio_loss": 7.981850624084473,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.07993783801794052,
      "rewards/margins": 1.9932165145874023,
      "rewards/rejected": -2.0731544494628906,
      "sft_loss": 0.0044608041644096375,
      "step": 9350
    },
    {
      "epoch": 0.7281213535589265,
      "grad_norm": 2.1387698650360107,
      "learning_rate": 8.578168464307206e-06,
      "logits/chosen": -0.47583675384521484,
      "logits/rejected": -1.1676183938980103,
      "logps/chosen": -0.6008586287498474,
      "logps/rejected": -16.23210334777832,
      "loss": 0.6062,
      "num_input_tokens_seen": 12230592,
      "odds_ratio_loss": 5.891040325164795,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.060085855424404144,
      "rewards/margins": 1.563124418258667,
      "rewards/rejected": -1.6232101917266846,
      "sft_loss": 0.017101094126701355,
      "step": 9360
    },
    {
      "epoch": 0.7288992609879424,
      "grad_norm": 4.9119110107421875,
      "learning_rate": 8.53215057146958e-06,
      "logits/chosen": -0.31183579564094543,
      "logits/rejected": -1.394217848777771,
      "logps/chosen": -1.281356930732727,
      "logps/rejected": -8.785355567932129,
      "loss": 1.3069,
      "num_input_tokens_seen": 12242032,
      "odds_ratio_loss": 12.887720108032227,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.12813569605350494,
      "rewards/margins": 0.7503998279571533,
      "rewards/rejected": -0.8785354495048523,
      "sft_loss": 0.018101245164871216,
      "step": 9370
    },
    {
      "epoch": 0.7296771684169584,
      "grad_norm": 6.540828704833984,
      "learning_rate": 8.486231032683498e-06,
      "logits/chosen": -0.2462223321199417,
      "logits/rejected": -1.6916329860687256,
      "logps/chosen": -0.9182268977165222,
      "logps/rejected": -23.39679527282715,
      "loss": 0.9206,
      "num_input_tokens_seen": 12259872,
      "odds_ratio_loss": 9.148286819458008,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09182268381118774,
      "rewards/margins": 2.247856616973877,
      "rewards/rejected": -2.339679479598999,
      "sft_loss": 0.005803549196571112,
      "step": 9380
    },
    {
      "epoch": 0.7304550758459744,
      "grad_norm": 3.1595852375030518,
      "learning_rate": 8.440410122202902e-06,
      "logits/chosen": -0.285053551197052,
      "logits/rejected": -1.308659315109253,
      "logps/chosen": -0.9199910163879395,
      "logps/rejected": -17.944133758544922,
      "loss": 0.9393,
      "num_input_tokens_seen": 12274096,
      "odds_ratio_loss": 9.294504165649414,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.09199909120798111,
      "rewards/margins": 1.7024141550064087,
      "rewards/rejected": -1.7944132089614868,
      "sft_loss": 0.009822076186537743,
      "step": 9390
    },
    {
      "epoch": 0.7312329832749903,
      "grad_norm": 3.8401002883911133,
      "learning_rate": 8.394688113692684e-06,
      "logits/chosen": -0.17555391788482666,
      "logits/rejected": -1.3844785690307617,
      "logps/chosen": -0.9503070712089539,
      "logps/rejected": -15.155308723449707,
      "loss": 0.9588,
      "num_input_tokens_seen": 12288448,
      "odds_ratio_loss": 9.44873332977295,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09503071010112762,
      "rewards/margins": 1.4205002784729004,
      "rewards/rejected": -1.5155308246612549,
      "sft_loss": 0.013938379473984241,
      "step": 9400
    },
    {
      "epoch": 0.7320108907040063,
      "grad_norm": 7.067780017852783,
      "learning_rate": 8.349065280227043e-06,
      "logits/chosen": -0.2124090939760208,
      "logits/rejected": -1.7307307720184326,
      "logps/chosen": -0.9263704419136047,
      "logps/rejected": -14.757944107055664,
      "loss": 0.9409,
      "num_input_tokens_seen": 12300544,
      "odds_ratio_loss": 9.307869911193848,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.09263704717159271,
      "rewards/margins": 1.3831573724746704,
      "rewards/rejected": -1.4757944345474243,
      "sft_loss": 0.010076623409986496,
      "step": 9410
    },
    {
      "epoch": 0.7327887981330222,
      "grad_norm": 1.2005184888839722,
      "learning_rate": 8.303541894287853e-06,
      "logits/chosen": -0.2487785816192627,
      "logits/rejected": -1.5799301862716675,
      "logps/chosen": -0.972506046295166,
      "logps/rejected": -15.554588317871094,
      "loss": 0.9839,
      "num_input_tokens_seen": 12310352,
      "odds_ratio_loss": 9.77066707611084,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09725061058998108,
      "rewards/margins": 1.4582083225250244,
      "rewards/rejected": -1.5554590225219727,
      "sft_loss": 0.006865913979709148,
      "step": 9420
    },
    {
      "epoch": 0.7335667055620381,
      "grad_norm": 23.47117805480957,
      "learning_rate": 8.258118227763042e-06,
      "logits/chosen": -0.18120557069778442,
      "logits/rejected": -1.3460719585418701,
      "logps/chosen": -0.9152958989143372,
      "logps/rejected": -7.53924036026001,
      "loss": 0.9301,
      "num_input_tokens_seen": 12325184,
      "odds_ratio_loss": 8.720853805541992,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.09152959287166595,
      "rewards/margins": 0.6623944044113159,
      "rewards/rejected": -0.7539240121841431,
      "sft_loss": 0.058042317628860474,
      "step": 9430
    },
    {
      "epoch": 0.734344612991054,
      "grad_norm": 2.655810594558716,
      "learning_rate": 8.212794551944968e-06,
      "logits/chosen": -0.3864406645298004,
      "logits/rejected": -1.237755298614502,
      "logps/chosen": -1.3730939626693726,
      "logps/rejected": -14.318827629089355,
      "loss": 1.3908,
      "num_input_tokens_seen": 12338624,
      "odds_ratio_loss": 13.32928466796875,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.13730938732624054,
      "rewards/margins": 1.2945735454559326,
      "rewards/rejected": -1.4318828582763672,
      "sft_loss": 0.05790317803621292,
      "step": 9440
    },
    {
      "epoch": 0.73512252042007,
      "grad_norm": 2.26202392578125,
      "learning_rate": 8.167571137528788e-06,
      "logits/chosen": -0.0685519352555275,
      "logits/rejected": -1.5295946598052979,
      "logps/chosen": -0.7410327792167664,
      "logps/rejected": -16.688684463500977,
      "loss": 0.7493,
      "num_input_tokens_seen": 12351760,
      "odds_ratio_loss": 7.458738803863525,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.07410328090190887,
      "rewards/margins": 1.5947654247283936,
      "rewards/rejected": -1.6688687801361084,
      "sft_loss": 0.003421871457248926,
      "step": 9450
    },
    {
      "epoch": 0.7359004278490859,
      "grad_norm": 1.3735111951828003,
      "learning_rate": 8.122448254610853e-06,
      "logits/chosen": -0.22400489449501038,
      "logits/rejected": -1.5753123760223389,
      "logps/chosen": -0.7171843647956848,
      "logps/rejected": -7.419655799865723,
      "loss": 0.7375,
      "num_input_tokens_seen": 12365376,
      "odds_ratio_loss": 7.273862361907959,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.07171843200922012,
      "rewards/margins": 0.6702471971511841,
      "rewards/rejected": -0.741965651512146,
      "sft_loss": 0.010080719366669655,
      "step": 9460
    },
    {
      "epoch": 0.7366783352781019,
      "grad_norm": 1.3101513385772705,
      "learning_rate": 8.077426172687092e-06,
      "logits/chosen": -0.2681163251399994,
      "logits/rejected": -1.6242246627807617,
      "logps/chosen": -0.9232928156852722,
      "logps/rejected": -21.1041202545166,
      "loss": 0.935,
      "num_input_tokens_seen": 12382208,
      "odds_ratio_loss": 9.310216903686523,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.09232927858829498,
      "rewards/margins": 2.018082857131958,
      "rewards/rejected": -2.110412120819092,
      "sft_loss": 0.003992895595729351,
      "step": 9470
    },
    {
      "epoch": 0.7374562427071178,
      "grad_norm": 3.2835025787353516,
      "learning_rate": 8.032505160651393e-06,
      "logits/chosen": -0.5149093866348267,
      "logits/rejected": -1.5588371753692627,
      "logps/chosen": -0.9430519938468933,
      "logps/rejected": -17.860750198364258,
      "loss": 0.9444,
      "num_input_tokens_seen": 12394752,
      "odds_ratio_loss": 9.215082168579102,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09430520981550217,
      "rewards/margins": 1.6917698383331299,
      "rewards/rejected": -1.7860748767852783,
      "sft_loss": 0.02293788641691208,
      "step": 9480
    },
    {
      "epoch": 0.7382341501361338,
      "grad_norm": 2.113330841064453,
      "learning_rate": 7.987685486794021e-06,
      "logits/chosen": -0.11559674888849258,
      "logits/rejected": -1.5787172317504883,
      "logps/chosen": -0.7669419646263123,
      "logps/rejected": -13.251312255859375,
      "loss": 0.7709,
      "num_input_tokens_seen": 12408784,
      "odds_ratio_loss": 7.644850254058838,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.07669420540332794,
      "rewards/margins": 1.2484371662139893,
      "rewards/rejected": -1.3251312971115112,
      "sft_loss": 0.006389009766280651,
      "step": 9490
    },
    {
      "epoch": 0.7390120575651498,
      "grad_norm": 3.922805070877075,
      "learning_rate": 7.942967418799987e-06,
      "logits/chosen": -0.2295842468738556,
      "logits/rejected": -1.2077304124832153,
      "logps/chosen": -0.8419455289840698,
      "logps/rejected": -8.025991439819336,
      "loss": 0.8703,
      "num_input_tokens_seen": 12420784,
      "odds_ratio_loss": 8.591384887695312,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.08419455587863922,
      "rewards/margins": 0.7184046506881714,
      "rewards/rejected": -0.8025991320610046,
      "sft_loss": 0.011183908209204674,
      "step": 9500
    },
    {
      "epoch": 0.7397899649941657,
      "grad_norm": 2.023562431335449,
      "learning_rate": 7.898351223747463e-06,
      "logits/chosen": -0.47920385003089905,
      "logits/rejected": -1.178824543952942,
      "logps/chosen": -0.8258765935897827,
      "logps/rejected": -21.803810119628906,
      "loss": 0.8267,
      "num_input_tokens_seen": 12437792,
      "odds_ratio_loss": 8.039857864379883,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08258765190839767,
      "rewards/margins": 2.0977935791015625,
      "rewards/rejected": -2.1803812980651855,
      "sft_loss": 0.022705521434545517,
      "step": 9510
    },
    {
      "epoch": 0.7405678724231817,
      "grad_norm": 2.659808874130249,
      "learning_rate": 7.85383716810619e-06,
      "logits/chosen": -0.14984755218029022,
      "logits/rejected": -1.4295680522918701,
      "logps/chosen": -0.8580425381660461,
      "logps/rejected": -13.875532150268555,
      "loss": 0.8718,
      "num_input_tokens_seen": 12450464,
      "odds_ratio_loss": 8.642005920410156,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08580423891544342,
      "rewards/margins": 1.3017492294311523,
      "rewards/rejected": -1.3875534534454346,
      "sft_loss": 0.007576326839625835,
      "step": 9520
    },
    {
      "epoch": 0.7413457798521976,
      "grad_norm": 7.571407794952393,
      "learning_rate": 7.809425517735882e-06,
      "logits/chosen": -0.4199492931365967,
      "logits/rejected": -1.2727622985839844,
      "logps/chosen": -0.7845070362091064,
      "logps/rejected": -7.90404748916626,
      "loss": 0.8038,
      "num_input_tokens_seen": 12460944,
      "odds_ratio_loss": 7.95632791519165,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.07845070958137512,
      "rewards/margins": 0.7119540572166443,
      "rewards/rejected": -0.790404736995697,
      "sft_loss": 0.008128446526825428,
      "step": 9530
    },
    {
      "epoch": 0.7421236872812136,
      "grad_norm": 2.4146907329559326,
      "learning_rate": 7.765116537884638e-06,
      "logits/chosen": -0.10155007988214493,
      "logits/rejected": -1.724957823753357,
      "logps/chosen": -0.7110482454299927,
      "logps/rejected": -16.225696563720703,
      "loss": 0.7145,
      "num_input_tokens_seen": 12476384,
      "odds_ratio_loss": 7.10809850692749,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.07110483199357986,
      "rewards/margins": 1.5514647960662842,
      "rewards/rejected": -1.6225694417953491,
      "sft_loss": 0.0037299140822142363,
      "step": 9540
    },
    {
      "epoch": 0.7429015947102294,
      "grad_norm": 4.062788009643555,
      "learning_rate": 7.720910493187353e-06,
      "logits/chosen": -0.11513806879520416,
      "logits/rejected": -1.4933720827102661,
      "logps/chosen": -0.8890712857246399,
      "logps/rejected": -15.664164543151855,
      "loss": 0.9005,
      "num_input_tokens_seen": 12491440,
      "odds_ratio_loss": 8.813547134399414,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08890712261199951,
      "rewards/margins": 1.4775094985961914,
      "rewards/rejected": -1.5664165019989014,
      "sft_loss": 0.01918819174170494,
      "step": 9550
    },
    {
      "epoch": 0.7436795021392454,
      "grad_norm": 17.29952049255371,
      "learning_rate": 7.676807647664156e-06,
      "logits/chosen": -0.6568484306335449,
      "logits/rejected": -1.1108663082122803,
      "logps/chosen": -0.9456981420516968,
      "logps/rejected": -18.975109100341797,
      "loss": 0.9569,
      "num_input_tokens_seen": 12499264,
      "odds_ratio_loss": 9.273554801940918,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09456981718540192,
      "rewards/margins": 1.802941083908081,
      "rewards/rejected": -1.8975108861923218,
      "sft_loss": 0.029570013284683228,
      "step": 9560
    },
    {
      "epoch": 0.7444574095682613,
      "grad_norm": 2.4996085166931152,
      "learning_rate": 7.632808264718805e-06,
      "logits/chosen": -0.0891091451048851,
      "logits/rejected": -1.5486935377120972,
      "logps/chosen": -0.9733103513717651,
      "logps/rejected": -11.993257522583008,
      "loss": 0.9894,
      "num_input_tokens_seen": 12513984,
      "odds_ratio_loss": 9.830720901489258,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.09733103215694427,
      "rewards/margins": 1.1019947528839111,
      "rewards/rejected": -1.199325680732727,
      "sft_loss": 0.006290743593126535,
      "step": 9570
    },
    {
      "epoch": 0.7452353169972773,
      "grad_norm": 12.195624351501465,
      "learning_rate": 7.588912607137141e-06,
      "logits/chosen": -0.19729329645633698,
      "logits/rejected": -1.429392695426941,
      "logps/chosen": -0.8851596117019653,
      "logps/rejected": -10.769949913024902,
      "loss": 0.9099,
      "num_input_tokens_seen": 12527952,
      "odds_ratio_loss": 9.021488189697266,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.08851595968008041,
      "rewards/margins": 0.9884791374206543,
      "rewards/rejected": -1.0769951343536377,
      "sft_loss": 0.007774298544973135,
      "step": 9580
    },
    {
      "epoch": 0.7460132244262933,
      "grad_norm": 2.7442383766174316,
      "learning_rate": 7.545120937085498e-06,
      "logits/chosen": -0.2869792878627777,
      "logits/rejected": -1.7530750036239624,
      "logps/chosen": -0.8515239953994751,
      "logps/rejected": -15.776219367980957,
      "loss": 0.8742,
      "num_input_tokens_seen": 12542816,
      "odds_ratio_loss": 8.661226272583008,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.08515240252017975,
      "rewards/margins": 1.4924695491790771,
      "rewards/rejected": -1.5776218175888062,
      "sft_loss": 0.008089212700724602,
      "step": 9590
    },
    {
      "epoch": 0.7467911318553092,
      "grad_norm": 8.69823169708252,
      "learning_rate": 7.501433516109152e-06,
      "logits/chosen": -0.3283597230911255,
      "logits/rejected": -1.450851321220398,
      "logps/chosen": -1.2167384624481201,
      "logps/rejected": -15.752424240112305,
      "loss": 1.2204,
      "num_input_tokens_seen": 12557728,
      "odds_ratio_loss": 10.35692024230957,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.12167384475469589,
      "rewards/margins": 1.4535688161849976,
      "rewards/rejected": -1.575242519378662,
      "sft_loss": 0.18467479944229126,
      "step": 9600
    },
    {
      "epoch": 0.7475690392843252,
      "grad_norm": 3.905773878097534,
      "learning_rate": 7.457850605130745e-06,
      "logits/chosen": -0.31992048025131226,
      "logits/rejected": -0.8100858926773071,
      "logps/chosen": -0.751301646232605,
      "logps/rejected": -14.805696487426758,
      "loss": 0.7602,
      "num_input_tokens_seen": 12571680,
      "odds_ratio_loss": 7.564237117767334,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.0751301646232605,
      "rewards/margins": 1.4054394960403442,
      "rewards/rejected": -1.48056960105896,
      "sft_loss": 0.003806706052273512,
      "step": 9610
    },
    {
      "epoch": 0.7483469467133411,
      "grad_norm": 2.715651035308838,
      "learning_rate": 7.414372464448743e-06,
      "logits/chosen": -0.25902432203292847,
      "logits/rejected": -1.471724271774292,
      "logps/chosen": -0.9195705652236938,
      "logps/rejected": -12.037912368774414,
      "loss": 0.9254,
      "num_input_tokens_seen": 12580880,
      "odds_ratio_loss": 9.145597457885742,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09195704758167267,
      "rewards/margins": 1.111834168434143,
      "rewards/rejected": -1.2037912607192993,
      "sft_loss": 0.010806088335812092,
      "step": 9620
    },
    {
      "epoch": 0.7491248541423571,
      "grad_norm": 4.897205352783203,
      "learning_rate": 7.370999353735869e-06,
      "logits/chosen": -0.6254599690437317,
      "logits/rejected": -0.6810623407363892,
      "logps/chosen": -0.8146526217460632,
      "logps/rejected": -13.211095809936523,
      "loss": 0.8365,
      "num_input_tokens_seen": 12591248,
      "odds_ratio_loss": 8.088262557983398,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.08146525919437408,
      "rewards/margins": 1.2396444082260132,
      "rewards/rejected": -1.321109652519226,
      "sft_loss": 0.027669047936797142,
      "step": 9630
    },
    {
      "epoch": 0.749902761571373,
      "grad_norm": 2.603041410446167,
      "learning_rate": 7.327731532037549e-06,
      "logits/chosen": -0.34888967871665955,
      "logits/rejected": -1.740108847618103,
      "logps/chosen": -1.137001633644104,
      "logps/rejected": -20.085874557495117,
      "loss": 1.148,
      "num_input_tokens_seen": 12606272,
      "odds_ratio_loss": 11.355992317199707,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.11370013654232025,
      "rewards/margins": 1.8948875665664673,
      "rewards/rejected": -2.008587598800659,
      "sft_loss": 0.012404505163431168,
      "step": 9640
    },
    {
      "epoch": 0.750680669000389,
      "grad_norm": 3.0989139080047607,
      "learning_rate": 7.284569257770382e-06,
      "logits/chosen": -0.3092551529407501,
      "logits/rejected": -1.3083518743515015,
      "logps/chosen": -1.007351279258728,
      "logps/rejected": -14.896029472351074,
      "loss": 1.0127,
      "num_input_tokens_seen": 12620336,
      "odds_ratio_loss": 10.002145767211914,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1007351279258728,
      "rewards/margins": 1.388867974281311,
      "rewards/rejected": -1.4896031618118286,
      "sft_loss": 0.012452312745153904,
      "step": 9650
    },
    {
      "epoch": 0.751458576429405,
      "grad_norm": 2.0114264488220215,
      "learning_rate": 7.241512788720578e-06,
      "logits/chosen": -0.3267592787742615,
      "logits/rejected": -1.3846229314804077,
      "logps/chosen": -0.8181226849555969,
      "logps/rejected": -11.771913528442383,
      "loss": 0.8315,
      "num_input_tokens_seen": 12630912,
      "odds_ratio_loss": 8.077900886535645,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.08181225508451462,
      "rewards/margins": 1.095379114151001,
      "rewards/rejected": -1.1771913766860962,
      "sft_loss": 0.023709949105978012,
      "step": 9660
    },
    {
      "epoch": 0.7522364838584209,
      "grad_norm": 1.943953037261963,
      "learning_rate": 7.198562382042434e-06,
      "logits/chosen": -0.1044231653213501,
      "logits/rejected": -1.8825289011001587,
      "logps/chosen": -0.9438576698303223,
      "logps/rejected": -14.98261833190918,
      "loss": 0.9609,
      "num_input_tokens_seen": 12644560,
      "odds_ratio_loss": 9.543760299682617,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.09438576549291611,
      "rewards/margins": 1.4038760662078857,
      "rewards/rejected": -1.4982619285583496,
      "sft_loss": 0.006542509887367487,
      "step": 9670
    },
    {
      "epoch": 0.7530143912874367,
      "grad_norm": 2.966200828552246,
      "learning_rate": 7.155718294256783e-06,
      "logits/chosen": 0.012543360702693462,
      "logits/rejected": -1.677483320236206,
      "logps/chosen": -1.0234817266464233,
      "logps/rejected": -11.815591812133789,
      "loss": 1.0293,
      "num_input_tokens_seen": 12659456,
      "odds_ratio_loss": 10.21452808380127,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.10234817117452621,
      "rewards/margins": 1.0792109966278076,
      "rewards/rejected": -1.1815592050552368,
      "sft_loss": 0.00782776903361082,
      "step": 9680
    },
    {
      "epoch": 0.7537922987164527,
      "grad_norm": 7.487515449523926,
      "learning_rate": 7.112980781249478e-06,
      "logits/chosen": -0.5516456961631775,
      "logits/rejected": -0.7924268841743469,
      "logps/chosen": -0.9835079312324524,
      "logps/rejected": -9.512467384338379,
      "loss": 1.0017,
      "num_input_tokens_seen": 12670160,
      "odds_ratio_loss": 9.811184883117676,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.09835079312324524,
      "rewards/margins": 0.8528959155082703,
      "rewards/rejected": -0.9512467384338379,
      "sft_loss": 0.020590314641594887,
      "step": 9690
    },
    {
      "epoch": 0.7545702061454687,
      "grad_norm": 2.7716739177703857,
      "learning_rate": 7.070350098269845e-06,
      "logits/chosen": -0.24449455738067627,
      "logits/rejected": -2.006038188934326,
      "logps/chosen": -0.8078185319900513,
      "logps/rejected": -17.701519012451172,
      "loss": 0.8097,
      "num_input_tokens_seen": 12680560,
      "odds_ratio_loss": 8.062870025634766,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08078185468912125,
      "rewards/margins": 1.6893701553344727,
      "rewards/rejected": -1.7701518535614014,
      "sft_loss": 0.003373973071575165,
      "step": 9700
    },
    {
      "epoch": 0.7553481135744846,
      "grad_norm": 2.9411494731903076,
      "learning_rate": 7.027826499929177e-06,
      "logits/chosen": -0.18928129971027374,
      "logits/rejected": -1.3758265972137451,
      "logps/chosen": -0.9373753666877747,
      "logps/rejected": -15.49957275390625,
      "loss": 0.9539,
      "num_input_tokens_seen": 12694608,
      "odds_ratio_loss": 9.455991744995117,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.09373755007982254,
      "rewards/margins": 1.4562196731567383,
      "rewards/rejected": -1.549957275390625,
      "sft_loss": 0.008260851725935936,
      "step": 9710
    },
    {
      "epoch": 0.7561260210035006,
      "grad_norm": 23.82400131225586,
      "learning_rate": 6.985410240199203e-06,
      "logits/chosen": -0.6237587928771973,
      "logits/rejected": -1.1116478443145752,
      "logps/chosen": -0.8208237886428833,
      "logps/rejected": -14.505549430847168,
      "loss": 0.8352,
      "num_input_tokens_seen": 12704256,
      "odds_ratio_loss": 7.775033473968506,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.08208237588405609,
      "rewards/margins": 1.368472695350647,
      "rewards/rejected": -1.4505550861358643,
      "sft_loss": 0.05770029500126839,
      "step": 9720
    },
    {
      "epoch": 0.7569039284325165,
      "grad_norm": 5.56984281539917,
      "learning_rate": 6.943101572410571e-06,
      "logits/chosen": -0.3685595393180847,
      "logits/rejected": -1.0919402837753296,
      "logps/chosen": -0.7797693014144897,
      "logps/rejected": -13.029932975769043,
      "loss": 0.7932,
      "num_input_tokens_seen": 12713744,
      "odds_ratio_loss": 7.839241027832031,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.07797692716121674,
      "rewards/margins": 1.2250163555145264,
      "rewards/rejected": -1.3029931783676147,
      "sft_loss": 0.009250042960047722,
      "step": 9730
    },
    {
      "epoch": 0.7576818358615325,
      "grad_norm": 1.6570754051208496,
      "learning_rate": 6.900900749251338e-06,
      "logits/chosen": -0.5102152228355408,
      "logits/rejected": -1.114384651184082,
      "logps/chosen": -0.7536827325820923,
      "logps/rejected": -12.92182731628418,
      "loss": 0.7615,
      "num_input_tokens_seen": 12727072,
      "odds_ratio_loss": 6.868859767913818,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.07536827772855759,
      "rewards/margins": 1.2168145179748535,
      "rewards/rejected": -1.2921828031539917,
      "sft_loss": 0.0746065154671669,
      "step": 9740
    },
    {
      "epoch": 0.7584597432905484,
      "grad_norm": 4.804180145263672,
      "learning_rate": 6.858808022765467e-06,
      "logits/chosen": -0.0006758645176887512,
      "logits/rejected": -1.5899875164031982,
      "logps/chosen": -0.8684228658676147,
      "logps/rejected": -13.665643692016602,
      "loss": 0.8872,
      "num_input_tokens_seen": 12737776,
      "odds_ratio_loss": 8.773876190185547,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.08684228360652924,
      "rewards/margins": 1.2797222137451172,
      "rewards/rejected": -1.3665645122528076,
      "sft_loss": 0.009812915697693825,
      "step": 9750
    },
    {
      "epoch": 0.7592376507195644,
      "grad_norm": 1.4910658597946167,
      "learning_rate": 6.816823644351303e-06,
      "logits/chosen": -0.5923045873641968,
      "logits/rejected": -1.2481071949005127,
      "logps/chosen": -0.7165006995201111,
      "logps/rejected": -9.771490097045898,
      "loss": 0.7448,
      "num_input_tokens_seen": 12749344,
      "odds_ratio_loss": 7.370150566101074,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.07165006548166275,
      "rewards/margins": 0.9054989814758301,
      "rewards/rejected": -0.9771490097045898,
      "sft_loss": 0.007765809539705515,
      "step": 9760
    },
    {
      "epoch": 0.7600155581485804,
      "grad_norm": 5.410368919372559,
      "learning_rate": 6.774947864760089e-06,
      "logits/chosen": -0.1797337532043457,
      "logits/rejected": -1.679751992225647,
      "logps/chosen": -1.0861485004425049,
      "logps/rejected": -14.003316879272461,
      "loss": 1.1028,
      "num_input_tokens_seen": 12760368,
      "odds_ratio_loss": 10.247908592224121,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.10861483961343765,
      "rewards/margins": 1.2917169332504272,
      "rewards/rejected": -1.400331735610962,
      "sft_loss": 0.07805103808641434,
      "step": 9770
    },
    {
      "epoch": 0.7607934655775963,
      "grad_norm": 17.647127151489258,
      "learning_rate": 6.733180934094466e-06,
      "logits/chosen": -0.40942397713661194,
      "logits/rejected": -1.2711374759674072,
      "logps/chosen": -0.953356921672821,
      "logps/rejected": -14.270413398742676,
      "loss": 0.9616,
      "num_input_tokens_seen": 12775856,
      "odds_ratio_loss": 9.490220069885254,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.0953356921672821,
      "rewards/margins": 1.3317056894302368,
      "rewards/rejected": -1.4270412921905518,
      "sft_loss": 0.012609261088073254,
      "step": 9780
    },
    {
      "epoch": 0.7615713730066123,
      "grad_norm": 1.5215857028961182,
      "learning_rate": 6.691523101806976e-06,
      "logits/chosen": -0.06620067358016968,
      "logits/rejected": -1.9449646472930908,
      "logps/chosen": -0.9097517728805542,
      "logps/rejected": -17.92283058166504,
      "loss": 0.9124,
      "num_input_tokens_seen": 12790816,
      "odds_ratio_loss": 8.980363845825195,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09097517281770706,
      "rewards/margins": 1.7013078927993774,
      "rewards/rejected": -1.792283058166504,
      "sft_loss": 0.014339657500386238,
      "step": 9790
    },
    {
      "epoch": 0.7623492804356281,
      "grad_norm": 15.784709930419922,
      "learning_rate": 6.649974616698562e-06,
      "logits/chosen": -0.2136622965335846,
      "logits/rejected": -1.6949361562728882,
      "logps/chosen": -1.03670334815979,
      "logps/rejected": -13.867294311523438,
      "loss": 1.0583,
      "num_input_tokens_seen": 12802160,
      "odds_ratio_loss": 10.493218421936035,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.10367034375667572,
      "rewards/margins": 1.2830591201782227,
      "rewards/rejected": -1.3867294788360596,
      "sft_loss": 0.008986649103462696,
      "step": 9800
    },
    {
      "epoch": 0.7631271878646441,
      "grad_norm": 15.24134635925293,
      "learning_rate": 6.6085357269171035e-06,
      "logits/chosen": -0.48242124915122986,
      "logits/rejected": -0.9146297574043274,
      "logps/chosen": -0.6837570071220398,
      "logps/rejected": -10.073492050170898,
      "loss": 0.7005,
      "num_input_tokens_seen": 12814800,
      "odds_ratio_loss": 6.878556728363037,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.06837569177150726,
      "rewards/margins": 0.938973605632782,
      "rewards/rejected": -1.0073492527008057,
      "sft_loss": 0.012609479017555714,
      "step": 9810
    },
    {
      "epoch": 0.76390509529366,
      "grad_norm": 5.1652512550354,
      "learning_rate": 6.567206679955917e-06,
      "logits/chosen": -0.18169178068637848,
      "logits/rejected": -1.6934398412704468,
      "logps/chosen": -0.8636601567268372,
      "logps/rejected": -13.6743803024292,
      "loss": 0.8859,
      "num_input_tokens_seen": 12827936,
      "odds_ratio_loss": 8.807854652404785,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.08636601269245148,
      "rewards/margins": 1.2810719013214111,
      "rewards/rejected": -1.3674379587173462,
      "sft_loss": 0.005116607528179884,
      "step": 9820
    },
    {
      "epoch": 0.764683002722676,
      "grad_norm": 7.437309265136719,
      "learning_rate": 6.525987722652286e-06,
      "logits/chosen": -0.21487879753112793,
      "logits/rejected": -1.2891901731491089,
      "logps/chosen": -0.8677453994750977,
      "logps/rejected": -15.33686351776123,
      "loss": 0.8864,
      "num_input_tokens_seen": 12842480,
      "odds_ratio_loss": 8.776076316833496,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.08677453547716141,
      "rewards/margins": 1.4469118118286133,
      "rewards/rejected": -1.5336863994598389,
      "sft_loss": 0.008795716799795628,
      "step": 9830
    },
    {
      "epoch": 0.7654609101516919,
      "grad_norm": 1.8497710227966309,
      "learning_rate": 6.484879101185986e-06,
      "logits/chosen": -0.2847957909107208,
      "logits/rejected": -2.0163838863372803,
      "logps/chosen": -0.8841788172721863,
      "logps/rejected": -16.62775421142578,
      "loss": 0.8887,
      "num_input_tokens_seen": 12857024,
      "odds_ratio_loss": 8.826251983642578,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.0884178951382637,
      "rewards/margins": 1.5743576288223267,
      "rewards/rejected": -1.6627753973007202,
      "sft_loss": 0.0060475775972008705,
      "step": 9840
    },
    {
      "epoch": 0.7662388175807079,
      "grad_norm": 2.8241870403289795,
      "learning_rate": 6.443881061077814e-06,
      "logits/chosen": -0.13205505907535553,
      "logits/rejected": -1.3500169515609741,
      "logps/chosen": -1.0866869688034058,
      "logps/rejected": -17.037189483642578,
      "loss": 1.0873,
      "num_input_tokens_seen": 12870720,
      "odds_ratio_loss": 10.3355131149292,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.10866870731115341,
      "rewards/margins": 1.5950504541397095,
      "rewards/rejected": -1.703718900680542,
      "sft_loss": 0.05375038459897041,
      "step": 9850
    },
    {
      "epoch": 0.7670167250097238,
      "grad_norm": 8.120436668395996,
      "learning_rate": 6.402993847188113e-06,
      "logits/chosen": -0.07916440069675446,
      "logits/rejected": -1.727804183959961,
      "logps/chosen": -0.8412625193595886,
      "logps/rejected": -12.190113067626953,
      "loss": 0.8442,
      "num_input_tokens_seen": 12886128,
      "odds_ratio_loss": 8.385359764099121,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08412624895572662,
      "rewards/margins": 1.1348850727081299,
      "rewards/rejected": -1.2190113067626953,
      "sft_loss": 0.005621061660349369,
      "step": 9860
    },
    {
      "epoch": 0.7677946324387398,
      "grad_norm": 8.273092269897461,
      "learning_rate": 6.3622177037153255e-06,
      "logits/chosen": -0.30196234583854675,
      "logits/rejected": -1.4886324405670166,
      "logps/chosen": -0.9431349039077759,
      "logps/rejected": -11.96256160736084,
      "loss": 0.9606,
      "num_input_tokens_seen": 12902208,
      "odds_ratio_loss": 9.528372764587402,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.09431348741054535,
      "rewards/margins": 1.101942777633667,
      "rewards/rejected": -1.196256160736084,
      "sft_loss": 0.007743525318801403,
      "step": 9870
    },
    {
      "epoch": 0.7685725398677558,
      "grad_norm": 4.558796405792236,
      "learning_rate": 6.3215528741945225e-06,
      "logits/chosen": -0.49385198950767517,
      "logits/rejected": -0.8861545324325562,
      "logps/chosen": -0.8803753852844238,
      "logps/rejected": -10.397488594055176,
      "loss": 0.8937,
      "num_input_tokens_seen": 12915488,
      "odds_ratio_loss": 8.755577087402344,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.08803753554821014,
      "rewards/margins": 0.9517113566398621,
      "rewards/rejected": -1.0397489070892334,
      "sft_loss": 0.018163876608014107,
      "step": 9880
    },
    {
      "epoch": 0.7693504472967717,
      "grad_norm": 1.6330955028533936,
      "learning_rate": 6.28099960149596e-06,
      "logits/chosen": -0.6630735397338867,
      "logits/rejected": -1.1244785785675049,
      "logps/chosen": -0.8058063387870789,
      "logps/rejected": -22.111560821533203,
      "loss": 0.8059,
      "num_input_tokens_seen": 12929568,
      "odds_ratio_loss": 7.960448265075684,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08058063685894012,
      "rewards/margins": 2.130575656890869,
      "rewards/rejected": -2.211156129837036,
      "sft_loss": 0.009810159914195538,
      "step": 9890
    },
    {
      "epoch": 0.7701283547257877,
      "grad_norm": 7.713956832885742,
      "learning_rate": 6.2405581278236145e-06,
      "logits/chosen": -0.44286996126174927,
      "logits/rejected": -1.3790740966796875,
      "logps/chosen": -0.9783164858818054,
      "logps/rejected": -20.651592254638672,
      "loss": 0.9787,
      "num_input_tokens_seen": 12940400,
      "odds_ratio_loss": 9.46540641784668,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09783165901899338,
      "rewards/margins": 1.9673277139663696,
      "rewards/rejected": -2.065159320831299,
      "sft_loss": 0.032199710607528687,
      "step": 9900
    },
    {
      "epoch": 0.7709062621548036,
      "grad_norm": 3.927487850189209,
      "learning_rate": 6.200228694713753e-06,
      "logits/chosen": -0.042903829365968704,
      "logits/rejected": -1.6210275888442993,
      "logps/chosen": -0.8930376172065735,
      "logps/rejected": -16.273311614990234,
      "loss": 0.8973,
      "num_input_tokens_seen": 12955424,
      "odds_ratio_loss": 8.926946640014648,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08930376172065735,
      "rewards/margins": 1.538027286529541,
      "rewards/rejected": -1.6273311376571655,
      "sft_loss": 0.0045908852480351925,
      "step": 9910
    },
    {
      "epoch": 0.7716841695838196,
      "grad_norm": 1.831764817237854,
      "learning_rate": 6.160011543033467e-06,
      "logits/chosen": -0.25490206480026245,
      "logits/rejected": -1.7062351703643799,
      "logps/chosen": -1.1320891380310059,
      "logps/rejected": -13.286028861999512,
      "loss": 1.1717,
      "num_input_tokens_seen": 12965680,
      "odds_ratio_loss": 11.279840469360352,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.11320891231298447,
      "rewards/margins": 1.2153937816619873,
      "rewards/rejected": -1.3286025524139404,
      "sft_loss": 0.043743547052145004,
      "step": 9920
    },
    {
      "epoch": 0.7724620770128354,
      "grad_norm": 4.164346694946289,
      "learning_rate": 6.119906912979264e-06,
      "logits/chosen": -0.2769322097301483,
      "logits/rejected": -1.6600967645645142,
      "logps/chosen": -0.7944921255111694,
      "logps/rejected": -17.34960174560547,
      "loss": 0.7974,
      "num_input_tokens_seen": 12976944,
      "odds_ratio_loss": 7.91311502456665,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.07944921404123306,
      "rewards/margins": 1.6555111408233643,
      "rewards/rejected": -1.7349601984024048,
      "sft_loss": 0.006089845206588507,
      "step": 9930
    },
    {
      "epoch": 0.7732399844418514,
      "grad_norm": 4.3693318367004395,
      "learning_rate": 6.0799150440756095e-06,
      "logits/chosen": -0.1835547685623169,
      "logits/rejected": -1.7030483484268188,
      "logps/chosen": -0.8427737951278687,
      "logps/rejected": -18.01861000061035,
      "loss": 0.8444,
      "num_input_tokens_seen": 12992592,
      "odds_ratio_loss": 8.359333992004395,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08427738398313522,
      "rewards/margins": 1.7175838947296143,
      "rewards/rejected": -1.801861047744751,
      "sft_loss": 0.008451655507087708,
      "step": 9940
    },
    {
      "epoch": 0.7740178918708673,
      "grad_norm": 1.813942790031433,
      "learning_rate": 6.040036175173505e-06,
      "logits/chosen": -0.07210500538349152,
      "logits/rejected": -1.649038314819336,
      "logps/chosen": -0.8709640502929688,
      "logps/rejected": -17.332408905029297,
      "loss": 0.8763,
      "num_input_tokens_seen": 13007312,
      "odds_ratio_loss": 8.719454765319824,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08709640055894852,
      "rewards/margins": 1.6461445093154907,
      "rewards/rejected": -1.7332408428192139,
      "sft_loss": 0.004315356258302927,
      "step": 9950
    },
    {
      "epoch": 0.7747957992998833,
      "grad_norm": 3.831186532974243,
      "learning_rate": 6.00027054444906e-06,
      "logits/chosen": -0.3818754553794861,
      "logits/rejected": -1.2485878467559814,
      "logps/chosen": -0.7479159832000732,
      "logps/rejected": -11.673700332641602,
      "loss": 0.7542,
      "num_input_tokens_seen": 13020864,
      "odds_ratio_loss": 7.26510763168335,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.07479159533977509,
      "rewards/margins": 1.0925782918930054,
      "rewards/rejected": -1.167370080947876,
      "sft_loss": 0.02773687243461609,
      "step": 9960
    },
    {
      "epoch": 0.7755737067288992,
      "grad_norm": 1.1907448768615723,
      "learning_rate": 5.960618389402075e-06,
      "logits/chosen": -0.005200075916945934,
      "logits/rejected": -1.7917149066925049,
      "logps/chosen": -1.2306063175201416,
      "logps/rejected": -19.59311294555664,
      "loss": 1.231,
      "num_input_tokens_seen": 13034928,
      "odds_ratio_loss": 12.255085945129395,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.12306062877178192,
      "rewards/margins": 1.836250901222229,
      "rewards/rejected": -1.9593114852905273,
      "sft_loss": 0.0055275773629546165,
      "step": 9970
    },
    {
      "epoch": 0.7763516141579152,
      "grad_norm": 5.450308799743652,
      "learning_rate": 5.92107994685461e-06,
      "logits/chosen": -0.529719889163971,
      "logits/rejected": -1.083274483680725,
      "logps/chosen": -0.8484355211257935,
      "logps/rejected": -10.391220092773438,
      "loss": 0.857,
      "num_input_tokens_seen": 13046672,
      "odds_ratio_loss": 8.189764022827148,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.08484355360269547,
      "rewards/margins": 0.9542785882949829,
      "rewards/rejected": -1.0391219854354858,
      "sft_loss": 0.03801070898771286,
      "step": 9980
    },
    {
      "epoch": 0.7771295215869312,
      "grad_norm": 7.595885753631592,
      "learning_rate": 5.881655452949583e-06,
      "logits/chosen": 0.03671069070696831,
      "logits/rejected": -2.0498228073120117,
      "logps/chosen": -0.898884654045105,
      "logps/rejected": -14.653169631958008,
      "loss": 0.9055,
      "num_input_tokens_seen": 13056912,
      "odds_ratio_loss": 9.006585121154785,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08988846838474274,
      "rewards/margins": 1.3754284381866455,
      "rewards/rejected": -1.4653170108795166,
      "sft_loss": 0.0048610251396894455,
      "step": 9990
    },
    {
      "epoch": 0.7779074290159471,
      "grad_norm": 3.6287105083465576,
      "learning_rate": 5.842345143149361e-06,
      "logits/chosen": -0.1152871698141098,
      "logits/rejected": -1.3964219093322754,
      "logps/chosen": -1.1272786855697632,
      "logps/rejected": -9.357419967651367,
      "loss": 1.1474,
      "num_input_tokens_seen": 13068000,
      "odds_ratio_loss": 10.209806442260742,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.11272789537906647,
      "rewards/margins": 0.8230140805244446,
      "rewards/rejected": -0.9357420206069946,
      "sft_loss": 0.12642648816108704,
      "step": 10000
    },
    {
      "epoch": 0.7786853364449631,
      "grad_norm": 4.439057350158691,
      "learning_rate": 5.803149252234336e-06,
      "logits/chosen": -0.22541113197803497,
      "logits/rejected": -1.4121778011322021,
      "logps/chosen": -1.1307504177093506,
      "logps/rejected": -19.595867156982422,
      "loss": 1.132,
      "num_input_tokens_seen": 13083008,
      "odds_ratio_loss": 9.42105484008789,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.11307503283023834,
      "rewards/margins": 1.8465116024017334,
      "rewards/rejected": -1.9595867395401,
      "sft_loss": 0.18993499875068665,
      "step": 10010
    },
    {
      "epoch": 0.779463243873979,
      "grad_norm": 1.8150360584259033,
      "learning_rate": 5.764068014301538e-06,
      "logits/chosen": -0.14730560779571533,
      "logits/rejected": -1.2366430759429932,
      "logps/chosen": -0.7479221224784851,
      "logps/rejected": -6.61281681060791,
      "loss": 0.7628,
      "num_input_tokens_seen": 13090528,
      "odds_ratio_loss": 7.487229824066162,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.07479222118854523,
      "rewards/margins": 0.5864894986152649,
      "rewards/rejected": -0.6612816452980042,
      "sft_loss": 0.014043653383851051,
      "step": 10020
    },
    {
      "epoch": 0.780241151302995,
      "grad_norm": 11.748605728149414,
      "learning_rate": 5.7251016627632435e-06,
      "logits/chosen": -0.6228640079498291,
      "logits/rejected": -1.2003719806671143,
      "logps/chosen": -0.8472782373428345,
      "logps/rejected": -18.236339569091797,
      "loss": 0.8512,
      "num_input_tokens_seen": 13106288,
      "odds_ratio_loss": 8.328832626342773,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08472783118486404,
      "rewards/margins": 1.7389062643051147,
      "rewards/rejected": -1.8236339092254639,
      "sft_loss": 0.018275240436196327,
      "step": 10030
    },
    {
      "epoch": 0.7810190587320109,
      "grad_norm": 8.976465225219727,
      "learning_rate": 5.686250430345555e-06,
      "logits/chosen": -0.40403372049331665,
      "logits/rejected": -1.1737014055252075,
      "logps/chosen": -0.8802043795585632,
      "logps/rejected": -11.703226089477539,
      "loss": 0.8928,
      "num_input_tokens_seen": 13119776,
      "odds_ratio_loss": 8.639188766479492,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.0880204439163208,
      "rewards/margins": 1.0823020935058594,
      "rewards/rejected": -1.1703224182128906,
      "sft_loss": 0.028895627707242966,
      "step": 10040
    },
    {
      "epoch": 0.7817969661610268,
      "grad_norm": 11.301222801208496,
      "learning_rate": 5.647514549087046e-06,
      "logits/chosen": -0.41604989767074585,
      "logits/rejected": -1.3308502435684204,
      "logps/chosen": -1.0168416500091553,
      "logps/rejected": -14.803606986999512,
      "loss": 1.0266,
      "num_input_tokens_seen": 13132656,
      "odds_ratio_loss": 10.065681457519531,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.10168415307998657,
      "rewards/margins": 1.3786766529083252,
      "rewards/rejected": -1.480360984802246,
      "sft_loss": 0.019993456080555916,
      "step": 10050
    },
    {
      "epoch": 0.7825748735900427,
      "grad_norm": 3.9200751781463623,
      "learning_rate": 5.6088942503373265e-06,
      "logits/chosen": -0.00039936602115631104,
      "logits/rejected": -1.7556613683700562,
      "logps/chosen": -0.8142244219779968,
      "logps/rejected": -15.882316589355469,
      "loss": 0.8212,
      "num_input_tokens_seen": 13148624,
      "odds_ratio_loss": 8.17491340637207,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08142244815826416,
      "rewards/margins": 1.506809115409851,
      "rewards/rejected": -1.5882318019866943,
      "sft_loss": 0.003715798957273364,
      "step": 10060
    },
    {
      "epoch": 0.7833527810190587,
      "grad_norm": 1.288907527923584,
      "learning_rate": 5.570389764755729e-06,
      "logits/chosen": -0.10343638807535172,
      "logits/rejected": -1.422821044921875,
      "logps/chosen": -0.8510880470275879,
      "logps/rejected": -13.416641235351562,
      "loss": 0.8577,
      "num_input_tokens_seen": 13168432,
      "odds_ratio_loss": 8.532673835754395,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08510880172252655,
      "rewards/margins": 1.256555438041687,
      "rewards/rejected": -1.3416640758514404,
      "sft_loss": 0.004443835467100143,
      "step": 10070
    },
    {
      "epoch": 0.7841306884480747,
      "grad_norm": 8.386462211608887,
      "learning_rate": 5.5320013223098684e-06,
      "logits/chosen": -0.10891640186309814,
      "logits/rejected": -1.3848743438720703,
      "logps/chosen": -0.6136741638183594,
      "logps/rejected": -19.104137420654297,
      "loss": 0.6158,
      "num_input_tokens_seen": 13179840,
      "odds_ratio_loss": 6.0931010246276855,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.061367422342300415,
      "rewards/margins": 1.8490467071533203,
      "rewards/rejected": -1.9104140996932983,
      "sft_loss": 0.006519814021885395,
      "step": 10080
    },
    {
      "epoch": 0.7849085958770906,
      "grad_norm": 1.7663533687591553,
      "learning_rate": 5.493729152274302e-06,
      "logits/chosen": -0.22560708224773407,
      "logits/rejected": -1.5404101610183716,
      "logps/chosen": -1.1439088582992554,
      "logps/rejected": -20.378692626953125,
      "loss": 1.1557,
      "num_input_tokens_seen": 13200592,
      "odds_ratio_loss": 11.499128341674805,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.11439088732004166,
      "rewards/margins": 1.923478364944458,
      "rewards/rejected": -2.0378692150115967,
      "sft_loss": 0.005769290495663881,
      "step": 10090
    },
    {
      "epoch": 0.7856865033061066,
      "grad_norm": 2.094831705093384,
      "learning_rate": 5.455573483229143e-06,
      "logits/chosen": -0.20372407138347626,
      "logits/rejected": -1.6087785959243774,
      "logps/chosen": -0.8583130836486816,
      "logps/rejected": -9.61543083190918,
      "loss": 0.8677,
      "num_input_tokens_seen": 13217472,
      "odds_ratio_loss": 8.596895217895508,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08583130687475204,
      "rewards/margins": 0.8757117390632629,
      "rewards/rejected": -0.961543083190918,
      "sft_loss": 0.00801418349146843,
      "step": 10100
    },
    {
      "epoch": 0.7864644107351225,
      "grad_norm": 2.245335578918457,
      "learning_rate": 5.417534543058716e-06,
      "logits/chosen": -0.3722991347312927,
      "logits/rejected": -1.39009428024292,
      "logps/chosen": -0.7631933689117432,
      "logps/rejected": -14.828773498535156,
      "loss": 0.7698,
      "num_input_tokens_seen": 13233008,
      "odds_ratio_loss": 7.651087760925293,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.07631933689117432,
      "rewards/margins": 1.4065582752227783,
      "rewards/rejected": -1.482877492904663,
      "sft_loss": 0.004690065048635006,
      "step": 10110
    },
    {
      "epoch": 0.7872423181641385,
      "grad_norm": 1.768862009048462,
      "learning_rate": 5.379612558950173e-06,
      "logits/chosen": 0.006290731020271778,
      "logits/rejected": -1.8056621551513672,
      "logps/chosen": -0.8933433294296265,
      "logps/rejected": -10.847860336303711,
      "loss": 0.9274,
      "num_input_tokens_seen": 13250944,
      "odds_ratio_loss": 9.240643501281738,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.08933433145284653,
      "rewards/margins": 0.995451807975769,
      "rewards/rejected": -1.0847861766815186,
      "sft_loss": 0.00336031848564744,
      "step": 10120
    },
    {
      "epoch": 0.7880202255931544,
      "grad_norm": 1.7466740608215332,
      "learning_rate": 5.341807757392153e-06,
      "logits/chosen": -0.26185980439186096,
      "logits/rejected": -1.625367522239685,
      "logps/chosen": -0.7085305452346802,
      "logps/rejected": -20.165281295776367,
      "loss": 0.7105,
      "num_input_tokens_seen": 13263552,
      "odds_ratio_loss": 7.040585994720459,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.07085305452346802,
      "rewards/margins": 1.9456751346588135,
      "rewards/rejected": -2.0165281295776367,
      "sft_loss": 0.006464931182563305,
      "step": 10130
    },
    {
      "epoch": 0.7887981330221704,
      "grad_norm": 1.9766432046890259,
      "learning_rate": 5.30412036417342e-06,
      "logits/chosen": -0.15813367068767548,
      "logits/rejected": -1.8555446863174438,
      "logps/chosen": -0.9007395505905151,
      "logps/rejected": -18.488134384155273,
      "loss": 0.9119,
      "num_input_tokens_seen": 13274960,
      "odds_ratio_loss": 9.050884246826172,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09007396548986435,
      "rewards/margins": 1.7587394714355469,
      "rewards/rejected": -1.8488136529922485,
      "sft_loss": 0.006835309322923422,
      "step": 10140
    },
    {
      "epoch": 0.7895760404511863,
      "grad_norm": 4.220992088317871,
      "learning_rate": 5.266550604381526e-06,
      "logits/chosen": -0.2565006613731384,
      "logits/rejected": -1.4154245853424072,
      "logps/chosen": -1.000119686126709,
      "logps/rejected": -15.364748001098633,
      "loss": 1.0084,
      "num_input_tokens_seen": 13288768,
      "odds_ratio_loss": 9.917895317077637,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.10001198202371597,
      "rewards/margins": 1.4364628791809082,
      "rewards/rejected": -1.5364747047424316,
      "sft_loss": 0.016632404178380966,
      "step": 10150
    },
    {
      "epoch": 0.7903539478802023,
      "grad_norm": 55.403568267822266,
      "learning_rate": 5.229098702401447e-06,
      "logits/chosen": -0.535290539264679,
      "logits/rejected": -0.9979928731918335,
      "logps/chosen": -1.0686589479446411,
      "logps/rejected": -15.240943908691406,
      "loss": 1.0706,
      "num_input_tokens_seen": 13303632,
      "odds_ratio_loss": 10.46064281463623,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.10686589777469635,
      "rewards/margins": 1.4172285795211792,
      "rewards/rejected": -1.524094581604004,
      "sft_loss": 0.024510297924280167,
      "step": 10160
    },
    {
      "epoch": 0.7911318553092183,
      "grad_norm": 2.378926992416382,
      "learning_rate": 5.19176488191426e-06,
      "logits/chosen": -0.089039646089077,
      "logits/rejected": -1.5092610120773315,
      "logps/chosen": -1.036505937576294,
      "logps/rejected": -14.474766731262207,
      "loss": 1.0391,
      "num_input_tokens_seen": 13317664,
      "odds_ratio_loss": 10.324877738952637,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.10365059226751328,
      "rewards/margins": 1.343826174736023,
      "rewards/rejected": -1.4474767446517944,
      "sft_loss": 0.00660539697855711,
      "step": 10170
    },
    {
      "epoch": 0.7919097627382341,
      "grad_norm": 3.639150619506836,
      "learning_rate": 5.154549365895814e-06,
      "logits/chosen": -0.09354257583618164,
      "logits/rejected": -1.6912329196929932,
      "logps/chosen": -0.8237224817276001,
      "logps/rejected": -22.405752182006836,
      "loss": 0.8254,
      "num_input_tokens_seen": 13334128,
      "odds_ratio_loss": 8.221444129943848,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08237224817276001,
      "rewards/margins": 2.158203125,
      "rewards/rejected": -2.2405755519866943,
      "sft_loss": 0.0032429061830043793,
      "step": 10180
    },
    {
      "epoch": 0.7926876701672501,
      "grad_norm": 2.631995916366577,
      "learning_rate": 5.117452376615345e-06,
      "logits/chosen": -0.07670159637928009,
      "logits/rejected": -1.7550842761993408,
      "logps/chosen": -1.188686490058899,
      "logps/rejected": -8.534611701965332,
      "loss": 1.2081,
      "num_input_tokens_seen": 13346160,
      "odds_ratio_loss": 11.93842887878418,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.11886866390705109,
      "rewards/margins": 0.7345924377441406,
      "rewards/rejected": -0.8534610867500305,
      "sft_loss": 0.014260287396609783,
      "step": 10190
    },
    {
      "epoch": 0.793465577596266,
      "grad_norm": 31.077972412109375,
      "learning_rate": 5.0804741356342445e-06,
      "logits/chosen": -0.42552217841148376,
      "logits/rejected": -0.9647568464279175,
      "logps/chosen": -1.0847182273864746,
      "logps/rejected": -12.32005500793457,
      "loss": 1.1025,
      "num_input_tokens_seen": 13361056,
      "odds_ratio_loss": 9.90789794921875,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.1084718257188797,
      "rewards/margins": 1.1235337257385254,
      "rewards/rejected": -1.2320055961608887,
      "sft_loss": 0.11174285411834717,
      "step": 10200
    },
    {
      "epoch": 0.794243485025282,
      "grad_norm": 2.0199506282806396,
      "learning_rate": 5.043614863804646e-06,
      "logits/chosen": -0.3819964826107025,
      "logits/rejected": -1.6330349445343018,
      "logps/chosen": -1.033983588218689,
      "logps/rejected": -24.93582534790039,
      "loss": 1.034,
      "num_input_tokens_seen": 13373008,
      "odds_ratio_loss": 9.945966720581055,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.10339836031198502,
      "rewards/margins": 2.390184164047241,
      "rewards/rejected": -2.4935827255249023,
      "sft_loss": 0.039412155747413635,
      "step": 10210
    },
    {
      "epoch": 0.7950213924542979,
      "grad_norm": 11.4447021484375,
      "learning_rate": 5.0068747812681505e-06,
      "logits/chosen": -0.1652594357728958,
      "logits/rejected": -1.1705518960952759,
      "logps/chosen": -0.7428327202796936,
      "logps/rejected": -13.094820022583008,
      "loss": 0.7596,
      "num_input_tokens_seen": 13383968,
      "odds_ratio_loss": 7.454339504241943,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.07428327202796936,
      "rewards/margins": 1.235198736190796,
      "rewards/rejected": -1.3094819784164429,
      "sft_loss": 0.014146876521408558,
      "step": 10220
    },
    {
      "epoch": 0.7957992998833139,
      "grad_norm": 7.466521739959717,
      "learning_rate": 4.970254107454497e-06,
      "logits/chosen": -0.33368849754333496,
      "logits/rejected": -1.018107533454895,
      "logps/chosen": -0.7646960616111755,
      "logps/rejected": -17.289798736572266,
      "loss": 0.7803,
      "num_input_tokens_seen": 13400320,
      "odds_ratio_loss": 7.598325252532959,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.07646960765123367,
      "rewards/margins": 1.6525102853775024,
      "rewards/rejected": -1.7289798259735107,
      "sft_loss": 0.020495515316724777,
      "step": 10230
    },
    {
      "epoch": 0.7965772073123298,
      "grad_norm": 3.564819812774658,
      "learning_rate": 4.933753061080265e-06,
      "logits/chosen": -0.4034608006477356,
      "logits/rejected": -1.3804579973220825,
      "logps/chosen": -1.7706609964370728,
      "logps/rejected": -17.545896530151367,
      "loss": 1.8045,
      "num_input_tokens_seen": 13419232,
      "odds_ratio_loss": 15.623266220092773,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.17706608772277832,
      "rewards/margins": 1.5775234699249268,
      "rewards/rejected": -1.7545896768569946,
      "sft_loss": 0.24220208823680878,
      "step": 10240
    },
    {
      "epoch": 0.7973551147413458,
      "grad_norm": 1.9408292770385742,
      "learning_rate": 4.897371860147556e-06,
      "logits/chosen": -0.10395392030477524,
      "logits/rejected": -1.6356542110443115,
      "logps/chosen": -1.13056218624115,
      "logps/rejected": -25.447980880737305,
      "loss": 1.1486,
      "num_input_tokens_seen": 13432848,
      "odds_ratio_loss": 11.428488731384277,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.1130562275648117,
      "rewards/margins": 2.43174147605896,
      "rewards/rejected": -2.544797897338867,
      "sft_loss": 0.005717330612242222,
      "step": 10250
    },
    {
      "epoch": 0.7981330221703618,
      "grad_norm": 1.4113094806671143,
      "learning_rate": 4.8611107219426996e-06,
      "logits/chosen": -0.5167861580848694,
      "logits/rejected": -1.1747467517852783,
      "logps/chosen": -0.9015966653823853,
      "logps/rejected": -15.947178840637207,
      "loss": 0.9024,
      "num_input_tokens_seen": 13445440,
      "odds_ratio_loss": 8.933000564575195,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09015966206789017,
      "rewards/margins": 1.5045583248138428,
      "rewards/rejected": -1.5947179794311523,
      "sft_loss": 0.009095687419176102,
      "step": 10260
    },
    {
      "epoch": 0.7989109295993777,
      "grad_norm": 6.226866245269775,
      "learning_rate": 4.824969863034942e-06,
      "logits/chosen": -0.2744348645210266,
      "logits/rejected": -1.526656985282898,
      "logps/chosen": -1.152288794517517,
      "logps/rejected": -21.838542938232422,
      "loss": 1.1687,
      "num_input_tokens_seen": 13459184,
      "odds_ratio_loss": 11.03105640411377,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.11522886902093887,
      "rewards/margins": 2.0686254501342773,
      "rewards/rejected": -2.1838538646698,
      "sft_loss": 0.0656442791223526,
      "step": 10270
    },
    {
      "epoch": 0.7996888370283937,
      "grad_norm": 3.539475202560425,
      "learning_rate": 4.788949499275177e-06,
      "logits/chosen": -0.30656367540359497,
      "logits/rejected": -0.8668183088302612,
      "logps/chosen": -0.8456679582595825,
      "logps/rejected": -16.91873550415039,
      "loss": 0.8515,
      "num_input_tokens_seen": 13470288,
      "odds_ratio_loss": 8.222433090209961,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08456679433584213,
      "rewards/margins": 1.6073070764541626,
      "rewards/rejected": -1.6918739080429077,
      "sft_loss": 0.029214253649115562,
      "step": 10280
    },
    {
      "epoch": 0.8004667444574096,
      "grad_norm": 1.8849167823791504,
      "learning_rate": 4.7530498457946325e-06,
      "logits/chosen": -0.3800256848335266,
      "logits/rejected": -1.3928248882293701,
      "logps/chosen": -0.7146269083023071,
      "logps/rejected": -13.00952434539795,
      "loss": 0.7249,
      "num_input_tokens_seen": 13484928,
      "odds_ratio_loss": 7.169774532318115,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.07146269083023071,
      "rewards/margins": 1.229489803314209,
      "rewards/rejected": -1.300952672958374,
      "sft_loss": 0.007878500036895275,
      "step": 10290
    },
    {
      "epoch": 0.8012446518864255,
      "grad_norm": 28.979467391967773,
      "learning_rate": 4.717271117003605e-06,
      "logits/chosen": -0.49607008695602417,
      "logits/rejected": -1.2129285335540771,
      "logps/chosen": -1.0439517498016357,
      "logps/rejected": -9.6376953125,
      "loss": 1.07,
      "num_input_tokens_seen": 13495120,
      "odds_ratio_loss": 10.457507133483887,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.10439516603946686,
      "rewards/margins": 0.8593744039535522,
      "rewards/rejected": -0.9637695550918579,
      "sft_loss": 0.024267733097076416,
      "step": 10300
    },
    {
      "epoch": 0.8020225593154414,
      "grad_norm": 0.30100616812705994,
      "learning_rate": 4.68161352659015e-06,
      "logits/chosen": -0.3668210804462433,
      "logits/rejected": -0.8683077096939087,
      "logps/chosen": -0.7981551885604858,
      "logps/rejected": -14.805597305297852,
      "loss": 0.8126,
      "num_input_tokens_seen": 13506912,
      "odds_ratio_loss": 8.055696487426758,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.07981552928686142,
      "rewards/margins": 1.400744080543518,
      "rewards/rejected": -1.4805595874786377,
      "sft_loss": 0.007010910660028458,
      "step": 10310
    },
    {
      "epoch": 0.8028004667444574,
      "grad_norm": 11.078510284423828,
      "learning_rate": 4.646077287518838e-06,
      "logits/chosen": -0.39960020780563354,
      "logits/rejected": -1.750201940536499,
      "logps/chosen": -0.8384559750556946,
      "logps/rejected": -13.797350883483887,
      "loss": 0.8468,
      "num_input_tokens_seen": 13516128,
      "odds_ratio_loss": 8.331451416015625,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.0838456004858017,
      "rewards/margins": 1.2958896160125732,
      "rewards/rejected": -1.3797352313995361,
      "sft_loss": 0.013614359311759472,
      "step": 10320
    },
    {
      "epoch": 0.8035783741734733,
      "grad_norm": 1.1385927200317383,
      "learning_rate": 4.610662612029482e-06,
      "logits/chosen": -0.13649150729179382,
      "logits/rejected": -1.0758001804351807,
      "logps/chosen": -0.8508375287055969,
      "logps/rejected": -12.534235954284668,
      "loss": 0.8695,
      "num_input_tokens_seen": 13528368,
      "odds_ratio_loss": 8.62982177734375,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08508376032114029,
      "rewards/margins": 1.1683398485183716,
      "rewards/rejected": -1.2534236907958984,
      "sft_loss": 0.0065626101568341255,
      "step": 10330
    },
    {
      "epoch": 0.8043562816024893,
      "grad_norm": 2.9482533931732178,
      "learning_rate": 4.575369711635843e-06,
      "logits/chosen": -0.10335323959589005,
      "logits/rejected": -1.4621038436889648,
      "logps/chosen": -0.7176315188407898,
      "logps/rejected": -24.72249984741211,
      "loss": 0.7204,
      "num_input_tokens_seen": 13545872,
      "odds_ratio_loss": 7.180578708648682,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.07176315784454346,
      "rewards/margins": 2.400486707687378,
      "rewards/rejected": -2.47225022315979,
      "sft_loss": 0.002294877776876092,
      "step": 10340
    },
    {
      "epoch": 0.8051341890315052,
      "grad_norm": 16.524131774902344,
      "learning_rate": 4.540198797124382e-06,
      "logits/chosen": -0.5208472013473511,
      "logits/rejected": -0.6951860189437866,
      "logps/chosen": -0.9725438952445984,
      "logps/rejected": -15.746805191040039,
      "loss": 0.9747,
      "num_input_tokens_seen": 13562416,
      "odds_ratio_loss": 9.463030815124512,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09725438058376312,
      "rewards/margins": 1.4774260520935059,
      "rewards/rejected": -1.5746803283691406,
      "sft_loss": 0.028433341532945633,
      "step": 10350
    },
    {
      "epoch": 0.8059120964605212,
      "grad_norm": 1.489711880683899,
      "learning_rate": 4.5051500785530024e-06,
      "logits/chosen": -0.309017539024353,
      "logits/rejected": -1.6297719478607178,
      "logps/chosen": -0.9229093790054321,
      "logps/rejected": -20.863962173461914,
      "loss": 0.9229,
      "num_input_tokens_seen": 13579456,
      "odds_ratio_loss": 9.180902481079102,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09229093790054321,
      "rewards/margins": 1.9941049814224243,
      "rewards/rejected": -2.086395740509033,
      "sft_loss": 0.0048326728865504265,
      "step": 10360
    },
    {
      "epoch": 0.8066900038895372,
      "grad_norm": 2.2888967990875244,
      "learning_rate": 4.470223765249796e-06,
      "logits/chosen": -0.23541298508644104,
      "logits/rejected": -1.2439138889312744,
      "logps/chosen": -0.7586043477058411,
      "logps/rejected": -13.460233688354492,
      "loss": 0.7655,
      "num_input_tokens_seen": 13594272,
      "odds_ratio_loss": 7.621101379394531,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.07586044073104858,
      "rewards/margins": 1.2701631784439087,
      "rewards/rejected": -1.346023440361023,
      "sft_loss": 0.0033806138671934605,
      "step": 10370
    },
    {
      "epoch": 0.8074679113185531,
      "grad_norm": 1.8455400466918945,
      "learning_rate": 4.435420065811791e-06,
      "logits/chosen": -0.10371305793523788,
      "logits/rejected": -1.3010910749435425,
      "logps/chosen": -0.8560296297073364,
      "logps/rejected": -5.94932746887207,
      "loss": 0.8874,
      "num_input_tokens_seen": 13610336,
      "odds_ratio_loss": 8.841263771057129,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.08560295403003693,
      "rewards/margins": 0.5093298554420471,
      "rewards/rejected": -0.5949327349662781,
      "sft_loss": 0.003240591613575816,
      "step": 10380
    },
    {
      "epoch": 0.8082458187475691,
      "grad_norm": 2.436837673187256,
      "learning_rate": 4.4007391881036975e-06,
      "logits/chosen": -0.5878714919090271,
      "logits/rejected": -1.4302802085876465,
      "logps/chosen": -1.0162811279296875,
      "logps/rejected": -21.074613571166992,
      "loss": 1.0227,
      "num_input_tokens_seen": 13622880,
      "odds_ratio_loss": 9.80493450164795,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.10162810981273651,
      "rewards/margins": 2.005833387374878,
      "rewards/rejected": -2.107461452484131,
      "sft_loss": 0.04224376007914543,
      "step": 10390
    },
    {
      "epoch": 0.809023726176585,
      "grad_norm": 3.158832550048828,
      "learning_rate": 4.3661813392566865e-06,
      "logits/chosen": -0.19001172482967377,
      "logits/rejected": -1.2967747449874878,
      "logps/chosen": -0.9470418691635132,
      "logps/rejected": -18.34066390991211,
      "loss": 0.9472,
      "num_input_tokens_seen": 13636208,
      "odds_ratio_loss": 9.365311622619629,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09470418840646744,
      "rewards/margins": 1.7393624782562256,
      "rewards/rejected": -1.83406662940979,
      "sft_loss": 0.010707495734095573,
      "step": 10400
    },
    {
      "epoch": 0.809801633605601,
      "grad_norm": 4.036028861999512,
      "learning_rate": 4.331746725667132e-06,
      "logits/chosen": -0.4446299970149994,
      "logits/rejected": -1.4362730979919434,
      "logps/chosen": -0.8088845014572144,
      "logps/rejected": -14.285024642944336,
      "loss": 0.8132,
      "num_input_tokens_seen": 13648832,
      "odds_ratio_loss": 7.9063849449157715,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08088846504688263,
      "rewards/margins": 1.347614049911499,
      "rewards/rejected": -1.4285024404525757,
      "sft_loss": 0.022565852850675583,
      "step": 10410
    },
    {
      "epoch": 0.8105795410346169,
      "grad_norm": 11.664623260498047,
      "learning_rate": 4.297435552995391e-06,
      "logits/chosen": -0.5774062871932983,
      "logits/rejected": -1.1489648818969727,
      "logps/chosen": -0.9516938924789429,
      "logps/rejected": -19.207168579101562,
      "loss": 0.9724,
      "num_input_tokens_seen": 13659936,
      "odds_ratio_loss": 9.546696662902832,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.09516939520835876,
      "rewards/margins": 1.825547456741333,
      "rewards/rejected": -1.9207168817520142,
      "sft_loss": 0.017754215747117996,
      "step": 10420
    },
    {
      "epoch": 0.8113574484636328,
      "grad_norm": 3.810201406478882,
      "learning_rate": 4.2632480261645705e-06,
      "logits/chosen": -0.31553104519844055,
      "logits/rejected": -1.482110857963562,
      "logps/chosen": -0.833909809589386,
      "logps/rejected": -22.53671646118164,
      "loss": 0.8341,
      "num_input_tokens_seen": 13672304,
      "odds_ratio_loss": 8.198694229125977,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.083390973508358,
      "rewards/margins": 2.1702804565429688,
      "rewards/rejected": -2.253671407699585,
      "sft_loss": 0.01419934444129467,
      "step": 10430
    },
    {
      "epoch": 0.8121353558926487,
      "grad_norm": 2.328805685043335,
      "learning_rate": 4.229184349359294e-06,
      "logits/chosen": -0.43258970975875854,
      "logits/rejected": -1.606311559677124,
      "logps/chosen": -0.7520237565040588,
      "logps/rejected": -16.893381118774414,
      "loss": 0.756,
      "num_input_tokens_seen": 13686560,
      "odds_ratio_loss": 7.460411071777344,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.07520237565040588,
      "rewards/margins": 1.6141357421875,
      "rewards/rejected": -1.6893380880355835,
      "sft_loss": 0.009956094436347485,
      "step": 10440
    },
    {
      "epoch": 0.8129132633216647,
      "grad_norm": 5.632044315338135,
      "learning_rate": 4.1952447260244995e-06,
      "logits/chosen": -0.2724373936653137,
      "logits/rejected": -1.5779287815093994,
      "logps/chosen": -0.842615008354187,
      "logps/rejected": -16.831958770751953,
      "loss": 0.8569,
      "num_input_tokens_seen": 13697088,
      "odds_ratio_loss": 8.503708839416504,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.08426151424646378,
      "rewards/margins": 1.5989340543746948,
      "rewards/rejected": -1.68319571018219,
      "sft_loss": 0.006496152840554714,
      "step": 10450
    },
    {
      "epoch": 0.8136911707506806,
      "grad_norm": 2.986135482788086,
      "learning_rate": 4.1614293588642324e-06,
      "logits/chosen": -0.21738651394844055,
      "logits/rejected": -1.6314799785614014,
      "logps/chosen": -0.982624888420105,
      "logps/rejected": -10.346254348754883,
      "loss": 1.0095,
      "num_input_tokens_seen": 13707952,
      "odds_ratio_loss": 10.010721206665039,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.0982624962925911,
      "rewards/margins": 0.9363630414009094,
      "rewards/rejected": -1.0346254110336304,
      "sft_loss": 0.008467676118016243,
      "step": 10460
    },
    {
      "epoch": 0.8144690781796966,
      "grad_norm": 3.291046142578125,
      "learning_rate": 4.127738449840396e-06,
      "logits/chosen": -0.13250042498111725,
      "logits/rejected": -1.6850128173828125,
      "logps/chosen": -0.9671382904052734,
      "logps/rejected": -14.425585746765137,
      "loss": 0.9935,
      "num_input_tokens_seen": 13720752,
      "odds_ratio_loss": 9.776313781738281,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.0967138260602951,
      "rewards/margins": 1.3458447456359863,
      "rewards/rejected": -1.4425586462020874,
      "sft_loss": 0.015822656452655792,
      "step": 10470
    },
    {
      "epoch": 0.8152469856087126,
      "grad_norm": 4.166664123535156,
      "learning_rate": 4.094172200171578e-06,
      "logits/chosen": -0.3846088945865631,
      "logits/rejected": -1.1095737218856812,
      "logps/chosen": -0.7261198163032532,
      "logps/rejected": -12.331188201904297,
      "loss": 0.7456,
      "num_input_tokens_seen": 13732592,
      "odds_ratio_loss": 7.3647050857543945,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.0726119726896286,
      "rewards/margins": 1.1605069637298584,
      "rewards/rejected": -1.2331188917160034,
      "sft_loss": 0.009162858128547668,
      "step": 10480
    },
    {
      "epoch": 0.8160248930377285,
      "grad_norm": 1.8134665489196777,
      "learning_rate": 4.060730810331839e-06,
      "logits/chosen": -0.12342437356710434,
      "logits/rejected": -1.7950830459594727,
      "logps/chosen": -0.8293974995613098,
      "logps/rejected": -15.393221855163574,
      "loss": 0.8456,
      "num_input_tokens_seen": 13745536,
      "odds_ratio_loss": 8.391043663024902,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.0829397514462471,
      "rewards/margins": 1.4563825130462646,
      "rewards/rejected": -1.5393221378326416,
      "sft_loss": 0.006471716798841953,
      "step": 10490
    },
    {
      "epoch": 0.8168028004667445,
      "grad_norm": 15.088645935058594,
      "learning_rate": 4.0274144800495216e-06,
      "logits/chosen": -0.47999221086502075,
      "logits/rejected": -1.3440254926681519,
      "logps/chosen": -0.7757308483123779,
      "logps/rejected": -13.22809886932373,
      "loss": 0.7924,
      "num_input_tokens_seen": 13759344,
      "odds_ratio_loss": 7.608541965484619,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.07757309079170227,
      "rewards/margins": 1.2452367544174194,
      "rewards/rejected": -1.3228099346160889,
      "sft_loss": 0.03156624361872673,
      "step": 10500
    },
    {
      "epoch": 0.8175807078957604,
      "grad_norm": 7.884941101074219,
      "learning_rate": 3.994223408306039e-06,
      "logits/chosen": -0.41769304871559143,
      "logits/rejected": -1.6686872243881226,
      "logps/chosen": -0.8849675059318542,
      "logps/rejected": -22.058025360107422,
      "loss": 0.8891,
      "num_input_tokens_seen": 13772224,
      "odds_ratio_loss": 8.78945255279541,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08849674463272095,
      "rewards/margins": 2.1173057556152344,
      "rewards/rejected": -2.2058024406433105,
      "sft_loss": 0.010153439827263355,
      "step": 10510
    },
    {
      "epoch": 0.8183586153247764,
      "grad_norm": 2.005485773086548,
      "learning_rate": 3.961157793334708e-06,
      "logits/chosen": -0.07831288129091263,
      "logits/rejected": -1.9417102336883545,
      "logps/chosen": -0.7648784518241882,
      "logps/rejected": -22.269865036010742,
      "loss": 0.7657,
      "num_input_tokens_seen": 13785824,
      "odds_ratio_loss": 7.601174831390381,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.07648786157369614,
      "rewards/margins": 2.150498867034912,
      "rewards/rejected": -2.2269864082336426,
      "sft_loss": 0.005598908755928278,
      "step": 10520
    },
    {
      "epoch": 0.8191365227537923,
      "grad_norm": 3.384782075881958,
      "learning_rate": 3.928217832619555e-06,
      "logits/chosen": -0.14389245212078094,
      "logits/rejected": -1.483295202255249,
      "logps/chosen": -0.7602868676185608,
      "logps/rejected": -17.16800880432129,
      "loss": 0.7805,
      "num_input_tokens_seen": 13799584,
      "odds_ratio_loss": 7.6541008949279785,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.07602868229150772,
      "rewards/margins": 1.6407722234725952,
      "rewards/rejected": -1.7168009281158447,
      "sft_loss": 0.015076467767357826,
      "step": 10530
    },
    {
      "epoch": 0.8199144301828083,
      "grad_norm": 3.2515947818756104,
      "learning_rate": 3.89540372289414e-06,
      "logits/chosen": -0.5360939502716064,
      "logits/rejected": -1.637587547302246,
      "logps/chosen": -0.8743669390678406,
      "logps/rejected": -22.532119750976562,
      "loss": 0.8786,
      "num_input_tokens_seen": 13815536,
      "odds_ratio_loss": 8.461068153381348,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08743669837713242,
      "rewards/margins": 2.1657752990722656,
      "rewards/rejected": -2.2532119750976562,
      "sft_loss": 0.03245777636766434,
      "step": 10540
    },
    {
      "epoch": 0.8206923376118241,
      "grad_norm": 7.553914546966553,
      "learning_rate": 3.862715660140376e-06,
      "logits/chosen": -0.23243942856788635,
      "logits/rejected": -1.265459418296814,
      "logps/chosen": -0.8319419622421265,
      "logps/rejected": -6.5457587242126465,
      "loss": 0.8717,
      "num_input_tokens_seen": 13831408,
      "odds_ratio_loss": 8.624181747436523,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.08319420367479324,
      "rewards/margins": 0.571381688117981,
      "rewards/rejected": -0.6545758843421936,
      "sft_loss": 0.00926089659333229,
      "step": 10550
    },
    {
      "epoch": 0.8214702450408401,
      "grad_norm": 1.2170356512069702,
      "learning_rate": 3.830153839587352e-06,
      "logits/chosen": -0.12456146627664566,
      "logits/rejected": -1.6984800100326538,
      "logps/chosen": -0.8001117706298828,
      "logps/rejected": -22.34385871887207,
      "loss": 0.824,
      "num_input_tokens_seen": 13842800,
      "odds_ratio_loss": 8.16563606262207,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.08001118153333664,
      "rewards/margins": 2.154374599456787,
      "rewards/rejected": -2.2343852519989014,
      "sft_loss": 0.007441601250320673,
      "step": 10560
    },
    {
      "epoch": 0.822248152469856,
      "grad_norm": 10.002525329589844,
      "learning_rate": 3.7977184557101975e-06,
      "logits/chosen": -0.6857665777206421,
      "logits/rejected": -1.205599069595337,
      "logps/chosen": -0.9053228497505188,
      "logps/rejected": -17.684093475341797,
      "loss": 0.9086,
      "num_input_tokens_seen": 13858096,
      "odds_ratio_loss": 8.848712921142578,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09053228795528412,
      "rewards/margins": 1.6778770685195923,
      "rewards/rejected": -1.7684093713760376,
      "sft_loss": 0.023730546236038208,
      "step": 10570
    },
    {
      "epoch": 0.823026059898872,
      "grad_norm": 18.812427520751953,
      "learning_rate": 3.765409702228881e-06,
      "logits/chosen": -0.17264044284820557,
      "logits/rejected": -1.8149480819702148,
      "logps/chosen": -0.9068514108657837,
      "logps/rejected": -22.119543075561523,
      "loss": 0.9093,
      "num_input_tokens_seen": 13872416,
      "odds_ratio_loss": 8.850945472717285,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09068514406681061,
      "rewards/margins": 2.1212692260742188,
      "rewards/rejected": -2.211954355239868,
      "sft_loss": 0.024195067584514618,
      "step": 10580
    },
    {
      "epoch": 0.823803967327888,
      "grad_norm": 5.036831378936768,
      "learning_rate": 3.7332277721070925e-06,
      "logits/chosen": 0.01380936335772276,
      "logits/rejected": -1.7724119424819946,
      "logps/chosen": -0.8250659108161926,
      "logps/rejected": -20.20041275024414,
      "loss": 0.8318,
      "num_input_tokens_seen": 13883168,
      "odds_ratio_loss": 8.276803016662598,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08250658214092255,
      "rewards/margins": 1.9375345706939697,
      "rewards/rejected": -2.0200412273406982,
      "sft_loss": 0.0041191792115569115,
      "step": 10590
    },
    {
      "epoch": 0.8245818747569039,
      "grad_norm": 2.521698474884033,
      "learning_rate": 3.7011728575510525e-06,
      "logits/chosen": -0.10515572875738144,
      "logits/rejected": -1.746950387954712,
      "logps/chosen": -0.7915820479393005,
      "logps/rejected": -24.0881404876709,
      "loss": 0.7929,
      "num_input_tokens_seen": 13897536,
      "odds_ratio_loss": 7.8814826011657715,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.07915820926427841,
      "rewards/margins": 2.329655885696411,
      "rewards/rejected": -2.408813953399658,
      "sft_loss": 0.00473423209041357,
      "step": 10600
    },
    {
      "epoch": 0.8253597821859199,
      "grad_norm": 1.3302189111709595,
      "learning_rate": 3.6692451500083908e-06,
      "logits/chosen": -0.2942681908607483,
      "logits/rejected": -1.1378456354141235,
      "logps/chosen": -0.9133954048156738,
      "logps/rejected": -15.458047866821289,
      "loss": 0.936,
      "num_input_tokens_seen": 13918352,
      "odds_ratio_loss": 9.19874095916748,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.09133954346179962,
      "rewards/margins": 1.4544650316238403,
      "rewards/rejected": -1.545804738998413,
      "sft_loss": 0.016142334789037704,
      "step": 10610
    },
    {
      "epoch": 0.8261376896149358,
      "grad_norm": 8.7985258102417,
      "learning_rate": 3.6374448401669952e-06,
      "logits/chosen": -0.34568607807159424,
      "logits/rejected": -1.3344576358795166,
      "logps/chosen": -0.8753089904785156,
      "logps/rejected": -14.121788024902344,
      "loss": 0.8878,
      "num_input_tokens_seen": 13934352,
      "odds_ratio_loss": 8.807962417602539,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.08753089606761932,
      "rewards/margins": 1.324648141860962,
      "rewards/rejected": -1.412178874015808,
      "sft_loss": 0.006974914111196995,
      "step": 10620
    },
    {
      "epoch": 0.8269155970439518,
      "grad_norm": 6.550417423248291,
      "learning_rate": 3.6057721179538678e-06,
      "logits/chosen": 0.013199204578995705,
      "logits/rejected": -1.4703713655471802,
      "logps/chosen": -0.8771390914916992,
      "logps/rejected": -9.699310302734375,
      "loss": 0.9111,
      "num_input_tokens_seen": 13947840,
      "odds_ratio_loss": 9.066411972045898,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.08771391212940216,
      "rewards/margins": 0.8822172284126282,
      "rewards/rejected": -0.9699310064315796,
      "sft_loss": 0.004470839165151119,
      "step": 10630
    },
    {
      "epoch": 0.8276935044729677,
      "grad_norm": 1.9927451610565186,
      "learning_rate": 3.574227172533995e-06,
      "logits/chosen": -0.4474928379058838,
      "logits/rejected": -1.1933181285858154,
      "logps/chosen": -3.157609701156616,
      "logps/rejected": -17.399599075317383,
      "loss": 3.1668,
      "num_input_tokens_seen": 13960624,
      "odds_ratio_loss": 31.481128692626953,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.3157609701156616,
      "rewards/margins": 1.424198865890503,
      "rewards/rejected": -1.739959955215454,
      "sft_loss": 0.018684662878513336,
      "step": 10640
    },
    {
      "epoch": 0.8284714119019837,
      "grad_norm": 2.234353542327881,
      "learning_rate": 3.5428101923092262e-06,
      "logits/chosen": -0.2922361493110657,
      "logits/rejected": -1.6244525909423828,
      "logps/chosen": -1.1844645738601685,
      "logps/rejected": -16.810644149780273,
      "loss": 1.2053,
      "num_input_tokens_seen": 13977696,
      "odds_ratio_loss": 12.007806777954102,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.11844644695520401,
      "rewards/margins": 1.5626181364059448,
      "rewards/rejected": -1.6810643672943115,
      "sft_loss": 0.004485656972974539,
      "step": 10650
    },
    {
      "epoch": 0.8292493193309997,
      "grad_norm": 1.6060020923614502,
      "learning_rate": 3.5115213649171306e-06,
      "logits/chosen": -0.4599802494049072,
      "logits/rejected": -1.4395368099212646,
      "logps/chosen": -0.9598096609115601,
      "logps/rejected": -4.537492752075195,
      "loss": 0.9772,
      "num_input_tokens_seen": 13987552,
      "odds_ratio_loss": 8.78867244720459,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.09598096460103989,
      "rewards/margins": 0.3577682673931122,
      "rewards/rejected": -0.45374923944473267,
      "sft_loss": 0.09836249053478241,
      "step": 10660
    },
    {
      "epoch": 0.8300272267600156,
      "grad_norm": 5.407586097717285,
      "learning_rate": 3.4803608772298907e-06,
      "logits/chosen": -0.6109030842781067,
      "logits/rejected": -1.033789873123169,
      "logps/chosen": -0.6999441981315613,
      "logps/rejected": -14.462682723999023,
      "loss": 0.7224,
      "num_input_tokens_seen": 13994992,
      "odds_ratio_loss": 7.111445426940918,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.06999442726373672,
      "rewards/margins": 1.3762738704681396,
      "rewards/rejected": -1.4462683200836182,
      "sft_loss": 0.0112073989585042,
      "step": 10670
    },
    {
      "epoch": 0.8308051341890315,
      "grad_norm": 14.045202255249023,
      "learning_rate": 3.4493289153531836e-06,
      "logits/chosen": -0.4810110926628113,
      "logits/rejected": -1.4211429357528687,
      "logps/chosen": -0.8086579442024231,
      "logps/rejected": -12.509838104248047,
      "loss": 0.8346,
      "num_input_tokens_seen": 14009648,
      "odds_ratio_loss": 8.250049591064453,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08086579293012619,
      "rewards/margins": 1.170117974281311,
      "rewards/rejected": -1.250983715057373,
      "sft_loss": 0.009561174549162388,
      "step": 10680
    },
    {
      "epoch": 0.8315830416180474,
      "grad_norm": 13.156830787658691,
      "learning_rate": 3.4184256646250524e-06,
      "logits/chosen": -0.25289249420166016,
      "logits/rejected": -1.280796766281128,
      "logps/chosen": -0.8776252865791321,
      "logps/rejected": -16.361913681030273,
      "loss": 0.9122,
      "num_input_tokens_seen": 14022320,
      "odds_ratio_loss": 8.958372116088867,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.08776251971721649,
      "rewards/margins": 1.5484288930892944,
      "rewards/rejected": -1.6361916065216064,
      "sft_loss": 0.016396891325712204,
      "step": 10690
    },
    {
      "epoch": 0.8323609490470634,
      "grad_norm": 4.0661540031433105,
      "learning_rate": 3.3876513096148338e-06,
      "logits/chosen": -0.14225341379642487,
      "logits/rejected": -1.3277254104614258,
      "logps/chosen": -0.9199932217597961,
      "logps/rejected": -11.546847343444824,
      "loss": 0.9276,
      "num_input_tokens_seen": 14031024,
      "odds_ratio_loss": 9.208245277404785,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09199932217597961,
      "rewards/margins": 1.062685489654541,
      "rewards/rejected": -1.1546847820281982,
      "sft_loss": 0.006757513619959354,
      "step": 10700
    },
    {
      "epoch": 0.8331388564760793,
      "grad_norm": 1.2136611938476562,
      "learning_rate": 3.3570060341220234e-06,
      "logits/chosen": -0.15510103106498718,
      "logits/rejected": -1.907846212387085,
      "logps/chosen": -0.9624649882316589,
      "logps/rejected": -20.84872055053711,
      "loss": 0.967,
      "num_input_tokens_seen": 14046720,
      "odds_ratio_loss": 9.59048080444336,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09624650329351425,
      "rewards/margins": 1.9886255264282227,
      "rewards/rejected": -2.084871768951416,
      "sft_loss": 0.007957112044095993,
      "step": 10710
    },
    {
      "epoch": 0.8339167639050953,
      "grad_norm": 6.186117172241211,
      "learning_rate": 3.326490021175205e-06,
      "logits/chosen": -0.11530347168445587,
      "logits/rejected": -1.3156671524047852,
      "logps/chosen": -0.6982190608978271,
      "logps/rejected": -6.572912693023682,
      "loss": 0.7118,
      "num_input_tokens_seen": 14060992,
      "odds_ratio_loss": 7.040630340576172,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.06982190907001495,
      "rewards/margins": 0.5874693393707275,
      "rewards/rejected": -0.6572912931442261,
      "sft_loss": 0.007760833017528057,
      "step": 10720
    },
    {
      "epoch": 0.8346946713341112,
      "grad_norm": 9.213081359863281,
      "learning_rate": 3.296103453030927e-06,
      "logits/chosen": -0.46214666962623596,
      "logits/rejected": -1.044995665550232,
      "logps/chosen": -0.7721800804138184,
      "logps/rejected": -7.03396463394165,
      "loss": 0.7835,
      "num_input_tokens_seen": 14072560,
      "odds_ratio_loss": 7.719517707824707,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.07721801102161407,
      "rewards/margins": 0.626178503036499,
      "rewards/rejected": -0.7033964395523071,
      "sft_loss": 0.011503453366458416,
      "step": 10730
    },
    {
      "epoch": 0.8354725787631272,
      "grad_norm": 3.0902130603790283,
      "learning_rate": 3.2658465111726405e-06,
      "logits/chosen": -0.3015756905078888,
      "logits/rejected": -0.9444737434387207,
      "logps/chosen": -1.0036855936050415,
      "logps/rejected": -8.566162109375,
      "loss": 1.0248,
      "num_input_tokens_seen": 14085216,
      "odds_ratio_loss": 10.090906143188477,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.10036854445934296,
      "rewards/margins": 0.7562476396560669,
      "rewards/rejected": -0.856616199016571,
      "sft_loss": 0.015708167105913162,
      "step": 10740
    },
    {
      "epoch": 0.8362504861921431,
      "grad_norm": 1.6802676916122437,
      "learning_rate": 3.2357193763095984e-06,
      "logits/chosen": -0.3023141026496887,
      "logits/rejected": -1.2213256359100342,
      "logps/chosen": -0.8354895710945129,
      "logps/rejected": -8.458917617797852,
      "loss": 0.8609,
      "num_input_tokens_seen": 14102064,
      "odds_ratio_loss": 8.399511337280273,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.08354896306991577,
      "rewards/margins": 0.7623429298400879,
      "rewards/rejected": -0.8458918333053589,
      "sft_loss": 0.020914433524012566,
      "step": 10750
    },
    {
      "epoch": 0.8370283936211591,
      "grad_norm": 2.0074286460876465,
      "learning_rate": 3.2057222283757877e-06,
      "logits/chosen": -0.1700136959552765,
      "logits/rejected": -1.258719801902771,
      "logps/chosen": -0.9247137904167175,
      "logps/rejected": -15.314018249511719,
      "loss": 0.9311,
      "num_input_tokens_seen": 14117840,
      "odds_ratio_loss": 8.977886199951172,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09247139096260071,
      "rewards/margins": 1.438930630683899,
      "rewards/rejected": -1.5314018726348877,
      "sft_loss": 0.03332524746656418,
      "step": 10760
    },
    {
      "epoch": 0.8378063010501751,
      "grad_norm": 22.7186336517334,
      "learning_rate": 3.175855246528847e-06,
      "logits/chosen": -0.2710705101490021,
      "logits/rejected": -1.2464778423309326,
      "logps/chosen": -1.6098449230194092,
      "logps/rejected": -8.80943775177002,
      "loss": 1.7031,
      "num_input_tokens_seen": 14128832,
      "odds_ratio_loss": 16.840373992919922,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.16098450124263763,
      "rewards/margins": 0.7199593782424927,
      "rewards/rejected": -0.880943775177002,
      "sft_loss": 0.01902218721807003,
      "step": 10770
    },
    {
      "epoch": 0.838584208479191,
      "grad_norm": 27.165802001953125,
      "learning_rate": 3.1461186091490015e-06,
      "logits/chosen": -0.4816823899745941,
      "logits/rejected": -1.344750165939331,
      "logps/chosen": -0.9202563166618347,
      "logps/rejected": -9.979837417602539,
      "loss": 0.9301,
      "num_input_tokens_seen": 14143392,
      "odds_ratio_loss": 9.173930168151855,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.09202565252780914,
      "rewards/margins": 0.9059581756591797,
      "rewards/rejected": -0.9979838132858276,
      "sft_loss": 0.012739296071231365,
      "step": 10780
    },
    {
      "epoch": 0.839362115908207,
      "grad_norm": 3.174644947052002,
      "learning_rate": 3.1165124938379947e-06,
      "logits/chosen": -0.2760254740715027,
      "logits/rejected": -1.6259291172027588,
      "logps/chosen": -0.772201418876648,
      "logps/rejected": -14.874751091003418,
      "loss": 0.7818,
      "num_input_tokens_seen": 14153776,
      "odds_ratio_loss": 7.721850395202637,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.07722015678882599,
      "rewards/margins": 1.4102550745010376,
      "rewards/rejected": -1.4874751567840576,
      "sft_loss": 0.009580541402101517,
      "step": 10790
    },
    {
      "epoch": 0.8401400233372228,
      "grad_norm": 1.4849799871444702,
      "learning_rate": 3.0870370774180315e-06,
      "logits/chosen": -0.20634207129478455,
      "logits/rejected": -1.5450636148452759,
      "logps/chosen": -0.9883825182914734,
      "logps/rejected": -11.121337890625,
      "loss": 1.0033,
      "num_input_tokens_seen": 14169824,
      "odds_ratio_loss": 9.974977493286133,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09883825480937958,
      "rewards/margins": 1.0132956504821777,
      "rewards/rejected": -1.1121338605880737,
      "sft_loss": 0.005842863116413355,
      "step": 10800
    },
    {
      "epoch": 0.8409179307662388,
      "grad_norm": 3.3578526973724365,
      "learning_rate": 3.0576925359307063e-06,
      "logits/chosen": -0.178543820977211,
      "logits/rejected": -1.4022276401519775,
      "logps/chosen": -0.7552523612976074,
      "logps/rejected": -9.426691055297852,
      "loss": 0.7738,
      "num_input_tokens_seen": 14181936,
      "odds_ratio_loss": 7.660872459411621,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.07552523910999298,
      "rewards/margins": 0.8671439290046692,
      "rewards/rejected": -0.9426690936088562,
      "sft_loss": 0.007728863507509232,
      "step": 10810
    },
    {
      "epoch": 0.8416958381952547,
      "grad_norm": 1.7538191080093384,
      "learning_rate": 3.0284790446359817e-06,
      "logits/chosen": -0.6491594314575195,
      "logits/rejected": -1.37003755569458,
      "logps/chosen": -1.3433282375335693,
      "logps/rejected": -15.055395126342773,
      "loss": 1.3577,
      "num_input_tokens_seen": 14194352,
      "odds_ratio_loss": 12.862637519836426,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1343328058719635,
      "rewards/margins": 1.3712069988250732,
      "rewards/rejected": -1.5055397748947144,
      "sft_loss": 0.0714530199766159,
      "step": 10820
    },
    {
      "epoch": 0.8424737456242707,
      "grad_norm": 1.6737394332885742,
      "learning_rate": 2.9993967780111126e-06,
      "logits/chosen": -0.3436136543750763,
      "logits/rejected": -1.3898818492889404,
      "logps/chosen": -0.7258359789848328,
      "logps/rejected": -13.831029891967773,
      "loss": 0.7328,
      "num_input_tokens_seen": 14213520,
      "odds_ratio_loss": 7.284417152404785,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.07258360087871552,
      "rewards/margins": 1.3105193376541138,
      "rewards/rejected": -1.3831030130386353,
      "sft_loss": 0.004367014858871698,
      "step": 10830
    },
    {
      "epoch": 0.8432516530532866,
      "grad_norm": 4.193950176239014,
      "learning_rate": 2.970445909749614e-06,
      "logits/chosen": -0.2003314197063446,
      "logits/rejected": -1.7939174175262451,
      "logps/chosen": -0.8816348314285278,
      "logps/rejected": -16.744592666625977,
      "loss": 0.9138,
      "num_input_tokens_seen": 14228208,
      "odds_ratio_loss": 9.093521118164062,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.08816348016262054,
      "rewards/margins": 1.5862958431243896,
      "rewards/rejected": -1.6744590997695923,
      "sft_loss": 0.004440158139914274,
      "step": 10840
    },
    {
      "epoch": 0.8440295604823026,
      "grad_norm": 1.9024956226348877,
      "learning_rate": 2.9416266127602484e-06,
      "logits/chosen": -0.10373874008655548,
      "logits/rejected": -1.681274652481079,
      "logps/chosen": -0.7674292325973511,
      "logps/rejected": -14.403085708618164,
      "loss": 0.7905,
      "num_input_tokens_seen": 14243744,
      "odds_ratio_loss": 7.8598175048828125,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.07674293220043182,
      "rewards/margins": 1.3635656833648682,
      "rewards/rejected": -1.4403088092803955,
      "sft_loss": 0.004566981457173824,
      "step": 10850
    },
    {
      "epoch": 0.8448074679113186,
      "grad_norm": 3.021470308303833,
      "learning_rate": 2.9129390591659422e-06,
      "logits/chosen": -0.182247593998909,
      "logits/rejected": -1.098589539527893,
      "logps/chosen": -0.9050281643867493,
      "logps/rejected": -12.335296630859375,
      "loss": 0.9084,
      "num_input_tokens_seen": 14253184,
      "odds_ratio_loss": 8.209359169006348,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09050282090902328,
      "rewards/margins": 1.1430268287658691,
      "rewards/rejected": -1.2335296869277954,
      "sft_loss": 0.08741842210292816,
      "step": 10860
    },
    {
      "epoch": 0.8455853753403345,
      "grad_norm": 1.78042733669281,
      "learning_rate": 2.8843834203027997e-06,
      "logits/chosen": -0.12538185715675354,
      "logits/rejected": -2.3496804237365723,
      "logps/chosen": -0.771729588508606,
      "logps/rejected": -11.53658390045166,
      "loss": 0.786,
      "num_input_tokens_seen": 14267296,
      "odds_ratio_loss": 7.821356773376465,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.07717296481132507,
      "rewards/margins": 1.0764853954315186,
      "rewards/rejected": -1.153658390045166,
      "sft_loss": 0.003909476101398468,
      "step": 10870
    },
    {
      "epoch": 0.8463632827693505,
      "grad_norm": 3.1438002586364746,
      "learning_rate": 2.855959866719063e-06,
      "logits/chosen": -0.27122312784194946,
      "logits/rejected": -1.5149050951004028,
      "logps/chosen": -0.8944989442825317,
      "logps/rejected": -16.33709716796875,
      "loss": 0.8987,
      "num_input_tokens_seen": 14276864,
      "odds_ratio_loss": 8.923892974853516,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08944989740848541,
      "rewards/margins": 1.544259786605835,
      "rewards/rejected": -1.6337096691131592,
      "sft_loss": 0.006331826094537973,
      "step": 10880
    },
    {
      "epoch": 0.8471411901983664,
      "grad_norm": 3.468661069869995,
      "learning_rate": 2.8276685681740966e-06,
      "logits/chosen": -0.2570245862007141,
      "logits/rejected": -1.0485957860946655,
      "logps/chosen": -0.8954674005508423,
      "logps/rejected": -10.502070426940918,
      "loss": 0.9112,
      "num_input_tokens_seen": 14289088,
      "odds_ratio_loss": 8.95766544342041,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.08954674750566483,
      "rewards/margins": 0.9606602787971497,
      "rewards/rejected": -1.0502071380615234,
      "sft_loss": 0.015441042371094227,
      "step": 10890
    },
    {
      "epoch": 0.8479190976273824,
      "grad_norm": 2.709669828414917,
      "learning_rate": 2.7995096936373728e-06,
      "logits/chosen": -0.47932925820350647,
      "logits/rejected": -1.279759168624878,
      "logps/chosen": -0.8552338480949402,
      "logps/rejected": -11.21165943145752,
      "loss": 0.8632,
      "num_input_tokens_seen": 14303184,
      "odds_ratio_loss": 8.557912826538086,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08552340418100357,
      "rewards/margins": 1.035642385482788,
      "rewards/rejected": -1.1211658716201782,
      "sft_loss": 0.007410106714814901,
      "step": 10900
    },
    {
      "epoch": 0.8486970050563983,
      "grad_norm": 2.0692508220672607,
      "learning_rate": 2.771483411287462e-06,
      "logits/chosen": -0.22692546248435974,
      "logits/rejected": -1.7416139841079712,
      "logps/chosen": -1.1929271221160889,
      "logps/rejected": -11.187047958374023,
      "loss": 1.1989,
      "num_input_tokens_seen": 14314880,
      "odds_ratio_loss": 11.927255630493164,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1192927211523056,
      "rewards/margins": 0.9994121789932251,
      "rewards/rejected": -1.1187047958374023,
      "sft_loss": 0.006167359184473753,
      "step": 10910
    },
    {
      "epoch": 0.8494749124854143,
      "grad_norm": 4.281651020050049,
      "learning_rate": 2.7435898885110332e-06,
      "logits/chosen": -0.37550920248031616,
      "logits/rejected": -1.643833875656128,
      "logps/chosen": -0.7633379101753235,
      "logps/rejected": -4.50496244430542,
      "loss": 0.7916,
      "num_input_tokens_seen": 14325008,
      "odds_ratio_loss": 7.83477783203125,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.07633379846811295,
      "rewards/margins": 0.374162495136261,
      "rewards/rejected": -0.45049628615379333,
      "sft_loss": 0.008120447397232056,
      "step": 10920
    },
    {
      "epoch": 0.8502528199144301,
      "grad_norm": 3.8543097972869873,
      "learning_rate": 2.715829291901853e-06,
      "logits/chosen": -0.18652768433094025,
      "logits/rejected": -1.3088423013687134,
      "logps/chosen": -1.8462274074554443,
      "logps/rejected": -13.132868766784668,
      "loss": 1.8577,
      "num_input_tokens_seen": 14345152,
      "odds_ratio_loss": 18.501949310302734,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.18462274968624115,
      "rewards/margins": 1.1286640167236328,
      "rewards/rejected": -1.3132869005203247,
      "sft_loss": 0.007510988973081112,
      "step": 10930
    },
    {
      "epoch": 0.8510307273434461,
      "grad_norm": 3.9623801708221436,
      "learning_rate": 2.6882017872597725e-06,
      "logits/chosen": -0.2059568464756012,
      "logits/rejected": -1.487349510192871,
      "logps/chosen": -0.9327132105827332,
      "logps/rejected": -15.16093921661377,
      "loss": 0.952,
      "num_input_tokens_seen": 14355328,
      "odds_ratio_loss": 9.465570449829102,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.09327132254838943,
      "rewards/margins": 1.4228225946426392,
      "rewards/rejected": -1.5160939693450928,
      "sft_loss": 0.005433371756225824,
      "step": 10940
    },
    {
      "epoch": 0.851808634772462,
      "grad_norm": 4.204638481140137,
      "learning_rate": 2.660707539589766e-06,
      "logits/chosen": -0.46590957045555115,
      "logits/rejected": -1.0994277000427246,
      "logps/chosen": -0.847273051738739,
      "logps/rejected": -9.813700675964355,
      "loss": 0.8647,
      "num_input_tokens_seen": 14369184,
      "odds_ratio_loss": 8.497247695922852,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08472730964422226,
      "rewards/margins": 0.8966428637504578,
      "rewards/rejected": -0.9813701510429382,
      "sft_loss": 0.014988129027187824,
      "step": 10950
    },
    {
      "epoch": 0.852586542201478,
      "grad_norm": 2.0978615283966064,
      "learning_rate": 2.633346713100929e-06,
      "logits/chosen": -0.3497718870639801,
      "logits/rejected": -1.447076678276062,
      "logps/chosen": -1.2323142290115356,
      "logps/rejected": -22.83798599243164,
      "loss": 1.2342,
      "num_input_tokens_seen": 14382736,
      "odds_ratio_loss": 11.533686637878418,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1232314333319664,
      "rewards/margins": 2.160567283630371,
      "rewards/rejected": -2.2837986946105957,
      "sft_loss": 0.08079709857702255,
      "step": 10960
    },
    {
      "epoch": 0.853364449630494,
      "grad_norm": 2.069700241088867,
      "learning_rate": 2.6061194712054927e-06,
      "logits/chosen": -0.10964655876159668,
      "logits/rejected": -1.6845324039459229,
      "logps/chosen": -0.8405076861381531,
      "logps/rejected": -19.095523834228516,
      "loss": 0.8513,
      "num_input_tokens_seen": 14395296,
      "odds_ratio_loss": 8.465188026428223,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.08405075967311859,
      "rewards/margins": 1.825501799583435,
      "rewards/rejected": -1.9095525741577148,
      "sft_loss": 0.004737283103168011,
      "step": 10970
    },
    {
      "epoch": 0.8541423570595099,
      "grad_norm": 34.36608123779297,
      "learning_rate": 2.5790259765178764e-06,
      "logits/chosen": -0.23506894707679749,
      "logits/rejected": -1.7545793056488037,
      "logps/chosen": -0.8853933215141296,
      "logps/rejected": -20.537443161010742,
      "loss": 0.8863,
      "num_input_tokens_seen": 14409472,
      "odds_ratio_loss": 8.764898300170898,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08853934705257416,
      "rewards/margins": 1.9652049541473389,
      "rewards/rejected": -2.053744316101074,
      "sft_loss": 0.009778345935046673,
      "step": 10980
    },
    {
      "epoch": 0.8549202644885259,
      "grad_norm": 6.184960842132568,
      "learning_rate": 2.55206639085368e-06,
      "logits/chosen": -0.20178565382957458,
      "logits/rejected": -1.3336738348007202,
      "logps/chosen": -1.004481554031372,
      "logps/rejected": -21.75019073486328,
      "loss": 1.0076,
      "num_input_tokens_seen": 14421104,
      "odds_ratio_loss": 9.94190788269043,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.10044815391302109,
      "rewards/margins": 2.074570894241333,
      "rewards/rejected": -2.175018787384033,
      "sft_loss": 0.013437485322356224,
      "step": 10990
    },
    {
      "epoch": 0.8556981719175418,
      "grad_norm": 174.71400451660156,
      "learning_rate": 2.5252408752287295e-06,
      "logits/chosen": -0.24337315559387207,
      "logits/rejected": -1.8969571590423584,
      "logps/chosen": -1.0682896375656128,
      "logps/rejected": -11.880017280578613,
      "loss": 1.072,
      "num_input_tokens_seen": 14435216,
      "odds_ratio_loss": 10.644633293151855,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1068289503455162,
      "rewards/margins": 1.0811727046966553,
      "rewards/rejected": -1.1880018711090088,
      "sft_loss": 0.007560241036117077,
      "step": 11000
    },
    {
      "epoch": 0.8564760793465578,
      "grad_norm": 2.8752541542053223,
      "learning_rate": 2.498549589858126e-06,
      "logits/chosen": -0.2807459533214569,
      "logits/rejected": -1.4375717639923096,
      "logps/chosen": -1.1921361684799194,
      "logps/rejected": -10.120828628540039,
      "loss": 1.2045,
      "num_input_tokens_seen": 14445648,
      "odds_ratio_loss": 11.83732795715332,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.11921361833810806,
      "rewards/margins": 0.8928692936897278,
      "rewards/rejected": -1.012082815170288,
      "sft_loss": 0.02073931321501732,
      "step": 11010
    },
    {
      "epoch": 0.8572539867755737,
      "grad_norm": 6.071030139923096,
      "learning_rate": 2.471992694155284e-06,
      "logits/chosen": -0.06983019411563873,
      "logits/rejected": -1.918835997581482,
      "logps/chosen": -0.9633820652961731,
      "logps/rejected": -21.31793785095215,
      "loss": 0.9651,
      "num_input_tokens_seen": 14461808,
      "odds_ratio_loss": 9.575490951538086,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.0963381975889206,
      "rewards/margins": 2.0354557037353516,
      "rewards/rejected": -2.131793737411499,
      "sft_loss": 0.0075905052945017815,
      "step": 11020
    },
    {
      "epoch": 0.8580318942045897,
      "grad_norm": 9.991567611694336,
      "learning_rate": 2.4455703467309743e-06,
      "logits/chosen": -0.17449812591075897,
      "logits/rejected": -1.6064729690551758,
      "logps/chosen": -0.9717493057250977,
      "logps/rejected": -8.299715042114258,
      "loss": 1.0341,
      "num_input_tokens_seen": 14475792,
      "odds_ratio_loss": 9.821267127990723,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.09717492759227753,
      "rewards/margins": 0.7327965497970581,
      "rewards/rejected": -0.8299714922904968,
      "sft_loss": 0.052019111812114716,
      "step": 11030
    },
    {
      "epoch": 0.8588098016336057,
      "grad_norm": 19.450063705444336,
      "learning_rate": 2.4192827053923754e-06,
      "logits/chosen": -0.2539360225200653,
      "logits/rejected": -1.26289963722229,
      "logps/chosen": -1.0389549732208252,
      "logps/rejected": -15.10387897491455,
      "loss": 1.0406,
      "num_input_tokens_seen": 14493312,
      "odds_ratio_loss": 10.181611061096191,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.10389550775289536,
      "rewards/margins": 1.4064924716949463,
      "rewards/rejected": -1.5103880167007446,
      "sft_loss": 0.022460829466581345,
      "step": 11040
    },
    {
      "epoch": 0.8595877090626215,
      "grad_norm": 10.944372177124023,
      "learning_rate": 2.3931299271421485e-06,
      "logits/chosen": -0.042080096900463104,
      "logits/rejected": -1.5874948501586914,
      "logps/chosen": -0.7615327835083008,
      "logps/rejected": -7.649228096008301,
      "loss": 0.7799,
      "num_input_tokens_seen": 14506896,
      "odds_ratio_loss": 7.745556831359863,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.07615328580141068,
      "rewards/margins": 0.6887694597244263,
      "rewards/rejected": -0.7649228572845459,
      "sft_loss": 0.005318674724549055,
      "step": 11050
    },
    {
      "epoch": 0.8603656164916375,
      "grad_norm": 3.6714460849761963,
      "learning_rate": 2.3671121681774655e-06,
      "logits/chosen": -0.15332692861557007,
      "logits/rejected": -1.4491770267486572,
      "logps/chosen": -0.9941250681877136,
      "logps/rejected": -13.707873344421387,
      "loss": 1.0027,
      "num_input_tokens_seen": 14519024,
      "odds_ratio_loss": 9.954381942749023,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09941250830888748,
      "rewards/margins": 1.2713749408721924,
      "rewards/rejected": -1.3707873821258545,
      "sft_loss": 0.007261206395924091,
      "step": 11060
    },
    {
      "epoch": 0.8611435239206534,
      "grad_norm": 1.2879174947738647,
      "learning_rate": 2.3412295838891123e-06,
      "logits/chosen": -0.004231536295264959,
      "logits/rejected": -1.5393919944763184,
      "logps/chosen": -0.8462308049201965,
      "logps/rejected": -18.241506576538086,
      "loss": 0.8474,
      "num_input_tokens_seen": 14537424,
      "odds_ratio_loss": 8.435163497924805,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08462308347225189,
      "rewards/margins": 1.7395274639129639,
      "rewards/rejected": -1.8241506814956665,
      "sft_loss": 0.003909712191671133,
      "step": 11070
    },
    {
      "epoch": 0.8619214313496694,
      "grad_norm": 8.174221992492676,
      "learning_rate": 2.315482328860544e-06,
      "logits/chosen": -0.4593052268028259,
      "logits/rejected": -1.0363997220993042,
      "logps/chosen": -1.0086044073104858,
      "logps/rejected": -9.086363792419434,
      "loss": 1.0211,
      "num_input_tokens_seen": 14548432,
      "odds_ratio_loss": 10.112740516662598,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.10086043924093246,
      "rewards/margins": 0.8077758550643921,
      "rewards/rejected": -0.9086363911628723,
      "sft_loss": 0.009857363067567348,
      "step": 11080
    },
    {
      "epoch": 0.8626993387786853,
      "grad_norm": 4.442285060882568,
      "learning_rate": 2.2898705568669605e-06,
      "logits/chosen": -0.3338615298271179,
      "logits/rejected": -1.3318408727645874,
      "logps/chosen": -0.9805607795715332,
      "logps/rejected": -15.963170051574707,
      "loss": 0.9877,
      "num_input_tokens_seen": 14558768,
      "odds_ratio_loss": 9.789030075073242,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09805609285831451,
      "rewards/margins": 1.4982608556747437,
      "rewards/rejected": -1.5963170528411865,
      "sft_loss": 0.008807901293039322,
      "step": 11090
    },
    {
      "epoch": 0.8634772462077013,
      "grad_norm": 2.3018581867218018,
      "learning_rate": 2.2643944208743912e-06,
      "logits/chosen": -0.09829515963792801,
      "logits/rejected": -1.4474740028381348,
      "logps/chosen": -0.8492511510848999,
      "logps/rejected": -22.195430755615234,
      "loss": 0.8501,
      "num_input_tokens_seen": 14576400,
      "odds_ratio_loss": 8.346790313720703,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08492513000965118,
      "rewards/margins": 2.134618043899536,
      "rewards/rejected": -2.219543218612671,
      "sft_loss": 0.015408432111144066,
      "step": 11100
    },
    {
      "epoch": 0.8642551536367172,
      "grad_norm": 1.9978951215744019,
      "learning_rate": 2.2390540730387768e-06,
      "logits/chosen": -0.23037929832935333,
      "logits/rejected": -1.7510086297988892,
      "logps/chosen": -0.8015023469924927,
      "logps/rejected": -8.597331047058105,
      "loss": 0.8109,
      "num_input_tokens_seen": 14587648,
      "odds_ratio_loss": 7.873522758483887,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08015023171901703,
      "rewards/margins": 0.7795829772949219,
      "rewards/rejected": -0.8597332239151001,
      "sft_loss": 0.023554079234600067,
      "step": 11110
    },
    {
      "epoch": 0.8650330610657332,
      "grad_norm": 1.5626524686813354,
      "learning_rate": 2.2138496647050783e-06,
      "logits/chosen": -0.2506280243396759,
      "logits/rejected": -1.3620119094848633,
      "logps/chosen": -0.7624492049217224,
      "logps/rejected": -13.497182846069336,
      "loss": 0.7796,
      "num_input_tokens_seen": 14596320,
      "odds_ratio_loss": 7.666055202484131,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.07624492794275284,
      "rewards/margins": 1.2734732627868652,
      "rewards/rejected": -1.3497183322906494,
      "sft_loss": 0.013004831969738007,
      "step": 11120
    },
    {
      "epoch": 0.8658109684947491,
      "grad_norm": 1.4940675497055054,
      "learning_rate": 2.1887813464063433e-06,
      "logits/chosen": -0.17947162687778473,
      "logits/rejected": -1.5230398178100586,
      "logps/chosen": -0.8179603815078735,
      "logps/rejected": -15.498629570007324,
      "loss": 0.8225,
      "num_input_tokens_seen": 14609632,
      "odds_ratio_loss": 8.163220405578613,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08179602771997452,
      "rewards/margins": 1.4680670499801636,
      "rewards/rejected": -1.5498631000518799,
      "sft_loss": 0.0061700670048594475,
      "step": 11130
    },
    {
      "epoch": 0.8665888759237651,
      "grad_norm": 2.0545926094055176,
      "learning_rate": 2.163849267862833e-06,
      "logits/chosen": -0.2326630800962448,
      "logits/rejected": -1.1611685752868652,
      "logps/chosen": -0.8332999348640442,
      "logps/rejected": -12.10827922821045,
      "loss": 0.8531,
      "num_input_tokens_seen": 14622960,
      "odds_ratio_loss": 8.454458236694336,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.08332999050617218,
      "rewards/margins": 1.127497911453247,
      "rewards/rejected": -1.2108280658721924,
      "sft_loss": 0.007665277924388647,
      "step": 11140
    },
    {
      "epoch": 0.8673667833527811,
      "grad_norm": 3.2545974254608154,
      "learning_rate": 2.139053577981115e-06,
      "logits/chosen": -0.12574820220470428,
      "logits/rejected": -1.5405362844467163,
      "logps/chosen": -0.94415682554245,
      "logps/rejected": -11.875570297241211,
      "loss": 0.9558,
      "num_input_tokens_seen": 14635344,
      "odds_ratio_loss": 9.519662857055664,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09441567957401276,
      "rewards/margins": 1.0931413173675537,
      "rewards/rejected": -1.1875572204589844,
      "sft_loss": 0.003794616786763072,
      "step": 11150
    },
    {
      "epoch": 0.868144690781797,
      "grad_norm": 3.8555874824523926,
      "learning_rate": 2.114394424853178e-06,
      "logits/chosen": -0.030388841405510902,
      "logits/rejected": -1.6810109615325928,
      "logps/chosen": -0.8759109377861023,
      "logps/rejected": -16.781967163085938,
      "loss": 0.8762,
      "num_input_tokens_seen": 14648704,
      "odds_ratio_loss": 8.72721004486084,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08759109675884247,
      "rewards/margins": 1.5906057357788086,
      "rewards/rejected": -1.678196668624878,
      "sft_loss": 0.003505890490487218,
      "step": 11160
    },
    {
      "epoch": 0.868922598210813,
      "grad_norm": 2.347398281097412,
      "learning_rate": 2.089871955755543e-06,
      "logits/chosen": -0.04993351176381111,
      "logits/rejected": -1.4341214895248413,
      "logps/chosen": -0.8593047261238098,
      "logps/rejected": -7.1145734786987305,
      "loss": 0.8976,
      "num_input_tokens_seen": 14659184,
      "odds_ratio_loss": 8.902770042419434,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.0859304741024971,
      "rewards/margins": 0.6255267858505249,
      "rewards/rejected": -0.7114574313163757,
      "sft_loss": 0.007294154725968838,
      "step": 11170
    },
    {
      "epoch": 0.8697005056398288,
      "grad_norm": 2.703765869140625,
      "learning_rate": 2.065486317148396e-06,
      "logits/chosen": -0.11260922253131866,
      "logits/rejected": -1.4905784130096436,
      "logps/chosen": -0.7861498594284058,
      "logps/rejected": -7.284165382385254,
      "loss": 0.8291,
      "num_input_tokens_seen": 14671552,
      "odds_ratio_loss": 8.243640899658203,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.0786149874329567,
      "rewards/margins": 0.6498016119003296,
      "rewards/rejected": -0.7284165620803833,
      "sft_loss": 0.004718432668596506,
      "step": 11180
    },
    {
      "epoch": 0.8704784130688448,
      "grad_norm": 1.5052175521850586,
      "learning_rate": 2.041237654674691e-06,
      "logits/chosen": -0.058899033814668655,
      "logits/rejected": -1.6328794956207275,
      "logps/chosen": -0.9910447001457214,
      "logps/rejected": -19.82402801513672,
      "loss": 0.9947,
      "num_input_tokens_seen": 14686624,
      "odds_ratio_loss": 9.896458625793457,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09910447150468826,
      "rewards/margins": 1.8832982778549194,
      "rewards/rejected": -1.9824028015136719,
      "sft_loss": 0.005023724865168333,
      "step": 11190
    },
    {
      "epoch": 0.8712563204978607,
      "grad_norm": 3.2280795574188232,
      "learning_rate": 2.017126113159301e-06,
      "logits/chosen": -0.1030881404876709,
      "logits/rejected": -1.545559048652649,
      "logps/chosen": -0.9980408549308777,
      "logps/rejected": -17.469547271728516,
      "loss": 1.012,
      "num_input_tokens_seen": 14702144,
      "odds_ratio_loss": 10.044103622436523,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.09980408847332001,
      "rewards/margins": 1.6471507549285889,
      "rewards/rejected": -1.7469545602798462,
      "sft_loss": 0.0076124765910208225,
      "step": 11200
    },
    {
      "epoch": 0.8720342279268767,
      "grad_norm": 7.864035129547119,
      "learning_rate": 1.993151836608151e-06,
      "logits/chosen": -0.17491404712200165,
      "logits/rejected": -1.3855005502700806,
      "logps/chosen": -0.5778329372406006,
      "logps/rejected": -13.704730033874512,
      "loss": 0.5797,
      "num_input_tokens_seen": 14715008,
      "odds_ratio_loss": 5.734278202056885,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.05778329446911812,
      "rewards/margins": 1.3126897811889648,
      "rewards/rejected": -1.3704731464385986,
      "sft_loss": 0.0062981559894979,
      "step": 11210
    },
    {
      "epoch": 0.8728121353558926,
      "grad_norm": 161.39295959472656,
      "learning_rate": 1.9693149682073393e-06,
      "logits/chosen": -0.49991482496261597,
      "logits/rejected": -1.12697434425354,
      "logps/chosen": -1.1875393390655518,
      "logps/rejected": -12.513293266296387,
      "loss": 1.1946,
      "num_input_tokens_seen": 14726608,
      "odds_ratio_loss": 11.83639144897461,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.11875393241643906,
      "rewards/margins": 1.132575511932373,
      "rewards/rejected": -1.2513294219970703,
      "sft_loss": 0.01099352352321148,
      "step": 11220
    },
    {
      "epoch": 0.8735900427849086,
      "grad_norm": 29.137767791748047,
      "learning_rate": 1.9456156503223107e-06,
      "logits/chosen": -0.20925652980804443,
      "logits/rejected": -1.4698998928070068,
      "logps/chosen": -0.962933361530304,
      "logps/rejected": -16.86948013305664,
      "loss": 0.9768,
      "num_input_tokens_seen": 14739920,
      "odds_ratio_loss": 9.696186065673828,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.09629333019256592,
      "rewards/margins": 1.5906546115875244,
      "rewards/rejected": -1.6869480609893799,
      "sft_loss": 0.0071391938254237175,
      "step": 11230
    },
    {
      "epoch": 0.8743679502139245,
      "grad_norm": 3.734955072402954,
      "learning_rate": 1.922054024496975e-06,
      "logits/chosen": -0.39152416586875916,
      "logits/rejected": -1.984805703163147,
      "logps/chosen": -1.262678861618042,
      "logps/rejected": -21.21131134033203,
      "loss": 1.2682,
      "num_input_tokens_seen": 14750096,
      "odds_ratio_loss": 12.499911308288574,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.12626788020133972,
      "rewards/margins": 1.9948632717132568,
      "rewards/rejected": -2.121131420135498,
      "sft_loss": 0.01821861043572426,
      "step": 11240
    },
    {
      "epoch": 0.8751458576429405,
      "grad_norm": 3.2851293087005615,
      "learning_rate": 1.898630231452897e-06,
      "logits/chosen": -0.4599381983280182,
      "logits/rejected": -1.0844638347625732,
      "logps/chosen": -0.9390494227409363,
      "logps/rejected": -15.288253784179688,
      "loss": 0.9424,
      "num_input_tokens_seen": 14762112,
      "odds_ratio_loss": 9.057352066040039,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09390494227409363,
      "rewards/margins": 1.4349205493927002,
      "rewards/rejected": -1.5288255214691162,
      "sft_loss": 0.03662865236401558,
      "step": 11250
    },
    {
      "epoch": 0.8759237650719565,
      "grad_norm": 1.8594413995742798,
      "learning_rate": 1.8753444110884172e-06,
      "logits/chosen": -0.2478855401277542,
      "logits/rejected": -1.322733998298645,
      "logps/chosen": -1.0074350833892822,
      "logps/rejected": -10.416272163391113,
      "loss": 1.0263,
      "num_input_tokens_seen": 14772640,
      "odds_ratio_loss": 10.212084770202637,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.10074350982904434,
      "rewards/margins": 0.9408837556838989,
      "rewards/rejected": -1.041627287864685,
      "sft_loss": 0.005066190846264362,
      "step": 11260
    },
    {
      "epoch": 0.8767016725009724,
      "grad_norm": 5.2853827476501465,
      "learning_rate": 1.8521967024778487e-06,
      "logits/chosen": -0.49381309747695923,
      "logits/rejected": -1.2733218669891357,
      "logps/chosen": -0.8687035441398621,
      "logps/rejected": -16.657901763916016,
      "loss": 0.8807,
      "num_input_tokens_seen": 14783424,
      "odds_ratio_loss": 8.629514694213867,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08687035739421844,
      "rewards/margins": 1.5789198875427246,
      "rewards/rejected": -1.6657902002334595,
      "sft_loss": 0.017769496887922287,
      "step": 11270
    },
    {
      "epoch": 0.8774795799299884,
      "grad_norm": 3.3902578353881836,
      "learning_rate": 1.829187243870628e-06,
      "logits/chosen": -0.3714262843132019,
      "logits/rejected": -1.1370850801467896,
      "logps/chosen": -0.8502224683761597,
      "logps/rejected": -13.264406204223633,
      "loss": 0.8601,
      "num_input_tokens_seen": 14798224,
      "odds_ratio_loss": 8.506078720092773,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.08502225577831268,
      "rewards/margins": 1.2414183616638184,
      "rewards/rejected": -1.3264405727386475,
      "sft_loss": 0.009510818868875504,
      "step": 11280
    },
    {
      "epoch": 0.8782574873590043,
      "grad_norm": 3.657287120819092,
      "learning_rate": 1.8063161726904976e-06,
      "logits/chosen": -0.20781084895133972,
      "logits/rejected": -1.3679994344711304,
      "logps/chosen": -0.8147785067558289,
      "logps/rejected": -9.873483657836914,
      "loss": 0.8365,
      "num_input_tokens_seen": 14809200,
      "odds_ratio_loss": 8.311952590942383,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.08147785812616348,
      "rewards/margins": 0.9058707356452942,
      "rewards/rejected": -0.9873485565185547,
      "sft_loss": 0.005294256377965212,
      "step": 11290
    },
    {
      "epoch": 0.8790353947880202,
      "grad_norm": 2.625351667404175,
      "learning_rate": 1.783583625534682e-06,
      "logits/chosen": 0.0041309236548841,
      "logits/rejected": -1.4668128490447998,
      "logps/chosen": -0.8829860687255859,
      "logps/rejected": -15.129183769226074,
      "loss": 0.9016,
      "num_input_tokens_seen": 14828032,
      "odds_ratio_loss": 8.983991622924805,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.08829861134290695,
      "rewards/margins": 1.4246199131011963,
      "rewards/rejected": -1.512918472290039,
      "sft_loss": 0.003154673147946596,
      "step": 11300
    },
    {
      "epoch": 0.8798133022170361,
      "grad_norm": 8.057369232177734,
      "learning_rate": 1.7609897381730662e-06,
      "logits/chosen": -0.4841797351837158,
      "logits/rejected": -1.6735862493515015,
      "logps/chosen": -0.9034742116928101,
      "logps/rejected": -19.795330047607422,
      "loss": 0.9042,
      "num_input_tokens_seen": 14840864,
      "odds_ratio_loss": 8.911484718322754,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09034740924835205,
      "rewards/margins": 1.8891855478286743,
      "rewards/rejected": -1.9795329570770264,
      "sft_loss": 0.013028261251747608,
      "step": 11310
    },
    {
      "epoch": 0.8805912096460521,
      "grad_norm": 3.4109108448028564,
      "learning_rate": 1.7385346455474016e-06,
      "logits/chosen": -0.47292718291282654,
      "logits/rejected": -1.4690214395523071,
      "logps/chosen": -0.978543758392334,
      "logps/rejected": -16.113407135009766,
      "loss": 0.9976,
      "num_input_tokens_seen": 14855616,
      "odds_ratio_loss": 9.500051498413086,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.097854383289814,
      "rewards/margins": 1.5134862661361694,
      "rewards/rejected": -1.611340880393982,
      "sft_loss": 0.04762210696935654,
      "step": 11320
    },
    {
      "epoch": 0.881369117075068,
      "grad_norm": 3.220376968383789,
      "learning_rate": 1.716218481770482e-06,
      "logits/chosen": 0.024894937872886658,
      "logits/rejected": -1.4938503503799438,
      "logps/chosen": -0.8936010599136353,
      "logps/rejected": -11.886364936828613,
      "loss": 0.8987,
      "num_input_tokens_seen": 14864624,
      "odds_ratio_loss": 8.929697036743164,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08936010301113129,
      "rewards/margins": 1.0992765426635742,
      "rewards/rejected": -1.1886365413665771,
      "sft_loss": 0.00577815156430006,
      "step": 11330
    },
    {
      "epoch": 0.882147024504084,
      "grad_norm": 1.3350600004196167,
      "learning_rate": 1.6940413801253524e-06,
      "logits/chosen": -0.13089770078659058,
      "logits/rejected": -1.4039313793182373,
      "logps/chosen": -0.8123601078987122,
      "logps/rejected": -11.132844924926758,
      "loss": 0.8339,
      "num_input_tokens_seen": 14880496,
      "odds_ratio_loss": 8.236737251281738,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.08123601227998734,
      "rewards/margins": 1.0320484638214111,
      "rewards/rejected": -1.1132845878601074,
      "sft_loss": 0.010246511548757553,
      "step": 11340
    },
    {
      "epoch": 0.8829249319331,
      "grad_norm": 3.229665994644165,
      "learning_rate": 1.6720034730645152e-06,
      "logits/chosen": -0.26986435055732727,
      "logits/rejected": -1.1807479858398438,
      "logps/chosen": -1.585399866104126,
      "logps/rejected": -12.02979564666748,
      "loss": 1.5936,
      "num_input_tokens_seen": 14891360,
      "odds_ratio_loss": 15.809844970703125,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15853998064994812,
      "rewards/margins": 1.044439673423767,
      "rewards/rejected": -1.2029794454574585,
      "sft_loss": 0.01263892650604248,
      "step": 11350
    },
    {
      "epoch": 0.8837028393621159,
      "grad_norm": 10.792923927307129,
      "learning_rate": 1.6501048922091255e-06,
      "logits/chosen": -0.7156990766525269,
      "logits/rejected": -1.0069489479064941,
      "logps/chosen": -1.0320378541946411,
      "logps/rejected": -12.793333053588867,
      "loss": 1.0346,
      "num_input_tokens_seen": 14902688,
      "odds_ratio_loss": 9.464272499084473,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.10320379585027695,
      "rewards/margins": 1.1761295795440674,
      "rewards/rejected": -1.2793333530426025,
      "sft_loss": 0.08816216140985489,
      "step": 11360
    },
    {
      "epoch": 0.8844807467911319,
      "grad_norm": 24.13997459411621,
      "learning_rate": 1.628345768348219e-06,
      "logits/chosen": -0.17151141166687012,
      "logits/rejected": -1.7036211490631104,
      "logps/chosen": -0.9553998112678528,
      "logps/rejected": -15.676922798156738,
      "loss": 0.9604,
      "num_input_tokens_seen": 14915024,
      "odds_ratio_loss": 8.78715705871582,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09553997963666916,
      "rewards/margins": 1.4721522331237793,
      "rewards/rejected": -1.5676923990249634,
      "sft_loss": 0.08164925128221512,
      "step": 11370
    },
    {
      "epoch": 0.8852586542201478,
      "grad_norm": 3.026034355163574,
      "learning_rate": 1.6067262314379327e-06,
      "logits/chosen": -0.3190793991088867,
      "logits/rejected": -1.447247862815857,
      "logps/chosen": -0.9018405079841614,
      "logps/rejected": -13.972577095031738,
      "loss": 0.9175,
      "num_input_tokens_seen": 14924096,
      "odds_ratio_loss": 9.034456253051758,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.0901840552687645,
      "rewards/margins": 1.3070735931396484,
      "rewards/rejected": -1.397257685661316,
      "sft_loss": 0.014050662517547607,
      "step": 11380
    },
    {
      "epoch": 0.8860365616491638,
      "grad_norm": 1.9133740663528442,
      "learning_rate": 1.585246410600713e-06,
      "logits/chosen": -0.0015375495422631502,
      "logits/rejected": -1.5664477348327637,
      "logps/chosen": -0.9613587260246277,
      "logps/rejected": -9.113849639892578,
      "loss": 0.9809,
      "num_input_tokens_seen": 14937840,
      "odds_ratio_loss": 9.752358436584473,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09613586962223053,
      "rewards/margins": 0.8152491450309753,
      "rewards/rejected": -0.9113848805427551,
      "sft_loss": 0.005694333929568529,
      "step": 11390
    },
    {
      "epoch": 0.8868144690781797,
      "grad_norm": 23.818763732910156,
      "learning_rate": 1.563906434124554e-06,
      "logits/chosen": -0.5499352216720581,
      "logits/rejected": -1.3355505466461182,
      "logps/chosen": -0.7920863032341003,
      "logps/rejected": -23.026779174804688,
      "loss": 0.7929,
      "num_input_tokens_seen": 14949600,
      "odds_ratio_loss": 7.54303503036499,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.0792086273431778,
      "rewards/margins": 2.2234692573547363,
      "rewards/rejected": -2.302677869796753,
      "sft_loss": 0.0385659821331501,
      "step": 11400
    },
    {
      "epoch": 0.8875923765071957,
      "grad_norm": 3.3071045875549316,
      "learning_rate": 1.5427064294622318e-06,
      "logits/chosen": -0.22468701004981995,
      "logits/rejected": -1.2597036361694336,
      "logps/chosen": -0.7308349609375,
      "logps/rejected": -8.260499954223633,
      "loss": 0.7525,
      "num_input_tokens_seen": 14962416,
      "odds_ratio_loss": 7.281064510345459,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.07308349758386612,
      "rewards/margins": 0.7529664635658264,
      "rewards/rejected": -0.8260499238967896,
      "sft_loss": 0.02442113682627678,
      "step": 11410
    },
    {
      "epoch": 0.8883702839362115,
      "grad_norm": 2.004885673522949,
      "learning_rate": 1.521646523230552e-06,
      "logits/chosen": -0.45084089040756226,
      "logits/rejected": -1.5717991590499878,
      "logps/chosen": -0.6790512204170227,
      "logps/rejected": -18.60866355895996,
      "loss": 0.6949,
      "num_input_tokens_seen": 14974592,
      "odds_ratio_loss": 6.8787360191345215,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.06790511310100555,
      "rewards/margins": 1.7929614782333374,
      "rewards/rejected": -1.8608665466308594,
      "sft_loss": 0.007033783011138439,
      "step": 11420
    },
    {
      "epoch": 0.8891481913652275,
      "grad_norm": 1.7734142541885376,
      "learning_rate": 1.5007268412095654e-06,
      "logits/chosen": -0.3790365755558014,
      "logits/rejected": -1.9984188079833984,
      "logps/chosen": -0.9335896372795105,
      "logps/rejected": -18.616718292236328,
      "loss": 0.9439,
      "num_input_tokens_seen": 14983280,
      "odds_ratio_loss": 8.583988189697266,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.09335896372795105,
      "rewards/margins": 1.7683128118515015,
      "rewards/rejected": -1.861671805381775,
      "sft_loss": 0.08547282963991165,
      "step": 11430
    },
    {
      "epoch": 0.8899260987942434,
      "grad_norm": 3.9545562267303467,
      "learning_rate": 1.4799475083418474e-06,
      "logits/chosen": -0.3433544635772705,
      "logits/rejected": -1.3527787923812866,
      "logps/chosen": -0.8042358160018921,
      "logps/rejected": -15.672798156738281,
      "loss": 0.8139,
      "num_input_tokens_seen": 14995552,
      "odds_ratio_loss": 8.090936660766602,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08042358607053757,
      "rewards/margins": 1.4868563413619995,
      "rewards/rejected": -1.5672800540924072,
      "sft_loss": 0.0047922274097800255,
      "step": 11440
    },
    {
      "epoch": 0.8907040062232594,
      "grad_norm": 11.453007698059082,
      "learning_rate": 1.4593086487317386e-06,
      "logits/chosen": -0.2827529311180115,
      "logits/rejected": -1.4444782733917236,
      "logps/chosen": -0.8965161442756653,
      "logps/rejected": -13.879638671875,
      "loss": 0.9004,
      "num_input_tokens_seen": 15005920,
      "odds_ratio_loss": 8.895330429077148,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08965159952640533,
      "rewards/margins": 1.2983124256134033,
      "rewards/rejected": -1.3879640102386475,
      "sft_loss": 0.010893709026277065,
      "step": 11450
    },
    {
      "epoch": 0.8914819136522754,
      "grad_norm": 2.555229902267456,
      "learning_rate": 1.4388103856446067e-06,
      "logits/chosen": -0.34105560183525085,
      "logits/rejected": -1.4867103099822998,
      "logps/chosen": -0.9608953595161438,
      "logps/rejected": -15.809675216674805,
      "loss": 0.972,
      "num_input_tokens_seen": 15018672,
      "odds_ratio_loss": 9.660136222839355,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.09608952701091766,
      "rewards/margins": 1.4848780632019043,
      "rewards/rejected": -1.580967664718628,
      "sft_loss": 0.0059747775085270405,
      "step": 11460
    },
    {
      "epoch": 0.8922598210812913,
      "grad_norm": 2.2812118530273438,
      "learning_rate": 1.4184528415060988e-06,
      "logits/chosen": -0.2780112624168396,
      "logits/rejected": -1.9391981363296509,
      "logps/chosen": -0.9633452296257019,
      "logps/rejected": -26.054580688476562,
      "loss": 0.9642,
      "num_input_tokens_seen": 15030528,
      "odds_ratio_loss": 9.586477279663086,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09633452445268631,
      "rewards/margins": 2.5091235637664795,
      "rewards/rejected": -2.6054580211639404,
      "sft_loss": 0.005508183501660824,
      "step": 11470
    },
    {
      "epoch": 0.8930377285103073,
      "grad_norm": 1.5275903940200806,
      "learning_rate": 1.3982361379014319e-06,
      "logits/chosen": -0.13214851915836334,
      "logits/rejected": -1.8039649724960327,
      "logps/chosen": -1.1114555597305298,
      "logps/rejected": -19.613750457763672,
      "loss": 1.1139,
      "num_input_tokens_seen": 15042128,
      "odds_ratio_loss": 10.567789077758789,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.11114554107189178,
      "rewards/margins": 1.8502298593521118,
      "rewards/rejected": -1.9613752365112305,
      "sft_loss": 0.057110778987407684,
      "step": 11480
    },
    {
      "epoch": 0.8938156359393232,
      "grad_norm": 3.1028542518615723,
      "learning_rate": 1.378160395574643e-06,
      "logits/chosen": -0.22435922920703888,
      "logits/rejected": -1.4496055841445923,
      "logps/chosen": -0.6374467611312866,
      "logps/rejected": -20.233562469482422,
      "loss": 0.6479,
      "num_input_tokens_seen": 15053104,
      "odds_ratio_loss": 6.419812202453613,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.06374466419219971,
      "rewards/margins": 1.9596116542816162,
      "rewards/rejected": -2.0233561992645264,
      "sft_loss": 0.005906237289309502,
      "step": 11490
    },
    {
      "epoch": 0.8945935433683392,
      "grad_norm": 1.0684871673583984,
      "learning_rate": 1.358225734427887e-06,
      "logits/chosen": -0.29106268286705017,
      "logits/rejected": -1.0870394706726074,
      "logps/chosen": -1.0519826412200928,
      "logps/rejected": -11.098809242248535,
      "loss": 1.0813,
      "num_input_tokens_seen": 15066512,
      "odds_ratio_loss": 9.962423324584961,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.10519824922084808,
      "rewards/margins": 1.0046826601028442,
      "rewards/rejected": -1.109881043434143,
      "sft_loss": 0.08502072095870972,
      "step": 11500
    },
    {
      "epoch": 0.8953714507973551,
      "grad_norm": 6.9853010177612305,
      "learning_rate": 1.3384322735207123e-06,
      "logits/chosen": -0.0020437941420823336,
      "logits/rejected": -1.803490400314331,
      "logps/chosen": -0.9121143221855164,
      "logps/rejected": -15.750417709350586,
      "loss": 0.9158,
      "num_input_tokens_seen": 15079392,
      "odds_ratio_loss": 9.098687171936035,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09121143072843552,
      "rewards/margins": 1.483830451965332,
      "rewards/rejected": -1.5750417709350586,
      "sft_loss": 0.005918119102716446,
      "step": 11510
    },
    {
      "epoch": 0.8961493582263711,
      "grad_norm": 5.0061211585998535,
      "learning_rate": 1.318780131069347e-06,
      "logits/chosen": -0.30716389417648315,
      "logits/rejected": -1.30451238155365,
      "logps/chosen": -0.6082115769386292,
      "logps/rejected": -13.56993579864502,
      "loss": 0.6161,
      "num_input_tokens_seen": 15092336,
      "odds_ratio_loss": 6.121800422668457,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.060821156948804855,
      "rewards/margins": 1.2961723804473877,
      "rewards/rejected": -1.356993556022644,
      "sft_loss": 0.003893594490364194,
      "step": 11520
    },
    {
      "epoch": 0.896927265655387,
      "grad_norm": 7.328099250793457,
      "learning_rate": 1.2992694244460008e-06,
      "logits/chosen": -0.4314120411872864,
      "logits/rejected": -1.3663753271102905,
      "logps/chosen": -0.7535328269004822,
      "logps/rejected": -16.93317413330078,
      "loss": 0.7547,
      "num_input_tokens_seen": 15107072,
      "odds_ratio_loss": 7.248821258544922,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.07535327970981598,
      "rewards/margins": 1.6179643869400024,
      "rewards/rejected": -1.6933176517486572,
      "sft_loss": 0.029783323407173157,
      "step": 11530
    },
    {
      "epoch": 0.897705173084403,
      "grad_norm": 2.2374439239501953,
      "learning_rate": 1.2799002701781526e-06,
      "logits/chosen": -0.08255377411842346,
      "logits/rejected": -1.3694899082183838,
      "logps/chosen": -1.6034971475601196,
      "logps/rejected": -10.680310249328613,
      "loss": 1.6142,
      "num_input_tokens_seen": 15122032,
      "odds_ratio_loss": 15.007932662963867,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.16034969687461853,
      "rewards/margins": 0.9076812863349915,
      "rewards/rejected": -1.0680310726165771,
      "sft_loss": 0.11340135335922241,
      "step": 11540
    },
    {
      "epoch": 0.8984830805134189,
      "grad_norm": 2.645416021347046,
      "learning_rate": 1.2606727839478721e-06,
      "logits/chosen": -0.17675833404064178,
      "logits/rejected": -1.2538154125213623,
      "logps/chosen": -0.646391749382019,
      "logps/rejected": -19.915109634399414,
      "loss": 0.6518,
      "num_input_tokens_seen": 15135648,
      "odds_ratio_loss": 6.432554721832275,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.06463916599750519,
      "rewards/margins": 1.9268718957901,
      "rewards/rejected": -1.9915109872817993,
      "sft_loss": 0.00850028544664383,
      "step": 11550
    },
    {
      "epoch": 0.8992609879424348,
      "grad_norm": 5.000986576080322,
      "learning_rate": 1.2415870805911056e-06,
      "logits/chosen": -0.20390114188194275,
      "logits/rejected": -1.4622828960418701,
      "logps/chosen": -0.9593456983566284,
      "logps/rejected": -12.142216682434082,
      "loss": 0.9793,
      "num_input_tokens_seen": 15146576,
      "odds_ratio_loss": 9.574671745300293,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.09593456983566284,
      "rewards/margins": 1.1182870864868164,
      "rewards/rejected": -1.2142215967178345,
      "sft_loss": 0.021797997877001762,
      "step": 11560
    },
    {
      "epoch": 0.9000388953714508,
      "grad_norm": 5.9946088790893555,
      "learning_rate": 1.2226432740970073e-06,
      "logits/chosen": -0.18088433146476746,
      "logits/rejected": -1.5347539186477661,
      "logps/chosen": -0.8887677192687988,
      "logps/rejected": -14.090968132019043,
      "loss": 0.9127,
      "num_input_tokens_seen": 15160624,
      "odds_ratio_loss": 9.078805923461914,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.08887676894664764,
      "rewards/margins": 1.3202202320098877,
      "rewards/rejected": -1.4090967178344727,
      "sft_loss": 0.004832427017390728,
      "step": 11570
    },
    {
      "epoch": 0.9008168028004667,
      "grad_norm": 5.074446678161621,
      "learning_rate": 1.2038414776072593e-06,
      "logits/chosen": -0.04299873486161232,
      "logits/rejected": -1.6535265445709229,
      "logps/chosen": -0.9011691808700562,
      "logps/rejected": -11.200329780578613,
      "loss": 0.9546,
      "num_input_tokens_seen": 15175184,
      "odds_ratio_loss": 9.490025520324707,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.09011692553758621,
      "rewards/margins": 1.0299160480499268,
      "rewards/rejected": -1.1200329065322876,
      "sft_loss": 0.005555834621191025,
      "step": 11580
    },
    {
      "epoch": 0.9015947102294827,
      "grad_norm": 4.478003978729248,
      "learning_rate": 1.1851818034153833e-06,
      "logits/chosen": -0.0886651873588562,
      "logits/rejected": -1.5455336570739746,
      "logps/chosen": -1.1897647380828857,
      "logps/rejected": -10.544865608215332,
      "loss": 1.2083,
      "num_input_tokens_seen": 15186752,
      "odds_ratio_loss": 11.775666236877441,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.11897647380828857,
      "rewards/margins": 0.9355101585388184,
      "rewards/rejected": -1.054486632347107,
      "sft_loss": 0.030766809359192848,
      "step": 11590
    },
    {
      "epoch": 0.9023726176584986,
      "grad_norm": 2.3363897800445557,
      "learning_rate": 1.1666643629660856e-06,
      "logits/chosen": -0.8766778707504272,
      "logits/rejected": -0.7064003944396973,
      "logps/chosen": -1.0922223329544067,
      "logps/rejected": -8.02529239654541,
      "loss": 1.095,
      "num_input_tokens_seen": 15196896,
      "odds_ratio_loss": 10.510167121887207,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.10922224819660187,
      "rewards/margins": 0.6933069229125977,
      "rewards/rejected": -0.8025292158126831,
      "sft_loss": 0.04393957182765007,
      "step": 11600
    },
    {
      "epoch": 0.9031505250875146,
      "grad_norm": 2.4352638721466064,
      "learning_rate": 1.1482892668545742e-06,
      "logits/chosen": -0.14885909855365753,
      "logits/rejected": -1.753119707107544,
      "logps/chosen": -0.7887487411499023,
      "logps/rejected": -13.43248176574707,
      "loss": 0.7962,
      "num_input_tokens_seen": 15210464,
      "odds_ratio_loss": 7.846124172210693,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.078874871134758,
      "rewards/margins": 1.264373540878296,
      "rewards/rejected": -1.3432481288909912,
      "sft_loss": 0.011604775674641132,
      "step": 11610
    },
    {
      "epoch": 0.9039284325165305,
      "grad_norm": 2.8514859676361084,
      "learning_rate": 1.1300566248259092e-06,
      "logits/chosen": -0.23481757938861847,
      "logits/rejected": -1.2238056659698486,
      "logps/chosen": -0.7990361452102661,
      "logps/rejected": -9.189723014831543,
      "loss": 0.8106,
      "num_input_tokens_seen": 15219536,
      "odds_ratio_loss": 8.021627426147461,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.07990361750125885,
      "rewards/margins": 0.8390687108039856,
      "rewards/rejected": -0.9189723134040833,
      "sft_loss": 0.008481072261929512,
      "step": 11620
    },
    {
      "epoch": 0.9047063399455465,
      "grad_norm": 1.4698771238327026,
      "learning_rate": 1.1119665457743484e-06,
      "logits/chosen": -0.05531783029437065,
      "logits/rejected": -1.8035624027252197,
      "logps/chosen": -1.8927888870239258,
      "logps/rejected": -19.5323429107666,
      "loss": 1.8981,
      "num_input_tokens_seen": 15231792,
      "odds_ratio_loss": 18.91992950439453,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.18927890062332153,
      "rewards/margins": 1.7639554738998413,
      "rewards/rejected": -1.953234314918518,
      "sft_loss": 0.006080970633774996,
      "step": 11630
    },
    {
      "epoch": 0.9054842473745625,
      "grad_norm": 8.166078567504883,
      "learning_rate": 1.0940191377426938e-06,
      "logits/chosen": -0.18938185274600983,
      "logits/rejected": -1.0948208570480347,
      "logps/chosen": -0.7792876958847046,
      "logps/rejected": -10.034112930297852,
      "loss": 0.7905,
      "num_input_tokens_seen": 15242496,
      "odds_ratio_loss": 7.8034515380859375,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.07792877405881882,
      "rewards/margins": 0.9254825711250305,
      "rewards/rejected": -1.0034112930297852,
      "sft_loss": 0.0101962611079216,
      "step": 11640
    },
    {
      "epoch": 0.9062621548035784,
      "grad_norm": 2.3272478580474854,
      "learning_rate": 1.076214507921644e-06,
      "logits/chosen": -0.2795869708061218,
      "logits/rejected": -1.4620563983917236,
      "logps/chosen": -0.9061124920845032,
      "logps/rejected": -14.130447387695312,
      "loss": 0.917,
      "num_input_tokens_seen": 15256656,
      "odds_ratio_loss": 9.063624382019043,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09061124175786972,
      "rewards/margins": 1.3224334716796875,
      "rewards/rejected": -1.4130446910858154,
      "sft_loss": 0.010650251060724258,
      "step": 11650
    },
    {
      "epoch": 0.9070400622325944,
      "grad_norm": 3.9123356342315674,
      "learning_rate": 1.0585527626491564e-06,
      "logits/chosen": -0.16889598965644836,
      "logits/rejected": -1.0222868919372559,
      "logps/chosen": -0.9573249816894531,
      "logps/rejected": -10.36623477935791,
      "loss": 0.9742,
      "num_input_tokens_seen": 15273632,
      "odds_ratio_loss": 9.666048049926758,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09573251754045486,
      "rewards/margins": 0.9408909678459167,
      "rewards/rejected": -1.036623477935791,
      "sft_loss": 0.007627577520906925,
      "step": 11660
    },
    {
      "epoch": 0.9078179696616102,
      "grad_norm": 2.5804312229156494,
      "learning_rate": 1.0410340074098162e-06,
      "logits/chosen": -0.29126492142677307,
      "logits/rejected": -1.21029794216156,
      "logps/chosen": -0.8057632446289062,
      "logps/rejected": -7.643909454345703,
      "loss": 0.8336,
      "num_input_tokens_seen": 15285872,
      "odds_ratio_loss": 8.258122444152832,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.0805763304233551,
      "rewards/margins": 0.6838146448135376,
      "rewards/rejected": -0.7643910050392151,
      "sft_loss": 0.007780120708048344,
      "step": 11670
    },
    {
      "epoch": 0.9085958770906262,
      "grad_norm": 1.9556622505187988,
      "learning_rate": 1.0236583468341914e-06,
      "logits/chosen": -0.3107249438762665,
      "logits/rejected": -1.642421007156372,
      "logps/chosen": -0.9437317848205566,
      "logps/rejected": -11.675893783569336,
      "loss": 0.9522,
      "num_input_tokens_seen": 15298096,
      "odds_ratio_loss": 9.248234748840332,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09437316656112671,
      "rewards/margins": 1.073216199874878,
      "rewards/rejected": -1.1675894260406494,
      "sft_loss": 0.027393311262130737,
      "step": 11680
    },
    {
      "epoch": 0.9093737845196421,
      "grad_norm": 1.9555792808532715,
      "learning_rate": 1.0064258846982282e-06,
      "logits/chosen": -0.3013989329338074,
      "logits/rejected": -1.2657264471054077,
      "logps/chosen": -1.101585865020752,
      "logps/rejected": -16.60849952697754,
      "loss": 1.1176,
      "num_input_tokens_seen": 15307776,
      "odds_ratio_loss": 10.710332870483398,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.11015859991312027,
      "rewards/margins": 1.5506913661956787,
      "rewards/rejected": -1.660850167274475,
      "sft_loss": 0.04653622955083847,
      "step": 11690
    },
    {
      "epoch": 0.9101516919486581,
      "grad_norm": 3.1522815227508545,
      "learning_rate": 9.893367239226208e-07,
      "logits/chosen": -0.09980596601963043,
      "logits/rejected": -1.88507878780365,
      "logps/chosen": -0.9663987159729004,
      "logps/rejected": -11.776286125183105,
      "loss": 0.9805,
      "num_input_tokens_seen": 15320480,
      "odds_ratio_loss": 9.767982482910156,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.09663987159729004,
      "rewards/margins": 1.0809887647628784,
      "rewards/rejected": -1.1776286363601685,
      "sft_loss": 0.0037159952335059643,
      "step": 11700
    },
    {
      "epoch": 0.910929599377674,
      "grad_norm": 2.2265751361846924,
      "learning_rate": 9.723909665721954e-07,
      "logits/chosen": -0.07370741665363312,
      "logits/rejected": -1.345320701599121,
      "logps/chosen": -0.670822024345398,
      "logps/rejected": -15.495183944702148,
      "loss": 0.6806,
      "num_input_tokens_seen": 15329120,
      "odds_ratio_loss": 6.744556427001953,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.06708221137523651,
      "rewards/margins": 1.4824364185333252,
      "rewards/rejected": -1.5495185852050781,
      "sft_loss": 0.006134716793894768,
      "step": 11710
    },
    {
      "epoch": 0.91170750680669,
      "grad_norm": 5.613677978515625,
      "learning_rate": 9.555887138553015e-07,
      "logits/chosen": -0.037133391946554184,
      "logits/rejected": -1.6061843633651733,
      "logps/chosen": -0.9063383340835571,
      "logps/rejected": -6.80635929107666,
      "loss": 0.919,
      "num_input_tokens_seen": 15341536,
      "odds_ratio_loss": 9.143983840942383,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.0906338319182396,
      "rewards/margins": 0.5900020599365234,
      "rewards/rejected": -0.6806358098983765,
      "sft_loss": 0.004566813353449106,
      "step": 11720
    },
    {
      "epoch": 0.912485414235706,
      "grad_norm": 1.902655005455017,
      "learning_rate": 9.389300661232142e-07,
      "logits/chosen": -0.09280941635370255,
      "logits/rejected": -1.4851089715957642,
      "logps/chosen": -0.75700843334198,
      "logps/rejected": -11.415763854980469,
      "loss": 0.7633,
      "num_input_tokens_seen": 15351520,
      "odds_ratio_loss": 7.47744607925415,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.07570084929466248,
      "rewards/margins": 1.06587553024292,
      "rewards/rejected": -1.1415762901306152,
      "sft_loss": 0.015602183528244495,
      "step": 11730
    },
    {
      "epoch": 0.9132633216647219,
      "grad_norm": 1.1491291522979736,
      "learning_rate": 9.224151228695216e-07,
      "logits/chosen": -0.21760201454162598,
      "logits/rejected": -1.598064661026001,
      "logps/chosen": -0.8968531489372253,
      "logps/rejected": -14.969205856323242,
      "loss": 0.9022,
      "num_input_tokens_seen": 15370992,
      "odds_ratio_loss": 8.735642433166504,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08968530595302582,
      "rewards/margins": 1.4072353839874268,
      "rewards/rejected": -1.4969207048416138,
      "sft_loss": 0.028603632003068924,
      "step": 11740
    },
    {
      "epoch": 0.9140412290937379,
      "grad_norm": 3.618295431137085,
      "learning_rate": 9.060439827295486e-07,
      "logits/chosen": -0.34190061688423157,
      "logits/rejected": -1.1805375814437866,
      "logps/chosen": -0.8074049949645996,
      "logps/rejected": -21.932445526123047,
      "loss": 0.8075,
      "num_input_tokens_seen": 15386080,
      "odds_ratio_loss": 7.9582200050354,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08074049651622772,
      "rewards/margins": 2.112504005432129,
      "rewards/rejected": -2.193244457244873,
      "sft_loss": 0.011641708202660084,
      "step": 11750
    },
    {
      "epoch": 0.9148191365227538,
      "grad_norm": 6.058084964752197,
      "learning_rate": 8.898167434797433e-07,
      "logits/chosen": -0.13754260540008545,
      "logits/rejected": -1.4365981817245483,
      "logps/chosen": -1.115795373916626,
      "logps/rejected": -14.768132209777832,
      "loss": 1.1229,
      "num_input_tokens_seen": 15399072,
      "odds_ratio_loss": 10.982545852661133,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.11157955229282379,
      "rewards/margins": 1.3652336597442627,
      "rewards/rejected": -1.4768131971359253,
      "sft_loss": 0.024679964408278465,
      "step": 11760
    },
    {
      "epoch": 0.9155970439517698,
      "grad_norm": 2.0407490730285645,
      "learning_rate": 8.737335020371218e-07,
      "logits/chosen": -0.06862975656986237,
      "logits/rejected": -1.5248997211456299,
      "logps/chosen": -0.7661813497543335,
      "logps/rejected": -13.9795560836792,
      "loss": 0.7756,
      "num_input_tokens_seen": 15412064,
      "odds_ratio_loss": 7.698896884918213,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.07661812752485275,
      "rewards/margins": 1.3213374614715576,
      "rewards/rejected": -1.3979556560516357,
      "sft_loss": 0.005741656757891178,
      "step": 11770
    },
    {
      "epoch": 0.9163749513807857,
      "grad_norm": 4.3902974128723145,
      "learning_rate": 8.577943544586686e-07,
      "logits/chosen": -0.1258087009191513,
      "logits/rejected": -1.358526587486267,
      "logps/chosen": -0.9527864456176758,
      "logps/rejected": -10.854793548583984,
      "loss": 0.9682,
      "num_input_tokens_seen": 15420832,
      "odds_ratio_loss": 9.32841968536377,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.09527863562107086,
      "rewards/margins": 0.9902006983757019,
      "rewards/rejected": -1.0854793787002563,
      "sft_loss": 0.03540115803480148,
      "step": 11780
    },
    {
      "epoch": 0.9171528588098017,
      "grad_norm": 3.496689558029175,
      "learning_rate": 8.419993959407652e-07,
      "logits/chosen": -0.13915188610553741,
      "logits/rejected": -1.347084641456604,
      "logps/chosen": -0.7917124032974243,
      "logps/rejected": -9.26572036743164,
      "loss": 0.7964,
      "num_input_tokens_seen": 15434000,
      "odds_ratio_loss": 7.921217918395996,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.07917124032974243,
      "rewards/margins": 0.8474008440971375,
      "rewards/rejected": -0.9265719652175903,
      "sft_loss": 0.004300933331251144,
      "step": 11790
    },
    {
      "epoch": 0.9179307662388175,
      "grad_norm": 1.8610042333602905,
      "learning_rate": 8.263487208186372e-07,
      "logits/chosen": -0.3398294746875763,
      "logits/rejected": -1.3658941984176636,
      "logps/chosen": -0.7725250124931335,
      "logps/rejected": -18.21072006225586,
      "loss": 0.7813,
      "num_input_tokens_seen": 15447920,
      "odds_ratio_loss": 7.75238561630249,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.07725249975919724,
      "rewards/margins": 1.7438195943832397,
      "rewards/rejected": -1.821071982383728,
      "sft_loss": 0.0060457391664385796,
      "step": 11800
    },
    {
      "epoch": 0.9187086736678335,
      "grad_norm": 3.241182327270508,
      "learning_rate": 8.108424225657635e-07,
      "logits/chosen": -0.3613438308238983,
      "logits/rejected": -1.475386619567871,
      "logps/chosen": -0.6234281659126282,
      "logps/rejected": -14.582191467285156,
      "loss": 0.6235,
      "num_input_tokens_seen": 15458496,
      "odds_ratio_loss": 6.197271823883057,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.0623428151011467,
      "rewards/margins": 1.3958762884140015,
      "rewards/rejected": -1.458219051361084,
      "sft_loss": 0.003796472679823637,
      "step": 11810
    },
    {
      "epoch": 0.9194865810968494,
      "grad_norm": 2.7631192207336426,
      "learning_rate": 7.954805937933435e-07,
      "logits/chosen": -0.07728441059589386,
      "logits/rejected": -1.3429712057113647,
      "logps/chosen": -0.9366790652275085,
      "logps/rejected": -7.251186370849609,
      "loss": 0.9534,
      "num_input_tokens_seen": 15471504,
      "odds_ratio_loss": 9.483495712280273,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.0936679020524025,
      "rewards/margins": 0.6314507722854614,
      "rewards/rejected": -0.7251186966896057,
      "sft_loss": 0.00500117614865303,
      "step": 11820
    },
    {
      "epoch": 0.9202644885258654,
      "grad_norm": 3.3543310165405273,
      "learning_rate": 7.802633262497333e-07,
      "logits/chosen": -0.2646792531013489,
      "logits/rejected": -1.4928174018859863,
      "logps/chosen": -0.8666411638259888,
      "logps/rejected": -13.072135925292969,
      "loss": 0.8795,
      "num_input_tokens_seen": 15483184,
      "odds_ratio_loss": 8.742317199707031,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.0866641029715538,
      "rewards/margins": 1.2205493450164795,
      "rewards/rejected": -1.3072136640548706,
      "sft_loss": 0.005289946682751179,
      "step": 11830
    },
    {
      "epoch": 0.9210423959548814,
      "grad_norm": 3.0707082748413086,
      "learning_rate": 7.651907108198992e-07,
      "logits/chosen": -0.19627130031585693,
      "logits/rejected": -1.3972231149673462,
      "logps/chosen": -1.4005322456359863,
      "logps/rejected": -15.51996898651123,
      "loss": 1.4278,
      "num_input_tokens_seen": 15495376,
      "odds_ratio_loss": 14.157544136047363,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.14005321264266968,
      "rewards/margins": 1.4119436740875244,
      "rewards/rejected": -1.5519969463348389,
      "sft_loss": 0.01207679696381092,
      "step": 11840
    },
    {
      "epoch": 0.9218203033838973,
      "grad_norm": 5.255340099334717,
      "learning_rate": 7.502628375248704e-07,
      "logits/chosen": -0.3192160725593567,
      "logits/rejected": -1.200587511062622,
      "logps/chosen": -0.9118558168411255,
      "logps/rejected": -15.272189140319824,
      "loss": 0.9122,
      "num_input_tokens_seen": 15510192,
      "odds_ratio_loss": 9.04362678527832,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09118557721376419,
      "rewards/margins": 1.4360332489013672,
      "rewards/rejected": -1.5272188186645508,
      "sft_loss": 0.007811239454895258,
      "step": 11850
    },
    {
      "epoch": 0.9225982108129133,
      "grad_norm": 39.124534606933594,
      "learning_rate": 7.354797955212073e-07,
      "logits/chosen": -0.28398269414901733,
      "logits/rejected": -1.553201675415039,
      "logps/chosen": -0.9308874011039734,
      "logps/rejected": -14.798632621765137,
      "loss": 0.937,
      "num_input_tokens_seen": 15522240,
      "odds_ratio_loss": 9.252286911010742,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09308873116970062,
      "rewards/margins": 1.3867747783660889,
      "rewards/rejected": -1.4798635244369507,
      "sft_loss": 0.011812415905296803,
      "step": 11860
    },
    {
      "epoch": 0.9233761182419292,
      "grad_norm": 2.2880563735961914,
      "learning_rate": 7.208416731004724e-07,
      "logits/chosen": -0.31729552149772644,
      "logits/rejected": -1.6201820373535156,
      "logps/chosen": -0.9650070071220398,
      "logps/rejected": -13.590810775756836,
      "loss": 0.9866,
      "num_input_tokens_seen": 15537056,
      "odds_ratio_loss": 9.812111854553223,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.0965007022023201,
      "rewards/margins": 1.2625805139541626,
      "rewards/rejected": -1.3590812683105469,
      "sft_loss": 0.005394432228058577,
      "step": 11870
    },
    {
      "epoch": 0.9241540256709452,
      "grad_norm": 4.735197067260742,
      "learning_rate": 7.063485576886852e-07,
      "logits/chosen": -0.34278565645217896,
      "logits/rejected": -1.0348883867263794,
      "logps/chosen": -0.8091912269592285,
      "logps/rejected": -11.004549980163574,
      "loss": 0.8228,
      "num_input_tokens_seen": 15548384,
      "odds_ratio_loss": 8.163427352905273,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08091913163661957,
      "rewards/margins": 1.0195358991622925,
      "rewards/rejected": -1.1004550457000732,
      "sft_loss": 0.0064763561822474,
      "step": 11880
    },
    {
      "epoch": 0.9249319330999611,
      "grad_norm": 1.6394617557525635,
      "learning_rate": 6.920005358458243e-07,
      "logits/chosen": -0.3871787190437317,
      "logits/rejected": -1.1279314756393433,
      "logps/chosen": -1.104557991027832,
      "logps/rejected": -11.183547019958496,
      "loss": 1.116,
      "num_input_tokens_seen": 15562736,
      "odds_ratio_loss": 11.037538528442383,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.11045581102371216,
      "rewards/margins": 1.0078990459442139,
      "rewards/rejected": -1.1183546781539917,
      "sft_loss": 0.012201225385069847,
      "step": 11890
    },
    {
      "epoch": 0.9257098405289771,
      "grad_norm": 1.6821295022964478,
      "learning_rate": 6.777976932652946e-07,
      "logits/chosen": -0.038701239973306656,
      "logits/rejected": -2.0137062072753906,
      "logps/chosen": -1.0222748517990112,
      "logps/rejected": -14.266027450561523,
      "loss": 1.0268,
      "num_input_tokens_seen": 15578576,
      "odds_ratio_loss": 10.136931419372559,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.10222749412059784,
      "rewards/margins": 1.3243753910064697,
      "rewards/rejected": -1.4266027212142944,
      "sft_loss": 0.013138200156390667,
      "step": 11900
    },
    {
      "epoch": 0.926487747957993,
      "grad_norm": 1.7491499185562134,
      "learning_rate": 6.637401147734173e-07,
      "logits/chosen": -0.10276250541210175,
      "logits/rejected": -1.3762527704238892,
      "logps/chosen": -0.9235138893127441,
      "logps/rejected": -15.89281940460205,
      "loss": 0.9288,
      "num_input_tokens_seen": 15592032,
      "odds_ratio_loss": 9.059270858764648,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09235139191150665,
      "rewards/margins": 1.4969305992126465,
      "rewards/rejected": -1.5892821550369263,
      "sft_loss": 0.02283237874507904,
      "step": 11910
    },
    {
      "epoch": 0.9272656553870089,
      "grad_norm": 12.425597190856934,
      "learning_rate": 6.498278843289324e-07,
      "logits/chosen": -0.2992832064628601,
      "logits/rejected": -1.7471001148223877,
      "logps/chosen": -1.0092207193374634,
      "logps/rejected": -16.291868209838867,
      "loss": 1.0104,
      "num_input_tokens_seen": 15604512,
      "odds_ratio_loss": 9.880388259887695,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.10092207044363022,
      "rewards/margins": 1.5282647609710693,
      "rewards/rejected": -1.629186987876892,
      "sft_loss": 0.022376861423254013,
      "step": 11920
    },
    {
      "epoch": 0.9280435628160248,
      "grad_norm": 1.985065221786499,
      "learning_rate": 6.360610850224741e-07,
      "logits/chosen": -0.15450415015220642,
      "logits/rejected": -1.873984694480896,
      "logps/chosen": -1.154370903968811,
      "logps/rejected": -19.407909393310547,
      "loss": 1.1551,
      "num_input_tokens_seen": 15620544,
      "odds_ratio_loss": 11.513033866882324,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1154371052980423,
      "rewards/margins": 1.825353980064392,
      "rewards/rejected": -1.9407908916473389,
      "sft_loss": 0.0037880162708461285,
      "step": 11930
    },
    {
      "epoch": 0.9288214702450408,
      "grad_norm": 1.0911420583724976,
      "learning_rate": 6.224397990761105e-07,
      "logits/chosen": -0.08118706941604614,
      "logits/rejected": -1.5405961275100708,
      "logps/chosen": -0.827751636505127,
      "logps/rejected": -11.274827003479004,
      "loss": 0.8369,
      "num_input_tokens_seen": 15638288,
      "odds_ratio_loss": 8.341136932373047,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08277516067028046,
      "rewards/margins": 1.0447075366973877,
      "rewards/rejected": -1.1274826526641846,
      "sft_loss": 0.002821421716362238,
      "step": 11940
    },
    {
      "epoch": 0.9295993776740568,
      "grad_norm": 1.6755629777908325,
      "learning_rate": 6.089641078428188e-07,
      "logits/chosen": -0.1963869035243988,
      "logits/rejected": -1.457040548324585,
      "logps/chosen": -0.9162726402282715,
      "logps/rejected": -11.366378784179688,
      "loss": 0.935,
      "num_input_tokens_seen": 15650016,
      "odds_ratio_loss": 9.271358489990234,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.09162726998329163,
      "rewards/margins": 1.0450108051300049,
      "rewards/rejected": -1.1366379261016846,
      "sft_loss": 0.007887369021773338,
      "step": 11950
    },
    {
      "epoch": 0.9303772851030727,
      "grad_norm": 4.658792972564697,
      "learning_rate": 5.956340918060188e-07,
      "logits/chosen": -0.6142601370811462,
      "logits/rejected": -1.1708184480667114,
      "logps/chosen": -0.6964209079742432,
      "logps/rejected": -22.007936477661133,
      "loss": 0.7011,
      "num_input_tokens_seen": 15664480,
      "odds_ratio_loss": 6.959555625915527,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.0696420893073082,
      "rewards/margins": 2.1311516761779785,
      "rewards/rejected": -2.200793743133545,
      "sft_loss": 0.005143972113728523,
      "step": 11960
    },
    {
      "epoch": 0.9311551925320887,
      "grad_norm": 2.936549425125122,
      "learning_rate": 5.824498305790904e-07,
      "logits/chosen": -0.06384022533893585,
      "logits/rejected": -1.625780463218689,
      "logps/chosen": -0.9296265840530396,
      "logps/rejected": -14.322857856750488,
      "loss": 0.9359,
      "num_input_tokens_seen": 15678096,
      "odds_ratio_loss": 9.310887336730957,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09296265989542007,
      "rewards/margins": 1.3393231630325317,
      "rewards/rejected": -1.4322856664657593,
      "sft_loss": 0.004858965985476971,
      "step": 11970
    },
    {
      "epoch": 0.9319330999611046,
      "grad_norm": 2.577427387237549,
      "learning_rate": 5.694114029048847e-07,
      "logits/chosen": -0.2968783378601074,
      "logits/rejected": -1.3519415855407715,
      "logps/chosen": -0.7401311993598938,
      "logps/rejected": -12.539484977722168,
      "loss": 0.7642,
      "num_input_tokens_seen": 15687184,
      "odds_ratio_loss": 7.474023342132568,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.0740131139755249,
      "rewards/margins": 1.1799354553222656,
      "rewards/rejected": -1.253948450088501,
      "sft_loss": 0.016818318516016006,
      "step": 11980
    },
    {
      "epoch": 0.9327110073901206,
      "grad_norm": 1.4747344255447388,
      "learning_rate": 5.565188866552718e-07,
      "logits/chosen": -0.20611505210399628,
      "logits/rejected": -1.4000139236450195,
      "logps/chosen": -1.1651408672332764,
      "logps/rejected": -16.422460556030273,
      "loss": 1.1732,
      "num_input_tokens_seen": 15703504,
      "odds_ratio_loss": 10.402894973754883,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.11651410162448883,
      "rewards/margins": 1.5257320404052734,
      "rewards/rejected": -1.6422462463378906,
      "sft_loss": 0.13292676210403442,
      "step": 11990
    },
    {
      "epoch": 0.9334889148191365,
      "grad_norm": 3.6748807430267334,
      "learning_rate": 5.437723588306659e-07,
      "logits/chosen": -0.31006914377212524,
      "logits/rejected": -1.5102678537368774,
      "logps/chosen": -1.2047278881072998,
      "logps/rejected": -19.45448875427246,
      "loss": 1.2573,
      "num_input_tokens_seen": 15720608,
      "odds_ratio_loss": 10.378247261047363,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.12047278881072998,
      "rewards/margins": 1.8249757289886475,
      "rewards/rejected": -1.945448875427246,
      "sft_loss": 0.2194538116455078,
      "step": 12000
    },
    {
      "epoch": 0.9342668222481525,
      "grad_norm": 2.386253833770752,
      "learning_rate": 5.311718955595651e-07,
      "logits/chosen": -0.2048005610704422,
      "logits/rejected": -1.4392439126968384,
      "logps/chosen": -0.904831051826477,
      "logps/rejected": -10.53231143951416,
      "loss": 0.9168,
      "num_input_tokens_seen": 15734576,
      "odds_ratio_loss": 9.017570495605469,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.09048312157392502,
      "rewards/margins": 0.9627480506896973,
      "rewards/rejected": -1.053231120109558,
      "sft_loss": 0.015087531879544258,
      "step": 12010
    },
    {
      "epoch": 0.9350447296771685,
      "grad_norm": 1.5931123495101929,
      "learning_rate": 5.187175720980957e-07,
      "logits/chosen": -0.21956145763397217,
      "logits/rejected": -1.9383878707885742,
      "logps/chosen": -0.6608736515045166,
      "logps/rejected": -16.625524520874023,
      "loss": 0.6628,
      "num_input_tokens_seen": 15750064,
      "odds_ratio_loss": 6.601613521575928,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.06608735024929047,
      "rewards/margins": 1.5964651107788086,
      "rewards/rejected": -1.6625524759292603,
      "sft_loss": 0.0026680524460971355,
      "step": 12020
    },
    {
      "epoch": 0.9358226371061844,
      "grad_norm": 2.137845754623413,
      "learning_rate": 5.064094628295741e-07,
      "logits/chosen": -0.4625570774078369,
      "logits/rejected": -1.1971545219421387,
      "logps/chosen": -1.0364329814910889,
      "logps/rejected": -17.639053344726562,
      "loss": 1.0443,
      "num_input_tokens_seen": 15765904,
      "odds_ratio_loss": 10.371828079223633,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.10364329814910889,
      "rewards/margins": 1.660262107849121,
      "rewards/rejected": -1.7639052867889404,
      "sft_loss": 0.007115009240806103,
      "step": 12030
    },
    {
      "epoch": 0.9366005445352004,
      "grad_norm": 4.474743843078613,
      "learning_rate": 4.942476412640479e-07,
      "logits/chosen": -0.4454718232154846,
      "logits/rejected": -1.316896915435791,
      "logps/chosen": -0.9748123288154602,
      "logps/rejected": -11.002370834350586,
      "loss": 0.9946,
      "num_input_tokens_seen": 15780736,
      "odds_ratio_loss": 9.50248908996582,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.09748123586177826,
      "rewards/margins": 1.002755880355835,
      "rewards/rejected": -1.1002370119094849,
      "sft_loss": 0.04437413066625595,
      "step": 12040
    },
    {
      "epoch": 0.9373784519642162,
      "grad_norm": 1.7154042720794678,
      "learning_rate": 4.822321800378643e-07,
      "logits/chosen": -0.22523906826972961,
      "logits/rejected": -2.0052857398986816,
      "logps/chosen": -0.9130438566207886,
      "logps/rejected": -11.436153411865234,
      "loss": 0.9204,
      "num_input_tokens_seen": 15794608,
      "odds_ratio_loss": 9.132055282592773,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09130439162254333,
      "rewards/margins": 1.052310824394226,
      "rewards/rejected": -1.1436152458190918,
      "sft_loss": 0.007221040781587362,
      "step": 12050
    },
    {
      "epoch": 0.9381563593932322,
      "grad_norm": 2.577000617980957,
      "learning_rate": 4.7036315091323337e-07,
      "logits/chosen": -0.4465862810611725,
      "logits/rejected": -1.540804386138916,
      "logps/chosen": -1.045684814453125,
      "logps/rejected": -16.930381774902344,
      "loss": 1.0583,
      "num_input_tokens_seen": 15806912,
      "odds_ratio_loss": 9.347288131713867,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.1045684814453125,
      "rewards/margins": 1.5884698629379272,
      "rewards/rejected": -1.6930382251739502,
      "sft_loss": 0.12357378005981445,
      "step": 12060
    },
    {
      "epoch": 0.9389342668222481,
      "grad_norm": 311.5629577636719,
      "learning_rate": 4.586406247778008e-07,
      "logits/chosen": -0.02665615640580654,
      "logits/rejected": -1.868891716003418,
      "logps/chosen": -1.5359704494476318,
      "logps/rejected": -16.84640121459961,
      "loss": 1.5365,
      "num_input_tokens_seen": 15822048,
      "odds_ratio_loss": 15.245460510253906,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15359702706336975,
      "rewards/margins": 1.5310429334640503,
      "rewards/rejected": -1.6846399307250977,
      "sft_loss": 0.011904889717698097,
      "step": 12070
    },
    {
      "epoch": 0.9397121742512641,
      "grad_norm": 1.5435036420822144,
      "learning_rate": 4.4706467164422894e-07,
      "logits/chosen": -0.10313475131988525,
      "logits/rejected": -1.3765543699264526,
      "logps/chosen": -0.7199575901031494,
      "logps/rejected": -14.565961837768555,
      "loss": 0.7322,
      "num_input_tokens_seen": 15838416,
      "odds_ratio_loss": 7.288201332092285,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.07199576497077942,
      "rewards/margins": 1.3846005201339722,
      "rewards/rejected": -1.4565962553024292,
      "sft_loss": 0.0034093856811523438,
      "step": 12080
    },
    {
      "epoch": 0.94049008168028,
      "grad_norm": 2.868589401245117,
      "learning_rate": 4.356353606497693e-07,
      "logits/chosen": -0.037973783910274506,
      "logits/rejected": -1.7042020559310913,
      "logps/chosen": -0.9549576640129089,
      "logps/rejected": -8.380606651306152,
      "loss": 0.9799,
      "num_input_tokens_seen": 15854096,
      "odds_ratio_loss": 9.75303840637207,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.09549576044082642,
      "rewards/margins": 0.7425647974014282,
      "rewards/rejected": -0.8380606770515442,
      "sft_loss": 0.00460343761369586,
      "step": 12090
    },
    {
      "epoch": 0.941267989109296,
      "grad_norm": 4.285772323608398,
      "learning_rate": 4.2435276005586e-07,
      "logits/chosen": -0.04987701028585434,
      "logits/rejected": -1.9801149368286133,
      "logps/chosen": -0.9888293147087097,
      "logps/rejected": -16.238651275634766,
      "loss": 0.991,
      "num_input_tokens_seen": 15867280,
      "odds_ratio_loss": 9.881705284118652,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09888292849063873,
      "rewards/margins": 1.524982213973999,
      "rewards/rejected": -1.6238653659820557,
      "sft_loss": 0.0028091927524656057,
      "step": 12100
    },
    {
      "epoch": 0.9420458965383119,
      "grad_norm": 12.25771427154541,
      "learning_rate": 4.132169372477124e-07,
      "logits/chosen": -0.25258541107177734,
      "logits/rejected": -0.953656792640686,
      "logps/chosen": -0.962501049041748,
      "logps/rejected": -15.844717025756836,
      "loss": 0.9771,
      "num_input_tokens_seen": 15879712,
      "odds_ratio_loss": 9.643468856811523,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.09625010937452316,
      "rewards/margins": 1.4882216453552246,
      "rewards/rejected": -1.5844718217849731,
      "sft_loss": 0.012737992219626904,
      "step": 12110
    },
    {
      "epoch": 0.9428238039673279,
      "grad_norm": 3.2377405166625977,
      "learning_rate": 4.0222795873390283e-07,
      "logits/chosen": -0.19470559060573578,
      "logits/rejected": -1.4375954866409302,
      "logps/chosen": -0.7517939805984497,
      "logps/rejected": -12.408689498901367,
      "loss": 0.7652,
      "num_input_tokens_seen": 15891616,
      "odds_ratio_loss": 7.581107139587402,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.07517940551042557,
      "rewards/margins": 1.1656897068023682,
      "rewards/rejected": -1.240869164466858,
      "sft_loss": 0.0070818522945046425,
      "step": 12120
    },
    {
      "epoch": 0.9436017113963439,
      "grad_norm": 2.6857593059539795,
      "learning_rate": 3.9138589014598717e-07,
      "logits/chosen": -0.07612824440002441,
      "logits/rejected": -1.4703692197799683,
      "logps/chosen": -0.9121390581130981,
      "logps/rejected": -5.44936990737915,
      "loss": 0.9324,
      "num_input_tokens_seen": 15904240,
      "odds_ratio_loss": 9.259724617004395,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.0912138968706131,
      "rewards/margins": 0.45372313261032104,
      "rewards/rejected": -0.544937014579773,
      "sft_loss": 0.006381546147167683,
      "step": 12130
    },
    {
      "epoch": 0.9443796188253598,
      "grad_norm": 2.4428467750549316,
      "learning_rate": 3.806907962381007e-07,
      "logits/chosen": -0.19738474488258362,
      "logits/rejected": -1.2523488998413086,
      "logps/chosen": -0.9463440775871277,
      "logps/rejected": -10.515398025512695,
      "loss": 0.9589,
      "num_input_tokens_seen": 15915312,
      "odds_ratio_loss": 9.487909317016602,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09463441371917725,
      "rewards/margins": 0.9569055438041687,
      "rewards/rejected": -1.0515398979187012,
      "sft_loss": 0.010065768845379353,
      "step": 12140
    },
    {
      "epoch": 0.9451575262543758,
      "grad_norm": 2.169193983078003,
      "learning_rate": 3.701427408865782e-07,
      "logits/chosen": -0.24170875549316406,
      "logits/rejected": -1.6161463260650635,
      "logps/chosen": -0.7222901582717896,
      "logps/rejected": -17.738393783569336,
      "loss": 0.7232,
      "num_input_tokens_seen": 15929984,
      "odds_ratio_loss": 7.122027397155762,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.07222901284694672,
      "rewards/margins": 1.7016105651855469,
      "rewards/rejected": -1.7738393545150757,
      "sft_loss": 0.010986115783452988,
      "step": 12150
    },
    {
      "epoch": 0.9459354336833917,
      "grad_norm": 1.722773790359497,
      "learning_rate": 3.597417870895653e-07,
      "logits/chosen": -0.008847051300108433,
      "logits/rejected": -1.943717360496521,
      "logps/chosen": -0.959466814994812,
      "logps/rejected": -19.27611541748047,
      "loss": 0.9622,
      "num_input_tokens_seen": 15945424,
      "odds_ratio_loss": 9.529878616333008,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09594668447971344,
      "rewards/margins": 1.831664800643921,
      "rewards/rejected": -1.9276115894317627,
      "sft_loss": 0.009181035682559013,
      "step": 12160
    },
    {
      "epoch": 0.9467133411124076,
      "grad_norm": 1.7755272388458252,
      "learning_rate": 3.494879969666437e-07,
      "logits/chosen": -0.3166256844997406,
      "logits/rejected": -1.0808227062225342,
      "logps/chosen": -0.9705327749252319,
      "logps/rejected": -7.702474117279053,
      "loss": 0.9803,
      "num_input_tokens_seen": 15956768,
      "odds_ratio_loss": 9.704435348510742,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.09705328941345215,
      "rewards/margins": 0.6731940507888794,
      "rewards/rejected": -0.7702473998069763,
      "sft_loss": 0.009903199970722198,
      "step": 12170
    },
    {
      "epoch": 0.9474912485414235,
      "grad_norm": 2.0175669193267822,
      "learning_rate": 3.393814317584648e-07,
      "logits/chosen": -0.19483661651611328,
      "logits/rejected": -1.8273557424545288,
      "logps/chosen": -1.0359901189804077,
      "logps/rejected": -13.904330253601074,
      "loss": 1.0365,
      "num_input_tokens_seen": 15969616,
      "odds_ratio_loss": 10.292900085449219,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.10359902679920197,
      "rewards/margins": 1.2868340015411377,
      "rewards/rejected": -1.3904330730438232,
      "sft_loss": 0.00725086173042655,
      "step": 12180
    },
    {
      "epoch": 0.9482691559704395,
      "grad_norm": 3.474292039871216,
      "learning_rate": 3.294221518263779e-07,
      "logits/chosen": -0.24808935821056366,
      "logits/rejected": -1.5131680965423584,
      "logps/chosen": -0.8394571542739868,
      "logps/rejected": -24.768207550048828,
      "loss": 0.841,
      "num_input_tokens_seen": 15983360,
      "odds_ratio_loss": 8.330411911010742,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08394570648670197,
      "rewards/margins": 2.3928751945495605,
      "rewards/rejected": -2.476820945739746,
      "sft_loss": 0.007986358366906643,
      "step": 12190
    },
    {
      "epoch": 0.9490470633994554,
      "grad_norm": 2.0075721740722656,
      "learning_rate": 3.196102166520776e-07,
      "logits/chosen": -0.4564247131347656,
      "logits/rejected": -1.2702101469039917,
      "logps/chosen": -1.1077969074249268,
      "logps/rejected": -15.049591064453125,
      "loss": 1.1184,
      "num_input_tokens_seen": 15995120,
      "odds_ratio_loss": 10.883405685424805,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.11077970266342163,
      "rewards/margins": 1.394179344177246,
      "rewards/rejected": -1.5049591064453125,
      "sft_loss": 0.030080148950219154,
      "step": 12200
    },
    {
      "epoch": 0.9498249708284714,
      "grad_norm": 4.91524076461792,
      "learning_rate": 3.0994568483723453e-07,
      "logits/chosen": -0.18206897377967834,
      "logits/rejected": -1.224856972694397,
      "logps/chosen": -0.7585890889167786,
      "logps/rejected": -9.87165641784668,
      "loss": 0.7673,
      "num_input_tokens_seen": 16005920,
      "odds_ratio_loss": 7.561065673828125,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.07585891336202621,
      "rewards/margins": 0.9113066792488098,
      "rewards/rejected": -0.987165629863739,
      "sft_loss": 0.011143623851239681,
      "step": 12210
    },
    {
      "epoch": 0.9506028782574873,
      "grad_norm": 156.9337615966797,
      "learning_rate": 3.004286141031626e-07,
      "logits/chosen": -0.17164663970470428,
      "logits/rejected": -1.5703425407409668,
      "logps/chosen": -0.9372615814208984,
      "logps/rejected": -18.172954559326172,
      "loss": 0.9459,
      "num_input_tokens_seen": 16019360,
      "odds_ratio_loss": 9.393759727478027,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.09372617304325104,
      "rewards/margins": 1.7235692739486694,
      "rewards/rejected": -1.8172953128814697,
      "sft_loss": 0.006557709537446499,
      "step": 12220
    },
    {
      "epoch": 0.9513807856865033,
      "grad_norm": 6.203927993774414,
      "learning_rate": 2.910590612904635e-07,
      "logits/chosen": -0.4827880859375,
      "logits/rejected": -0.7654441595077515,
      "logps/chosen": -0.8061830401420593,
      "logps/rejected": -6.185842037200928,
      "loss": 0.834,
      "num_input_tokens_seen": 16029104,
      "odds_ratio_loss": 8.147764205932617,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.08061830699443817,
      "rewards/margins": 0.5379658937454224,
      "rewards/rejected": -0.6185842156410217,
      "sft_loss": 0.01920284889638424,
      "step": 12230
    },
    {
      "epoch": 0.9521586931155193,
      "grad_norm": 3.568031072616577,
      "learning_rate": 2.81837082358688e-07,
      "logits/chosen": -0.18624334037303925,
      "logits/rejected": -1.3989797830581665,
      "logps/chosen": -0.9238718748092651,
      "logps/rejected": -12.488553047180176,
      "loss": 0.9304,
      "num_input_tokens_seen": 16043568,
      "odds_ratio_loss": 9.237310409545898,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09238718450069427,
      "rewards/margins": 1.156468152999878,
      "rewards/rejected": -1.2488553524017334,
      "sft_loss": 0.0066881850361824036,
      "step": 12240
    },
    {
      "epoch": 0.9529366005445352,
      "grad_norm": 1.8548458814620972,
      "learning_rate": 2.72762732386006e-07,
      "logits/chosen": -0.2077573537826538,
      "logits/rejected": -1.9637689590454102,
      "logps/chosen": -0.9647920727729797,
      "logps/rejected": -13.010641098022461,
      "loss": 0.9775,
      "num_input_tokens_seen": 16057312,
      "odds_ratio_loss": 9.729558944702148,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.09647920727729797,
      "rewards/margins": 1.2045848369598389,
      "rewards/rejected": -1.3010642528533936,
      "sft_loss": 0.004576006438583136,
      "step": 12250
    },
    {
      "epoch": 0.9537145079735512,
      "grad_norm": 4.205636501312256,
      "learning_rate": 2.63836065568876e-07,
      "logits/chosen": -0.6094192266464233,
      "logits/rejected": -1.2007038593292236,
      "logps/chosen": -0.7853139042854309,
      "logps/rejected": -14.346330642700195,
      "loss": 0.8093,
      "num_input_tokens_seen": 16068064,
      "odds_ratio_loss": 7.98935079574585,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.07853139191865921,
      "rewards/margins": 1.356101632118225,
      "rewards/rejected": -1.4346331357955933,
      "sft_loss": 0.01034087035804987,
      "step": 12260
    },
    {
      "epoch": 0.9544924154025671,
      "grad_norm": 3.066114902496338,
      "learning_rate": 2.5505713522171457e-07,
      "logits/chosen": -0.52752685546875,
      "logits/rejected": -1.322624921798706,
      "logps/chosen": -1.0002782344818115,
      "logps/rejected": -16.155460357666016,
      "loss": 1.0088,
      "num_input_tokens_seen": 16080032,
      "odds_ratio_loss": 9.885408401489258,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.10002782195806503,
      "rewards/margins": 1.515518307685852,
      "rewards/rejected": -1.6155462265014648,
      "sft_loss": 0.02022591605782509,
      "step": 12270
    },
    {
      "epoch": 0.9552703228315831,
      "grad_norm": 9.273058891296387,
      "learning_rate": 2.464259937765889e-07,
      "logits/chosen": -0.6523706316947937,
      "logits/rejected": -1.2843421697616577,
      "logps/chosen": -0.9229806065559387,
      "logps/rejected": -22.3568172454834,
      "loss": 0.9232,
      "num_input_tokens_seen": 16090400,
      "odds_ratio_loss": 9.077737808227539,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09229806065559387,
      "rewards/margins": 2.143383502960205,
      "rewards/rejected": -2.2356815338134766,
      "sft_loss": 0.015461449511349201,
      "step": 12280
    },
    {
      "epoch": 0.956048230260599,
      "grad_norm": 17.13327980041504,
      "learning_rate": 2.3794269278289982e-07,
      "logits/chosen": -0.6235595345497131,
      "logits/rejected": -0.8743928670883179,
      "logps/chosen": -0.841866135597229,
      "logps/rejected": -14.01153564453125,
      "loss": 0.8555,
      "num_input_tokens_seen": 16101152,
      "odds_ratio_loss": 8.061131477355957,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.0841866135597229,
      "rewards/margins": 1.3169668912887573,
      "rewards/rejected": -1.4011536836624146,
      "sft_loss": 0.04942449554800987,
      "step": 12290
    },
    {
      "epoch": 0.9568261376896149,
      "grad_norm": 23.418697357177734,
      "learning_rate": 2.2960728290706836e-07,
      "logits/chosen": -0.31731849908828735,
      "logits/rejected": -1.3659123182296753,
      "logps/chosen": -1.1973960399627686,
      "logps/rejected": -12.80005168914795,
      "loss": 1.2045,
      "num_input_tokens_seen": 16114400,
      "odds_ratio_loss": 11.009976387023926,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.11973961442708969,
      "rewards/margins": 1.160265564918518,
      "rewards/rejected": -1.2800052165985107,
      "sft_loss": 0.10345391929149628,
      "step": 12300
    },
    {
      "epoch": 0.9576040451186308,
      "grad_norm": 8.194009780883789,
      "learning_rate": 2.2141981393223876e-07,
      "logits/chosen": -0.21181325614452362,
      "logits/rejected": -1.5768486261367798,
      "logps/chosen": -1.3449286222457886,
      "logps/rejected": -16.770870208740234,
      "loss": 1.3834,
      "num_input_tokens_seen": 16124784,
      "odds_ratio_loss": 11.494306564331055,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.1344928741455078,
      "rewards/margins": 1.5425941944122314,
      "rewards/rejected": -1.6770871877670288,
      "sft_loss": 0.23394536972045898,
      "step": 12310
    },
    {
      "epoch": 0.9583819525476468,
      "grad_norm": 4.417098522186279,
      "learning_rate": 2.1338033475798158e-07,
      "logits/chosen": -0.12259181588888168,
      "logits/rejected": -1.6844972372055054,
      "logps/chosen": -1.0650262832641602,
      "logps/rejected": -13.395116806030273,
      "loss": 1.0797,
      "num_input_tokens_seen": 16137648,
      "odds_ratio_loss": 10.725841522216797,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.10650262981653214,
      "rewards/margins": 1.2330090999603271,
      "rewards/rejected": -1.3395118713378906,
      "sft_loss": 0.007141365669667721,
      "step": 12320
    },
    {
      "epoch": 0.9591598599766628,
      "grad_norm": 1.7791497707366943,
      "learning_rate": 2.0548889339999654e-07,
      "logits/chosen": -0.3604108393192291,
      "logits/rejected": -1.5325709581375122,
      "logps/chosen": -0.8818775415420532,
      "logps/rejected": -20.89768409729004,
      "loss": 0.8836,
      "num_input_tokens_seen": 16150128,
      "odds_ratio_loss": 7.5479888916015625,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08818776160478592,
      "rewards/margins": 2.0015807151794434,
      "rewards/rejected": -2.089768171310425,
      "sft_loss": 0.12881183624267578,
      "step": 12330
    },
    {
      "epoch": 0.9599377674056787,
      "grad_norm": 6.587460517883301,
      "learning_rate": 1.9774553698983512e-07,
      "logits/chosen": -0.38189247250556946,
      "logits/rejected": -1.1007959842681885,
      "logps/chosen": -0.7522581219673157,
      "logps/rejected": -6.890204429626465,
      "loss": 0.7582,
      "num_input_tokens_seen": 16160560,
      "odds_ratio_loss": 7.452940464019775,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.0752258226275444,
      "rewards/margins": 0.6137946844100952,
      "rewards/rejected": -0.6890205144882202,
      "sft_loss": 0.01288747787475586,
      "step": 12340
    },
    {
      "epoch": 0.9607156748346947,
      "grad_norm": 5.235318183898926,
      "learning_rate": 1.9015031177460619e-07,
      "logits/chosen": -0.1968858242034912,
      "logits/rejected": -1.5090630054473877,
      "logps/chosen": -0.9505993127822876,
      "logps/rejected": -14.273775100708008,
      "loss": 0.9609,
      "num_input_tokens_seen": 16175728,
      "odds_ratio_loss": 9.44192886352539,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09505993127822876,
      "rewards/margins": 1.3323174715042114,
      "rewards/rejected": -1.4273775815963745,
      "sft_loss": 0.01673353649675846,
      "step": 12350
    },
    {
      "epoch": 0.9614935822637106,
      "grad_norm": 6.162037372589111,
      "learning_rate": 1.827032631167097e-07,
      "logits/chosen": -0.5855585336685181,
      "logits/rejected": -1.5549615621566772,
      "logps/chosen": -0.9294301867485046,
      "logps/rejected": -9.373213768005371,
      "loss": 0.9568,
      "num_input_tokens_seen": 16187920,
      "odds_ratio_loss": 8.704675674438477,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.09294302016496658,
      "rewards/margins": 0.8443782925605774,
      "rewards/rejected": -0.9373214840888977,
      "sft_loss": 0.08634259551763535,
      "step": 12360
    },
    {
      "epoch": 0.9622714896927266,
      "grad_norm": 5.875050067901611,
      "learning_rate": 1.7540443549356188e-07,
      "logits/chosen": -0.6409750580787659,
      "logits/rejected": -0.7209451794624329,
      "logps/chosen": -0.8512948751449585,
      "logps/rejected": -20.480974197387695,
      "loss": 0.859,
      "num_input_tokens_seen": 16201888,
      "odds_ratio_loss": 8.250844955444336,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.0851295068860054,
      "rewards/margins": 1.962967872619629,
      "rewards/rejected": -2.0480971336364746,
      "sft_loss": 0.03389885649085045,
      "step": 12370
    },
    {
      "epoch": 0.9630493971217425,
      "grad_norm": 7.472117900848389,
      "learning_rate": 1.6825387249733704e-07,
      "logits/chosen": -0.42525428533554077,
      "logits/rejected": -1.3136355876922607,
      "logps/chosen": -0.6914771795272827,
      "logps/rejected": -8.531336784362793,
      "loss": 0.7037,
      "num_input_tokens_seen": 16214576,
      "odds_ratio_loss": 6.910098075866699,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.06914772093296051,
      "rewards/margins": 0.7839860320091248,
      "rewards/rejected": -0.8531337976455688,
      "sft_loss": 0.012650936841964722,
      "step": 12380
    },
    {
      "epoch": 0.9638273045507585,
      "grad_norm": 3.758302927017212,
      "learning_rate": 1.6125161683468726e-07,
      "logits/chosen": -0.42221713066101074,
      "logits/rejected": -1.5183813571929932,
      "logps/chosen": -1.3994024991989136,
      "logps/rejected": -12.888708114624023,
      "loss": 1.4037,
      "num_input_tokens_seen": 16225616,
      "odds_ratio_loss": 13.958964347839355,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13994024693965912,
      "rewards/margins": 1.148930549621582,
      "rewards/rejected": -1.288870930671692,
      "sft_loss": 0.007841033861041069,
      "step": 12390
    },
    {
      "epoch": 0.9646052119797744,
      "grad_norm": 13.455365180969238,
      "learning_rate": 1.5439771032651485e-07,
      "logits/chosen": -0.23251929879188538,
      "logits/rejected": -1.6611559391021729,
      "logps/chosen": -0.786699652671814,
      "logps/rejected": -14.888818740844727,
      "loss": 0.8089,
      "num_input_tokens_seen": 16238272,
      "odds_ratio_loss": 7.606795310974121,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.07866998016834259,
      "rewards/margins": 1.4102119207382202,
      "rewards/rejected": -1.4888819456100464,
      "sft_loss": 0.048230789601802826,
      "step": 12400
    },
    {
      "epoch": 0.9653831194087904,
      "grad_norm": 8.305849075317383,
      "learning_rate": 1.476921939076975e-07,
      "logits/chosen": -0.09227391332387924,
      "logits/rejected": -1.1034928560256958,
      "logps/chosen": -0.9128230214118958,
      "logps/rejected": -7.003422737121582,
      "loss": 0.9533,
      "num_input_tokens_seen": 16249104,
      "odds_ratio_loss": 8.971957206726074,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.09128230065107346,
      "rewards/margins": 0.6090599298477173,
      "rewards/rejected": -0.7003422975540161,
      "sft_loss": 0.0561177060008049,
      "step": 12410
    },
    {
      "epoch": 0.9661610268378062,
      "grad_norm": 0.31519854068756104,
      "learning_rate": 1.41135107626858e-07,
      "logits/chosen": -0.36162978410720825,
      "logits/rejected": -0.9066141843795776,
      "logps/chosen": -0.8489063382148743,
      "logps/rejected": -8.324593544006348,
      "loss": 0.8506,
      "num_input_tokens_seen": 16259392,
      "odds_ratio_loss": 8.225701332092285,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08489063382148743,
      "rewards/margins": 0.7475687265396118,
      "rewards/rejected": -0.8324593305587769,
      "sft_loss": 0.028013313189148903,
      "step": 12420
    },
    {
      "epoch": 0.9669389342668222,
      "grad_norm": 21.114967346191406,
      "learning_rate": 1.3472649064612818e-07,
      "logits/chosen": -0.39116179943084717,
      "logits/rejected": -1.32956862449646,
      "logps/chosen": -0.8328605890274048,
      "logps/rejected": -11.167863845825195,
      "loss": 0.8505,
      "num_input_tokens_seen": 16270256,
      "odds_ratio_loss": 8.265515327453613,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.08328606188297272,
      "rewards/margins": 1.03350031375885,
      "rewards/rejected": -1.116786241531372,
      "sft_loss": 0.02390402927994728,
      "step": 12430
    },
    {
      "epoch": 0.9677168416958382,
      "grad_norm": 2.498950481414795,
      "learning_rate": 1.2846638124089928e-07,
      "logits/chosen": -0.25835880637168884,
      "logits/rejected": -1.3009707927703857,
      "logps/chosen": -0.7393811345100403,
      "logps/rejected": -14.841459274291992,
      "loss": 0.7486,
      "num_input_tokens_seen": 16280016,
      "odds_ratio_loss": 7.436209678649902,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.07393811643123627,
      "rewards/margins": 1.410207986831665,
      "rewards/rejected": -1.484145998954773,
      "sft_loss": 0.00495152547955513,
      "step": 12440
    },
    {
      "epoch": 0.9684947491248541,
      "grad_norm": 43.50111770629883,
      "learning_rate": 1.2235481679960803e-07,
      "logits/chosen": -0.0818486213684082,
      "logits/rejected": -1.5897998809814453,
      "logps/chosen": -0.9484689831733704,
      "logps/rejected": -16.3314208984375,
      "loss": 0.9589,
      "num_input_tokens_seen": 16295184,
      "odds_ratio_loss": 9.431092262268066,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09484690427780151,
      "rewards/margins": 1.538295030593872,
      "rewards/rejected": -1.633142113685608,
      "sft_loss": 0.015782522037625313,
      "step": 12450
    },
    {
      "epoch": 0.9692726565538701,
      "grad_norm": 4.413215637207031,
      "learning_rate": 1.1639183382350649e-07,
      "logits/chosen": -0.6921803951263428,
      "logits/rejected": -0.9575518369674683,
      "logps/chosen": -1.122647762298584,
      "logps/rejected": -12.000131607055664,
      "loss": 1.1484,
      "num_input_tokens_seen": 16307840,
      "odds_ratio_loss": 10.92697811126709,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.11226477473974228,
      "rewards/margins": 1.0877482891082764,
      "rewards/rejected": -1.2000130414962769,
      "sft_loss": 0.05570362135767937,
      "step": 12460
    },
    {
      "epoch": 0.970050563982886,
      "grad_norm": 9.900873184204102,
      "learning_rate": 1.1057746792644541e-07,
      "logits/chosen": -0.11293037235736847,
      "logits/rejected": -1.5653810501098633,
      "logps/chosen": -0.8573525547981262,
      "logps/rejected": -10.341009140014648,
      "loss": 0.8724,
      "num_input_tokens_seen": 16319312,
      "odds_ratio_loss": 8.643301963806152,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.0857352465391159,
      "rewards/margins": 0.9483655691146851,
      "rewards/rejected": -1.0341007709503174,
      "sft_loss": 0.0080681461840868,
      "step": 12470
    },
    {
      "epoch": 0.970828471411902,
      "grad_norm": 2.6572036743164062,
      "learning_rate": 1.0491175383466056e-07,
      "logits/chosen": -0.30273956060409546,
      "logits/rejected": -1.4701600074768066,
      "logps/chosen": -0.9312847852706909,
      "logps/rejected": -18.610532760620117,
      "loss": 0.9361,
      "num_input_tokens_seen": 16333312,
      "odds_ratio_loss": 9.294637680053711,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.0931284949183464,
      "rewards/margins": 1.7679250240325928,
      "rewards/rejected": -1.861053228378296,
      "sft_loss": 0.006683858577162027,
      "step": 12480
    },
    {
      "epoch": 0.9716063788409179,
      "grad_norm": 1.9054896831512451,
      "learning_rate": 9.939472538657013e-08,
      "logits/chosen": -0.9018571972846985,
      "logits/rejected": -0.8955874443054199,
      "logps/chosen": -0.6079337000846863,
      "logps/rejected": -12.151650428771973,
      "loss": 0.6204,
      "num_input_tokens_seen": 16346000,
      "odds_ratio_loss": 6.025877952575684,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.060793377459049225,
      "rewards/margins": 1.1543715000152588,
      "rewards/rejected": -1.2151648998260498,
      "sft_loss": 0.017820732668042183,
      "step": 12490
    },
    {
      "epoch": 0.9723842862699339,
      "grad_norm": 5.731510639190674,
      "learning_rate": 9.402641553256375e-08,
      "logits/chosen": -0.10272973775863647,
      "logits/rejected": -1.1582386493682861,
      "logps/chosen": -0.7395635843276978,
      "logps/rejected": -10.743902206420898,
      "loss": 0.7431,
      "num_input_tokens_seen": 16354496,
      "odds_ratio_loss": 7.344037055969238,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.07395635545253754,
      "rewards/margins": 1.0004339218139648,
      "rewards/rejected": -1.0743902921676636,
      "sft_loss": 0.008720135316252708,
      "step": 12500
    },
    {
      "epoch": 0.9731621936989499,
      "grad_norm": 32.50389099121094,
      "learning_rate": 8.880685633481656e-08,
      "logits/chosen": -0.18928098678588867,
      "logits/rejected": -1.7569280862808228,
      "logps/chosen": -0.8919623494148254,
      "logps/rejected": -20.336193084716797,
      "loss": 0.895,
      "num_input_tokens_seen": 16367104,
      "odds_ratio_loss": 8.890974044799805,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08919622749090195,
      "rewards/margins": 1.9444230794906616,
      "rewards/rejected": -2.0336194038391113,
      "sft_loss": 0.0058658611960709095,
      "step": 12510
    },
    {
      "epoch": 0.9739401011279658,
      "grad_norm": 2.3467681407928467,
      "learning_rate": 8.373607896708657e-08,
      "logits/chosen": -0.04171920567750931,
      "logits/rejected": -1.8098809719085693,
      "logps/chosen": -0.8330823183059692,
      "logps/rejected": -14.04285717010498,
      "loss": 0.8349,
      "num_input_tokens_seen": 16378816,
      "odds_ratio_loss": 8.255022048950195,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08330823481082916,
      "rewards/margins": 1.3209774494171143,
      "rewards/rejected": -1.4042856693267822,
      "sft_loss": 0.009358550421893597,
      "step": 12520
    },
    {
      "epoch": 0.9747180085569818,
      "grad_norm": 1.3713639974594116,
      "learning_rate": 7.881411371453707e-08,
      "logits/chosen": -0.28759709000587463,
      "logits/rejected": -1.3889890909194946,
      "logps/chosen": -0.9286860227584839,
      "logps/rejected": -7.614617347717285,
      "loss": 0.9356,
      "num_input_tokens_seen": 16390256,
      "odds_ratio_loss": 9.244423866271973,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.0928686186671257,
      "rewards/margins": 0.668593168258667,
      "rewards/rejected": -0.7614617347717285,
      "sft_loss": 0.01112511195242405,
      "step": 12530
    },
    {
      "epoch": 0.9754959159859977,
      "grad_norm": 4.426767826080322,
      "learning_rate": 7.404098997355335e-08,
      "logits/chosen": -0.35824424028396606,
      "logits/rejected": -1.33821439743042,
      "logps/chosen": -0.8309882879257202,
      "logps/rejected": -18.37192153930664,
      "loss": 0.8335,
      "num_input_tokens_seen": 16403888,
      "odds_ratio_loss": 8.10359001159668,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08309882879257202,
      "rewards/margins": 1.7540934085845947,
      "rewards/rejected": -1.8371922969818115,
      "sft_loss": 0.023182952776551247,
      "step": 12540
    },
    {
      "epoch": 0.9762738234150136,
      "grad_norm": 313.7215576171875,
      "learning_rate": 6.94167362515652e-08,
      "logits/chosen": -0.6350809335708618,
      "logits/rejected": -1.0927356481552124,
      "logps/chosen": -1.0203081369400024,
      "logps/rejected": -13.360295295715332,
      "loss": 1.0235,
      "num_input_tokens_seen": 16419856,
      "odds_ratio_loss": 9.871319770812988,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.10203079879283905,
      "rewards/margins": 1.2339987754821777,
      "rewards/rejected": -1.3360296487808228,
      "sft_loss": 0.03635597229003906,
      "step": 12550
    },
    {
      "epoch": 0.9770517308440295,
      "grad_norm": 1.7516701221466064,
      "learning_rate": 6.494138016687468e-08,
      "logits/chosen": 0.02967851236462593,
      "logits/rejected": -1.5164343118667603,
      "logps/chosen": -0.6879326105117798,
      "logps/rejected": -13.3673677444458,
      "loss": 0.6919,
      "num_input_tokens_seen": 16438128,
      "odds_ratio_loss": 6.892519474029541,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.06879327446222305,
      "rewards/margins": 1.2679435014724731,
      "rewards/rejected": -1.3367364406585693,
      "sft_loss": 0.00268021272495389,
      "step": 12560
    },
    {
      "epoch": 0.9778296382730455,
      "grad_norm": 10.79580307006836,
      "learning_rate": 6.06149484485008e-08,
      "logits/chosen": -0.2581590712070465,
      "logits/rejected": -1.425004482269287,
      "logps/chosen": -0.7572137117385864,
      "logps/rejected": -18.632070541381836,
      "loss": 0.7878,
      "num_input_tokens_seen": 16449392,
      "odds_ratio_loss": 7.764173984527588,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.0757213681936264,
      "rewards/margins": 1.7874854803085327,
      "rewards/rejected": -1.8632068634033203,
      "sft_loss": 0.011387160047888756,
      "step": 12570
    },
    {
      "epoch": 0.9786075457020614,
      "grad_norm": 1.9176056385040283,
      "learning_rate": 5.643746693600738e-08,
      "logits/chosen": -0.3222039043903351,
      "logits/rejected": -1.4580246210098267,
      "logps/chosen": -1.1318254470825195,
      "logps/rejected": -13.271893501281738,
      "loss": 1.1372,
      "num_input_tokens_seen": 16460176,
      "odds_ratio_loss": 11.175423622131348,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.11318254470825195,
      "rewards/margins": 1.214006781578064,
      "rewards/rejected": -1.327189326286316,
      "sft_loss": 0.019621601328253746,
      "step": 12580
    },
    {
      "epoch": 0.9793854531310774,
      "grad_norm": 3.187330961227417,
      "learning_rate": 5.240896057935596e-08,
      "logits/chosen": -0.09735438972711563,
      "logits/rejected": -1.3586671352386475,
      "logps/chosen": -0.9365214109420776,
      "logps/rejected": -11.869119644165039,
      "loss": 0.9705,
      "num_input_tokens_seen": 16474832,
      "odds_ratio_loss": 9.631131172180176,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.09365214407444,
      "rewards/margins": 1.0932599306106567,
      "rewards/rejected": -1.186911940574646,
      "sft_loss": 0.007435616105794907,
      "step": 12590
    },
    {
      "epoch": 0.9801633605600933,
      "grad_norm": 1.5250585079193115,
      "learning_rate": 4.852945343876147e-08,
      "logits/chosen": -0.2481372356414795,
      "logits/rejected": -1.2218652963638306,
      "logps/chosen": -1.1172990798950195,
      "logps/rejected": -14.136247634887695,
      "loss": 1.1277,
      "num_input_tokens_seen": 16487808,
      "odds_ratio_loss": 10.643720626831055,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.11172991991043091,
      "rewards/margins": 1.3018949031829834,
      "rewards/rejected": -1.413624882698059,
      "sft_loss": 0.06337771564722061,
      "step": 12600
    },
    {
      "epoch": 0.9809412679891093,
      "grad_norm": 5.009439945220947,
      "learning_rate": 4.4798968684536816e-08,
      "logits/chosen": -0.12432391941547394,
      "logits/rejected": -1.9330546855926514,
      "logps/chosen": -1.0320632457733154,
      "logps/rejected": -15.102289199829102,
      "loss": 1.0389,
      "num_input_tokens_seen": 16500688,
      "odds_ratio_loss": 10.34919261932373,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1032063215970993,
      "rewards/margins": 1.407022476196289,
      "rewards/rejected": -1.5102288722991943,
      "sft_loss": 0.003973107319325209,
      "step": 12610
    },
    {
      "epoch": 0.9817191754181253,
      "grad_norm": 1.481100082397461,
      "learning_rate": 4.12175285969596e-08,
      "logits/chosen": -0.18140652775764465,
      "logits/rejected": -1.433404564857483,
      "logps/chosen": -0.9879193305969238,
      "logps/rejected": -12.899325370788574,
      "loss": 1.0095,
      "num_input_tokens_seen": 16513120,
      "odds_ratio_loss": 9.938295364379883,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.0987919345498085,
      "rewards/margins": 1.1911407709121704,
      "rewards/rejected": -1.2899327278137207,
      "sft_loss": 0.015678998082876205,
      "step": 12620
    },
    {
      "epoch": 0.9824970828471412,
      "grad_norm": 3.2705838680267334,
      "learning_rate": 3.7785154566141734e-08,
      "logits/chosen": -0.16677150130271912,
      "logits/rejected": -2.082714796066284,
      "logps/chosen": -0.9549390077590942,
      "logps/rejected": -22.509668350219727,
      "loss": 0.955,
      "num_input_tokens_seen": 16529152,
      "odds_ratio_loss": 9.085559844970703,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09549389779567719,
      "rewards/margins": 2.155473232269287,
      "rewards/rejected": -2.250967025756836,
      "sft_loss": 0.046428509056568146,
      "step": 12630
    },
    {
      "epoch": 0.9832749902761572,
      "grad_norm": 1.5827522277832031,
      "learning_rate": 3.450186709190173e-08,
      "logits/chosen": 0.0006435498362407088,
      "logits/rejected": -1.7825992107391357,
      "logps/chosen": -1.3783544301986694,
      "logps/rejected": -12.360357284545898,
      "loss": 1.4183,
      "num_input_tokens_seen": 16545968,
      "odds_ratio_loss": 14.124194145202637,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.13783542811870575,
      "rewards/margins": 1.0982002019882202,
      "rewards/rejected": -1.236035704612732,
      "sft_loss": 0.005850933492183685,
      "step": 12640
    },
    {
      "epoch": 0.9840528977051731,
      "grad_norm": 2.583888053894043,
      "learning_rate": 3.1367685783637027e-08,
      "logits/chosen": -0.08986677974462509,
      "logits/rejected": -1.441861629486084,
      "logps/chosen": -0.8815041780471802,
      "logps/rejected": -8.078205108642578,
      "loss": 0.8927,
      "num_input_tokens_seen": 16559760,
      "odds_ratio_loss": 8.863988876342773,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08815042674541473,
      "rewards/margins": 0.7196700572967529,
      "rewards/rejected": -0.8078204393386841,
      "sft_loss": 0.006350231356918812,
      "step": 12650
    },
    {
      "epoch": 0.9848308051341891,
      "grad_norm": 3.0049779415130615,
      "learning_rate": 2.8382629360212964e-08,
      "logits/chosen": -0.47108444571495056,
      "logits/rejected": -1.0828173160552979,
      "logps/chosen": -0.7092539072036743,
      "logps/rejected": -14.784589767456055,
      "loss": 0.7103,
      "num_input_tokens_seen": 16573264,
      "odds_ratio_loss": 6.922545433044434,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.07092537730932236,
      "rewards/margins": 1.4075334072113037,
      "rewards/rejected": -1.4784588813781738,
      "sft_loss": 0.01802605763077736,
      "step": 12660
    },
    {
      "epoch": 0.9856087125632049,
      "grad_norm": 3.0464465618133545,
      "learning_rate": 2.5546715649843454e-08,
      "logits/chosen": -0.06019148975610733,
      "logits/rejected": -1.9300884008407593,
      "logps/chosen": -0.8260527849197388,
      "logps/rejected": -6.8024725914001465,
      "loss": 0.8865,
      "num_input_tokens_seen": 16584240,
      "odds_ratio_loss": 8.806262016296387,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.0826052725315094,
      "rewards/margins": 0.5976420044898987,
      "rewards/rejected": -0.6802473664283752,
      "sft_loss": 0.00587079580873251,
      "step": 12670
    },
    {
      "epoch": 0.9863866199922209,
      "grad_norm": 2.0222275257110596,
      "learning_rate": 2.2859961589996592e-08,
      "logits/chosen": -0.03779103606939316,
      "logits/rejected": -1.8276106119155884,
      "logps/chosen": -0.968870997428894,
      "logps/rejected": -21.92473030090332,
      "loss": 0.9782,
      "num_input_tokens_seen": 16599776,
      "odds_ratio_loss": 9.445577621459961,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.09688709676265717,
      "rewards/margins": 2.095586061477661,
      "rewards/rejected": -2.1924729347229004,
      "sft_loss": 0.03368552029132843,
      "step": 12680
    },
    {
      "epoch": 0.9871645274212368,
      "grad_norm": 2.786588430404663,
      "learning_rate": 2.0322383227278085e-08,
      "logits/chosen": -0.07717380672693253,
      "logits/rejected": -1.7762495279312134,
      "logps/chosen": -0.8848506808280945,
      "logps/rejected": -15.007204055786133,
      "loss": 0.8853,
      "num_input_tokens_seen": 16613616,
      "odds_ratio_loss": 8.807829856872559,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08848506212234497,
      "rewards/margins": 1.4122353792190552,
      "rewards/rejected": -1.5007203817367554,
      "sft_loss": 0.004489428363740444,
      "step": 12690
    },
    {
      "epoch": 0.9879424348502528,
      "grad_norm": 1.800603985786438,
      "learning_rate": 1.7933995717347996e-08,
      "logits/chosen": -0.2031760960817337,
      "logits/rejected": -1.3554677963256836,
      "logps/chosen": -0.7762188911437988,
      "logps/rejected": -14.93109130859375,
      "loss": 0.7783,
      "num_input_tokens_seen": 16624656,
      "odds_ratio_loss": 7.481186866760254,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.07762189209461212,
      "rewards/margins": 1.4154871702194214,
      "rewards/rejected": -1.4931089878082275,
      "sft_loss": 0.03014567494392395,
      "step": 12700
    },
    {
      "epoch": 0.9887203422792687,
      "grad_norm": 3.7643752098083496,
      "learning_rate": 1.569481332482636e-08,
      "logits/chosen": 0.05465736240148544,
      "logits/rejected": -1.5168482065200806,
      "logps/chosen": -0.9140115976333618,
      "logps/rejected": -10.084342956542969,
      "loss": 0.9228,
      "num_input_tokens_seen": 16641136,
      "odds_ratio_loss": 9.198801040649414,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09140116721391678,
      "rewards/margins": 0.917033314704895,
      "rewards/rejected": -1.0084344148635864,
      "sft_loss": 0.002949314657598734,
      "step": 12710
    },
    {
      "epoch": 0.9894982497082847,
      "grad_norm": 3.4115965366363525,
      "learning_rate": 1.3604849423201616e-08,
      "logits/chosen": -0.2617233991622925,
      "logits/rejected": -1.460564136505127,
      "logps/chosen": -0.8608863949775696,
      "logps/rejected": -11.34762191772461,
      "loss": 0.8719,
      "num_input_tokens_seen": 16655472,
      "odds_ratio_loss": 8.64155387878418,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.0860886350274086,
      "rewards/margins": 1.0486735105514526,
      "rewards/rejected": -1.1347620487213135,
      "sft_loss": 0.007763334549963474,
      "step": 12720
    },
    {
      "epoch": 0.9902761571373007,
      "grad_norm": 2.1883881092071533,
      "learning_rate": 1.166411649476673e-08,
      "logits/chosen": -0.4011611044406891,
      "logits/rejected": -0.961927592754364,
      "logps/chosen": -0.7738547921180725,
      "logps/rejected": -7.4190521240234375,
      "loss": 0.8047,
      "num_input_tokens_seen": 16666992,
      "odds_ratio_loss": 7.953970432281494,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.07738548517227173,
      "rewards/margins": 0.664519727230072,
      "rewards/rejected": -0.7419052124023438,
      "sft_loss": 0.009307285770773888,
      "step": 12730
    },
    {
      "epoch": 0.9910540645663166,
      "grad_norm": 4.376555919647217,
      "learning_rate": 9.87262613052209e-09,
      "logits/chosen": -0.19814147055149078,
      "logits/rejected": -1.5912666320800781,
      "logps/chosen": -0.7829992175102234,
      "logps/rejected": -14.966654777526855,
      "loss": 0.785,
      "num_input_tokens_seen": 16684736,
      "odds_ratio_loss": 7.817573547363281,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.07829991728067398,
      "rewards/margins": 1.4183655977249146,
      "rewards/rejected": -1.496665596961975,
      "sft_loss": 0.003252812195569277,
      "step": 12740
    },
    {
      "epoch": 0.9918319719953326,
      "grad_norm": 2.65453839302063,
      "learning_rate": 8.230389030128293e-09,
      "logits/chosen": -0.28969889879226685,
      "logits/rejected": -1.453749179840088,
      "logps/chosen": -0.9020107388496399,
      "logps/rejected": -15.950787544250488,
      "loss": 0.9037,
      "num_input_tokens_seen": 16701296,
      "odds_ratio_loss": 8.81907844543457,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09020107239484787,
      "rewards/margins": 1.5048778057098389,
      "rewards/rejected": -1.595078706741333,
      "sft_loss": 0.02180342748761177,
      "step": 12750
    },
    {
      "epoch": 0.9926098794243485,
      "grad_norm": 2.612891435623169,
      "learning_rate": 6.7374150018256754e-09,
      "logits/chosen": -0.2980986535549164,
      "logits/rejected": -1.6294724941253662,
      "logps/chosen": -0.9495716094970703,
      "logps/rejected": -17.207256317138672,
      "loss": 0.9526,
      "num_input_tokens_seen": 16713664,
      "odds_ratio_loss": 9.388467788696289,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09495715796947479,
      "rewards/margins": 1.6257686614990234,
      "rewards/rejected": -1.7207257747650146,
      "sft_loss": 0.013775520026683807,
      "step": 12760
    },
    {
      "epoch": 0.9933877868533645,
      "grad_norm": 2.1296916007995605,
      "learning_rate": 5.39371296238711e-09,
      "logits/chosen": -0.03332703188061714,
      "logits/rejected": -1.5227935314178467,
      "logps/chosen": -0.6612142324447632,
      "logps/rejected": -12.962564468383789,
      "loss": 0.6679,
      "num_input_tokens_seen": 16727856,
      "odds_ratio_loss": 6.621134281158447,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.0661214217543602,
      "rewards/margins": 1.230135202407837,
      "rewards/rejected": -1.296256422996521,
      "sft_loss": 0.005786126479506493,
      "step": 12770
    },
    {
      "epoch": 0.9941656942823804,
      "grad_norm": 1.1724915504455566,
      "learning_rate": 4.199290937056955e-09,
      "logits/chosen": -0.24994008243083954,
      "logits/rejected": -1.2555500268936157,
      "logps/chosen": -1.1840388774871826,
      "logps/rejected": -13.798118591308594,
      "loss": 1.2418,
      "num_input_tokens_seen": 16740848,
      "odds_ratio_loss": 11.353900909423828,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.11840388923883438,
      "rewards/margins": 1.2614080905914307,
      "rewards/rejected": -1.3798120021820068,
      "sft_loss": 0.10643362998962402,
      "step": 12780
    },
    {
      "epoch": 0.9949436017113964,
      "grad_norm": 1.7221916913986206,
      "learning_rate": 3.154156059512192e-09,
      "logits/chosen": -0.31520482897758484,
      "logits/rejected": -1.4942389726638794,
      "logps/chosen": -0.9685817956924438,
      "logps/rejected": -15.389326095581055,
      "loss": 0.9696,
      "num_input_tokens_seen": 16754048,
      "odds_ratio_loss": 9.584997177124023,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.0968581885099411,
      "rewards/margins": 1.4420745372772217,
      "rewards/rejected": -1.5389328002929688,
      "sft_loss": 0.011118141934275627,
      "step": 12790
    },
    {
      "epoch": 0.9957215091404122,
      "grad_norm": 1.8254626989364624,
      "learning_rate": 2.258314571804143e-09,
      "logits/chosen": -0.04389285668730736,
      "logits/rejected": -2.0369677543640137,
      "logps/chosen": -0.956657886505127,
      "logps/rejected": -13.710368156433105,
      "loss": 0.98,
      "num_input_tokens_seen": 16769024,
      "odds_ratio_loss": 9.739141464233398,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.09566578269004822,
      "rewards/margins": 1.2753711938858032,
      "rewards/rejected": -1.3710368871688843,
      "sft_loss": 0.006069186609238386,
      "step": 12800
    },
    {
      "epoch": 0.9964994165694282,
      "grad_norm": 1.4335428476333618,
      "learning_rate": 1.511771824336261e-09,
      "logits/chosen": -0.19139555096626282,
      "logits/rejected": -1.8160762786865234,
      "logps/chosen": -0.9348017573356628,
      "logps/rejected": -13.172691345214844,
      "loss": 0.9398,
      "num_input_tokens_seen": 16784912,
      "odds_ratio_loss": 9.348949432373047,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.093480184674263,
      "rewards/margins": 1.2237889766693115,
      "rewards/rejected": -1.3172690868377686,
      "sft_loss": 0.00491794990375638,
      "step": 12810
    },
    {
      "epoch": 0.9972773239984442,
      "grad_norm": 2.756385326385498,
      "learning_rate": 9.145322758308261e-10,
      "logits/chosen": -0.37790343165397644,
      "logits/rejected": -1.0679850578308105,
      "logps/chosen": -1.01203191280365,
      "logps/rejected": -18.016756057739258,
      "loss": 1.0175,
      "num_input_tokens_seen": 16799360,
      "odds_ratio_loss": 8.870290756225586,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.10120320320129395,
      "rewards/margins": 1.7004724740982056,
      "rewards/rejected": -1.80167555809021,
      "sft_loss": 0.13049033284187317,
      "step": 12820
    },
    {
      "epoch": 0.9980552314274601,
      "grad_norm": 4.3858795166015625,
      "learning_rate": 4.665994932956385e-10,
      "logits/chosen": -0.1261058747768402,
      "logits/rejected": -1.3934625387191772,
      "logps/chosen": -0.8405144810676575,
      "logps/rejected": -18.738025665283203,
      "loss": 0.8406,
      "num_input_tokens_seen": 16817520,
      "odds_ratio_loss": 8.351066589355469,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08405144512653351,
      "rewards/margins": 1.7897508144378662,
      "rewards/rejected": -1.8738024234771729,
      "sft_loss": 0.005541075021028519,
      "step": 12830
    },
    {
      "epoch": 0.9988331388564761,
      "grad_norm": 107.50608825683594,
      "learning_rate": 1.679761519990386e-10,
      "logits/chosen": -0.2797563970088959,
      "logits/rejected": -1.545263648033142,
      "logps/chosen": -1.2055482864379883,
      "logps/rejected": -11.273192405700684,
      "loss": 1.2627,
      "num_input_tokens_seen": 16830640,
      "odds_ratio_loss": 12.5514554977417,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.12055482715368271,
      "rewards/margins": 1.00676429271698,
      "rewards/rejected": -1.1273192167282104,
      "sft_loss": 0.007532855961471796,
      "step": 12840
    },
    {
      "epoch": 0.999611046285492,
      "grad_norm": 1.6087356805801392,
      "learning_rate": 1.8664035467130802e-11,
      "logits/chosen": -0.23476950824260712,
      "logits/rejected": -1.456777811050415,
      "logps/chosen": -0.7568179965019226,
      "logps/rejected": -14.984893798828125,
      "loss": 0.758,
      "num_input_tokens_seen": 16844576,
      "odds_ratio_loss": 7.390267848968506,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.07568180561065674,
      "rewards/margins": 1.4228076934814453,
      "rewards/rejected": -1.4984893798828125,
      "sft_loss": 0.018998144194483757,
      "step": 12850
    },
    {
      "epoch": 1.0,
      "num_input_tokens_seen": 16848032,
      "step": 12855,
      "total_flos": 7.671391583498404e+17,
      "train_loss": 0.9753038262820161,
      "train_runtime": 20926.3402,
      "train_samples_per_second": 0.614,
      "train_steps_per_second": 0.614
    }
  ],
  "logging_steps": 10,
  "max_steps": 12855,
  "num_input_tokens_seen": 16848032,
  "num_train_epochs": 1,
  "save_steps": 1000,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": true
      },
      "attributes": {}
    }
  },
  "total_flos": 7.671391583498404e+17,
  "train_batch_size": 1,
  "trial_name": null,
  "trial_params": null
}