{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 2.9618082618862043,
  "eval_steps": 200,
  "global_step": 7600,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.03897116134060795,
      "grad_norm": 0.21226342022418976,
      "learning_rate": 0.00025,
      "logits/chosen": -19.687898635864258,
      "logits/rejected": -18.633106231689453,
      "logps/chosen": -343.7872619628906,
      "logps/rejected": -250.41775512695312,
      "loss": 0.4838,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": 0.6970043182373047,
      "rewards/margins": 0.9049937725067139,
      "rewards/rejected": -0.2079893946647644,
      "step": 100
    },
    {
      "epoch": 0.0779423226812159,
      "grad_norm": 1.92564857006073,
      "learning_rate": 0.0005,
      "logits/chosen": -19.59960174560547,
      "logits/rejected": -18.517173767089844,
      "logps/chosen": -349.5354309082031,
      "logps/rejected": -277.8677062988281,
      "loss": 0.2445,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": 1.4490649700164795,
      "rewards/margins": 3.1850626468658447,
      "rewards/rejected": -1.7359976768493652,
      "step": 200
    },
    {
      "epoch": 0.0779423226812159,
      "eval_logits/chosen": -16.551204681396484,
      "eval_logits/rejected": -16.51001739501953,
      "eval_logps/chosen": -340.37274169921875,
      "eval_logps/rejected": -285.42059326171875,
      "eval_loss": 0.19902318716049194,
      "eval_rewards/accuracies": 0.9711538553237915,
      "eval_rewards/chosen": 0.12859317660331726,
      "eval_rewards/margins": 1.9796644449234009,
      "eval_rewards/rejected": -1.8510712385177612,
      "eval_runtime": 183.9267,
      "eval_samples_per_second": 1.131,
      "eval_steps_per_second": 0.565,
      "step": 200
    },
    {
      "epoch": 0.11691348402182385,
      "grad_norm": 1.4603872299194336,
      "learning_rate": 0.0004997805905390462,
      "logits/chosen": -19.685632705688477,
      "logits/rejected": -18.630176544189453,
      "logps/chosen": -347.72479248046875,
      "logps/rejected": -297.4257507324219,
      "loss": 0.195,
      "rewards/accuracies": 0.8799999952316284,
      "rewards/chosen": 0.857117235660553,
      "rewards/margins": 4.945696830749512,
      "rewards/rejected": -4.088580131530762,
      "step": 300
    },
    {
      "epoch": 0.1558846453624318,
      "grad_norm": 2.9201486110687256,
      "learning_rate": 0.0004991227472802768,
      "logits/chosen": -19.715879440307617,
      "logits/rejected": -18.654006958007812,
      "logps/chosen": -347.0750427246094,
      "logps/rejected": -297.565185546875,
      "loss": 0.1965,
      "rewards/accuracies": 0.8949999809265137,
      "rewards/chosen": 0.27759745717048645,
      "rewards/margins": 5.794647693634033,
      "rewards/rejected": -5.517050266265869,
      "step": 400
    },
    {
      "epoch": 0.1558846453624318,
      "eval_logits/chosen": -16.683916091918945,
      "eval_logits/rejected": -16.602561950683594,
      "eval_logps/chosen": -343.2335510253906,
      "eval_logps/rejected": -309.0007629394531,
      "eval_loss": 0.04625505581498146,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": -0.15748834609985352,
      "eval_rewards/margins": 4.051597595214844,
      "eval_rewards/rejected": -4.2090864181518555,
      "eval_runtime": 183.7705,
      "eval_samples_per_second": 1.132,
      "eval_steps_per_second": 0.566,
      "step": 400
    },
    {
      "epoch": 0.19485580670303976,
      "grad_norm": 0.7862759232521057,
      "learning_rate": 0.0004980276249199705,
      "logits/chosen": -19.461082458496094,
      "logits/rejected": -18.434755325317383,
      "logps/chosen": -370.48394775390625,
      "logps/rejected": -314.01043701171875,
      "loss": 0.1873,
      "rewards/accuracies": 0.8974999785423279,
      "rewards/chosen": 0.1783198118209839,
      "rewards/margins": 5.953300952911377,
      "rewards/rejected": -5.774980545043945,
      "step": 500
    },
    {
      "epoch": 0.2338269680436477,
      "grad_norm": 0.5820030570030212,
      "learning_rate": 0.0004964971456997812,
      "logits/chosen": -19.28722381591797,
      "logits/rejected": -18.402751922607422,
      "logps/chosen": -369.10980224609375,
      "logps/rejected": -328.44403076171875,
      "loss": 0.19,
      "rewards/accuracies": 0.9024999737739563,
      "rewards/chosen": -0.12349352240562439,
      "rewards/margins": 6.251636505126953,
      "rewards/rejected": -6.375130653381348,
      "step": 600
    },
    {
      "epoch": 0.2338269680436477,
      "eval_logits/chosen": -16.357208251953125,
      "eval_logits/rejected": -16.30510139465332,
      "eval_logps/chosen": -336.88409423828125,
      "eval_logps/rejected": -304.0970764160156,
      "eval_loss": 0.036708563566207886,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": 0.47745734453201294,
      "eval_rewards/margins": 4.196179389953613,
      "eval_rewards/rejected": -3.718721866607666,
      "eval_runtime": 183.7701,
      "eval_samples_per_second": 1.132,
      "eval_steps_per_second": 0.566,
      "step": 600
    },
    {
      "epoch": 0.2727981293842556,
      "grad_norm": 0.3981655538082123,
      "learning_rate": 0.0004945339960326746,
      "logits/chosen": -19.472816467285156,
      "logits/rejected": -18.500709533691406,
      "logps/chosen": -368.1885681152344,
      "logps/rejected": -322.74298095703125,
      "loss": 0.171,
      "rewards/accuracies": 0.9012500047683716,
      "rewards/chosen": -0.00018103599722962826,
      "rewards/margins": 6.297857761383057,
      "rewards/rejected": -6.298038959503174,
      "step": 700
    },
    {
      "epoch": 0.3117692907248636,
      "grad_norm": 0.429470956325531,
      "learning_rate": 0.0004921416217875326,
      "logits/chosen": -19.675304412841797,
      "logits/rejected": -18.701597213745117,
      "logps/chosen": -354.1678771972656,
      "logps/rejected": -312.2345886230469,
      "loss": 0.1742,
      "rewards/accuracies": 0.8912500143051147,
      "rewards/chosen": 0.06210971996188164,
      "rewards/margins": 6.645627498626709,
      "rewards/rejected": -6.583518028259277,
      "step": 800
    },
    {
      "epoch": 0.3117692907248636,
      "eval_logits/chosen": -16.51410484313965,
      "eval_logits/rejected": -16.480632781982422,
      "eval_logps/chosen": -338.15936279296875,
      "eval_logps/rejected": -315.7767333984375,
      "eval_loss": 0.023502787575125694,
      "eval_rewards/accuracies": 0.995192289352417,
      "eval_rewards/chosen": 0.34993284940719604,
      "eval_rewards/margins": 5.236622333526611,
      "eval_rewards/rejected": -4.886689186096191,
      "eval_runtime": 183.2137,
      "eval_samples_per_second": 1.135,
      "eval_steps_per_second": 0.568,
      "step": 800
    },
    {
      "epoch": 0.35074045206547155,
      "grad_norm": 0.3401203453540802,
      "learning_rate": 0.0004893242222407031,
      "logits/chosen": -19.789016723632812,
      "logits/rejected": -18.94336700439453,
      "logps/chosen": -343.6241760253906,
      "logps/rejected": -323.7110595703125,
      "loss": 0.1681,
      "rewards/accuracies": 0.9087499976158142,
      "rewards/chosen": 0.2733341455459595,
      "rewards/margins": 6.843422889709473,
      "rewards/rejected": -6.5700883865356445,
      "step": 900
    },
    {
      "epoch": 0.3897116134060795,
      "grad_norm": 1.2149152755737305,
      "learning_rate": 0.00048608674270511344,
      "logits/chosen": -19.847705841064453,
      "logits/rejected": -18.942218780517578,
      "logps/chosen": -359.46551513671875,
      "logps/rejected": -334.84136962890625,
      "loss": 0.1605,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.1338280439376831,
      "rewards/margins": 7.973731517791748,
      "rewards/rejected": -7.839903354644775,
      "step": 1000
    },
    {
      "epoch": 0.3897116134060795,
      "eval_logits/chosen": -16.609466552734375,
      "eval_logits/rejected": -16.567523956298828,
      "eval_logps/chosen": -334.4723205566406,
      "eval_logps/rejected": -314.89141845703125,
      "eval_loss": 0.010429470799863338,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": 0.7186338901519775,
      "eval_rewards/margins": 5.516786575317383,
      "eval_rewards/rejected": -4.798152446746826,
      "eval_runtime": 183.9441,
      "eval_samples_per_second": 1.131,
      "eval_steps_per_second": 0.565,
      "step": 1000
    },
    {
      "epoch": 0.4286827747466875,
      "grad_norm": 15.479838371276855,
      "learning_rate": 0.00048243486584988133,
      "logits/chosen": -19.61993408203125,
      "logits/rejected": -18.709421157836914,
      "logps/chosen": -361.1015625,
      "logps/rejected": -337.32342529296875,
      "loss": 0.173,
      "rewards/accuracies": 0.9037500023841858,
      "rewards/chosen": -0.6482878923416138,
      "rewards/margins": 7.439374923706055,
      "rewards/rejected": -8.087662696838379,
      "step": 1100
    },
    {
      "epoch": 0.4676539360872954,
      "grad_norm": 2.251624822616577,
      "learning_rate": 0.00047837500172566503,
      "logits/chosen": -19.768835067749023,
      "logits/rejected": -18.762409210205078,
      "logps/chosen": -365.1903076171875,
      "logps/rejected": -328.1022644042969,
      "loss": 0.19,
      "rewards/accuracies": 0.9200000166893005,
      "rewards/chosen": -0.6630336046218872,
      "rewards/margins": 6.923609733581543,
      "rewards/rejected": -7.586642265319824,
      "step": 1200
    },
    {
      "epoch": 0.4676539360872954,
      "eval_logits/chosen": -16.668439865112305,
      "eval_logits/rejected": -16.63960075378418,
      "eval_logps/chosen": -348.08587646484375,
      "eval_logps/rejected": -331.30828857421875,
      "eval_loss": 0.013935078866779804,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": -0.6427211761474609,
      "eval_rewards/margins": 5.797122478485107,
      "eval_rewards/rejected": -6.43984317779541,
      "eval_runtime": 183.93,
      "eval_samples_per_second": 1.131,
      "eval_steps_per_second": 0.565,
      "step": 1200
    },
    {
      "epoch": 0.5066250974279034,
      "grad_norm": 2.569606304168701,
      "learning_rate": 0.00047391427651325704,
      "logits/chosen": -20.112619400024414,
      "logits/rejected": -19.097084045410156,
      "logps/chosen": -361.98895263671875,
      "logps/rejected": -335.0581970214844,
      "loss": 0.1348,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -0.5764763355255127,
      "rewards/margins": 7.914137363433838,
      "rewards/rejected": -8.49061393737793,
      "step": 1300
    },
    {
      "epoch": 0.5455962587685113,
      "grad_norm": 1.0773099660873413,
      "learning_rate": 0.00046906052001517164,
      "logits/chosen": -20.002775192260742,
      "logits/rejected": -19.024526596069336,
      "logps/chosen": -369.9784851074219,
      "logps/rejected": -346.8582763671875,
      "loss": 0.1693,
      "rewards/accuracies": 0.9237499833106995,
      "rewards/chosen": -1.2993569374084473,
      "rewards/margins": 8.027504920959473,
      "rewards/rejected": -9.326862335205078,
      "step": 1400
    },
    {
      "epoch": 0.5455962587685113,
      "eval_logits/chosen": -16.792936325073242,
      "eval_logits/rejected": -16.73811912536621,
      "eval_logps/chosen": -335.2118835449219,
      "eval_logps/rejected": -317.7545471191406,
      "eval_loss": 0.019140072166919708,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": 0.6446801424026489,
      "eval_rewards/margins": 5.729149341583252,
      "eval_rewards/rejected": -5.084469318389893,
      "eval_runtime": 183.9981,
      "eval_samples_per_second": 1.13,
      "eval_steps_per_second": 0.565,
      "step": 1400
    },
    {
      "epoch": 0.5845674201091192,
      "grad_norm": 3.4432373046875,
      "learning_rate": 0.00046382225191218373,
      "logits/chosen": -20.071659088134766,
      "logits/rejected": -19.052783966064453,
      "logps/chosen": -363.5613708496094,
      "logps/rejected": -345.07879638671875,
      "loss": 0.1425,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -0.769163966178894,
      "rewards/margins": 8.191140174865723,
      "rewards/rejected": -8.96030330657959,
      "step": 1500
    },
    {
      "epoch": 0.6235385814497272,
      "grad_norm": 1.9013152122497559,
      "learning_rate": 0.0004582086668089399,
      "logits/chosen": -19.665212631225586,
      "logits/rejected": -18.75629234313965,
      "logps/chosen": -367.5525817871094,
      "logps/rejected": -342.85516357421875,
      "loss": 0.1263,
      "rewards/accuracies": 0.9325000047683716,
      "rewards/chosen": -0.8581268191337585,
      "rewards/margins": 8.124137878417969,
      "rewards/rejected": -8.982264518737793,
      "step": 1600
    },
    {
      "epoch": 0.6235385814497272,
      "eval_logits/chosen": -16.445354461669922,
      "eval_logits/rejected": -16.387128829956055,
      "eval_logps/chosen": -327.33563232421875,
      "eval_logps/rejected": -312.2492980957031,
      "eval_loss": 0.0060209427028894424,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": 1.4323018789291382,
      "eval_rewards/margins": 5.966245174407959,
      "eval_rewards/rejected": -4.5339436531066895,
      "eval_runtime": 184.0733,
      "eval_samples_per_second": 1.13,
      "eval_steps_per_second": 0.565,
      "step": 1600
    },
    {
      "epoch": 0.6625097427903351,
      "grad_norm": 0.8941299915313721,
      "learning_rate": 0.0004522296180948922,
      "logits/chosen": -19.424589157104492,
      "logits/rejected": -18.495784759521484,
      "logps/chosen": -374.550537109375,
      "logps/rejected": -353.1033630371094,
      "loss": 0.125,
      "rewards/accuracies": 0.9387500286102295,
      "rewards/chosen": -0.5644223690032959,
      "rewards/margins": 8.560413360595703,
      "rewards/rejected": -9.124835014343262,
      "step": 1700
    },
    {
      "epoch": 0.7014809041309431,
      "grad_norm": 1.1238608360290527,
      "learning_rate": 0.00044589560064888347,
      "logits/chosen": -19.239290237426758,
      "logits/rejected": -18.375873565673828,
      "logps/chosen": -369.72027587890625,
      "logps/rejected": -349.2558898925781,
      "loss": 0.1232,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.9773778319358826,
      "rewards/margins": 8.248481750488281,
      "rewards/rejected": -9.225860595703125,
      "step": 1800
    },
    {
      "epoch": 0.7014809041309431,
      "eval_logits/chosen": -16.073932647705078,
      "eval_logits/rejected": -16.034406661987305,
      "eval_logps/chosen": -343.9805908203125,
      "eval_logps/rejected": -342.4121398925781,
      "eval_loss": 0.005078889429569244,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": -0.23218610882759094,
      "eval_rewards/margins": 7.318041801452637,
      "eval_rewards/rejected": -7.550227642059326,
      "eval_runtime": 183.2955,
      "eval_samples_per_second": 1.135,
      "eval_steps_per_second": 0.567,
      "step": 1800
    },
    {
      "epoch": 0.7404520654715511,
      "grad_norm": 3.126462697982788,
      "learning_rate": 0.00043921773241774185,
      "logits/chosen": -19.31768226623535,
      "logits/rejected": -18.37215232849121,
      "logps/chosen": -348.50390625,
      "logps/rejected": -336.8278503417969,
      "loss": 0.0923,
      "rewards/accuracies": 0.9587500095367432,
      "rewards/chosen": -0.21741238236427307,
      "rewards/margins": 8.6088228225708,
      "rewards/rejected": -8.826234817504883,
      "step": 1900
    },
    {
      "epoch": 0.779423226812159,
      "grad_norm": 0.6362993121147156,
      "learning_rate": 0.0004322077349012186,
      "logits/chosen": -19.6059513092041,
      "logits/rejected": -18.7484073638916,
      "logps/chosen": -359.93206787109375,
      "logps/rejected": -353.2133483886719,
      "loss": 0.0983,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -1.2052465677261353,
      "rewards/margins": 8.998382568359375,
      "rewards/rejected": -10.203628540039062,
      "step": 2000
    },
    {
      "epoch": 0.779423226812159,
      "eval_logits/chosen": -16.276912689208984,
      "eval_logits/rejected": -16.23944854736328,
      "eval_logps/chosen": -340.11883544921875,
      "eval_logps/rejected": -336.9291687011719,
      "eval_loss": 0.003970090765506029,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": 0.15398484468460083,
      "eval_rewards/margins": 7.155914783477783,
      "eval_rewards/rejected": -7.001929759979248,
      "eval_runtime": 184.065,
      "eval_samples_per_second": 1.13,
      "eval_steps_per_second": 0.565,
      "step": 2000
    },
    {
      "epoch": 0.818394388152767,
      "grad_norm": 3.1369752883911133,
      "learning_rate": 0.00042487791257752493,
      "logits/chosen": -19.724271774291992,
      "logits/rejected": -18.804405212402344,
      "logps/chosen": -371.5325012207031,
      "logps/rejected": -357.1531677246094,
      "loss": 0.1049,
      "rewards/accuracies": 0.9512500166893005,
      "rewards/chosen": -1.3020814657211304,
      "rewards/margins": 8.643760681152344,
      "rewards/rejected": -9.945842742919922,
      "step": 2100
    },
    {
      "epoch": 0.857365549493375,
      "grad_norm": 1.0814727544784546,
      "learning_rate": 0.0004172411313055802,
      "logits/chosen": -19.90325164794922,
      "logits/rejected": -19.001785278320312,
      "logps/chosen": -376.0804748535156,
      "logps/rejected": -359.4775390625,
      "loss": 0.0924,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -0.4394536316394806,
      "rewards/margins": 8.659097671508789,
      "rewards/rejected": -9.098551750183105,
      "step": 2200
    },
    {
      "epoch": 0.857365549493375,
      "eval_logits/chosen": -16.71300506591797,
      "eval_logits/rejected": -16.685504913330078,
      "eval_logps/chosen": -334.0523986816406,
      "eval_logps/rejected": -324.8212890625,
      "eval_loss": 0.00510050356388092,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": 0.760625958442688,
      "eval_rewards/margins": 6.551769733428955,
      "eval_rewards/rejected": -5.791143417358398,
      "eval_runtime": 183.3621,
      "eval_samples_per_second": 1.134,
      "eval_steps_per_second": 0.567,
      "step": 2200
    },
    {
      "epoch": 0.8963367108339828,
      "grad_norm": 2.7771525382995605,
      "learning_rate": 0.0004093107957418828,
      "logits/chosen": -19.735408782958984,
      "logits/rejected": -18.8337459564209,
      "logps/chosen": -367.6742858886719,
      "logps/rejected": -356.5105895996094,
      "loss": 0.0789,
      "rewards/accuracies": 0.9612500071525574,
      "rewards/chosen": -1.1817975044250488,
      "rewards/margins": 9.191315650939941,
      "rewards/rejected": -10.373113632202148,
      "step": 2300
    },
    {
      "epoch": 0.9353078721745908,
      "grad_norm": 0.0016880702460184693,
      "learning_rate": 0.0004011008258116425,
      "logits/chosen": -19.32187271118164,
      "logits/rejected": -18.563722610473633,
      "logps/chosen": -373.0425109863281,
      "logps/rejected": -362.72589111328125,
      "loss": 0.0848,
      "rewards/accuracies": 0.9649999737739563,
      "rewards/chosen": -2.054009437561035,
      "rewards/margins": 9.20674991607666,
      "rewards/rejected": -11.260760307312012,
      "step": 2400
    },
    {
      "epoch": 0.9353078721745908,
      "eval_logits/chosen": -16.159223556518555,
      "eval_logits/rejected": -16.161327362060547,
      "eval_logps/chosen": -337.83880615234375,
      "eval_logps/rejected": -339.8623046875,
      "eval_loss": 0.0020727256778627634,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": 0.38198867440223694,
      "eval_rewards/margins": 7.677234172821045,
      "eval_rewards/rejected": -7.295245170593262,
      "eval_runtime": 183.8954,
      "eval_samples_per_second": 1.131,
      "eval_steps_per_second": 0.566,
      "step": 2400
    },
    {
      "epoch": 0.9742790335151987,
      "grad_norm": 1.488077998161316,
      "learning_rate": 0.00039262563227547396,
      "logits/chosen": -19.280136108398438,
      "logits/rejected": -18.451894760131836,
      "logps/chosen": -376.1581115722656,
      "logps/rejected": -366.6534118652344,
      "loss": 0.0662,
      "rewards/accuracies": 0.9725000262260437,
      "rewards/chosen": -1.176061987876892,
      "rewards/margins": 8.959639549255371,
      "rewards/rejected": -10.135702133178711,
      "step": 2500
    },
    {
      "epoch": 1.0132501948558068,
      "grad_norm": 2.0352509021759033,
      "learning_rate": 0.0003839000914345393,
      "logits/chosen": -19.330242156982422,
      "logits/rejected": -18.416221618652344,
      "logps/chosen": -373.55523681640625,
      "logps/rejected": -357.8737487792969,
      "loss": 0.0373,
      "rewards/accuracies": 0.9837499856948853,
      "rewards/chosen": -0.7969868183135986,
      "rewards/margins": 9.221331596374512,
      "rewards/rejected": -10.018318176269531,
      "step": 2600
    },
    {
      "epoch": 1.0132501948558068,
      "eval_logits/chosen": -15.664155006408691,
      "eval_logits/rejected": -15.684382438659668,
      "eval_logps/chosen": -348.93939208984375,
      "eval_logps/rejected": -357.6224365234375,
      "eval_loss": 0.001914023538120091,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": -0.7280683517456055,
      "eval_rewards/margins": 8.34318733215332,
      "eval_rewards/rejected": -9.071255683898926,
      "eval_runtime": 183.3393,
      "eval_samples_per_second": 1.135,
      "eval_steps_per_second": 0.567,
      "step": 2600
    },
    {
      "epoch": 1.0522213561964147,
      "grad_norm": 1.1277427673339844,
      "learning_rate": 0.00037493951901853797,
      "logits/chosen": -19.345949172973633,
      "logits/rejected": -18.49242401123047,
      "logps/chosen": -365.28350830078125,
      "logps/rejected": -361.580810546875,
      "loss": 0.021,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -0.9318392872810364,
      "rewards/margins": 9.834463119506836,
      "rewards/rejected": -10.766302108764648,
      "step": 2700
    },
    {
      "epoch": 1.0911925175370225,
      "grad_norm": 0.07178483158349991,
      "learning_rate": 0.00036575964330237904,
      "logits/chosen": -19.510316848754883,
      "logits/rejected": -18.55198860168457,
      "logps/chosen": -367.44598388671875,
      "logps/rejected": -363.74078369140625,
      "loss": 0.0151,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -1.267324447631836,
      "rewards/margins": 10.227116584777832,
      "rewards/rejected": -11.494441032409668,
      "step": 2800
    },
    {
      "epoch": 1.0911925175370225,
      "eval_logits/chosen": -15.850824356079102,
      "eval_logits/rejected": -15.864118576049805,
      "eval_logps/chosen": -359.7252502441406,
      "eval_logps/rejected": -381.7374572753906,
      "eval_loss": 0.0014377026818692684,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": -1.8066591024398804,
      "eval_rewards/margins": 9.676095962524414,
      "eval_rewards/rejected": -11.482755661010742,
      "eval_runtime": 183.4769,
      "eval_samples_per_second": 1.134,
      "eval_steps_per_second": 0.567,
      "step": 2800
    },
    {
      "epoch": 1.1301636788776306,
      "grad_norm": 1.4340250492095947,
      "learning_rate": 0.00035637657749872255,
      "logits/chosen": -19.317781448364258,
      "logits/rejected": -18.438331604003906,
      "logps/chosen": -375.9151306152344,
      "logps/rejected": -366.2434387207031,
      "loss": 0.0137,
      "rewards/accuracies": 0.9962499737739563,
      "rewards/chosen": -2.468919277191162,
      "rewards/margins": 10.26855754852295,
      "rewards/rejected": -12.737476348876953,
      "step": 2900
    },
    {
      "epoch": 1.1691348402182384,
      "grad_norm": 0.2114144116640091,
      "learning_rate": 0.00034680679147484916,
      "logits/chosen": -19.166513442993164,
      "logits/rejected": -18.25996208190918,
      "logps/chosen": -371.7158508300781,
      "logps/rejected": -375.24481201171875,
      "loss": 0.0177,
      "rewards/accuracies": 0.9925000071525574,
      "rewards/chosen": -1.933343768119812,
      "rewards/margins": 10.125692367553711,
      "rewards/rejected": -12.059035301208496,
      "step": 3000
    },
    {
      "epoch": 1.1691348402182384,
      "eval_logits/chosen": -16.009708404541016,
      "eval_logits/rejected": -15.956798553466797,
      "eval_logps/chosen": -332.91302490234375,
      "eval_logps/rejected": -345.1927490234375,
      "eval_loss": 0.0008918559760786593,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": 0.874567449092865,
      "eval_rewards/margins": 8.702858924865723,
      "eval_rewards/rejected": -7.8282904624938965,
      "eval_runtime": 184.0136,
      "eval_samples_per_second": 1.13,
      "eval_steps_per_second": 0.565,
      "step": 3000
    },
    {
      "epoch": 1.2081060015588465,
      "grad_norm": 1.6861902475357056,
      "learning_rate": 0.00033706708284350227,
      "logits/chosen": -19.25012969970703,
      "logits/rejected": -18.424623489379883,
      "logps/chosen": -372.6192932128906,
      "logps/rejected": -379.19549560546875,
      "loss": 0.0209,
      "rewards/accuracies": 0.9950000047683716,
      "rewards/chosen": -1.3639909029006958,
      "rewards/margins": 10.262293815612793,
      "rewards/rejected": -11.626285552978516,
      "step": 3100
    },
    {
      "epoch": 1.2470771628994544,
      "grad_norm": 0.16628094017505646,
      "learning_rate": 0.00032717454747844735,
      "logits/chosen": -19.219282150268555,
      "logits/rejected": -18.238243103027344,
      "logps/chosen": -365.1196594238281,
      "logps/rejected": -364.3663330078125,
      "loss": 0.0168,
      "rewards/accuracies": 0.9912499785423279,
      "rewards/chosen": -1.1836423873901367,
      "rewards/margins": 10.353758811950684,
      "rewards/rejected": -11.53740119934082,
      "step": 3200
    },
    {
      "epoch": 1.2470771628994544,
      "eval_logits/chosen": -15.836195945739746,
      "eval_logits/rejected": -15.786764144897461,
      "eval_logps/chosen": -340.96075439453125,
      "eval_logps/rejected": -357.6408996582031,
      "eval_loss": 0.0008726614178158343,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": 0.06979300826787949,
      "eval_rewards/margins": 9.142892837524414,
      "eval_rewards/rejected": -9.073100090026855,
      "eval_runtime": 183.8457,
      "eval_samples_per_second": 1.131,
      "eval_steps_per_second": 0.566,
      "step": 3200
    },
    {
      "epoch": 1.2860483242400624,
      "grad_norm": 0.0003638103371486068,
      "learning_rate": 0.00031714654950649947,
      "logits/chosen": -19.43454933166504,
      "logits/rejected": -18.49201011657715,
      "logps/chosen": -371.1318664550781,
      "logps/rejected": -372.17938232421875,
      "loss": 0.0163,
      "rewards/accuracies": 0.9912499785423279,
      "rewards/chosen": -1.4684104919433594,
      "rewards/margins": 10.204544067382812,
      "rewards/rejected": -11.672954559326172,
      "step": 3300
    },
    {
      "epoch": 1.3250194855806703,
      "grad_norm": 0.0004742901655845344,
      "learning_rate": 0.0003070006908286945,
      "logits/chosen": -19.53050994873047,
      "logits/rejected": -18.605304718017578,
      "logps/chosen": -383.9638671875,
      "logps/rejected": -382.642333984375,
      "loss": 0.0159,
      "rewards/accuracies": 0.9925000071525574,
      "rewards/chosen": -1.5249592065811157,
      "rewards/margins": 10.782021522521973,
      "rewards/rejected": -12.306981086730957,
      "step": 3400
    },
    {
      "epoch": 1.3250194855806703,
      "eval_logits/chosen": -15.98876953125,
      "eval_logits/rejected": -15.942709922790527,
      "eval_logps/chosen": -337.0641174316406,
      "eval_logps/rejected": -357.69342041015625,
      "eval_loss": 0.0007918892079032958,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": 0.4594591557979584,
      "eval_rewards/margins": 9.53781795501709,
      "eval_rewards/rejected": -9.078359603881836,
      "eval_runtime": 183.8285,
      "eval_samples_per_second": 1.131,
      "eval_steps_per_second": 0.566,
      "step": 3400
    },
    {
      "epoch": 1.3639906469212781,
      "grad_norm": 3.363085985183716,
      "learning_rate": 0.0002967547802240997,
      "logits/chosen": -19.533775329589844,
      "logits/rejected": -18.48381233215332,
      "logps/chosen": -369.41827392578125,
      "logps/rejected": -367.9228820800781,
      "loss": 0.0152,
      "rewards/accuracies": 0.9950000047683716,
      "rewards/chosen": -0.9302732944488525,
      "rewards/margins": 10.337385177612305,
      "rewards/rejected": -11.267658233642578,
      "step": 3500
    },
    {
      "epoch": 1.4029618082618862,
      "grad_norm": 0.014138066209852695,
      "learning_rate": 0.00028642680209049715,
      "logits/chosen": -19.569242477416992,
      "logits/rejected": -18.72142791748047,
      "logps/chosen": -355.16339111328125,
      "logps/rejected": -369.4004211425781,
      "loss": 0.0106,
      "rewards/accuracies": 0.9987499713897705,
      "rewards/chosen": -0.6573285460472107,
      "rewards/margins": 10.500960350036621,
      "rewards/rejected": -11.15829086303711,
      "step": 3600
    },
    {
      "epoch": 1.4029618082618862,
      "eval_logits/chosen": -16.05227279663086,
      "eval_logits/rejected": -16.000469207763672,
      "eval_logps/chosen": -334.4986572265625,
      "eval_logps/rejected": -352.8683166503906,
      "eval_loss": 0.0009424146264791489,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": 0.7160030603408813,
      "eval_rewards/margins": 9.311846733093262,
      "eval_rewards/rejected": -8.595843315124512,
      "eval_runtime": 183.6848,
      "eval_samples_per_second": 1.132,
      "eval_steps_per_second": 0.566,
      "step": 3600
    },
    {
      "epoch": 1.4419329696024943,
      "grad_norm": 0.009088713675737381,
      "learning_rate": 0.00027603488487680684,
      "logits/chosen": -19.91989517211914,
      "logits/rejected": -18.872535705566406,
      "logps/chosen": -372.661865234375,
      "logps/rejected": -367.7705078125,
      "loss": 0.0184,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -1.1601972579956055,
      "rewards/margins": 10.311330795288086,
      "rewards/rejected": -11.471526145935059,
      "step": 3700
    },
    {
      "epoch": 1.4809041309431021,
      "grad_norm": 0.010365006513893604,
      "learning_rate": 0.00026559726926266204,
      "logits/chosen": -19.595075607299805,
      "logits/rejected": -18.658893585205078,
      "logps/chosen": -390.93206787109375,
      "logps/rejected": -393.7774963378906,
      "loss": 0.0095,
      "rewards/accuracies": 0.9950000047683716,
      "rewards/chosen": -2.7839019298553467,
      "rewards/margins": 10.808964729309082,
      "rewards/rejected": -13.592867851257324,
      "step": 3800
    },
    {
      "epoch": 1.4809041309431021,
      "eval_logits/chosen": -15.648859977722168,
      "eval_logits/rejected": -15.611715316772461,
      "eval_logps/chosen": -346.69561767578125,
      "eval_logps/rejected": -376.822265625,
      "eval_loss": 0.0005593308596871793,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": -0.5036950707435608,
      "eval_rewards/margins": 10.487546920776367,
      "eval_rewards/rejected": -10.991241455078125,
      "eval_runtime": 184.1015,
      "eval_samples_per_second": 1.13,
      "eval_steps_per_second": 0.565,
      "step": 3800
    },
    {
      "epoch": 1.51987529228371,
      "grad_norm": 0.12639912962913513,
      "learning_rate": 0.00025513227614098707,
      "logits/chosen": -19.499780654907227,
      "logits/rejected": -18.518644332885742,
      "logps/chosen": -366.645263671875,
      "logps/rejected": -380.7431640625,
      "loss": 0.0067,
      "rewards/accuracies": 0.9975000023841858,
      "rewards/chosen": -1.6605476140975952,
      "rewards/margins": 10.881020545959473,
      "rewards/rejected": -12.541565895080566,
      "step": 3900
    },
    {
      "epoch": 1.558846453624318,
      "grad_norm": 0.32782620191574097,
      "learning_rate": 0.00024465827445977964,
      "logits/chosen": -19.350522994995117,
      "logits/rejected": -18.31049346923828,
      "logps/chosen": -380.0142822265625,
      "logps/rejected": -377.0948791503906,
      "loss": 0.0107,
      "rewards/accuracies": 0.9962499737739563,
      "rewards/chosen": -1.4461276531219482,
      "rewards/margins": 10.850979804992676,
      "rewards/rejected": -12.297107696533203,
      "step": 4000
    },
    {
      "epoch": 1.558846453624318,
      "eval_logits/chosen": -15.838950157165527,
      "eval_logits/rejected": -15.806854248046875,
      "eval_logps/chosen": -336.4289245605469,
      "eval_logps/rejected": -364.8307189941406,
      "eval_loss": 0.00040981665370054543,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": 0.5229762196540833,
      "eval_rewards/margins": 10.315059661865234,
      "eval_rewards/rejected": -9.792083740234375,
      "eval_runtime": 184.0487,
      "eval_samples_per_second": 1.13,
      "eval_steps_per_second": 0.565,
      "step": 4000
    },
    {
      "epoch": 1.597817614964926,
      "grad_norm": 1.2895886898040771,
      "learning_rate": 0.00023419364897954285,
      "logits/chosen": -19.494075775146484,
      "logits/rejected": -18.560813903808594,
      "logps/chosen": -361.8075256347656,
      "logps/rejected": -380.0975646972656,
      "loss": 0.0108,
      "rewards/accuracies": 0.9950000047683716,
      "rewards/chosen": -1.1338391304016113,
      "rewards/margins": 11.00627326965332,
      "rewards/rejected": -12.140111923217773,
      "step": 4100
    },
    {
      "epoch": 1.6367887763055338,
      "grad_norm": 0.00033555322443135083,
      "learning_rate": 0.00022375676800296247,
      "logits/chosen": -19.587282180786133,
      "logits/rejected": -18.464643478393555,
      "logps/chosen": -379.6680603027344,
      "logps/rejected": -372.1777648925781,
      "loss": 0.0124,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -1.0544594526290894,
      "rewards/margins": 11.037619590759277,
      "rewards/rejected": -12.092079162597656,
      "step": 4200
    },
    {
      "epoch": 1.6367887763055338,
      "eval_logits/chosen": -15.83126163482666,
      "eval_logits/rejected": -15.794379234313965,
      "eval_logps/chosen": -330.6876525878906,
      "eval_logps/rejected": -356.1661682128906,
      "eval_loss": 0.0004220473056193441,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": 1.0971003770828247,
      "eval_rewards/margins": 10.022727012634277,
      "eval_rewards/rejected": -8.925626754760742,
      "eval_runtime": 183.9983,
      "eval_samples_per_second": 1.13,
      "eval_steps_per_second": 0.565,
      "step": 4200
    },
    {
      "epoch": 1.6757599376461418,
      "grad_norm": 1.2421733140945435,
      "learning_rate": 0.00021336595113347144,
      "logits/chosen": -19.402978897094727,
      "logits/rejected": -18.351083755493164,
      "logps/chosen": -370.1186828613281,
      "logps/rejected": -374.5020446777344,
      "loss": 0.009,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -1.1320860385894775,
      "rewards/margins": 10.839487075805664,
      "rewards/rejected": -11.971571922302246,
      "step": 4300
    },
    {
      "epoch": 1.71473109898675,
      "grad_norm": 0.16441740095615387,
      "learning_rate": 0.00020303943711929526,
      "logits/chosen": -19.135257720947266,
      "logits/rejected": -18.16560173034668,
      "logps/chosen": -375.6964111328125,
      "logps/rejected": -390.9460144042969,
      "loss": 0.021,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -2.0195655822753906,
      "rewards/margins": 11.327723503112793,
      "rewards/rejected": -13.3472900390625,
      "step": 4400
    },
    {
      "epoch": 1.71473109898675,
      "eval_logits/chosen": -15.488088607788086,
      "eval_logits/rejected": -15.465119361877441,
      "eval_logps/chosen": -344.2342529296875,
      "eval_logps/rejected": -376.2817687988281,
      "eval_loss": 0.0005891394102945924,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": -0.25755709409713745,
      "eval_rewards/margins": 10.679630279541016,
      "eval_rewards/rejected": -10.937186241149902,
      "eval_runtime": 184.1206,
      "eval_samples_per_second": 1.13,
      "eval_steps_per_second": 0.565,
      "step": 4400
    },
    {
      "epoch": 1.7537022603273578,
      "grad_norm": 0.02925615757703781,
      "learning_rate": 0.00019279535183942101,
      "logits/chosen": -19.17304039001465,
      "logits/rejected": -18.184049606323242,
      "logps/chosen": -385.8622131347656,
      "logps/rejected": -391.7120361328125,
      "loss": 0.0083,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -1.9831608533859253,
      "rewards/margins": 11.25387954711914,
      "rewards/rejected": -13.237038612365723,
      "step": 4500
    },
    {
      "epoch": 1.7926734216679656,
      "grad_norm": 0.05078015476465225,
      "learning_rate": 0.00018265167648768259,
      "logits/chosen": -19.064685821533203,
      "logits/rejected": -18.237638473510742,
      "logps/chosen": -376.9311218261719,
      "logps/rejected": -387.96905517578125,
      "loss": 0.0061,
      "rewards/accuracies": 0.9950000047683716,
      "rewards/chosen": -2.660050392150879,
      "rewards/margins": 11.354269027709961,
      "rewards/rejected": -14.014320373535156,
      "step": 4600
    },
    {
      "epoch": 1.7926734216679656,
      "eval_logits/chosen": -15.26915454864502,
      "eval_logits/rejected": -15.260515213012695,
      "eval_logps/chosen": -355.8375549316406,
      "eval_logps/rejected": -390.41119384765625,
      "eval_loss": 0.0009181927889585495,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": -1.417887568473816,
      "eval_rewards/margins": 10.932249069213867,
      "eval_rewards/rejected": -12.350136756896973,
      "eval_runtime": 183.689,
      "eval_samples_per_second": 1.132,
      "eval_steps_per_second": 0.566,
      "step": 4600
    },
    {
      "epoch": 1.8316445830085737,
      "grad_norm": 0.2935536205768585,
      "learning_rate": 0.00017262621601080811,
      "logits/chosen": -19.163440704345703,
      "logits/rejected": -18.167402267456055,
      "logps/chosen": -379.1352233886719,
      "logps/rejected": -383.6081237792969,
      "loss": 0.0126,
      "rewards/accuracies": 0.9950000047683716,
      "rewards/chosen": -2.3724796772003174,
      "rewards/margins": 11.351487159729004,
      "rewards/rejected": -13.723965644836426,
      "step": 4700
    },
    {
      "epoch": 1.8706157443491818,
      "grad_norm": 0.02069302648305893,
      "learning_rate": 0.00016273656785582984,
      "logits/chosen": -18.975814819335938,
      "logits/rejected": -18.142276763916016,
      "logps/chosen": -383.4511413574219,
      "logps/rejected": -393.5997009277344,
      "loss": 0.0102,
      "rewards/accuracies": 0.9950000047683716,
      "rewards/chosen": -2.359323024749756,
      "rewards/margins": 11.0841064453125,
      "rewards/rejected": -13.443428993225098,
      "step": 4800
    },
    {
      "epoch": 1.8706157443491818,
      "eval_logits/chosen": -15.47636604309082,
      "eval_logits/rejected": -15.429464340209961,
      "eval_logps/chosen": -341.96356201171875,
      "eval_logps/rejected": -369.5177307128906,
      "eval_loss": 0.0004053489537909627,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": -0.030484315007925034,
      "eval_rewards/margins": 10.230300903320312,
      "eval_rewards/rejected": -10.260786056518555,
      "eval_runtime": 183.2032,
      "eval_samples_per_second": 1.135,
      "eval_steps_per_second": 0.568,
      "step": 4800
    },
    {
      "epoch": 1.9095869056897894,
      "grad_norm": 0.06283093243837357,
      "learning_rate": 0.00015300009108171347,
      "logits/chosen": -19.286584854125977,
      "logits/rejected": -18.341442108154297,
      "logps/chosen": -376.3005065917969,
      "logps/rejected": -383.4756774902344,
      "loss": 0.0044,
      "rewards/accuracies": 0.9987499713897705,
      "rewards/chosen": -1.3465908765792847,
      "rewards/margins": 11.345978736877441,
      "rewards/rejected": -12.692569732666016,
      "step": 4900
    },
    {
      "epoch": 1.9485580670303975,
      "grad_norm": 0.04220004007220268,
      "learning_rate": 0.00014343387588942392,
      "logits/chosen": -19.211626052856445,
      "logits/rejected": -18.220531463623047,
      "logps/chosen": -389.9902648925781,
      "logps/rejected": -397.5294494628906,
      "loss": 0.0057,
      "rewards/accuracies": 0.9962499737739563,
      "rewards/chosen": -1.984826922416687,
      "rewards/margins": 11.562482833862305,
      "rewards/rejected": -13.547309875488281,
      "step": 5000
    },
    {
      "epoch": 1.9485580670303975,
      "eval_logits/chosen": -15.311283111572266,
      "eval_logits/rejected": -15.284743309020996,
      "eval_logps/chosen": -350.1474304199219,
      "eval_logps/rejected": -386.7135314941406,
      "eval_loss": 0.00032912034657783806,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": -0.8488726019859314,
      "eval_rewards/margins": 11.131490707397461,
      "eval_rewards/rejected": -11.980363845825195,
      "eval_runtime": 183.6747,
      "eval_samples_per_second": 1.132,
      "eval_steps_per_second": 0.566,
      "step": 5000
    },
    {
      "epoch": 1.9875292283710055,
      "grad_norm": 0.05133282020688057,
      "learning_rate": 0.00013405471362391068,
      "logits/chosen": -19.146968841552734,
      "logits/rejected": -18.36100196838379,
      "logps/chosen": -375.5859680175781,
      "logps/rejected": -405.4987487792969,
      "loss": 0.0055,
      "rewards/accuracies": 0.9962499737739563,
      "rewards/chosen": -2.0062613487243652,
      "rewards/margins": 11.725434303283691,
      "rewards/rejected": -13.731695175170898,
      "step": 5100
    },
    {
      "epoch": 2.0265003897116136,
      "grad_norm": 0.003434022655710578,
      "learning_rate": 0.00012487906730066888,
      "logits/chosen": -19.30167579650879,
      "logits/rejected": -18.223770141601562,
      "logps/chosen": -376.9130859375,
      "logps/rejected": -380.2550354003906,
      "loss": 0.0025,
      "rewards/accuracies": 0.9975000023841858,
      "rewards/chosen": -1.750290870666504,
      "rewards/margins": 11.900991439819336,
      "rewards/rejected": -13.651283264160156,
      "step": 5200
    },
    {
      "epoch": 2.0265003897116136,
      "eval_logits/chosen": -15.498884201049805,
      "eval_logits/rejected": -15.481465339660645,
      "eval_logps/chosen": -349.6870422363281,
      "eval_logps/rejected": -385.3147888183594,
      "eval_loss": 0.0003257237549405545,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": -0.8028372526168823,
      "eval_rewards/margins": 11.037657737731934,
      "eval_rewards/rejected": -11.840494155883789,
      "eval_runtime": 183.7823,
      "eval_samples_per_second": 1.132,
      "eval_steps_per_second": 0.566,
      "step": 5200
    },
    {
      "epoch": 2.0654715510522212,
      "grad_norm": 0.026280131191015244,
      "learning_rate": 0.00011592304270860795,
      "logits/chosen": -19.42230796813965,
      "logits/rejected": -18.394390106201172,
      "logps/chosen": -400.98980712890625,
      "logps/rejected": -409.95098876953125,
      "loss": 0.005,
      "rewards/accuracies": 0.9962499737739563,
      "rewards/chosen": -2.090318202972412,
      "rewards/margins": 11.665027618408203,
      "rewards/rejected": -13.755346298217773,
      "step": 5300
    },
    {
      "epoch": 2.1044427123928293,
      "grad_norm": 0.00010202277189819142,
      "learning_rate": 0.00010720236013995221,
      "logits/chosen": -19.477603912353516,
      "logits/rejected": -18.485740661621094,
      "logps/chosen": -365.8956298828125,
      "logps/rejected": -397.5658264160156,
      "loss": 0.0015,
      "rewards/accuracies": 0.9987499713897705,
      "rewards/chosen": -1.8521977663040161,
      "rewards/margins": 12.37298583984375,
      "rewards/rejected": -14.225183486938477,
      "step": 5400
    },
    {
      "epoch": 2.1044427123928293,
      "eval_logits/chosen": -15.460063934326172,
      "eval_logits/rejected": -15.440217018127441,
      "eval_logps/chosen": -351.2416687011719,
      "eval_logps/rejected": -389.2104797363281,
      "eval_loss": 0.00034331483766436577,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": -0.958295464515686,
      "eval_rewards/margins": 11.27176570892334,
      "eval_rewards/rejected": -12.230061531066895,
      "eval_runtime": 183.6747,
      "eval_samples_per_second": 1.132,
      "eval_steps_per_second": 0.566,
      "step": 5400
    },
    {
      "epoch": 2.1434138737334374,
      "grad_norm": 0.005236598663032055,
      "learning_rate": 9.873232679679392e-05,
      "logits/chosen": -19.37502670288086,
      "logits/rejected": -18.3719482421875,
      "logps/chosen": -381.2720947265625,
      "logps/rejected": -403.01593017578125,
      "loss": 0.0024,
      "rewards/accuracies": 0.9975000023841858,
      "rewards/chosen": -2.047292709350586,
      "rewards/margins": 12.199724197387695,
      "rewards/rejected": -14.247018814086914,
      "step": 5500
    },
    {
      "epoch": 2.182385035074045,
      "grad_norm": 0.029128100723028183,
      "learning_rate": 9.052780992273379e-05,
      "logits/chosen": -19.223974227905273,
      "logits/rejected": -18.315275192260742,
      "logps/chosen": -378.5389404296875,
      "logps/rejected": -399.8840637207031,
      "loss": 0.0021,
      "rewards/accuracies": 0.9975000023841858,
      "rewards/chosen": -1.8436551094055176,
      "rewards/margins": 12.235860824584961,
      "rewards/rejected": -14.07951545715332,
      "step": 5600
    },
    {
      "epoch": 2.182385035074045,
      "eval_logits/chosen": -15.46290111541748,
      "eval_logits/rejected": -15.436098098754883,
      "eval_logps/chosen": -349.15423583984375,
      "eval_logps/rejected": -387.6983947753906,
      "eval_loss": 0.00030186420190148056,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": -0.7495561242103577,
      "eval_rewards/margins": 11.329290390014648,
      "eval_rewards/rejected": -12.078847885131836,
      "eval_runtime": 183.2993,
      "eval_samples_per_second": 1.135,
      "eval_steps_per_second": 0.567,
      "step": 5600
    },
    {
      "epoch": 2.221356196414653,
      "grad_norm": 0.011133521795272827,
      "learning_rate": 8.26032107067696e-05,
      "logits/chosen": -19.451305389404297,
      "logits/rejected": -18.38578224182129,
      "logps/chosen": -374.8850402832031,
      "logps/rejected": -388.83575439453125,
      "loss": 0.0014,
      "rewards/accuracies": 0.9987499713897705,
      "rewards/chosen": -1.683659315109253,
      "rewards/margins": 12.163559913635254,
      "rewards/rejected": -13.847216606140137,
      "step": 5700
    },
    {
      "epoch": 2.260327357755261,
      "grad_norm": 7.387495134025812e-05,
      "learning_rate": 7.497243900523937e-05,
      "logits/chosen": -19.474388122558594,
      "logits/rejected": -18.39251708984375,
      "logps/chosen": -382.3981628417969,
      "logps/rejected": -390.441162109375,
      "loss": 0.0017,
      "rewards/accuracies": 0.9987499713897705,
      "rewards/chosen": -1.4145100116729736,
      "rewards/margins": 11.843048095703125,
      "rewards/rejected": -13.257555961608887,
      "step": 5800
    },
    {
      "epoch": 2.260327357755261,
      "eval_logits/chosen": -15.475095748901367,
      "eval_logits/rejected": -15.438820838928223,
      "eval_logps/chosen": -347.4687805175781,
      "eval_logps/rejected": -385.78497314453125,
      "eval_loss": 0.0002713102730922401,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": -0.5810099244117737,
      "eval_rewards/margins": 11.306499481201172,
      "eval_rewards/rejected": -11.887508392333984,
      "eval_runtime": 183.4323,
      "eval_samples_per_second": 1.134,
      "eval_steps_per_second": 0.567,
      "step": 5800
    },
    {
      "epoch": 2.2992985190958692,
      "grad_norm": 0.003995438106358051,
      "learning_rate": 6.764888892618867e-05,
      "logits/chosen": -19.462810516357422,
      "logits/rejected": -18.417314529418945,
      "logps/chosen": -375.923583984375,
      "logps/rejected": -394.6671447753906,
      "loss": 0.0051,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -1.9151861667633057,
      "rewards/margins": 11.908782005310059,
      "rewards/rejected": -13.823966026306152,
      "step": 5900
    },
    {
      "epoch": 2.338269680436477,
      "grad_norm": 0.003454476362094283,
      "learning_rate": 6.064541531901838e-05,
      "logits/chosen": -19.41205406188965,
      "logits/rejected": -18.35369873046875,
      "logps/chosen": -374.3619384765625,
      "logps/rejected": -392.4114990234375,
      "loss": 0.004,
      "rewards/accuracies": 0.9962499737739563,
      "rewards/chosen": -2.0450844764709473,
      "rewards/margins": 11.8558988571167,
      "rewards/rejected": -13.900982856750488,
      "step": 6000
    },
    {
      "epoch": 2.338269680436477,
      "eval_logits/chosen": -15.477252960205078,
      "eval_logits/rejected": -15.445527076721191,
      "eval_logps/chosen": -347.4806213378906,
      "eval_logps/rejected": -386.5325927734375,
      "eval_loss": 0.00025467213708907366,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": -0.5821936130523682,
      "eval_rewards/margins": 11.38007926940918,
      "eval_rewards/rejected": -11.962273597717285,
      "eval_runtime": 183.2231,
      "eval_samples_per_second": 1.135,
      "eval_steps_per_second": 0.568,
      "step": 6000
    },
    {
      "epoch": 2.377240841777085,
      "grad_norm": 0.0025488168466836214,
      "learning_rate": 5.397431121068011e-05,
      "logits/chosen": -19.279691696166992,
      "logits/rejected": -18.215682983398438,
      "logps/chosen": -391.84844970703125,
      "logps/rejected": -398.8818359375,
      "loss": 0.003,
      "rewards/accuracies": 0.9962499737739563,
      "rewards/chosen": -2.172652006149292,
      "rewards/margins": 12.266448020935059,
      "rewards/rejected": -14.439099311828613,
      "step": 6100
    },
    {
      "epoch": 2.416212003117693,
      "grad_norm": 0.0001306094927713275,
      "learning_rate": 4.7647286228024944e-05,
      "logits/chosen": -19.264507293701172,
      "logits/rejected": -18.2991886138916,
      "logps/chosen": -368.3695373535156,
      "logps/rejected": -395.1746826171875,
      "loss": 0.0021,
      "rewards/accuracies": 0.9975000023841858,
      "rewards/chosen": -2.2215933799743652,
      "rewards/margins": 12.452881813049316,
      "rewards/rejected": -14.674474716186523,
      "step": 6200
    },
    {
      "epoch": 2.416212003117693,
      "eval_logits/chosen": -15.429259300231934,
      "eval_logits/rejected": -15.400547981262207,
      "eval_logps/chosen": -350.2945556640625,
      "eval_logps/rejected": -390.7493896484375,
      "eval_loss": 0.0002859699307009578,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": -0.8635876178741455,
      "eval_rewards/margins": 11.520365715026855,
      "eval_rewards/rejected": -12.383952140808105,
      "eval_runtime": 183.7971,
      "eval_samples_per_second": 1.132,
      "eval_steps_per_second": 0.566,
      "step": 6200
    },
    {
      "epoch": 2.455183164458301,
      "grad_norm": 0.0013745080213993788,
      "learning_rate": 4.167544604418003e-05,
      "logits/chosen": -19.316665649414062,
      "logits/rejected": -18.3441162109375,
      "logps/chosen": -378.2589416503906,
      "logps/rejected": -407.154052734375,
      "loss": 0.0012,
      "rewards/accuracies": 0.9987499713897705,
      "rewards/chosen": -2.0685226917266846,
      "rewards/margins": 12.507061958312988,
      "rewards/rejected": -14.575584411621094,
      "step": 6300
    },
    {
      "epoch": 2.4941543257989087,
      "grad_norm": 0.002951019676402211,
      "learning_rate": 3.6069272885030256e-05,
      "logits/chosen": -19.417644500732422,
      "logits/rejected": -18.45380210876465,
      "logps/chosen": -384.16253662109375,
      "logps/rejected": -404.4368896484375,
      "loss": 0.0038,
      "rewards/accuracies": 0.9975000023841858,
      "rewards/chosen": -2.470909357070923,
      "rewards/margins": 12.393091201782227,
      "rewards/rejected": -14.864001274108887,
      "step": 6400
    },
    {
      "epoch": 2.4941543257989087,
      "eval_logits/chosen": -15.429072380065918,
      "eval_logits/rejected": -15.39948844909668,
      "eval_logps/chosen": -350.1909484863281,
      "eval_logps/rejected": -391.09564208984375,
      "eval_loss": 0.00028344389284029603,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": -0.8532273769378662,
      "eval_rewards/margins": 11.565349578857422,
      "eval_rewards/rejected": -12.418577194213867,
      "eval_runtime": 183.8395,
      "eval_samples_per_second": 1.131,
      "eval_steps_per_second": 0.566,
      "step": 6400
    },
    {
      "epoch": 2.533125487139517,
      "grad_norm": 0.011729140765964985,
      "learning_rate": 3.083860713002276e-05,
      "logits/chosen": -19.238452911376953,
      "logits/rejected": -18.347043991088867,
      "logps/chosen": -370.990478515625,
      "logps/rejected": -400.6825256347656,
      "loss": 0.0039,
      "rewards/accuracies": 0.9950000047683716,
      "rewards/chosen": -2.3387069702148438,
      "rewards/margins": 12.353742599487305,
      "rewards/rejected": -14.692447662353516,
      "step": 6500
    },
    {
      "epoch": 2.572096648480125,
      "grad_norm": 0.014655795879662037,
      "learning_rate": 2.5992630039587377e-05,
      "logits/chosen": -19.383378982543945,
      "logits/rejected": -18.424985885620117,
      "logps/chosen": -380.931396484375,
      "logps/rejected": -399.5527648925781,
      "loss": 0.004,
      "rewards/accuracies": 0.9950000047683716,
      "rewards/chosen": -2.246741771697998,
      "rewards/margins": 12.052864074707031,
      "rewards/rejected": -14.299607276916504,
      "step": 6600
    },
    {
      "epoch": 2.572096648480125,
      "eval_logits/chosen": -15.413681983947754,
      "eval_logits/rejected": -15.38673210144043,
      "eval_logps/chosen": -350.96832275390625,
      "eval_logps/rejected": -392.92138671875,
      "eval_loss": 0.0002713745925575495,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": -0.9309618473052979,
      "eval_rewards/margins": 11.670186996459961,
      "eval_rewards/rejected": -12.601149559020996,
      "eval_runtime": 183.927,
      "eval_samples_per_second": 1.131,
      "eval_steps_per_second": 0.565,
      "step": 6600
    },
    {
      "epoch": 2.6110678098207325,
      "grad_norm": 0.007622725795954466,
      "learning_rate": 2.153984763949371e-05,
      "logits/chosen": -19.36250114440918,
      "logits/rejected": -18.28182029724121,
      "logps/chosen": -383.64678955078125,
      "logps/rejected": -392.4764404296875,
      "loss": 0.0051,
      "rewards/accuracies": 0.9950000047683716,
      "rewards/chosen": -2.286069631576538,
      "rewards/margins": 12.199385643005371,
      "rewards/rejected": -14.485455513000488,
      "step": 6700
    },
    {
      "epoch": 2.6500389711613406,
      "grad_norm": 0.01298923883587122,
      "learning_rate": 1.7488075790430934e-05,
      "logits/chosen": -19.342336654663086,
      "logits/rejected": -18.265600204467773,
      "logps/chosen": -386.0182800292969,
      "logps/rejected": -406.8291015625,
      "loss": 0.0025,
      "rewards/accuracies": 0.9975000023841858,
      "rewards/chosen": -2.360260009765625,
      "rewards/margins": 12.328630447387695,
      "rewards/rejected": -14.68889045715332,
      "step": 6800
    },
    {
      "epoch": 2.6500389711613406,
      "eval_logits/chosen": -15.421009063720703,
      "eval_logits/rejected": -15.389812469482422,
      "eval_logps/chosen": -350.44403076171875,
      "eval_logps/rejected": -392.24505615234375,
      "eval_loss": 0.00027188131934963167,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": -0.8785340785980225,
      "eval_rewards/margins": 11.654979705810547,
      "eval_rewards/rejected": -12.533516883850098,
      "eval_runtime": 183.8793,
      "eval_samples_per_second": 1.131,
      "eval_steps_per_second": 0.566,
      "step": 6800
    },
    {
      "epoch": 2.6890101325019486,
      "grad_norm": 0.001023565884679556,
      "learning_rate": 1.3844426469017707e-05,
      "logits/chosen": -19.42437171936035,
      "logits/rejected": -18.330951690673828,
      "logps/chosen": -389.0,
      "logps/rejected": -406.2248840332031,
      "loss": 0.0029,
      "rewards/accuracies": 0.9962499737739563,
      "rewards/chosen": -2.3574695587158203,
      "rewards/margins": 12.130472183227539,
      "rewards/rejected": -14.48794174194336,
      "step": 6900
    },
    {
      "epoch": 2.7279812938425563,
      "grad_norm": 0.030132969841361046,
      "learning_rate": 1.061529528432198e-05,
      "logits/chosen": -19.441736221313477,
      "logits/rejected": -18.430885314941406,
      "logps/chosen": -375.5260009765625,
      "logps/rejected": -398.24298095703125,
      "loss": 0.0012,
      "rewards/accuracies": 0.9987499713897705,
      "rewards/chosen": -2.37859845161438,
      "rewards/margins": 12.304988861083984,
      "rewards/rejected": -14.683588981628418,
      "step": 7000
    },
    {
      "epoch": 2.7279812938425563,
      "eval_logits/chosen": -15.412691116333008,
      "eval_logits/rejected": -15.382089614868164,
      "eval_logps/chosen": -351.04168701171875,
      "eval_logps/rejected": -393.3232421875,
      "eval_loss": 0.00028384948382154107,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": -0.9383015036582947,
      "eval_rewards/margins": 11.703031539916992,
      "eval_rewards/rejected": -12.641332626342773,
      "eval_runtime": 183.1659,
      "eval_samples_per_second": 1.136,
      "eval_steps_per_second": 0.568,
      "step": 7000
    },
    {
      "epoch": 2.7669524551831643,
      "grad_norm": 0.29376310110092163,
      "learning_rate": 7.806350251804484e-06,
      "logits/chosen": -19.37668228149414,
      "logits/rejected": -18.328815460205078,
      "logps/chosen": -368.5742492675781,
      "logps/rejected": -397.5633544921875,
      "loss": 0.003,
      "rewards/accuracies": 0.9962499737739563,
      "rewards/chosen": -2.3045263290405273,
      "rewards/margins": 12.480491638183594,
      "rewards/rejected": -14.785019874572754,
      "step": 7100
    },
    {
      "epoch": 2.8059236165237724,
      "grad_norm": 0.002321546198800206,
      "learning_rate": 5.422521844388683e-06,
      "logits/chosen": -19.359325408935547,
      "logits/rejected": -18.383474349975586,
      "logps/chosen": -373.9019470214844,
      "logps/rejected": -399.3661193847656,
      "loss": 0.0047,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -2.3699896335601807,
      "rewards/margins": 12.351306915283203,
      "rewards/rejected": -14.721295356750488,
      "step": 7200
    },
    {
      "epoch": 2.8059236165237724,
      "eval_logits/chosen": -15.414888381958008,
      "eval_logits/rejected": -15.384535789489746,
      "eval_logps/chosen": -350.7855224609375,
      "eval_logps/rejected": -393.17767333984375,
      "eval_loss": 0.00028104818193241954,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": -0.9126843214035034,
      "eval_rewards/margins": 11.714097023010254,
      "eval_rewards/rejected": -12.626781463623047,
      "eval_runtime": 183.8851,
      "eval_samples_per_second": 1.131,
      "eval_steps_per_second": 0.566,
      "step": 7200
    },
    {
      "epoch": 2.8448947778643805,
      "grad_norm": 0.027766738086938858,
      "learning_rate": 3.4679943381216438e-06,
      "logits/chosen": -19.325197219848633,
      "logits/rejected": -18.378944396972656,
      "logps/chosen": -379.8866271972656,
      "logps/rejected": -401.4820251464844,
      "loss": 0.0004,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.4111533164978027,
      "rewards/margins": 12.260860443115234,
      "rewards/rejected": -14.672014236450195,
      "step": 7300
    },
    {
      "epoch": 2.8838659392049886,
      "grad_norm": 0.0003760048421099782,
      "learning_rate": 1.9461984676158727e-06,
      "logits/chosen": -19.34538459777832,
      "logits/rejected": -18.292407989501953,
      "logps/chosen": -386.7870788574219,
      "logps/rejected": -399.7149353027344,
      "loss": 0.003,
      "rewards/accuracies": 0.9975000023841858,
      "rewards/chosen": -2.498373508453369,
      "rewards/margins": 12.094259262084961,
      "rewards/rejected": -14.592632293701172,
      "step": 7400
    },
    {
      "epoch": 2.8838659392049886,
      "eval_logits/chosen": -15.413555145263672,
      "eval_logits/rejected": -15.383076667785645,
      "eval_logps/chosen": -350.8740539550781,
      "eval_logps/rejected": -393.42462158203125,
      "eval_loss": 0.0002813572355080396,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": -0.9215376377105713,
      "eval_rewards/margins": 11.729934692382812,
      "eval_rewards/rejected": -12.651473045349121,
      "eval_runtime": 184.2047,
      "eval_samples_per_second": 1.129,
      "eval_steps_per_second": 0.565,
      "step": 7400
    },
    {
      "epoch": 2.922837100545596,
      "grad_norm": 0.0007408323581330478,
      "learning_rate": 8.598054041644155e-07,
      "logits/chosen": -19.447223663330078,
      "logits/rejected": -18.43070411682129,
      "logps/chosen": -382.068359375,
      "logps/rejected": -402.309814453125,
      "loss": 0.002,
      "rewards/accuracies": 0.9987499713897705,
      "rewards/chosen": -2.2234978675842285,
      "rewards/margins": 12.309508323669434,
      "rewards/rejected": -14.533007621765137,
      "step": 7500
    },
    {
      "epoch": 2.9618082618862043,
      "grad_norm": 0.0027539017610251904,
      "learning_rate": 2.107220670987675e-07,
      "logits/chosen": -19.346935272216797,
      "logits/rejected": -18.29763412475586,
      "logps/chosen": -388.3053894042969,
      "logps/rejected": -401.0611572265625,
      "loss": 0.003,
      "rewards/accuracies": 0.9975000023841858,
      "rewards/chosen": -2.2870028018951416,
      "rewards/margins": 12.200346946716309,
      "rewards/rejected": -14.487349510192871,
      "step": 7600
    },
    {
      "epoch": 2.9618082618862043,
      "eval_logits/chosen": -15.41310977935791,
      "eval_logits/rejected": -15.382636070251465,
      "eval_logps/chosen": -350.8951721191406,
      "eval_logps/rejected": -393.48431396484375,
      "eval_loss": 0.0002806605480145663,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": -0.9236502051353455,
      "eval_rewards/margins": 11.733796119689941,
      "eval_rewards/rejected": -12.657448768615723,
      "eval_runtime": 183.4198,
      "eval_samples_per_second": 1.134,
      "eval_steps_per_second": 0.567,
      "step": 7600
    }
  ],
  "logging_steps": 100,
  "max_steps": 7698,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 3,
  "save_steps": 200,
  "total_flos": 0.0,
  "train_batch_size": 2,
  "trial_name": null,
  "trial_params": null
}