{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 0.9995965030262273,
  "eval_steps": 300,
  "global_step": 1858,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.0,
      "learning_rate": 2.6881720430107528e-09,
      "logits/chosen": -2.670260429382324,
      "logits/rejected": -2.1533777713775635,
      "logps/chosen": -299.33551025390625,
      "logps/rejected": -186.81130981445312,
      "loss": 13125.0,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "rewards/safe_rewards": 0.0,
      "rewards/unsafe_rewards": 0.0,
      "step": 1
    },
    {
      "epoch": 0.01,
      "learning_rate": 2.6881720430107527e-08,
      "logits/chosen": -2.4763858318328857,
      "logits/rejected": -2.354341745376587,
      "logps/chosen": -201.82504272460938,
      "logps/rejected": -189.45822143554688,
      "loss": 14971.9097,
      "rewards/accuracies": 0.4930555522441864,
      "rewards/chosen": 9.199242413160391e-06,
      "rewards/margins": -8.218608854804188e-05,
      "rewards/rejected": 9.138535824604332e-05,
      "rewards/safe_rewards": 1.8223654478788376e-05,
      "rewards/unsafe_rewards": 1.7482994962847442e-07,
      "step": 10
    },
    {
      "epoch": 0.01,
      "learning_rate": 5.3763440860215054e-08,
      "logits/chosen": -2.44279408454895,
      "logits/rejected": -2.218726873397827,
      "logps/chosen": -226.34970092773438,
      "logps/rejected": -181.1803436279297,
      "loss": 15239.6516,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.0002221722388640046,
      "rewards/margins": -0.0001655549422139302,
      "rewards/rejected": -5.6617216614540666e-05,
      "rewards/safe_rewards": -0.0002462912234477699,
      "rewards/unsafe_rewards": -0.0001980532251764089,
      "step": 20
    },
    {
      "epoch": 0.02,
      "learning_rate": 8.064516129032257e-08,
      "logits/chosen": -2.4326512813568115,
      "logits/rejected": -2.2918035984039307,
      "logps/chosen": -215.0783233642578,
      "logps/rejected": -189.28309631347656,
      "loss": 14993.0922,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.0004244670271873474,
      "rewards/margins": -0.000839566346257925,
      "rewards/rejected": 0.0004150994645897299,
      "rewards/safe_rewards": -0.0002224749478045851,
      "rewards/unsafe_rewards": -0.0006264590774662793,
      "step": 30
    },
    {
      "epoch": 0.02,
      "learning_rate": 1.0752688172043011e-07,
      "logits/chosen": -2.375096082687378,
      "logits/rejected": -2.157045841217041,
      "logps/chosen": -180.7657470703125,
      "logps/rejected": -173.87054443359375,
      "loss": 14723.7563,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -0.00011304272629786283,
      "rewards/margins": 0.0008593280799686909,
      "rewards/rejected": -0.0009723707917146385,
      "rewards/safe_rewards": 0.0004990470479242504,
      "rewards/unsafe_rewards": -0.0007251326460391283,
      "step": 40
    },
    {
      "epoch": 0.03,
      "learning_rate": 1.3440860215053762e-07,
      "logits/chosen": -2.489865303039551,
      "logits/rejected": -2.1804325580596924,
      "logps/chosen": -209.59664916992188,
      "logps/rejected": -167.6984100341797,
      "loss": 14816.0938,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": 0.0011160348076373339,
      "rewards/margins": 0.0046949503012001514,
      "rewards/rejected": -0.0035789154935628176,
      "rewards/safe_rewards": 0.00042248546378687024,
      "rewards/unsafe_rewards": 0.0018095843261107802,
      "step": 50
    },
    {
      "epoch": 0.03,
      "learning_rate": 1.6129032258064515e-07,
      "logits/chosen": -2.4289777278900146,
      "logits/rejected": -2.2897238731384277,
      "logps/chosen": -185.92630004882812,
      "logps/rejected": -185.1912841796875,
      "loss": 14593.3875,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -0.0020237788558006287,
      "rewards/margins": 0.002217882312834263,
      "rewards/rejected": -0.004241660702973604,
      "rewards/safe_rewards": -0.001639070687815547,
      "rewards/unsafe_rewards": -0.002408486558124423,
      "step": 60
    },
    {
      "epoch": 0.04,
      "learning_rate": 1.8817204301075268e-07,
      "logits/chosen": -2.423598289489746,
      "logits/rejected": -2.2320618629455566,
      "logps/chosen": -202.7837677001953,
      "logps/rejected": -184.62033081054688,
      "loss": 14276.2172,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -0.005977807100862265,
      "rewards/margins": 0.005661749746650457,
      "rewards/rejected": -0.011639557778835297,
      "rewards/safe_rewards": -0.00685838982462883,
      "rewards/unsafe_rewards": -0.005097225774079561,
      "step": 70
    },
    {
      "epoch": 0.04,
      "learning_rate": 2.1505376344086022e-07,
      "logits/chosen": -2.4342398643493652,
      "logits/rejected": -2.271530866622925,
      "logps/chosen": -221.3511505126953,
      "logps/rejected": -196.20684814453125,
      "loss": 14430.5766,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.017918918281793594,
      "rewards/margins": 0.013557764701545238,
      "rewards/rejected": -0.03147668391466141,
      "rewards/safe_rewards": -0.020387938246130943,
      "rewards/unsafe_rewards": -0.015449894592165947,
      "step": 80
    },
    {
      "epoch": 0.05,
      "learning_rate": 2.4193548387096775e-07,
      "logits/chosen": -2.4415786266326904,
      "logits/rejected": -2.301713466644287,
      "logps/chosen": -211.0481414794922,
      "logps/rejected": -171.99159240722656,
      "loss": 14501.5938,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.05115891247987747,
      "rewards/margins": 0.025721188634634018,
      "rewards/rejected": -0.07688009738922119,
      "rewards/safe_rewards": -0.04981667920947075,
      "rewards/unsafe_rewards": -0.05250114947557449,
      "step": 90
    },
    {
      "epoch": 0.05,
      "learning_rate": 2.6881720430107523e-07,
      "logits/chosen": -2.4574618339538574,
      "logits/rejected": -2.3291263580322266,
      "logps/chosen": -204.63087463378906,
      "logps/rejected": -179.63002014160156,
      "loss": 14364.2516,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.08542316406965256,
      "rewards/margins": 0.03905550017952919,
      "rewards/rejected": -0.12447866052389145,
      "rewards/safe_rewards": -0.08581903576850891,
      "rewards/unsafe_rewards": -0.0850272923707962,
      "step": 100
    },
    {
      "epoch": 0.06,
      "learning_rate": 2.956989247311828e-07,
      "logits/chosen": -2.429097890853882,
      "logits/rejected": -2.2684311866760254,
      "logps/chosen": -222.95913696289062,
      "logps/rejected": -187.91293334960938,
      "loss": 14761.6688,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.11019313335418701,
      "rewards/margins": 0.048916272819042206,
      "rewards/rejected": -0.159109428524971,
      "rewards/safe_rewards": -0.10632093250751495,
      "rewards/unsafe_rewards": -0.11406532675027847,
      "step": 110
    },
    {
      "epoch": 0.06,
      "learning_rate": 3.225806451612903e-07,
      "logits/chosen": -2.4783873558044434,
      "logits/rejected": -2.262389659881592,
      "logps/chosen": -220.35800170898438,
      "logps/rejected": -169.4020233154297,
      "loss": 13980.0906,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.09087369590997696,
      "rewards/margins": 0.07458756119012833,
      "rewards/rejected": -0.1654612421989441,
      "rewards/safe_rewards": -0.08417809009552002,
      "rewards/unsafe_rewards": -0.0975693017244339,
      "step": 120
    },
    {
      "epoch": 0.07,
      "learning_rate": 3.4946236559139783e-07,
      "logits/chosen": -2.47499942779541,
      "logits/rejected": -2.3391237258911133,
      "logps/chosen": -224.6880645751953,
      "logps/rejected": -181.26388549804688,
      "loss": 13439.1469,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.10466276109218597,
      "rewards/margins": 0.0889045000076294,
      "rewards/rejected": -0.19356727600097656,
      "rewards/safe_rewards": -0.07197071611881256,
      "rewards/unsafe_rewards": -0.13735483586788177,
      "step": 130
    },
    {
      "epoch": 0.08,
      "learning_rate": 3.7634408602150537e-07,
      "logits/chosen": -2.5111544132232666,
      "logits/rejected": -2.323676347732544,
      "logps/chosen": -232.02120971679688,
      "logps/rejected": -207.3649444580078,
      "loss": 13156.9813,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.20595864951610565,
      "rewards/margins": 0.0673115998506546,
      "rewards/rejected": -0.27327024936676025,
      "rewards/safe_rewards": -0.19269119203090668,
      "rewards/unsafe_rewards": -0.21922609210014343,
      "step": 140
    },
    {
      "epoch": 0.08,
      "learning_rate": 4.0322580645161285e-07,
      "logits/chosen": -2.440796375274658,
      "logits/rejected": -2.294173002243042,
      "logps/chosen": -229.3905792236328,
      "logps/rejected": -201.4041748046875,
      "loss": 12286.168,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.2244747132062912,
      "rewards/margins": 0.09208513051271439,
      "rewards/rejected": -0.3165598511695862,
      "rewards/safe_rewards": -0.2293473184108734,
      "rewards/unsafe_rewards": -0.21960210800170898,
      "step": 150
    },
    {
      "epoch": 0.09,
      "learning_rate": 4.3010752688172043e-07,
      "logits/chosen": -2.4001574516296387,
      "logits/rejected": -2.2244515419006348,
      "logps/chosen": -242.0061492919922,
      "logps/rejected": -220.7064666748047,
      "loss": 13450.5672,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.2182481735944748,
      "rewards/margins": 0.09150904417037964,
      "rewards/rejected": -0.30975720286369324,
      "rewards/safe_rewards": -0.20791450142860413,
      "rewards/unsafe_rewards": -0.22858186066150665,
      "step": 160
    },
    {
      "epoch": 0.09,
      "learning_rate": 4.569892473118279e-07,
      "logits/chosen": -2.4316458702087402,
      "logits/rejected": -2.3187336921691895,
      "logps/chosen": -229.7932891845703,
      "logps/rejected": -191.04086303710938,
      "loss": 12580.4609,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.15405751764774323,
      "rewards/margins": 0.08396536856889725,
      "rewards/rejected": -0.23802292346954346,
      "rewards/safe_rewards": -0.13653624057769775,
      "rewards/unsafe_rewards": -0.1715788096189499,
      "step": 170
    },
    {
      "epoch": 0.1,
      "learning_rate": 4.838709677419355e-07,
      "logits/chosen": -2.473118305206299,
      "logits/rejected": -2.3191967010498047,
      "logps/chosen": -217.89785766601562,
      "logps/rejected": -209.47775268554688,
      "loss": 12886.1875,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.2591598629951477,
      "rewards/margins": 0.1223045364022255,
      "rewards/rejected": -0.3814643919467926,
      "rewards/safe_rewards": -0.2669682800769806,
      "rewards/unsafe_rewards": -0.25135138630867004,
      "step": 180
    },
    {
      "epoch": 0.1,
      "learning_rate": 4.999929391798331e-07,
      "logits/chosen": -2.4944000244140625,
      "logits/rejected": -2.323819637298584,
      "logps/chosen": -224.4022674560547,
      "logps/rejected": -209.09341430664062,
      "loss": 12964.0625,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.30063313245773315,
      "rewards/margins": 0.11855147778987885,
      "rewards/rejected": -0.4191845953464508,
      "rewards/safe_rewards": -0.30268028378486633,
      "rewards/unsafe_rewards": -0.2985859215259552,
      "step": 190
    },
    {
      "epoch": 0.11,
      "learning_rate": 4.9991350953333e-07,
      "logits/chosen": -2.4175021648406982,
      "logits/rejected": -2.261751413345337,
      "logps/chosen": -257.97100830078125,
      "logps/rejected": -248.92697143554688,
      "loss": 12408.5,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.3828560709953308,
      "rewards/margins": 0.09374293684959412,
      "rewards/rejected": -0.4765990376472473,
      "rewards/safe_rewards": -0.37296319007873535,
      "rewards/unsafe_rewards": -0.39274901151657104,
      "step": 200
    },
    {
      "epoch": 0.11,
      "learning_rate": 4.997458523498236e-07,
      "logits/chosen": -2.4530012607574463,
      "logits/rejected": -2.3131752014160156,
      "logps/chosen": -222.4829559326172,
      "logps/rejected": -198.89447021484375,
      "loss": 12109.7687,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.3033100664615631,
      "rewards/margins": 0.09060867875814438,
      "rewards/rejected": -0.3939187526702881,
      "rewards/safe_rewards": -0.29972246289253235,
      "rewards/unsafe_rewards": -0.30689769983291626,
      "step": 210
    },
    {
      "epoch": 0.12,
      "learning_rate": 4.99490026817712e-07,
      "logits/chosen": -2.415255308151245,
      "logits/rejected": -2.290353775024414,
      "logps/chosen": -225.1759490966797,
      "logps/rejected": -215.8201446533203,
      "loss": 12286.7969,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.308417946100235,
      "rewards/margins": 0.15336203575134277,
      "rewards/rejected": -0.46177998185157776,
      "rewards/safe_rewards": -0.2879489064216614,
      "rewards/unsafe_rewards": -0.3288869559764862,
      "step": 220
    },
    {
      "epoch": 0.12,
      "learning_rate": 4.991461232516674e-07,
      "logits/chosen": -2.403742551803589,
      "logits/rejected": -2.2323994636535645,
      "logps/chosen": -265.6297607421875,
      "logps/rejected": -247.0828399658203,
      "loss": 13271.4391,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.35555627942085266,
      "rewards/margins": 0.13768498599529266,
      "rewards/rejected": -0.49324122071266174,
      "rewards/safe_rewards": -0.34540650248527527,
      "rewards/unsafe_rewards": -0.36570602655410767,
      "step": 230
    },
    {
      "epoch": 0.13,
      "learning_rate": 4.98714263060751e-07,
      "logits/chosen": -2.5071444511413574,
      "logits/rejected": -2.31742525100708,
      "logps/chosen": -209.05905151367188,
      "logps/rejected": -179.74855041503906,
      "loss": 12854.6859,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.2065323293209076,
      "rewards/margins": 0.12446670234203339,
      "rewards/rejected": -0.33099907636642456,
      "rewards/safe_rewards": -0.21961939334869385,
      "rewards/unsafe_rewards": -0.19344526529312134,
      "step": 240
    },
    {
      "epoch": 0.13,
      "learning_rate": 4.98194598705552e-07,
      "logits/chosen": -2.460813045501709,
      "logits/rejected": -2.3778514862060547,
      "logps/chosen": -238.38693237304688,
      "logps/rejected": -223.18795776367188,
      "loss": 13078.5102,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.48650604486465454,
      "rewards/margins": 0.07239247858524323,
      "rewards/rejected": -0.5588985681533813,
      "rewards/safe_rewards": -0.4960268437862396,
      "rewards/unsafe_rewards": -0.47698527574539185,
      "step": 250
    },
    {
      "epoch": 0.14,
      "learning_rate": 4.975873136443648e-07,
      "logits/chosen": -2.5023112297058105,
      "logits/rejected": -2.350739002227783,
      "logps/chosen": -268.8946228027344,
      "logps/rejected": -247.2296600341797,
      "loss": 11995.3344,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.3785567581653595,
      "rewards/margins": 0.12816599011421204,
      "rewards/rejected": -0.5067228078842163,
      "rewards/safe_rewards": -0.39122840762138367,
      "rewards/unsafe_rewards": -0.36588507890701294,
      "step": 260
    },
    {
      "epoch": 0.15,
      "learning_rate": 4.968926222684212e-07,
      "logits/chosen": -2.4438796043395996,
      "logits/rejected": -2.3461410999298096,
      "logps/chosen": -229.61386108398438,
      "logps/rejected": -223.92630004882812,
      "loss": 12123.7047,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.294075608253479,
      "rewards/margins": 0.13579340279102325,
      "rewards/rejected": -0.42986902594566345,
      "rewards/safe_rewards": -0.300296813249588,
      "rewards/unsafe_rewards": -0.2878544330596924,
      "step": 270
    },
    {
      "epoch": 0.15,
      "learning_rate": 4.961107698262044e-07,
      "logits/chosen": -2.4015512466430664,
      "logits/rejected": -2.263901948928833,
      "logps/chosen": -255.80538940429688,
      "logps/rejected": -228.9020233154297,
      "loss": 12352.2758,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.375516414642334,
      "rewards/margins": 0.10626471042633057,
      "rewards/rejected": -0.48178109526634216,
      "rewards/safe_rewards": -0.37126559019088745,
      "rewards/unsafe_rewards": -0.37976714968681335,
      "step": 280
    },
    {
      "epoch": 0.16,
      "learning_rate": 4.952420323368673e-07,
      "logits/chosen": -2.4498801231384277,
      "logits/rejected": -2.382075548171997,
      "logps/chosen": -239.36386108398438,
      "logps/rejected": -241.67861938476562,
      "loss": 12562.6453,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.47904032468795776,
      "rewards/margins": 0.12548959255218506,
      "rewards/rejected": -0.6045299172401428,
      "rewards/safe_rewards": -0.4896617829799652,
      "rewards/unsafe_rewards": -0.46841883659362793,
      "step": 290
    },
    {
      "epoch": 0.16,
      "learning_rate": 4.942867164927899e-07,
      "logits/chosen": -2.448484182357788,
      "logits/rejected": -2.3499300479888916,
      "logps/chosen": -233.24447631835938,
      "logps/rejected": -213.6923828125,
      "loss": 13096.7359,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.3095199167728424,
      "rewards/margins": 0.13528046011924744,
      "rewards/rejected": -0.44480031728744507,
      "rewards/safe_rewards": -0.2981899082660675,
      "rewards/unsafe_rewards": -0.3208498954772949,
      "step": 300
    },
    {
      "epoch": 0.16,
      "eval_logits/chosen": -2.319474458694458,
      "eval_logits/rejected": -2.1815290451049805,
      "eval_logps/chosen": -170.00274658203125,
      "eval_logps/rejected": -140.18304443359375,
      "eval_loss": 4529.67333984375,
      "eval_rewards/accuracies": 0.6583978533744812,
      "eval_rewards/chosen": -0.39572206139564514,
      "eval_rewards/margins": 0.08145187050104141,
      "eval_rewards/rejected": -0.4771738648414612,
      "eval_rewards/safe_rewards": -0.39296460151672363,
      "eval_rewards/unsafe_rewards": -0.3956325650215149,
      "eval_runtime": 996.6135,
      "eval_samples_per_second": 33.156,
      "eval_steps_per_second": 1.037,
      "step": 300
    },
    {
      "epoch": 0.17,
      "learning_rate": 4.932451595513062e-07,
      "logits/chosen": -2.464543342590332,
      "logits/rejected": -2.3167717456817627,
      "logps/chosen": -244.10421752929688,
      "logps/rejected": -229.1746368408203,
      "loss": 11442.1586,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.25108686089515686,
      "rewards/margins": 0.15766267478466034,
      "rewards/rejected": -0.408749520778656,
      "rewards/safe_rewards": -0.2550010681152344,
      "rewards/unsafe_rewards": -0.24717266857624054,
      "step": 310
    },
    {
      "epoch": 0.17,
      "learning_rate": 4.921177292156419e-07,
      "logits/chosen": -2.486952543258667,
      "logits/rejected": -2.317573070526123,
      "logps/chosen": -234.8188934326172,
      "logps/rejected": -228.7853240966797,
      "loss": 11535.2578,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.2879149317741394,
      "rewards/margins": 0.18317629396915436,
      "rewards/rejected": -0.47109121084213257,
      "rewards/safe_rewards": -0.2716086506843567,
      "rewards/unsafe_rewards": -0.30422115325927734,
      "step": 320
    },
    {
      "epoch": 0.18,
      "learning_rate": 4.909048235051033e-07,
      "logits/chosen": -2.408730983734131,
      "logits/rejected": -2.313384532928467,
      "logps/chosen": -240.1051483154297,
      "logps/rejected": -238.6286163330078,
      "loss": 11998.6938,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.4070356488227844,
      "rewards/margins": 0.15228822827339172,
      "rewards/rejected": -0.559323787689209,
      "rewards/safe_rewards": -0.38683491945266724,
      "rewards/unsafe_rewards": -0.42723625898361206,
      "step": 330
    },
    {
      "epoch": 0.18,
      "learning_rate": 4.896068706145631e-07,
      "logits/chosen": -2.430915355682373,
      "logits/rejected": -2.3069653511047363,
      "logps/chosen": -263.82440185546875,
      "logps/rejected": -228.1202392578125,
      "loss": 12415.2109,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.5314391255378723,
      "rewards/margins": 0.14294834434986115,
      "rewards/rejected": -0.674387514591217,
      "rewards/safe_rewards": -0.5249046087265015,
      "rewards/unsafe_rewards": -0.5379736423492432,
      "step": 340
    },
    {
      "epoch": 0.19,
      "learning_rate": 4.882243287632946e-07,
      "logits/chosen": -2.5057172775268555,
      "logits/rejected": -2.3827576637268066,
      "logps/chosen": -237.1790771484375,
      "logps/rejected": -237.27392578125,
      "loss": 11917.3391,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.44174814224243164,
      "rewards/margins": 0.1188526302576065,
      "rewards/rejected": -0.5606008172035217,
      "rewards/safe_rewards": -0.4454471170902252,
      "rewards/unsafe_rewards": -0.4380492568016052,
      "step": 350
    },
    {
      "epoch": 0.19,
      "learning_rate": 4.867576860332048e-07,
      "logits/chosen": -2.5187079906463623,
      "logits/rejected": -2.4128689765930176,
      "logps/chosen": -207.39404296875,
      "logps/rejected": -220.6639404296875,
      "loss": 11619.4062,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.353740930557251,
      "rewards/margins": 0.16230328381061554,
      "rewards/rejected": -0.5160442590713501,
      "rewards/safe_rewards": -0.37073829770088196,
      "rewards/unsafe_rewards": -0.33674362301826477,
      "step": 360
    },
    {
      "epoch": 0.2,
      "learning_rate": 4.85207460196526e-07,
      "logits/chosen": -2.5497612953186035,
      "logits/rejected": -2.4256086349487305,
      "logps/chosen": -251.84036254882812,
      "logps/rejected": -246.384033203125,
      "loss": 11846.7016,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.47489339113235474,
      "rewards/margins": 0.13903862237930298,
      "rewards/rejected": -0.6139320135116577,
      "rewards/safe_rewards": -0.4880181849002838,
      "rewards/unsafe_rewards": -0.46176856756210327,
      "step": 370
    },
    {
      "epoch": 0.2,
      "learning_rate": 4.835741985330259e-07,
      "logits/chosen": -2.5404162406921387,
      "logits/rejected": -2.4239704608917236,
      "logps/chosen": -246.53939819335938,
      "logps/rejected": -233.34353637695312,
      "loss": 11351.725,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.45782560110092163,
      "rewards/margins": 0.15662749111652374,
      "rewards/rejected": -0.614453136920929,
      "rewards/safe_rewards": -0.42890676856040955,
      "rewards/unsafe_rewards": -0.48674440383911133,
      "step": 380
    },
    {
      "epoch": 0.21,
      "learning_rate": 4.818584776367992e-07,
      "logits/chosen": -2.5068519115448,
      "logits/rejected": -2.4303596019744873,
      "logps/chosen": -244.3930206298828,
      "logps/rejected": -249.14187622070312,
      "loss": 11726.7609,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.40478819608688354,
      "rewards/margins": 0.16650350391864777,
      "rewards/rejected": -0.5712917447090149,
      "rewards/safe_rewards": -0.43173927068710327,
      "rewards/unsafe_rewards": -0.377837210893631,
      "step": 390
    },
    {
      "epoch": 0.22,
      "learning_rate": 4.800609032127122e-07,
      "logits/chosen": -2.4628491401672363,
      "logits/rejected": -2.374331474304199,
      "logps/chosen": -254.27822875976562,
      "logps/rejected": -228.73330688476562,
      "loss": 12046.5695,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.48730865120887756,
      "rewards/margins": 0.11220350116491318,
      "rewards/rejected": -0.5995121598243713,
      "rewards/safe_rewards": -0.4591635763645172,
      "rewards/unsafe_rewards": -0.5154536962509155,
      "step": 400
    },
    {
      "epoch": 0.22,
      "learning_rate": 4.78182109862569e-07,
      "logits/chosen": -2.4446866512298584,
      "logits/rejected": -2.408660650253296,
      "logps/chosen": -230.39208984375,
      "logps/rejected": -231.1884765625,
      "loss": 12854.2617,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.3896816670894623,
      "rewards/margins": 0.12279312312602997,
      "rewards/rejected": -0.5124748945236206,
      "rewards/safe_rewards": -0.35095858573913574,
      "rewards/unsafe_rewards": -0.4284047484397888,
      "step": 410
    },
    {
      "epoch": 0.23,
      "learning_rate": 4.7622276086107677e-07,
      "logits/chosen": -2.4900124073028564,
      "logits/rejected": -2.3827693462371826,
      "logps/chosen": -251.73007202148438,
      "logps/rejected": -243.5518341064453,
      "loss": 12053.4797,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.4146080017089844,
      "rewards/margins": 0.16079458594322205,
      "rewards/rejected": -0.5754026174545288,
      "rewards/safe_rewards": -0.4179013669490814,
      "rewards/unsafe_rewards": -0.41131457686424255,
      "step": 420
    },
    {
      "epoch": 0.23,
      "learning_rate": 4.741835479216879e-07,
      "logits/chosen": -2.445971727371216,
      "logits/rejected": -2.286686420440674,
      "logps/chosen": -294.9362487792969,
      "logps/rejected": -262.58404541015625,
      "loss": 11785.9859,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.48062044382095337,
      "rewards/margins": 0.16858412325382233,
      "rewards/rejected": -0.6492044925689697,
      "rewards/safe_rewards": -0.5107543468475342,
      "rewards/unsafe_rewards": -0.450486421585083,
      "step": 430
    },
    {
      "epoch": 0.24,
      "learning_rate": 4.720651909524036e-07,
      "logits/chosen": -2.3577795028686523,
      "logits/rejected": -2.2147059440612793,
      "logps/chosen": -236.0032196044922,
      "logps/rejected": -218.00820922851562,
      "loss": 11980.575,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.3994632661342621,
      "rewards/margins": 0.1237812489271164,
      "rewards/rejected": -0.5232445001602173,
      "rewards/safe_rewards": -0.4401687681674957,
      "rewards/unsafe_rewards": -0.3587578237056732,
      "step": 440
    },
    {
      "epoch": 0.24,
      "learning_rate": 4.698684378016222e-07,
      "logits/chosen": -2.2142910957336426,
      "logits/rejected": -2.0808310508728027,
      "logps/chosen": -249.1070556640625,
      "logps/rejected": -246.5348663330078,
      "loss": 12188.9062,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.6178480982780457,
      "rewards/margins": 0.13151735067367554,
      "rewards/rejected": -0.7493655681610107,
      "rewards/safe_rewards": -0.6169500350952148,
      "rewards/unsafe_rewards": -0.6187463402748108,
      "step": 450
    },
    {
      "epoch": 0.25,
      "learning_rate": 4.675940639941256e-07,
      "logits/chosen": -2.2552475929260254,
      "logits/rejected": -2.087047815322876,
      "logps/chosen": -268.5553283691406,
      "logps/rejected": -256.51849365234375,
      "loss": 11744.7563,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.6386964917182922,
      "rewards/margins": 0.15222086012363434,
      "rewards/rejected": -0.7909173965454102,
      "rewards/safe_rewards": -0.6151617765426636,
      "rewards/unsafe_rewards": -0.6622311472892761,
      "step": 460
    },
    {
      "epoch": 0.25,
      "learning_rate": 4.6524287245729286e-07,
      "logits/chosen": -2.2173266410827637,
      "logits/rejected": -2.0759875774383545,
      "logps/chosen": -247.02444458007812,
      "logps/rejected": -235.70431518554688,
      "loss": 11539.7117,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.5241425633430481,
      "rewards/margins": 0.1680615395307541,
      "rewards/rejected": -0.6922041177749634,
      "rewards/safe_rewards": -0.5475858449935913,
      "rewards/unsafe_rewards": -0.5006993412971497,
      "step": 470
    },
    {
      "epoch": 0.26,
      "learning_rate": 4.628156932376418e-07,
      "logits/chosen": -2.254505157470703,
      "logits/rejected": -2.057737112045288,
      "logps/chosen": -252.7125244140625,
      "logps/rejected": -227.06204223632812,
      "loss": 11626.5016,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.5887488126754761,
      "rewards/margins": 0.16838543117046356,
      "rewards/rejected": -0.7571342587471008,
      "rewards/safe_rewards": -0.5993281602859497,
      "rewards/unsafe_rewards": -0.5781695246696472,
      "step": 480
    },
    {
      "epoch": 0.26,
      "learning_rate": 4.603133832077953e-07,
      "logits/chosen": -2.2982699871063232,
      "logits/rejected": -2.1872057914733887,
      "logps/chosen": -293.5793762207031,
      "logps/rejected": -291.3560791015625,
      "loss": 11510.6367,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.609082043170929,
      "rewards/margins": 0.1590423583984375,
      "rewards/rejected": -0.7681244015693665,
      "rewards/safe_rewards": -0.6237068176269531,
      "rewards/unsafe_rewards": -0.5944572687149048,
      "step": 490
    },
    {
      "epoch": 0.27,
      "learning_rate": 4.5773682576397776e-07,
      "logits/chosen": -2.2795727252960205,
      "logits/rejected": -2.150864839553833,
      "logps/chosen": -247.63088989257812,
      "logps/rejected": -234.78408813476562,
      "loss": 11965.7203,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.47748714685440063,
      "rewards/margins": 0.13509830832481384,
      "rewards/rejected": -0.6125854849815369,
      "rewards/safe_rewards": -0.48745498061180115,
      "rewards/unsafe_rewards": -0.4675193428993225,
      "step": 500
    },
    {
      "epoch": 0.27,
      "learning_rate": 4.5508693051414774e-07,
      "logits/chosen": -2.3894762992858887,
      "logits/rejected": -2.2791035175323486,
      "logps/chosen": -240.63394165039062,
      "logps/rejected": -236.6797332763672,
      "loss": 11414.6078,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.4428652822971344,
      "rewards/margins": 0.16295991837978363,
      "rewards/rejected": -0.6058252453804016,
      "rewards/safe_rewards": -0.4437985420227051,
      "rewards/unsafe_rewards": -0.44193196296691895,
      "step": 510
    },
    {
      "epoch": 0.28,
      "learning_rate": 4.52364632956877e-07,
      "logits/chosen": -2.376743793487549,
      "logits/rejected": -2.2216954231262207,
      "logps/chosen": -262.9234313964844,
      "logps/rejected": -226.41116333007812,
      "loss": 12766.6898,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.4680143892765045,
      "rewards/margins": 0.11080136150121689,
      "rewards/rejected": -0.578815758228302,
      "rewards/safe_rewards": -0.4691530764102936,
      "rewards/unsafe_rewards": -0.4668757915496826,
      "step": 520
    },
    {
      "epoch": 0.29,
      "learning_rate": 4.4957089415108895e-07,
      "logits/chosen": -2.3553097248077393,
      "logits/rejected": -2.2464723587036133,
      "logps/chosen": -242.263671875,
      "logps/rejected": -255.6951141357422,
      "loss": 11431.9516,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.4732840061187744,
      "rewards/margins": 0.19163894653320312,
      "rewards/rejected": -0.6649229526519775,
      "rewards/safe_rewards": -0.4660876393318176,
      "rewards/unsafe_rewards": -0.4804803729057312,
      "step": 530
    },
    {
      "epoch": 0.29,
      "learning_rate": 4.467067003767745e-07,
      "logits/chosen": -2.3562865257263184,
      "logits/rejected": -2.181304931640625,
      "logps/chosen": -247.70449829101562,
      "logps/rejected": -240.8135528564453,
      "loss": 12058.2938,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.5476623177528381,
      "rewards/margins": 0.19659535586833954,
      "rewards/rejected": -0.7442576885223389,
      "rewards/safe_rewards": -0.5530461072921753,
      "rewards/unsafe_rewards": -0.5422784686088562,
      "step": 540
    },
    {
      "epoch": 0.3,
      "learning_rate": 4.437730627868027e-07,
      "logits/chosen": -2.3266587257385254,
      "logits/rejected": -2.120724678039551,
      "logps/chosen": -231.5970916748047,
      "logps/rejected": -221.3756103515625,
      "loss": 11200.6187,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.5469081997871399,
      "rewards/margins": 0.20715579390525818,
      "rewards/rejected": -0.7540639638900757,
      "rewards/safe_rewards": -0.5482960939407349,
      "rewards/unsafe_rewards": -0.5455203652381897,
      "step": 550
    },
    {
      "epoch": 0.3,
      "learning_rate": 4.4077101704995163e-07,
      "logits/chosen": -2.3897128105163574,
      "logits/rejected": -2.2368927001953125,
      "logps/chosen": -238.13796997070312,
      "logps/rejected": -232.872314453125,
      "loss": 11633.2375,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.4596262574195862,
      "rewards/margins": 0.13524024188518524,
      "rewards/rejected": -0.5948664546012878,
      "rewards/safe_rewards": -0.4546341300010681,
      "rewards/unsafe_rewards": -0.4646182954311371,
      "step": 560
    },
    {
      "epoch": 0.31,
      "learning_rate": 4.3770162298528356e-07,
      "logits/chosen": -2.396641969680786,
      "logits/rejected": -2.2687771320343018,
      "logps/chosen": -244.2262420654297,
      "logps/rejected": -219.45248413085938,
      "loss": 12088.3172,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.493887335062027,
      "rewards/margins": 0.15973922610282898,
      "rewards/rejected": -0.653626561164856,
      "rewards/safe_rewards": -0.48037633299827576,
      "rewards/unsafe_rewards": -0.5073983073234558,
      "step": 570
    },
    {
      "epoch": 0.31,
      "learning_rate": 4.3456596418799476e-07,
      "logits/chosen": -2.3043763637542725,
      "logits/rejected": -2.173457145690918,
      "logps/chosen": -257.493896484375,
      "logps/rejected": -241.50222778320312,
      "loss": 10751.843,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.5369631052017212,
      "rewards/margins": 0.15752388536930084,
      "rewards/rejected": -0.6944869756698608,
      "rewards/safe_rewards": -0.5310293436050415,
      "rewards/unsafe_rewards": -0.5428968667984009,
      "step": 580
    },
    {
      "epoch": 0.32,
      "learning_rate": 4.313651476468715e-07,
      "logits/chosen": -2.270984172821045,
      "logits/rejected": -2.1114678382873535,
      "logps/chosen": -245.8052520751953,
      "logps/rejected": -238.4367218017578,
      "loss": 11156.9102,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.4919458329677582,
      "rewards/margins": 0.16596952080726624,
      "rewards/rejected": -0.6579153537750244,
      "rewards/safe_rewards": -0.533513605594635,
      "rewards/unsafe_rewards": -0.45037803053855896,
      "step": 590
    },
    {
      "epoch": 0.32,
      "learning_rate": 4.2810030335348693e-07,
      "logits/chosen": -2.199920177459717,
      "logits/rejected": -2.001626491546631,
      "logps/chosen": -263.06634521484375,
      "logps/rejected": -236.2901153564453,
      "loss": 11584.7875,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.5653510689735413,
      "rewards/margins": 0.14779801666736603,
      "rewards/rejected": -0.7131490707397461,
      "rewards/safe_rewards": -0.5651000142097473,
      "rewards/unsafe_rewards": -0.56560218334198,
      "step": 600
    },
    {
      "epoch": 0.32,
      "eval_logits/chosen": -1.993375301361084,
      "eval_logits/rejected": -1.7938207387924194,
      "eval_logps/chosen": -211.25747680664062,
      "eval_logps/rejected": -180.6571044921875,
      "eval_loss": 4406.71337890625,
      "eval_rewards/accuracies": 0.6338334679603577,
      "eval_rewards/chosen": -0.8082689642906189,
      "eval_rewards/margins": 0.07364560663700104,
      "eval_rewards/rejected": -0.8819145560264587,
      "eval_rewards/safe_rewards": -0.8027816414833069,
      "eval_rewards/unsafe_rewards": -0.8050090670585632,
      "eval_runtime": 995.4426,
      "eval_samples_per_second": 33.195,
      "eval_steps_per_second": 1.038,
      "step": 600
    },
    {
      "epoch": 0.33,
      "learning_rate": 4.2477258390327806e-07,
      "logits/chosen": -2.2489686012268066,
      "logits/rejected": -2.072410821914673,
      "logps/chosen": -241.9006805419922,
      "logps/rejected": -251.07376098632812,
      "loss": 11304.5898,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.5690335035324097,
      "rewards/margins": 0.20418229699134827,
      "rewards/rejected": -0.7732157707214355,
      "rewards/safe_rewards": -0.5881134271621704,
      "rewards/unsafe_rewards": -0.5499535799026489,
      "step": 610
    },
    {
      "epoch": 0.33,
      "learning_rate": 4.2138316408864197e-07,
      "logits/chosen": -2.334721803665161,
      "logits/rejected": -2.150754928588867,
      "logps/chosen": -246.8868865966797,
      "logps/rejected": -237.44033813476562,
      "loss": 10365.4461,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.48651427030563354,
      "rewards/margins": 0.2631617486476898,
      "rewards/rejected": -0.7496760487556458,
      "rewards/safe_rewards": -0.4882384240627289,
      "rewards/unsafe_rewards": -0.4847901463508606,
      "step": 620
    },
    {
      "epoch": 0.34,
      "learning_rate": 4.179332404841962e-07,
      "logits/chosen": -2.2644031047821045,
      "logits/rejected": -2.0903661251068115,
      "logps/chosen": -267.8985290527344,
      "logps/rejected": -251.5312042236328,
      "loss": 10894.2219,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.5778486728668213,
      "rewards/margins": 0.22166796028614044,
      "rewards/rejected": -0.7995165586471558,
      "rewards/safe_rewards": -0.5826883316040039,
      "rewards/unsafe_rewards": -0.5730089545249939,
      "step": 630
    },
    {
      "epoch": 0.34,
      "learning_rate": 4.1442403102434954e-07,
      "logits/chosen": -2.282378911972046,
      "logits/rejected": -2.113847255706787,
      "logps/chosen": -270.49737548828125,
      "logps/rejected": -250.124755859375,
      "loss": 11471.2906,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.5048245787620544,
      "rewards/margins": 0.20923128724098206,
      "rewards/rejected": -0.7140558958053589,
      "rewards/safe_rewards": -0.5091570615768433,
      "rewards/unsafe_rewards": -0.5004920959472656,
      "step": 640
    },
    {
      "epoch": 0.35,
      "learning_rate": 4.108567745733318e-07,
      "logits/chosen": -2.283024311065674,
      "logits/rejected": -2.1171913146972656,
      "logps/chosen": -216.49264526367188,
      "logps/rejected": -221.82421875,
      "loss": 11842.7375,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.4298115372657776,
      "rewards/margins": 0.178109809756279,
      "rewards/rejected": -0.6079213619232178,
      "rewards/safe_rewards": -0.41675862669944763,
      "rewards/unsafe_rewards": -0.4428643584251404,
      "step": 650
    },
    {
      "epoch": 0.36,
      "learning_rate": 4.0723273048783426e-07,
      "logits/chosen": -2.3322463035583496,
      "logits/rejected": -2.1912810802459717,
      "logps/chosen": -271.88299560546875,
      "logps/rejected": -232.814208984375,
      "loss": 11859.6484,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.47567135095596313,
      "rewards/margins": 0.16224288940429688,
      "rewards/rejected": -0.6379141807556152,
      "rewards/safe_rewards": -0.4415673315525055,
      "rewards/unsafe_rewards": -0.5097752809524536,
      "step": 660
    },
    {
      "epoch": 0.36,
      "learning_rate": 4.0355317817241697e-07,
      "logits/chosen": -2.3272366523742676,
      "logits/rejected": -2.1486003398895264,
      "logps/chosen": -275.4678039550781,
      "logps/rejected": -224.7365264892578,
      "loss": 11542.4758,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.465405136346817,
      "rewards/margins": 0.16741091012954712,
      "rewards/rejected": -0.6328160166740417,
      "rewards/safe_rewards": -0.43667951226234436,
      "rewards/unsafe_rewards": -0.4941307008266449,
      "step": 670
    },
    {
      "epoch": 0.37,
      "learning_rate": 3.998194166278367e-07,
      "logits/chosen": -2.378793239593506,
      "logits/rejected": -2.2765145301818848,
      "logps/chosen": -242.05722045898438,
      "logps/rejected": -228.02822875976562,
      "loss": 12028.9203,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.5077196359634399,
      "rewards/margins": 0.10326583683490753,
      "rewards/rejected": -0.6109854578971863,
      "rewards/safe_rewards": -0.519899845123291,
      "rewards/unsafe_rewards": -0.4955393671989441,
      "step": 680
    },
    {
      "epoch": 0.37,
      "learning_rate": 3.9603276399245855e-07,
      "logits/chosen": -2.3500876426696777,
      "logits/rejected": -2.2054431438446045,
      "logps/chosen": -266.26800537109375,
      "logps/rejected": -240.6906280517578,
      "loss": 11937.7102,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.5368236303329468,
      "rewards/margins": 0.19003283977508545,
      "rewards/rejected": -0.7268564105033875,
      "rewards/safe_rewards": -0.5230244994163513,
      "rewards/unsafe_rewards": -0.5506226420402527,
      "step": 690
    },
    {
      "epoch": 0.38,
      "learning_rate": 3.9219455707691e-07,
      "logits/chosen": -2.3385093212127686,
      "logits/rejected": -2.194088935852051,
      "logps/chosen": -262.49957275390625,
      "logps/rejected": -248.57937622070312,
      "loss": 11237.9461,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.5866261720657349,
      "rewards/margins": 0.18088310956954956,
      "rewards/rejected": -0.7675093412399292,
      "rewards/safe_rewards": -0.5994827747344971,
      "rewards/unsafe_rewards": -0.5737696886062622,
      "step": 700
    },
    {
      "epoch": 0.38,
      "learning_rate": 3.883061508921439e-07,
      "logits/chosen": -2.4109256267547607,
      "logits/rejected": -2.309866428375244,
      "logps/chosen": -253.4579620361328,
      "logps/rejected": -269.43060302734375,
      "loss": 11465.2805,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.5475128889083862,
      "rewards/margins": 0.13771310448646545,
      "rewards/rejected": -0.6852259635925293,
      "rewards/safe_rewards": -0.5719602704048157,
      "rewards/unsafe_rewards": -0.5230655670166016,
      "step": 710
    },
    {
      "epoch": 0.39,
      "learning_rate": 3.8436891817107555e-07,
      "logits/chosen": -2.3683512210845947,
      "logits/rejected": -2.29433012008667,
      "logps/chosen": -238.59597778320312,
      "logps/rejected": -241.7389678955078,
      "loss": 11748.5664,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.481682687997818,
      "rewards/margins": 0.18611426651477814,
      "rewards/rejected": -0.6677969694137573,
      "rewards/safe_rewards": -0.4765700697898865,
      "rewards/unsafe_rewards": -0.4867952764034271,
      "step": 720
    },
    {
      "epoch": 0.39,
      "learning_rate": 3.8038424888396414e-07,
      "logits/chosen": -2.3680453300476074,
      "logits/rejected": -2.233586072921753,
      "logps/chosen": -255.9365692138672,
      "logps/rejected": -252.37393188476562,
      "loss": 10985.4922,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.5011948347091675,
      "rewards/margins": 0.1678631603717804,
      "rewards/rejected": -0.6690580248832703,
      "rewards/safe_rewards": -0.5107179880142212,
      "rewards/unsafe_rewards": -0.49167174100875854,
      "step": 730
    },
    {
      "epoch": 0.4,
      "learning_rate": 3.763535497477079e-07,
      "logits/chosen": -2.376601457595825,
      "logits/rejected": -2.2197787761688232,
      "logps/chosen": -260.691650390625,
      "logps/rejected": -245.3418731689453,
      "loss": 11114.0781,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.5256686210632324,
      "rewards/margins": 0.21716609597206116,
      "rewards/rejected": -0.742834746837616,
      "rewards/safe_rewards": -0.5268703103065491,
      "rewards/unsafe_rewards": -0.524466872215271,
      "step": 740
    },
    {
      "epoch": 0.4,
      "learning_rate": 3.7227824372922795e-07,
      "logits/chosen": -2.3355045318603516,
      "logits/rejected": -2.202324867248535,
      "logps/chosen": -244.21817016601562,
      "logps/rejected": -234.26766967773438,
      "loss": 11000.375,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.5600683093070984,
      "rewards/margins": 0.15717127919197083,
      "rewards/rejected": -0.7172395586967468,
      "rewards/safe_rewards": -0.5515316724777222,
      "rewards/unsafe_rewards": -0.5686048269271851,
      "step": 750
    },
    {
      "epoch": 0.41,
      "learning_rate": 3.681597695431148e-07,
      "logits/chosen": -2.351304292678833,
      "logits/rejected": -2.2070152759552,
      "logps/chosen": -249.6879425048828,
      "logps/rejected": -252.49722290039062,
      "loss": 11395.6094,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.5104349851608276,
      "rewards/margins": 0.22875532507896423,
      "rewards/rejected": -0.7391902804374695,
      "rewards/safe_rewards": -0.5327340364456177,
      "rewards/unsafe_rewards": -0.4881359040737152,
      "step": 760
    },
    {
      "epoch": 0.41,
      "learning_rate": 3.639995811437159e-07,
      "logits/chosen": -2.3204877376556396,
      "logits/rejected": -2.2027907371520996,
      "logps/chosen": -252.81082153320312,
      "logps/rejected": -258.1892395019531,
      "loss": 11175.6797,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.5397385358810425,
      "rewards/margins": 0.19889900088310242,
      "rewards/rejected": -0.7386375069618225,
      "rewards/safe_rewards": -0.5588012337684631,
      "rewards/unsafe_rewards": -0.520675778388977,
      "step": 770
    },
    {
      "epoch": 0.42,
      "learning_rate": 3.597991472118426e-07,
      "logits/chosen": -2.3824949264526367,
      "logits/rejected": -2.2542223930358887,
      "logps/chosen": -267.04437255859375,
      "logps/rejected": -254.86337280273438,
      "loss": 11511.8547,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.5231243371963501,
      "rewards/margins": 0.17550811171531677,
      "rewards/rejected": -0.6986324191093445,
      "rewards/safe_rewards": -0.5508791208267212,
      "rewards/unsafe_rewards": -0.49536967277526855,
      "step": 780
    },
    {
      "epoch": 0.43,
      "learning_rate": 3.5555995063627836e-07,
      "logits/chosen": -2.3838717937469482,
      "logits/rejected": -2.264705181121826,
      "logps/chosen": -288.45098876953125,
      "logps/rejected": -261.7911071777344,
      "loss": 11126.725,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.5807030200958252,
      "rewards/margins": 0.1779719889163971,
      "rewards/rejected": -0.7586749792098999,
      "rewards/safe_rewards": -0.5821424722671509,
      "rewards/unsafe_rewards": -0.5792635679244995,
      "step": 790
    },
    {
      "epoch": 0.43,
      "learning_rate": 3.512834879902715e-07,
      "logits/chosen": -2.325700521469116,
      "logits/rejected": -2.20318603515625,
      "logps/chosen": -262.4079284667969,
      "logps/rejected": -256.8218078613281,
      "loss": 11113.3211,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.6240984797477722,
      "rewards/margins": 0.1907762587070465,
      "rewards/rejected": -0.8148747682571411,
      "rewards/safe_rewards": -0.6181318759918213,
      "rewards/unsafe_rewards": -0.6300650835037231,
      "step": 800
    },
    {
      "epoch": 0.44,
      "learning_rate": 3.4697126900319616e-07,
      "logits/chosen": -2.2967851161956787,
      "logits/rejected": -2.1394267082214355,
      "logps/chosen": -259.38018798828125,
      "logps/rejected": -245.7625732421875,
      "loss": 11771.4461,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.5871919393539429,
      "rewards/margins": 0.2232244461774826,
      "rewards/rejected": -0.8104164004325867,
      "rewards/safe_rewards": -0.5566633939743042,
      "rewards/unsafe_rewards": -0.6177204251289368,
      "step": 810
    },
    {
      "epoch": 0.44,
      "learning_rate": 3.426248160275693e-07,
      "logits/chosen": -2.33948016166687,
      "logits/rejected": -2.2020907402038574,
      "logps/chosen": -251.8537139892578,
      "logps/rejected": -247.29443359375,
      "loss": 11619.0344,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.5438165068626404,
      "rewards/margins": 0.17501583695411682,
      "rewards/rejected": -0.7188323736190796,
      "rewards/safe_rewards": -0.5689524412155151,
      "rewards/unsafe_rewards": -0.5186805725097656,
      "step": 820
    },
    {
      "epoch": 0.45,
      "learning_rate": 3.3824566350161094e-07,
      "logits/chosen": -2.3640289306640625,
      "logits/rejected": -2.195350408554077,
      "logps/chosen": -256.7790222167969,
      "logps/rejected": -235.6261444091797,
      "loss": 11238.1625,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.4845482409000397,
      "rewards/margins": 0.1882091611623764,
      "rewards/rejected": -0.6727573871612549,
      "rewards/safe_rewards": -0.48699751496315,
      "rewards/unsafe_rewards": -0.48209887742996216,
      "step": 830
    },
    {
      "epoch": 0.45,
      "learning_rate": 3.338353574075381e-07,
      "logits/chosen": -2.319687843322754,
      "logits/rejected": -2.2427151203155518,
      "logps/chosen": -229.2741241455078,
      "logps/rejected": -230.6369171142578,
      "loss": 13030.1109,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.5387877225875854,
      "rewards/margins": 0.1425672322511673,
      "rewards/rejected": -0.6813548803329468,
      "rewards/safe_rewards": -0.5679432153701782,
      "rewards/unsafe_rewards": -0.5096321702003479,
      "step": 840
    },
    {
      "epoch": 0.46,
      "learning_rate": 3.2939545472578314e-07,
      "logits/chosen": -2.3902430534362793,
      "logits/rejected": -2.1937272548675537,
      "logps/chosen": -279.47705078125,
      "logps/rejected": -250.4775848388672,
      "loss": 11581.1313,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.5328287482261658,
      "rewards/margins": 0.13031499087810516,
      "rewards/rejected": -0.6631438136100769,
      "rewards/safe_rewards": -0.5133041143417358,
      "rewards/unsafe_rewards": -0.5523533821105957,
      "step": 850
    },
    {
      "epoch": 0.46,
      "learning_rate": 3.2492752288532916e-07,
      "logits/chosen": -2.325867176055908,
      "logits/rejected": -2.1522111892700195,
      "logps/chosen": -253.1811981201172,
      "logps/rejected": -233.44174194335938,
      "loss": 11311.8109,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.4949401319026947,
      "rewards/margins": 0.15677447617053986,
      "rewards/rejected": -0.6517146825790405,
      "rewards/safe_rewards": -0.47377505898475647,
      "rewards/unsafe_rewards": -0.5161052942276001,
      "step": 860
    },
    {
      "epoch": 0.47,
      "learning_rate": 3.204331392103574e-07,
      "logits/chosen": -2.337698459625244,
      "logits/rejected": -2.1352732181549072,
      "logps/chosen": -268.2446594238281,
      "logps/rejected": -234.0172882080078,
      "loss": 11545.3641,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.5469434261322021,
      "rewards/margins": 0.15391257405281067,
      "rewards/rejected": -0.7008560299873352,
      "rewards/safe_rewards": -0.5454758405685425,
      "rewards/unsafe_rewards": -0.5484111905097961,
      "step": 870
    },
    {
      "epoch": 0.47,
      "learning_rate": 3.159138903634006e-07,
      "logits/chosen": -2.1978166103363037,
      "logits/rejected": -2.0318470001220703,
      "logps/chosen": -280.46319580078125,
      "logps/rejected": -256.2702941894531,
      "loss": 11248.9852,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.7179843187332153,
      "rewards/margins": 0.11917723715305328,
      "rewards/rejected": -0.837161660194397,
      "rewards/safe_rewards": -0.7153393030166626,
      "rewards/unsafe_rewards": -0.7206293940544128,
      "step": 880
    },
    {
      "epoch": 0.48,
      "learning_rate": 3.1137137178519977e-07,
      "logits/chosen": -2.1384453773498535,
      "logits/rejected": -2.003512382507324,
      "logps/chosen": -253.1010284423828,
      "logps/rejected": -262.6402893066406,
      "loss": 11450.3414,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.7104918956756592,
      "rewards/margins": 0.18712057173252106,
      "rewards/rejected": -0.897612452507019,
      "rewards/safe_rewards": -0.6939632296562195,
      "rewards/unsafe_rewards": -0.7270206212997437,
      "step": 890
    },
    {
      "epoch": 0.48,
      "learning_rate": 3.068071871314626e-07,
      "logits/chosen": -2.135709524154663,
      "logits/rejected": -2.028604507446289,
      "logps/chosen": -242.0713653564453,
      "logps/rejected": -236.3900909423828,
      "loss": 10862.3484,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.6334865689277649,
      "rewards/margins": 0.13563254475593567,
      "rewards/rejected": -0.7691190838813782,
      "rewards/safe_rewards": -0.63458251953125,
      "rewards/unsafe_rewards": -0.6323906183242798,
      "step": 900
    },
    {
      "epoch": 0.48,
      "eval_logits/chosen": -1.9376695156097412,
      "eval_logits/rejected": -1.7468173503875732,
      "eval_logps/chosen": -218.71400451660156,
      "eval_logps/rejected": -187.7608642578125,
      "eval_loss": 4377.5634765625,
      "eval_rewards/accuracies": 0.6195546984672546,
      "eval_rewards/chosen": -0.8828346133232117,
      "eval_rewards/margins": 0.07011755555868149,
      "eval_rewards/rejected": -0.9529521465301514,
      "eval_rewards/safe_rewards": -0.8774688839912415,
      "eval_rewards/unsafe_rewards": -0.8777852058410645,
      "eval_runtime": 994.9448,
      "eval_samples_per_second": 33.212,
      "eval_steps_per_second": 1.038,
      "step": 900
    },
    {
      "epoch": 0.49,
      "learning_rate": 3.022229477067205e-07,
      "logits/chosen": -2.1737711429595947,
      "logits/rejected": -1.9998136758804321,
      "logps/chosen": -283.02313232421875,
      "logps/rejected": -255.57656860351562,
      "loss": 10507.2695,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.6587027311325073,
      "rewards/margins": 0.193914994597435,
      "rewards/rejected": -0.8526177406311035,
      "rewards/safe_rewards": -0.6458637714385986,
      "rewards/unsafe_rewards": -0.6715416312217712,
      "step": 910
    },
    {
      "epoch": 0.49,
      "learning_rate": 2.976202718954869e-07,
      "logits/chosen": -2.183156728744507,
      "logits/rejected": -1.9853588342666626,
      "logps/chosen": -282.3530578613281,
      "logps/rejected": -271.27069091796875,
      "loss": 11861.3852,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.7082624435424805,
      "rewards/margins": 0.18091826140880585,
      "rewards/rejected": -0.8891807794570923,
      "rewards/safe_rewards": -0.7145110368728638,
      "rewards/unsafe_rewards": -0.7020138502120972,
      "step": 920
    },
    {
      "epoch": 0.5,
      "learning_rate": 2.930007845909146e-07,
      "logits/chosen": -2.177462339401245,
      "logits/rejected": -2.0282797813415527,
      "logps/chosen": -268.3103942871094,
      "logps/rejected": -266.30548095703125,
      "loss": 11683.3781,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.7064036726951599,
      "rewards/margins": 0.13596151769161224,
      "rewards/rejected": -0.8423651456832886,
      "rewards/safe_rewards": -0.7049628496170044,
      "rewards/unsafe_rewards": -0.7078445553779602,
      "step": 930
    },
    {
      "epoch": 0.51,
      "learning_rate": 2.8836611662115634e-07,
      "logits/chosen": -2.1535885334014893,
      "logits/rejected": -1.9535331726074219,
      "logps/chosen": -287.9471130371094,
      "logps/rejected": -254.46157836914062,
      "loss": 12032.6703,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.6779440641403198,
      "rewards/margins": 0.1851780265569687,
      "rewards/rejected": -0.8631221652030945,
      "rewards/safe_rewards": -0.6615537405014038,
      "rewards/unsafe_rewards": -0.6943344473838806,
      "step": 940
    },
    {
      "epoch": 0.51,
      "learning_rate": 2.8371790417362986e-07,
      "logits/chosen": -2.1404051780700684,
      "logits/rejected": -1.9921302795410156,
      "logps/chosen": -250.58895874023438,
      "logps/rejected": -252.75021362304688,
      "loss": 12575.8664,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.6325485110282898,
      "rewards/margins": 0.13314202427864075,
      "rewards/rejected": -0.7656905651092529,
      "rewards/safe_rewards": -0.615986168384552,
      "rewards/unsafe_rewards": -0.6491108536720276,
      "step": 950
    },
    {
      "epoch": 0.52,
      "learning_rate": 2.7905778821739056e-07,
      "logits/chosen": -2.1147265434265137,
      "logits/rejected": -1.9771724939346313,
      "logps/chosen": -253.51351928710938,
      "logps/rejected": -233.7639923095703,
      "loss": 11670.4188,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.5872529745101929,
      "rewards/margins": 0.1395425647497177,
      "rewards/rejected": -0.7267955541610718,
      "rewards/safe_rewards": -0.5655093193054199,
      "rewards/unsafe_rewards": -0.6089966297149658,
      "step": 960
    },
    {
      "epoch": 0.52,
      "learning_rate": 2.74387413923817e-07,
      "logits/chosen": -2.056212902069092,
      "logits/rejected": -1.9303762912750244,
      "logps/chosen": -294.0152893066406,
      "logps/rejected": -272.24615478515625,
      "loss": 11716.1664,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.6957898736000061,
      "rewards/margins": 0.17264129221439362,
      "rewards/rejected": -0.8684310913085938,
      "rewards/safe_rewards": -0.688141405582428,
      "rewards/unsafe_rewards": -0.7034383416175842,
      "step": 970
    },
    {
      "epoch": 0.53,
      "learning_rate": 2.69708430085812e-07,
      "logits/chosen": -2.0949666500091553,
      "logits/rejected": -1.8358476161956787,
      "logps/chosen": -303.7754821777344,
      "logps/rejected": -279.3113708496094,
      "loss": 12070.975,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.8039963841438293,
      "rewards/margins": 0.2226048707962036,
      "rewards/rejected": -1.0266011953353882,
      "rewards/safe_rewards": -0.7967602014541626,
      "rewards/unsafe_rewards": -0.8112322688102722,
      "step": 980
    },
    {
      "epoch": 0.53,
      "learning_rate": 2.6502248853572504e-07,
      "logits/chosen": -2.018751859664917,
      "logits/rejected": -1.81708562374115,
      "logps/chosen": -268.05535888671875,
      "logps/rejected": -263.68487548828125,
      "loss": 12117.2781,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.8013098835945129,
      "rewards/margins": 0.21267572045326233,
      "rewards/rejected": -1.013985514640808,
      "rewards/safe_rewards": -0.7930777668952942,
      "rewards/unsafe_rewards": -0.8095420002937317,
      "step": 990
    },
    {
      "epoch": 0.54,
      "learning_rate": 2.6033124356220325e-07,
      "logits/chosen": -2.040283441543579,
      "logits/rejected": -1.8361635208129883,
      "logps/chosen": -266.68841552734375,
      "logps/rejected": -244.3878631591797,
      "loss": 11242.6422,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.639050304889679,
      "rewards/margins": 0.21101757884025574,
      "rewards/rejected": -0.8500677943229675,
      "rewards/safe_rewards": -0.6206936836242676,
      "rewards/unsafe_rewards": -0.6574069261550903,
      "step": 1000
    },
    {
      "epoch": 0.54,
      "learning_rate": 2.55636351326173e-07,
      "logits/chosen": -2.074812412261963,
      "logits/rejected": -1.9070053100585938,
      "logps/chosen": -267.3968505859375,
      "logps/rejected": -242.2158660888672,
      "loss": 10413.1977,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.6132654547691345,
      "rewards/margins": 0.21324153244495392,
      "rewards/rejected": -0.8265069723129272,
      "rewards/safe_rewards": -0.638529360294342,
      "rewards/unsafe_rewards": -0.588001549243927,
      "step": 1010
    },
    {
      "epoch": 0.55,
      "learning_rate": 2.509394692761622e-07,
      "logits/chosen": -2.1109747886657715,
      "logits/rejected": -1.8711456060409546,
      "logps/chosen": -266.0138244628906,
      "logps/rejected": -246.04159545898438,
      "loss": 11730.4734,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.6028767824172974,
      "rewards/margins": 0.20766966044902802,
      "rewards/rejected": -0.8105465173721313,
      "rewards/safe_rewards": -0.5901221632957458,
      "rewards/unsafe_rewards": -0.6156314611434937,
      "step": 1020
    },
    {
      "epoch": 0.55,
      "learning_rate": 2.462422555631674e-07,
      "logits/chosen": -2.073171854019165,
      "logits/rejected": -1.8314094543457031,
      "logps/chosen": -267.2673034667969,
      "logps/rejected": -234.9337921142578,
      "loss": 11384.8359,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.6479398012161255,
      "rewards/margins": 0.20152795314788818,
      "rewards/rejected": -0.8494676351547241,
      "rewards/safe_rewards": -0.6472254991531372,
      "rewards/unsafe_rewards": -0.6486541032791138,
      "step": 1030
    },
    {
      "epoch": 0.56,
      "learning_rate": 2.415463684552728e-07,
      "logits/chosen": -2.0377871990203857,
      "logits/rejected": -1.8571240901947021,
      "logps/chosen": -264.3353576660156,
      "logps/rejected": -250.5574493408203,
      "loss": 11869.4391,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.6739367246627808,
      "rewards/margins": 0.1584298014640808,
      "rewards/rejected": -0.8323665857315063,
      "rewards/safe_rewards": -0.6686100363731384,
      "rewards/unsafe_rewards": -0.6792632341384888,
      "step": 1040
    },
    {
      "epoch": 0.56,
      "learning_rate": 2.3685346575222807e-07,
      "logits/chosen": -2.111082077026367,
      "logits/rejected": -1.8854366540908813,
      "logps/chosen": -280.7798767089844,
      "logps/rejected": -254.7810821533203,
      "loss": 11278.0977,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.6307443976402283,
      "rewards/margins": 0.16730864346027374,
      "rewards/rejected": -0.7980531454086304,
      "rewards/safe_rewards": -0.6317691802978516,
      "rewards/unsafe_rewards": -0.6297196745872498,
      "step": 1050
    },
    {
      "epoch": 0.57,
      "learning_rate": 2.321652042001919e-07,
      "logits/chosen": -2.124929189682007,
      "logits/rejected": -1.8838306665420532,
      "logps/chosen": -272.2227783203125,
      "logps/rejected": -273.8869323730469,
      "loss": 11306.2313,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.6101773381233215,
      "rewards/margins": 0.20483005046844482,
      "rewards/rejected": -0.8150073885917664,
      "rewards/safe_rewards": -0.6046205759048462,
      "rewards/unsafe_rewards": -0.6157340407371521,
      "step": 1060
    },
    {
      "epoch": 0.58,
      "learning_rate": 2.2748323890684662e-07,
      "logits/chosen": -2.1761631965637207,
      "logits/rejected": -1.8986377716064453,
      "logps/chosen": -270.1111145019531,
      "logps/rejected": -247.43576049804688,
      "loss": 11255.1672,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.5767061710357666,
      "rewards/margins": 0.2480432540178299,
      "rewards/rejected": -0.8247492909431458,
      "rewards/safe_rewards": -0.5869646072387695,
      "rewards/unsafe_rewards": -0.5664476156234741,
      "step": 1070
    },
    {
      "epoch": 0.58,
      "learning_rate": 2.2280922275709213e-07,
      "logits/chosen": -2.1585605144500732,
      "logits/rejected": -2.006892681121826,
      "logps/chosen": -272.51593017578125,
      "logps/rejected": -259.193603515625,
      "loss": 11610.1609,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.6135168671607971,
      "rewards/margins": 0.16986827552318573,
      "rewards/rejected": -0.783385157585144,
      "rewards/safe_rewards": -0.5932785272598267,
      "rewards/unsafe_rewards": -0.6337552666664124,
      "step": 1080
    },
    {
      "epoch": 0.59,
      "learning_rate": 2.1814480582952375e-07,
      "logits/chosen": -2.176574230194092,
      "logits/rejected": -2.0007762908935547,
      "logps/chosen": -264.99627685546875,
      "logps/rejected": -260.7152099609375,
      "loss": 11244.9641,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.6294008493423462,
      "rewards/margins": 0.1786467581987381,
      "rewards/rejected": -0.8080476522445679,
      "rewards/safe_rewards": -0.610293984413147,
      "rewards/unsafe_rewards": -0.6485077738761902,
      "step": 1090
    },
    {
      "epoch": 0.59,
      "learning_rate": 2.1349163481390187e-07,
      "logits/chosen": -2.1403450965881348,
      "logits/rejected": -1.9742145538330078,
      "logps/chosen": -258.41851806640625,
      "logps/rejected": -253.95596313476562,
      "loss": 11432.9805,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.579275369644165,
      "rewards/margins": 0.1978916972875595,
      "rewards/rejected": -0.7771671414375305,
      "rewards/safe_rewards": -0.5507979393005371,
      "rewards/unsafe_rewards": -0.6077528595924377,
      "step": 1100
    },
    {
      "epoch": 0.6,
      "learning_rate": 2.0885135242981647e-07,
      "logits/chosen": -2.110644817352295,
      "logits/rejected": -1.8921115398406982,
      "logps/chosen": -295.12310791015625,
      "logps/rejected": -242.7633056640625,
      "loss": 10866.6008,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.6292051672935486,
      "rewards/margins": 0.1930016279220581,
      "rewards/rejected": -0.8222068548202515,
      "rewards/safe_rewards": -0.6636418104171753,
      "rewards/unsafe_rewards": -0.5947686433792114,
      "step": 1110
    },
    {
      "epoch": 0.6,
      "learning_rate": 2.0422559684675494e-07,
      "logits/chosen": -2.07668137550354,
      "logits/rejected": -1.870774269104004,
      "logps/chosen": -279.4585876464844,
      "logps/rejected": -252.98422241210938,
      "loss": 11123.7453,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.622269868850708,
      "rewards/margins": 0.18196968734264374,
      "rewards/rejected": -0.8042396306991577,
      "rewards/safe_rewards": -0.6149351596832275,
      "rewards/unsafe_rewards": -0.629604697227478,
      "step": 1120
    },
    {
      "epoch": 0.61,
      "learning_rate": 1.9961600110577457e-07,
      "logits/chosen": -2.0477161407470703,
      "logits/rejected": -1.8454376459121704,
      "logps/chosen": -281.2010803222656,
      "logps/rejected": -276.59503173828125,
      "loss": 11451.4273,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.7229688763618469,
      "rewards/margins": 0.15784484148025513,
      "rewards/rejected": -0.880813717842102,
      "rewards/safe_rewards": -0.6920644640922546,
      "rewards/unsafe_rewards": -0.7538732290267944,
      "step": 1130
    },
    {
      "epoch": 0.61,
      "learning_rate": 1.950241925429867e-07,
      "logits/chosen": -2.1420865058898926,
      "logits/rejected": -1.9303150177001953,
      "logps/chosen": -264.2613830566406,
      "logps/rejected": -247.3074493408203,
      "loss": 11185.8914,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.6415218114852905,
      "rewards/margins": 0.21913623809814453,
      "rewards/rejected": -0.8606580495834351,
      "rewards/safe_rewards": -0.6352616548538208,
      "rewards/unsafe_rewards": -0.647782027721405,
      "step": 1140
    },
    {
      "epoch": 0.62,
      "learning_rate": 1.9045179221505495e-07,
      "logits/chosen": -2.1051979064941406,
      "logits/rejected": -1.9602015018463135,
      "logps/chosen": -292.8426513671875,
      "logps/rejected": -272.5882568359375,
      "loss": 10919.0359,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.6359490156173706,
      "rewards/margins": 0.21196213364601135,
      "rewards/rejected": -0.8479111790657043,
      "rewards/safe_rewards": -0.605754554271698,
      "rewards/unsafe_rewards": -0.666143536567688,
      "step": 1150
    },
    {
      "epoch": 0.62,
      "learning_rate": 1.8590041432690893e-07,
      "logits/chosen": -2.0412395000457764,
      "logits/rejected": -1.9340324401855469,
      "logps/chosen": -256.9598693847656,
      "logps/rejected": -250.4047393798828,
      "loss": 11277.7938,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.6930029988288879,
      "rewards/margins": 0.13424697518348694,
      "rewards/rejected": -0.8272498846054077,
      "rewards/safe_rewards": -0.6941550970077515,
      "rewards/unsafe_rewards": -0.6918508410453796,
      "step": 1160
    },
    {
      "epoch": 0.63,
      "learning_rate": 1.813716656618788e-07,
      "logits/chosen": -2.0102741718292236,
      "logits/rejected": -1.9039026498794556,
      "logps/chosen": -259.2917175292969,
      "logps/rejected": -252.873046875,
      "loss": 11988.2266,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.7098476886749268,
      "rewards/margins": 0.1709776222705841,
      "rewards/rejected": -0.8808252215385437,
      "rewards/safe_rewards": -0.7011247873306274,
      "rewards/unsafe_rewards": -0.7185705304145813,
      "step": 1170
    },
    {
      "epoch": 0.63,
      "learning_rate": 1.7686714501444788e-07,
      "logits/chosen": -2.084367275238037,
      "logits/rejected": -1.8202540874481201,
      "logps/chosen": -288.35626220703125,
      "logps/rejected": -254.33651733398438,
      "loss": 11612.3062,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.7599023580551147,
      "rewards/margins": 0.16640828549861908,
      "rewards/rejected": -0.926310658454895,
      "rewards/safe_rewards": -0.7765257954597473,
      "rewards/unsafe_rewards": -0.7432790994644165,
      "step": 1180
    },
    {
      "epoch": 0.64,
      "learning_rate": 1.7238844262582768e-07,
      "logits/chosen": -2.052704095840454,
      "logits/rejected": -1.9864280223846436,
      "logps/chosen": -275.9404602050781,
      "logps/rejected": -276.5259094238281,
      "loss": 10682.7672,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.6891142129898071,
      "rewards/margins": 0.15954537689685822,
      "rewards/rejected": -0.8486596345901489,
      "rewards/safe_rewards": -0.6521228551864624,
      "rewards/unsafe_rewards": -0.7261057496070862,
      "step": 1190
    },
    {
      "epoch": 0.65,
      "learning_rate": 1.679371396225504e-07,
      "logits/chosen": -2.0941474437713623,
      "logits/rejected": -1.8897063732147217,
      "logps/chosen": -271.2891540527344,
      "logps/rejected": -270.78924560546875,
      "loss": 11671.4219,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.7015270590782166,
      "rewards/margins": 0.19374321401119232,
      "rewards/rejected": -0.8952702283859253,
      "rewards/safe_rewards": -0.6884527802467346,
      "rewards/unsafe_rewards": -0.7146013379096985,
      "step": 1200
    },
    {
      "epoch": 0.65,
      "eval_logits/chosen": -1.8665262460708618,
      "eval_logits/rejected": -1.6740189790725708,
      "eval_logps/chosen": -228.5369415283203,
      "eval_logps/rejected": -197.5587921142578,
      "eval_loss": 4346.4052734375,
      "eval_rewards/accuracies": 0.6158034801483154,
      "eval_rewards/chosen": -0.9810636043548584,
      "eval_rewards/margins": 0.06986771523952484,
      "eval_rewards/rejected": -1.0509313344955444,
      "eval_rewards/safe_rewards": -0.9763943552970886,
      "eval_rewards/unsafe_rewards": -0.9767957329750061,
      "eval_runtime": 992.8374,
      "eval_samples_per_second": 33.282,
      "eval_steps_per_second": 1.04,
      "step": 1200
    },
    {
      "epoch": 0.65,
      "learning_rate": 1.6351480745828096e-07,
      "logits/chosen": -2.1039538383483887,
      "logits/rejected": -1.9394419193267822,
      "logps/chosen": -265.77984619140625,
      "logps/rejected": -253.25137329101562,
      "loss": 10276.6594,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.6483036279678345,
      "rewards/margins": 0.1812000572681427,
      "rewards/rejected": -0.8295037150382996,
      "rewards/safe_rewards": -0.6423134803771973,
      "rewards/unsafe_rewards": -0.6542937755584717,
      "step": 1210
    },
    {
      "epoch": 0.66,
      "learning_rate": 1.5912300735904248e-07,
      "logits/chosen": -2.1374449729919434,
      "logits/rejected": -1.983070969581604,
      "logps/chosen": -290.1751403808594,
      "logps/rejected": -259.19097900390625,
      "loss": 11657.7609,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.6737682223320007,
      "rewards/margins": 0.14703810214996338,
      "rewards/rejected": -0.8208063244819641,
      "rewards/safe_rewards": -0.6786805987358093,
      "rewards/unsafe_rewards": -0.6688558459281921,
      "step": 1220
    },
    {
      "epoch": 0.66,
      "learning_rate": 1.5476328977205395e-07,
      "logits/chosen": -2.1445086002349854,
      "logits/rejected": -1.9767332077026367,
      "logps/chosen": -271.8307189941406,
      "logps/rejected": -251.6882781982422,
      "loss": 11008.7156,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.6544319987297058,
      "rewards/margins": 0.21836614608764648,
      "rewards/rejected": -0.8727981448173523,
      "rewards/safe_rewards": -0.6966055631637573,
      "rewards/unsafe_rewards": -0.6122584342956543,
      "step": 1230
    },
    {
      "epoch": 0.67,
      "learning_rate": 1.5043719381837112e-07,
      "logits/chosen": -2.08951473236084,
      "logits/rejected": -1.9547208547592163,
      "logps/chosen": -287.0835266113281,
      "logps/rejected": -271.43670654296875,
      "loss": 11362.9422,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.6643306612968445,
      "rewards/margins": 0.17835856974124908,
      "rewards/rejected": -0.8426891565322876,
      "rewards/safe_rewards": -0.6574736833572388,
      "rewards/unsafe_rewards": -0.6711875200271606,
      "step": 1240
    },
    {
      "epoch": 0.67,
      "learning_rate": 1.461462467495284e-07,
      "logits/chosen": -2.0471625328063965,
      "logits/rejected": -1.8923746347427368,
      "logps/chosen": -252.0188446044922,
      "logps/rejected": -260.54522705078125,
      "loss": 10503.6422,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.7411776185035706,
      "rewards/margins": 0.21499836444854736,
      "rewards/rejected": -0.9561759829521179,
      "rewards/safe_rewards": -0.7554097175598145,
      "rewards/unsafe_rewards": -0.7269455194473267,
      "step": 1250
    },
    {
      "epoch": 0.68,
      "learning_rate": 1.4189196340836865e-07,
      "logits/chosen": -2.1763851642608643,
      "logits/rejected": -1.9493157863616943,
      "logps/chosen": -269.4758605957031,
      "logps/rejected": -258.123291015625,
      "loss": 10853.5781,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.7069443464279175,
      "rewards/margins": 0.17843110859394073,
      "rewards/rejected": -0.8853754997253418,
      "rewards/safe_rewards": -0.7389132380485535,
      "rewards/unsafe_rewards": -0.6749754548072815,
      "step": 1260
    },
    {
      "epoch": 0.68,
      "learning_rate": 1.3767584569425561e-07,
      "logits/chosen": -2.19474458694458,
      "logits/rejected": -1.9729722738265991,
      "logps/chosen": -277.60992431640625,
      "logps/rejected": -258.0458984375,
      "loss": 11364.5312,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.6931368708610535,
      "rewards/margins": 0.18750238418579102,
      "rewards/rejected": -0.8806392550468445,
      "rewards/safe_rewards": -0.7000595927238464,
      "rewards/unsafe_rewards": -0.6862143278121948,
      "step": 1270
    },
    {
      "epoch": 0.69,
      "learning_rate": 1.334993820328541e-07,
      "logits/chosen": -2.0907111167907715,
      "logits/rejected": -1.9239038228988647,
      "logps/chosen": -246.83642578125,
      "logps/rejected": -245.0505828857422,
      "loss": 10652.2328,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.7036882638931274,
      "rewards/margins": 0.24492530524730682,
      "rewards/rejected": -0.9486135244369507,
      "rewards/safe_rewards": -0.6736655235290527,
      "rewards/unsafe_rewards": -0.7337108850479126,
      "step": 1280
    },
    {
      "epoch": 0.69,
      "learning_rate": 1.2936404685066852e-07,
      "logits/chosen": -2.0505154132843018,
      "logits/rejected": -1.9092705249786377,
      "logps/chosen": -286.58990478515625,
      "logps/rejected": -280.3271789550781,
      "loss": 11526.9,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.68525230884552,
      "rewards/margins": 0.18679992854595184,
      "rewards/rejected": -0.8720521926879883,
      "rewards/safe_rewards": -0.7125786542892456,
      "rewards/unsafe_rewards": -0.6579257249832153,
      "step": 1290
    },
    {
      "epoch": 0.7,
      "learning_rate": 1.252713000545221e-07,
      "logits/chosen": -2.2164149284362793,
      "logits/rejected": -2.019528388977051,
      "logps/chosen": -282.0648498535156,
      "logps/rejected": -258.7922058105469,
      "loss": 10187.7117,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.6330300569534302,
      "rewards/margins": 0.23740728199481964,
      "rewards/rejected": -0.8704373240470886,
      "rewards/safe_rewards": -0.6281386613845825,
      "rewards/unsafe_rewards": -0.6379214525222778,
      "step": 1300
    },
    {
      "epoch": 0.7,
      "learning_rate": 1.2122258651616304e-07,
      "logits/chosen": -2.183960437774658,
      "logits/rejected": -1.976144790649414,
      "logps/chosen": -275.6617126464844,
      "logps/rejected": -244.86434936523438,
      "loss": 10813.9688,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6538507342338562,
      "rewards/margins": 0.17538292706012726,
      "rewards/rejected": -0.8292337656021118,
      "rewards/safe_rewards": -0.6337584853172302,
      "rewards/unsafe_rewards": -0.673943042755127,
      "step": 1310
    },
    {
      "epoch": 0.71,
      "learning_rate": 1.1721933556217792e-07,
      "logits/chosen": -2.167842388153076,
      "logits/rejected": -2.0031533241271973,
      "logps/chosen": -265.863525390625,
      "logps/rejected": -261.1900329589844,
      "loss": 11171.5523,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.6531133651733398,
      "rewards/margins": 0.19861795008182526,
      "rewards/rejected": -0.8517313003540039,
      "rewards/safe_rewards": -0.6670490503311157,
      "rewards/unsafe_rewards": -0.639177680015564,
      "step": 1320
    },
    {
      "epoch": 0.72,
      "learning_rate": 1.1326296046939333e-07,
      "logits/chosen": -2.134734869003296,
      "logits/rejected": -1.9698076248168945,
      "logps/chosen": -257.0758972167969,
      "logps/rejected": -242.34640502929688,
      "loss": 10608.8219,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.6214333772659302,
      "rewards/margins": 0.21942774951457977,
      "rewards/rejected": -0.8408611416816711,
      "rewards/safe_rewards": -0.6008444428443909,
      "rewards/unsafe_rewards": -0.6420222520828247,
      "step": 1330
    },
    {
      "epoch": 0.72,
      "learning_rate": 1.0935485796594351e-07,
      "logits/chosen": -2.174132823944092,
      "logits/rejected": -1.9792499542236328,
      "logps/chosen": -291.65814208984375,
      "logps/rejected": -260.0698547363281,
      "loss": 11895.8789,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.6717311143875122,
      "rewards/margins": 0.22725185751914978,
      "rewards/rejected": -0.8989830017089844,
      "rewards/safe_rewards": -0.6783491373062134,
      "rewards/unsafe_rewards": -0.665113091468811,
      "step": 1340
    },
    {
      "epoch": 0.73,
      "learning_rate": 1.0549640773818028e-07,
      "logits/chosen": -2.139070749282837,
      "logits/rejected": -2.040194034576416,
      "logps/chosen": -272.8217468261719,
      "logps/rejected": -252.80606079101562,
      "loss": 11776.8328,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.724553108215332,
      "rewards/margins": 0.14907190203666687,
      "rewards/rejected": -0.8736250996589661,
      "rewards/safe_rewards": -0.764702320098877,
      "rewards/unsafe_rewards": -0.6844038963317871,
      "step": 1350
    },
    {
      "epoch": 0.73,
      "learning_rate": 1.0168897194359921e-07,
      "logits/chosen": -2.1789050102233887,
      "logits/rejected": -1.9954169988632202,
      "logps/chosen": -300.97711181640625,
      "logps/rejected": -274.7542724609375,
      "loss": 10877.8953,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.7322704195976257,
      "rewards/margins": 0.15806125104427338,
      "rewards/rejected": -0.8903317451477051,
      "rewards/safe_rewards": -0.6902467608451843,
      "rewards/unsafe_rewards": -0.7742940783500671,
      "step": 1360
    },
    {
      "epoch": 0.74,
      "learning_rate": 9.793389472995392e-08,
      "logits/chosen": -2.155348539352417,
      "logits/rejected": -1.9421207904815674,
      "logps/chosen": -279.4602966308594,
      "logps/rejected": -240.51791381835938,
      "loss": 10089.3242,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.639030933380127,
      "rewards/margins": 0.252288281917572,
      "rewards/rejected": -0.8913192749023438,
      "rewards/safe_rewards": -0.6204690933227539,
      "rewards/unsafe_rewards": -0.6575928330421448,
      "step": 1370
    },
    {
      "epoch": 0.74,
      "learning_rate": 9.423250176072874e-08,
      "logits/chosen": -2.1420297622680664,
      "logits/rejected": -1.946412444114685,
      "logps/chosen": -260.1219177246094,
      "logps/rejected": -234.1204833984375,
      "loss": 12708.7586,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.71236252784729,
      "rewards/margins": 0.1536431610584259,
      "rewards/rejected": -0.8660055994987488,
      "rewards/safe_rewards": -0.6938896179199219,
      "rewards/unsafe_rewards": -0.7308354377746582,
      "step": 1380
    },
    {
      "epoch": 0.75,
      "learning_rate": 9.058609974713654e-08,
      "logits/chosen": -2.1647183895111084,
      "logits/rejected": -1.9520107507705688,
      "logps/chosen": -270.0279235839844,
      "logps/rejected": -260.9911193847656,
      "loss": 10654.6578,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.6530768275260925,
      "rewards/margins": 0.2193538248538971,
      "rewards/rejected": -0.872430682182312,
      "rewards/safe_rewards": -0.6400060653686523,
      "rewards/unsafe_rewards": -0.6661475896835327,
      "step": 1390
    },
    {
      "epoch": 0.75,
      "learning_rate": 8.699597598680753e-08,
      "logits/chosen": -2.1252872943878174,
      "logits/rejected": -1.9515492916107178,
      "logps/chosen": -257.53680419921875,
      "logps/rejected": -247.7872314453125,
      "loss": 10438.4352,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.6546493768692017,
      "rewards/margins": 0.19867801666259766,
      "rewards/rejected": -0.8533273935317993,
      "rewards/safe_rewards": -0.6472882032394409,
      "rewards/unsafe_rewards": -0.6620105504989624,
      "step": 1400
    },
    {
      "epoch": 0.76,
      "learning_rate": 8.346339790933166e-08,
      "logits/chosen": -2.163567543029785,
      "logits/rejected": -1.9771531820297241,
      "logps/chosen": -261.906982421875,
      "logps/rejected": -245.43344116210938,
      "loss": 11502.7297,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.6902798414230347,
      "rewards/margins": 0.19094708561897278,
      "rewards/rejected": -0.8812268376350403,
      "rewards/safe_rewards": -0.6901260018348694,
      "rewards/unsafe_rewards": -0.6904336214065552,
      "step": 1410
    },
    {
      "epoch": 0.76,
      "learning_rate": 7.998961262881506e-08,
      "logits/chosen": -2.136033535003662,
      "logits/rejected": -1.915771245956421,
      "logps/chosen": -282.3092346191406,
      "logps/rejected": -245.1779022216797,
      "loss": 10907.2641,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.6517400741577148,
      "rewards/margins": 0.2158946692943573,
      "rewards/rejected": -0.8676347732543945,
      "rewards/safe_rewards": -0.6872564554214478,
      "rewards/unsafe_rewards": -0.6162236928939819,
      "step": 1420
    },
    {
      "epoch": 0.77,
      "learning_rate": 7.657584650360846e-08,
      "logits/chosen": -2.068126678466797,
      "logits/rejected": -1.9587552547454834,
      "logps/chosen": -251.7869873046875,
      "logps/rejected": -236.9489288330078,
      "loss": 11409.3359,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.6544846296310425,
      "rewards/margins": 0.2086174488067627,
      "rewards/rejected": -0.8631020784378052,
      "rewards/safe_rewards": -0.6824710965156555,
      "rewards/unsafe_rewards": -0.6264981627464294,
      "step": 1430
    },
    {
      "epoch": 0.77,
      "learning_rate": 7.322330470336313e-08,
      "logits/chosen": -2.122067928314209,
      "logits/rejected": -1.9209505319595337,
      "logps/chosen": -269.77520751953125,
      "logps/rejected": -268.40972900390625,
      "loss": 10821.843,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.6080222129821777,
      "rewards/margins": 0.25041937828063965,
      "rewards/rejected": -0.8584416508674622,
      "rewards/safe_rewards": -0.5671194195747375,
      "rewards/unsafe_rewards": -0.6489250659942627,
      "step": 1440
    },
    {
      "epoch": 0.78,
      "learning_rate": 6.993317078356709e-08,
      "logits/chosen": -2.1524910926818848,
      "logits/rejected": -2.056199789047241,
      "logps/chosen": -279.29583740234375,
      "logps/rejected": -255.84521484375,
      "loss": 11391.4539,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.691224217414856,
      "rewards/margins": 0.13461162149906158,
      "rewards/rejected": -0.8258358240127563,
      "rewards/safe_rewards": -0.7272582054138184,
      "rewards/unsafe_rewards": -0.6551901698112488,
      "step": 1450
    },
    {
      "epoch": 0.79,
      "learning_rate": 6.67066062677118e-08,
      "logits/chosen": -2.191707134246826,
      "logits/rejected": -2.0195794105529785,
      "logps/chosen": -263.27642822265625,
      "logps/rejected": -235.50973510742188,
      "loss": 11926.3453,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.6236655116081238,
      "rewards/margins": 0.16336306929588318,
      "rewards/rejected": -0.7870286107063293,
      "rewards/safe_rewards": -0.6308915615081787,
      "rewards/unsafe_rewards": -0.6164394617080688,
      "step": 1460
    },
    {
      "epoch": 0.79,
      "learning_rate": 6.354475023723685e-08,
      "logits/chosen": -2.1155574321746826,
      "logits/rejected": -1.9720067977905273,
      "logps/chosen": -301.7005920410156,
      "logps/rejected": -267.5169372558594,
      "loss": 11094.9625,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.6763675808906555,
      "rewards/margins": 0.23334193229675293,
      "rewards/rejected": -0.9097094535827637,
      "rewards/safe_rewards": -0.6679859757423401,
      "rewards/unsafe_rewards": -0.6847492456436157,
      "step": 1470
    },
    {
      "epoch": 0.8,
      "learning_rate": 6.044871892939746e-08,
      "logits/chosen": -2.184211254119873,
      "logits/rejected": -1.9999923706054688,
      "logps/chosen": -281.41900634765625,
      "logps/rejected": -268.82464599609375,
      "loss": 10924.2703,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.6950246691703796,
      "rewards/margins": 0.19827482104301453,
      "rewards/rejected": -0.8932995796203613,
      "rewards/safe_rewards": -0.6960557699203491,
      "rewards/unsafe_rewards": -0.6939936280250549,
      "step": 1480
    },
    {
      "epoch": 0.8,
      "learning_rate": 5.741960534319676e-08,
      "logits/chosen": -2.134298801422119,
      "logits/rejected": -2.02238130569458,
      "logps/chosen": -241.8282012939453,
      "logps/rejected": -240.76290893554688,
      "loss": 11064.8711,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.687341570854187,
      "rewards/margins": 0.16167931258678436,
      "rewards/rejected": -0.8490209579467773,
      "rewards/safe_rewards": -0.7233562469482422,
      "rewards/unsafe_rewards": -0.6513269543647766,
      "step": 1490
    },
    {
      "epoch": 0.81,
      "learning_rate": 5.44584788535217e-08,
      "logits/chosen": -2.1548972129821777,
      "logits/rejected": -2.0020337104797363,
      "logps/chosen": -288.1227111816406,
      "logps/rejected": -265.52337646484375,
      "loss": 10202.4125,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.7084766626358032,
      "rewards/margins": 0.21978096663951874,
      "rewards/rejected": -0.9282576441764832,
      "rewards/safe_rewards": -0.6754325032234192,
      "rewards/unsafe_rewards": -0.741520881652832,
      "step": 1500
    },
    {
      "epoch": 0.81,
      "eval_logits/chosen": -1.9419655799865723,
      "eval_logits/rejected": -1.764477252960205,
      "eval_logps/chosen": -226.97747802734375,
      "eval_logps/rejected": -195.17942810058594,
      "eval_loss": 4320.98779296875,
      "eval_rewards/accuracies": 0.6022507548332214,
      "eval_rewards/chosen": -0.9654689431190491,
      "eval_rewards/margins": 0.061668772250413895,
      "eval_rewards/rejected": -1.0271376371383667,
      "eval_rewards/safe_rewards": -0.9610660672187805,
      "eval_rewards/unsafe_rewards": -0.9618369936943054,
      "eval_runtime": 993.2958,
      "eval_samples_per_second": 33.267,
      "eval_steps_per_second": 1.04,
      "step": 1500
    },
    {
      "epoch": 0.81,
      "learning_rate": 5.156638483361933e-08,
      "logits/chosen": -2.2049641609191895,
      "logits/rejected": -2.0106749534606934,
      "logps/chosen": -285.2719421386719,
      "logps/rejected": -273.9066467285156,
      "loss": 11107.9672,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.6739949584007263,
      "rewards/margins": 0.23203198611736298,
      "rewards/rejected": -0.9060269594192505,
      "rewards/safe_rewards": -0.6972517967224121,
      "rewards/unsafe_rewards": -0.6507382988929749,
      "step": 1510
    },
    {
      "epoch": 0.82,
      "learning_rate": 4.8744344286046236e-08,
      "logits/chosen": -2.148716926574707,
      "logits/rejected": -2.0059280395507812,
      "logps/chosen": -285.7190856933594,
      "logps/rejected": -266.8186950683594,
      "loss": 11427.2711,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.6893398761749268,
      "rewards/margins": 0.15746144950389862,
      "rewards/rejected": -0.8468014001846313,
      "rewards/safe_rewards": -0.7316367030143738,
      "rewards/unsafe_rewards": -0.6470431089401245,
      "step": 1520
    },
    {
      "epoch": 0.82,
      "learning_rate": 4.599335348222169e-08,
      "logits/chosen": -2.174553871154785,
      "logits/rejected": -2.05711030960083,
      "logps/chosen": -288.70849609375,
      "logps/rejected": -289.8013610839844,
      "loss": 10682.943,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.6674157977104187,
      "rewards/margins": 0.21851542592048645,
      "rewards/rejected": -0.8859313130378723,
      "rewards/safe_rewards": -0.6756128668785095,
      "rewards/unsafe_rewards": -0.6592189073562622,
      "step": 1530
    },
    {
      "epoch": 0.83,
      "learning_rate": 4.331438361071163e-08,
      "logits/chosen": -2.1525204181671143,
      "logits/rejected": -2.059199571609497,
      "logps/chosen": -302.69195556640625,
      "logps/rejected": -287.97161865234375,
      "loss": 11478.0016,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.6294053792953491,
      "rewards/margins": 0.17254997789859772,
      "rewards/rejected": -0.8019553422927856,
      "rewards/safe_rewards": -0.6268101334571838,
      "rewards/unsafe_rewards": -0.6320004463195801,
      "step": 1540
    },
    {
      "epoch": 0.83,
      "learning_rate": 4.0708380434367864e-08,
      "logits/chosen": -2.1821987628936768,
      "logits/rejected": -1.9993159770965576,
      "logps/chosen": -265.56854248046875,
      "logps/rejected": -261.4121398925781,
      "loss": 10532.0812,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.6764332056045532,
      "rewards/margins": 0.19545750319957733,
      "rewards/rejected": -0.8718908429145813,
      "rewards/safe_rewards": -0.6992810964584351,
      "rewards/unsafe_rewards": -0.6535855531692505,
      "step": 1550
    },
    {
      "epoch": 0.84,
      "learning_rate": 3.817626395644305e-08,
      "logits/chosen": -2.1872050762176514,
      "logits/rejected": -2.0328307151794434,
      "logps/chosen": -265.10205078125,
      "logps/rejected": -248.8824462890625,
      "loss": 11308.0203,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.6716850399971008,
      "rewards/margins": 0.13508987426757812,
      "rewards/rejected": -0.806774914264679,
      "rewards/safe_rewards": -0.6651741862297058,
      "rewards/unsafe_rewards": -0.6781958341598511,
      "step": 1560
    },
    {
      "epoch": 0.84,
      "learning_rate": 3.571892809580013e-08,
      "logits/chosen": -2.157172203063965,
      "logits/rejected": -2.0089964866638184,
      "logps/chosen": -260.57525634765625,
      "logps/rejected": -251.6233367919922,
      "loss": 11400.9055,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.6940633058547974,
      "rewards/margins": 0.136986643075943,
      "rewards/rejected": -0.831049919128418,
      "rewards/safe_rewards": -0.6672950983047485,
      "rewards/unsafe_rewards": -0.7208314538002014,
      "step": 1570
    },
    {
      "epoch": 0.85,
      "learning_rate": 3.333724037132976e-08,
      "logits/chosen": -2.180004596710205,
      "logits/rejected": -2.0329430103302,
      "logps/chosen": -274.8149108886719,
      "logps/rejected": -270.170654296875,
      "loss": 11022.3203,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.6869722008705139,
      "rewards/margins": 0.17142841219902039,
      "rewards/rejected": -0.8584005236625671,
      "rewards/safe_rewards": -0.664495587348938,
      "rewards/unsafe_rewards": -0.7094486951828003,
      "step": 1580
    },
    {
      "epoch": 0.86,
      "learning_rate": 3.1032041595688506e-08,
      "logits/chosen": -2.107234477996826,
      "logits/rejected": -1.9175293445587158,
      "logps/chosen": -265.2936706542969,
      "logps/rejected": -258.67987060546875,
      "loss": 10853.2336,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.6839567422866821,
      "rewards/margins": 0.1879226267337799,
      "rewards/rejected": -0.8718793988227844,
      "rewards/safe_rewards": -0.6944083571434021,
      "rewards/unsafe_rewards": -0.6735051870346069,
      "step": 1590
    },
    {
      "epoch": 0.86,
      "learning_rate": 2.880414557846453e-08,
      "logits/chosen": -2.131779432296753,
      "logits/rejected": -2.045048236846924,
      "logps/chosen": -249.33114624023438,
      "logps/rejected": -240.8230438232422,
      "loss": 10239.6547,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.6429909467697144,
      "rewards/margins": 0.1874755173921585,
      "rewards/rejected": -0.8304663896560669,
      "rewards/safe_rewards": -0.6290788650512695,
      "rewards/unsafe_rewards": -0.6569029092788696,
      "step": 1600
    },
    {
      "epoch": 0.87,
      "learning_rate": 2.6654338838876662e-08,
      "logits/chosen": -2.2182207107543945,
      "logits/rejected": -1.985608458518982,
      "logps/chosen": -280.660400390625,
      "logps/rejected": -240.7994384765625,
      "loss": 11038.0031,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.6624557971954346,
      "rewards/margins": 0.23739679157733917,
      "rewards/rejected": -0.8998525738716125,
      "rewards/safe_rewards": -0.68045574426651,
      "rewards/unsafe_rewards": -0.6444558501243591,
      "step": 1610
    },
    {
      "epoch": 0.87,
      "learning_rate": 2.4583380328107805e-08,
      "logits/chosen": -2.1470413208007812,
      "logits/rejected": -1.985733985900879,
      "logps/chosen": -291.9235534667969,
      "logps/rejected": -258.4531555175781,
      "loss": 11528.2828,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.6623538136482239,
      "rewards/margins": 0.20262321829795837,
      "rewards/rejected": -0.8649770021438599,
      "rewards/safe_rewards": -0.6641503572463989,
      "rewards/unsafe_rewards": -0.660557210445404,
      "step": 1620
    },
    {
      "epoch": 0.88,
      "learning_rate": 2.259200116137039e-08,
      "logits/chosen": -2.1394782066345215,
      "logits/rejected": -1.9972803592681885,
      "logps/chosen": -284.652099609375,
      "logps/rejected": -278.02423095703125,
      "loss": 11545.0234,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.6758708357810974,
      "rewards/margins": 0.17819495499134064,
      "rewards/rejected": -0.8540657758712769,
      "rewards/safe_rewards": -0.6568797826766968,
      "rewards/unsafe_rewards": -0.6948619484901428,
      "step": 1630
    },
    {
      "epoch": 0.88,
      "learning_rate": 2.068090435979958e-08,
      "logits/chosen": -2.1189610958099365,
      "logits/rejected": -2.0093135833740234,
      "logps/chosen": -268.87371826171875,
      "logps/rejected": -250.8845672607422,
      "loss": 11243.7469,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.6307465434074402,
      "rewards/margins": 0.16744786500930786,
      "rewards/rejected": -0.798194408416748,
      "rewards/safe_rewards": -0.6202843189239502,
      "rewards/unsafe_rewards": -0.6412087678909302,
      "step": 1640
    },
    {
      "epoch": 0.89,
      "learning_rate": 1.8850764602263423e-08,
      "logits/chosen": -2.1497740745544434,
      "logits/rejected": -1.9683231115341187,
      "logps/chosen": -263.89495849609375,
      "logps/rejected": -266.2047424316406,
      "loss": 11196.7602,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.6697946786880493,
      "rewards/margins": 0.1570041924715042,
      "rewards/rejected": -0.8267987966537476,
      "rewards/safe_rewards": -0.6728789806365967,
      "rewards/unsafe_rewards": -0.666710376739502,
      "step": 1650
    },
    {
      "epoch": 0.89,
      "learning_rate": 1.710222798718028e-08,
      "logits/chosen": -2.16522479057312,
      "logits/rejected": -2.045431613922119,
      "logps/chosen": -276.67889404296875,
      "logps/rejected": -280.6698303222656,
      "loss": 10997.7469,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.6575156450271606,
      "rewards/margins": 0.1838151067495346,
      "rewards/rejected": -0.8413307070732117,
      "rewards/safe_rewards": -0.6694937944412231,
      "rewards/unsafe_rewards": -0.6455374360084534,
      "step": 1660
    },
    {
      "epoch": 0.9,
      "learning_rate": 1.5435911804424356e-08,
      "logits/chosen": -2.159721851348877,
      "logits/rejected": -2.048668384552002,
      "logps/chosen": -290.70916748046875,
      "logps/rejected": -270.44683837890625,
      "loss": 11924.0805,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.6324605345726013,
      "rewards/margins": 0.19088464975357056,
      "rewards/rejected": -0.8233451843261719,
      "rewards/safe_rewards": -0.6401357054710388,
      "rewards/unsafe_rewards": -0.624785304069519,
      "step": 1670
    },
    {
      "epoch": 0.9,
      "learning_rate": 1.3852404317403199e-08,
      "logits/chosen": -2.108271360397339,
      "logits/rejected": -1.9915637969970703,
      "logps/chosen": -250.95361328125,
      "logps/rejected": -268.6639099121094,
      "loss": 11402.2625,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.6399115324020386,
      "rewards/margins": 0.16807249188423157,
      "rewards/rejected": -0.8079840540885925,
      "rewards/safe_rewards": -0.6549087762832642,
      "rewards/unsafe_rewards": -0.6249145269393921,
      "step": 1680
    },
    {
      "epoch": 0.91,
      "learning_rate": 1.235226455538113e-08,
      "logits/chosen": -2.1952316761016846,
      "logits/rejected": -2.0874345302581787,
      "logps/chosen": -270.4241943359375,
      "logps/rejected": -263.0255432128906,
      "loss": 11680.2766,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.6519337892532349,
      "rewards/margins": 0.18009743094444275,
      "rewards/rejected": -0.8320311307907104,
      "rewards/safe_rewards": -0.662136435508728,
      "rewards/unsafe_rewards": -0.6417311429977417,
      "step": 1690
    },
    {
      "epoch": 0.91,
      "learning_rate": 1.0936022116124321e-08,
      "logits/chosen": -2.155649185180664,
      "logits/rejected": -2.0004172325134277,
      "logps/chosen": -262.668212890625,
      "logps/rejected": -252.78994750976562,
      "loss": 10840.3375,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.6261528134346008,
      "rewards/margins": 0.2126920521259308,
      "rewards/rejected": -0.838844895362854,
      "rewards/safe_rewards": -0.6410089731216431,
      "rewards/unsafe_rewards": -0.6112965941429138,
      "step": 1700
    },
    {
      "epoch": 0.92,
      "learning_rate": 9.60417697893534e-09,
      "logits/chosen": -2.155482292175293,
      "logits/rejected": -2.0352649688720703,
      "logps/chosen": -259.88397216796875,
      "logps/rejected": -262.71746826171875,
      "loss": 11428.7422,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.632883608341217,
      "rewards/margins": 0.17666089534759521,
      "rewards/rejected": -0.8095444440841675,
      "rewards/safe_rewards": -0.5953446626663208,
      "rewards/unsafe_rewards": -0.6704224348068237,
      "step": 1710
    },
    {
      "epoch": 0.93,
      "learning_rate": 8.357199328144576e-09,
      "logits/chosen": -2.1424155235290527,
      "logits/rejected": -2.0332658290863037,
      "logps/chosen": -308.6863708496094,
      "logps/rejected": -289.75830078125,
      "loss": 10265.0281,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.6669701337814331,
      "rewards/margins": 0.17594949901103973,
      "rewards/rejected": -0.8429197072982788,
      "rewards/safe_rewards": -0.7015780210494995,
      "rewards/unsafe_rewards": -0.6323622465133667,
      "step": 1720
    },
    {
      "epoch": 0.93,
      "learning_rate": 7.1955293871198144e-09,
      "logits/chosen": -2.112140417098999,
      "logits/rejected": -2.030024528503418,
      "logps/chosen": -241.85476684570312,
      "logps/rejected": -241.1482391357422,
      "loss": 11823.4781,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.6754211187362671,
      "rewards/margins": 0.13721489906311035,
      "rewards/rejected": -0.8126360177993774,
      "rewards/safe_rewards": -0.6722853779792786,
      "rewards/unsafe_rewards": -0.6785567998886108,
      "step": 1730
    },
    {
      "epoch": 0.94,
      "learning_rate": 6.119577262853254e-09,
      "logits/chosen": -2.1218502521514893,
      "logits/rejected": -1.944239854812622,
      "logps/chosen": -251.3634490966797,
      "logps/rejected": -230.8335723876953,
      "loss": 11556.3273,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.638350248336792,
      "rewards/margins": 0.21106202900409698,
      "rewards/rejected": -0.8494122624397278,
      "rewards/safe_rewards": -0.613598108291626,
      "rewards/unsafe_rewards": -0.6631024479866028,
      "step": 1740
    },
    {
      "epoch": 0.94,
      "learning_rate": 5.129722801180542e-09,
      "logits/chosen": -2.1147098541259766,
      "logits/rejected": -2.0030481815338135,
      "logps/chosen": -270.8954162597656,
      "logps/rejected": -255.99600219726562,
      "loss": 10173.3906,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.6771684885025024,
      "rewards/margins": 0.1781720221042633,
      "rewards/rejected": -0.8553404808044434,
      "rewards/safe_rewards": -0.7092324495315552,
      "rewards/unsafe_rewards": -0.6451044082641602,
      "step": 1750
    },
    {
      "epoch": 0.95,
      "learning_rate": 4.226315452682816e-09,
      "logits/chosen": -2.1494641304016113,
      "logits/rejected": -2.025930404663086,
      "logps/chosen": -264.47100830078125,
      "logps/rejected": -256.65093994140625,
      "loss": 10950.5703,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.6349852681159973,
      "rewards/margins": 0.19823208451271057,
      "rewards/rejected": -0.833217442035675,
      "rewards/safe_rewards": -0.6437097787857056,
      "rewards/unsafe_rewards": -0.6262607574462891,
      "step": 1760
    },
    {
      "epoch": 0.95,
      "learning_rate": 3.4096741493194193e-09,
      "logits/chosen": -2.2127685546875,
      "logits/rejected": -2.0972578525543213,
      "logps/chosen": -264.87530517578125,
      "logps/rejected": -257.4256286621094,
      "loss": 11913.2609,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.6370722651481628,
      "rewards/margins": 0.15080325305461884,
      "rewards/rejected": -0.7878755927085876,
      "rewards/safe_rewards": -0.617976725101471,
      "rewards/unsafe_rewards": -0.6561679840087891,
      "step": 1770
    },
    {
      "epoch": 0.96,
      "learning_rate": 2.6800871918346846e-09,
      "logits/chosen": -2.220578670501709,
      "logits/rejected": -2.04236102104187,
      "logps/chosen": -274.6324768066406,
      "logps/rejected": -260.865234375,
      "loss": 11005.4078,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.6086142659187317,
      "rewards/margins": 0.19890496134757996,
      "rewards/rejected": -0.8075191378593445,
      "rewards/safe_rewards": -0.6277071237564087,
      "rewards/unsafe_rewards": -0.5895212888717651,
      "step": 1780
    },
    {
      "epoch": 0.96,
      "learning_rate": 2.0378121479783796e-09,
      "logits/chosen": -2.132037401199341,
      "logits/rejected": -1.972651481628418,
      "logps/chosen": -265.41961669921875,
      "logps/rejected": -256.274658203125,
      "loss": 11459.243,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.6822707056999207,
      "rewards/margins": 0.2049897462129593,
      "rewards/rejected": -0.8872605562210083,
      "rewards/safe_rewards": -0.6853871941566467,
      "rewards/unsafe_rewards": -0.6791542768478394,
      "step": 1790
    },
    {
      "epoch": 0.97,
      "learning_rate": 1.4830757615760247e-09,
      "logits/chosen": -2.1159682273864746,
      "logits/rejected": -1.9551427364349365,
      "logps/chosen": -281.0558166503906,
      "logps/rejected": -260.3757629394531,
      "loss": 11785.8336,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.6576239466667175,
      "rewards/margins": 0.15307986736297607,
      "rewards/rejected": -0.8107039332389832,
      "rewards/safe_rewards": -0.689802348613739,
      "rewards/unsafe_rewards": -0.6254457235336304,
      "step": 1800
    },
    {
      "epoch": 0.97,
      "eval_logits/chosen": -1.954970121383667,
      "eval_logits/rejected": -1.7745355367660522,
      "eval_logps/chosen": -224.6013946533203,
      "eval_logps/rejected": -193.1151123046875,
      "eval_loss": 4320.82080078125,
      "eval_rewards/accuracies": 0.6027347445487976,
      "eval_rewards/chosen": -0.9417080879211426,
      "eval_rewards/margins": 0.06478659808635712,
      "eval_rewards/rejected": -1.0064946413040161,
      "eval_rewards/safe_rewards": -0.9369282722473145,
      "eval_rewards/unsafe_rewards": -0.9372634291648865,
      "eval_runtime": 993.1393,
      "eval_samples_per_second": 33.272,
      "eval_steps_per_second": 1.04,
      "step": 1800
    },
    {
      "epoch": 0.97,
      "learning_rate": 1.0160738724809548e-09,
      "logits/chosen": -2.1704697608947754,
      "logits/rejected": -1.9731838703155518,
      "logps/chosen": -259.1123046875,
      "logps/rejected": -257.42254638671875,
      "loss": 10364.2625,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.6492170691490173,
      "rewards/margins": 0.19766607880592346,
      "rewards/rejected": -0.8468831777572632,
      "rewards/safe_rewards": -0.6583928465843201,
      "rewards/unsafe_rewards": -0.6400412321090698,
      "step": 1810
    },
    {
      "epoch": 0.98,
      "learning_rate": 6.369713474366212e-10,
      "logits/chosen": -2.1475830078125,
      "logits/rejected": -1.9969937801361084,
      "logps/chosen": -294.9414367675781,
      "logps/rejected": -288.337890625,
      "loss": 10068.2195,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.6980792880058289,
      "rewards/margins": 0.2098824679851532,
      "rewards/rejected": -0.9079617261886597,
      "rewards/safe_rewards": -0.7020989656448364,
      "rewards/unsafe_rewards": -0.6940596699714661,
      "step": 1820
    },
    {
      "epoch": 0.98,
      "learning_rate": 3.459020218731512e-10,
      "logits/chosen": -2.1376261711120605,
      "logits/rejected": -2.0157809257507324,
      "logps/chosen": -254.5150604248047,
      "logps/rejected": -243.1461181640625,
      "loss": 10131.9508,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.6527233719825745,
      "rewards/margins": 0.2155275046825409,
      "rewards/rejected": -0.8682507276535034,
      "rewards/safe_rewards": -0.6437439322471619,
      "rewards/unsafe_rewards": -0.6617026329040527,
      "step": 1830
    },
    {
      "epoch": 0.99,
      "learning_rate": 1.429686526593088e-10,
      "logits/chosen": -2.116459608078003,
      "logits/rejected": -1.9903713464736938,
      "logps/chosen": -270.08428955078125,
      "logps/rejected": -265.38336181640625,
      "loss": 11322.0812,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.6661794185638428,
      "rewards/margins": 0.18438491225242615,
      "rewards/rejected": -0.8505643606185913,
      "rewards/safe_rewards": -0.6831308603286743,
      "rewards/unsafe_rewards": -0.6492279767990112,
      "step": 1840
    },
    {
      "epoch": 1.0,
      "learning_rate": 2.824288182584622e-11,
      "logits/chosen": -2.1975011825561523,
      "logits/rejected": -2.020158290863037,
      "logps/chosen": -284.1047668457031,
      "logps/rejected": -259.551513671875,
      "loss": 10472.5969,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.6722114682197571,
      "rewards/margins": 0.17123940587043762,
      "rewards/rejected": -0.8434508442878723,
      "rewards/safe_rewards": -0.6627709865570068,
      "rewards/unsafe_rewards": -0.6816519498825073,
      "step": 1850
    },
    {
      "epoch": 1.0,
      "step": 1858,
      "total_flos": 0.0,
      "train_loss": 11692.422362755651,
      "train_runtime": 23508.9776,
      "train_samples_per_second": 2.53,
      "train_steps_per_second": 0.079
    }
  ],
  "logging_steps": 10,
  "max_steps": 1858,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 300,
  "total_flos": 0.0,
  "train_batch_size": 2,
  "trial_name": null,
  "trial_params": null
}