{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 0.998691442030882,
  "eval_steps": 500,
  "global_step": 477,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.010468463752944255,
      "grad_norm": 10.600991740982659,
      "learning_rate": 5.208333333333333e-08,
      "logits/chosen": -0.7836824655532837,
      "logits/rejected": -0.8418347239494324,
      "logps/chosen": -1.174766182899475,
      "logps/rejected": -1.3588011264801025,
      "loss": -0.0349,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -1.174766182899475,
      "rewards/margins": 0.1840350478887558,
      "rewards/rejected": -1.3588011264801025,
      "step": 5
    },
    {
      "epoch": 0.02093692750588851,
      "grad_norm": 2.7100936762262937,
      "learning_rate": 1.0416666666666667e-07,
      "logits/chosen": -0.8232773542404175,
      "logits/rejected": -0.8316025733947754,
      "logps/chosen": -1.1583973169326782,
      "logps/rejected": -1.2626698017120361,
      "loss": -0.0305,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.1583973169326782,
      "rewards/margins": 0.10427255928516388,
      "rewards/rejected": -1.2626698017120361,
      "step": 10
    },
    {
      "epoch": 0.031405391258832765,
      "grad_norm": 5.660375571877667,
      "learning_rate": 1.5624999999999999e-07,
      "logits/chosen": -0.7926589846611023,
      "logits/rejected": -0.8048545718193054,
      "logps/chosen": -1.1033742427825928,
      "logps/rejected": -1.3587285280227661,
      "loss": -0.0465,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -1.1033742427825928,
      "rewards/margins": 0.25535425543785095,
      "rewards/rejected": -1.3587285280227661,
      "step": 15
    },
    {
      "epoch": 0.04187385501177702,
      "grad_norm": 3.7357000754298184,
      "learning_rate": 2.0833333333333333e-07,
      "logits/chosen": -0.7191373705863953,
      "logits/rejected": -0.8023589253425598,
      "logps/chosen": -1.1485874652862549,
      "logps/rejected": -1.253619909286499,
      "loss": -0.0339,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -1.1485874652862549,
      "rewards/margins": 0.10503246635198593,
      "rewards/rejected": -1.253619909286499,
      "step": 20
    },
    {
      "epoch": 0.05234231876472128,
      "grad_norm": 5.416875927919006,
      "learning_rate": 2.604166666666667e-07,
      "logits/chosen": -0.806380569934845,
      "logits/rejected": -0.8102810978889465,
      "logps/chosen": -1.1436618566513062,
      "logps/rejected": -1.2122325897216797,
      "loss": -0.0392,
      "rewards/accuracies": 0.518750011920929,
      "rewards/chosen": -1.1436618566513062,
      "rewards/margins": 0.06857069581747055,
      "rewards/rejected": -1.2122325897216797,
      "step": 25
    },
    {
      "epoch": 0.06281078251766553,
      "grad_norm": 5.913782183964252,
      "learning_rate": 3.1249999999999997e-07,
      "logits/chosen": -0.7805474996566772,
      "logits/rejected": -0.8070163726806641,
      "logps/chosen": -1.0784746408462524,
      "logps/rejected": -1.2096612453460693,
      "loss": -0.0302,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -1.0784746408462524,
      "rewards/margins": 0.13118666410446167,
      "rewards/rejected": -1.2096612453460693,
      "step": 30
    },
    {
      "epoch": 0.07327924627060979,
      "grad_norm": 2.6541139134020133,
      "learning_rate": 3.645833333333333e-07,
      "logits/chosen": -0.814271092414856,
      "logits/rejected": -0.7939152121543884,
      "logps/chosen": -1.046288013458252,
      "logps/rejected": -1.1717827320098877,
      "loss": -0.043,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -1.046288013458252,
      "rewards/margins": 0.12549468874931335,
      "rewards/rejected": -1.1717827320098877,
      "step": 35
    },
    {
      "epoch": 0.08374771002355404,
      "grad_norm": 2.3868591785059694,
      "learning_rate": 4.1666666666666667e-07,
      "logits/chosen": -0.7904576063156128,
      "logits/rejected": -0.9059640765190125,
      "logps/chosen": -0.966705322265625,
      "logps/rejected": -1.2806286811828613,
      "loss": -0.0588,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.966705322265625,
      "rewards/margins": 0.31392329931259155,
      "rewards/rejected": -1.2806286811828613,
      "step": 40
    },
    {
      "epoch": 0.0942161737764983,
      "grad_norm": 3.8242068571740493,
      "learning_rate": 4.6874999999999996e-07,
      "logits/chosen": -0.7696360349655151,
      "logits/rejected": -0.834786057472229,
      "logps/chosen": -0.9762159585952759,
      "logps/rejected": -1.1492093801498413,
      "loss": -0.0549,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.9762159585952759,
      "rewards/margins": 0.17299343645572662,
      "rewards/rejected": -1.1492093801498413,
      "step": 45
    },
    {
      "epoch": 0.10468463752944256,
      "grad_norm": 3.349329434355001,
      "learning_rate": 4.999731868769026e-07,
      "logits/chosen": -0.8303428888320923,
      "logits/rejected": -0.9099453687667847,
      "logps/chosen": -1.0054728984832764,
      "logps/rejected": -1.2581647634506226,
      "loss": -0.0438,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -1.0054728984832764,
      "rewards/margins": 0.2526918649673462,
      "rewards/rejected": -1.2581647634506226,
      "step": 50
    },
    {
      "epoch": 0.11515310128238682,
      "grad_norm": 17.990360349960177,
      "learning_rate": 4.996716052911017e-07,
      "logits/chosen": -0.8441118001937866,
      "logits/rejected": -0.8858628273010254,
      "logps/chosen": -0.9657171368598938,
      "logps/rejected": -1.1507956981658936,
      "loss": -0.0487,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -0.9657171368598938,
      "rewards/margins": 0.18507853150367737,
      "rewards/rejected": -1.1507956981658936,
      "step": 55
    },
    {
      "epoch": 0.12562156503533106,
      "grad_norm": 2.4604104425775093,
      "learning_rate": 4.990353313429303e-07,
      "logits/chosen": -0.866470456123352,
      "logits/rejected": -0.9195329546928406,
      "logps/chosen": -1.0146195888519287,
      "logps/rejected": -1.1302388906478882,
      "loss": -0.0599,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -1.0146195888519287,
      "rewards/margins": 0.11561928689479828,
      "rewards/rejected": -1.1302388906478882,
      "step": 60
    },
    {
      "epoch": 0.1360900287882753,
      "grad_norm": 2.386753258565622,
      "learning_rate": 4.980652179769217e-07,
      "logits/chosen": -0.8147915601730347,
      "logits/rejected": -0.8957066535949707,
      "logps/chosen": -0.9518657922744751,
      "logps/rejected": -1.295042872428894,
      "loss": -0.0521,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.9518657922744751,
      "rewards/margins": 0.3431771397590637,
      "rewards/rejected": -1.295042872428894,
      "step": 65
    },
    {
      "epoch": 0.14655849254121958,
      "grad_norm": 2.547052916904684,
      "learning_rate": 4.967625656594781e-07,
      "logits/chosen": -0.8104137182235718,
      "logits/rejected": -0.8524875640869141,
      "logps/chosen": -0.9991258382797241,
      "logps/rejected": -1.169449806213379,
      "loss": -0.0541,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.9991258382797241,
      "rewards/margins": 0.17032404243946075,
      "rewards/rejected": -1.169449806213379,
      "step": 70
    },
    {
      "epoch": 0.15702695629416383,
      "grad_norm": 3.4870962135797243,
      "learning_rate": 4.951291206355559e-07,
      "logits/chosen": -0.8058147430419922,
      "logits/rejected": -0.8437595367431641,
      "logps/chosen": -0.9659625291824341,
      "logps/rejected": -1.2181332111358643,
      "loss": -0.0584,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.9659625291824341,
      "rewards/margins": 0.25217074155807495,
      "rewards/rejected": -1.2181332111358643,
      "step": 75
    },
    {
      "epoch": 0.16749542004710807,
      "grad_norm": 11.125744861715972,
      "learning_rate": 4.93167072587771e-07,
      "logits/chosen": -0.8071163892745972,
      "logits/rejected": -0.8949975967407227,
      "logps/chosen": -0.9791210293769836,
      "logps/rejected": -1.2237764596939087,
      "loss": -0.0513,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.9791210293769836,
      "rewards/margins": 0.24465537071228027,
      "rewards/rejected": -1.2237764596939087,
      "step": 80
    },
    {
      "epoch": 0.17796388380005235,
      "grad_norm": 3.898865350054578,
      "learning_rate": 4.908790517010636e-07,
      "logits/chosen": -0.8957134485244751,
      "logits/rejected": -0.9214810132980347,
      "logps/chosen": -0.9375537633895874,
      "logps/rejected": -1.1478058099746704,
      "loss": -0.057,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.9375537633895874,
      "rewards/margins": 0.21025200188159943,
      "rewards/rejected": -1.1478058099746704,
      "step": 85
    },
    {
      "epoch": 0.1884323475529966,
      "grad_norm": 3.582473358962524,
      "learning_rate": 4.882681251368548e-07,
      "logits/chosen": -0.8433510661125183,
      "logits/rejected": -0.8857067227363586,
      "logps/chosen": -1.0094842910766602,
      "logps/rejected": -1.2096831798553467,
      "loss": -0.0644,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.0094842910766602,
      "rewards/margins": 0.20019885897636414,
      "rewards/rejected": -1.2096831798553467,
      "step": 90
    },
    {
      "epoch": 0.19890081130594087,
      "grad_norm": 2.6501926608400797,
      "learning_rate": 4.853377929214243e-07,
      "logits/chosen": -0.8576142191886902,
      "logits/rejected": -0.8764857053756714,
      "logps/chosen": -0.9934334754943848,
      "logps/rejected": -1.1720221042633057,
      "loss": -0.0505,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -0.9934334754943848,
      "rewards/margins": 0.1785886585712433,
      "rewards/rejected": -1.1720221042633057,
      "step": 95
    },
    {
      "epoch": 0.2093692750588851,
      "grad_norm": 33.88769391502933,
      "learning_rate": 4.820919832540181e-07,
      "logits/chosen": -0.8933563232421875,
      "logits/rejected": -0.9612926244735718,
      "logps/chosen": -1.0248340368270874,
      "logps/rejected": -1.2969132661819458,
      "loss": -0.0529,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -1.0248340368270874,
      "rewards/margins": 0.2720790505409241,
      "rewards/rejected": -1.2969132661819458,
      "step": 100
    },
    {
      "epoch": 0.21983773881182936,
      "grad_norm": 5.123142096970102,
      "learning_rate": 4.785350472409791e-07,
      "logits/chosen": -0.8836784362792969,
      "logits/rejected": -0.9229451417922974,
      "logps/chosen": -0.9697669744491577,
      "logps/rejected": -1.3045259714126587,
      "loss": -0.063,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.9697669744491577,
      "rewards/margins": 0.3347588777542114,
      "rewards/rejected": -1.3045259714126587,
      "step": 105
    },
    {
      "epoch": 0.23030620256477363,
      "grad_norm": 4.05979796610575,
      "learning_rate": 4.7467175306295647e-07,
      "logits/chosen": -0.9353694915771484,
      "logits/rejected": -0.98784339427948,
      "logps/chosen": -1.0010242462158203,
      "logps/rejected": -1.3293567895889282,
      "loss": -0.0709,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.0010242462158203,
      "rewards/margins": 0.3283325731754303,
      "rewards/rejected": -1.3293567895889282,
      "step": 110
    },
    {
      "epoch": 0.24077466631771788,
      "grad_norm": 4.515920103518208,
      "learning_rate": 4.70507279583015e-07,
      "logits/chosen": -0.8709474802017212,
      "logits/rejected": -0.9586912393569946,
      "logps/chosen": -0.9947507977485657,
      "logps/rejected": -1.400367021560669,
      "loss": -0.0838,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.9947507977485657,
      "rewards/margins": 0.40561628341674805,
      "rewards/rejected": -1.400367021560669,
      "step": 115
    },
    {
      "epoch": 0.2512431300706621,
      "grad_norm": 4.110081424436634,
      "learning_rate": 4.6604720940421207e-07,
      "logits/chosen": -0.8883617520332336,
      "logits/rejected": -0.9269152879714966,
      "logps/chosen": -1.0870624780654907,
      "logps/rejected": -1.3331050872802734,
      "loss": -0.0626,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -1.0870624780654907,
      "rewards/margins": 0.24604257941246033,
      "rewards/rejected": -1.3331050872802734,
      "step": 120
    },
    {
      "epoch": 0.26171159382360637,
      "grad_norm": 3.7209294499373624,
      "learning_rate": 4.612975213859487e-07,
      "logits/chosen": -0.9534390568733215,
      "logits/rejected": -0.9193525314331055,
      "logps/chosen": -1.002967119216919,
      "logps/rejected": -1.2403877973556519,
      "loss": -0.0683,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -1.002967119216919,
      "rewards/margins": 0.23742058873176575,
      "rewards/rejected": -1.2403877973556519,
      "step": 125
    },
    {
      "epoch": 0.2721800575765506,
      "grad_norm": 3.499758787421745,
      "learning_rate": 4.5626458262912735e-07,
      "logits/chosen": -0.8969480395317078,
      "logits/rejected": -0.9590781927108765,
      "logps/chosen": -1.0254695415496826,
      "logps/rejected": -1.1852182149887085,
      "loss": -0.0663,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -1.0254695415496826,
      "rewards/margins": 0.15974867343902588,
      "rewards/rejected": -1.1852182149887085,
      "step": 130
    },
    {
      "epoch": 0.2826485213294949,
      "grad_norm": 4.321280373798233,
      "learning_rate": 4.5095513994085974e-07,
      "logits/chosen": -0.8857036828994751,
      "logits/rejected": -0.9329354166984558,
      "logps/chosen": -1.0760549306869507,
      "logps/rejected": -1.5539740324020386,
      "loss": -0.0777,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -1.0760549306869507,
      "rewards/margins": 0.47791916131973267,
      "rewards/rejected": -1.5539740324020386,
      "step": 135
    },
    {
      "epoch": 0.29311698508243916,
      "grad_norm": 3.205749007042699,
      "learning_rate": 4.453763107901675e-07,
      "logits/chosen": -0.9192848205566406,
      "logits/rejected": -0.9621114730834961,
      "logps/chosen": -1.0559959411621094,
      "logps/rejected": -1.453352451324463,
      "loss": -0.0865,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.0559959411621094,
      "rewards/margins": 0.39735639095306396,
      "rewards/rejected": -1.453352451324463,
      "step": 140
    },
    {
      "epoch": 0.3035854488353834,
      "grad_norm": 5.90034906637265,
      "learning_rate": 4.395355737667985e-07,
      "logits/chosen": -0.9266332387924194,
      "logits/rejected": -1.0155680179595947,
      "logps/chosen": -1.074566125869751,
      "logps/rejected": -1.3885366916656494,
      "loss": -0.0784,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -1.074566125869751,
      "rewards/margins": 0.3139706254005432,
      "rewards/rejected": -1.3885366916656494,
      "step": 145
    },
    {
      "epoch": 0.31405391258832765,
      "grad_norm": 4.108356388429939,
      "learning_rate": 4.3344075855595097e-07,
      "logits/chosen": -0.9286434054374695,
      "logits/rejected": -0.9394516944885254,
      "logps/chosen": -0.9996128082275391,
      "logps/rejected": -1.3331079483032227,
      "loss": -0.0833,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.9996128082275391,
      "rewards/margins": 0.3334951102733612,
      "rewards/rejected": -1.3331079483032227,
      "step": 150
    },
    {
      "epoch": 0.3245223763412719,
      "grad_norm": 12.905833082101557,
      "learning_rate": 4.271000354423425e-07,
      "logits/chosen": -0.9297893643379211,
      "logits/rejected": -0.9530170559883118,
      "logps/chosen": -1.0625531673431396,
      "logps/rejected": -1.3642576932907104,
      "loss": -0.087,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -1.0625531673431396,
      "rewards/margins": 0.3017045855522156,
      "rewards/rejected": -1.3642576932907104,
      "step": 155
    },
    {
      "epoch": 0.33499084009421615,
      "grad_norm": 4.043990046291937,
      "learning_rate": 4.2052190435769554e-07,
      "logits/chosen": -0.9249881505966187,
      "logits/rejected": -0.9575279355049133,
      "logps/chosen": -1.0861815214157104,
      "logps/rejected": -1.4002734422683716,
      "loss": -0.0807,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -1.0861815214157104,
      "rewards/margins": 0.31409186124801636,
      "rewards/rejected": -1.4002734422683716,
      "step": 160
    },
    {
      "epoch": 0.34545930384716045,
      "grad_norm": 4.916459068211759,
      "learning_rate": 4.137151834863213e-07,
      "logits/chosen": -0.9621366262435913,
      "logits/rejected": -0.995520293712616,
      "logps/chosen": -1.0501563549041748,
      "logps/rejected": -1.4903188943862915,
      "loss": -0.0907,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.0501563549041748,
      "rewards/margins": 0.4401625096797943,
      "rewards/rejected": -1.4903188943862915,
      "step": 165
    },
    {
      "epoch": 0.3559277676001047,
      "grad_norm": 5.335366984629706,
      "learning_rate": 4.0668899744407567e-07,
      "logits/chosen": -0.9441369771957397,
      "logits/rejected": -0.9942566156387329,
      "logps/chosen": -1.1764615774154663,
      "logps/rejected": -1.5894583463668823,
      "loss": -0.08,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.1764615774154663,
      "rewards/margins": 0.41299691796302795,
      "rewards/rejected": -1.5894583463668823,
      "step": 170
    },
    {
      "epoch": 0.36639623135304894,
      "grad_norm": 6.274854639434234,
      "learning_rate": 3.994527650465352e-07,
      "logits/chosen": -0.9332780838012695,
      "logits/rejected": -1.0104329586029053,
      "logps/chosen": -1.1270277500152588,
      "logps/rejected": -1.563685655593872,
      "loss": -0.0818,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -1.1270277500152588,
      "rewards/margins": 0.4366579055786133,
      "rewards/rejected": -1.563685655593872,
      "step": 175
    },
    {
      "epoch": 0.3768646951059932,
      "grad_norm": 7.8424698369148444,
      "learning_rate": 3.920161866827889e-07,
      "logits/chosen": -0.9343840479850769,
      "logits/rejected": -1.0072158575057983,
      "logps/chosen": -1.1227374076843262,
      "logps/rejected": -1.5470798015594482,
      "loss": -0.0868,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -1.1227374076843262,
      "rewards/margins": 0.42434263229370117,
      "rewards/rejected": -1.5470798015594482,
      "step": 180
    },
    {
      "epoch": 0.38733315885893743,
      "grad_norm": 4.338056644815217,
      "learning_rate": 3.8438923131177237e-07,
      "logits/chosen": -0.8708566427230835,
      "logits/rejected": -0.9427552223205566,
      "logps/chosen": -1.187537670135498,
      "logps/rejected": -1.823570966720581,
      "loss": -0.1045,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -1.187537670135498,
      "rewards/margins": 0.636033296585083,
      "rewards/rejected": -1.823570966720581,
      "step": 185
    },
    {
      "epoch": 0.39780162261188173,
      "grad_norm": 6.673093826325478,
      "learning_rate": 3.765821230985757e-07,
      "logits/chosen": -0.9710279703140259,
      "logits/rejected": -0.9876633882522583,
      "logps/chosen": -1.1750192642211914,
      "logps/rejected": -1.6105972528457642,
      "loss": -0.0772,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.1750192642211914,
      "rewards/margins": 0.435577929019928,
      "rewards/rejected": -1.6105972528457642,
      "step": 190
    },
    {
      "epoch": 0.408270086364826,
      "grad_norm": 6.960384794729192,
      "learning_rate": 3.6860532770864005e-07,
      "logits/chosen": -0.9228804707527161,
      "logits/rejected": -0.9380979537963867,
      "logps/chosen": -1.299344778060913,
      "logps/rejected": -1.6751635074615479,
      "loss": -0.0845,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -1.299344778060913,
      "rewards/margins": 0.37581872940063477,
      "rewards/rejected": -1.6751635074615479,
      "step": 195
    },
    {
      "epoch": 0.4187385501177702,
      "grad_norm": 6.471834662041886,
      "learning_rate": 3.604695382782159e-07,
      "logits/chosen": -0.9913051724433899,
      "logits/rejected": -0.992074191570282,
      "logps/chosen": -1.221297264099121,
      "logps/rejected": -1.7224781513214111,
      "loss": -0.0952,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -1.221297264099121,
      "rewards/margins": 0.5011808276176453,
      "rewards/rejected": -1.7224781513214111,
      "step": 200
    },
    {
      "epoch": 0.42920701387071447,
      "grad_norm": 6.306403211422552,
      "learning_rate": 3.5218566107988867e-07,
      "logits/chosen": -0.883250892162323,
      "logits/rejected": -0.9081627726554871,
      "logps/chosen": -1.1198192834854126,
      "logps/rejected": -1.7429187297821045,
      "loss": -0.1047,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -1.1198192834854126,
      "rewards/margins": 0.6230995059013367,
      "rewards/rejected": -1.7429187297821045,
      "step": 205
    },
    {
      "epoch": 0.4396754776236587,
      "grad_norm": 25.678467464556803,
      "learning_rate": 3.4376480090239047e-07,
      "logits/chosen": -0.8838554620742798,
      "logits/rejected": -0.9190862774848938,
      "logps/chosen": -1.416511058807373,
      "logps/rejected": -1.6893377304077148,
      "loss": -0.0809,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -1.416511058807373,
      "rewards/margins": 0.27282652258872986,
      "rewards/rejected": -1.6893377304077148,
      "step": 210
    },
    {
      "epoch": 0.45014394137660296,
      "grad_norm": 16.57780152635486,
      "learning_rate": 3.3521824616429284e-07,
      "logits/chosen": -0.8571985363960266,
      "logits/rejected": -0.9235578775405884,
      "logps/chosen": -1.1465059518814087,
      "logps/rejected": -1.7085803747177124,
      "loss": -0.0954,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -1.1465059518814087,
      "rewards/margins": 0.5620743036270142,
      "rewards/rejected": -1.7085803747177124,
      "step": 215
    },
    {
      "epoch": 0.46061240512954726,
      "grad_norm": 14.206901054524915,
      "learning_rate": 3.265574537815398e-07,
      "logits/chosen": -0.8814586400985718,
      "logits/rejected": -0.9272589683532715,
      "logps/chosen": -1.1567518711090088,
      "logps/rejected": -1.6502008438110352,
      "loss": -0.0723,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -1.1567518711090088,
      "rewards/margins": 0.4934488832950592,
      "rewards/rejected": -1.6502008438110352,
      "step": 220
    },
    {
      "epoch": 0.4710808688824915,
      "grad_norm": 6.6240353334989495,
      "learning_rate": 3.1779403380910425e-07,
      "logits/chosen": -0.8806575536727905,
      "logits/rejected": -0.9517351388931274,
      "logps/chosen": -1.1226614713668823,
      "logps/rejected": -1.7587499618530273,
      "loss": -0.0895,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -1.1226614713668823,
      "rewards/margins": 0.6360886096954346,
      "rewards/rejected": -1.7587499618530273,
      "step": 225
    },
    {
      "epoch": 0.48154933263543576,
      "grad_norm": 4.308434013872677,
      "learning_rate": 3.0893973387735683e-07,
      "logits/chosen": -0.8706935048103333,
      "logits/rejected": -0.9313523173332214,
      "logps/chosen": -1.2426685094833374,
      "logps/rejected": -2.1209378242492676,
      "loss": -0.1094,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.2426685094833374,
      "rewards/margins": 0.878269374370575,
      "rewards/rejected": -2.1209378242492676,
      "step": 230
    },
    {
      "epoch": 0.49201779638838,
      "grad_norm": 6.844252123006064,
      "learning_rate": 3.000064234440111e-07,
      "logits/chosen": -0.8923465609550476,
      "logits/rejected": -0.9771193265914917,
      "logps/chosen": -1.096273422241211,
      "logps/rejected": -1.7108914852142334,
      "loss": -0.1035,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -1.096273422241211,
      "rewards/margins": 0.6146181225776672,
      "rewards/rejected": -1.7108914852142334,
      "step": 235
    },
    {
      "epoch": 0.5024862601413242,
      "grad_norm": 4.117878427703188,
      "learning_rate": 2.910060778827554e-07,
      "logits/chosen": -0.8880583047866821,
      "logits/rejected": -0.9422343373298645,
      "logps/chosen": -1.187822699546814,
      "logps/rejected": -1.7649528980255127,
      "loss": -0.0888,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -1.187822699546814,
      "rewards/margins": 0.5771301984786987,
      "rewards/rejected": -1.7649528980255127,
      "step": 240
    },
    {
      "epoch": 0.5129547238942685,
      "grad_norm": 11.906403629187743,
      "learning_rate": 2.8195076242990116e-07,
      "logits/chosen": -0.9159537553787231,
      "logits/rejected": -0.959700882434845,
      "logps/chosen": -1.292806625366211,
      "logps/rejected": -1.880102515220642,
      "loss": -0.1005,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -1.292806625366211,
      "rewards/margins": 0.5872958898544312,
      "rewards/rejected": -1.880102515220642,
      "step": 245
    },
    {
      "epoch": 0.5234231876472127,
      "grad_norm": 9.645796474062738,
      "learning_rate": 2.7285261601056697e-07,
      "logits/chosen": -0.8488172292709351,
      "logits/rejected": -0.9300807118415833,
      "logps/chosen": -1.2770051956176758,
      "logps/rejected": -1.9150631427764893,
      "loss": -0.0977,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -1.2770051956176758,
      "rewards/margins": 0.6380580067634583,
      "rewards/rejected": -1.9150631427764893,
      "step": 250
    },
    {
      "epoch": 0.533891651400157,
      "grad_norm": 4.731035112524077,
      "learning_rate": 2.6372383496608186e-07,
      "logits/chosen": -0.9005299806594849,
      "logits/rejected": -0.9532705545425415,
      "logps/chosen": -1.309429407119751,
      "logps/rejected": -2.0657031536102295,
      "loss": -0.1139,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -1.309429407119751,
      "rewards/margins": 0.7562738656997681,
      "rewards/rejected": -2.0657031536102295,
      "step": 255
    },
    {
      "epoch": 0.5443601151531012,
      "grad_norm": 7.325832832673913,
      "learning_rate": 2.5457665670441937e-07,
      "logits/chosen": -0.8450838327407837,
      "logits/rejected": -0.9038773775100708,
      "logps/chosen": -1.0662903785705566,
      "logps/rejected": -1.9864327907562256,
      "loss": -0.106,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -1.0662903785705566,
      "rewards/margins": 0.9201424717903137,
      "rewards/rejected": -1.9864327907562256,
      "step": 260
    },
    {
      "epoch": 0.5548285789060455,
      "grad_norm": 11.31094245733987,
      "learning_rate": 2.454233432955807e-07,
      "logits/chosen": -0.8820160031318665,
      "logits/rejected": -0.9507255554199219,
      "logps/chosen": -1.221882700920105,
      "logps/rejected": -1.6403167247772217,
      "loss": -0.0993,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -1.221882700920105,
      "rewards/margins": 0.4184340536594391,
      "rewards/rejected": -1.6403167247772217,
      "step": 265
    },
    {
      "epoch": 0.5652970426589898,
      "grad_norm": 7.9126915493471675,
      "learning_rate": 2.3627616503391812e-07,
      "logits/chosen": -0.911536693572998,
      "logits/rejected": -0.9098671674728394,
      "logps/chosen": -1.3050767183303833,
      "logps/rejected": -1.6615794897079468,
      "loss": -0.0834,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.3050767183303833,
      "rewards/margins": 0.35650262236595154,
      "rewards/rejected": -1.6615794897079468,
      "step": 270
    },
    {
      "epoch": 0.575765506411934,
      "grad_norm": 4.215844646514211,
      "learning_rate": 2.2714738398943308e-07,
      "logits/chosen": -0.9258629083633423,
      "logits/rejected": -0.9768645167350769,
      "logps/chosen": -1.1248077154159546,
      "logps/rejected": -1.950279951095581,
      "loss": -0.1036,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -1.1248077154159546,
      "rewards/margins": 0.8254722356796265,
      "rewards/rejected": -1.950279951095581,
      "step": 275
    },
    {
      "epoch": 0.5862339701648783,
      "grad_norm": 4.592191084246013,
      "learning_rate": 2.1804923757009882e-07,
      "logits/chosen": -0.9678147435188293,
      "logits/rejected": -1.0440599918365479,
      "logps/chosen": -1.1835848093032837,
      "logps/rejected": -1.6567246913909912,
      "loss": -0.0968,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -1.1835848093032837,
      "rewards/margins": 0.4731399416923523,
      "rewards/rejected": -1.6567246913909912,
      "step": 280
    },
    {
      "epoch": 0.5967024339178225,
      "grad_norm": 12.623165618800098,
      "learning_rate": 2.089939221172446e-07,
      "logits/chosen": -0.9032037854194641,
      "logits/rejected": -0.9397640228271484,
      "logps/chosen": -1.2246944904327393,
      "logps/rejected": -1.975319504737854,
      "loss": -0.1062,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.2246944904327393,
      "rewards/margins": 0.7506248950958252,
      "rewards/rejected": -1.975319504737854,
      "step": 285
    },
    {
      "epoch": 0.6071708976707668,
      "grad_norm": 5.38289989385453,
      "learning_rate": 1.9999357655598891e-07,
      "logits/chosen": -0.909401535987854,
      "logits/rejected": -1.0137306451797485,
      "logps/chosen": -1.3724336624145508,
      "logps/rejected": -2.2477662563323975,
      "loss": -0.1042,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -1.3724336624145508,
      "rewards/margins": 0.8753326535224915,
      "rewards/rejected": -2.2477662563323975,
      "step": 290
    },
    {
      "epoch": 0.6176393614237111,
      "grad_norm": 7.401969971775179,
      "learning_rate": 1.9106026612264315e-07,
      "logits/chosen": -0.8931488990783691,
      "logits/rejected": -0.9808778762817383,
      "logps/chosen": -1.287915825843811,
      "logps/rejected": -2.0314321517944336,
      "loss": -0.0964,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -1.287915825843811,
      "rewards/margins": 0.7435161471366882,
      "rewards/rejected": -2.0314321517944336,
      "step": 295
    },
    {
      "epoch": 0.6281078251766553,
      "grad_norm": 6.296173137370066,
      "learning_rate": 1.8220596619089573e-07,
      "logits/chosen": -0.9120398759841919,
      "logits/rejected": -0.9633253216743469,
      "logps/chosen": -1.3466119766235352,
      "logps/rejected": -1.948876976966858,
      "loss": -0.0998,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.3466119766235352,
      "rewards/margins": 0.6022650003433228,
      "rewards/rejected": -1.948876976966858,
      "step": 300
    },
    {
      "epoch": 0.6385762889295996,
      "grad_norm": 6.12276434326567,
      "learning_rate": 1.7344254621846017e-07,
      "logits/chosen": -0.8898782730102539,
      "logits/rejected": -0.9864279627799988,
      "logps/chosen": -1.3288440704345703,
      "logps/rejected": -2.1605937480926514,
      "loss": -0.1155,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -1.3288440704345703,
      "rewards/margins": 0.8317497372627258,
      "rewards/rejected": -2.1605937480926514,
      "step": 305
    },
    {
      "epoch": 0.6490447526825438,
      "grad_norm": 35.82187771529538,
      "learning_rate": 1.647817538357072e-07,
      "logits/chosen": -0.917158305644989,
      "logits/rejected": -0.9460798501968384,
      "logps/chosen": -1.1947271823883057,
      "logps/rejected": -1.9670846462249756,
      "loss": -0.1061,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -1.1947271823883057,
      "rewards/margins": 0.7723575830459595,
      "rewards/rejected": -1.9670846462249756,
      "step": 310
    },
    {
      "epoch": 0.6595132164354881,
      "grad_norm": 7.813472688510903,
      "learning_rate": 1.562351990976095e-07,
      "logits/chosen": -0.9020414352416992,
      "logits/rejected": -0.9904358983039856,
      "logps/chosen": -1.1299612522125244,
      "logps/rejected": -2.129662036895752,
      "loss": -0.1206,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -1.1299612522125244,
      "rewards/margins": 0.9997007250785828,
      "rewards/rejected": -2.129662036895752,
      "step": 315
    },
    {
      "epoch": 0.6699816801884323,
      "grad_norm": 10.278529226899488,
      "learning_rate": 1.478143389201113e-07,
      "logits/chosen": -0.9329323768615723,
      "logits/rejected": -0.9159154891967773,
      "logps/chosen": -1.2115180492401123,
      "logps/rejected": -1.8157265186309814,
      "loss": -0.1025,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.2115180492401123,
      "rewards/margins": 0.6042084097862244,
      "rewards/rejected": -1.8157265186309814,
      "step": 320
    },
    {
      "epoch": 0.6804501439413766,
      "grad_norm": 4.623648192830834,
      "learning_rate": 1.3953046172178413e-07,
      "logits/chosen": -0.9241701364517212,
      "logits/rejected": -0.9985548257827759,
      "logps/chosen": -1.2149182558059692,
      "logps/rejected": -1.8975908756256104,
      "loss": -0.1173,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.2149182558059692,
      "rewards/margins": 0.6826726198196411,
      "rewards/rejected": -1.8975908756256104,
      "step": 325
    },
    {
      "epoch": 0.6909186076943209,
      "grad_norm": 4.284608818953695,
      "learning_rate": 1.3139467229135998e-07,
      "logits/chosen": -0.9073573350906372,
      "logits/rejected": -0.970782458782196,
      "logps/chosen": -1.1777896881103516,
      "logps/rejected": -1.862392783164978,
      "loss": -0.0988,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.1777896881103516,
      "rewards/margins": 0.6846030950546265,
      "rewards/rejected": -1.862392783164978,
      "step": 330
    },
    {
      "epoch": 0.7013870714472651,
      "grad_norm": 11.616905967515871,
      "learning_rate": 1.2341787690142435e-07,
      "logits/chosen": -0.9223908185958862,
      "logits/rejected": -0.9389724731445312,
      "logps/chosen": -1.1395795345306396,
      "logps/rejected": -1.7211112976074219,
      "loss": -0.0961,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -1.1395795345306396,
      "rewards/margins": 0.5815316438674927,
      "rewards/rejected": -1.7211112976074219,
      "step": 335
    },
    {
      "epoch": 0.7118555352002094,
      "grad_norm": 6.760822618737345,
      "learning_rate": 1.1561076868822755e-07,
      "logits/chosen": -0.8945438265800476,
      "logits/rejected": -0.9830290079116821,
      "logps/chosen": -1.3890964984893799,
      "logps/rejected": -2.09273624420166,
      "loss": -0.1051,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -1.3890964984893799,
      "rewards/margins": 0.7036401033401489,
      "rewards/rejected": -2.09273624420166,
      "step": 340
    },
    {
      "epoch": 0.7223239989531536,
      "grad_norm": 5.837717059446264,
      "learning_rate": 1.0798381331721107e-07,
      "logits/chosen": -0.920322060585022,
      "logits/rejected": -0.977412223815918,
      "logps/chosen": -1.2339359521865845,
      "logps/rejected": -2.0405447483062744,
      "loss": -0.1013,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.2339359521865845,
      "rewards/margins": 0.8066087961196899,
      "rewards/rejected": -2.0405447483062744,
      "step": 345
    },
    {
      "epoch": 0.7327924627060979,
      "grad_norm": 5.994873994092868,
      "learning_rate": 1.0054723495346482e-07,
      "logits/chosen": -0.9375583529472351,
      "logits/rejected": -0.9326186180114746,
      "logps/chosen": -1.276450753211975,
      "logps/rejected": -1.994940161705017,
      "loss": -0.0972,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -1.276450753211975,
      "rewards/margins": 0.7184895277023315,
      "rewards/rejected": -1.994940161705017,
      "step": 350
    },
    {
      "epoch": 0.7432609264590422,
      "grad_norm": 7.791758915935874,
      "learning_rate": 9.331100255592436e-08,
      "logits/chosen": -0.9378272294998169,
      "logits/rejected": -0.9758423566818237,
      "logps/chosen": -1.1630890369415283,
      "logps/rejected": -2.039836883544922,
      "loss": -0.1098,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -1.1630890369415283,
      "rewards/margins": 0.876747727394104,
      "rewards/rejected": -2.039836883544922,
      "step": 355
    },
    {
      "epoch": 0.7537293902119864,
      "grad_norm": 13.09750518578683,
      "learning_rate": 8.628481651367875e-08,
      "logits/chosen": -0.880013108253479,
      "logits/rejected": -0.9332892298698425,
      "logps/chosen": -1.1081321239471436,
      "logps/rejected": -1.8383991718292236,
      "loss": -0.1127,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -1.1081321239471436,
      "rewards/margins": 0.7302672266960144,
      "rewards/rejected": -1.8383991718292236,
      "step": 360
    },
    {
      "epoch": 0.7641978539649307,
      "grad_norm": 9.380534898584633,
      "learning_rate": 7.947809564230445e-08,
      "logits/chosen": -0.9200069308280945,
      "logits/rejected": -0.9297819137573242,
      "logps/chosen": -1.3429687023162842,
      "logps/rejected": -2.1255431175231934,
      "loss": -0.1104,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -1.3429687023162842,
      "rewards/margins": 0.7825741171836853,
      "rewards/rejected": -2.1255431175231934,
      "step": 365
    },
    {
      "epoch": 0.7746663177178749,
      "grad_norm": 8.565425388220058,
      "learning_rate": 7.289996455765748e-08,
      "logits/chosen": -0.9139026403427124,
      "logits/rejected": -0.935566246509552,
      "logps/chosen": -1.2977614402770996,
      "logps/rejected": -2.170118808746338,
      "loss": -0.1132,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -1.2977614402770996,
      "rewards/margins": 0.8723572492599487,
      "rewards/rejected": -2.170118808746338,
      "step": 370
    },
    {
      "epoch": 0.7851347814708192,
      "grad_norm": 8.601153230537866,
      "learning_rate": 6.655924144404906e-08,
      "logits/chosen": -0.9767621755599976,
      "logits/rejected": -0.948783278465271,
      "logps/chosen": -1.324704885482788,
      "logps/rejected": -1.9237314462661743,
      "loss": -0.1008,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -1.324704885482788,
      "rewards/margins": 0.5990264415740967,
      "rewards/rejected": -1.9237314462661743,
      "step": 375
    },
    {
      "epoch": 0.7956032452237635,
      "grad_norm": 6.863136889215709,
      "learning_rate": 6.046442623320145e-08,
      "logits/chosen": -0.9905835390090942,
      "logits/rejected": -0.9240586161613464,
      "logps/chosen": -1.3711433410644531,
      "logps/rejected": -1.975137710571289,
      "loss": -0.1128,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -1.3711433410644531,
      "rewards/margins": 0.6039943695068359,
      "rewards/rejected": -1.975137710571289,
      "step": 380
    },
    {
      "epoch": 0.8060717089767077,
      "grad_norm": 10.366780250831454,
      "learning_rate": 5.4623689209832484e-08,
      "logits/chosen": -0.9187868237495422,
      "logits/rejected": -0.9430438876152039,
      "logps/chosen": -1.181815505027771,
      "logps/rejected": -1.8992712497711182,
      "loss": -0.1144,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -1.181815505027771,
      "rewards/margins": 0.7174557447433472,
      "rewards/rejected": -1.8992712497711182,
      "step": 385
    },
    {
      "epoch": 0.816540172729652,
      "grad_norm": 7.927037671256472,
      "learning_rate": 4.904486005914027e-08,
      "logits/chosen": -0.907934308052063,
      "logits/rejected": -0.995517909526825,
      "logps/chosen": -1.231623888015747,
      "logps/rejected": -2.0885114669799805,
      "loss": -0.1057,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -1.231623888015747,
      "rewards/margins": 0.8568875193595886,
      "rewards/rejected": -2.0885114669799805,
      "step": 390
    },
    {
      "epoch": 0.8270086364825961,
      "grad_norm": 9.128577656956017,
      "learning_rate": 4.373541737087263e-08,
      "logits/chosen": -0.9012504816055298,
      "logits/rejected": -0.9742817878723145,
      "logps/chosen": -1.2018259763717651,
      "logps/rejected": -2.0016086101531982,
      "loss": -0.0976,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -1.2018259763717651,
      "rewards/margins": 0.7997827529907227,
      "rewards/rejected": -2.0016086101531982,
      "step": 395
    },
    {
      "epoch": 0.8374771002355405,
      "grad_norm": 7.6959080621814175,
      "learning_rate": 3.8702478614051345e-08,
      "logits/chosen": -0.8954795598983765,
      "logits/rejected": -0.9198480844497681,
      "logps/chosen": -1.1422559022903442,
      "logps/rejected": -1.9656658172607422,
      "loss": -0.1147,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -1.1422559022903442,
      "rewards/margins": 0.823409914970398,
      "rewards/rejected": -1.9656658172607422,
      "step": 400
    },
    {
      "epoch": 0.8479455639884846,
      "grad_norm": 37.567595483922176,
      "learning_rate": 3.3952790595787986e-08,
      "logits/chosen": -0.9429194331169128,
      "logits/rejected": -0.9478951692581177,
      "logps/chosen": -1.3765652179718018,
      "logps/rejected": -2.215216636657715,
      "loss": -0.0958,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -1.3765652179718018,
      "rewards/margins": 0.8386514782905579,
      "rewards/rejected": -2.215216636657715,
      "step": 405
    },
    {
      "epoch": 0.8584140277414289,
      "grad_norm": 7.455321380026374,
      "learning_rate": 2.9492720416985e-08,
      "logits/chosen": -0.9236493110656738,
      "logits/rejected": -0.9541531801223755,
      "logps/chosen": -1.2001044750213623,
      "logps/rejected": -1.9708802700042725,
      "loss": -0.1062,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.2001044750213623,
      "rewards/margins": 0.7707759737968445,
      "rewards/rejected": -1.9708802700042725,
      "step": 410
    },
    {
      "epoch": 0.8688824914943732,
      "grad_norm": 6.175605994677344,
      "learning_rate": 2.5328246937043525e-08,
      "logits/chosen": -0.9655213356018066,
      "logits/rejected": -1.0396745204925537,
      "logps/chosen": -1.2312039136886597,
      "logps/rejected": -2.045647382736206,
      "loss": -0.113,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -1.2312039136886597,
      "rewards/margins": 0.8144432306289673,
      "rewards/rejected": -2.045647382736206,
      "step": 415
    },
    {
      "epoch": 0.8793509552473174,
      "grad_norm": 10.76836628721284,
      "learning_rate": 2.1464952759020856e-08,
      "logits/chosen": -0.9188562631607056,
      "logits/rejected": -0.925468921661377,
      "logps/chosen": -1.2931550741195679,
      "logps/rejected": -2.1757659912109375,
      "loss": -0.1001,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -1.2931550741195679,
      "rewards/margins": 0.8826107978820801,
      "rewards/rejected": -2.1757659912109375,
      "step": 420
    },
    {
      "epoch": 0.8898194190002617,
      "grad_norm": 18.620013597425544,
      "learning_rate": 1.7908016745981856e-08,
      "logits/chosen": -0.9154056310653687,
      "logits/rejected": -0.9359448552131653,
      "logps/chosen": -1.3699318170547485,
      "logps/rejected": -2.101792097091675,
      "loss": -0.1114,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.3699318170547485,
      "rewards/margins": 0.7318603992462158,
      "rewards/rejected": -2.101792097091675,
      "step": 425
    },
    {
      "epoch": 0.9002878827532059,
      "grad_norm": 9.476057330917156,
      "learning_rate": 1.4662207078575684e-08,
      "logits/chosen": -0.916486918926239,
      "logits/rejected": -0.9570449590682983,
      "logps/chosen": -1.3201847076416016,
      "logps/rejected": -2.078465223312378,
      "loss": -0.107,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -1.3201847076416016,
      "rewards/margins": 0.7582806348800659,
      "rewards/rejected": -2.078465223312378,
      "step": 430
    },
    {
      "epoch": 0.9107563465061502,
      "grad_norm": 24.560875979085978,
      "learning_rate": 1.1731874863145142e-08,
      "logits/chosen": -0.9874083399772644,
      "logits/rejected": -1.0102823972702026,
      "logps/chosen": -1.3655614852905273,
      "logps/rejected": -2.2522528171539307,
      "loss": -0.1182,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.3655614852905273,
      "rewards/margins": 0.8866912126541138,
      "rewards/rejected": -2.2522528171539307,
      "step": 435
    },
    {
      "epoch": 0.9212248102590945,
      "grad_norm": 8.256652917309696,
      "learning_rate": 9.12094829893642e-09,
      "logits/chosen": -0.9290092587471008,
      "logits/rejected": -0.9523330926895142,
      "logps/chosen": -1.283372163772583,
      "logps/rejected": -1.9379644393920898,
      "loss": -0.1093,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.283372163772583,
      "rewards/margins": 0.6545922160148621,
      "rewards/rejected": -1.9379644393920898,
      "step": 440
    },
    {
      "epoch": 0.9316932740120387,
      "grad_norm": 8.586204837874826,
      "learning_rate": 6.832927412229017e-09,
      "logits/chosen": -0.9241235852241516,
      "logits/rejected": -0.9258465766906738,
      "logps/chosen": -1.1934940814971924,
      "logps/rejected": -1.9461805820465088,
      "loss": -0.0992,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -1.1934940814971924,
      "rewards/margins": 0.7526863813400269,
      "rewards/rejected": -1.9461805820465088,
      "step": 445
    },
    {
      "epoch": 0.942161737764983,
      "grad_norm": 8.342614252776851,
      "learning_rate": 4.8708793644441086e-09,
      "logits/chosen": -0.9078584909439087,
      "logits/rejected": -0.8857234716415405,
      "logps/chosen": -1.2411367893218994,
      "logps/rejected": -1.9459234476089478,
      "loss": -0.109,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -1.2411367893218994,
      "rewards/margins": 0.7047866582870483,
      "rewards/rejected": -1.9459234476089478,
      "step": 450
    },
    {
      "epoch": 0.9526302015179272,
      "grad_norm": 24.648524026578897,
      "learning_rate": 3.2374343405217884e-09,
      "logits/chosen": -0.8968909382820129,
      "logits/rejected": -0.9244266748428345,
      "logps/chosen": -1.4007362127304077,
      "logps/rejected": -1.9055297374725342,
      "loss": -0.1035,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -1.4007362127304077,
      "rewards/margins": 0.504793643951416,
      "rewards/rejected": -1.9055297374725342,
      "step": 455
    },
    {
      "epoch": 0.9630986652708715,
      "grad_norm": 21.758607746420935,
      "learning_rate": 1.9347820230782295e-09,
      "logits/chosen": -0.9594353437423706,
      "logits/rejected": -0.9764599800109863,
      "logps/chosen": -1.2421373128890991,
      "logps/rejected": -2.0354504585266113,
      "loss": -0.1121,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -1.2421373128890991,
      "rewards/margins": 0.7933129668235779,
      "rewards/rejected": -2.0354504585266113,
      "step": 460
    },
    {
      "epoch": 0.9735671290238157,
      "grad_norm": 12.46171751515778,
      "learning_rate": 9.64668657069706e-10,
      "logits/chosen": -0.9806089401245117,
      "logits/rejected": -1.0108145475387573,
      "logps/chosen": -1.3329870700836182,
      "logps/rejected": -2.020784854888916,
      "loss": -0.1072,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -1.3329870700836182,
      "rewards/margins": 0.6877978444099426,
      "rewards/rejected": -2.020784854888916,
      "step": 465
    },
    {
      "epoch": 0.98403559277676,
      "grad_norm": 11.866295749167707,
      "learning_rate": 3.2839470889836627e-10,
      "logits/chosen": -0.9077326059341431,
      "logits/rejected": -0.9319435358047485,
      "logps/chosen": -1.2216429710388184,
      "logps/rejected": -1.813685655593872,
      "loss": -0.096,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.2216429710388184,
      "rewards/margins": 0.5920424461364746,
      "rewards/rejected": -1.813685655593872,
      "step": 470
    },
    {
      "epoch": 0.9945040565297043,
      "grad_norm": 20.786464029224504,
      "learning_rate": 2.6813123097352287e-11,
      "logits/chosen": -0.9626280069351196,
      "logits/rejected": -0.9883280992507935,
      "logps/chosen": -1.3163354396820068,
      "logps/rejected": -1.988018274307251,
      "loss": -0.0924,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -1.3163354396820068,
      "rewards/margins": 0.6716827154159546,
      "rewards/rejected": -1.988018274307251,
      "step": 475
    },
    {
      "epoch": 0.998691442030882,
      "step": 477,
      "total_flos": 0.0,
      "train_loss": -0.08717740175609069,
      "train_runtime": 14342.367,
      "train_samples_per_second": 4.263,
      "train_steps_per_second": 0.033
    }
  ],
  "logging_steps": 5,
  "max_steps": 477,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 1000000,
  "total_flos": 0.0,
  "train_batch_size": 2,
  "trial_name": null,
  "trial_params": null
}