{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 1.0,
  "eval_steps": 1000,
  "global_step": 313,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.003194888178913738,
      "grad_norm": 707.0901555036106,
      "learning_rate": 1.5625e-08,
      "logits/generated": -2.801048994064331,
      "logits/real": -2.5173754692077637,
      "logps/generated": -448.5753173828125,
      "logps/real": -119.54361724853516,
      "loss": 0.8855,
      "rewards/accuracies": 0.0,
      "rewards/generated": 0.0,
      "rewards/margins": 0.0,
      "rewards/real": 0.0,
      "step": 1
    },
    {
      "epoch": 0.03194888178913738,
      "grad_norm": 26.033036236831965,
      "learning_rate": 1.5624999999999999e-07,
      "logits/generated": -2.4279539585113525,
      "logits/real": -2.416008234024048,
      "logps/generated": -441.2065734863281,
      "logps/real": -192.90866088867188,
      "loss": 0.5492,
      "rewards/accuracies": 0.8611111044883728,
      "rewards/generated": -1.0092414617538452,
      "rewards/margins": 1.2109122276306152,
      "rewards/real": 0.20167087018489838,
      "step": 10
    },
    {
      "epoch": 0.06389776357827476,
      "grad_norm": 2.607641570692266,
      "learning_rate": 3.1249999999999997e-07,
      "logits/generated": -2.0776472091674805,
      "logits/real": -2.2696104049682617,
      "logps/generated": -528.232666015625,
      "logps/real": -141.57501220703125,
      "loss": 0.1444,
      "rewards/accuracies": 1.0,
      "rewards/generated": -8.630824089050293,
      "rewards/margins": 10.818326950073242,
      "rewards/real": 2.187502384185791,
      "step": 20
    },
    {
      "epoch": 0.09584664536741214,
      "grad_norm": 1.6797275561770264,
      "learning_rate": 4.6874999999999996e-07,
      "logits/generated": -2.0287694931030273,
      "logits/real": -2.202587604522705,
      "logps/generated": -561.3330078125,
      "logps/real": -106.47761535644531,
      "loss": 0.1182,
      "rewards/accuracies": 1.0,
      "rewards/generated": -12.7778902053833,
      "rewards/margins": 15.439559936523438,
      "rewards/real": 2.661670207977295,
      "step": 30
    },
    {
      "epoch": 0.12779552715654952,
      "grad_norm": 1.6877722773345887,
      "learning_rate": 4.857651245551601e-07,
      "logits/generated": -2.2078909873962402,
      "logits/real": -2.292013168334961,
      "logps/generated": -599.3379516601562,
      "logps/real": -124.68473052978516,
      "loss": 0.1381,
      "rewards/accuracies": 1.0,
      "rewards/generated": -13.965204238891602,
      "rewards/margins": 16.917484283447266,
      "rewards/real": 2.9522790908813477,
      "step": 40
    },
    {
      "epoch": 0.1597444089456869,
      "grad_norm": 1.97067074811704,
      "learning_rate": 4.679715302491103e-07,
      "logits/generated": -2.162761926651001,
      "logits/real": -2.3204751014709473,
      "logps/generated": -582.5556640625,
      "logps/real": -136.84011840820312,
      "loss": 0.143,
      "rewards/accuracies": 1.0,
      "rewards/generated": -14.243738174438477,
      "rewards/margins": 17.179973602294922,
      "rewards/real": 2.936236619949341,
      "step": 50
    },
    {
      "epoch": 0.19169329073482427,
      "grad_norm": 1.8076057502854346,
      "learning_rate": 4.501779359430605e-07,
      "logits/generated": -2.0397229194641113,
      "logits/real": -2.299813747406006,
      "logps/generated": -601.5155029296875,
      "logps/real": -158.8765869140625,
      "loss": 0.1215,
      "rewards/accuracies": 1.0,
      "rewards/generated": -15.693455696105957,
      "rewards/margins": 18.398269653320312,
      "rewards/real": 2.7048134803771973,
      "step": 60
    },
    {
      "epoch": 0.22364217252396165,
      "grad_norm": 1.5331566397357441,
      "learning_rate": 4.3238434163701063e-07,
      "logits/generated": -2.803489923477173,
      "logits/real": -2.78045654296875,
      "logps/generated": -619.4425048828125,
      "logps/real": -137.57711791992188,
      "loss": 0.1436,
      "rewards/accuracies": 1.0,
      "rewards/generated": -15.494806289672852,
      "rewards/margins": 18.42218780517578,
      "rewards/real": 2.927382707595825,
      "step": 70
    },
    {
      "epoch": 0.25559105431309903,
      "grad_norm": 1.511875674922715,
      "learning_rate": 4.1459074733096083e-07,
      "logits/generated": -2.690621852874756,
      "logits/real": -2.5948469638824463,
      "logps/generated": -596.6559448242188,
      "logps/real": -109.74491882324219,
      "loss": 0.1233,
      "rewards/accuracies": 1.0,
      "rewards/generated": -16.492372512817383,
      "rewards/margins": 19.500394821166992,
      "rewards/real": 3.008024215698242,
      "step": 80
    },
    {
      "epoch": 0.28753993610223644,
      "grad_norm": 1.3167801498083793,
      "learning_rate": 3.9679715302491103e-07,
      "logits/generated": -2.7257370948791504,
      "logits/real": -2.683110237121582,
      "logps/generated": -610.23583984375,
      "logps/real": -146.4161834716797,
      "loss": 0.1269,
      "rewards/accuracies": 1.0,
      "rewards/generated": -16.799118041992188,
      "rewards/margins": 19.806835174560547,
      "rewards/real": 3.007721185684204,
      "step": 90
    },
    {
      "epoch": 0.3194888178913738,
      "grad_norm": 1.6900650289885557,
      "learning_rate": 3.790035587188612e-07,
      "logits/generated": -2.734708309173584,
      "logits/real": -2.7301347255706787,
      "logps/generated": -614.1470336914062,
      "logps/real": -153.92105102539062,
      "loss": 0.1487,
      "rewards/accuracies": 1.0,
      "rewards/generated": -17.10015106201172,
      "rewards/margins": 20.158763885498047,
      "rewards/real": 3.058610677719116,
      "step": 100
    },
    {
      "epoch": 0.3514376996805112,
      "grad_norm": 1.3021727337855808,
      "learning_rate": 3.6120996441281137e-07,
      "logits/generated": -2.6350350379943848,
      "logits/real": -2.7262301445007324,
      "logps/generated": -596.9154663085938,
      "logps/real": -122.75111389160156,
      "loss": 0.1255,
      "rewards/accuracies": 1.0,
      "rewards/generated": -16.06097412109375,
      "rewards/margins": 19.084787368774414,
      "rewards/real": 3.023812770843506,
      "step": 110
    },
    {
      "epoch": 0.38338658146964855,
      "grad_norm": 1.6018857166027896,
      "learning_rate": 3.4341637010676157e-07,
      "logits/generated": -2.7175872325897217,
      "logits/real": -2.7377209663391113,
      "logps/generated": -641.0619506835938,
      "logps/real": -134.5575408935547,
      "loss": 0.1344,
      "rewards/accuracies": 1.0,
      "rewards/generated": -18.354572296142578,
      "rewards/margins": 21.459613800048828,
      "rewards/real": 3.1050407886505127,
      "step": 120
    },
    {
      "epoch": 0.41533546325878595,
      "grad_norm": 1.957334214600403,
      "learning_rate": 3.256227758007117e-07,
      "logits/generated": -2.6532580852508545,
      "logits/real": -2.6732802391052246,
      "logps/generated": -593.9332885742188,
      "logps/real": -146.0222625732422,
      "loss": 0.1467,
      "rewards/accuracies": 1.0,
      "rewards/generated": -16.877891540527344,
      "rewards/margins": 19.932432174682617,
      "rewards/real": 3.054542064666748,
      "step": 130
    },
    {
      "epoch": 0.4472843450479233,
      "grad_norm": 1.3941013687837516,
      "learning_rate": 3.078291814946619e-07,
      "logits/generated": -2.655681848526001,
      "logits/real": -2.6176505088806152,
      "logps/generated": -563.309814453125,
      "logps/real": -146.92739868164062,
      "loss": 0.1287,
      "rewards/accuracies": 1.0,
      "rewards/generated": -14.66212272644043,
      "rewards/margins": 17.799179077148438,
      "rewards/real": 3.1370577812194824,
      "step": 140
    },
    {
      "epoch": 0.4792332268370607,
      "grad_norm": 1.4882690191731298,
      "learning_rate": 2.900355871886121e-07,
      "logits/generated": -2.747220993041992,
      "logits/real": -2.701289176940918,
      "logps/generated": -579.5242919921875,
      "logps/real": -130.2000274658203,
      "loss": 0.1296,
      "rewards/accuracies": 1.0,
      "rewards/generated": -16.728702545166016,
      "rewards/margins": 19.850534439086914,
      "rewards/real": 3.1218338012695312,
      "step": 150
    },
    {
      "epoch": 0.5111821086261981,
      "grad_norm": 1.5998263047919852,
      "learning_rate": 2.7224199288256225e-07,
      "logits/generated": -2.689751148223877,
      "logits/real": -2.7137889862060547,
      "logps/generated": -598.3092041015625,
      "logps/real": -124.75559997558594,
      "loss": 0.1466,
      "rewards/accuracies": 1.0,
      "rewards/generated": -16.721622467041016,
      "rewards/margins": 19.949649810791016,
      "rewards/real": 3.2280242443084717,
      "step": 160
    },
    {
      "epoch": 0.5431309904153354,
      "grad_norm": 1.828096060541728,
      "learning_rate": 2.5444839857651245e-07,
      "logits/generated": -2.631895065307617,
      "logits/real": -2.652967929840088,
      "logps/generated": -586.2786254882812,
      "logps/real": -147.9235382080078,
      "loss": 0.1406,
      "rewards/accuracies": 1.0,
      "rewards/generated": -15.885740280151367,
      "rewards/margins": 19.117094039916992,
      "rewards/real": 3.231353282928467,
      "step": 170
    },
    {
      "epoch": 0.5750798722044729,
      "grad_norm": 1.4197910889449137,
      "learning_rate": 2.366548042704626e-07,
      "logits/generated": -2.669239044189453,
      "logits/real": -2.614443063735962,
      "logps/generated": -651.5192260742188,
      "logps/real": -123.7098617553711,
      "loss": 0.1397,
      "rewards/accuracies": 1.0,
      "rewards/generated": -17.95551109313965,
      "rewards/margins": 21.130229949951172,
      "rewards/real": 3.1747171878814697,
      "step": 180
    },
    {
      "epoch": 0.6070287539936102,
      "grad_norm": 1.623253543306337,
      "learning_rate": 2.188612099644128e-07,
      "logits/generated": -2.6021995544433594,
      "logits/real": -2.6068227291107178,
      "logps/generated": -603.6708374023438,
      "logps/real": -133.12811279296875,
      "loss": 0.1504,
      "rewards/accuracies": 1.0,
      "rewards/generated": -17.770116806030273,
      "rewards/margins": 21.024080276489258,
      "rewards/real": 3.2539620399475098,
      "step": 190
    },
    {
      "epoch": 0.6389776357827476,
      "grad_norm": 1.6934472036753703,
      "learning_rate": 2.0106761565836297e-07,
      "logits/generated": -2.6339528560638428,
      "logits/real": -2.5701403617858887,
      "logps/generated": -586.7174072265625,
      "logps/real": -126.43733978271484,
      "loss": 0.1329,
      "rewards/accuracies": 1.0,
      "rewards/generated": -16.226985931396484,
      "rewards/margins": 19.362442016601562,
      "rewards/real": 3.1354544162750244,
      "step": 200
    },
    {
      "epoch": 0.670926517571885,
      "grad_norm": 1.498510937597327,
      "learning_rate": 1.8327402135231316e-07,
      "logits/generated": -2.8424220085144043,
      "logits/real": -2.6752820014953613,
      "logps/generated": -605.6686401367188,
      "logps/real": -122.51239013671875,
      "loss": 0.1198,
      "rewards/accuracies": 1.0,
      "rewards/generated": -16.924907684326172,
      "rewards/margins": 20.049123764038086,
      "rewards/real": 3.1242165565490723,
      "step": 210
    },
    {
      "epoch": 0.7028753993610224,
      "grad_norm": 1.4317115723777145,
      "learning_rate": 1.6548042704626334e-07,
      "logits/generated": -2.7989039421081543,
      "logits/real": -2.6403791904449463,
      "logps/generated": -624.21826171875,
      "logps/real": -137.1665496826172,
      "loss": 0.1386,
      "rewards/accuracies": 1.0,
      "rewards/generated": -18.16226577758789,
      "rewards/margins": 21.232851028442383,
      "rewards/real": 3.070587635040283,
      "step": 220
    },
    {
      "epoch": 0.7348242811501597,
      "grad_norm": 1.303859194823977,
      "learning_rate": 1.476868327402135e-07,
      "logits/generated": -2.635648250579834,
      "logits/real": -2.5836873054504395,
      "logps/generated": -611.7884521484375,
      "logps/real": -139.7376708984375,
      "loss": 0.1379,
      "rewards/accuracies": 1.0,
      "rewards/generated": -16.920841217041016,
      "rewards/margins": 20.23355484008789,
      "rewards/real": 3.312713623046875,
      "step": 230
    },
    {
      "epoch": 0.7667731629392971,
      "grad_norm": 1.671858324479984,
      "learning_rate": 1.298932384341637e-07,
      "logits/generated": -2.835536003112793,
      "logits/real": -2.5438485145568848,
      "logps/generated": -604.9158325195312,
      "logps/real": -90.45211029052734,
      "loss": 0.1236,
      "rewards/accuracies": 1.0,
      "rewards/generated": -18.167110443115234,
      "rewards/margins": 21.106388092041016,
      "rewards/real": 2.939276933670044,
      "step": 240
    },
    {
      "epoch": 0.7987220447284346,
      "grad_norm": 1.4302765337757193,
      "learning_rate": 1.1209964412811388e-07,
      "logits/generated": -2.773268938064575,
      "logits/real": -2.7092514038085938,
      "logps/generated": -657.0368041992188,
      "logps/real": -131.35690307617188,
      "loss": 0.1284,
      "rewards/accuracies": 1.0,
      "rewards/generated": -19.432615280151367,
      "rewards/margins": 22.64904022216797,
      "rewards/real": 3.2164230346679688,
      "step": 250
    },
    {
      "epoch": 0.8306709265175719,
      "grad_norm": 1.443073941719056,
      "learning_rate": 9.430604982206406e-08,
      "logits/generated": -2.756702184677124,
      "logits/real": -2.677367687225342,
      "logps/generated": -634.17041015625,
      "logps/real": -103.53001403808594,
      "loss": 0.1157,
      "rewards/accuracies": 1.0,
      "rewards/generated": -18.820682525634766,
      "rewards/margins": 21.887561798095703,
      "rewards/real": 3.0668790340423584,
      "step": 260
    },
    {
      "epoch": 0.8626198083067093,
      "grad_norm": 1.8796921765501535,
      "learning_rate": 7.651245551601423e-08,
      "logits/generated": -2.7278664112091064,
      "logits/real": -2.7242510318756104,
      "logps/generated": -606.2032470703125,
      "logps/real": -182.83935546875,
      "loss": 0.1603,
      "rewards/accuracies": 1.0,
      "rewards/generated": -18.959697723388672,
      "rewards/margins": 22.330434799194336,
      "rewards/real": 3.3707375526428223,
      "step": 270
    },
    {
      "epoch": 0.8945686900958466,
      "grad_norm": 1.553843219847652,
      "learning_rate": 5.871886120996441e-08,
      "logits/generated": -2.738983154296875,
      "logits/real": -2.6845531463623047,
      "logps/generated": -631.7647705078125,
      "logps/real": -175.20382690429688,
      "loss": 0.1264,
      "rewards/accuracies": 1.0,
      "rewards/generated": -19.304214477539062,
      "rewards/margins": 22.55462646484375,
      "rewards/real": 3.2504119873046875,
      "step": 280
    },
    {
      "epoch": 0.9265175718849841,
      "grad_norm": 1.8666524089612875,
      "learning_rate": 4.092526690391459e-08,
      "logits/generated": -2.6846232414245605,
      "logits/real": -2.6559813022613525,
      "logps/generated": -589.41796875,
      "logps/real": -121.3634262084961,
      "loss": 0.1282,
      "rewards/accuracies": 1.0,
      "rewards/generated": -18.461414337158203,
      "rewards/margins": 21.56503677368164,
      "rewards/real": 3.103621006011963,
      "step": 290
    },
    {
      "epoch": 0.9584664536741214,
      "grad_norm": 1.9808591697359095,
      "learning_rate": 2.3131672597864765e-08,
      "logits/generated": -2.789712905883789,
      "logits/real": -2.6096606254577637,
      "logps/generated": -629.02294921875,
      "logps/real": -189.27085876464844,
      "loss": 0.1435,
      "rewards/accuracies": 1.0,
      "rewards/generated": -18.036930084228516,
      "rewards/margins": 21.457435607910156,
      "rewards/real": 3.4205050468444824,
      "step": 300
    },
    {
      "epoch": 0.9904153354632588,
      "grad_norm": 1.5167523084769354,
      "learning_rate": 5.338078291814947e-09,
      "logits/generated": -2.786799192428589,
      "logits/real": -2.7553584575653076,
      "logps/generated": -612.6634521484375,
      "logps/real": -153.3998260498047,
      "loss": 0.1292,
      "rewards/accuracies": 1.0,
      "rewards/generated": -17.887155532836914,
      "rewards/margins": 21.20827293395996,
      "rewards/real": 3.321115016937256,
      "step": 310
    },
    {
      "epoch": 1.0,
      "step": 313,
      "total_flos": 0.0,
      "train_loss": 0.14877043307398835,
      "train_runtime": 1689.625,
      "train_samples_per_second": 2.959,
      "train_steps_per_second": 0.185
    }
  ],
  "logging_steps": 10,
  "max_steps": 313,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 1000,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": true
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 4,
  "trial_name": null,
  "trial_params": null
}