{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 1.0,
  "eval_steps": 500,
  "global_step": 1563,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.0,
      "learning_rate": 3.1847133757961784e-09,
      "logits/generated": -Infinity,
      "logits/real": -Infinity,
      "logps/generated": -539.3351440429688,
      "logps/real": -367.9522399902344,
      "loss": 1.6159,
      "rewards/accuracies": 0.625,
      "rewards/generated": -21.398611068725586,
      "rewards/margins": 5.284192085266113,
      "rewards/real": -16.114418029785156,
      "step": 1
    },
    {
      "epoch": 0.01,
      "learning_rate": 3.184713375796178e-08,
      "logits/generated": -Infinity,
      "logits/real": -Infinity,
      "logps/generated": -466.6612548828125,
      "logps/real": -352.3954162597656,
      "loss": 2.9155,
      "rewards/accuracies": 0.6944444179534912,
      "rewards/generated": -16.62398910522461,
      "rewards/margins": 6.3880696296691895,
      "rewards/real": -10.235919952392578,
      "step": 10
    },
    {
      "epoch": 0.01,
      "learning_rate": 6.369426751592356e-08,
      "logits/generated": -Infinity,
      "logits/real": -Infinity,
      "logps/generated": -494.484375,
      "logps/real": -385.1441955566406,
      "loss": 3.3003,
      "rewards/accuracies": 0.699999988079071,
      "rewards/generated": -20.39480972290039,
      "rewards/margins": 6.397805213928223,
      "rewards/real": -13.997003555297852,
      "step": 20
    },
    {
      "epoch": 0.02,
      "learning_rate": 9.554140127388536e-08,
      "logits/generated": -Infinity,
      "logits/real": -Infinity,
      "logps/generated": -502.4696350097656,
      "logps/real": -407.5299987792969,
      "loss": 4.0551,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/generated": -21.359249114990234,
      "rewards/margins": 6.890555381774902,
      "rewards/real": -14.4686918258667,
      "step": 30
    },
    {
      "epoch": 0.03,
      "learning_rate": 1.2738853503184713e-07,
      "logits/generated": -Infinity,
      "logits/real": -Infinity,
      "logps/generated": -453.08099365234375,
      "logps/real": -369.57183837890625,
      "loss": 3.4836,
      "rewards/accuracies": 0.675000011920929,
      "rewards/generated": -16.532052993774414,
      "rewards/margins": 5.033951759338379,
      "rewards/real": -11.498100280761719,
      "step": 40
    },
    {
      "epoch": 0.03,
      "learning_rate": 1.592356687898089e-07,
      "logits/generated": -Infinity,
      "logits/real": -Infinity,
      "logps/generated": -496.9840393066406,
      "logps/real": -375.58489990234375,
      "loss": 3.6846,
      "rewards/accuracies": 0.762499988079071,
      "rewards/generated": -21.554412841796875,
      "rewards/margins": 8.116512298583984,
      "rewards/real": -13.437899589538574,
      "step": 50
    },
    {
      "epoch": 0.04,
      "learning_rate": 1.9108280254777072e-07,
      "logits/generated": -Infinity,
      "logits/real": -Infinity,
      "logps/generated": -510.9554138183594,
      "logps/real": -394.2112731933594,
      "loss": 2.9311,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/generated": -22.012584686279297,
      "rewards/margins": 7.591399192810059,
      "rewards/real": -14.421185493469238,
      "step": 60
    },
    {
      "epoch": 0.04,
      "learning_rate": 2.2292993630573247e-07,
      "logits/generated": -Infinity,
      "logits/real": -Infinity,
      "logps/generated": -500.39190673828125,
      "logps/real": -409.89202880859375,
      "loss": 3.2596,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/generated": -21.793880462646484,
      "rewards/margins": 6.847817897796631,
      "rewards/real": -14.946063041687012,
      "step": 70
    },
    {
      "epoch": 0.05,
      "learning_rate": 2.5477707006369425e-07,
      "logits/generated": -Infinity,
      "logits/real": -Infinity,
      "logps/generated": -502.09320068359375,
      "logps/real": -387.62890625,
      "loss": 2.3412,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/generated": -21.89040184020996,
      "rewards/margins": 7.6944780349731445,
      "rewards/real": -14.19592571258545,
      "step": 80
    },
    {
      "epoch": 0.06,
      "learning_rate": 2.86624203821656e-07,
      "logits/generated": -Infinity,
      "logits/real": -Infinity,
      "logps/generated": -557.3800048828125,
      "logps/real": -414.1551818847656,
      "loss": 1.9592,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/generated": -26.10390281677246,
      "rewards/margins": 10.303709983825684,
      "rewards/real": -15.800193786621094,
      "step": 90
    },
    {
      "epoch": 0.06,
      "learning_rate": 3.184713375796178e-07,
      "logits/generated": -Infinity,
      "logits/real": -Infinity,
      "logps/generated": -543.3717651367188,
      "logps/real": -368.0298156738281,
      "loss": 1.7186,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/generated": -25.229318618774414,
      "rewards/margins": 12.685417175292969,
      "rewards/real": -12.543901443481445,
      "step": 100
    },
    {
      "epoch": 0.07,
      "learning_rate": 3.5031847133757957e-07,
      "logits/generated": -Infinity,
      "logits/real": -Infinity,
      "logps/generated": -596.9669189453125,
      "logps/real": -413.40350341796875,
      "loss": 1.4592,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/generated": -30.341022491455078,
      "rewards/margins": 15.404217720031738,
      "rewards/real": -14.936800956726074,
      "step": 110
    },
    {
      "epoch": 0.08,
      "learning_rate": 3.8216560509554143e-07,
      "logits/generated": -Infinity,
      "logits/real": -Infinity,
      "logps/generated": -569.759033203125,
      "logps/real": -374.6027526855469,
      "loss": 1.1146,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/generated": -26.69822120666504,
      "rewards/margins": 14.171666145324707,
      "rewards/real": -12.526556015014648,
      "step": 120
    },
    {
      "epoch": 0.08,
      "learning_rate": 4.140127388535032e-07,
      "logits/generated": -Infinity,
      "logits/real": -Infinity,
      "logps/generated": -593.4500122070312,
      "logps/real": -421.9783630371094,
      "loss": 1.2263,
      "rewards/accuracies": 0.862500011920929,
      "rewards/generated": -31.279077529907227,
      "rewards/margins": 15.886209487915039,
      "rewards/real": -15.39286994934082,
      "step": 130
    },
    {
      "epoch": 0.09,
      "learning_rate": 4.4585987261146494e-07,
      "logits/generated": -Infinity,
      "logits/real": -Infinity,
      "logps/generated": -648.861328125,
      "logps/real": -401.9364013671875,
      "loss": 0.8198,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/generated": -34.970481872558594,
      "rewards/margins": 18.90291976928711,
      "rewards/real": -16.067562103271484,
      "step": 140
    },
    {
      "epoch": 0.1,
      "learning_rate": 4.777070063694267e-07,
      "logits/generated": -Infinity,
      "logits/real": -Infinity,
      "logps/generated": -638.9237670898438,
      "logps/real": -419.0517578125,
      "loss": 1.0322,
      "rewards/accuracies": 0.862500011920929,
      "rewards/generated": -34.63279342651367,
      "rewards/margins": 18.07627296447754,
      "rewards/real": -16.556522369384766,
      "step": 150
    },
    {
      "epoch": 0.1,
      "learning_rate": 4.989331436699858e-07,
      "logits/generated": -Infinity,
      "logits/real": -Infinity,
      "logps/generated": -667.6073608398438,
      "logps/real": -418.95562744140625,
      "loss": 1.075,
      "rewards/accuracies": 0.875,
      "rewards/generated": -37.12474822998047,
      "rewards/margins": 20.29035186767578,
      "rewards/real": -16.834400177001953,
      "step": 160
    },
    {
      "epoch": 0.11,
      "learning_rate": 4.953769559032717e-07,
      "logits/generated": -Infinity,
      "logits/real": -Infinity,
      "logps/generated": -700.0158081054688,
      "logps/real": -375.8499755859375,
      "loss": 1.0279,
      "rewards/accuracies": 0.925000011920929,
      "rewards/generated": -40.3820915222168,
      "rewards/margins": 25.6898136138916,
      "rewards/real": -14.692278861999512,
      "step": 170
    },
    {
      "epoch": 0.12,
      "learning_rate": 4.918207681365576e-07,
      "logits/generated": -Infinity,
      "logits/real": -Infinity,
      "logps/generated": -680.1141357421875,
      "logps/real": -433.60052490234375,
      "loss": 0.7383,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/generated": -36.7303352355957,
      "rewards/margins": 21.14933967590332,
      "rewards/real": -15.5809965133667,
      "step": 180
    },
    {
      "epoch": 0.12,
      "learning_rate": 4.882645803698435e-07,
      "logits/generated": -Infinity,
      "logits/real": -Infinity,
      "logps/generated": -654.9556274414062,
      "logps/real": -447.13519287109375,
      "loss": 0.8003,
      "rewards/accuracies": 0.887499988079071,
      "rewards/generated": -37.65789031982422,
      "rewards/margins": 19.25699806213379,
      "rewards/real": -18.40089225769043,
      "step": 190
    },
    {
      "epoch": 0.13,
      "learning_rate": 4.847083926031294e-07,
      "logits/generated": -Infinity,
      "logits/real": -Infinity,
      "logps/generated": -667.2574462890625,
      "logps/real": -390.1756591796875,
      "loss": 0.4891,
      "rewards/accuracies": 0.949999988079071,
      "rewards/generated": -37.77182388305664,
      "rewards/margins": 22.858360290527344,
      "rewards/real": -14.913459777832031,
      "step": 200
    },
    {
      "epoch": 0.13,
      "learning_rate": 4.811522048364154e-07,
      "logits/generated": -Infinity,
      "logits/real": -Infinity,
      "logps/generated": -710.2974853515625,
      "logps/real": -364.0440673828125,
      "loss": 0.4879,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/generated": -42.257808685302734,
      "rewards/margins": 28.898040771484375,
      "rewards/real": -13.359766960144043,
      "step": 210
    },
    {
      "epoch": 0.14,
      "learning_rate": 4.775960170697012e-07,
      "logits/generated": -Infinity,
      "logits/real": -Infinity,
      "logps/generated": -648.4423828125,
      "logps/real": -361.45953369140625,
      "loss": 0.543,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/generated": -36.720672607421875,
      "rewards/margins": 22.87405014038086,
      "rewards/real": -13.846624374389648,
      "step": 220
    },
    {
      "epoch": 0.15,
      "learning_rate": 4.7403982930298717e-07,
      "logits/generated": -Infinity,
      "logits/real": -Infinity,
      "logps/generated": -702.8461303710938,
      "logps/real": -383.67425537109375,
      "loss": 0.4983,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/generated": -40.814414978027344,
      "rewards/margins": 26.6862850189209,
      "rewards/real": -14.128130912780762,
      "step": 230
    },
    {
      "epoch": 0.15,
      "learning_rate": 4.7048364153627306e-07,
      "logits/generated": -Infinity,
      "logits/real": -Infinity,
      "logps/generated": -666.6107788085938,
      "logps/real": -442.3255310058594,
      "loss": 0.5706,
      "rewards/accuracies": 0.9375,
      "rewards/generated": -37.09008026123047,
      "rewards/margins": 21.38768196105957,
      "rewards/real": -15.702404975891113,
      "step": 240
    },
    {
      "epoch": 0.16,
      "learning_rate": 4.66927453769559e-07,
      "logits/generated": -Infinity,
      "logits/real": -Infinity,
      "logps/generated": -685.1495971679688,
      "logps/real": -359.0677795410156,
      "loss": 0.5271,
      "rewards/accuracies": 0.9375,
      "rewards/generated": -38.3968505859375,
      "rewards/margins": 26.170297622680664,
      "rewards/real": -12.226548194885254,
      "step": 250
    },
    {
      "epoch": 0.17,
      "learning_rate": 4.633712660028449e-07,
      "logits/generated": -Infinity,
      "logits/real": -Infinity,
      "logps/generated": -679.1738891601562,
      "logps/real": -367.4692077636719,
      "loss": 0.4291,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/generated": -38.32642364501953,
      "rewards/margins": 25.107736587524414,
      "rewards/real": -13.218683242797852,
      "step": 260
    },
    {
      "epoch": 0.17,
      "learning_rate": 4.5981507823613085e-07,
      "logits/generated": -Infinity,
      "logits/real": -Infinity,
      "logps/generated": -681.1062622070312,
      "logps/real": -395.8605651855469,
      "loss": 0.6404,
      "rewards/accuracies": 0.9375,
      "rewards/generated": -39.41220474243164,
      "rewards/margins": 23.57551383972168,
      "rewards/real": -15.836690902709961,
      "step": 270
    },
    {
      "epoch": 0.18,
      "learning_rate": 4.562588904694168e-07,
      "logits/generated": -Infinity,
      "logits/real": -Infinity,
      "logps/generated": -720.9105224609375,
      "logps/real": -410.877197265625,
      "loss": 0.3044,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/generated": -43.70952606201172,
      "rewards/margins": 28.780017852783203,
      "rewards/real": -14.929506301879883,
      "step": 280
    },
    {
      "epoch": 0.19,
      "learning_rate": 4.5270270270270264e-07,
      "logits/generated": -Infinity,
      "logits/real": -Infinity,
      "logps/generated": -714.8857421875,
      "logps/real": -394.2933654785156,
      "loss": 0.2007,
      "rewards/accuracies": 0.949999988079071,
      "rewards/generated": -41.24427032470703,
      "rewards/margins": 28.501968383789062,
      "rewards/real": -12.742300987243652,
      "step": 290
    },
    {
      "epoch": 0.19,
      "learning_rate": 4.491465149359886e-07,
      "logits/generated": -Infinity,
      "logits/real": -Infinity,
      "logps/generated": -682.9447021484375,
      "logps/real": -367.23590087890625,
      "loss": 0.4911,
      "rewards/accuracies": 0.9375,
      "rewards/generated": -39.695213317871094,
      "rewards/margins": 26.347553253173828,
      "rewards/real": -13.34765911102295,
      "step": 300
    },
    {
      "epoch": 0.2,
      "learning_rate": 4.4559032716927454e-07,
      "logits/generated": -Infinity,
      "logits/real": -Infinity,
      "logps/generated": -709.5479736328125,
      "logps/real": -430.88763427734375,
      "loss": 0.1828,
      "rewards/accuracies": 0.9375,
      "rewards/generated": -42.66682052612305,
      "rewards/margins": 27.468265533447266,
      "rewards/real": -15.19856071472168,
      "step": 310
    },
    {
      "epoch": 0.2,
      "learning_rate": 4.420341394025605e-07,
      "logits/generated": -Infinity,
      "logits/real": -Infinity,
      "logps/generated": -755.2057495117188,
      "logps/real": -418.7355041503906,
      "loss": 0.3104,
      "rewards/accuracies": 0.949999988079071,
      "rewards/generated": -46.417259216308594,
      "rewards/margins": 30.384328842163086,
      "rewards/real": -16.032928466796875,
      "step": 320
    },
    {
      "epoch": 0.21,
      "learning_rate": 4.384779516358463e-07,
      "logits/generated": -Infinity,
      "logits/real": -Infinity,
      "logps/generated": -744.1261596679688,
      "logps/real": -432.3663024902344,
      "loss": 0.4376,
      "rewards/accuracies": 0.887499988079071,
      "rewards/generated": -45.77213668823242,
      "rewards/margins": 28.370372772216797,
      "rewards/real": -17.401762008666992,
      "step": 330
    },
    {
      "epoch": 0.22,
      "learning_rate": 4.3492176386913227e-07,
      "logits/generated": -Infinity,
      "logits/real": -Infinity,
      "logps/generated": -778.1282348632812,
      "logps/real": -449.26776123046875,
      "loss": 0.2628,
      "rewards/accuracies": 0.949999988079071,
      "rewards/generated": -48.66665267944336,
      "rewards/margins": 31.182621002197266,
      "rewards/real": -17.484031677246094,
      "step": 340
    },
    {
      "epoch": 0.22,
      "learning_rate": 4.313655761024182e-07,
      "logits/generated": -Infinity,
      "logits/real": -Infinity,
      "logps/generated": -756.3446655273438,
      "logps/real": -430.87493896484375,
      "loss": 0.3925,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/generated": -46.581642150878906,
      "rewards/margins": 29.290760040283203,
      "rewards/real": -17.290876388549805,
      "step": 350
    },
    {
      "epoch": 0.23,
      "learning_rate": 4.278093883357041e-07,
      "logits/generated": -Infinity,
      "logits/real": -Infinity,
      "logps/generated": -777.9512939453125,
      "logps/real": -414.18450927734375,
      "loss": 0.4194,
      "rewards/accuracies": 0.987500011920929,
      "rewards/generated": -48.169349670410156,
      "rewards/margins": 32.56087875366211,
      "rewards/real": -15.608467102050781,
      "step": 360
    },
    {
      "epoch": 0.24,
      "learning_rate": 4.2425320056899e-07,
      "logits/generated": -Infinity,
      "logits/real": -Infinity,
      "logps/generated": -776.2547607421875,
      "logps/real": -459.26971435546875,
      "loss": 0.1642,
      "rewards/accuracies": 0.987500011920929,
      "rewards/generated": -48.74531173706055,
      "rewards/margins": 30.371551513671875,
      "rewards/real": -18.373756408691406,
      "step": 370
    },
    {
      "epoch": 0.24,
      "learning_rate": 4.2069701280227595e-07,
      "logits/generated": -Infinity,
      "logits/real": -Infinity,
      "logps/generated": -787.847900390625,
      "logps/real": -431.939697265625,
      "loss": 0.1822,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/generated": -48.518409729003906,
      "rewards/margins": 31.980022430419922,
      "rewards/real": -16.538381576538086,
      "step": 380
    },
    {
      "epoch": 0.25,
      "learning_rate": 4.1714082503556185e-07,
      "logits/generated": -Infinity,
      "logits/real": -Infinity,
      "logps/generated": -782.6195678710938,
      "logps/real": -409.2000732421875,
      "loss": 0.1507,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/generated": -49.06304168701172,
      "rewards/margins": 32.46137237548828,
      "rewards/real": -16.601669311523438,
      "step": 390
    },
    {
      "epoch": 0.26,
      "learning_rate": 4.135846372688478e-07,
      "logits/generated": -Infinity,
      "logits/real": -Infinity,
      "logps/generated": -743.0278930664062,
      "logps/real": -435.90399169921875,
      "loss": 0.3316,
      "rewards/accuracies": 0.9375,
      "rewards/generated": -46.87841796875,
      "rewards/margins": 29.552413940429688,
      "rewards/real": -17.32600975036621,
      "step": 400
    },
    {
      "epoch": 0.26,
      "learning_rate": 4.100284495021337e-07,
      "logits/generated": -Infinity,
      "logits/real": -Infinity,
      "logps/generated": -779.1712646484375,
      "logps/real": -406.473876953125,
      "loss": 0.333,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/generated": -48.82392883300781,
      "rewards/margins": 33.1125602722168,
      "rewards/real": -15.711362838745117,
      "step": 410
    },
    {
      "epoch": 0.27,
      "learning_rate": 4.064722617354196e-07,
      "logits/generated": -Infinity,
      "logits/real": -Infinity,
      "logps/generated": -774.0535888671875,
      "logps/real": -417.33184814453125,
      "loss": 0.2446,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/generated": -49.45975875854492,
      "rewards/margins": 33.8248405456543,
      "rewards/real": -15.634923934936523,
      "step": 420
    },
    {
      "epoch": 0.28,
      "learning_rate": 4.0291607396870553e-07,
      "logits/generated": -Infinity,
      "logits/real": -Infinity,
      "logps/generated": -767.9479370117188,
      "logps/real": -403.72357177734375,
      "loss": 0.2645,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/generated": -48.79080581665039,
      "rewards/margins": 33.18938064575195,
      "rewards/real": -15.601428031921387,
      "step": 430
    },
    {
      "epoch": 0.28,
      "learning_rate": 3.993598862019915e-07,
      "logits/generated": -Infinity,
      "logits/real": -Infinity,
      "logps/generated": -754.3994140625,
      "logps/real": -410.26470947265625,
      "loss": 0.2166,
      "rewards/accuracies": 0.987500011920929,
      "rewards/generated": -46.57341766357422,
      "rewards/margins": 30.815637588500977,
      "rewards/real": -15.757779121398926,
      "step": 440
    },
    {
      "epoch": 0.29,
      "learning_rate": 3.9580369843527737e-07,
      "logits/generated": -Infinity,
      "logits/real": -Infinity,
      "logps/generated": -870.4566650390625,
      "logps/real": -408.28997802734375,
      "loss": 0.2063,
      "rewards/accuracies": 0.987500011920929,
      "rewards/generated": -57.19744110107422,
      "rewards/margins": 40.77977752685547,
      "rewards/real": -16.41766357421875,
      "step": 450
    },
    {
      "epoch": 0.29,
      "learning_rate": 3.9224751066856327e-07,
      "logits/generated": -Infinity,
      "logits/real": -Infinity,
      "logps/generated": -876.7347412109375,
      "logps/real": -472.7223205566406,
      "loss": 0.3418,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/generated": -57.74506378173828,
      "rewards/margins": 37.913551330566406,
      "rewards/real": -19.831506729125977,
      "step": 460
    },
    {
      "epoch": 0.3,
      "learning_rate": 3.886913229018492e-07,
      "logits/generated": -Infinity,
      "logits/real": -Infinity,
      "logps/generated": -798.3450317382812,
      "logps/real": -431.00628662109375,
      "loss": 0.5482,
      "rewards/accuracies": 0.949999988079071,
      "rewards/generated": -50.90120315551758,
      "rewards/margins": 32.8853645324707,
      "rewards/real": -18.01584243774414,
      "step": 470
    },
    {
      "epoch": 0.31,
      "learning_rate": 3.851351351351351e-07,
      "logits/generated": -Infinity,
      "logits/real": -Infinity,
      "logps/generated": -804.5404052734375,
      "logps/real": -420.87066650390625,
      "loss": 0.1632,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/generated": -51.20261764526367,
      "rewards/margins": 34.63499069213867,
      "rewards/real": -16.567626953125,
      "step": 480
    },
    {
      "epoch": 0.31,
      "learning_rate": 3.8157894736842105e-07,
      "logits/generated": -Infinity,
      "logits/real": -Infinity,
      "logps/generated": -808.2142333984375,
      "logps/real": -412.75909423828125,
      "loss": 0.067,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/generated": -51.9697265625,
      "rewards/margins": 33.47379684448242,
      "rewards/real": -18.49593162536621,
      "step": 490
    },
    {
      "epoch": 0.32,
      "learning_rate": 3.7802275960170695e-07,
      "logits/generated": -Infinity,
      "logits/real": -Infinity,
      "logps/generated": -833.8155517578125,
      "logps/real": -421.85693359375,
      "loss": 0.3303,
      "rewards/accuracies": 0.949999988079071,
      "rewards/generated": -53.8129768371582,
      "rewards/margins": 36.602169036865234,
      "rewards/real": -17.210813522338867,
      "step": 500
    },
    {
      "epoch": 0.32,
      "eval_logits/generated": -Infinity,
      "eval_logits/real": -Infinity,
      "eval_logps/generated": -529.861328125,
      "eval_logps/real": -300.84613037109375,
      "eval_loss": 0.20025908946990967,
      "eval_rewards/accuracies": 0.9371019005775452,
      "eval_rewards/generated": -23.842620849609375,
      "eval_rewards/margins": 18.996700286865234,
      "eval_rewards/real": -4.845921039581299,
      "eval_runtime": 816.3836,
      "eval_samples_per_second": 6.125,
      "eval_steps_per_second": 0.192,
      "step": 500
    },
    {
      "epoch": 0.33,
      "learning_rate": 3.7446657183499284e-07,
      "logits/generated": -Infinity,
      "logits/real": -Infinity,
      "logps/generated": -865.2786254882812,
      "logps/real": -424.654052734375,
      "loss": 0.113,
      "rewards/accuracies": 0.987500011920929,
      "rewards/generated": -56.876319885253906,
      "rewards/margins": 39.13337326049805,
      "rewards/real": -17.74294662475586,
      "step": 510
    },
    {
      "epoch": 0.33,
      "learning_rate": 3.709103840682788e-07,
      "logits/generated": -Infinity,
      "logits/real": -Infinity,
      "logps/generated": -779.5700073242188,
      "logps/real": -393.60321044921875,
      "loss": 0.1035,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/generated": -49.612606048583984,
      "rewards/margins": 33.626441955566406,
      "rewards/real": -15.986169815063477,
      "step": 520
    },
    {
      "epoch": 0.34,
      "learning_rate": 3.6735419630156474e-07,
      "logits/generated": -Infinity,
      "logits/real": -Infinity,
      "logps/generated": -776.9521484375,
      "logps/real": -400.86016845703125,
      "loss": 0.138,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/generated": -49.583370208740234,
      "rewards/margins": 32.01411819458008,
      "rewards/real": -17.569255828857422,
      "step": 530
    },
    {
      "epoch": 0.35,
      "learning_rate": 3.637980085348506e-07,
      "logits/generated": -Infinity,
      "logits/real": -Infinity,
      "logps/generated": -768.1486206054688,
      "logps/real": -421.4039611816406,
      "loss": 0.3281,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/generated": -48.634918212890625,
      "rewards/margins": 31.545923233032227,
      "rewards/real": -17.088993072509766,
      "step": 540
    },
    {
      "epoch": 0.35,
      "learning_rate": 3.602418207681365e-07,
      "logits/generated": -Infinity,
      "logits/real": -Infinity,
      "logps/generated": -798.0943603515625,
      "logps/real": -438.1573791503906,
      "loss": 0.1217,
      "rewards/accuracies": 0.949999988079071,
      "rewards/generated": -49.93975067138672,
      "rewards/margins": 32.817298889160156,
      "rewards/real": -17.12244987487793,
      "step": 550
    },
    {
      "epoch": 0.36,
      "learning_rate": 3.5668563300142247e-07,
      "logits/generated": -Infinity,
      "logits/real": -Infinity,
      "logps/generated": -784.5931396484375,
      "logps/real": -432.72039794921875,
      "loss": 0.1697,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/generated": -49.35939407348633,
      "rewards/margins": 30.70552635192871,
      "rewards/real": -18.65386962890625,
      "step": 560
    },
    {
      "epoch": 0.36,
      "learning_rate": 3.5312944523470837e-07,
      "logits/generated": -Infinity,
      "logits/real": -Infinity,
      "logps/generated": -830.1300659179688,
      "logps/real": -439.60400390625,
      "loss": 0.1078,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/generated": -53.280181884765625,
      "rewards/margins": 33.769203186035156,
      "rewards/real": -19.510984420776367,
      "step": 570
    },
    {
      "epoch": 0.37,
      "learning_rate": 3.495732574679943e-07,
      "logits/generated": -Infinity,
      "logits/real": -Infinity,
      "logps/generated": -795.7073364257812,
      "logps/real": -419.2820739746094,
      "loss": 0.3486,
      "rewards/accuracies": 0.987500011920929,
      "rewards/generated": -51.35248947143555,
      "rewards/margins": 34.13421630859375,
      "rewards/real": -17.2182674407959,
      "step": 580
    },
    {
      "epoch": 0.38,
      "learning_rate": 3.460170697012802e-07,
      "logits/generated": -Infinity,
      "logits/real": -Infinity,
      "logps/generated": -867.7032470703125,
      "logps/real": -411.7303771972656,
      "loss": 0.1504,
      "rewards/accuracies": 0.987500011920929,
      "rewards/generated": -57.48298263549805,
      "rewards/margins": 41.10187530517578,
      "rewards/real": -16.3811092376709,
      "step": 590
    },
    {
      "epoch": 0.38,
      "learning_rate": 3.424608819345661e-07,
      "logits/generated": -Infinity,
      "logits/real": -Infinity,
      "logps/generated": -883.2786865234375,
      "logps/real": -425.11041259765625,
      "loss": 0.26,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/generated": -58.78193283081055,
      "rewards/margins": 40.10378646850586,
      "rewards/real": -18.67814064025879,
      "step": 600
    },
    {
      "epoch": 0.39,
      "learning_rate": 3.3890469416785205e-07,
      "logits/generated": -Infinity,
      "logits/real": -Infinity,
      "logps/generated": -849.3904418945312,
      "logps/real": -451.054931640625,
      "loss": 0.1136,
      "rewards/accuracies": 0.949999988079071,
      "rewards/generated": -56.99665451049805,
      "rewards/margins": 37.19718933105469,
      "rewards/real": -19.79946517944336,
      "step": 610
    },
    {
      "epoch": 0.4,
      "learning_rate": 3.35348506401138e-07,
      "logits/generated": -Infinity,
      "logits/real": -Infinity,
      "logps/generated": -894.3114013671875,
      "logps/real": -414.6637268066406,
      "loss": 0.2349,
      "rewards/accuracies": 0.987500011920929,
      "rewards/generated": -59.90185546875,
      "rewards/margins": 41.685302734375,
      "rewards/real": -18.216556549072266,
      "step": 620
    },
    {
      "epoch": 0.4,
      "learning_rate": 3.3179231863442384e-07,
      "logits/generated": -Infinity,
      "logits/real": -Infinity,
      "logps/generated": -826.98681640625,
      "logps/real": -439.54559326171875,
      "loss": 0.1956,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/generated": -53.87934494018555,
      "rewards/margins": 35.65498733520508,
      "rewards/real": -18.2243595123291,
      "step": 630
    },
    {
      "epoch": 0.41,
      "learning_rate": 3.282361308677098e-07,
      "logits/generated": -Infinity,
      "logits/real": -Infinity,
      "logps/generated": -865.1568603515625,
      "logps/real": -431.68914794921875,
      "loss": 0.132,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/generated": -56.41038131713867,
      "rewards/margins": 37.95232391357422,
      "rewards/real": -18.458057403564453,
      "step": 640
    },
    {
      "epoch": 0.42,
      "learning_rate": 3.2467994310099573e-07,
      "logits/generated": -Infinity,
      "logits/real": -Infinity,
      "logps/generated": -910.2918090820312,
      "logps/real": -463.79034423828125,
      "loss": 0.0608,
      "rewards/accuracies": 1.0,
      "rewards/generated": -61.92814254760742,
      "rewards/margins": 42.76028823852539,
      "rewards/real": -19.167850494384766,
      "step": 650
    },
    {
      "epoch": 0.42,
      "learning_rate": 3.211237553342817e-07,
      "logits/generated": -Infinity,
      "logits/real": -Infinity,
      "logps/generated": -792.5632934570312,
      "logps/real": -417.36187744140625,
      "loss": 0.0828,
      "rewards/accuracies": 0.987500011920929,
      "rewards/generated": -51.489219665527344,
      "rewards/margins": 34.43842697143555,
      "rewards/real": -17.050796508789062,
      "step": 660
    },
    {
      "epoch": 0.43,
      "learning_rate": 3.175675675675675e-07,
      "logits/generated": -Infinity,
      "logits/real": -Infinity,
      "logps/generated": -870.7483520507812,
      "logps/real": -379.50860595703125,
      "loss": 0.2451,
      "rewards/accuracies": 1.0,
      "rewards/generated": -59.225135803222656,
      "rewards/margins": 44.224327087402344,
      "rewards/real": -15.000802993774414,
      "step": 670
    },
    {
      "epoch": 0.44,
      "learning_rate": 3.1401137980085347e-07,
      "logits/generated": -Infinity,
      "logits/real": -Infinity,
      "logps/generated": -930.4283447265625,
      "logps/real": -410.5220642089844,
      "loss": 0.1281,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/generated": -63.19865036010742,
      "rewards/margins": 46.53211212158203,
      "rewards/real": -16.666542053222656,
      "step": 680
    },
    {
      "epoch": 0.44,
      "learning_rate": 3.104551920341394e-07,
      "logits/generated": -Infinity,
      "logits/real": -Infinity,
      "logps/generated": -847.4844970703125,
      "logps/real": -407.2476806640625,
      "loss": 0.2284,
      "rewards/accuracies": 0.987500011920929,
      "rewards/generated": -55.74333572387695,
      "rewards/margins": 40.407684326171875,
      "rewards/real": -15.335647583007812,
      "step": 690
    },
    {
      "epoch": 0.45,
      "learning_rate": 3.068990042674253e-07,
      "logits/generated": -Infinity,
      "logits/real": -Infinity,
      "logps/generated": -811.4222412109375,
      "logps/real": -391.034912109375,
      "loss": 0.142,
      "rewards/accuracies": 0.987500011920929,
      "rewards/generated": -53.94512939453125,
      "rewards/margins": 37.505123138427734,
      "rewards/real": -16.440000534057617,
      "step": 700
    },
    {
      "epoch": 0.45,
      "learning_rate": 3.033428165007112e-07,
      "logits/generated": -Infinity,
      "logits/real": -Infinity,
      "logps/generated": -877.0598754882812,
      "logps/real": -428.3775329589844,
      "loss": 0.0922,
      "rewards/accuracies": 1.0,
      "rewards/generated": -59.027740478515625,
      "rewards/margins": 40.57575988769531,
      "rewards/real": -18.45196533203125,
      "step": 710
    },
    {
      "epoch": 0.46,
      "learning_rate": 2.9978662873399715e-07,
      "logits/generated": -Infinity,
      "logits/real": -Infinity,
      "logps/generated": -855.3162841796875,
      "logps/real": -435.65301513671875,
      "loss": 0.2617,
      "rewards/accuracies": 0.949999988079071,
      "rewards/generated": -56.983558654785156,
      "rewards/margins": 38.14686965942383,
      "rewards/real": -18.836681365966797,
      "step": 720
    },
    {
      "epoch": 0.47,
      "learning_rate": 2.9623044096728305e-07,
      "logits/generated": -Infinity,
      "logits/real": -Infinity,
      "logps/generated": -818.3048095703125,
      "logps/real": -438.0257263183594,
      "loss": 0.281,
      "rewards/accuracies": 0.887499988079071,
      "rewards/generated": -54.11011505126953,
      "rewards/margins": 37.23180389404297,
      "rewards/real": -16.878313064575195,
      "step": 730
    },
    {
      "epoch": 0.47,
      "learning_rate": 2.92674253200569e-07,
      "logits/generated": -Infinity,
      "logits/real": -Infinity,
      "logps/generated": -871.099609375,
      "logps/real": -420.02880859375,
      "loss": 0.1622,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/generated": -58.3928337097168,
      "rewards/margins": 41.17832565307617,
      "rewards/real": -17.21451187133789,
      "step": 740
    },
    {
      "epoch": 0.48,
      "learning_rate": 2.8911806543385494e-07,
      "logits/generated": -Infinity,
      "logits/real": -Infinity,
      "logps/generated": -907.7469482421875,
      "logps/real": -472.86614990234375,
      "loss": 0.2422,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/generated": -61.027503967285156,
      "rewards/margins": 42.720279693603516,
      "rewards/real": -18.30722427368164,
      "step": 750
    },
    {
      "epoch": 0.49,
      "learning_rate": 2.855618776671408e-07,
      "logits/generated": -Infinity,
      "logits/real": -Infinity,
      "logps/generated": -920.8482666015625,
      "logps/real": -464.93218994140625,
      "loss": 0.1303,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/generated": -63.184059143066406,
      "rewards/margins": 44.418663024902344,
      "rewards/real": -18.765398025512695,
      "step": 760
    },
    {
      "epoch": 0.49,
      "learning_rate": 2.8200568990042673e-07,
      "logits/generated": -Infinity,
      "logits/real": -Infinity,
      "logps/generated": -957.7092895507812,
      "logps/real": -442.88897705078125,
      "loss": 0.3922,
      "rewards/accuracies": 0.949999988079071,
      "rewards/generated": -65.42414093017578,
      "rewards/margins": 46.468353271484375,
      "rewards/real": -18.955785751342773,
      "step": 770
    },
    {
      "epoch": 0.5,
      "learning_rate": 2.784495021337127e-07,
      "logits/generated": -Infinity,
      "logits/real": -Infinity,
      "logps/generated": -908.4993286132812,
      "logps/real": -413.3855895996094,
      "loss": 0.1754,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/generated": -63.04656982421875,
      "rewards/margins": 44.62030792236328,
      "rewards/real": -18.426259994506836,
      "step": 780
    },
    {
      "epoch": 0.51,
      "learning_rate": 2.7489331436699857e-07,
      "logits/generated": -Infinity,
      "logits/real": -Infinity,
      "logps/generated": -863.7523193359375,
      "logps/real": -465.85009765625,
      "loss": 0.1958,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/generated": -56.57744216918945,
      "rewards/margins": 37.282875061035156,
      "rewards/real": -19.2945613861084,
      "step": 790
    },
    {
      "epoch": 0.51,
      "learning_rate": 2.7133712660028446e-07,
      "logits/generated": -Infinity,
      "logits/real": -Infinity,
      "logps/generated": -967.6433715820312,
      "logps/real": -416.7491149902344,
      "loss": 0.163,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/generated": -66.82258605957031,
      "rewards/margins": 49.467437744140625,
      "rewards/real": -17.355154037475586,
      "step": 800
    },
    {
      "epoch": 0.52,
      "learning_rate": 2.677809388335704e-07,
      "logits/generated": -Infinity,
      "logits/real": -Infinity,
      "logps/generated": -902.4373168945312,
      "logps/real": -438.209716796875,
      "loss": 0.1638,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/generated": -61.510841369628906,
      "rewards/margins": 41.37336349487305,
      "rewards/real": -20.13747787475586,
      "step": 810
    },
    {
      "epoch": 0.52,
      "learning_rate": 2.642247510668563e-07,
      "logits/generated": -Infinity,
      "logits/real": -Infinity,
      "logps/generated": -977.4361572265625,
      "logps/real": -426.0132751464844,
      "loss": 0.1951,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/generated": -67.98822784423828,
      "rewards/margins": 50.39772415161133,
      "rewards/real": -17.59050941467285,
      "step": 820
    },
    {
      "epoch": 0.53,
      "learning_rate": 2.6066856330014225e-07,
      "logits/generated": -Infinity,
      "logits/real": -Infinity,
      "logps/generated": -933.1921997070312,
      "logps/real": -432.3758239746094,
      "loss": 0.1135,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/generated": -64.10530090332031,
      "rewards/margins": 46.14706802368164,
      "rewards/real": -17.958240509033203,
      "step": 830
    },
    {
      "epoch": 0.54,
      "learning_rate": 2.5711237553342815e-07,
      "logits/generated": -Infinity,
      "logits/real": -Infinity,
      "logps/generated": -933.9869995117188,
      "logps/real": -469.08074951171875,
      "loss": 0.1554,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/generated": -62.14231491088867,
      "rewards/margins": 42.210784912109375,
      "rewards/real": -19.931529998779297,
      "step": 840
    },
    {
      "epoch": 0.54,
      "learning_rate": 2.5355618776671404e-07,
      "logits/generated": -Infinity,
      "logits/real": -Infinity,
      "logps/generated": -905.9029541015625,
      "logps/real": -442.23382568359375,
      "loss": 0.0891,
      "rewards/accuracies": 0.987500011920929,
      "rewards/generated": -61.816429138183594,
      "rewards/margins": 43.42655944824219,
      "rewards/real": -18.38986587524414,
      "step": 850
    },
    {
      "epoch": 0.55,
      "learning_rate": 2.5e-07,
      "logits/generated": -Infinity,
      "logits/real": -Infinity,
      "logps/generated": -903.47314453125,
      "logps/real": -426.867431640625,
      "loss": 0.2286,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/generated": -61.971595764160156,
      "rewards/margins": 43.64350891113281,
      "rewards/real": -18.328088760375977,
      "step": 860
    },
    {
      "epoch": 0.56,
      "learning_rate": 2.4644381223328594e-07,
      "logits/generated": -Infinity,
      "logits/real": -Infinity,
      "logps/generated": -904.4534301757812,
      "logps/real": -448.7220153808594,
      "loss": 0.0816,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/generated": -62.04343795776367,
      "rewards/margins": 42.920806884765625,
      "rewards/real": -19.122634887695312,
      "step": 870
    },
    {
      "epoch": 0.56,
      "learning_rate": 2.4288762446657183e-07,
      "logits/generated": -Infinity,
      "logits/real": -Infinity,
      "logps/generated": -967.4172973632812,
      "logps/real": -439.7066955566406,
      "loss": 0.1422,
      "rewards/accuracies": 1.0,
      "rewards/generated": -68.501220703125,
      "rewards/margins": 50.839942932128906,
      "rewards/real": -17.66128158569336,
      "step": 880
    },
    {
      "epoch": 0.57,
      "learning_rate": 2.393314366998578e-07,
      "logits/generated": -Infinity,
      "logits/real": -Infinity,
      "logps/generated": -949.60546875,
      "logps/real": -431.2792053222656,
      "loss": 0.2069,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/generated": -65.57563781738281,
      "rewards/margins": 47.400535583496094,
      "rewards/real": -18.175098419189453,
      "step": 890
    },
    {
      "epoch": 0.58,
      "learning_rate": 2.3577524893314365e-07,
      "logits/generated": -Infinity,
      "logits/real": -Infinity,
      "logps/generated": -958.1769409179688,
      "logps/real": -407.71038818359375,
      "loss": 0.0272,
      "rewards/accuracies": 1.0,
      "rewards/generated": -65.54978942871094,
      "rewards/margins": 49.18589401245117,
      "rewards/real": -16.363895416259766,
      "step": 900
    },
    {
      "epoch": 0.58,
      "learning_rate": 2.322190611664296e-07,
      "logits/generated": -Infinity,
      "logits/real": -Infinity,
      "logps/generated": -854.2259521484375,
      "logps/real": -461.548583984375,
      "loss": 0.2207,
      "rewards/accuracies": 0.925000011920929,
      "rewards/generated": -57.03990936279297,
      "rewards/margins": 37.42232131958008,
      "rewards/real": -19.617586135864258,
      "step": 910
    },
    {
      "epoch": 0.59,
      "learning_rate": 2.2866287339971549e-07,
      "logits/generated": -Infinity,
      "logits/real": -Infinity,
      "logps/generated": -913.2406005859375,
      "logps/real": -424.36065673828125,
      "loss": 0.1078,
      "rewards/accuracies": 0.987500011920929,
      "rewards/generated": -61.39838790893555,
      "rewards/margins": 44.66621780395508,
      "rewards/real": -16.732179641723633,
      "step": 920
    },
    {
      "epoch": 0.6,
      "learning_rate": 2.251066856330014e-07,
      "logits/generated": -Infinity,
      "logits/real": -Infinity,
      "logps/generated": -903.1484375,
      "logps/real": -405.7799987792969,
      "loss": 0.2241,
      "rewards/accuracies": 0.987500011920929,
      "rewards/generated": -62.2490234375,
      "rewards/margins": 45.68339920043945,
      "rewards/real": -16.565624237060547,
      "step": 930
    },
    {
      "epoch": 0.6,
      "learning_rate": 2.2155049786628733e-07,
      "logits/generated": -Infinity,
      "logits/real": -Infinity,
      "logps/generated": -916.8377075195312,
      "logps/real": -432.683837890625,
      "loss": 0.1258,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/generated": -61.457244873046875,
      "rewards/margins": 44.813026428222656,
      "rewards/real": -16.64422035217285,
      "step": 940
    },
    {
      "epoch": 0.61,
      "learning_rate": 2.1799431009957325e-07,
      "logits/generated": -Infinity,
      "logits/real": -Infinity,
      "logps/generated": -894.6036987304688,
      "logps/real": -406.5244445800781,
      "loss": 0.029,
      "rewards/accuracies": 0.987500011920929,
      "rewards/generated": -60.447288513183594,
      "rewards/margins": 43.23609161376953,
      "rewards/real": -17.211193084716797,
      "step": 950
    },
    {
      "epoch": 0.61,
      "learning_rate": 2.1443812233285914e-07,
      "logits/generated": -Infinity,
      "logits/real": -Infinity,
      "logps/generated": -938.0587158203125,
      "logps/real": -458.41143798828125,
      "loss": 0.1849,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/generated": -64.76756286621094,
      "rewards/margins": 45.54555892944336,
      "rewards/real": -19.222000122070312,
      "step": 960
    },
    {
      "epoch": 0.62,
      "learning_rate": 2.108819345661451e-07,
      "logits/generated": -Infinity,
      "logits/real": -Infinity,
      "logps/generated": -990.7427978515625,
      "logps/real": -451.2483825683594,
      "loss": 0.2407,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/generated": -68.94884490966797,
      "rewards/margins": 48.71486282348633,
      "rewards/real": -20.23398208618164,
      "step": 970
    },
    {
      "epoch": 0.63,
      "learning_rate": 2.0732574679943098e-07,
      "logits/generated": -Infinity,
      "logits/real": -Infinity,
      "logps/generated": -833.5616455078125,
      "logps/real": -393.88153076171875,
      "loss": 0.162,
      "rewards/accuracies": 0.9375,
      "rewards/generated": -54.51671600341797,
      "rewards/margins": 37.748043060302734,
      "rewards/real": -16.768672943115234,
      "step": 980
    },
    {
      "epoch": 0.63,
      "learning_rate": 2.0376955903271693e-07,
      "logits/generated": -Infinity,
      "logits/real": -Infinity,
      "logps/generated": -875.89453125,
      "logps/real": -475.2669982910156,
      "loss": 0.1122,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/generated": -59.66436004638672,
      "rewards/margins": 39.77927780151367,
      "rewards/real": -19.88509178161621,
      "step": 990
    },
    {
      "epoch": 0.64,
      "learning_rate": 2.0021337126600283e-07,
      "logits/generated": -Infinity,
      "logits/real": -Infinity,
      "logps/generated": -917.2653198242188,
      "logps/real": -410.7203674316406,
      "loss": 0.0933,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/generated": -63.5474853515625,
      "rewards/margins": 45.195716857910156,
      "rewards/real": -18.35177230834961,
      "step": 1000
    },
    {
      "epoch": 0.64,
      "eval_logits/generated": -Infinity,
      "eval_logits/real": -Infinity,
      "eval_logps/generated": -639.9600219726562,
      "eval_logps/real": -298.9767761230469,
      "eval_loss": 0.15978027880191803,
      "eval_rewards/accuracies": 0.9609872698783875,
      "eval_rewards/generated": -34.85248565673828,
      "eval_rewards/margins": 30.193498611450195,
      "eval_rewards/real": -4.658985614776611,
      "eval_runtime": 516.5493,
      "eval_samples_per_second": 9.68,
      "eval_steps_per_second": 0.304,
      "step": 1000
    },
    {
      "epoch": 0.65,
      "learning_rate": 1.9665718349928875e-07,
      "logits/generated": -Infinity,
      "logits/real": -Infinity,
      "logps/generated": -931.1051025390625,
      "logps/real": -400.90509033203125,
      "loss": 0.1599,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/generated": -65.04185485839844,
      "rewards/margins": 48.49779510498047,
      "rewards/real": -16.544055938720703,
      "step": 1010
    },
    {
      "epoch": 0.65,
      "learning_rate": 1.931009957325747e-07,
      "logits/generated": -Infinity,
      "logits/real": -Infinity,
      "logps/generated": -962.6014404296875,
      "logps/real": -403.762939453125,
      "loss": 0.1063,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/generated": -66.96389770507812,
      "rewards/margins": 49.26046371459961,
      "rewards/real": -17.703428268432617,
      "step": 1020
    },
    {
      "epoch": 0.66,
      "learning_rate": 1.895448079658606e-07,
      "logits/generated": -Infinity,
      "logits/real": -Infinity,
      "logps/generated": -907.5885620117188,
      "logps/real": -444.3199768066406,
      "loss": 0.1852,
      "rewards/accuracies": 0.949999988079071,
      "rewards/generated": -62.3195915222168,
      "rewards/margins": 43.62810134887695,
      "rewards/real": -18.691490173339844,
      "step": 1030
    },
    {
      "epoch": 0.67,
      "learning_rate": 1.859886201991465e-07,
      "logits/generated": -Infinity,
      "logits/real": -Infinity,
      "logps/generated": -943.9777221679688,
      "logps/real": -450.1148986816406,
      "loss": 0.237,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/generated": -65.29325866699219,
      "rewards/margins": 46.713497161865234,
      "rewards/real": -18.57975959777832,
      "step": 1040
    },
    {
      "epoch": 0.67,
      "learning_rate": 1.8243243243243243e-07,
      "logits/generated": -Infinity,
      "logits/real": -Infinity,
      "logps/generated": -988.4417724609375,
      "logps/real": -427.38702392578125,
      "loss": 0.0954,
      "rewards/accuracies": 0.987500011920929,
      "rewards/generated": -69.71871948242188,
      "rewards/margins": 52.45946502685547,
      "rewards/real": -17.259246826171875,
      "step": 1050
    },
    {
      "epoch": 0.68,
      "learning_rate": 1.7887624466571835e-07,
      "logits/generated": -Infinity,
      "logits/real": -Infinity,
      "logps/generated": -911.5084228515625,
      "logps/real": -415.616455078125,
      "loss": 0.1856,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/generated": -62.075843811035156,
      "rewards/margins": 45.55692672729492,
      "rewards/real": -16.51891326904297,
      "step": 1060
    },
    {
      "epoch": 0.68,
      "learning_rate": 1.7532005689900424e-07,
      "logits/generated": -Infinity,
      "logits/real": -Infinity,
      "logps/generated": -979.6092529296875,
      "logps/real": -422.32720947265625,
      "loss": 0.2408,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/generated": -68.75633239746094,
      "rewards/margins": 51.622406005859375,
      "rewards/real": -17.13392448425293,
      "step": 1070
    },
    {
      "epoch": 0.69,
      "learning_rate": 1.717638691322902e-07,
      "logits/generated": -Infinity,
      "logits/real": -Infinity,
      "logps/generated": -931.5291748046875,
      "logps/real": -425.2210388183594,
      "loss": 0.2356,
      "rewards/accuracies": 0.9375,
      "rewards/generated": -65.39581298828125,
      "rewards/margins": 47.234867095947266,
      "rewards/real": -18.160947799682617,
      "step": 1080
    },
    {
      "epoch": 0.7,
      "learning_rate": 1.6820768136557609e-07,
      "logits/generated": -Infinity,
      "logits/real": -Infinity,
      "logps/generated": -958.6076049804688,
      "logps/real": -406.325927734375,
      "loss": 0.0331,
      "rewards/accuracies": 1.0,
      "rewards/generated": -66.18185424804688,
      "rewards/margins": 50.36635208129883,
      "rewards/real": -15.815505981445312,
      "step": 1090
    },
    {
      "epoch": 0.7,
      "learning_rate": 1.64651493598862e-07,
      "logits/generated": -Infinity,
      "logits/real": -Infinity,
      "logps/generated": -857.2650146484375,
      "logps/real": -447.1287536621094,
      "loss": 0.1216,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/generated": -55.9595947265625,
      "rewards/margins": 37.767311096191406,
      "rewards/real": -18.192276000976562,
      "step": 1100
    },
    {
      "epoch": 0.71,
      "learning_rate": 1.6109530583214793e-07,
      "logits/generated": -Infinity,
      "logits/real": -Infinity,
      "logps/generated": -849.4425048828125,
      "logps/real": -399.19500732421875,
      "loss": 0.1411,
      "rewards/accuracies": 0.987500011920929,
      "rewards/generated": -58.21696090698242,
      "rewards/margins": 42.287139892578125,
      "rewards/real": -15.929832458496094,
      "step": 1110
    },
    {
      "epoch": 0.72,
      "learning_rate": 1.5753911806543385e-07,
      "logits/generated": -Infinity,
      "logits/real": -Infinity,
      "logps/generated": -1003.6033325195312,
      "logps/real": -437.3089294433594,
      "loss": 0.2411,
      "rewards/accuracies": 0.949999988079071,
      "rewards/generated": -69.50968933105469,
      "rewards/margins": 51.704078674316406,
      "rewards/real": -17.805606842041016,
      "step": 1120
    },
    {
      "epoch": 0.72,
      "learning_rate": 1.5398293029871974e-07,
      "logits/generated": -Infinity,
      "logits/real": -Infinity,
      "logps/generated": -963.5474853515625,
      "logps/real": -435.29095458984375,
      "loss": 0.175,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/generated": -66.86241149902344,
      "rewards/margins": 49.975059509277344,
      "rewards/real": -16.887353897094727,
      "step": 1130
    },
    {
      "epoch": 0.73,
      "learning_rate": 1.504267425320057e-07,
      "logits/generated": -Infinity,
      "logits/real": -Infinity,
      "logps/generated": -963.6954956054688,
      "logps/real": -415.65960693359375,
      "loss": 0.0674,
      "rewards/accuracies": 1.0,
      "rewards/generated": -67.89158630371094,
      "rewards/margins": 50.22557830810547,
      "rewards/real": -17.666006088256836,
      "step": 1140
    },
    {
      "epoch": 0.74,
      "learning_rate": 1.4687055476529158e-07,
      "logits/generated": -Infinity,
      "logits/real": -Infinity,
      "logps/generated": -963.4187622070312,
      "logps/real": -441.03094482421875,
      "loss": 0.256,
      "rewards/accuracies": 0.949999988079071,
      "rewards/generated": -65.62948608398438,
      "rewards/margins": 47.720008850097656,
      "rewards/real": -17.909475326538086,
      "step": 1150
    },
    {
      "epoch": 0.74,
      "learning_rate": 1.4331436699857753e-07,
      "logits/generated": -Infinity,
      "logits/real": -Infinity,
      "logps/generated": -948.861328125,
      "logps/real": -444.0084533691406,
      "loss": 0.2541,
      "rewards/accuracies": 0.987500011920929,
      "rewards/generated": -63.19414138793945,
      "rewards/margins": 43.93315887451172,
      "rewards/real": -19.26097869873047,
      "step": 1160
    },
    {
      "epoch": 0.75,
      "learning_rate": 1.3975817923186345e-07,
      "logits/generated": -Infinity,
      "logits/real": -Infinity,
      "logps/generated": -920.9754028320312,
      "logps/real": -451.8453063964844,
      "loss": 0.1545,
      "rewards/accuracies": 1.0,
      "rewards/generated": -62.874755859375,
      "rewards/margins": 45.2825927734375,
      "rewards/real": -17.592164993286133,
      "step": 1170
    },
    {
      "epoch": 0.75,
      "learning_rate": 1.3620199146514935e-07,
      "logits/generated": -Infinity,
      "logits/real": -Infinity,
      "logps/generated": -962.2693481445312,
      "logps/real": -381.82183837890625,
      "loss": 0.1726,
      "rewards/accuracies": 0.987500011920929,
      "rewards/generated": -66.71905517578125,
      "rewards/margins": 50.402278900146484,
      "rewards/real": -16.3167667388916,
      "step": 1180
    },
    {
      "epoch": 0.76,
      "learning_rate": 1.326458036984353e-07,
      "logits/generated": -Infinity,
      "logits/real": -Infinity,
      "logps/generated": -844.4295043945312,
      "logps/real": -408.8478088378906,
      "loss": 0.0521,
      "rewards/accuracies": 0.987500011920929,
      "rewards/generated": -55.36383819580078,
      "rewards/margins": 38.550174713134766,
      "rewards/real": -16.813655853271484,
      "step": 1190
    },
    {
      "epoch": 0.77,
      "learning_rate": 1.290896159317212e-07,
      "logits/generated": -Infinity,
      "logits/real": -Infinity,
      "logps/generated": -1029.0372314453125,
      "logps/real": -405.693359375,
      "loss": 0.0738,
      "rewards/accuracies": 1.0,
      "rewards/generated": -73.27754211425781,
      "rewards/margins": 56.71906661987305,
      "rewards/real": -16.558481216430664,
      "step": 1200
    },
    {
      "epoch": 0.77,
      "learning_rate": 1.255334281650071e-07,
      "logits/generated": -Infinity,
      "logits/real": -Infinity,
      "logps/generated": -955.4649658203125,
      "logps/real": -442.18377685546875,
      "loss": 0.1159,
      "rewards/accuracies": 0.987500011920929,
      "rewards/generated": -66.71927642822266,
      "rewards/margins": 48.399192810058594,
      "rewards/real": -18.320079803466797,
      "step": 1210
    },
    {
      "epoch": 0.78,
      "learning_rate": 1.2197724039829303e-07,
      "logits/generated": -Infinity,
      "logits/real": -Infinity,
      "logps/generated": -925.6458129882812,
      "logps/real": -418.9791564941406,
      "loss": 0.115,
      "rewards/accuracies": 0.987500011920929,
      "rewards/generated": -63.2381477355957,
      "rewards/margins": 45.51228332519531,
      "rewards/real": -17.725872039794922,
      "step": 1220
    },
    {
      "epoch": 0.79,
      "learning_rate": 1.1842105263157894e-07,
      "logits/generated": -Infinity,
      "logits/real": -Infinity,
      "logps/generated": -915.9041137695312,
      "logps/real": -407.2160949707031,
      "loss": 0.1614,
      "rewards/accuracies": 0.987500011920929,
      "rewards/generated": -62.57989501953125,
      "rewards/margins": 44.657745361328125,
      "rewards/real": -17.922143936157227,
      "step": 1230
    },
    {
      "epoch": 0.79,
      "learning_rate": 1.1486486486486487e-07,
      "logits/generated": -Infinity,
      "logits/real": -Infinity,
      "logps/generated": -895.3005981445312,
      "logps/real": -435.64947509765625,
      "loss": 0.1643,
      "rewards/accuracies": 0.949999988079071,
      "rewards/generated": -61.3538932800293,
      "rewards/margins": 43.27713394165039,
      "rewards/real": -18.076759338378906,
      "step": 1240
    },
    {
      "epoch": 0.8,
      "learning_rate": 1.1130867709815078e-07,
      "logits/generated": -Infinity,
      "logits/real": -Infinity,
      "logps/generated": -926.37255859375,
      "logps/real": -427.218505859375,
      "loss": 0.1664,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/generated": -63.830535888671875,
      "rewards/margins": 45.601234436035156,
      "rewards/real": -18.229299545288086,
      "step": 1250
    },
    {
      "epoch": 0.81,
      "learning_rate": 1.077524893314367e-07,
      "logits/generated": -Infinity,
      "logits/real": -Infinity,
      "logps/generated": -968.36279296875,
      "logps/real": -382.15057373046875,
      "loss": 0.1394,
      "rewards/accuracies": 1.0,
      "rewards/generated": -66.76223754882812,
      "rewards/margins": 51.3797721862793,
      "rewards/real": -15.382467269897461,
      "step": 1260
    },
    {
      "epoch": 0.81,
      "learning_rate": 1.0419630156472262e-07,
      "logits/generated": -Infinity,
      "logits/real": -Infinity,
      "logps/generated": -958.3225708007812,
      "logps/real": -481.63055419921875,
      "loss": 0.3112,
      "rewards/accuracies": 0.925000011920929,
      "rewards/generated": -67.70050048828125,
      "rewards/margins": 46.94557189941406,
      "rewards/real": -20.754926681518555,
      "step": 1270
    },
    {
      "epoch": 0.82,
      "learning_rate": 1.0064011379800854e-07,
      "logits/generated": -Infinity,
      "logits/real": -Infinity,
      "logps/generated": -937.0823364257812,
      "logps/real": -458.1944274902344,
      "loss": 0.088,
      "rewards/accuracies": 0.9375,
      "rewards/generated": -64.70439910888672,
      "rewards/margins": 45.66063690185547,
      "rewards/real": -19.043764114379883,
      "step": 1280
    },
    {
      "epoch": 0.83,
      "learning_rate": 9.708392603129445e-08,
      "logits/generated": -Infinity,
      "logits/real": -Infinity,
      "logps/generated": -929.6287841796875,
      "logps/real": -412.03436279296875,
      "loss": 0.1778,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/generated": -65.4493408203125,
      "rewards/margins": 48.98451614379883,
      "rewards/real": -16.46481704711914,
      "step": 1290
    },
    {
      "epoch": 0.83,
      "learning_rate": 9.352773826458037e-08,
      "logits/generated": -Infinity,
      "logits/real": -Infinity,
      "logps/generated": -1027.0244140625,
      "logps/real": -444.76092529296875,
      "loss": 0.1457,
      "rewards/accuracies": 0.987500011920929,
      "rewards/generated": -72.01765441894531,
      "rewards/margins": 53.2171745300293,
      "rewards/real": -18.800477981567383,
      "step": 1300
    },
    {
      "epoch": 0.84,
      "learning_rate": 8.997155049786629e-08,
      "logits/generated": -Infinity,
      "logits/real": -Infinity,
      "logps/generated": -937.8785400390625,
      "logps/real": -400.3741760253906,
      "loss": 0.0517,
      "rewards/accuracies": 0.987500011920929,
      "rewards/generated": -64.58556365966797,
      "rewards/margins": 48.209327697753906,
      "rewards/real": -16.37624168395996,
      "step": 1310
    },
    {
      "epoch": 0.84,
      "learning_rate": 8.64153627311522e-08,
      "logits/generated": -Infinity,
      "logits/real": -Infinity,
      "logps/generated": -962.7506103515625,
      "logps/real": -421.57952880859375,
      "loss": 0.2135,
      "rewards/accuracies": 0.987500011920929,
      "rewards/generated": -67.5987777709961,
      "rewards/margins": 49.87495803833008,
      "rewards/real": -17.72382164001465,
      "step": 1320
    },
    {
      "epoch": 0.85,
      "learning_rate": 8.285917496443812e-08,
      "logits/generated": -Infinity,
      "logits/real": -Infinity,
      "logps/generated": -929.7264404296875,
      "logps/real": -431.411376953125,
      "loss": 0.0276,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/generated": -63.279151916503906,
      "rewards/margins": 45.76356506347656,
      "rewards/real": -17.515583038330078,
      "step": 1330
    },
    {
      "epoch": 0.86,
      "learning_rate": 7.930298719772404e-08,
      "logits/generated": -Infinity,
      "logits/real": -Infinity,
      "logps/generated": -960.4884033203125,
      "logps/real": -449.7841796875,
      "loss": 0.1428,
      "rewards/accuracies": 1.0,
      "rewards/generated": -67.2632827758789,
      "rewards/margins": 49.2481689453125,
      "rewards/real": -18.015111923217773,
      "step": 1340
    },
    {
      "epoch": 0.86,
      "learning_rate": 7.574679943100994e-08,
      "logits/generated": -Infinity,
      "logits/real": -Infinity,
      "logps/generated": -918.400390625,
      "logps/real": -389.68658447265625,
      "loss": 0.2082,
      "rewards/accuracies": 0.987500011920929,
      "rewards/generated": -64.85960388183594,
      "rewards/margins": 48.430328369140625,
      "rewards/real": -16.429283142089844,
      "step": 1350
    },
    {
      "epoch": 0.87,
      "learning_rate": 7.219061166429587e-08,
      "logits/generated": -Infinity,
      "logits/real": -Infinity,
      "logps/generated": -938.8449096679688,
      "logps/real": -409.8709411621094,
      "loss": 0.0844,
      "rewards/accuracies": 0.987500011920929,
      "rewards/generated": -66.37425994873047,
      "rewards/margins": 50.338191986083984,
      "rewards/real": -16.036067962646484,
      "step": 1360
    },
    {
      "epoch": 0.88,
      "learning_rate": 6.863442389758179e-08,
      "logits/generated": -Infinity,
      "logits/real": -Infinity,
      "logps/generated": -961.57958984375,
      "logps/real": -456.1361389160156,
      "loss": 0.1468,
      "rewards/accuracies": 0.925000011920929,
      "rewards/generated": -67.77473449707031,
      "rewards/margins": 49.818424224853516,
      "rewards/real": -17.956310272216797,
      "step": 1370
    },
    {
      "epoch": 0.88,
      "learning_rate": 6.507823613086771e-08,
      "logits/generated": -Infinity,
      "logits/real": -Infinity,
      "logps/generated": -1005.2136840820312,
      "logps/real": -452.8905334472656,
      "loss": 0.2335,
      "rewards/accuracies": 0.987500011920929,
      "rewards/generated": -70.96513366699219,
      "rewards/margins": 52.77602005004883,
      "rewards/real": -18.189117431640625,
      "step": 1380
    },
    {
      "epoch": 0.89,
      "learning_rate": 6.152204836415363e-08,
      "logits/generated": -Infinity,
      "logits/real": -Infinity,
      "logps/generated": -981.8912963867188,
      "logps/real": -449.5849609375,
      "loss": 0.102,
      "rewards/accuracies": 0.987500011920929,
      "rewards/generated": -69.54399871826172,
      "rewards/margins": 50.90169143676758,
      "rewards/real": -18.642309188842773,
      "step": 1390
    },
    {
      "epoch": 0.9,
      "learning_rate": 5.796586059743954e-08,
      "logits/generated": -Infinity,
      "logits/real": -Infinity,
      "logps/generated": -939.12841796875,
      "logps/real": -418.45855712890625,
      "loss": 0.053,
      "rewards/accuracies": 0.987500011920929,
      "rewards/generated": -64.34689331054688,
      "rewards/margins": 47.31360626220703,
      "rewards/real": -17.03328514099121,
      "step": 1400
    },
    {
      "epoch": 0.9,
      "learning_rate": 5.4409672830725456e-08,
      "logits/generated": -Infinity,
      "logits/real": -Infinity,
      "logps/generated": -996.6143798828125,
      "logps/real": -419.6319885253906,
      "loss": 0.1242,
      "rewards/accuracies": 1.0,
      "rewards/generated": -69.24372863769531,
      "rewards/margins": 51.62445831298828,
      "rewards/real": -17.619264602661133,
      "step": 1410
    },
    {
      "epoch": 0.91,
      "learning_rate": 5.0853485064011376e-08,
      "logits/generated": -Infinity,
      "logits/real": -Infinity,
      "logps/generated": -1000.04052734375,
      "logps/real": -417.8561096191406,
      "loss": 0.0612,
      "rewards/accuracies": 0.987500011920929,
      "rewards/generated": -70.85627746582031,
      "rewards/margins": 52.63084030151367,
      "rewards/real": -18.22542953491211,
      "step": 1420
    },
    {
      "epoch": 0.91,
      "learning_rate": 4.72972972972973e-08,
      "logits/generated": -Infinity,
      "logits/real": -Infinity,
      "logps/generated": -934.1195068359375,
      "logps/real": -418.022705078125,
      "loss": 0.2932,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/generated": -66.13401794433594,
      "rewards/margins": 50.10154342651367,
      "rewards/real": -16.0324764251709,
      "step": 1430
    },
    {
      "epoch": 0.92,
      "learning_rate": 4.374110953058322e-08,
      "logits/generated": -Infinity,
      "logits/real": -Infinity,
      "logps/generated": -913.4429931640625,
      "logps/real": -439.560791015625,
      "loss": 0.3329,
      "rewards/accuracies": 0.925000011920929,
      "rewards/generated": -63.869712829589844,
      "rewards/margins": 44.8972053527832,
      "rewards/real": -18.972515106201172,
      "step": 1440
    },
    {
      "epoch": 0.93,
      "learning_rate": 4.018492176386913e-08,
      "logits/generated": -Infinity,
      "logits/real": -Infinity,
      "logps/generated": -911.0625,
      "logps/real": -418.2806091308594,
      "loss": 0.2745,
      "rewards/accuracies": 0.949999988079071,
      "rewards/generated": -61.86652755737305,
      "rewards/margins": 44.91896438598633,
      "rewards/real": -16.94756317138672,
      "step": 1450
    },
    {
      "epoch": 0.93,
      "learning_rate": 3.6628733997155046e-08,
      "logits/generated": -Infinity,
      "logits/real": -Infinity,
      "logps/generated": -981.3883666992188,
      "logps/real": -376.0027770996094,
      "loss": 0.3734,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/generated": -68.52994537353516,
      "rewards/margins": 52.24712371826172,
      "rewards/real": -16.282825469970703,
      "step": 1460
    },
    {
      "epoch": 0.94,
      "learning_rate": 3.3072546230440967e-08,
      "logits/generated": -Infinity,
      "logits/real": -Infinity,
      "logps/generated": -1051.210205078125,
      "logps/real": -414.3362731933594,
      "loss": 0.1266,
      "rewards/accuracies": 0.987500011920929,
      "rewards/generated": -75.56956481933594,
      "rewards/margins": 58.789466857910156,
      "rewards/real": -16.780101776123047,
      "step": 1470
    },
    {
      "epoch": 0.95,
      "learning_rate": 2.9516358463726884e-08,
      "logits/generated": -Infinity,
      "logits/real": -Infinity,
      "logps/generated": -964.0816650390625,
      "logps/real": -437.93572998046875,
      "loss": 0.1666,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/generated": -67.49874877929688,
      "rewards/margins": 50.19408416748047,
      "rewards/real": -17.304664611816406,
      "step": 1480
    },
    {
      "epoch": 0.95,
      "learning_rate": 2.59601706970128e-08,
      "logits/generated": -Infinity,
      "logits/real": -Infinity,
      "logps/generated": -985.89404296875,
      "logps/real": -457.82720947265625,
      "loss": 0.2917,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/generated": -68.58557891845703,
      "rewards/margins": 50.40437698364258,
      "rewards/real": -18.181203842163086,
      "step": 1490
    },
    {
      "epoch": 0.96,
      "learning_rate": 2.240398293029872e-08,
      "logits/generated": -Infinity,
      "logits/real": -Infinity,
      "logps/generated": -966.505859375,
      "logps/real": -419.21337890625,
      "loss": 0.2065,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/generated": -67.9041519165039,
      "rewards/margins": 51.61236572265625,
      "rewards/real": -16.291778564453125,
      "step": 1500
    },
    {
      "epoch": 0.96,
      "eval_logits/generated": -Infinity,
      "eval_logits/real": -Infinity,
      "eval_logps/generated": -659.6319580078125,
      "eval_logps/real": -295.5523376464844,
      "eval_loss": 0.1642562448978424,
      "eval_rewards/accuracies": 0.962579607963562,
      "eval_rewards/generated": -36.8196907043457,
      "eval_rewards/margins": 32.503150939941406,
      "eval_rewards/real": -4.316542148590088,
      "eval_runtime": 514.2854,
      "eval_samples_per_second": 9.722,
      "eval_steps_per_second": 0.305,
      "step": 1500
    },
    {
      "epoch": 0.97,
      "learning_rate": 1.8847795163584636e-08,
      "logits/generated": -Infinity,
      "logits/real": -Infinity,
      "logps/generated": -906.3391723632812,
      "logps/real": -424.758544921875,
      "loss": 0.1254,
      "rewards/accuracies": 0.987500011920929,
      "rewards/generated": -62.673912048339844,
      "rewards/margins": 45.69168472290039,
      "rewards/real": -16.98223114013672,
      "step": 1510
    },
    {
      "epoch": 0.97,
      "learning_rate": 1.5291607396870554e-08,
      "logits/generated": -Infinity,
      "logits/real": -Infinity,
      "logps/generated": -930.1663208007812,
      "logps/real": -418.9947204589844,
      "loss": 0.1585,
      "rewards/accuracies": 0.987500011920929,
      "rewards/generated": -64.94093322753906,
      "rewards/margins": 47.36656951904297,
      "rewards/real": -17.574371337890625,
      "step": 1520
    },
    {
      "epoch": 0.98,
      "learning_rate": 1.1735419630156473e-08,
      "logits/generated": -Infinity,
      "logits/real": -Infinity,
      "logps/generated": -991.5504760742188,
      "logps/real": -432.77117919921875,
      "loss": 0.0505,
      "rewards/accuracies": 0.987500011920929,
      "rewards/generated": -69.73775482177734,
      "rewards/margins": 52.04203414916992,
      "rewards/real": -17.69571876525879,
      "step": 1530
    },
    {
      "epoch": 0.99,
      "learning_rate": 8.179231863442388e-09,
      "logits/generated": -Infinity,
      "logits/real": -Infinity,
      "logps/generated": -957.3475341796875,
      "logps/real": -448.859619140625,
      "loss": 0.1641,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/generated": -66.77298736572266,
      "rewards/margins": 47.40740203857422,
      "rewards/real": -19.36557960510254,
      "step": 1540
    },
    {
      "epoch": 0.99,
      "learning_rate": 4.623044096728307e-09,
      "logits/generated": -Infinity,
      "logits/real": -Infinity,
      "logps/generated": -999.8143310546875,
      "logps/real": -424.55364990234375,
      "loss": 0.0578,
      "rewards/accuracies": 0.987500011920929,
      "rewards/generated": -71.12496948242188,
      "rewards/margins": 54.26820755004883,
      "rewards/real": -16.85675621032715,
      "step": 1550
    },
    {
      "epoch": 1.0,
      "learning_rate": 1.0668563300142248e-09,
      "logits/generated": -Infinity,
      "logits/real": -Infinity,
      "logps/generated": -885.8681640625,
      "logps/real": -414.7955627441406,
      "loss": 0.0539,
      "rewards/accuracies": 1.0,
      "rewards/generated": -60.05329513549805,
      "rewards/margins": 43.077919006347656,
      "rewards/real": -16.975379943847656,
      "step": 1560
    },
    {
      "epoch": 1.0,
      "step": 1563,
      "total_flos": 0.0,
      "train_loss": 0.42768371397759275,
      "train_runtime": 16836.7545,
      "train_samples_per_second": 2.97,
      "train_steps_per_second": 0.093
    }
  ],
  "logging_steps": 10,
  "max_steps": 1563,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 100,
  "total_flos": 0.0,
  "train_batch_size": 8,
  "trial_name": null,
  "trial_params": null
}