{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 1.986206896551724,
  "eval_steps": 16,
  "global_step": 114,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.017422867513611617,
      "grad_norm": 0.39681947231292725,
      "kl": 0.0,
      "learning_rate": 2.5e-06,
      "logps/chosen": -11.77645784980106,
      "logps/rejected": -10.591483425911125,
      "loss": 0.5,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 1
    },
    {
      "epoch": 0.03484573502722323,
      "grad_norm": 0.1452127993106842,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logps/chosen": -11.32990430748981,
      "logps/rejected": -10.54532470703125,
      "loss": 0.5,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 2
    },
    {
      "epoch": 0.052268602540834846,
      "grad_norm": 0.3266962170600891,
      "kl": 0.0029318034648895264,
      "learning_rate": 4.955357142857144e-06,
      "logps/chosen": -11.723511402027027,
      "logps/rejected": -10.767559722440327,
      "loss": 0.5049,
      "rewards/chosen": -0.000146248413098825,
      "rewards/margins": -0.00010965315489224665,
      "rewards/rejected": -3.6595258206578356e-05,
      "step": 3
    },
    {
      "epoch": 0.06969147005444647,
      "grad_norm": 0.222149059176445,
      "kl": 0.0028855118434876204,
      "learning_rate": 4.910714285714286e-06,
      "logps/chosen": -11.060244668496622,
      "logps/rejected": -10.751804677685302,
      "loss": 0.5038,
      "rewards/chosen": -6.2011333333479394e-06,
      "rewards/margins": 0.0002193756961875533,
      "rewards/rejected": -0.00022557682952090125,
      "step": 4
    },
    {
      "epoch": 0.08711433756805807,
      "grad_norm": 0.23216503858566284,
      "kl": 0.003043340751901269,
      "learning_rate": 4.866071428571429e-06,
      "logps/chosen": -11.470826369065504,
      "logps/rejected": -10.402360395951705,
      "loss": 0.5049,
      "rewards/chosen": -6.513710384472058e-05,
      "rewards/margins": -2.0336450510690883e-05,
      "rewards/rejected": -4.48006533340297e-05,
      "step": 5
    },
    {
      "epoch": 0.10453720508166969,
      "grad_norm": 0.2607983946800232,
      "kl": 0.0025751441717147827,
      "learning_rate": 4.821428571428572e-06,
      "logps/chosen": -11.465345632399025,
      "logps/rejected": -10.64196449037286,
      "loss": 0.5036,
      "rewards/chosen": 0.00024762971736594494,
      "rewards/margins": 0.0002127551913608797,
      "rewards/rejected": 3.4874526005065234e-05,
      "step": 6
    },
    {
      "epoch": 0.12196007259528131,
      "grad_norm": 0.335457444190979,
      "kl": 0.0026024316903203726,
      "learning_rate": 4.776785714285715e-06,
      "logps/chosen": -11.233088825579573,
      "logps/rejected": -10.439390931995257,
      "loss": 0.5047,
      "rewards/chosen": -1.1002241089977418e-05,
      "rewards/margins": 5.897991155423426e-05,
      "rewards/rejected": -6.998215264421168e-05,
      "step": 7
    },
    {
      "epoch": 0.13938294010889293,
      "grad_norm": 0.21088895201683044,
      "kl": 0.004203550051897764,
      "learning_rate": 4.732142857142857e-06,
      "logps/chosen": -11.42248280843099,
      "logps/rejected": -10.589916229248047,
      "loss": 0.5049,
      "rewards/chosen": -0.00014668223836148778,
      "rewards/margins": 8.747742200891179e-05,
      "rewards/rejected": -0.00023415966037039956,
      "step": 8
    },
    {
      "epoch": 0.15680580762250454,
      "grad_norm": 0.2334163337945938,
      "kl": 0.0023759554605931044,
      "learning_rate": 4.6875000000000004e-06,
      "logps/chosen": -11.515217052940232,
      "logps/rejected": -10.479249335216029,
      "loss": 0.5047,
      "rewards/chosen": 1.3712330321496433e-05,
      "rewards/margins": 0.00026004273796865194,
      "rewards/rejected": -0.0002463304076471555,
      "step": 9
    },
    {
      "epoch": 0.17422867513611615,
      "grad_norm": 0.2604079246520996,
      "kl": 0.002872834214940667,
      "learning_rate": 4.642857142857144e-06,
      "logps/chosen": -11.413588686342592,
      "logps/rejected": -10.579748568784435,
      "loss": 0.5047,
      "rewards/chosen": -1.3620075252321032e-05,
      "rewards/margins": 0.00010424906420423088,
      "rewards/rejected": -0.00011786913945655192,
      "step": 10
    },
    {
      "epoch": 0.19165154264972778,
      "grad_norm": 0.2593174874782562,
      "kl": 0.0022250190377235413,
      "learning_rate": 4.5982142857142854e-06,
      "logps/chosen": -11.424675119011056,
      "logps/rejected": -10.487614428237535,
      "loss": 0.5044,
      "rewards/chosen": 0.00014462312843641426,
      "rewards/margins": 0.0003136567206988408,
      "rewards/rejected": -0.00016903359226242657,
      "step": 11
    },
    {
      "epoch": 0.20907441016333939,
      "grad_norm": 0.1917509138584137,
      "kl": 0.0026314922142773867,
      "learning_rate": 4.553571428571429e-06,
      "logps/chosen": -11.320460956057007,
      "logps/rejected": -10.401315122928674,
      "loss": 0.5045,
      "rewards/chosen": 0.00016278723993663833,
      "rewards/margins": 0.0006349250786245901,
      "rewards/rejected": -0.0004721378386879517,
      "step": 12
    },
    {
      "epoch": 0.226497277676951,
      "grad_norm": 0.2281549721956253,
      "kl": 0.0019408013904467225,
      "learning_rate": 4.508928571428572e-06,
      "logps/chosen": -11.184846626695736,
      "logps/rejected": -10.86145371965223,
      "loss": 0.5041,
      "rewards/chosen": 0.00017400297699664605,
      "rewards/margins": 0.0004404290903305609,
      "rewards/rejected": -0.00026642611333391484,
      "step": 13
    },
    {
      "epoch": 0.24392014519056263,
      "grad_norm": 0.29345858097076416,
      "kl": 0.0032534776255488396,
      "learning_rate": 4.464285714285715e-06,
      "logps/chosen": -11.328454192120928,
      "logps/rejected": -10.352222804454607,
      "loss": 0.5055,
      "rewards/chosen": -0.00023024088532703563,
      "rewards/margins": 0.00022275829169079655,
      "rewards/rejected": -0.0004529991770178322,
      "step": 14
    },
    {
      "epoch": 0.2613430127041742,
      "grad_norm": 0.20449300110340118,
      "kl": 0.003026653779670596,
      "learning_rate": 4.419642857142857e-06,
      "logps/chosen": -11.203996222636421,
      "logps/rejected": -10.722607944100936,
      "loss": 0.5043,
      "rewards/chosen": 0.00018860261679300803,
      "rewards/margins": 0.0004592195771910241,
      "rewards/rejected": -0.0002706169603980161,
      "step": 15
    },
    {
      "epoch": 0.27876588021778587,
      "grad_norm": 0.201906219124794,
      "kl": 0.002389234723523259,
      "learning_rate": 4.3750000000000005e-06,
      "logps/chosen": -11.154755261479592,
      "logps/rejected": -10.722290688372674,
      "loss": 0.5042,
      "rewards/chosen": 0.00029305163390782414,
      "rewards/margins": 0.0007897253288582932,
      "rewards/rejected": -0.000496673694950469,
      "step": 16
    },
    {
      "epoch": 0.27876588021778587,
      "eval_kl": 0.002219570567831397,
      "eval_logps/chosen": -11.288407461983818,
      "eval_logps/rejected": -10.652853829520089,
      "eval_loss": 0.5038086175918579,
      "eval_rewards/chosen": 0.0004127062524535826,
      "eval_rewards/margins": 0.0008178721514663526,
      "eval_rewards/rejected": -0.00040516589901276996,
      "eval_runtime": 41.7888,
      "eval_samples_per_second": 10.673,
      "eval_steps_per_second": 0.67,
      "step": 16
    },
    {
      "epoch": 0.2961887477313975,
      "grad_norm": 0.2981337308883667,
      "kl": 0.0040084668435156345,
      "learning_rate": 4.330357142857143e-06,
      "logps/chosen": -11.60596080662049,
      "logps/rejected": -10.618717876233553,
      "loss": 0.5046,
      "rewards/chosen": 0.00017366750338642867,
      "rewards/margins": 0.0006427841346935287,
      "rewards/rejected": -0.0004691166313071,
      "step": 17
    },
    {
      "epoch": 0.3136116152450091,
      "grad_norm": 0.1905021220445633,
      "kl": 0.0029678388964384794,
      "learning_rate": 4.2857142857142855e-06,
      "logps/chosen": -11.358947906876567,
      "logps/rejected": -10.434524660188009,
      "loss": 0.5048,
      "rewards/chosen": 2.532637675752616e-05,
      "rewards/margins": 0.0007196114555040427,
      "rewards/rejected": -0.0006942850787465166,
      "step": 18
    },
    {
      "epoch": 0.3310344827586207,
      "grad_norm": 0.21423697471618652,
      "kl": 0.002790730679407716,
      "learning_rate": 4.241071428571429e-06,
      "logps/chosen": -11.554482736895162,
      "logps/rejected": -11.022931956281566,
      "loss": 0.504,
      "rewards/chosen": 0.00010277342892462207,
      "rewards/margins": 0.0007622132545750395,
      "rewards/rejected": -0.0006594398256504174,
      "step": 19
    },
    {
      "epoch": 0.3484573502722323,
      "grad_norm": 0.19702628254890442,
      "kl": 0.002327132970094681,
      "learning_rate": 4.196428571428572e-06,
      "logps/chosen": -11.411150614420572,
      "logps/rejected": -10.28190549214681,
      "loss": 0.504,
      "rewards/chosen": 0.000137356905421863,
      "rewards/margins": 0.0005323387255581716,
      "rewards/rejected": -0.00039498182013630867,
      "step": 20
    },
    {
      "epoch": 0.3658802177858439,
      "grad_norm": 0.20904229581356049,
      "kl": 0.002573903650045395,
      "learning_rate": 4.151785714285715e-06,
      "logps/chosen": -11.281766930404974,
      "logps/rejected": -10.511914841672207,
      "loss": 0.504,
      "rewards/chosen": 0.00020978350800519087,
      "rewards/margins": 0.0008185515050823616,
      "rewards/rejected": -0.0006087679970771708,
      "step": 21
    },
    {
      "epoch": 0.38330308529945556,
      "grad_norm": 0.19066323339939117,
      "kl": 0.004084563348442316,
      "learning_rate": 4.107142857142857e-06,
      "logps/chosen": -11.57584678589191,
      "logps/rejected": -11.04019266504156,
      "loss": 0.5042,
      "rewards/chosen": 0.00028499441137364435,
      "rewards/margins": 0.0007805728857745476,
      "rewards/rejected": -0.0004955784744009032,
      "step": 22
    },
    {
      "epoch": 0.40072595281306717,
      "grad_norm": 0.21532228589057922,
      "kl": 0.0018439519917592406,
      "learning_rate": 4.0625000000000005e-06,
      "logps/chosen": -11.395969251063482,
      "logps/rejected": -10.348430416126943,
      "loss": 0.5039,
      "rewards/chosen": 0.0002833760065558069,
      "rewards/margins": 0.000828726542101749,
      "rewards/rejected": -0.0005453505355459421,
      "step": 23
    },
    {
      "epoch": 0.41814882032667877,
      "grad_norm": 0.1852329671382904,
      "kl": 0.002771291183307767,
      "learning_rate": 4.017857142857143e-06,
      "logps/chosen": -11.074297150088029,
      "logps/rejected": -10.626986228813559,
      "loss": 0.5035,
      "rewards/chosen": 0.00041746350241379,
      "rewards/margins": 0.0010044164534086372,
      "rewards/rejected": -0.0005869529509948472,
      "step": 24
    },
    {
      "epoch": 0.4355716878402904,
      "grad_norm": 0.2030268907546997,
      "kl": 0.004407358821481466,
      "learning_rate": 3.9732142857142855e-06,
      "logps/chosen": -11.303015870474727,
      "logps/rejected": -10.537113018889926,
      "loss": 0.5033,
      "rewards/chosen": 0.000330610088800472,
      "rewards/margins": 0.0013877044688375805,
      "rewards/rejected": -0.0010570943800371085,
      "step": 25
    },
    {
      "epoch": 0.452994555353902,
      "grad_norm": 0.330771803855896,
      "kl": 0.0028855446726083755,
      "learning_rate": 3.928571428571429e-06,
      "logps/chosen": -11.326861509342784,
      "logps/rejected": -10.294227359169408,
      "loss": 0.5044,
      "rewards/chosen": 0.0002357424380853004,
      "rewards/margins": 0.0008208984113942651,
      "rewards/rejected": -0.0005851559733089647,
      "step": 26
    },
    {
      "epoch": 0.4704174228675136,
      "grad_norm": 0.204763263463974,
      "kl": 0.0029878586065024137,
      "learning_rate": 3.883928571428572e-06,
      "logps/chosen": -11.238988330696202,
      "logps/rejected": -10.748736097729557,
      "loss": 0.5038,
      "rewards/chosen": 0.0004333936338183246,
      "rewards/margins": 0.0012179940220870076,
      "rewards/rejected": -0.0007846003882686829,
      "step": 27
    },
    {
      "epoch": 0.48784029038112525,
      "grad_norm": 0.4104137420654297,
      "kl": 0.002435252070426941,
      "learning_rate": 3.839285714285715e-06,
      "logps/chosen": -11.65949259124677,
      "logps/rejected": -10.41943679769521,
      "loss": 0.505,
      "rewards/chosen": -4.84786946952189e-05,
      "rewards/margins": 0.0007131273978328765,
      "rewards/rejected": -0.0007616060925280954,
      "step": 28
    },
    {
      "epoch": 0.5052631578947369,
      "grad_norm": 0.1900724321603775,
      "kl": 0.003689997596666217,
      "learning_rate": 3.794642857142857e-06,
      "logps/chosen": -11.369253954475308,
      "logps/rejected": -10.650356996814738,
      "loss": 0.5035,
      "rewards/chosen": 0.0005172402034571141,
      "rewards/margins": 0.0012426704372049484,
      "rewards/rejected": -0.0007254302337478343,
      "step": 29
    },
    {
      "epoch": 0.5226860254083484,
      "grad_norm": 0.25149452686309814,
      "kl": 0.003089333651587367,
      "learning_rate": 3.7500000000000005e-06,
      "logps/chosen": -11.706775306138677,
      "logps/rejected": -10.319467447916667,
      "loss": 0.5048,
      "rewards/chosen": 0.00012126840821659293,
      "rewards/margins": 0.0005743338433082473,
      "rewards/rejected": -0.00045306543509165444,
      "step": 30
    },
    {
      "epoch": 0.5401088929219601,
      "grad_norm": 0.2375558316707611,
      "kl": 0.0021258096676319838,
      "learning_rate": 3.7053571428571434e-06,
      "logps/chosen": -11.040724951044387,
      "logps/rejected": -10.158553799715909,
      "loss": 0.504,
      "rewards/chosen": 0.00032178575158430455,
      "rewards/margins": 0.001046311172281009,
      "rewards/rejected": -0.0007245254206967044,
      "step": 31
    },
    {
      "epoch": 0.5575317604355717,
      "grad_norm": 0.20534147322177887,
      "kl": 0.002767733531072736,
      "learning_rate": 3.660714285714286e-06,
      "logps/chosen": -11.517271468581537,
      "logps/rejected": -10.606380126338161,
      "loss": 0.5037,
      "rewards/chosen": 0.0005935063821607523,
      "rewards/margins": 0.0013072239592998884,
      "rewards/rejected": -0.0007137175771391362,
      "step": 32
    },
    {
      "epoch": 0.5575317604355717,
      "eval_kl": 0.0013082654913887382,
      "eval_logps/chosen": -11.286455426897321,
      "eval_logps/rejected": -10.656499590192523,
      "eval_loss": 0.5032945275306702,
      "eval_rewards/chosen": 0.0006079860031604767,
      "eval_rewards/margins": 0.001377601841730731,
      "eval_rewards/rejected": -0.0007696158385702542,
      "eval_runtime": 41.5226,
      "eval_samples_per_second": 10.741,
      "eval_steps_per_second": 0.674,
      "step": 32
    },
    {
      "epoch": 0.5749546279491833,
      "grad_norm": 0.240458145737648,
      "kl": 0.002010853262618184,
      "learning_rate": 3.616071428571429e-06,
      "logps/chosen": -11.25503524547906,
      "logps/rejected": -11.051669034090908,
      "loss": 0.5043,
      "rewards/chosen": 0.00019917678711983152,
      "rewards/margins": 0.0007490985231613238,
      "rewards/rejected": -0.0005499217360414923,
      "step": 33
    },
    {
      "epoch": 0.592377495462795,
      "grad_norm": 0.18933577835559845,
      "kl": 0.0021181467454880476,
      "learning_rate": 3.5714285714285718e-06,
      "logps/chosen": -11.664008105974741,
      "logps/rejected": -10.828131391115837,
      "loss": 0.5032,
      "rewards/chosen": 0.0005863340947912147,
      "rewards/margins": 0.0013070002798906317,
      "rewards/rejected": -0.000720666185099417,
      "step": 34
    },
    {
      "epoch": 0.6098003629764065,
      "grad_norm": 0.20784388482570648,
      "kl": 0.002850498305633664,
      "learning_rate": 3.5267857142857147e-06,
      "logps/chosen": -11.464662464007452,
      "logps/rejected": -10.13366056743421,
      "loss": 0.5041,
      "rewards/chosen": 0.00017015952970277326,
      "rewards/margins": 0.0008830348158433109,
      "rewards/rejected": -0.0007128752861405376,
      "step": 35
    },
    {
      "epoch": 0.6272232304900182,
      "grad_norm": 0.1830260157585144,
      "kl": 0.003351332386955619,
      "learning_rate": 3.482142857142857e-06,
      "logps/chosen": -11.203702368776113,
      "logps/rejected": -10.977577265433986,
      "loss": 0.5036,
      "rewards/chosen": 0.00040133928356064395,
      "rewards/margins": 0.0009519243287689104,
      "rewards/rejected": -0.0005505850452082664,
      "step": 36
    },
    {
      "epoch": 0.6446460980036298,
      "grad_norm": 0.18368099629878998,
      "kl": 0.0037287927698343992,
      "learning_rate": 3.4375e-06,
      "logps/chosen": -11.507929849361188,
      "logps/rejected": -10.265622594673646,
      "loss": 0.503,
      "rewards/chosen": 0.00047660240316917883,
      "rewards/margins": 0.0012168469473675392,
      "rewards/rejected": -0.0007402445441983603,
      "step": 37
    },
    {
      "epoch": 0.6620689655172414,
      "grad_norm": 0.22299793362617493,
      "kl": 0.0025776519905775785,
      "learning_rate": 3.3928571428571435e-06,
      "logps/chosen": -11.776427912833121,
      "logps/rejected": -10.537473627590241,
      "loss": 0.5041,
      "rewards/chosen": 0.00029345284107372844,
      "rewards/margins": 0.0009656724311435042,
      "rewards/rejected": -0.0006722195900697759,
      "step": 38
    },
    {
      "epoch": 0.679491833030853,
      "grad_norm": 0.21307651698589325,
      "kl": 0.0029651224613189697,
      "learning_rate": 3.3482142857142855e-06,
      "logps/chosen": -11.337782663316583,
      "logps/rejected": -10.489964500633446,
      "loss": 0.5041,
      "rewards/chosen": 0.000353607026176836,
      "rewards/margins": 0.0009334429436754503,
      "rewards/rejected": -0.0005798359174986143,
      "step": 39
    },
    {
      "epoch": 0.6969147005444646,
      "grad_norm": 0.21936196088790894,
      "kl": 0.0019360833102837205,
      "learning_rate": 3.303571428571429e-06,
      "logps/chosen": -12.052929194286616,
      "logps/rejected": -10.637656591271842,
      "loss": 0.5044,
      "rewards/chosen": 0.000453127905575916,
      "rewards/margins": 0.0010775220099412956,
      "rewards/rejected": -0.0006243941043653796,
      "step": 40
    },
    {
      "epoch": 0.7143375680580762,
      "grad_norm": 0.5484824180603027,
      "kl": 0.0032773923594504595,
      "learning_rate": 3.258928571428572e-06,
      "logps/chosen": -11.568930269491792,
      "logps/rejected": -10.71394266108031,
      "loss": 0.5054,
      "rewards/chosen": 5.686828264533871e-05,
      "rewards/margins": 0.0006258111605199182,
      "rewards/rejected": -0.0005689428778745795,
      "step": 41
    },
    {
      "epoch": 0.7317604355716878,
      "grad_norm": 0.20393657684326172,
      "kl": 0.003910336643457413,
      "learning_rate": 3.2142857142857147e-06,
      "logps/chosen": -11.504363593319558,
      "logps/rejected": -10.101014539930556,
      "loss": 0.5036,
      "rewards/chosen": 0.00029671770974624257,
      "rewards/margins": 0.001082045991996541,
      "rewards/rejected": -0.0007853282822502984,
      "step": 42
    },
    {
      "epoch": 0.7491833030852995,
      "grad_norm": 0.16519714891910553,
      "kl": 0.003650219878181815,
      "learning_rate": 3.1696428571428572e-06,
      "logps/chosen": -10.816071234809028,
      "logps/rejected": -10.322287166819853,
      "loss": 0.503,
      "rewards/chosen": 0.0007183290190166897,
      "rewards/margins": 0.0015814549685303683,
      "rewards/rejected": -0.0008631259495136785,
      "step": 43
    },
    {
      "epoch": 0.7666061705989111,
      "grad_norm": 0.20090065896511078,
      "kl": 0.002337512793019414,
      "learning_rate": 3.125e-06,
      "logps/chosen": -11.918630044291339,
      "logps/rejected": -11.223259346737725,
      "loss": 0.5041,
      "rewards/chosen": 0.00022355469036602912,
      "rewards/margins": 0.0007338886578213189,
      "rewards/rejected": -0.0005103339674552898,
      "step": 44
    },
    {
      "epoch": 0.7840290381125227,
      "grad_norm": 0.23820927739143372,
      "kl": 0.0014156276592984796,
      "learning_rate": 3.080357142857143e-06,
      "logps/chosen": -11.657449557086615,
      "logps/rejected": -10.76135790132429,
      "loss": 0.5037,
      "rewards/chosen": 0.0006081545994350604,
      "rewards/margins": 0.0009453997203017225,
      "rewards/rejected": -0.0003372451208666622,
      "step": 45
    },
    {
      "epoch": 0.8014519056261343,
      "grad_norm": 0.2038673311471939,
      "kl": 0.0020267742220312357,
      "learning_rate": 3.0357142857142856e-06,
      "logps/chosen": -11.508970748546512,
      "logps/rejected": -10.532100327058727,
      "loss": 0.5043,
      "rewards/chosen": 0.0003054198599600977,
      "rewards/margins": 0.0006779901515518521,
      "rewards/rejected": -0.0003725702915917544,
      "step": 46
    },
    {
      "epoch": 0.8188747731397459,
      "grad_norm": 0.2023131251335144,
      "kl": 0.0036333007737994194,
      "learning_rate": 2.991071428571429e-06,
      "logps/chosen": -11.018499706769436,
      "logps/rejected": -10.779378461234177,
      "loss": 0.5039,
      "rewards/chosen": 0.00048798394586701177,
      "rewards/margins": 0.0010197794913363682,
      "rewards/rejected": -0.0005317955454693565,
      "step": 47
    },
    {
      "epoch": 0.8362976406533575,
      "grad_norm": 0.19009532034397125,
      "kl": 0.0020073249470442533,
      "learning_rate": 2.946428571428572e-06,
      "logps/chosen": -11.3989532127809,
      "logps/rejected": -11.23740068454187,
      "loss": 0.5035,
      "rewards/chosen": 0.0002772415035896087,
      "rewards/margins": 0.0007655857580138798,
      "rewards/rejected": -0.0004883442544242711,
      "step": 48
    },
    {
      "epoch": 0.8362976406533575,
      "eval_kl": 0.001558297430165112,
      "eval_logps/chosen": -11.289903913225446,
      "eval_logps/rejected": -10.654574802943639,
      "eval_loss": 0.5040844082832336,
      "eval_rewards/chosen": 0.00026316663050757986,
      "eval_rewards/margins": 0.0008403176741142358,
      "eval_rewards/rejected": -0.000577151043606656,
      "eval_runtime": 41.5721,
      "eval_samples_per_second": 10.728,
      "eval_steps_per_second": 0.674,
      "step": 48
    },
    {
      "epoch": 0.8537205081669691,
      "grad_norm": 0.2040846198797226,
      "kl": 0.0028173942118883133,
      "learning_rate": 2.9017857142857148e-06,
      "logps/chosen": -11.502028847760695,
      "logps/rejected": -9.920858160493337,
      "loss": 0.5037,
      "rewards/chosen": 0.0003798981720113499,
      "rewards/margins": 0.001099527059744577,
      "rewards/rejected": -0.0007196288877332271,
      "step": 49
    },
    {
      "epoch": 0.8711433756805808,
      "grad_norm": 0.46055036783218384,
      "kl": 0.001712739816866815,
      "learning_rate": 2.8571428571428573e-06,
      "logps/chosen": -11.66295957325691,
      "logps/rejected": -10.606267155827704,
      "loss": 0.5046,
      "rewards/chosen": 8.328470527826242e-05,
      "rewards/margins": 0.0005694089115119315,
      "rewards/rejected": -0.000486124206233669,
      "step": 50
    },
    {
      "epoch": 0.8885662431941924,
      "grad_norm": 0.2116948515176773,
      "kl": 0.0014734516153112054,
      "learning_rate": 2.8125e-06,
      "logps/chosen": -11.087164900249377,
      "logps/rejected": -10.773697606224456,
      "loss": 0.5046,
      "rewards/chosen": 0.0002402601545291054,
      "rewards/margins": 0.0008381397699308074,
      "rewards/rejected": -0.000597879615401702,
      "step": 51
    },
    {
      "epoch": 0.905989110707804,
      "grad_norm": 0.2204010933637619,
      "kl": 0.0023574617225676775,
      "learning_rate": 2.767857142857143e-06,
      "logps/chosen": -11.274248798076924,
      "logps/rejected": -10.153245391038359,
      "loss": 0.5032,
      "rewards/chosen": 0.0005002290392533327,
      "rewards/margins": 0.0011188293800006614,
      "rewards/rejected": -0.0006186003407473286,
      "step": 52
    },
    {
      "epoch": 0.9234119782214156,
      "grad_norm": 0.22388231754302979,
      "kl": 0.002206307603046298,
      "learning_rate": 2.7232142857142856e-06,
      "logps/chosen": -11.911312705592104,
      "logps/rejected": -10.432602086018042,
      "loss": 0.5041,
      "rewards/chosen": 0.0004630535840988159,
      "rewards/margins": 0.0009620043727540478,
      "rewards/rejected": -0.0004989507886552319,
      "step": 53
    },
    {
      "epoch": 0.9408348457350272,
      "grad_norm": 0.20113378763198853,
      "kl": 0.0021755837369710207,
      "learning_rate": 2.6785714285714285e-06,
      "logps/chosen": -11.37064292594178,
      "logps/rejected": -10.506611788539082,
      "loss": 0.5036,
      "rewards/chosen": 0.00027320436010622,
      "rewards/margins": 0.0005519093562507254,
      "rewards/rejected": -0.0002787049961445054,
      "step": 54
    },
    {
      "epoch": 0.9582577132486388,
      "grad_norm": 0.7052344083786011,
      "kl": 0.002290615811944008,
      "learning_rate": 2.633928571428572e-06,
      "logps/chosen": -11.785455354221204,
      "logps/rejected": -10.28837688228627,
      "loss": 0.5038,
      "rewards/chosen": 0.0004812158371141444,
      "rewards/margins": 0.0010049399622622454,
      "rewards/rejected": -0.0005237241251481011,
      "step": 55
    },
    {
      "epoch": 0.9756805807622505,
      "grad_norm": 0.18455247581005096,
      "kl": 0.001295599271543324,
      "learning_rate": 2.5892857142857148e-06,
      "logps/chosen": -11.051934689891581,
      "logps/rejected": -10.790322161735372,
      "loss": 0.5038,
      "rewards/chosen": 0.00037732340243397925,
      "rewards/margins": 0.0009394810824965168,
      "rewards/rejected": -0.0005621576800625375,
      "step": 56
    },
    {
      "epoch": 0.993103448275862,
      "grad_norm": 0.1891961693763733,
      "kl": 0.00294533371925354,
      "learning_rate": 2.5446428571428573e-06,
      "logps/chosen": -11.06938274166127,
      "logps/rejected": -10.141628405186518,
      "loss": 0.5034,
      "rewards/chosen": 0.00042142599357842164,
      "rewards/margins": 0.0006545582687216406,
      "rewards/rejected": -0.0002331322751432189,
      "step": 57
    },
    {
      "epoch": 1.0105263157894737,
      "grad_norm": 0.22368647158145905,
      "kl": 0.002181227086111903,
      "learning_rate": 2.5e-06,
      "logps/chosen": -11.29225919699367,
      "logps/rejected": -10.528809902815013,
      "loss": 0.5036,
      "rewards/chosen": 0.0002977928406075586,
      "rewards/margins": 0.0008452181976182237,
      "rewards/rejected": -0.000547425357010665,
      "step": 58
    },
    {
      "epoch": 1.0279491833030854,
      "grad_norm": 0.19894687831401825,
      "kl": 0.0022480275947600603,
      "learning_rate": 2.455357142857143e-06,
      "logps/chosen": -11.208153912401574,
      "logps/rejected": -10.815308563468992,
      "loss": 0.5037,
      "rewards/chosen": 0.00028297573015132913,
      "rewards/margins": 0.0009500956342332799,
      "rewards/rejected": -0.0006671199040819508,
      "step": 59
    },
    {
      "epoch": 1.0453720508166968,
      "grad_norm": 0.19452357292175293,
      "kl": 0.003947921562939882,
      "learning_rate": 2.410714285714286e-06,
      "logps/chosen": -11.397494791666666,
      "logps/rejected": -10.136647309359097,
      "loss": 0.5036,
      "rewards/chosen": 0.0006445555686950684,
      "rewards/margins": 0.0011657039269842872,
      "rewards/rejected": -0.0005211483582892187,
      "step": 60
    },
    {
      "epoch": 1.0627949183303085,
      "grad_norm": 0.2021968960762024,
      "kl": 0.00338422111235559,
      "learning_rate": 2.3660714285714285e-06,
      "logps/chosen": -11.75505078125,
      "logps/rejected": -10.338195024252544,
      "loss": 0.5035,
      "rewards/chosen": 0.0006066491603851318,
      "rewards/margins": 0.0008883069659614078,
      "rewards/rejected": -0.0002816578055762759,
      "step": 61
    },
    {
      "epoch": 1.0802177858439201,
      "grad_norm": 0.18729077279567719,
      "kl": 0.0022495612502098083,
      "learning_rate": 2.321428571428572e-06,
      "logps/chosen": -11.218356103012242,
      "logps/rejected": -10.509180972450658,
      "loss": 0.5034,
      "rewards/chosen": 0.0004960870005420803,
      "rewards/margins": 0.0008710139801466005,
      "rewards/rejected": -0.0003749269796045203,
      "step": 62
    },
    {
      "epoch": 1.0976406533575318,
      "grad_norm": 0.19914378225803375,
      "kl": 0.003042086260393262,
      "learning_rate": 2.2767857142857144e-06,
      "logps/chosen": -11.236100905012377,
      "logps/rejected": -10.941490760216347,
      "loss": 0.5034,
      "rewards/chosen": 0.0013145432613863803,
      "rewards/margins": 0.0017898373715980775,
      "rewards/rejected": -0.00047529411021169726,
      "step": 63
    },
    {
      "epoch": 1.1150635208711435,
      "grad_norm": 0.19482098519802094,
      "kl": 0.002518598223105073,
      "learning_rate": 2.2321428571428573e-06,
      "logps/chosen": -11.394892939814815,
      "logps/rejected": -10.390007762419872,
      "loss": 0.5037,
      "rewards/chosen": 0.0003466351480080337,
      "rewards/margins": 0.0006846176696153592,
      "rewards/rejected": -0.0003379825216073256,
      "step": 64
    },
    {
      "epoch": 1.1150635208711435,
      "eval_kl": 0.0016980862710624933,
      "eval_logps/chosen": -11.287184579031807,
      "eval_logps/rejected": -10.654030936104911,
      "eval_loss": 0.5034511089324951,
      "eval_rewards/chosen": 0.0005350507396672453,
      "eval_rewards/margins": 0.0010578856537384646,
      "eval_rewards/rejected": -0.0005228349140712193,
      "eval_runtime": 41.5198,
      "eval_samples_per_second": 10.742,
      "eval_steps_per_second": 0.674,
      "step": 64
    },
    {
      "epoch": 1.132486388384755,
      "grad_norm": 0.20668481290340424,
      "kl": 0.003911910578608513,
      "learning_rate": 2.1875000000000002e-06,
      "logps/chosen": -11.54857183807525,
      "logps/rejected": -11.033686736893785,
      "loss": 0.504,
      "rewards/chosen": 0.0006883917608071322,
      "rewards/margins": 0.0012867070528502448,
      "rewards/rejected": -0.0005983152920431127,
      "step": 65
    },
    {
      "epoch": 1.1499092558983666,
      "grad_norm": 0.20961228013038635,
      "kl": 0.0017986433813348413,
      "learning_rate": 2.1428571428571427e-06,
      "logps/chosen": -11.831485829454786,
      "logps/rejected": -10.89035345583546,
      "loss": 0.504,
      "rewards/chosen": 0.0002542631502481217,
      "rewards/margins": 0.0006354290375722992,
      "rewards/rejected": -0.0003811658873241775,
      "step": 66
    },
    {
      "epoch": 1.1673321234119782,
      "grad_norm": 0.19809454679489136,
      "kl": 0.002224177587777376,
      "learning_rate": 2.098214285714286e-06,
      "logps/chosen": -11.331428947029533,
      "logps/rejected": -10.59391437190594,
      "loss": 0.5034,
      "rewards/chosen": 0.0005333201325201727,
      "rewards/margins": 0.001024618166864742,
      "rewards/rejected": -0.0004912980343445693,
      "step": 67
    },
    {
      "epoch": 1.18475499092559,
      "grad_norm": 0.19940558075904846,
      "kl": 0.00197195284999907,
      "learning_rate": 2.0535714285714286e-06,
      "logps/chosen": -11.550331552282016,
      "logps/rejected": -10.52055773651808,
      "loss": 0.503,
      "rewards/chosen": 0.0006965530340937892,
      "rewards/margins": 0.0013385297657675268,
      "rewards/rejected": -0.0006419767316737377,
      "step": 68
    },
    {
      "epoch": 1.2021778584392013,
      "grad_norm": 0.21953897178173065,
      "kl": 0.0020015963818877935,
      "learning_rate": 2.0089285714285715e-06,
      "logps/chosen": -10.969684954573935,
      "logps/rejected": -10.10690514375847,
      "loss": 0.5042,
      "rewards/chosen": 0.0001828334385291078,
      "rewards/margins": 0.0006383278937348235,
      "rewards/rejected": -0.00045549445520571576,
      "step": 69
    },
    {
      "epoch": 1.219600725952813,
      "grad_norm": 0.19710642099380493,
      "kl": 0.00231179385446012,
      "learning_rate": 1.9642857142857144e-06,
      "logps/chosen": -11.595243038467848,
      "logps/rejected": -10.112245387193152,
      "loss": 0.5033,
      "rewards/chosen": 0.00039139722432364314,
      "rewards/margins": 0.0010068912661115128,
      "rewards/rejected": -0.0006154940417878696,
      "step": 70
    },
    {
      "epoch": 1.2370235934664247,
      "grad_norm": 0.1987324059009552,
      "kl": 0.002300393534824252,
      "learning_rate": 1.9196428571428573e-06,
      "logps/chosen": -11.305307889174403,
      "logps/rejected": -10.520050751278772,
      "loss": 0.5035,
      "rewards/chosen": 0.0004280993729750737,
      "rewards/margins": 0.0009470002607156162,
      "rewards/rejected": -0.0005189008877405425,
      "step": 71
    },
    {
      "epoch": 1.2544464609800363,
      "grad_norm": 0.18199442327022552,
      "kl": 0.0023830088321119547,
      "learning_rate": 1.8750000000000003e-06,
      "logps/chosen": -11.361725593834231,
      "logps/rejected": -10.366453233233628,
      "loss": 0.5029,
      "rewards/chosen": 0.000802040341086786,
      "rewards/margins": 0.0015088396506467594,
      "rewards/rejected": -0.0007067993095599734,
      "step": 72
    },
    {
      "epoch": 1.271869328493648,
      "grad_norm": 0.2903735339641571,
      "kl": 0.0014767992543056607,
      "learning_rate": 1.830357142857143e-06,
      "logps/chosen": -11.19476151596653,
      "logps/rejected": -10.959830972092663,
      "loss": 0.5034,
      "rewards/chosen": 0.00034413270937289043,
      "rewards/margins": 0.0008636754093954977,
      "rewards/rejected": -0.0005195427000226073,
      "step": 73
    },
    {
      "epoch": 1.2892921960072594,
      "grad_norm": 0.20160308480262756,
      "kl": 0.003358997404575348,
      "learning_rate": 1.7857142857142859e-06,
      "logps/chosen": -10.99349352108535,
      "logps/rejected": -10.71151579269255,
      "loss": 0.5032,
      "rewards/chosen": 0.0006134921023922582,
      "rewards/margins": 0.0010400663268667124,
      "rewards/rejected": -0.00042657422447445415,
      "step": 74
    },
    {
      "epoch": 1.306715063520871,
      "grad_norm": 0.23630282282829285,
      "kl": 0.0023285781498998404,
      "learning_rate": 1.7410714285714286e-06,
      "logps/chosen": -11.506769643439947,
      "logps/rejected": -10.45025872564935,
      "loss": 0.5036,
      "rewards/chosen": 0.0006310967801445147,
      "rewards/margins": 0.0013577277155657525,
      "rewards/rejected": -0.0007266309354212377,
      "step": 75
    },
    {
      "epoch": 1.3241379310344827,
      "grad_norm": 0.18052582442760468,
      "kl": 0.0017558973049744964,
      "learning_rate": 1.6964285714285717e-06,
      "logps/chosen": -11.271184662723785,
      "logps/rejected": -10.48470008910809,
      "loss": 0.5033,
      "rewards/chosen": 0.0006234512076048595,
      "rewards/margins": 0.0013483937273518372,
      "rewards/rejected": -0.0007249425197469777,
      "step": 76
    },
    {
      "epoch": 1.3415607985480944,
      "grad_norm": 0.18041782081127167,
      "kl": 0.0014101862907409668,
      "learning_rate": 1.6517857142857144e-06,
      "logps/chosen": -11.356517080872704,
      "logps/rejected": -10.676619024547804,
      "loss": 0.5033,
      "rewards/chosen": 0.0006575664979579255,
      "rewards/margins": 0.001282264682831324,
      "rewards/rejected": -0.0006246981848733986,
      "step": 77
    },
    {
      "epoch": 1.358983666061706,
      "grad_norm": 0.2689516544342041,
      "kl": 0.0018190672853961587,
      "learning_rate": 1.6071428571428574e-06,
      "logps/chosen": -11.690565026324728,
      "logps/rejected": -10.228720703125,
      "loss": 0.5034,
      "rewards/chosen": 0.0008241140648074772,
      "rewards/margins": 0.0013455211176820424,
      "rewards/rejected": -0.0005214070528745652,
      "step": 78
    },
    {
      "epoch": 1.3764065335753175,
      "grad_norm": 0.20166873931884766,
      "kl": 0.0017292723059654236,
      "learning_rate": 1.5625e-06,
      "logps/chosen": -11.26230097903481,
      "logps/rejected": -11.479227756367292,
      "loss": 0.5034,
      "rewards/chosen": 0.0006177344654179827,
      "rewards/margins": 0.0012838053047731855,
      "rewards/rejected": -0.0006660708393552028,
      "step": 79
    },
    {
      "epoch": 1.3938294010889292,
      "grad_norm": 0.19607041776180267,
      "kl": 0.0031144104432314634,
      "learning_rate": 1.5178571428571428e-06,
      "logps/chosen": -11.208266241253181,
      "logps/rejected": -10.324174479166667,
      "loss": 0.5036,
      "rewards/chosen": 0.0007915615883795663,
      "rewards/margins": 0.0014803941892910246,
      "rewards/rejected": -0.0006888326009114583,
      "step": 80
    },
    {
      "epoch": 1.3938294010889292,
      "eval_kl": 0.0010345944901928306,
      "eval_logps/chosen": -11.287428719656807,
      "eval_logps/rejected": -10.653524126325335,
      "eval_loss": 0.5035938024520874,
      "eval_rewards/chosen": 0.0005107134514089141,
      "eval_rewards/margins": 0.0009828850826514618,
      "eval_rewards/rejected": -0.0004721716312425477,
      "eval_runtime": 41.5877,
      "eval_samples_per_second": 10.724,
      "eval_steps_per_second": 0.673,
      "step": 80
    },
    {
      "epoch": 1.4112522686025408,
      "grad_norm": 0.2139582335948944,
      "kl": 0.0026578225661069155,
      "learning_rate": 1.473214285714286e-06,
      "logps/chosen": -11.404391783887467,
      "logps/rejected": -10.326293621518568,
      "loss": 0.5037,
      "rewards/chosen": 0.0004199028701123679,
      "rewards/margins": 0.0008872187001457688,
      "rewards/rejected": -0.00046731583003340095,
      "step": 81
    },
    {
      "epoch": 1.4286751361161525,
      "grad_norm": 0.3459619879722595,
      "kl": 0.001983026973903179,
      "learning_rate": 1.4285714285714286e-06,
      "logps/chosen": -11.670273977308417,
      "logps/rejected": -10.357556482263513,
      "loss": 0.5033,
      "rewards/chosen": 0.0005598089203762649,
      "rewards/margins": 0.0007428220220635673,
      "rewards/rejected": -0.00018301310168730245,
      "step": 82
    },
    {
      "epoch": 1.4460980036297642,
      "grad_norm": 0.21906645596027374,
      "kl": 0.0028725937008857727,
      "learning_rate": 1.3839285714285715e-06,
      "logps/chosen": -11.741632576689632,
      "logps/rejected": -10.43268595062177,
      "loss": 0.5033,
      "rewards/chosen": 0.0006233549962832234,
      "rewards/margins": 0.001042224365689024,
      "rewards/rejected": -0.0004188693694058007,
      "step": 83
    },
    {
      "epoch": 1.4635208711433756,
      "grad_norm": 0.17626065015792847,
      "kl": 0.00280402391217649,
      "learning_rate": 1.3392857142857143e-06,
      "logps/chosen": -11.461028108891753,
      "logps/rejected": -10.382724480879935,
      "loss": 0.5028,
      "rewards/chosen": 0.000817755371639409,
      "rewards/margins": 0.0010563352754176859,
      "rewards/rejected": -0.00023857990377827696,
      "step": 84
    },
    {
      "epoch": 1.4809437386569873,
      "grad_norm": 0.21237261593341827,
      "kl": 0.002868844196200371,
      "learning_rate": 1.2946428571428574e-06,
      "logps/chosen": -11.554072961329156,
      "logps/rejected": -10.606980036953727,
      "loss": 0.5036,
      "rewards/chosen": 0.00040721653476554046,
      "rewards/margins": 0.0007238921818244932,
      "rewards/rejected": -0.00031667564705895274,
      "step": 85
    },
    {
      "epoch": 1.498366606170599,
      "grad_norm": 0.21006950736045837,
      "kl": 0.0021779753733426332,
      "learning_rate": 1.25e-06,
      "logps/chosen": -11.395751308954486,
      "logps/rejected": -10.509847842022815,
      "loss": 0.5039,
      "rewards/chosen": 0.0002953462833464932,
      "rewards/margins": 0.0009452639913045675,
      "rewards/rejected": -0.0006499177079580743,
      "step": 86
    },
    {
      "epoch": 1.5157894736842106,
      "grad_norm": 0.2044811248779297,
      "kl": 0.0027360066305845976,
      "learning_rate": 1.205357142857143e-06,
      "logps/chosen": -11.286482900943396,
      "logps/rejected": -10.226171383028968,
      "loss": 0.5036,
      "rewards/chosen": 0.00042764071345007963,
      "rewards/margins": 0.000754331256658454,
      "rewards/rejected": -0.00032669054320837447,
      "step": 87
    },
    {
      "epoch": 1.5332123411978222,
      "grad_norm": 0.1994449943304062,
      "kl": 0.0012993334094062448,
      "learning_rate": 1.160714285714286e-06,
      "logps/chosen": -11.429487209875607,
      "logps/rejected": -10.525211634260884,
      "loss": 0.5034,
      "rewards/chosen": 0.0006345899649036741,
      "rewards/margins": 0.0010494590537087978,
      "rewards/rejected": -0.0004148690888051237,
      "step": 88
    },
    {
      "epoch": 1.5506352087114337,
      "grad_norm": 0.2269240915775299,
      "kl": 0.0028356958646327257,
      "learning_rate": 1.1160714285714287e-06,
      "logps/chosen": -11.494557624315693,
      "logps/rejected": -10.647711232930673,
      "loss": 0.5039,
      "rewards/chosen": 0.0005171763055806032,
      "rewards/margins": 0.0010993786980465312,
      "rewards/rejected": -0.000582202392465928,
      "step": 89
    },
    {
      "epoch": 1.5680580762250453,
      "grad_norm": 0.17898203432559967,
      "kl": 0.0027232880238443613,
      "learning_rate": 1.0714285714285714e-06,
      "logps/chosen": -11.28170191988032,
      "logps/rejected": -10.416350695551658,
      "loss": 0.5031,
      "rewards/chosen": 0.0005683141978497201,
      "rewards/margins": 0.0011697266601863967,
      "rewards/rejected": -0.0006014124623366765,
      "step": 90
    },
    {
      "epoch": 1.585480943738657,
      "grad_norm": 0.30514323711395264,
      "kl": 0.003049398772418499,
      "learning_rate": 1.0267857142857143e-06,
      "logps/chosen": -11.509693008814102,
      "logps/rejected": -10.68894675925926,
      "loss": 0.5045,
      "rewards/chosen": 0.0003556648890177409,
      "rewards/margins": 0.0007876505769749797,
      "rewards/rejected": -0.0004319856879572389,
      "step": 91
    },
    {
      "epoch": 1.6029038112522684,
      "grad_norm": 0.22825570404529572,
      "kl": 0.001876774593256414,
      "learning_rate": 9.821428571428572e-07,
      "logps/chosen": -11.782056884765625,
      "logps/rejected": -10.345401929772418,
      "loss": 0.5035,
      "rewards/chosen": 0.0005325853452086449,
      "rewards/margins": 0.0011105023842790853,
      "rewards/rejected": -0.0005779170390704404,
      "step": 92
    },
    {
      "epoch": 1.6203266787658803,
      "grad_norm": 0.1923515647649765,
      "kl": 0.0026991041377186775,
      "learning_rate": 9.375000000000001e-07,
      "logps/chosen": -11.152751884075126,
      "logps/rejected": -9.97277569514449,
      "loss": 0.5036,
      "rewards/chosen": 0.0005818240043490824,
      "rewards/margins": 0.0010237199423968812,
      "rewards/rejected": -0.0004418959380477987,
      "step": 93
    },
    {
      "epoch": 1.6377495462794918,
      "grad_norm": 0.21557894349098206,
      "kl": 0.001241236925125122,
      "learning_rate": 8.928571428571429e-07,
      "logps/chosen": -11.167566068672839,
      "logps/rejected": -11.65214332601584,
      "loss": 0.5034,
      "rewards/chosen": 0.0005508374652744811,
      "rewards/margins": 0.0011452616979097689,
      "rewards/rejected": -0.0005944242326352879,
      "step": 94
    },
    {
      "epoch": 1.6551724137931034,
      "grad_norm": 0.15490390360355377,
      "kl": 0.0023578277323395014,
      "learning_rate": 8.482142857142859e-07,
      "logps/chosen": -11.630532976752022,
      "logps/rejected": -10.874779842962846,
      "loss": 0.5024,
      "rewards/chosen": 0.001025584428458201,
      "rewards/margins": 0.0015268274366321843,
      "rewards/rejected": -0.0005012430081739834,
      "step": 95
    },
    {
      "epoch": 1.672595281306715,
      "grad_norm": 0.21339836716651917,
      "kl": 0.0030644694343209267,
      "learning_rate": 8.035714285714287e-07,
      "logps/chosen": -11.422396998881075,
      "logps/rejected": -10.11446374544098,
      "loss": 0.5032,
      "rewards/chosen": 0.0006199539698603208,
      "rewards/margins": 0.0011987829361300795,
      "rewards/rejected": -0.0005788289662697587,
      "step": 96
    },
    {
      "epoch": 1.672595281306715,
      "eval_kl": 0.0012286369455978274,
      "eval_logps/chosen": -11.286660330636161,
      "eval_logps/rejected": -10.654095241001674,
      "eval_loss": 0.503455400466919,
      "eval_rewards/chosen": 0.0005875181938920702,
      "eval_rewards/margins": 0.0011167954320886306,
      "eval_rewards/rejected": -0.0005292772381965603,
      "eval_runtime": 41.5081,
      "eval_samples_per_second": 10.745,
      "eval_steps_per_second": 0.675,
      "step": 96
    },
    {
      "epoch": 1.6900181488203265,
      "grad_norm": 0.1822512149810791,
      "kl": 0.0018086197087541223,
      "learning_rate": 7.589285714285714e-07,
      "logps/chosen": -11.162734347797702,
      "logps/rejected": -10.495764249694377,
      "loss": 0.5024,
      "rewards/chosen": 0.0007518338957868911,
      "rewards/margins": 0.0012704190070132039,
      "rewards/rejected": -0.0005185851112263127,
      "step": 97
    },
    {
      "epoch": 1.7074410163339384,
      "grad_norm": 0.25983884930610657,
      "kl": 0.001475659548304975,
      "learning_rate": 7.142857142857143e-07,
      "logps/chosen": -11.400912198153408,
      "logps/rejected": -10.339601098037347,
      "loss": 0.5035,
      "rewards/chosen": 0.0009949224916371432,
      "rewards/margins": 0.0013851757258375045,
      "rewards/rejected": -0.00039025323420036127,
      "step": 98
    },
    {
      "epoch": 1.7248638838475499,
      "grad_norm": 0.18693022429943085,
      "kl": 0.002064686268568039,
      "learning_rate": 6.696428571428571e-07,
      "logps/chosen": -11.613883205191799,
      "logps/rejected": -10.563515374599358,
      "loss": 0.5033,
      "rewards/chosen": 0.0007188759310535653,
      "rewards/margins": 0.0014389748514110983,
      "rewards/rejected": -0.000720098920357533,
      "step": 99
    },
    {
      "epoch": 1.7422867513611615,
      "grad_norm": 0.2225840985774994,
      "kl": 0.0023758753668516874,
      "learning_rate": 6.25e-07,
      "logps/chosen": -11.64061211658971,
      "logps/rejected": -10.474972761688624,
      "loss": 0.503,
      "rewards/chosen": 0.0006008470986323495,
      "rewards/margins": 0.0013913730829674177,
      "rewards/rejected": -0.0007905259843350682,
      "step": 100
    },
    {
      "epoch": 1.7597096188747732,
      "grad_norm": 0.20069453120231628,
      "kl": 0.001968987053260207,
      "learning_rate": 5.80357142857143e-07,
      "logps/chosen": -11.33251953125,
      "logps/rejected": -10.533562193525599,
      "loss": 0.5042,
      "rewards/chosen": 0.0002462785234864877,
      "rewards/margins": 0.0005758832525453824,
      "rewards/rejected": -0.0003296047290588947,
      "step": 101
    },
    {
      "epoch": 1.7771324863883846,
      "grad_norm": 0.1894039809703827,
      "kl": 0.003226263215765357,
      "learning_rate": 5.357142857142857e-07,
      "logps/chosen": -11.40087890625,
      "logps/rejected": -10.571763671875,
      "loss": 0.504,
      "rewards/chosen": 0.00039598065174869607,
      "rewards/margins": 0.0011867985482737612,
      "rewards/rejected": -0.0007908178965250651,
      "step": 102
    },
    {
      "epoch": 1.7945553539019965,
      "grad_norm": 0.16288532316684723,
      "kl": 0.002215030835941434,
      "learning_rate": 4.910714285714286e-07,
      "logps/chosen": -11.702183131167763,
      "logps/rejected": -10.757870389014176,
      "loss": 0.5027,
      "rewards/chosen": 0.0007084523376665617,
      "rewards/margins": 0.001386926632437186,
      "rewards/rejected": -0.0006784742947706242,
      "step": 103
    },
    {
      "epoch": 1.811978221415608,
      "grad_norm": 0.18168872594833374,
      "kl": 0.0029251668602228165,
      "learning_rate": 4.4642857142857147e-07,
      "logps/chosen": -11.455708379330842,
      "logps/rejected": -11.033857421875,
      "loss": 0.5029,
      "rewards/chosen": 0.0005165521136444548,
      "rewards/margins": 0.0011331064335030059,
      "rewards/rejected": -0.0006165543198585511,
      "step": 104
    },
    {
      "epoch": 1.8294010889292196,
      "grad_norm": 0.17783309519290924,
      "kl": 0.0027887674514204264,
      "learning_rate": 4.0178571428571434e-07,
      "logps/chosen": -11.572852601396276,
      "logps/rejected": -9.61759107940051,
      "loss": 0.5029,
      "rewards/chosen": 0.0006706785965473093,
      "rewards/margins": 0.001759196485623659,
      "rewards/rejected": -0.0010885178890763497,
      "step": 105
    },
    {
      "epoch": 1.8468239564428313,
      "grad_norm": 0.18048368394374847,
      "kl": 0.0018043291056528687,
      "learning_rate": 3.5714285714285716e-07,
      "logps/chosen": -11.548997631962436,
      "logps/rejected": -11.18207649910013,
      "loss": 0.5038,
      "rewards/chosen": 0.00041619074468167953,
      "rewards/margins": 0.0005713092191551835,
      "rewards/rejected": -0.00015511847447350388,
      "step": 106
    },
    {
      "epoch": 1.8642468239564427,
      "grad_norm": 0.4144538938999176,
      "kl": 0.0020457797218114138,
      "learning_rate": 3.125e-07,
      "logps/chosen": -10.976453261876685,
      "logps/rejected": -10.985627066278338,
      "loss": 0.5048,
      "rewards/chosen": 0.00022244479537331512,
      "rewards/margins": 0.0006958120098764712,
      "rewards/rejected": -0.00047336721450315616,
      "step": 107
    },
    {
      "epoch": 1.8816696914700546,
      "grad_norm": 0.19312834739685059,
      "kl": 0.0014575794339179993,
      "learning_rate": 2.6785714285714284e-07,
      "logps/chosen": -11.37527155066906,
      "logps/rejected": -10.887072595373377,
      "loss": 0.5035,
      "rewards/chosen": 0.0006041409686091052,
      "rewards/margins": 0.0013794575971519186,
      "rewards/rejected": -0.0007753166285428134,
      "step": 108
    },
    {
      "epoch": 1.899092558983666,
      "grad_norm": 0.24906954169273376,
      "kl": 0.002399616641923785,
      "learning_rate": 2.2321428571428574e-07,
      "logps/chosen": -11.331646469763092,
      "logps/rejected": -10.6643505460235,
      "loss": 0.5046,
      "rewards/chosen": 0.0009270152843503881,
      "rewards/margins": 0.001616924349477436,
      "rewards/rejected": -0.0006899090651270479,
      "step": 109
    },
    {
      "epoch": 1.9165154264972777,
      "grad_norm": 0.1880314201116562,
      "kl": 0.0026240963488817215,
      "learning_rate": 1.7857142857142858e-07,
      "logps/chosen": -11.29613597972973,
      "logps/rejected": -10.615358285568467,
      "loss": 0.5031,
      "rewards/chosen": 0.0010521258051330979,
      "rewards/margins": 0.0013450532624429453,
      "rewards/rejected": -0.0002929274573098475,
      "step": 110
    },
    {
      "epoch": 1.9339382940108893,
      "grad_norm": 0.18609453737735748,
      "kl": 0.002484673634171486,
      "learning_rate": 1.3392857142857142e-07,
      "logps/chosen": -11.604450023938023,
      "logps/rejected": -10.78327236781785,
      "loss": 0.5028,
      "rewards/chosen": 0.0006836153836635494,
      "rewards/margins": 0.001532551150396621,
      "rewards/rejected": -0.0008489357667330717,
      "step": 111
    },
    {
      "epoch": 1.9513611615245008,
      "grad_norm": 0.20338238775730133,
      "kl": 0.0025711385533213615,
      "learning_rate": 8.928571428571429e-08,
      "logps/chosen": -11.439998413705585,
      "logps/rejected": -10.535859296666109,
      "loss": 0.5036,
      "rewards/chosen": 0.0005479781760782154,
      "rewards/margins": 0.0011845395663098528,
      "rewards/rejected": -0.0006365613902316374,
      "step": 112
    },
    {
      "epoch": 1.9513611615245008,
      "eval_kl": 0.000932498718611896,
      "eval_logps/chosen": -11.286882672991071,
      "eval_logps/rejected": -10.654556274414062,
      "eval_loss": 0.5036724805831909,
      "eval_rewards/chosen": 0.0005651861429214478,
      "eval_rewards/margins": 0.0011406249499746732,
      "eval_rewards/rejected": -0.0005754388070532254,
      "eval_runtime": 41.5261,
      "eval_samples_per_second": 10.74,
      "eval_steps_per_second": 0.674,
      "step": 112
    },
    {
      "epoch": 1.9687840290381127,
      "grad_norm": 0.18693436682224274,
      "kl": 0.0030837843660265207,
      "learning_rate": 4.4642857142857145e-08,
      "logps/chosen": -11.600828793174342,
      "logps/rejected": -10.943336722777062,
      "loss": 0.5031,
      "rewards/chosen": 0.0006475970541176044,
      "rewards/margins": 0.0013402001844336014,
      "rewards/rejected": -0.0006926031303159969,
      "step": 113
    },
    {
      "epoch": 1.986206896551724,
      "grad_norm": 0.1876712143421173,
      "kl": 0.0018025139579549432,
      "learning_rate": 0.0,
      "logps/chosen": -11.169455056662088,
      "logps/rejected": -10.222750522122524,
      "loss": 0.5029,
      "rewards/chosen": 0.0007852382221064725,
      "rewards/margins": 0.0014990826276489785,
      "rewards/rejected": -0.0007138444055425059,
      "step": 114
    },
    {
      "epoch": 1.986206896551724,
      "step": 114,
      "total_flos": 2.0165964255461376e+18,
      "train_loss": 0.5037083285942412,
      "train_runtime": 19753.9777,
      "train_samples_per_second": 4.463,
      "train_steps_per_second": 0.006
    }
  ],
  "logging_steps": 1,
  "max_steps": 114,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 2,
  "save_steps": 500,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": true
      },
      "attributes": {}
    }
  },
  "total_flos": 2.0165964255461376e+18,
  "train_batch_size": 2,
  "trial_name": null,
  "trial_params": null
}