{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 2.988679245283019,
  "eval_steps": 500,
  "global_step": 396,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.01,
      "learning_rate": 1.25e-08,
      "logps/chosen": -22.472335815429688,
      "logps/rejected": -25.36812400817871,
      "loss": 0.6931,
      "losses/dpo": 0.6931471824645996,
      "losses/sft": 0.7711470723152161,
      "losses/total": 0.6931471824645996,
      "ref_logps/chosen": -22.472335815429688,
      "ref_logps/rejected": -25.36812400817871,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 1
    },
    {
      "epoch": 0.02,
      "learning_rate": 2.5e-08,
      "logps/chosen": -21.278339385986328,
      "logps/rejected": -25.130128860473633,
      "loss": 0.6931,
      "losses/dpo": 0.6931471824645996,
      "losses/sft": 0.8523496985435486,
      "losses/total": 0.6931471824645996,
      "ref_logps/chosen": -21.278339385986328,
      "ref_logps/rejected": -25.130128860473633,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 2
    },
    {
      "epoch": 0.02,
      "learning_rate": 3.75e-08,
      "logps/chosen": -21.53506851196289,
      "logps/rejected": -26.44188690185547,
      "loss": 0.693,
      "losses/dpo": 0.6928481459617615,
      "losses/sft": 0.6631997227668762,
      "losses/total": 0.6928481459617615,
      "ref_logps/chosen": -21.54958152770996,
      "ref_logps/rejected": -26.452028274536133,
      "rewards/accuracies": 0.515625,
      "rewards/chosen": 0.001451290212571621,
      "rewards/margins": 0.00043702672701328993,
      "rewards/rejected": 0.0010142631363123655,
      "step": 3
    },
    {
      "epoch": 0.03,
      "learning_rate": 5e-08,
      "logps/chosen": -21.846920013427734,
      "logps/rejected": -26.232192993164062,
      "loss": 0.6935,
      "losses/dpo": 0.6933612823486328,
      "losses/sft": 0.819932758808136,
      "losses/total": 0.6933612823486328,
      "ref_logps/chosen": -21.842269897460938,
      "ref_logps/rejected": -26.234174728393555,
      "rewards/accuracies": 0.4609375,
      "rewards/chosen": -0.00046504498459398746,
      "rewards/margins": -0.000663207727484405,
      "rewards/rejected": 0.0001981628010980785,
      "step": 4
    },
    {
      "epoch": 0.04,
      "learning_rate": 6.25e-08,
      "logps/chosen": -23.82025146484375,
      "logps/rejected": -26.558738708496094,
      "loss": 0.693,
      "losses/dpo": 0.6929464340209961,
      "losses/sft": 0.7624120712280273,
      "losses/total": 0.6929464340209961,
      "ref_logps/chosen": -23.817665100097656,
      "ref_logps/rejected": -26.55132293701172,
      "rewards/accuracies": 0.5234375,
      "rewards/chosen": -0.00025857496075332165,
      "rewards/margins": 0.0004831284750252962,
      "rewards/rejected": -0.0007417035521939397,
      "step": 5
    },
    {
      "epoch": 0.05,
      "learning_rate": 7.5e-08,
      "logps/chosen": -25.088871002197266,
      "logps/rejected": -29.653806686401367,
      "loss": 0.6923,
      "losses/dpo": 0.6934427618980408,
      "losses/sft": 0.7273141741752625,
      "losses/total": 0.6934427618980408,
      "ref_logps/chosen": -25.0992431640625,
      "ref_logps/rejected": -29.64551544189453,
      "rewards/accuracies": 0.6015625,
      "rewards/chosen": 0.0010370061499997973,
      "rewards/margins": 0.0018662326037883759,
      "rewards/rejected": -0.0008292265702039003,
      "step": 6
    },
    {
      "epoch": 0.05,
      "learning_rate": 8.75e-08,
      "logps/chosen": -23.075027465820312,
      "logps/rejected": -27.50556182861328,
      "loss": 0.693,
      "losses/dpo": 0.6948896646499634,
      "losses/sft": 0.6432714462280273,
      "losses/total": 0.6948896646499634,
      "ref_logps/chosen": -23.066946029663086,
      "ref_logps/rejected": -27.4930362701416,
      "rewards/accuracies": 0.4765625,
      "rewards/chosen": -0.0008082209387794137,
      "rewards/margins": 0.0004443599027581513,
      "rewards/rejected": -0.001252580899745226,
      "step": 7
    },
    {
      "epoch": 0.06,
      "learning_rate": 1e-07,
      "logps/chosen": -21.430335998535156,
      "logps/rejected": -29.949260711669922,
      "loss": 0.6933,
      "losses/dpo": 0.6911635398864746,
      "losses/sft": 0.8042243123054504,
      "losses/total": 0.6911635398864746,
      "ref_logps/chosen": -21.44394302368164,
      "ref_logps/rejected": -29.96406364440918,
      "rewards/accuracies": 0.4921875,
      "rewards/chosen": 0.0013606649590656161,
      "rewards/margins": -0.00011985772289335728,
      "rewards/rejected": 0.0014805227983742952,
      "step": 8
    },
    {
      "epoch": 0.07,
      "learning_rate": 1.125e-07,
      "logps/chosen": -23.053390502929688,
      "logps/rejected": -27.866111755371094,
      "loss": 0.6923,
      "losses/dpo": 0.6914368271827698,
      "losses/sft": 0.8787165284156799,
      "losses/total": 0.6914368271827698,
      "ref_logps/chosen": -23.060134887695312,
      "ref_logps/rejected": -27.85537338256836,
      "rewards/accuracies": 0.5546875,
      "rewards/chosen": 0.0006745259161107242,
      "rewards/margins": 0.0017485294956713915,
      "rewards/rejected": -0.0010740034049376845,
      "step": 9
    },
    {
      "epoch": 0.08,
      "learning_rate": 1.25e-07,
      "logps/chosen": -23.637466430664062,
      "logps/rejected": -29.587308883666992,
      "loss": 0.6922,
      "losses/dpo": 0.690066397190094,
      "losses/sft": 1.0419297218322754,
      "losses/total": 0.690066397190094,
      "ref_logps/chosen": -23.649028778076172,
      "ref_logps/rejected": -29.579374313354492,
      "rewards/accuracies": 0.546875,
      "rewards/chosen": 0.0011563875013962388,
      "rewards/margins": 0.0019498697947710752,
      "rewards/rejected": -0.0007934823515824974,
      "step": 10
    },
    {
      "epoch": 0.08,
      "learning_rate": 1.375e-07,
      "logps/chosen": -22.38899040222168,
      "logps/rejected": -24.971160888671875,
      "loss": 0.6926,
      "losses/dpo": 0.6951523423194885,
      "losses/sft": 0.9443475008010864,
      "losses/total": 0.6951523423194885,
      "ref_logps/chosen": -22.398780822753906,
      "ref_logps/rejected": -24.969751358032227,
      "rewards/accuracies": 0.5078125,
      "rewards/chosen": 0.000978996278718114,
      "rewards/margins": 0.0011200353037565947,
      "rewards/rejected": -0.00014103890862315893,
      "step": 11
    },
    {
      "epoch": 0.09,
      "learning_rate": 1.5e-07,
      "logps/chosen": -20.165252685546875,
      "logps/rejected": -26.619457244873047,
      "loss": 0.6946,
      "losses/dpo": 0.6987805962562561,
      "losses/sft": 0.876471221446991,
      "losses/total": 0.6987805962562561,
      "ref_logps/chosen": -20.14897918701172,
      "ref_logps/rejected": -26.63131332397461,
      "rewards/accuracies": 0.421875,
      "rewards/chosen": -0.001627539866603911,
      "rewards/margins": -0.002813115483149886,
      "rewards/rejected": 0.0011855755001306534,
      "step": 12
    },
    {
      "epoch": 0.1,
      "learning_rate": 1.625e-07,
      "logps/chosen": -25.07573699951172,
      "logps/rejected": -25.939855575561523,
      "loss": 0.6936,
      "losses/dpo": 0.6952416896820068,
      "losses/sft": 0.9322817325592041,
      "losses/total": 0.6952416896820068,
      "ref_logps/chosen": -25.075220108032227,
      "ref_logps/rejected": -25.947521209716797,
      "rewards/accuracies": 0.515625,
      "rewards/chosen": -5.185510963201523e-05,
      "rewards/margins": -0.0008183673489838839,
      "rewards/rejected": 0.0007665121229365468,
      "step": 13
    },
    {
      "epoch": 0.11,
      "learning_rate": 1.75e-07,
      "logps/chosen": -22.58213233947754,
      "logps/rejected": -27.590843200683594,
      "loss": 0.692,
      "losses/dpo": 0.6901522874832153,
      "losses/sft": 0.8234641551971436,
      "losses/total": 0.6901522874832153,
      "ref_logps/chosen": -22.58617401123047,
      "ref_logps/rejected": -27.570602416992188,
      "rewards/accuracies": 0.578125,
      "rewards/chosen": 0.00040407240157946944,
      "rewards/margins": 0.002427991945296526,
      "rewards/rejected": -0.0020239197183400393,
      "step": 14
    },
    {
      "epoch": 0.11,
      "learning_rate": 1.875e-07,
      "logps/chosen": -23.004196166992188,
      "logps/rejected": -25.858173370361328,
      "loss": 0.6925,
      "losses/dpo": 0.6923660039901733,
      "losses/sft": 0.7345502376556396,
      "losses/total": 0.6923660039901733,
      "ref_logps/chosen": -23.010601043701172,
      "ref_logps/rejected": -25.85067367553711,
      "rewards/accuracies": 0.515625,
      "rewards/chosen": 0.0006403709994629025,
      "rewards/margins": 0.0013903947547078133,
      "rewards/rejected": -0.0007500239298678935,
      "step": 15
    },
    {
      "epoch": 0.12,
      "learning_rate": 2e-07,
      "logps/chosen": -21.546062469482422,
      "logps/rejected": -25.777360916137695,
      "loss": 0.6931,
      "losses/dpo": 0.6901232004165649,
      "losses/sft": 0.8039647936820984,
      "losses/total": 0.6901232004165649,
      "ref_logps/chosen": -21.53840446472168,
      "ref_logps/rejected": -25.766767501831055,
      "rewards/accuracies": 0.5390625,
      "rewards/chosen": -0.0007656853413209319,
      "rewards/margins": 0.0002937153331004083,
      "rewards/rejected": -0.0010594006162136793,
      "step": 16
    },
    {
      "epoch": 0.13,
      "learning_rate": 2.1249999999999998e-07,
      "logps/chosen": -22.206989288330078,
      "logps/rejected": -27.877731323242188,
      "loss": 0.6937,
      "losses/dpo": 0.6932737827301025,
      "losses/sft": 0.7667961716651917,
      "losses/total": 0.6932737827301025,
      "ref_logps/chosen": -22.19771957397461,
      "ref_logps/rejected": -27.87958335876465,
      "rewards/accuracies": 0.4453125,
      "rewards/chosen": -0.0009270801674574614,
      "rewards/margins": -0.0011123311705887318,
      "rewards/rejected": 0.0001852509449236095,
      "step": 17
    },
    {
      "epoch": 0.14,
      "learning_rate": 2.25e-07,
      "logps/chosen": -21.215139389038086,
      "logps/rejected": -25.75381088256836,
      "loss": 0.693,
      "losses/dpo": 0.6932240724563599,
      "losses/sft": 0.736687421798706,
      "losses/total": 0.6932240724563599,
      "ref_logps/chosen": -21.212387084960938,
      "ref_logps/rejected": -25.746326446533203,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.00027507508639246225,
      "rewards/margins": 0.000473553518531844,
      "rewards/rejected": -0.0007486287504434586,
      "step": 18
    },
    {
      "epoch": 0.14,
      "learning_rate": 2.3749999999999998e-07,
      "logps/chosen": -22.499832153320312,
      "logps/rejected": -26.145751953125,
      "loss": 0.6932,
      "losses/dpo": 0.6942628622055054,
      "losses/sft": 0.7466978430747986,
      "losses/total": 0.6942628622055054,
      "ref_logps/chosen": -22.496463775634766,
      "ref_logps/rejected": -26.141849517822266,
      "rewards/accuracies": 0.484375,
      "rewards/chosen": -0.0003368390607647598,
      "rewards/margins": 5.3280091378837824e-05,
      "rewards/rejected": -0.0003901191521435976,
      "step": 19
    },
    {
      "epoch": 0.15,
      "learning_rate": 2.5e-07,
      "logps/chosen": -21.5505428314209,
      "logps/rejected": -25.036113739013672,
      "loss": 0.693,
      "losses/dpo": 0.688271164894104,
      "losses/sft": 0.8725596070289612,
      "losses/total": 0.688271164894104,
      "ref_logps/chosen": -21.558109283447266,
      "ref_logps/rejected": -25.038726806640625,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.0007566105341538787,
      "rewards/margins": 0.0004949538852088153,
      "rewards/rejected": 0.0002616568235680461,
      "step": 20
    },
    {
      "epoch": 0.16,
      "learning_rate": 2.625e-07,
      "logps/chosen": -21.649169921875,
      "logps/rejected": -25.741392135620117,
      "loss": 0.6917,
      "losses/dpo": 0.6939514875411987,
      "losses/sft": 0.7525328993797302,
      "losses/total": 0.6939514875411987,
      "ref_logps/chosen": -21.666126251220703,
      "ref_logps/rejected": -25.72817611694336,
      "rewards/accuracies": 0.640625,
      "rewards/chosen": 0.0016953760059550405,
      "rewards/margins": 0.003016936592757702,
      "rewards/rejected": -0.0013215603539720178,
      "step": 21
    },
    {
      "epoch": 0.17,
      "learning_rate": 2.75e-07,
      "logps/chosen": -21.422496795654297,
      "logps/rejected": -26.453773498535156,
      "loss": 0.695,
      "losses/dpo": 0.699163019657135,
      "losses/sft": 0.7248706221580505,
      "losses/total": 0.699163019657135,
      "ref_logps/chosen": -21.396032333374023,
      "ref_logps/rejected": -26.464006423950195,
      "rewards/accuracies": 0.4296875,
      "rewards/chosen": -0.002646287204697728,
      "rewards/margins": -0.003669553902000189,
      "rewards/rejected": 0.0010232668137177825,
      "step": 22
    },
    {
      "epoch": 0.17,
      "learning_rate": 2.8749999999999995e-07,
      "logps/chosen": -21.21988868713379,
      "logps/rejected": -25.13469886779785,
      "loss": 0.6929,
      "losses/dpo": 0.6908746957778931,
      "losses/sft": 0.7899657487869263,
      "losses/total": 0.6908746957778931,
      "ref_logps/chosen": -21.22311782836914,
      "ref_logps/rejected": -25.131580352783203,
      "rewards/accuracies": 0.515625,
      "rewards/chosen": 0.0003229643334634602,
      "rewards/margins": 0.0006348754977807403,
      "rewards/rejected": -0.00031191116431728005,
      "step": 23
    },
    {
      "epoch": 0.18,
      "learning_rate": 3e-07,
      "logps/chosen": -24.172225952148438,
      "logps/rejected": -27.93877410888672,
      "loss": 0.6936,
      "losses/dpo": 0.6931849718093872,
      "losses/sft": 0.7270597219467163,
      "losses/total": 0.6931849718093872,
      "ref_logps/chosen": -24.16461944580078,
      "ref_logps/rejected": -27.940391540527344,
      "rewards/accuracies": 0.4765625,
      "rewards/chosen": -0.000760397466365248,
      "rewards/margins": -0.0009220357751473784,
      "rewards/rejected": 0.00016163833788596094,
      "step": 24
    },
    {
      "epoch": 0.19,
      "learning_rate": 3.1249999999999997e-07,
      "logps/chosen": -23.023677825927734,
      "logps/rejected": -23.77918243408203,
      "loss": 0.6929,
      "losses/dpo": 0.6930486559867859,
      "losses/sft": 0.779391884803772,
      "losses/total": 0.6930486559867859,
      "ref_logps/chosen": -23.028684616088867,
      "ref_logps/rejected": -23.778560638427734,
      "rewards/accuracies": 0.546875,
      "rewards/chosen": 0.0005005812272429466,
      "rewards/margins": 0.0005629429360851645,
      "rewards/rejected": -6.23615924268961e-05,
      "step": 25
    },
    {
      "epoch": 0.2,
      "learning_rate": 3.25e-07,
      "logps/chosen": -24.240978240966797,
      "logps/rejected": -30.183570861816406,
      "loss": 0.6923,
      "losses/dpo": 0.6919558644294739,
      "losses/sft": 0.8828473091125488,
      "losses/total": 0.6919558644294739,
      "ref_logps/chosen": -24.253870010375977,
      "ref_logps/rejected": -30.17804718017578,
      "rewards/accuracies": 0.5078125,
      "rewards/chosen": 0.001289202249608934,
      "rewards/margins": 0.0018417320679873228,
      "rewards/rejected": -0.0005525298183783889,
      "step": 26
    },
    {
      "epoch": 0.2,
      "learning_rate": 3.375e-07,
      "logps/chosen": -22.371261596679688,
      "logps/rejected": -28.10503387451172,
      "loss": 0.6911,
      "losses/dpo": 0.6919010281562805,
      "losses/sft": 0.9361266493797302,
      "losses/total": 0.6919010281562805,
      "ref_logps/chosen": -22.4020938873291,
      "ref_logps/rejected": -28.094257354736328,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": 0.0030831946060061455,
      "rewards/margins": 0.004160974640399218,
      "rewards/rejected": -0.001077780150808394,
      "step": 27
    },
    {
      "epoch": 0.21,
      "learning_rate": 3.5e-07,
      "logps/chosen": -21.107967376708984,
      "logps/rejected": -27.053752899169922,
      "loss": 0.6921,
      "losses/dpo": 0.6916664838790894,
      "losses/sft": 0.8491181135177612,
      "losses/total": 0.6916664838790894,
      "ref_logps/chosen": -21.1080379486084,
      "ref_logps/rejected": -27.03229331970215,
      "rewards/accuracies": 0.5390625,
      "rewards/chosen": 7.087946869432926e-06,
      "rewards/margins": 0.0021530785597860813,
      "rewards/rejected": -0.00214599072933197,
      "step": 28
    },
    {
      "epoch": 0.22,
      "learning_rate": 3.6249999999999997e-07,
      "logps/chosen": -23.424461364746094,
      "logps/rejected": -27.092483520507812,
      "loss": 0.6906,
      "losses/dpo": 0.6926239728927612,
      "losses/sft": 0.7789149284362793,
      "losses/total": 0.6926239728927612,
      "ref_logps/chosen": -23.46218490600586,
      "ref_logps/rejected": -27.07909393310547,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": 0.0037725979927927256,
      "rewards/margins": 0.005111560225486755,
      "rewards/rejected": -0.0013389625819399953,
      "step": 29
    },
    {
      "epoch": 0.23,
      "learning_rate": 3.75e-07,
      "logps/chosen": -22.859556198120117,
      "logps/rejected": -27.201662063598633,
      "loss": 0.6933,
      "losses/dpo": 0.6948127746582031,
      "losses/sft": 0.7969105243682861,
      "losses/total": 0.6948127746582031,
      "ref_logps/chosen": -22.869096755981445,
      "ref_logps/rejected": -27.212430953979492,
      "rewards/accuracies": 0.515625,
      "rewards/chosen": 0.0009542852640151978,
      "rewards/margins": -0.00012272456660866737,
      "rewards/rejected": 0.001077009947039187,
      "step": 30
    },
    {
      "epoch": 0.23,
      "learning_rate": 3.875e-07,
      "logps/chosen": -22.666168212890625,
      "logps/rejected": -25.310596466064453,
      "loss": 0.6918,
      "losses/dpo": 0.6922581195831299,
      "losses/sft": 0.7759775519371033,
      "losses/total": 0.6922581195831299,
      "ref_logps/chosen": -22.68026351928711,
      "ref_logps/rejected": -25.297521591186523,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.001409594900906086,
      "rewards/margins": 0.002717201365157962,
      "rewards/rejected": -0.001307606347836554,
      "step": 31
    },
    {
      "epoch": 0.24,
      "learning_rate": 4e-07,
      "logps/chosen": -23.281084060668945,
      "logps/rejected": -28.84569549560547,
      "loss": 0.693,
      "losses/dpo": 0.6980300545692444,
      "losses/sft": 0.7636886835098267,
      "losses/total": 0.6980300545692444,
      "ref_logps/chosen": -23.299869537353516,
      "ref_logps/rejected": -28.859834671020508,
      "rewards/accuracies": 0.5390625,
      "rewards/chosen": 0.0018782642437145114,
      "rewards/margins": 0.000464284501504153,
      "rewards/rejected": 0.0014139798004180193,
      "step": 32
    },
    {
      "epoch": 0.25,
      "learning_rate": 4.1249999999999997e-07,
      "logps/chosen": -20.922544479370117,
      "logps/rejected": -27.139453887939453,
      "loss": 0.6914,
      "losses/dpo": 0.6892759799957275,
      "losses/sft": 0.7832686901092529,
      "losses/total": 0.6892759799957275,
      "ref_logps/chosen": -20.949806213378906,
      "ref_logps/rejected": -27.13178253173828,
      "rewards/accuracies": 0.6484375,
      "rewards/chosen": 0.0027261325158178806,
      "rewards/margins": 0.003493295982480049,
      "rewards/rejected": -0.0007671635248698294,
      "step": 33
    },
    {
      "epoch": 0.26,
      "learning_rate": 4.2499999999999995e-07,
      "logps/chosen": -22.535436630249023,
      "logps/rejected": -26.6143798828125,
      "loss": 0.6926,
      "losses/dpo": 0.6938276290893555,
      "losses/sft": 0.7895969152450562,
      "losses/total": 0.6938276290893555,
      "ref_logps/chosen": -22.540180206298828,
      "ref_logps/rejected": -26.607288360595703,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": 0.0004745282931253314,
      "rewards/margins": 0.0011834825854748487,
      "rewards/rejected": -0.0007089540013112128,
      "step": 34
    },
    {
      "epoch": 0.26,
      "learning_rate": 4.375e-07,
      "logps/chosen": -21.444934844970703,
      "logps/rejected": -27.329378128051758,
      "loss": 0.6928,
      "losses/dpo": 0.6910836100578308,
      "losses/sft": 0.7998620271682739,
      "losses/total": 0.6910836100578308,
      "ref_logps/chosen": -21.460729598999023,
      "ref_logps/rejected": -27.336944580078125,
      "rewards/accuracies": 0.515625,
      "rewards/chosen": 0.0015797324012964964,
      "rewards/margins": 0.0008232423570007086,
      "rewards/rejected": 0.0007564900442957878,
      "step": 35
    },
    {
      "epoch": 0.27,
      "learning_rate": 4.5e-07,
      "logps/chosen": -22.847640991210938,
      "logps/rejected": -26.22686195373535,
      "loss": 0.6938,
      "losses/dpo": 0.6915764808654785,
      "losses/sft": 0.7927474975585938,
      "losses/total": 0.6915764808654785,
      "ref_logps/chosen": -22.84987449645996,
      "ref_logps/rejected": -26.240699768066406,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.00022311191423796117,
      "rewards/margins": -0.001160716055892408,
      "rewards/rejected": 0.0013838279992341995,
      "step": 36
    },
    {
      "epoch": 0.28,
      "learning_rate": 4.625e-07,
      "logps/chosen": -23.097599029541016,
      "logps/rejected": -25.179964065551758,
      "loss": 0.6925,
      "losses/dpo": 0.6903287768363953,
      "losses/sft": 0.8005999326705933,
      "losses/total": 0.6903287768363953,
      "ref_logps/chosen": -23.103515625,
      "ref_logps/rejected": -25.171833038330078,
      "rewards/accuracies": 0.4921875,
      "rewards/chosen": 0.000591703865211457,
      "rewards/margins": 0.001405149232596159,
      "rewards/rejected": -0.000813445309177041,
      "step": 37
    },
    {
      "epoch": 0.29,
      "learning_rate": 4.7499999999999995e-07,
      "logps/chosen": -23.07529640197754,
      "logps/rejected": -26.14615821838379,
      "loss": 0.6912,
      "losses/dpo": 0.6978300213813782,
      "losses/sft": 0.7380209565162659,
      "losses/total": 0.6978300213813782,
      "ref_logps/chosen": -23.113910675048828,
      "ref_logps/rejected": -26.144914627075195,
      "rewards/accuracies": 0.578125,
      "rewards/chosen": 0.003861566074192524,
      "rewards/margins": 0.003985891118645668,
      "rewards/rejected": -0.00012432527728378773,
      "step": 38
    },
    {
      "epoch": 0.29,
      "learning_rate": 4.875e-07,
      "logps/chosen": -23.091575622558594,
      "logps/rejected": -28.207073211669922,
      "loss": 0.6933,
      "losses/dpo": 0.6926023960113525,
      "losses/sft": 0.7966833710670471,
      "losses/total": 0.6926023960113525,
      "ref_logps/chosen": -23.102182388305664,
      "ref_logps/rejected": -28.21949577331543,
      "rewards/accuracies": 0.5234375,
      "rewards/chosen": 0.0010607184376567602,
      "rewards/margins": -0.00018126872600987554,
      "rewards/rejected": 0.0012419875711202621,
      "step": 39
    },
    {
      "epoch": 0.3,
      "learning_rate": 5e-07,
      "logps/chosen": -21.683151245117188,
      "logps/rejected": -27.111900329589844,
      "loss": 0.6903,
      "losses/dpo": 0.6866278648376465,
      "losses/sft": 0.887488842010498,
      "losses/total": 0.6866278648376465,
      "ref_logps/chosen": -21.714126586914062,
      "ref_logps/rejected": -27.08427619934082,
      "rewards/accuracies": 0.609375,
      "rewards/chosen": 0.003097555134445429,
      "rewards/margins": 0.0058600143529474735,
      "rewards/rejected": -0.0027624592185020447,
      "step": 40
    },
    {
      "epoch": 0.31,
      "learning_rate": 4.985955056179775e-07,
      "logps/chosen": -23.24443817138672,
      "logps/rejected": -24.057823181152344,
      "loss": 0.6924,
      "losses/dpo": 0.6903232336044312,
      "losses/sft": 0.7454457879066467,
      "losses/total": 0.6903232336044312,
      "ref_logps/chosen": -23.264373779296875,
      "ref_logps/rejected": -24.060710906982422,
      "rewards/accuracies": 0.4765625,
      "rewards/chosen": 0.0019934140145778656,
      "rewards/margins": 0.0017045673448592424,
      "rewards/rejected": 0.0002888469025492668,
      "step": 41
    },
    {
      "epoch": 0.32,
      "learning_rate": 4.97191011235955e-07,
      "logps/chosen": -22.751291275024414,
      "logps/rejected": -23.993690490722656,
      "loss": 0.692,
      "losses/dpo": 0.6917561292648315,
      "losses/sft": 0.8527467846870422,
      "losses/total": 0.6917561292648315,
      "ref_logps/chosen": -22.75712013244629,
      "ref_logps/rejected": -23.975711822509766,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.0005830166628584266,
      "rewards/margins": 0.0023808996193110943,
      "rewards/rejected": -0.0017978833056986332,
      "step": 42
    },
    {
      "epoch": 0.32,
      "learning_rate": 4.957865168539325e-07,
      "logps/chosen": -24.575613021850586,
      "logps/rejected": -27.22784996032715,
      "loss": 0.6922,
      "losses/dpo": 0.6912024021148682,
      "losses/sft": 0.8869270086288452,
      "losses/total": 0.6912024021148682,
      "ref_logps/chosen": -24.60643196105957,
      "ref_logps/rejected": -27.23748779296875,
      "rewards/accuracies": 0.5234375,
      "rewards/chosen": 0.0030817545484751463,
      "rewards/margins": 0.002117899712175131,
      "rewards/rejected": 0.0009638546616770327,
      "step": 43
    },
    {
      "epoch": 0.33,
      "learning_rate": 4.943820224719101e-07,
      "logps/chosen": -23.449739456176758,
      "logps/rejected": -29.683177947998047,
      "loss": 0.6913,
      "losses/dpo": 0.690817654132843,
      "losses/sft": 0.7518939971923828,
      "losses/total": 0.690817654132843,
      "ref_logps/chosen": -23.47886848449707,
      "ref_logps/rejected": -29.673599243164062,
      "rewards/accuracies": 0.5546875,
      "rewards/chosen": 0.0029130401089787483,
      "rewards/margins": 0.0038708222564309835,
      "rewards/rejected": -0.0009577819146215916,
      "step": 44
    },
    {
      "epoch": 0.34,
      "learning_rate": 4.929775280898877e-07,
      "logps/chosen": -21.53199577331543,
      "logps/rejected": -26.939178466796875,
      "loss": 0.6923,
      "losses/dpo": 0.6913425922393799,
      "losses/sft": 0.6940815448760986,
      "losses/total": 0.6913425922393799,
      "ref_logps/chosen": -21.567256927490234,
      "ref_logps/rejected": -26.955793380737305,
      "rewards/accuracies": 0.5078125,
      "rewards/chosen": 0.0035258703865110874,
      "rewards/margins": 0.0018642698414623737,
      "rewards/rejected": 0.0016616008942946792,
      "step": 45
    },
    {
      "epoch": 0.35,
      "learning_rate": 4.915730337078651e-07,
      "logps/chosen": -25.476314544677734,
      "logps/rejected": -28.62994956970215,
      "loss": 0.6903,
      "losses/dpo": 0.6909126043319702,
      "losses/sft": 0.9766503572463989,
      "losses/total": 0.6909126043319702,
      "ref_logps/chosen": -25.53481674194336,
      "ref_logps/rejected": -28.629175186157227,
      "rewards/accuracies": 0.5703125,
      "rewards/chosen": 0.005850302986800671,
      "rewards/margins": 0.005927846767008305,
      "rewards/rejected": -7.754407124593854e-05,
      "step": 46
    },
    {
      "epoch": 0.35,
      "learning_rate": 4.901685393258427e-07,
      "logps/chosen": -24.225303649902344,
      "logps/rejected": -25.871768951416016,
      "loss": 0.6919,
      "losses/dpo": 0.6921358704566956,
      "losses/sft": 0.8468361496925354,
      "losses/total": 0.6921358704566956,
      "ref_logps/chosen": -24.27114486694336,
      "ref_logps/rejected": -25.892141342163086,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.004584114067256451,
      "rewards/margins": 0.0025467565283179283,
      "rewards/rejected": 0.0020373575389385223,
      "step": 47
    },
    {
      "epoch": 0.36,
      "learning_rate": 4.887640449438202e-07,
      "logps/chosen": -21.586902618408203,
      "logps/rejected": -27.604530334472656,
      "loss": 0.6902,
      "losses/dpo": 0.6886686086654663,
      "losses/sft": 0.7169030904769897,
      "losses/total": 0.6886686086654663,
      "ref_logps/chosen": -21.621915817260742,
      "ref_logps/rejected": -27.578147888183594,
      "rewards/accuracies": 0.5703125,
      "rewards/chosen": 0.003501205239444971,
      "rewards/margins": 0.006139571778476238,
      "rewards/rejected": -0.00263836607336998,
      "step": 48
    },
    {
      "epoch": 0.37,
      "learning_rate": 4.873595505617978e-07,
      "logps/chosen": -24.600910186767578,
      "logps/rejected": -30.10862922668457,
      "loss": 0.6909,
      "losses/dpo": 0.6952996850013733,
      "losses/sft": 0.7813842296600342,
      "losses/total": 0.6952996850013733,
      "ref_logps/chosen": -24.644847869873047,
      "ref_logps/rejected": -30.104995727539062,
      "rewards/accuracies": 0.5859375,
      "rewards/chosen": 0.004394051153212786,
      "rewards/margins": 0.004757395945489407,
      "rewards/rejected": -0.0003633448213804513,
      "step": 49
    },
    {
      "epoch": 0.38,
      "learning_rate": 4.859550561797752e-07,
      "logps/chosen": -20.754308700561523,
      "logps/rejected": -24.876815795898438,
      "loss": 0.6914,
      "losses/dpo": 0.6889626979827881,
      "losses/sft": 0.8148602843284607,
      "losses/total": 0.6889626979827881,
      "ref_logps/chosen": -20.81591796875,
      "ref_logps/rejected": -24.90121078491211,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.006161023862659931,
      "rewards/margins": 0.0037216043565422297,
      "rewards/rejected": 0.0024394195061177015,
      "step": 50
    },
    {
      "epoch": 0.38,
      "learning_rate": 4.845505617977528e-07,
      "logps/chosen": -23.585115432739258,
      "logps/rejected": -24.949783325195312,
      "loss": 0.6915,
      "losses/dpo": 0.6886854767799377,
      "losses/sft": 0.8582803010940552,
      "losses/total": 0.6886854767799377,
      "ref_logps/chosen": -23.63630485534668,
      "ref_logps/rejected": -24.965686798095703,
      "rewards/accuracies": 0.5546875,
      "rewards/chosen": 0.00511885154992342,
      "rewards/margins": 0.00352850160561502,
      "rewards/rejected": 0.0015903504099696875,
      "step": 51
    },
    {
      "epoch": 0.39,
      "learning_rate": 4.831460674157303e-07,
      "logps/chosen": -20.576318740844727,
      "logps/rejected": -24.87842559814453,
      "loss": 0.6916,
      "losses/dpo": 0.6899633407592773,
      "losses/sft": 0.6870510578155518,
      "losses/total": 0.6899633407592773,
      "ref_logps/chosen": -20.60286521911621,
      "ref_logps/rejected": -24.87299346923828,
      "rewards/accuracies": 0.5546875,
      "rewards/chosen": 0.0026545142754912376,
      "rewards/margins": 0.0031977419275790453,
      "rewards/rejected": -0.0005432275356724858,
      "step": 52
    },
    {
      "epoch": 0.4,
      "learning_rate": 4.817415730337078e-07,
      "logps/chosen": -24.051544189453125,
      "logps/rejected": -25.128353118896484,
      "loss": 0.6887,
      "losses/dpo": 0.6841185092926025,
      "losses/sft": 0.833280622959137,
      "losses/total": 0.6841185092926025,
      "ref_logps/chosen": -24.10638427734375,
      "ref_logps/rejected": -25.090627670288086,
      "rewards/accuracies": 0.6171875,
      "rewards/chosen": 0.005483907647430897,
      "rewards/margins": 0.009256447665393353,
      "rewards/rejected": -0.003772540483623743,
      "step": 53
    },
    {
      "epoch": 0.41,
      "learning_rate": 4.803370786516854e-07,
      "logps/chosen": -21.564958572387695,
      "logps/rejected": -26.20134735107422,
      "loss": 0.6914,
      "losses/dpo": 0.687272846698761,
      "losses/sft": 0.7218018770217896,
      "losses/total": 0.687272846698761,
      "ref_logps/chosen": -21.621246337890625,
      "ref_logps/rejected": -26.221445083618164,
      "rewards/accuracies": 0.546875,
      "rewards/chosen": 0.00562882237136364,
      "rewards/margins": 0.003618879709392786,
      "rewards/rejected": 0.002009942661970854,
      "step": 54
    },
    {
      "epoch": 0.42,
      "learning_rate": 4.789325842696629e-07,
      "logps/chosen": -23.699432373046875,
      "logps/rejected": -26.1567325592041,
      "loss": 0.6884,
      "losses/dpo": 0.6862033605575562,
      "losses/sft": 0.9426325559616089,
      "losses/total": 0.6862033605575562,
      "ref_logps/chosen": -23.775989532470703,
      "ref_logps/rejected": -26.134971618652344,
      "rewards/accuracies": 0.5859375,
      "rewards/chosen": 0.007655493449419737,
      "rewards/margins": 0.009831697680056095,
      "rewards/rejected": -0.0021762042306363583,
      "step": 55
    },
    {
      "epoch": 0.42,
      "learning_rate": 4.775280898876405e-07,
      "logps/chosen": -23.076374053955078,
      "logps/rejected": -27.695213317871094,
      "loss": 0.6881,
      "losses/dpo": 0.6900283098220825,
      "losses/sft": 0.8505688905715942,
      "losses/total": 0.6900283098220825,
      "ref_logps/chosen": -23.134254455566406,
      "ref_logps/rejected": -27.64853858947754,
      "rewards/accuracies": 0.6640625,
      "rewards/chosen": 0.005787987262010574,
      "rewards/margins": 0.010455346666276455,
      "rewards/rejected": -0.004667359404265881,
      "step": 56
    },
    {
      "epoch": 0.43,
      "learning_rate": 4.7612359550561797e-07,
      "logps/chosen": -21.54006576538086,
      "logps/rejected": -24.36727523803711,
      "loss": 0.6911,
      "losses/dpo": 0.6942879557609558,
      "losses/sft": 0.7311047911643982,
      "losses/total": 0.6942879557609558,
      "ref_logps/chosen": -21.592029571533203,
      "ref_logps/rejected": -24.37733268737793,
      "rewards/accuracies": 0.6015625,
      "rewards/chosen": 0.0051962630823254585,
      "rewards/margins": 0.004190489184111357,
      "rewards/rejected": 0.0010057740146294236,
      "step": 57
    },
    {
      "epoch": 0.44,
      "learning_rate": 4.747191011235955e-07,
      "logps/chosen": -21.678865432739258,
      "logps/rejected": -28.501548767089844,
      "loss": 0.6906,
      "losses/dpo": 0.6889323592185974,
      "losses/sft": 0.7590615749359131,
      "losses/total": 0.6889323592185974,
      "ref_logps/chosen": -21.72535514831543,
      "ref_logps/rejected": -28.493976593017578,
      "rewards/accuracies": 0.6328125,
      "rewards/chosen": 0.004648969508707523,
      "rewards/margins": 0.005405961070209742,
      "rewards/rejected": -0.0007569912704639137,
      "step": 58
    },
    {
      "epoch": 0.45,
      "learning_rate": 4.7331460674157303e-07,
      "logps/chosen": -23.9781436920166,
      "logps/rejected": -26.515047073364258,
      "loss": 0.684,
      "losses/dpo": 0.6820257902145386,
      "losses/sft": 0.8394409418106079,
      "losses/total": 0.6820257902145386,
      "ref_logps/chosen": -24.077434539794922,
      "ref_logps/rejected": -26.427589416503906,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.009928906336426735,
      "rewards/margins": 0.01867445930838585,
      "rewards/rejected": -0.008745552971959114,
      "step": 59
    },
    {
      "epoch": 0.45,
      "learning_rate": 4.7191011235955054e-07,
      "logps/chosen": -22.162433624267578,
      "logps/rejected": -30.391559600830078,
      "loss": 0.6894,
      "losses/dpo": 0.6909818053245544,
      "losses/sft": 0.7433596253395081,
      "losses/total": 0.6909818053245544,
      "ref_logps/chosen": -22.250009536743164,
      "ref_logps/rejected": -30.40111541748047,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.00875765923410654,
      "rewards/margins": 0.007802051026374102,
      "rewards/rejected": 0.0009556080331094563,
      "step": 60
    },
    {
      "epoch": 0.46,
      "learning_rate": 4.705056179775281e-07,
      "logps/chosen": -24.088329315185547,
      "logps/rejected": -26.851608276367188,
      "loss": 0.6888,
      "losses/dpo": 0.6858267188072205,
      "losses/sft": 0.6961312294006348,
      "losses/total": 0.6858267188072205,
      "ref_logps/chosen": -24.163042068481445,
      "ref_logps/rejected": -26.837688446044922,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.007471038028597832,
      "rewards/margins": 0.008863050490617752,
      "rewards/rejected": -0.0013920125784352422,
      "step": 61
    },
    {
      "epoch": 0.47,
      "learning_rate": 4.691011235955056e-07,
      "logps/chosen": -23.13729476928711,
      "logps/rejected": -28.607454299926758,
      "loss": 0.6896,
      "losses/dpo": 0.6952353715896606,
      "losses/sft": 0.8425909280776978,
      "losses/total": 0.6952353715896606,
      "ref_logps/chosen": -23.206546783447266,
      "ref_logps/rejected": -28.603229522705078,
      "rewards/accuracies": 0.578125,
      "rewards/chosen": 0.006925276480615139,
      "rewards/margins": 0.007347787730395794,
      "rewards/rejected": -0.00042251107515767217,
      "step": 62
    },
    {
      "epoch": 0.48,
      "learning_rate": 4.6769662921348315e-07,
      "logps/chosen": -22.758800506591797,
      "logps/rejected": -25.503629684448242,
      "loss": 0.6882,
      "losses/dpo": 0.690306544303894,
      "losses/sft": 0.7292711734771729,
      "losses/total": 0.690306544303894,
      "ref_logps/chosen": -22.867115020751953,
      "ref_logps/rejected": -25.50885009765625,
      "rewards/accuracies": 0.640625,
      "rewards/chosen": 0.010831332765519619,
      "rewards/margins": 0.010309312492609024,
      "rewards/rejected": 0.0005220210296101868,
      "step": 63
    },
    {
      "epoch": 0.48,
      "learning_rate": 4.662921348314606e-07,
      "logps/chosen": -22.957290649414062,
      "logps/rejected": -27.15595245361328,
      "loss": 0.6868,
      "losses/dpo": 0.6876275539398193,
      "losses/sft": 0.9537997245788574,
      "losses/total": 0.6876275539398193,
      "ref_logps/chosen": -23.08481788635254,
      "ref_logps/rejected": -27.15395736694336,
      "rewards/accuracies": 0.6484375,
      "rewards/chosen": 0.012752560898661613,
      "rewards/margins": 0.012952261604368687,
      "rewards/rejected": -0.0001996997743844986,
      "step": 64
    },
    {
      "epoch": 0.49,
      "learning_rate": 4.6488764044943816e-07,
      "logps/chosen": -21.856212615966797,
      "logps/rejected": -28.90016746520996,
      "loss": 0.688,
      "losses/dpo": 0.6866365075111389,
      "losses/sft": 0.748786211013794,
      "losses/total": 0.6866365075111389,
      "ref_logps/chosen": -21.946701049804688,
      "ref_logps/rejected": -28.884090423583984,
      "rewards/accuracies": 0.5546875,
      "rewards/chosen": 0.009048780426383018,
      "rewards/margins": 0.010656429454684258,
      "rewards/rejected": -0.0016076482133939862,
      "step": 65
    },
    {
      "epoch": 0.5,
      "learning_rate": 4.634831460674157e-07,
      "logps/chosen": -21.727970123291016,
      "logps/rejected": -24.484195709228516,
      "loss": 0.6866,
      "losses/dpo": 0.6858303546905518,
      "losses/sft": 0.7428255677223206,
      "losses/total": 0.6858303546905518,
      "ref_logps/chosen": -21.827533721923828,
      "ref_logps/rejected": -24.44991111755371,
      "rewards/accuracies": 0.6484375,
      "rewards/chosen": 0.009956244379281998,
      "rewards/margins": 0.013384684920310974,
      "rewards/rejected": -0.0034284412395209074,
      "step": 66
    },
    {
      "epoch": 0.51,
      "learning_rate": 4.620786516853932e-07,
      "logps/chosen": -23.145030975341797,
      "logps/rejected": -25.03292465209961,
      "loss": 0.685,
      "losses/dpo": 0.6789939403533936,
      "losses/sft": 0.718001127243042,
      "losses/total": 0.6789939403533936,
      "ref_logps/chosen": -23.27937889099121,
      "ref_logps/rejected": -24.999483108520508,
      "rewards/accuracies": 0.671875,
      "rewards/chosen": 0.013434557244181633,
      "rewards/margins": 0.016778716817498207,
      "rewards/rejected": -0.0033441600389778614,
      "step": 67
    },
    {
      "epoch": 0.51,
      "learning_rate": 4.606741573033708e-07,
      "logps/chosen": -21.208370208740234,
      "logps/rejected": -25.74646759033203,
      "loss": 0.6852,
      "losses/dpo": 0.6921157836914062,
      "losses/sft": 0.8621765971183777,
      "losses/total": 0.6921157836914062,
      "ref_logps/chosen": -21.325489044189453,
      "ref_logps/rejected": -25.700342178344727,
      "rewards/accuracies": 0.640625,
      "rewards/chosen": 0.011711984872817993,
      "rewards/margins": 0.016324326395988464,
      "rewards/rejected": -0.0046123419888317585,
      "step": 68
    },
    {
      "epoch": 0.52,
      "learning_rate": 4.592696629213483e-07,
      "logps/chosen": -22.621421813964844,
      "logps/rejected": -28.81465721130371,
      "loss": 0.6885,
      "losses/dpo": 0.689292848110199,
      "losses/sft": 0.7215853929519653,
      "losses/total": 0.689292848110199,
      "ref_logps/chosen": -22.720117568969727,
      "ref_logps/rejected": -28.81524658203125,
      "rewards/accuracies": 0.609375,
      "rewards/chosen": 0.009869576431810856,
      "rewards/margins": 0.009810445830225945,
      "rewards/rejected": 5.913013592362404e-05,
      "step": 69
    },
    {
      "epoch": 0.53,
      "learning_rate": 4.5786516853932584e-07,
      "logps/chosen": -22.636703491210938,
      "logps/rejected": -28.595046997070312,
      "loss": 0.6872,
      "losses/dpo": 0.6876038312911987,
      "losses/sft": 0.7616434097290039,
      "losses/total": 0.6876038312911987,
      "ref_logps/chosen": -22.73769187927246,
      "ref_logps/rejected": -28.57284164428711,
      "rewards/accuracies": 0.6328125,
      "rewards/chosen": 0.01009867899119854,
      "rewards/margins": 0.01231930311769247,
      "rewards/rejected": -0.002220625290647149,
      "step": 70
    },
    {
      "epoch": 0.54,
      "learning_rate": 4.5646067415730334e-07,
      "logps/chosen": -23.055517196655273,
      "logps/rejected": -28.524490356445312,
      "loss": 0.6849,
      "losses/dpo": 0.6818934082984924,
      "losses/sft": 0.8828948736190796,
      "losses/total": 0.6818934082984924,
      "ref_logps/chosen": -23.18179702758789,
      "ref_logps/rejected": -28.479928970336914,
      "rewards/accuracies": 0.6796875,
      "rewards/chosen": 0.012628016993403435,
      "rewards/margins": 0.01708414778113365,
      "rewards/rejected": -0.004456131719052792,
      "step": 71
    },
    {
      "epoch": 0.54,
      "learning_rate": 4.550561797752809e-07,
      "logps/chosen": -25.802350997924805,
      "logps/rejected": -29.403223037719727,
      "loss": 0.682,
      "losses/dpo": 0.6922101974487305,
      "losses/sft": 0.7417640089988708,
      "losses/total": 0.6922101974487305,
      "ref_logps/chosen": -25.971485137939453,
      "ref_logps/rejected": -29.342666625976562,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.016913428902626038,
      "rewards/margins": 0.02296869084239006,
      "rewards/rejected": -0.006055259145796299,
      "step": 72
    },
    {
      "epoch": 0.55,
      "learning_rate": 4.536516853932584e-07,
      "logps/chosen": -22.979541778564453,
      "logps/rejected": -31.861392974853516,
      "loss": 0.6849,
      "losses/dpo": 0.6843876242637634,
      "losses/sft": 0.6335030198097229,
      "losses/total": 0.6843876242637634,
      "ref_logps/chosen": -23.086105346679688,
      "ref_logps/rejected": -31.796035766601562,
      "rewards/accuracies": 0.609375,
      "rewards/chosen": 0.010656386613845825,
      "rewards/margins": 0.017192194238305092,
      "rewards/rejected": -0.006535808090120554,
      "step": 73
    },
    {
      "epoch": 0.56,
      "learning_rate": 4.522471910112359e-07,
      "logps/chosen": -21.333240509033203,
      "logps/rejected": -25.32451629638672,
      "loss": 0.6842,
      "losses/dpo": 0.6832489967346191,
      "losses/sft": 0.8737274408340454,
      "losses/total": 0.6832489967346191,
      "ref_logps/chosen": -21.456268310546875,
      "ref_logps/rejected": -25.263538360595703,
      "rewards/accuracies": 0.703125,
      "rewards/chosen": 0.012302841059863567,
      "rewards/margins": 0.01840106211602688,
      "rewards/rejected": -0.006098220124840736,
      "step": 74
    },
    {
      "epoch": 0.57,
      "learning_rate": 4.5084269662921347e-07,
      "logps/chosen": -21.905548095703125,
      "logps/rejected": -25.504837036132812,
      "loss": 0.6845,
      "losses/dpo": 0.6803750991821289,
      "losses/sft": 0.7227590084075928,
      "losses/total": 0.6803750991821289,
      "ref_logps/chosen": -22.001012802124023,
      "ref_logps/rejected": -25.422731399536133,
      "rewards/accuracies": 0.6953125,
      "rewards/chosen": 0.009546317160129547,
      "rewards/margins": 0.01775689423084259,
      "rewards/rejected": -0.008210576139390469,
      "step": 75
    },
    {
      "epoch": 0.57,
      "learning_rate": 4.4943820224719097e-07,
      "logps/chosen": -22.212453842163086,
      "logps/rejected": -25.56966209411621,
      "loss": 0.6845,
      "losses/dpo": 0.6879241466522217,
      "losses/sft": 0.936349093914032,
      "losses/total": 0.6879241466522217,
      "ref_logps/chosen": -22.337223052978516,
      "ref_logps/rejected": -25.51331901550293,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.012477071955800056,
      "rewards/margins": 0.018111376091837883,
      "rewards/rejected": -0.005634305067360401,
      "step": 76
    },
    {
      "epoch": 0.58,
      "learning_rate": 4.4803370786516853e-07,
      "logps/chosen": -20.199138641357422,
      "logps/rejected": -26.30996322631836,
      "loss": 0.6818,
      "losses/dpo": 0.6872521638870239,
      "losses/sft": 0.6872013211250305,
      "losses/total": 0.6872521638870239,
      "ref_logps/chosen": -20.368690490722656,
      "ref_logps/rejected": -26.24540138244629,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.016955075785517693,
      "rewards/margins": 0.02341129444539547,
      "rewards/rejected": -0.006456219125539064,
      "step": 77
    },
    {
      "epoch": 0.59,
      "learning_rate": 4.4662921348314603e-07,
      "logps/chosen": -22.031774520874023,
      "logps/rejected": -26.07961082458496,
      "loss": 0.6813,
      "losses/dpo": 0.6833238005638123,
      "losses/sft": 0.7775546312332153,
      "losses/total": 0.6833238005638123,
      "ref_logps/chosen": -22.163204193115234,
      "ref_logps/rejected": -25.965608596801758,
      "rewards/accuracies": 0.7109375,
      "rewards/chosen": 0.01314287818968296,
      "rewards/margins": 0.02454320341348648,
      "rewards/rejected": -0.011400324292480946,
      "step": 78
    },
    {
      "epoch": 0.6,
      "learning_rate": 4.452247191011236e-07,
      "logps/chosen": -22.522083282470703,
      "logps/rejected": -26.621906280517578,
      "loss": 0.6801,
      "losses/dpo": 0.6835525035858154,
      "losses/sft": 0.7558909058570862,
      "losses/total": 0.6835525035858154,
      "ref_logps/chosen": -22.656497955322266,
      "ref_logps/rejected": -26.48514747619629,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.013441269285976887,
      "rewards/margins": 0.027117114514112473,
      "rewards/rejected": -0.013675847090780735,
      "step": 79
    },
    {
      "epoch": 0.6,
      "learning_rate": 4.438202247191011e-07,
      "logps/chosen": -22.05775260925293,
      "logps/rejected": -26.428781509399414,
      "loss": 0.6836,
      "losses/dpo": 0.6767468452453613,
      "losses/sft": 0.8101401329040527,
      "losses/total": 0.6767468452453613,
      "ref_logps/chosen": -22.211511611938477,
      "ref_logps/rejected": -26.38385772705078,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": 0.015375516377389431,
      "rewards/margins": 0.019868001341819763,
      "rewards/rejected": -0.004492484033107758,
      "step": 80
    },
    {
      "epoch": 0.61,
      "learning_rate": 4.4241573033707865e-07,
      "logps/chosen": -22.327136993408203,
      "logps/rejected": -27.90719985961914,
      "loss": 0.6803,
      "losses/dpo": 0.6830211281776428,
      "losses/sft": 0.7352213263511658,
      "losses/total": 0.6830211281776428,
      "ref_logps/chosen": -22.457595825195312,
      "ref_logps/rejected": -27.77078628540039,
      "rewards/accuracies": 0.6484375,
      "rewards/chosen": 0.013045946136116982,
      "rewards/margins": 0.026687312871217728,
      "rewards/rejected": -0.013641366735100746,
      "step": 81
    },
    {
      "epoch": 0.62,
      "learning_rate": 4.410112359550562e-07,
      "logps/chosen": -23.738140106201172,
      "logps/rejected": -26.4810791015625,
      "loss": 0.6802,
      "losses/dpo": 0.673937976360321,
      "losses/sft": 0.7962872385978699,
      "losses/total": 0.673937976360321,
      "ref_logps/chosen": -23.89459991455078,
      "ref_logps/rejected": -26.366804122924805,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": 0.015645721927285194,
      "rewards/margins": 0.027073292061686516,
      "rewards/rejected": -0.011427570134401321,
      "step": 82
    },
    {
      "epoch": 0.63,
      "learning_rate": 4.3960674157303366e-07,
      "logps/chosen": -21.008014678955078,
      "logps/rejected": -24.34069061279297,
      "loss": 0.6805,
      "losses/dpo": 0.6789628863334656,
      "losses/sft": 0.9124815464019775,
      "losses/total": 0.6789628863334656,
      "ref_logps/chosen": -21.115734100341797,
      "ref_logps/rejected": -24.184371948242188,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": 0.01077171228826046,
      "rewards/margins": 0.026403725147247314,
      "rewards/rejected": -0.015632012858986855,
      "step": 83
    },
    {
      "epoch": 0.63,
      "learning_rate": 4.382022471910112e-07,
      "logps/chosen": -20.62143325805664,
      "logps/rejected": -26.963245391845703,
      "loss": 0.6833,
      "losses/dpo": 0.6907744407653809,
      "losses/sft": 0.7639827728271484,
      "losses/total": 0.6907744407653809,
      "ref_logps/chosen": -20.739776611328125,
      "ref_logps/rejected": -26.87374496459961,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.011834252625703812,
      "rewards/margins": 0.020784219726920128,
      "rewards/rejected": -0.008949968963861465,
      "step": 84
    },
    {
      "epoch": 0.64,
      "learning_rate": 4.367977528089887e-07,
      "logps/chosen": -21.591964721679688,
      "logps/rejected": -24.5494384765625,
      "loss": 0.6846,
      "losses/dpo": 0.6878204345703125,
      "losses/sft": 0.6917088627815247,
      "losses/total": 0.6878204345703125,
      "ref_logps/chosen": -21.644916534423828,
      "ref_logps/rejected": -24.421239852905273,
      "rewards/accuracies": 0.6328125,
      "rewards/chosen": 0.005295174196362495,
      "rewards/margins": 0.01811503805220127,
      "rewards/rejected": -0.012819863855838776,
      "step": 85
    },
    {
      "epoch": 0.65,
      "learning_rate": 4.353932584269663e-07,
      "logps/chosen": -24.759811401367188,
      "logps/rejected": -28.227123260498047,
      "loss": 0.6825,
      "losses/dpo": 0.6937445402145386,
      "losses/sft": 0.9424384832382202,
      "losses/total": 0.6937445402145386,
      "ref_logps/chosen": -24.891460418701172,
      "ref_logps/rejected": -28.136310577392578,
      "rewards/accuracies": 0.6484375,
      "rewards/chosen": 0.013164759613573551,
      "rewards/margins": 0.022245781496167183,
      "rewards/rejected": -0.009081022813916206,
      "step": 86
    },
    {
      "epoch": 0.66,
      "learning_rate": 4.339887640449438e-07,
      "logps/chosen": -22.8006591796875,
      "logps/rejected": -26.10009002685547,
      "loss": 0.6795,
      "losses/dpo": 0.6909404993057251,
      "losses/sft": 0.8603497743606567,
      "losses/total": 0.6909404993057251,
      "ref_logps/chosen": -22.96673583984375,
      "ref_logps/rejected": -25.977882385253906,
      "rewards/accuracies": 0.7109375,
      "rewards/chosen": 0.01660749316215515,
      "rewards/margins": 0.028828214854002,
      "rewards/rejected": -0.012220719829201698,
      "step": 87
    },
    {
      "epoch": 0.66,
      "learning_rate": 4.3258426966292134e-07,
      "logps/chosen": -24.15732765197754,
      "logps/rejected": -28.13039779663086,
      "loss": 0.6752,
      "losses/dpo": 0.6638558506965637,
      "losses/sft": 0.8455443382263184,
      "losses/total": 0.6638558506965637,
      "ref_logps/chosen": -24.341087341308594,
      "ref_logps/rejected": -27.938106536865234,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": 0.018375899642705917,
      "rewards/margins": 0.03760489821434021,
      "rewards/rejected": -0.019228998571634293,
      "step": 88
    },
    {
      "epoch": 0.67,
      "learning_rate": 4.311797752808989e-07,
      "logps/chosen": -21.290430068969727,
      "logps/rejected": -25.207626342773438,
      "loss": 0.6771,
      "losses/dpo": 0.6774411797523499,
      "losses/sft": 0.9257520437240601,
      "losses/total": 0.6774411797523499,
      "ref_logps/chosen": -21.449438095092773,
      "ref_logps/rejected": -25.032873153686523,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": 0.015900880098342896,
      "rewards/margins": 0.033375710248947144,
      "rewards/rejected": -0.017474830150604248,
      "step": 89
    },
    {
      "epoch": 0.68,
      "learning_rate": 4.297752808988764e-07,
      "logps/chosen": -24.241390228271484,
      "logps/rejected": -27.57483673095703,
      "loss": 0.681,
      "losses/dpo": 0.6869298219680786,
      "losses/sft": 0.8004887104034424,
      "losses/total": 0.6869298219680786,
      "ref_logps/chosen": -24.399887084960938,
      "ref_logps/rejected": -27.475460052490234,
      "rewards/accuracies": 0.640625,
      "rewards/chosen": 0.015849877148866653,
      "rewards/margins": 0.02578754723072052,
      "rewards/rejected": -0.009937671013176441,
      "step": 90
    },
    {
      "epoch": 0.69,
      "learning_rate": 4.2837078651685396e-07,
      "logps/chosen": -21.290605545043945,
      "logps/rejected": -25.188884735107422,
      "loss": 0.6843,
      "losses/dpo": 0.6896719336509705,
      "losses/sft": 0.7865870594978333,
      "losses/total": 0.6896719336509705,
      "ref_logps/chosen": -21.39483642578125,
      "ref_logps/rejected": -25.098407745361328,
      "rewards/accuracies": 0.6328125,
      "rewards/chosen": 0.010422902181744576,
      "rewards/margins": 0.01947084441781044,
      "rewards/rejected": -0.00904794316738844,
      "step": 91
    },
    {
      "epoch": 0.69,
      "learning_rate": 4.269662921348314e-07,
      "logps/chosen": -21.395389556884766,
      "logps/rejected": -24.005056381225586,
      "loss": 0.6863,
      "losses/dpo": 0.6820717453956604,
      "losses/sft": 0.8161361813545227,
      "losses/total": 0.6820717453956604,
      "ref_logps/chosen": -21.495037078857422,
      "ref_logps/rejected": -23.9505558013916,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": 0.009964808821678162,
      "rewards/margins": 0.015414956025779247,
      "rewards/rejected": -0.00545014813542366,
      "step": 92
    },
    {
      "epoch": 0.7,
      "learning_rate": 4.2556179775280896e-07,
      "logps/chosen": -20.948806762695312,
      "logps/rejected": -24.735366821289062,
      "loss": 0.6786,
      "losses/dpo": 0.6868577599525452,
      "losses/sft": 0.7177249193191528,
      "losses/total": 0.6868577599525452,
      "ref_logps/chosen": -21.072193145751953,
      "ref_logps/rejected": -24.555286407470703,
      "rewards/accuracies": 0.6953125,
      "rewards/chosen": 0.012338603846728802,
      "rewards/margins": 0.030346699059009552,
      "rewards/rejected": -0.018008096143603325,
      "step": 93
    },
    {
      "epoch": 0.71,
      "learning_rate": 4.2415730337078647e-07,
      "logps/chosen": -24.245830535888672,
      "logps/rejected": -28.811023712158203,
      "loss": 0.6783,
      "losses/dpo": 0.6721839904785156,
      "losses/sft": 0.816402018070221,
      "losses/total": 0.6721839904785156,
      "ref_logps/chosen": -24.40906524658203,
      "ref_logps/rejected": -28.65966033935547,
      "rewards/accuracies": 0.703125,
      "rewards/chosen": 0.01632346771657467,
      "rewards/margins": 0.03146028146147728,
      "rewards/rejected": -0.01513681747019291,
      "step": 94
    },
    {
      "epoch": 0.72,
      "learning_rate": 4.22752808988764e-07,
      "logps/chosen": -22.48372459411621,
      "logps/rejected": -29.088359832763672,
      "loss": 0.6709,
      "losses/dpo": 0.6718644499778748,
      "losses/sft": 0.823063313961029,
      "losses/total": 0.6718644499778748,
      "ref_logps/chosen": -22.634136199951172,
      "ref_logps/rejected": -28.77379608154297,
      "rewards/accuracies": 0.7109375,
      "rewards/chosen": 0.01504128985106945,
      "rewards/margins": 0.04649777710437775,
      "rewards/rejected": -0.03145648539066315,
      "step": 95
    },
    {
      "epoch": 0.72,
      "learning_rate": 4.2134831460674153e-07,
      "logps/chosen": -20.869436264038086,
      "logps/rejected": -27.790451049804688,
      "loss": 0.6785,
      "losses/dpo": 0.6842025518417358,
      "losses/sft": 0.8330531120300293,
      "losses/total": 0.6842025518417358,
      "ref_logps/chosen": -20.964067459106445,
      "ref_logps/rejected": -27.572711944580078,
      "rewards/accuracies": 0.6484375,
      "rewards/chosen": 0.009463240392506123,
      "rewards/margins": 0.031237438321113586,
      "rewards/rejected": -0.02177419885993004,
      "step": 96
    },
    {
      "epoch": 0.73,
      "learning_rate": 4.199438202247191e-07,
      "logps/chosen": -22.02164077758789,
      "logps/rejected": -28.644880294799805,
      "loss": 0.6772,
      "losses/dpo": 0.6933009028434753,
      "losses/sft": 0.7342395186424255,
      "losses/total": 0.6933009028434753,
      "ref_logps/chosen": -22.146129608154297,
      "ref_logps/rejected": -28.42925262451172,
      "rewards/accuracies": 0.640625,
      "rewards/chosen": 0.012448801659047604,
      "rewards/margins": 0.03401148319244385,
      "rewards/rejected": -0.02156267873942852,
      "step": 97
    },
    {
      "epoch": 0.74,
      "learning_rate": 4.1853932584269664e-07,
      "logps/chosen": -21.086360931396484,
      "logps/rejected": -23.74181365966797,
      "loss": 0.6834,
      "losses/dpo": 0.7061095833778381,
      "losses/sft": 0.6976662278175354,
      "losses/total": 0.7061095833778381,
      "ref_logps/chosen": -21.240116119384766,
      "ref_logps/rejected": -23.683523178100586,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.015375564806163311,
      "rewards/margins": 0.021204624325037003,
      "rewards/rejected": -0.0058290609158575535,
      "step": 98
    },
    {
      "epoch": 0.75,
      "learning_rate": 4.1713483146067415e-07,
      "logps/chosen": -21.535640716552734,
      "logps/rejected": -28.555763244628906,
      "loss": 0.6749,
      "losses/dpo": 0.6546899080276489,
      "losses/sft": 0.8132616281509399,
      "losses/total": 0.6546899080276489,
      "ref_logps/chosen": -21.68370819091797,
      "ref_logps/rejected": -28.313589096069336,
      "rewards/accuracies": 0.6640625,
      "rewards/chosen": 0.014806646853685379,
      "rewards/margins": 0.03902393952012062,
      "rewards/rejected": -0.02421729266643524,
      "step": 99
    },
    {
      "epoch": 0.75,
      "learning_rate": 4.157303370786517e-07,
      "logps/chosen": -22.314010620117188,
      "logps/rejected": -26.403512954711914,
      "loss": 0.6777,
      "losses/dpo": 0.6830233931541443,
      "losses/sft": 0.7298552393913269,
      "losses/total": 0.6830233931541443,
      "ref_logps/chosen": -22.442527770996094,
      "ref_logps/rejected": -26.1983699798584,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.012851729989051819,
      "rewards/margins": 0.033365827053785324,
      "rewards/rejected": -0.020514097064733505,
      "step": 100
    },
    {
      "epoch": 0.76,
      "learning_rate": 4.1432584269662915e-07,
      "logps/chosen": -23.65606117248535,
      "logps/rejected": -27.6639461517334,
      "loss": 0.6787,
      "losses/dpo": 0.66861492395401,
      "losses/sft": 0.7538549900054932,
      "losses/total": 0.66861492395401,
      "ref_logps/chosen": -23.742881774902344,
      "ref_logps/rejected": -27.43739128112793,
      "rewards/accuracies": 0.6328125,
      "rewards/chosen": 0.008682135492563248,
      "rewards/margins": 0.03133738413453102,
      "rewards/rejected": -0.022655250504612923,
      "step": 101
    },
    {
      "epoch": 0.77,
      "learning_rate": 4.129213483146067e-07,
      "logps/chosen": -21.20174789428711,
      "logps/rejected": -27.045516967773438,
      "loss": 0.6736,
      "losses/dpo": 0.6594799757003784,
      "losses/sft": 0.7625675201416016,
      "losses/total": 0.6594799757003784,
      "ref_logps/chosen": -21.360929489135742,
      "ref_logps/rejected": -26.788671493530273,
      "rewards/accuracies": 0.6640625,
      "rewards/chosen": 0.015918483957648277,
      "rewards/margins": 0.04160304740071297,
      "rewards/rejected": -0.02568456158041954,
      "step": 102
    },
    {
      "epoch": 0.78,
      "learning_rate": 4.115168539325842e-07,
      "logps/chosen": -25.287567138671875,
      "logps/rejected": -27.158187866210938,
      "loss": 0.6789,
      "losses/dpo": 0.6871756315231323,
      "losses/sft": 0.7897288799285889,
      "losses/total": 0.6871756315231323,
      "ref_logps/chosen": -25.39737319946289,
      "ref_logps/rejected": -26.95665740966797,
      "rewards/accuracies": 0.6015625,
      "rewards/chosen": 0.010980643332004547,
      "rewards/margins": 0.03113364614546299,
      "rewards/rejected": -0.020153000950813293,
      "step": 103
    },
    {
      "epoch": 0.78,
      "learning_rate": 4.1011235955056177e-07,
      "logps/chosen": -20.239051818847656,
      "logps/rejected": -27.055557250976562,
      "loss": 0.6766,
      "losses/dpo": 0.6560062170028687,
      "losses/sft": 0.7211654186248779,
      "losses/total": 0.6560062170028687,
      "ref_logps/chosen": -20.345287322998047,
      "ref_logps/rejected": -26.804546356201172,
      "rewards/accuracies": 0.640625,
      "rewards/chosen": 0.010623706504702568,
      "rewards/margins": 0.03572461009025574,
      "rewards/rejected": -0.02510090172290802,
      "step": 104
    },
    {
      "epoch": 0.79,
      "learning_rate": 4.0870786516853933e-07,
      "logps/chosen": -22.816429138183594,
      "logps/rejected": -28.331439971923828,
      "loss": 0.6728,
      "losses/dpo": 0.6975245475769043,
      "losses/sft": 0.8287545442581177,
      "losses/total": 0.6975245475769043,
      "ref_logps/chosen": -22.96261215209961,
      "ref_logps/rejected": -28.04006576538086,
      "rewards/accuracies": 0.7109375,
      "rewards/chosen": 0.014618270099163055,
      "rewards/margins": 0.043755702674388885,
      "rewards/rejected": -0.02913743630051613,
      "step": 105
    },
    {
      "epoch": 0.8,
      "learning_rate": 4.0730337078651683e-07,
      "logps/chosen": -22.864845275878906,
      "logps/rejected": -27.868162155151367,
      "loss": 0.6776,
      "losses/dpo": 0.6524635553359985,
      "losses/sft": 0.8967273235321045,
      "losses/total": 0.6524635553359985,
      "ref_logps/chosen": -22.934465408325195,
      "ref_logps/rejected": -27.60092544555664,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": 0.006961943581700325,
      "rewards/margins": 0.03368568420410156,
      "rewards/rejected": -0.026723740622401237,
      "step": 106
    },
    {
      "epoch": 0.81,
      "learning_rate": 4.058988764044944e-07,
      "logps/chosen": -26.633420944213867,
      "logps/rejected": -29.40836524963379,
      "loss": 0.6785,
      "losses/dpo": 0.6883168816566467,
      "losses/sft": 0.9007142782211304,
      "losses/total": 0.6883168816566467,
      "ref_logps/chosen": -26.658733367919922,
      "ref_logps/rejected": -29.11638641357422,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.0025312139187008142,
      "rewards/margins": 0.031729087233543396,
      "rewards/rejected": -0.029197873547673225,
      "step": 107
    },
    {
      "epoch": 0.82,
      "learning_rate": 4.044943820224719e-07,
      "logps/chosen": -21.93716049194336,
      "logps/rejected": -26.78734016418457,
      "loss": 0.6678,
      "losses/dpo": 0.6620572805404663,
      "losses/sft": 0.7277075052261353,
      "losses/total": 0.6620572805404663,
      "ref_logps/chosen": -22.14274787902832,
      "ref_logps/rejected": -26.450454711914062,
      "rewards/accuracies": 0.7265625,
      "rewards/chosen": 0.02055862732231617,
      "rewards/margins": 0.05424723029136658,
      "rewards/rejected": -0.03368859738111496,
      "step": 108
    },
    {
      "epoch": 0.82,
      "learning_rate": 4.0308988764044945e-07,
      "logps/chosen": -23.479236602783203,
      "logps/rejected": -25.321468353271484,
      "loss": 0.6732,
      "losses/dpo": 0.6536136865615845,
      "losses/sft": 0.793202817440033,
      "losses/total": 0.6536136865615845,
      "ref_logps/chosen": -23.628402709960938,
      "ref_logps/rejected": -25.03476905822754,
      "rewards/accuracies": 0.6171875,
      "rewards/chosen": 0.014916517771780491,
      "rewards/margins": 0.043586596846580505,
      "rewards/rejected": -0.02867007628083229,
      "step": 109
    },
    {
      "epoch": 0.83,
      "learning_rate": 4.0168539325842696e-07,
      "logps/chosen": -21.36187744140625,
      "logps/rejected": -26.808046340942383,
      "loss": 0.6677,
      "losses/dpo": 0.658541202545166,
      "losses/sft": 0.6240718364715576,
      "losses/total": 0.658541202545166,
      "ref_logps/chosen": -21.525625228881836,
      "ref_logps/rejected": -26.417198181152344,
      "rewards/accuracies": 0.640625,
      "rewards/chosen": 0.01637459173798561,
      "rewards/margins": 0.05545924976468086,
      "rewards/rejected": -0.03908466175198555,
      "step": 110
    },
    {
      "epoch": 0.84,
      "learning_rate": 4.0028089887640446e-07,
      "logps/chosen": -22.143728256225586,
      "logps/rejected": -26.035858154296875,
      "loss": 0.6732,
      "losses/dpo": 0.6707695126533508,
      "losses/sft": 0.8353971838951111,
      "losses/total": 0.6707695126533508,
      "ref_logps/chosen": -22.292274475097656,
      "ref_logps/rejected": -25.744632720947266,
      "rewards/accuracies": 0.6484375,
      "rewards/chosen": 0.014854478649795055,
      "rewards/margins": 0.043976958841085434,
      "rewards/rejected": -0.029122481122612953,
      "step": 111
    },
    {
      "epoch": 0.85,
      "learning_rate": 3.9887640449438196e-07,
      "logps/chosen": -22.15041732788086,
      "logps/rejected": -24.53826332092285,
      "loss": 0.6688,
      "losses/dpo": 0.6656994819641113,
      "losses/sft": 0.8727293014526367,
      "losses/total": 0.6656994819641113,
      "ref_logps/chosen": -22.210494995117188,
      "ref_logps/rejected": -24.07231330871582,
      "rewards/accuracies": 0.7109375,
      "rewards/chosen": 0.0060077933594584465,
      "rewards/margins": 0.05260289087891579,
      "rewards/rejected": -0.046595096588134766,
      "step": 112
    },
    {
      "epoch": 0.85,
      "learning_rate": 3.974719101123595e-07,
      "logps/chosen": -23.314592361450195,
      "logps/rejected": -27.797752380371094,
      "loss": 0.675,
      "losses/dpo": 0.6690158247947693,
      "losses/sft": 0.7370929718017578,
      "losses/total": 0.6690158247947693,
      "ref_logps/chosen": -23.396080017089844,
      "ref_logps/rejected": -27.483016967773438,
      "rewards/accuracies": 0.609375,
      "rewards/chosen": 0.008148876950144768,
      "rewards/margins": 0.03962232545018196,
      "rewards/rejected": -0.031473446637392044,
      "step": 113
    },
    {
      "epoch": 0.86,
      "learning_rate": 3.960674157303371e-07,
      "logps/chosen": -21.854373931884766,
      "logps/rejected": -26.652328491210938,
      "loss": 0.6706,
      "losses/dpo": 0.645140528678894,
      "losses/sft": 0.77164226770401,
      "losses/total": 0.645140528678894,
      "ref_logps/chosen": -21.949893951416016,
      "ref_logps/rejected": -26.255746841430664,
      "rewards/accuracies": 0.6484375,
      "rewards/chosen": 0.009551877155900002,
      "rewards/margins": 0.0492100827395916,
      "rewards/rejected": -0.03965820372104645,
      "step": 114
    },
    {
      "epoch": 0.87,
      "learning_rate": 3.946629213483146e-07,
      "logps/chosen": -23.778413772583008,
      "logps/rejected": -28.40381622314453,
      "loss": 0.6634,
      "losses/dpo": 0.6699668169021606,
      "losses/sft": 0.8002771139144897,
      "losses/total": 0.6699668169021606,
      "ref_logps/chosen": -23.903501510620117,
      "ref_logps/rejected": -27.89557647705078,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 0.0125090591609478,
      "rewards/margins": 0.06333282589912415,
      "rewards/rejected": -0.050823770463466644,
      "step": 115
    },
    {
      "epoch": 0.88,
      "learning_rate": 3.9325842696629214e-07,
      "logps/chosen": -24.59353256225586,
      "logps/rejected": -28.184139251708984,
      "loss": 0.6658,
      "losses/dpo": 0.6745936870574951,
      "losses/sft": 0.8017398715019226,
      "losses/total": 0.6745936870574951,
      "ref_logps/chosen": -24.77825164794922,
      "ref_logps/rejected": -27.77378273010254,
      "rewards/accuracies": 0.671875,
      "rewards/chosen": 0.018472209572792053,
      "rewards/margins": 0.059507861733436584,
      "rewards/rejected": -0.04103565216064453,
      "step": 116
    },
    {
      "epoch": 0.88,
      "learning_rate": 3.9185393258426964e-07,
      "logps/chosen": -20.781490325927734,
      "logps/rejected": -25.704240798950195,
      "loss": 0.6641,
      "losses/dpo": 0.6748782396316528,
      "losses/sft": 0.6509857177734375,
      "losses/total": 0.6748782396316528,
      "ref_logps/chosen": -20.93104362487793,
      "ref_logps/rejected": -25.223262786865234,
      "rewards/accuracies": 0.703125,
      "rewards/chosen": 0.014955190010368824,
      "rewards/margins": 0.06305292248725891,
      "rewards/rejected": -0.04809773340821266,
      "step": 117
    },
    {
      "epoch": 0.89,
      "learning_rate": 3.904494382022472e-07,
      "logps/chosen": -22.889171600341797,
      "logps/rejected": -28.954145431518555,
      "loss": 0.6719,
      "losses/dpo": 0.6790695190429688,
      "losses/sft": 0.7899962663650513,
      "losses/total": 0.6790695190429688,
      "ref_logps/chosen": -22.998294830322266,
      "ref_logps/rejected": -28.596576690673828,
      "rewards/accuracies": 0.640625,
      "rewards/chosen": 0.010911967605352402,
      "rewards/margins": 0.04666893184185028,
      "rewards/rejected": -0.03575696796178818,
      "step": 118
    },
    {
      "epoch": 0.9,
      "learning_rate": 3.890449438202247e-07,
      "logps/chosen": -22.229143142700195,
      "logps/rejected": -24.892658233642578,
      "loss": 0.6713,
      "losses/dpo": 0.6665077209472656,
      "losses/sft": 0.8753491044044495,
      "losses/total": 0.6665077209472656,
      "ref_logps/chosen": -22.402416229248047,
      "ref_logps/rejected": -24.568809509277344,
      "rewards/accuracies": 0.6484375,
      "rewards/chosen": 0.01732712611556053,
      "rewards/margins": 0.0497119314968586,
      "rewards/rejected": -0.032384805381298065,
      "step": 119
    },
    {
      "epoch": 0.91,
      "learning_rate": 3.876404494382022e-07,
      "logps/chosen": -22.233783721923828,
      "logps/rejected": -29.53872299194336,
      "loss": 0.6637,
      "losses/dpo": 0.6545946002006531,
      "losses/sft": 0.8056938052177429,
      "losses/total": 0.6545946002006531,
      "ref_logps/chosen": -22.328821182250977,
      "ref_logps/rejected": -28.996824264526367,
      "rewards/accuracies": 0.7265625,
      "rewards/chosen": 0.009503833949565887,
      "rewards/margins": 0.06369376927614212,
      "rewards/rejected": -0.05418993532657623,
      "step": 120
    },
    {
      "epoch": 0.91,
      "learning_rate": 3.8623595505617977e-07,
      "logps/chosen": -24.073867797851562,
      "logps/rejected": -27.632476806640625,
      "loss": 0.6778,
      "losses/dpo": 0.6500009298324585,
      "losses/sft": 0.9210071563720703,
      "losses/total": 0.6500009298324585,
      "ref_logps/chosen": -24.12955093383789,
      "ref_logps/rejected": -27.32662582397461,
      "rewards/accuracies": 0.5859375,
      "rewards/chosen": 0.00556858628988266,
      "rewards/margins": 0.03615354374051094,
      "rewards/rejected": -0.03058495745062828,
      "step": 121
    },
    {
      "epoch": 0.92,
      "learning_rate": 3.8483146067415727e-07,
      "logps/chosen": -21.38442039489746,
      "logps/rejected": -31.358665466308594,
      "loss": 0.6601,
      "losses/dpo": 0.6630659103393555,
      "losses/sft": 0.8758641481399536,
      "losses/total": 0.6630659103393555,
      "ref_logps/chosen": -21.540292739868164,
      "ref_logps/rejected": -30.79846954345703,
      "rewards/accuracies": 0.703125,
      "rewards/chosen": 0.01558714546263218,
      "rewards/margins": 0.07160677015781403,
      "rewards/rejected": -0.056019626557826996,
      "step": 122
    },
    {
      "epoch": 0.93,
      "learning_rate": 3.834269662921348e-07,
      "logps/chosen": -21.09262466430664,
      "logps/rejected": -25.64166831970215,
      "loss": 0.6622,
      "losses/dpo": 0.6400080919265747,
      "losses/sft": 0.8849148750305176,
      "losses/total": 0.6400080919265747,
      "ref_logps/chosen": -21.179445266723633,
      "ref_logps/rejected": -25.056869506835938,
      "rewards/accuracies": 0.6796875,
      "rewards/chosen": 0.00868179090321064,
      "rewards/margins": 0.06716156005859375,
      "rewards/rejected": -0.05847976729273796,
      "step": 123
    },
    {
      "epoch": 0.94,
      "learning_rate": 3.8202247191011233e-07,
      "logps/chosen": -25.65859603881836,
      "logps/rejected": -28.025104522705078,
      "loss": 0.6765,
      "losses/dpo": 0.6927012205123901,
      "losses/sft": 0.8673559427261353,
      "losses/total": 0.6927012205123901,
      "ref_logps/chosen": -25.61608123779297,
      "ref_logps/rejected": -27.600624084472656,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.004251426085829735,
      "rewards/margins": 0.038196537643671036,
      "rewards/rejected": -0.04244796186685562,
      "step": 124
    },
    {
      "epoch": 0.94,
      "learning_rate": 3.806179775280899e-07,
      "logps/chosen": -23.93341636657715,
      "logps/rejected": -29.840375900268555,
      "loss": 0.6647,
      "losses/dpo": 0.7150436639785767,
      "losses/sft": 0.9468034505844116,
      "losses/total": 0.7150436639785767,
      "ref_logps/chosen": -23.979652404785156,
      "ref_logps/rejected": -29.25320816040039,
      "rewards/accuracies": 0.703125,
      "rewards/chosen": 0.004623853601515293,
      "rewards/margins": 0.06334076821804047,
      "rewards/rejected": -0.058716922998428345,
      "step": 125
    },
    {
      "epoch": 0.95,
      "learning_rate": 3.792134831460674e-07,
      "logps/chosen": -25.031259536743164,
      "logps/rejected": -28.292198181152344,
      "loss": 0.6559,
      "losses/dpo": 0.6770719289779663,
      "losses/sft": 0.9255229234695435,
      "losses/total": 0.6770719289779663,
      "ref_logps/chosen": -25.22754669189453,
      "ref_logps/rejected": -27.667905807495117,
      "rewards/accuracies": 0.7265625,
      "rewards/chosen": 0.019628863781690598,
      "rewards/margins": 0.08205802738666534,
      "rewards/rejected": -0.06242916360497475,
      "step": 126
    },
    {
      "epoch": 0.96,
      "learning_rate": 3.7780898876404495e-07,
      "logps/chosen": -21.68558692932129,
      "logps/rejected": -26.84676742553711,
      "loss": 0.6765,
      "losses/dpo": 0.635480523109436,
      "losses/sft": 0.7413178086280823,
      "losses/total": 0.635480523109436,
      "ref_logps/chosen": -21.638694763183594,
      "ref_logps/rejected": -26.392860412597656,
      "rewards/accuracies": 0.5390625,
      "rewards/chosen": -0.004689330700784922,
      "rewards/margins": 0.04070135951042175,
      "rewards/rejected": -0.04539068788290024,
      "step": 127
    },
    {
      "epoch": 0.97,
      "learning_rate": 3.7640449438202245e-07,
      "logps/chosen": -22.910152435302734,
      "logps/rejected": -26.53976821899414,
      "loss": 0.6587,
      "losses/dpo": 0.6835530400276184,
      "losses/sft": 0.9732310771942139,
      "losses/total": 0.6835530400276184,
      "ref_logps/chosen": -23.018016815185547,
      "ref_logps/rejected": -25.88375473022461,
      "rewards/accuracies": 0.703125,
      "rewards/chosen": 0.010786494240164757,
      "rewards/margins": 0.07638738304376602,
      "rewards/rejected": -0.06560088694095612,
      "step": 128
    },
    {
      "epoch": 0.97,
      "learning_rate": 3.75e-07,
      "logps/chosen": -23.20888900756836,
      "logps/rejected": -26.875211715698242,
      "loss": 0.6617,
      "losses/dpo": 0.6463422775268555,
      "losses/sft": 0.7454620599746704,
      "losses/total": 0.6463422775268555,
      "ref_logps/chosen": -23.336442947387695,
      "ref_logps/rejected": -26.300058364868164,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.012755412608385086,
      "rewards/margins": 0.0702708438038826,
      "rewards/rejected": -0.057515427470207214,
      "step": 129
    },
    {
      "epoch": 0.98,
      "learning_rate": 3.735955056179775e-07,
      "logps/chosen": -22.396747589111328,
      "logps/rejected": -29.472164154052734,
      "loss": 0.6784,
      "losses/dpo": 0.6625787019729614,
      "losses/sft": 0.7854889631271362,
      "losses/total": 0.6625787019729614,
      "ref_logps/chosen": -22.277332305908203,
      "ref_logps/rejected": -28.998043060302734,
      "rewards/accuracies": 0.6328125,
      "rewards/chosen": -0.01194157637655735,
      "rewards/margins": 0.03547064587473869,
      "rewards/rejected": -0.047412216663360596,
      "step": 130
    },
    {
      "epoch": 0.99,
      "learning_rate": 3.72191011235955e-07,
      "logps/chosen": -18.81739044189453,
      "logps/rejected": -24.600296020507812,
      "loss": 0.6612,
      "losses/dpo": 0.6598723530769348,
      "losses/sft": 0.8644169569015503,
      "losses/total": 0.6598723530769348,
      "ref_logps/chosen": -18.89391326904297,
      "ref_logps/rejected": -23.96309471130371,
      "rewards/accuracies": 0.671875,
      "rewards/chosen": 0.0076522137969732285,
      "rewards/margins": 0.07137227803468704,
      "rewards/rejected": -0.06372006982564926,
      "step": 131
    },
    {
      "epoch": 1.0,
      "learning_rate": 3.707865168539326e-07,
      "logps/chosen": -25.24700927734375,
      "logps/rejected": -29.2607364654541,
      "loss": 0.6576,
      "losses/dpo": 0.6264052391052246,
      "losses/sft": 0.7484258413314819,
      "losses/total": 0.6264052391052246,
      "ref_logps/chosen": -25.243091583251953,
      "ref_logps/rejected": -28.458097457885742,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.00039180926978588104,
      "rewards/margins": 0.07987209409475327,
      "rewards/rejected": -0.0802639052271843,
      "step": 132
    },
    {
      "epoch": 1.0,
      "learning_rate": 3.693820224719101e-07,
      "logps/chosen": -24.664264678955078,
      "logps/rejected": -29.071331024169922,
      "loss": 0.6596,
      "losses/dpo": 0.6850643157958984,
      "losses/sft": 0.7063156366348267,
      "losses/total": 0.6850643157958984,
      "ref_logps/chosen": -24.58011245727539,
      "ref_logps/rejected": -28.208541870117188,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.008415229618549347,
      "rewards/margins": 0.07786377519369125,
      "rewards/rejected": -0.08627899736166,
      "step": 133
    },
    {
      "epoch": 1.01,
      "learning_rate": 3.6797752808988764e-07,
      "logps/chosen": -21.803192138671875,
      "logps/rejected": -25.79207992553711,
      "loss": 0.6529,
      "losses/dpo": 0.6567816734313965,
      "losses/sft": 0.8528650403022766,
      "losses/total": 0.6567816734313965,
      "ref_logps/chosen": -21.88966941833496,
      "ref_logps/rejected": -24.97705841064453,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.00864771381020546,
      "rewards/margins": 0.0901501327753067,
      "rewards/rejected": -0.08150242269039154,
      "step": 134
    },
    {
      "epoch": 1.02,
      "learning_rate": 3.6657303370786514e-07,
      "logps/chosen": -20.78626823425293,
      "logps/rejected": -27.048810958862305,
      "loss": 0.6442,
      "losses/dpo": 0.6402660608291626,
      "losses/sft": 0.7653439044952393,
      "losses/total": 0.6402660608291626,
      "ref_logps/chosen": -20.915481567382812,
      "ref_logps/rejected": -26.105587005615234,
      "rewards/accuracies": 0.765625,
      "rewards/chosen": 0.012921325862407684,
      "rewards/margins": 0.10724389553070068,
      "rewards/rejected": -0.0943225771188736,
      "step": 135
    },
    {
      "epoch": 1.03,
      "learning_rate": 3.651685393258427e-07,
      "logps/chosen": -23.661598205566406,
      "logps/rejected": -26.884532928466797,
      "loss": 0.6563,
      "losses/dpo": 0.6588989496231079,
      "losses/sft": 0.8334387540817261,
      "losses/total": 0.6588989496231079,
      "ref_logps/chosen": -23.68170166015625,
      "ref_logps/rejected": -26.042449951171875,
      "rewards/accuracies": 0.671875,
      "rewards/chosen": 0.0020102611742913723,
      "rewards/margins": 0.08621874451637268,
      "rewards/rejected": -0.08420848101377487,
      "step": 136
    },
    {
      "epoch": 1.03,
      "learning_rate": 3.637640449438202e-07,
      "logps/chosen": -21.846914291381836,
      "logps/rejected": -26.843595504760742,
      "loss": 0.6414,
      "losses/dpo": 0.610801100730896,
      "losses/sft": 0.6104759573936462,
      "losses/total": 0.610801100730896,
      "ref_logps/chosen": -21.904037475585938,
      "ref_logps/rejected": -25.758628845214844,
      "rewards/accuracies": 0.7734375,
      "rewards/chosen": 0.005712391808629036,
      "rewards/margins": 0.11420895159244537,
      "rewards/rejected": -0.10849656164646149,
      "step": 137
    },
    {
      "epoch": 1.04,
      "learning_rate": 3.6235955056179776e-07,
      "logps/chosen": -23.79953384399414,
      "logps/rejected": -26.24932861328125,
      "loss": 0.6507,
      "losses/dpo": 0.6711180806159973,
      "losses/sft": 0.8334028720855713,
      "losses/total": 0.6711180806159973,
      "ref_logps/chosen": -23.89289093017578,
      "ref_logps/rejected": -25.393817901611328,
      "rewards/accuracies": 0.6953125,
      "rewards/chosen": 0.009335671551525593,
      "rewards/margins": 0.09488671272993088,
      "rewards/rejected": -0.08555103838443756,
      "step": 138
    },
    {
      "epoch": 1.05,
      "learning_rate": 3.6095505617977526e-07,
      "logps/chosen": -20.413612365722656,
      "logps/rejected": -28.091732025146484,
      "loss": 0.6393,
      "losses/dpo": 0.6086191534996033,
      "losses/sft": 0.7045127749443054,
      "losses/total": 0.6086191534996033,
      "ref_logps/chosen": -20.591529846191406,
      "ref_logps/rejected": -27.054677963256836,
      "rewards/accuracies": 0.703125,
      "rewards/chosen": 0.0177919864654541,
      "rewards/margins": 0.12149728834629059,
      "rewards/rejected": -0.10370529443025589,
      "step": 139
    },
    {
      "epoch": 1.06,
      "learning_rate": 3.5955056179775277e-07,
      "logps/chosen": -23.96946907043457,
      "logps/rejected": -25.42624282836914,
      "loss": 0.6574,
      "losses/dpo": 0.6771029233932495,
      "losses/sft": 0.8275946378707886,
      "losses/total": 0.6771029233932495,
      "ref_logps/chosen": -23.84187126159668,
      "ref_logps/rejected": -24.474294662475586,
      "rewards/accuracies": 0.703125,
      "rewards/chosen": -0.012759597972035408,
      "rewards/margins": 0.0824354737997055,
      "rewards/rejected": -0.09519506990909576,
      "step": 140
    },
    {
      "epoch": 1.06,
      "learning_rate": 3.581460674157303e-07,
      "logps/chosen": -20.24493980407715,
      "logps/rejected": -26.33192253112793,
      "loss": 0.6403,
      "losses/dpo": 0.60587477684021,
      "losses/sft": 0.7718257904052734,
      "losses/total": 0.60587477684021,
      "ref_logps/chosen": -20.375638961791992,
      "ref_logps/rejected": -25.299020767211914,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.013069930486381054,
      "rewards/margins": 0.11636004596948624,
      "rewards/rejected": -0.10329011082649231,
      "step": 141
    },
    {
      "epoch": 1.07,
      "learning_rate": 3.5674157303370783e-07,
      "logps/chosen": -22.9414119720459,
      "logps/rejected": -28.200380325317383,
      "loss": 0.6384,
      "losses/dpo": 0.6827423572540283,
      "losses/sft": 0.8567611575126648,
      "losses/total": 0.6827423572540283,
      "ref_logps/chosen": -23.111347198486328,
      "ref_logps/rejected": -27.142616271972656,
      "rewards/accuracies": 0.7421875,
      "rewards/chosen": 0.01699351891875267,
      "rewards/margins": 0.12276984751224518,
      "rewards/rejected": -0.10577632486820221,
      "step": 142
    },
    {
      "epoch": 1.08,
      "learning_rate": 3.553370786516854e-07,
      "logps/chosen": -23.226070404052734,
      "logps/rejected": -27.77198028564453,
      "loss": 0.6624,
      "losses/dpo": 0.6864386796951294,
      "losses/sft": 0.8041479587554932,
      "losses/total": 0.6864386796951294,
      "ref_logps/chosen": -22.92740249633789,
      "ref_logps/rejected": -26.72946548461914,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.029866419732570648,
      "rewards/margins": 0.07438516616821289,
      "rewards/rejected": -0.10425157845020294,
      "step": 143
    },
    {
      "epoch": 1.09,
      "learning_rate": 3.539325842696629e-07,
      "logps/chosen": -21.75617027282715,
      "logps/rejected": -28.53704833984375,
      "loss": 0.6455,
      "losses/dpo": 0.6347097158432007,
      "losses/sft": 0.6569658517837524,
      "losses/total": 0.6347097158432007,
      "ref_logps/chosen": -21.872474670410156,
      "ref_logps/rejected": -27.540082931518555,
      "rewards/accuracies": 0.7109375,
      "rewards/chosen": 0.011630430817604065,
      "rewards/margins": 0.11132718622684479,
      "rewards/rejected": -0.09969674795866013,
      "step": 144
    },
    {
      "epoch": 1.09,
      "learning_rate": 3.5252808988764045e-07,
      "logps/chosen": -24.18975830078125,
      "logps/rejected": -29.736862182617188,
      "loss": 0.6407,
      "losses/dpo": 0.6530706286430359,
      "losses/sft": 0.8703383207321167,
      "losses/total": 0.6530706286430359,
      "ref_logps/chosen": -24.138484954833984,
      "ref_logps/rejected": -28.495933532714844,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.005126964300870895,
      "rewards/margins": 0.11896562576293945,
      "rewards/rejected": -0.12409258633852005,
      "step": 145
    },
    {
      "epoch": 1.1,
      "learning_rate": 3.51123595505618e-07,
      "logps/chosen": -24.84428596496582,
      "logps/rejected": -29.576303482055664,
      "loss": 0.647,
      "losses/dpo": 0.6477080583572388,
      "losses/sft": 0.8653473854064941,
      "losses/total": 0.6477080583572388,
      "ref_logps/chosen": -24.755064010620117,
      "ref_logps/rejected": -28.43872833251953,
      "rewards/accuracies": 0.7265625,
      "rewards/chosen": -0.008922239765524864,
      "rewards/margins": 0.10483534634113312,
      "rewards/rejected": -0.11375758051872253,
      "step": 146
    },
    {
      "epoch": 1.11,
      "learning_rate": 3.497191011235955e-07,
      "logps/chosen": -24.983165740966797,
      "logps/rejected": -27.753063201904297,
      "loss": 0.6095,
      "losses/dpo": 0.6273882389068604,
      "losses/sft": 0.8987213373184204,
      "losses/total": 0.6273882389068604,
      "ref_logps/chosen": -25.17366600036621,
      "ref_logps/rejected": -26.05366325378418,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.019050076603889465,
      "rewards/margins": 0.1889900416135788,
      "rewards/rejected": -0.16993993520736694,
      "step": 147
    },
    {
      "epoch": 1.12,
      "learning_rate": 3.48314606741573e-07,
      "logps/chosen": -22.61692237854004,
      "logps/rejected": -27.743179321289062,
      "loss": 0.6583,
      "losses/dpo": 0.6790063381195068,
      "losses/sft": 0.7648496627807617,
      "losses/total": 0.6790063381195068,
      "ref_logps/chosen": -22.40664291381836,
      "ref_logps/rejected": -26.67925262451172,
      "rewards/accuracies": 0.671875,
      "rewards/chosen": -0.02102772891521454,
      "rewards/margins": 0.08536479622125626,
      "rewards/rejected": -0.1063925176858902,
      "step": 148
    },
    {
      "epoch": 1.12,
      "learning_rate": 3.469101123595505e-07,
      "logps/chosen": -22.846782684326172,
      "logps/rejected": -29.590002059936523,
      "loss": 0.6261,
      "losses/dpo": 0.6479306221008301,
      "losses/sft": 0.8049210906028748,
      "losses/total": 0.6479306221008301,
      "ref_logps/chosen": -23.011579513549805,
      "ref_logps/rejected": -28.2562198638916,
      "rewards/accuracies": 0.7734375,
      "rewards/chosen": 0.016479745507240295,
      "rewards/margins": 0.14985813200473785,
      "rewards/rejected": -0.13337840139865875,
      "step": 149
    },
    {
      "epoch": 1.13,
      "learning_rate": 3.4550561797752807e-07,
      "logps/chosen": -21.699583053588867,
      "logps/rejected": -27.46141815185547,
      "loss": 0.6277,
      "losses/dpo": 0.6358213424682617,
      "losses/sft": 0.8344307541847229,
      "losses/total": 0.6358213424682617,
      "ref_logps/chosen": -21.698383331298828,
      "ref_logps/rejected": -25.974313735961914,
      "rewards/accuracies": 0.7734375,
      "rewards/chosen": -0.00012012943625450134,
      "rewards/margins": 0.14859014749526978,
      "rewards/rejected": -0.14871028065681458,
      "step": 150
    },
    {
      "epoch": 1.14,
      "learning_rate": 3.441011235955056e-07,
      "logps/chosen": -20.88718032836914,
      "logps/rejected": -25.436817169189453,
      "loss": 0.654,
      "losses/dpo": 0.6406779289245605,
      "losses/sft": 0.8018806576728821,
      "losses/total": 0.6406779289245605,
      "ref_logps/chosen": -20.712448120117188,
      "ref_logps/rejected": -24.37557029724121,
      "rewards/accuracies": 0.671875,
      "rewards/chosen": -0.01747327297925949,
      "rewards/margins": 0.08865140378475189,
      "rewards/rejected": -0.10612466931343079,
      "step": 151
    },
    {
      "epoch": 1.15,
      "learning_rate": 3.4269662921348313e-07,
      "logps/chosen": -22.312236785888672,
      "logps/rejected": -30.142927169799805,
      "loss": 0.6355,
      "losses/dpo": 0.5932921171188354,
      "losses/sft": 0.6528638005256653,
      "losses/total": 0.5932921171188354,
      "ref_logps/chosen": -22.210554122924805,
      "ref_logps/rejected": -28.713180541992188,
      "rewards/accuracies": 0.703125,
      "rewards/chosen": -0.010168392211198807,
      "rewards/margins": 0.13280624151229858,
      "rewards/rejected": -0.1429746299982071,
      "step": 152
    },
    {
      "epoch": 1.15,
      "learning_rate": 3.4129213483146064e-07,
      "logps/chosen": -26.28810691833496,
      "logps/rejected": -29.10406494140625,
      "loss": 0.6359,
      "losses/dpo": 0.6205468773841858,
      "losses/sft": 0.8744308352470398,
      "losses/total": 0.6205468773841858,
      "ref_logps/chosen": -26.150360107421875,
      "ref_logps/rejected": -27.655288696289062,
      "rewards/accuracies": 0.7890625,
      "rewards/chosen": -0.013774631544947624,
      "rewards/margins": 0.1311032772064209,
      "rewards/rejected": -0.14487791061401367,
      "step": 153
    },
    {
      "epoch": 1.16,
      "learning_rate": 3.398876404494382e-07,
      "logps/chosen": -22.283679962158203,
      "logps/rejected": -26.302614212036133,
      "loss": 0.6679,
      "losses/dpo": 0.6655905246734619,
      "losses/sft": 0.8864909410476685,
      "losses/total": 0.6655905246734619,
      "ref_logps/chosen": -21.867923736572266,
      "ref_logps/rejected": -25.230295181274414,
      "rewards/accuracies": 0.6484375,
      "rewards/chosen": -0.04157543182373047,
      "rewards/margins": 0.0656563863158226,
      "rewards/rejected": -0.10723182559013367,
      "step": 154
    },
    {
      "epoch": 1.17,
      "learning_rate": 3.3848314606741575e-07,
      "logps/chosen": -22.68756103515625,
      "logps/rejected": -28.45652961730957,
      "loss": 0.6559,
      "losses/dpo": 0.6645406484603882,
      "losses/sft": 0.794353723526001,
      "losses/total": 0.6645406484603882,
      "ref_logps/chosen": -22.445241928100586,
      "ref_logps/rejected": -27.26552963256836,
      "rewards/accuracies": 0.6328125,
      "rewards/chosen": -0.024232013151049614,
      "rewards/margins": 0.09486782550811768,
      "rewards/rejected": -0.11909983307123184,
      "step": 155
    },
    {
      "epoch": 1.18,
      "learning_rate": 3.3707865168539325e-07,
      "logps/chosen": -22.336397171020508,
      "logps/rejected": -27.09580421447754,
      "loss": 0.6194,
      "losses/dpo": 0.5837043523788452,
      "losses/sft": 0.9716494083404541,
      "losses/total": 0.5837043523788452,
      "ref_logps/chosen": -22.35472297668457,
      "ref_logps/rejected": -25.422115325927734,
      "rewards/accuracies": 0.7734375,
      "rewards/chosen": 0.001832372508943081,
      "rewards/margins": 0.16920123994350433,
      "rewards/rejected": -0.16736885905265808,
      "step": 156
    },
    {
      "epoch": 1.18,
      "learning_rate": 3.356741573033708e-07,
      "logps/chosen": -22.49996566772461,
      "logps/rejected": -28.435253143310547,
      "loss": 0.6425,
      "losses/dpo": 0.6696836948394775,
      "losses/sft": 0.773880660533905,
      "losses/total": 0.6696836948394775,
      "ref_logps/chosen": -22.170368194580078,
      "ref_logps/rejected": -26.929363250732422,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.032960131764411926,
      "rewards/margins": 0.11762877553701401,
      "rewards/rejected": -0.15058889985084534,
      "step": 157
    },
    {
      "epoch": 1.19,
      "learning_rate": 3.3426966292134826e-07,
      "logps/chosen": -22.498619079589844,
      "logps/rejected": -30.868057250976562,
      "loss": 0.6295,
      "losses/dpo": 0.6400988101959229,
      "losses/sft": 0.724359929561615,
      "losses/total": 0.6400988101959229,
      "ref_logps/chosen": -22.199575424194336,
      "ref_logps/rejected": -29.1011962890625,
      "rewards/accuracies": 0.7109375,
      "rewards/chosen": -0.029904408380389214,
      "rewards/margins": 0.1467815339565277,
      "rewards/rejected": -0.17668592929840088,
      "step": 158
    },
    {
      "epoch": 1.2,
      "learning_rate": 3.328651685393258e-07,
      "logps/chosen": -24.872241973876953,
      "logps/rejected": -29.327089309692383,
      "loss": 0.6331,
      "losses/dpo": 0.6349748373031616,
      "losses/sft": 0.7728020548820496,
      "losses/total": 0.6349748373031616,
      "ref_logps/chosen": -24.600563049316406,
      "ref_logps/rejected": -27.632978439331055,
      "rewards/accuracies": 0.6953125,
      "rewards/chosen": -0.027167750522494316,
      "rewards/margins": 0.14224328100681305,
      "rewards/rejected": -0.16941101849079132,
      "step": 159
    },
    {
      "epoch": 1.21,
      "learning_rate": 3.314606741573033e-07,
      "logps/chosen": -25.719676971435547,
      "logps/rejected": -28.384960174560547,
      "loss": 0.6269,
      "losses/dpo": 0.6175022721290588,
      "losses/sft": 0.8887324929237366,
      "losses/total": 0.6175022721290588,
      "ref_logps/chosen": -25.583393096923828,
      "ref_logps/rejected": -26.621837615966797,
      "rewards/accuracies": 0.703125,
      "rewards/chosen": -0.013628311455249786,
      "rewards/margins": 0.16268408298492432,
      "rewards/rejected": -0.1763123720884323,
      "step": 160
    },
    {
      "epoch": 1.22,
      "learning_rate": 3.300561797752809e-07,
      "logps/chosen": -20.547767639160156,
      "logps/rejected": -26.39871597290039,
      "loss": 0.6418,
      "losses/dpo": 0.604182243347168,
      "losses/sft": 0.63340824842453,
      "losses/total": 0.604182243347168,
      "ref_logps/chosen": -20.272342681884766,
      "ref_logps/rejected": -24.899860382080078,
      "rewards/accuracies": 0.7109375,
      "rewards/chosen": -0.027542442083358765,
      "rewards/margins": 0.12234312295913696,
      "rewards/rejected": -0.14988556504249573,
      "step": 161
    },
    {
      "epoch": 1.22,
      "learning_rate": 3.2865168539325844e-07,
      "logps/chosen": -22.42629623413086,
      "logps/rejected": -27.69287872314453,
      "loss": 0.6111,
      "losses/dpo": 0.5942946672439575,
      "losses/sft": 0.9472201466560364,
      "losses/total": 0.5942946672439575,
      "ref_logps/chosen": -22.135528564453125,
      "ref_logps/rejected": -25.469520568847656,
      "rewards/accuracies": 0.7421875,
      "rewards/chosen": -0.029076654464006424,
      "rewards/margins": 0.1932588517665863,
      "rewards/rejected": -0.22233551740646362,
      "step": 162
    },
    {
      "epoch": 1.23,
      "learning_rate": 3.2724719101123594e-07,
      "logps/chosen": -23.306896209716797,
      "logps/rejected": -28.64287567138672,
      "loss": 0.6467,
      "losses/dpo": 0.6821013689041138,
      "losses/sft": 0.9050745368003845,
      "losses/total": 0.6821013689041138,
      "ref_logps/chosen": -22.86626625061035,
      "ref_logps/rejected": -27.0958251953125,
      "rewards/accuracies": 0.671875,
      "rewards/chosen": -0.04406279698014259,
      "rewards/margins": 0.11064193397760391,
      "rewards/rejected": -0.1547047346830368,
      "step": 163
    },
    {
      "epoch": 1.24,
      "learning_rate": 3.258426966292135e-07,
      "logps/chosen": -24.126543045043945,
      "logps/rejected": -26.020713806152344,
      "loss": 0.6214,
      "losses/dpo": 0.6081950664520264,
      "losses/sft": 0.827450692653656,
      "losses/total": 0.6081950664520264,
      "ref_logps/chosen": -23.934072494506836,
      "ref_logps/rejected": -24.092741012573242,
      "rewards/accuracies": 0.765625,
      "rewards/chosen": -0.01924710161983967,
      "rewards/margins": 0.17355017364025116,
      "rewards/rejected": -0.19279725849628448,
      "step": 164
    },
    {
      "epoch": 1.25,
      "learning_rate": 3.24438202247191e-07,
      "logps/chosen": -23.07083511352539,
      "logps/rejected": -29.666513442993164,
      "loss": 0.6401,
      "losses/dpo": 0.6096771955490112,
      "losses/sft": 0.7951339483261108,
      "losses/total": 0.6096771955490112,
      "ref_logps/chosen": -22.55157470703125,
      "ref_logps/rejected": -27.86458396911621,
      "rewards/accuracies": 0.6796875,
      "rewards/chosen": -0.05192602425813675,
      "rewards/margins": 0.12826718389987946,
      "rewards/rejected": -0.1801932007074356,
      "step": 165
    },
    {
      "epoch": 1.25,
      "learning_rate": 3.2303370786516856e-07,
      "logps/chosen": -23.97926139831543,
      "logps/rejected": -26.387611389160156,
      "loss": 0.6543,
      "losses/dpo": 0.5806229710578918,
      "losses/sft": 0.9021787047386169,
      "losses/total": 0.5806229710578918,
      "ref_logps/chosen": -23.097957611083984,
      "ref_logps/rejected": -24.48831558227539,
      "rewards/accuracies": 0.609375,
      "rewards/chosen": -0.0881301686167717,
      "rewards/margins": 0.10179921984672546,
      "rewards/rejected": -0.18992936611175537,
      "step": 166
    },
    {
      "epoch": 1.26,
      "learning_rate": 3.21629213483146e-07,
      "logps/chosen": -24.496349334716797,
      "logps/rejected": -28.20893669128418,
      "loss": 0.6439,
      "losses/dpo": 0.5786381959915161,
      "losses/sft": 0.9020153284072876,
      "losses/total": 0.5786381959915161,
      "ref_logps/chosen": -24.026926040649414,
      "ref_logps/rejected": -26.54248046875,
      "rewards/accuracies": 0.6484375,
      "rewards/chosen": -0.04694243520498276,
      "rewards/margins": 0.11970352381467819,
      "rewards/rejected": -0.16664597392082214,
      "step": 167
    },
    {
      "epoch": 1.27,
      "learning_rate": 3.2022471910112357e-07,
      "logps/chosen": -23.227306365966797,
      "logps/rejected": -29.19955825805664,
      "loss": 0.6389,
      "losses/dpo": 0.6521559953689575,
      "losses/sft": 0.9907703399658203,
      "losses/total": 0.6521559953689575,
      "ref_logps/chosen": -22.804248809814453,
      "ref_logps/rejected": -27.36874008178711,
      "rewards/accuracies": 0.640625,
      "rewards/chosen": -0.042305897921323776,
      "rewards/margins": 0.14077602326869965,
      "rewards/rejected": -0.18308192491531372,
      "step": 168
    },
    {
      "epoch": 1.28,
      "learning_rate": 3.1882022471910107e-07,
      "logps/chosen": -22.211841583251953,
      "logps/rejected": -27.533721923828125,
      "loss": 0.6512,
      "losses/dpo": 0.6903020143508911,
      "losses/sft": 0.8463045358657837,
      "losses/total": 0.6903020143508911,
      "ref_logps/chosen": -21.630611419677734,
      "ref_logps/rejected": -25.909526824951172,
      "rewards/accuracies": 0.6640625,
      "rewards/chosen": -0.05812288075685501,
      "rewards/margins": 0.10429678112268448,
      "rewards/rejected": -0.1624196618795395,
      "step": 169
    },
    {
      "epoch": 1.28,
      "learning_rate": 3.1741573033707863e-07,
      "logps/chosen": -22.332489013671875,
      "logps/rejected": -28.400074005126953,
      "loss": 0.6155,
      "losses/dpo": 0.6296464204788208,
      "losses/sft": 0.6626120805740356,
      "losses/total": 0.6296464204788208,
      "ref_logps/chosen": -22.126087188720703,
      "ref_logps/rejected": -26.414535522460938,
      "rewards/accuracies": 0.796875,
      "rewards/chosen": -0.020640213042497635,
      "rewards/margins": 0.17791378498077393,
      "rewards/rejected": -0.19855400919914246,
      "step": 170
    },
    {
      "epoch": 1.29,
      "learning_rate": 3.160112359550562e-07,
      "logps/chosen": -23.771900177001953,
      "logps/rejected": -30.088207244873047,
      "loss": 0.5971,
      "losses/dpo": 0.6422166228294373,
      "losses/sft": 0.7472187876701355,
      "losses/total": 0.6422166228294373,
      "ref_logps/chosen": -23.688966751098633,
      "ref_logps/rejected": -27.655853271484375,
      "rewards/accuracies": 0.7734375,
      "rewards/chosen": -0.008293594233691692,
      "rewards/margins": 0.23494186997413635,
      "rewards/rejected": -0.24323543906211853,
      "step": 171
    },
    {
      "epoch": 1.3,
      "learning_rate": 3.146067415730337e-07,
      "logps/chosen": -23.348037719726562,
      "logps/rejected": -27.53687286376953,
      "loss": 0.6459,
      "losses/dpo": 0.6455183029174805,
      "losses/sft": 0.8395851850509644,
      "losses/total": 0.6455183029174805,
      "ref_logps/chosen": -22.63860511779785,
      "ref_logps/rejected": -25.60868263244629,
      "rewards/accuracies": 0.6640625,
      "rewards/chosen": -0.0709431990981102,
      "rewards/margins": 0.12187594175338745,
      "rewards/rejected": -0.19281914830207825,
      "step": 172
    },
    {
      "epoch": 1.31,
      "learning_rate": 3.1320224719101125e-07,
      "logps/chosen": -24.17770767211914,
      "logps/rejected": -30.49142074584961,
      "loss": 0.627,
      "losses/dpo": 0.6627662181854248,
      "losses/sft": 0.9079832434654236,
      "losses/total": 0.6627662181854248,
      "ref_logps/chosen": -23.401166915893555,
      "ref_logps/rejected": -28.0411376953125,
      "rewards/accuracies": 0.6796875,
      "rewards/chosen": -0.07765418291091919,
      "rewards/margins": 0.16737422347068787,
      "rewards/rejected": -0.24502840638160706,
      "step": 173
    },
    {
      "epoch": 1.31,
      "learning_rate": 3.1179775280898875e-07,
      "logps/chosen": -24.392324447631836,
      "logps/rejected": -27.670101165771484,
      "loss": 0.6251,
      "losses/dpo": 0.6143248081207275,
      "losses/sft": 0.6558141112327576,
      "losses/total": 0.6143248081207275,
      "ref_logps/chosen": -23.9196720123291,
      "ref_logps/rejected": -25.53693389892578,
      "rewards/accuracies": 0.7265625,
      "rewards/chosen": -0.047265198081731796,
      "rewards/margins": 0.16605158150196075,
      "rewards/rejected": -0.21331676840782166,
      "step": 174
    },
    {
      "epoch": 1.32,
      "learning_rate": 3.103932584269663e-07,
      "logps/chosen": -24.742660522460938,
      "logps/rejected": -33.37188720703125,
      "loss": 0.6157,
      "losses/dpo": 0.5933184623718262,
      "losses/sft": 0.9941530227661133,
      "losses/total": 0.5933184623718262,
      "ref_logps/chosen": -24.150442123413086,
      "ref_logps/rejected": -30.931093215942383,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.05922209471464157,
      "rewards/margins": 0.1848573535680771,
      "rewards/rejected": -0.24407947063446045,
      "step": 175
    },
    {
      "epoch": 1.33,
      "learning_rate": 3.0898876404494376e-07,
      "logps/chosen": -23.68863296508789,
      "logps/rejected": -28.516223907470703,
      "loss": 0.6428,
      "losses/dpo": 0.6548395156860352,
      "losses/sft": 0.9564076066017151,
      "losses/total": 0.6548395156860352,
      "ref_logps/chosen": -22.73943519592285,
      "ref_logps/rejected": -26.3284912109375,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.09491994976997375,
      "rewards/margins": 0.12385320663452148,
      "rewards/rejected": -0.21877314150333405,
      "step": 176
    },
    {
      "epoch": 1.34,
      "learning_rate": 3.075842696629213e-07,
      "logps/chosen": -22.527427673339844,
      "logps/rejected": -31.241607666015625,
      "loss": 0.6179,
      "losses/dpo": 0.5700336694717407,
      "losses/sft": 0.8869008421897888,
      "losses/total": 0.5700336694717407,
      "ref_logps/chosen": -21.73688507080078,
      "ref_logps/rejected": -28.55950164794922,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.07905411720275879,
      "rewards/margins": 0.18915657699108124,
      "rewards/rejected": -0.2682107090950012,
      "step": 177
    },
    {
      "epoch": 1.34,
      "learning_rate": 3.0617977528089887e-07,
      "logps/chosen": -22.916969299316406,
      "logps/rejected": -26.000946044921875,
      "loss": 0.6425,
      "losses/dpo": 0.651595413684845,
      "losses/sft": 0.8127326369285583,
      "losses/total": 0.651595413684845,
      "ref_logps/chosen": -22.194671630859375,
      "ref_logps/rejected": -23.969348907470703,
      "rewards/accuracies": 0.671875,
      "rewards/chosen": -0.07222998142242432,
      "rewards/margins": 0.1309295892715454,
      "rewards/rejected": -0.20315957069396973,
      "step": 178
    },
    {
      "epoch": 1.35,
      "learning_rate": 3.047752808988764e-07,
      "logps/chosen": -20.320987701416016,
      "logps/rejected": -27.46251106262207,
      "loss": 0.6217,
      "losses/dpo": 0.7334872484207153,
      "losses/sft": 0.9430239200592041,
      "losses/total": 0.7334872484207153,
      "ref_logps/chosen": -19.424144744873047,
      "ref_logps/rejected": -24.763113021850586,
      "rewards/accuracies": 0.6953125,
      "rewards/chosen": -0.08968427777290344,
      "rewards/margins": 0.18025556206703186,
      "rewards/rejected": -0.2699398398399353,
      "step": 179
    },
    {
      "epoch": 1.36,
      "learning_rate": 3.0337078651685393e-07,
      "logps/chosen": -23.853857040405273,
      "logps/rejected": -27.422889709472656,
      "loss": 0.6381,
      "losses/dpo": 0.6393001079559326,
      "losses/sft": 0.766620397567749,
      "losses/total": 0.6393001079559326,
      "ref_logps/chosen": -22.798233032226562,
      "ref_logps/rejected": -24.980205535888672,
      "rewards/accuracies": 0.6796875,
      "rewards/chosen": -0.10556241869926453,
      "rewards/margins": 0.13870559632778168,
      "rewards/rejected": -0.2442680299282074,
      "step": 180
    },
    {
      "epoch": 1.37,
      "learning_rate": 3.0196629213483144e-07,
      "logps/chosen": -23.360549926757812,
      "logps/rejected": -27.110477447509766,
      "loss": 0.6234,
      "losses/dpo": 0.6311055421829224,
      "losses/sft": 0.9324018955230713,
      "losses/total": 0.6311055421829224,
      "ref_logps/chosen": -22.890331268310547,
      "ref_logps/rejected": -24.903316497802734,
      "rewards/accuracies": 0.7421875,
      "rewards/chosen": -0.04702185466885567,
      "rewards/margins": 0.17369432747364044,
      "rewards/rejected": -0.2207161784172058,
      "step": 181
    },
    {
      "epoch": 1.37,
      "learning_rate": 3.00561797752809e-07,
      "logps/chosen": -23.004093170166016,
      "logps/rejected": -31.04292106628418,
      "loss": 0.5926,
      "losses/dpo": 0.6243355870246887,
      "losses/sft": 0.8456003665924072,
      "losses/total": 0.6243355870246887,
      "ref_logps/chosen": -22.555362701416016,
      "ref_logps/rejected": -28.14826011657715,
      "rewards/accuracies": 0.7578125,
      "rewards/chosen": -0.04487309604883194,
      "rewards/margins": 0.24459321796894073,
      "rewards/rejected": -0.28946632146835327,
      "step": 182
    },
    {
      "epoch": 1.38,
      "learning_rate": 2.991573033707865e-07,
      "logps/chosen": -26.668237686157227,
      "logps/rejected": -30.511489868164062,
      "loss": 0.6099,
      "losses/dpo": 0.6743872761726379,
      "losses/sft": 0.836949348449707,
      "losses/total": 0.6743872761726379,
      "ref_logps/chosen": -25.680599212646484,
      "ref_logps/rejected": -27.35342788696289,
      "rewards/accuracies": 0.7265625,
      "rewards/chosen": -0.09876400232315063,
      "rewards/margins": 0.2170422226190567,
      "rewards/rejected": -0.31580623984336853,
      "step": 183
    },
    {
      "epoch": 1.39,
      "learning_rate": 2.9775280898876406e-07,
      "logps/chosen": -23.974590301513672,
      "logps/rejected": -28.162975311279297,
      "loss": 0.6119,
      "losses/dpo": 0.5823447704315186,
      "losses/sft": 0.8065779805183411,
      "losses/total": 0.5823447704315186,
      "ref_logps/chosen": -23.180667877197266,
      "ref_logps/rejected": -25.296037673950195,
      "rewards/accuracies": 0.7109375,
      "rewards/chosen": -0.07939236611127853,
      "rewards/margins": 0.20730134844779968,
      "rewards/rejected": -0.2866936922073364,
      "step": 184
    },
    {
      "epoch": 1.4,
      "learning_rate": 2.9634831460674156e-07,
      "logps/chosen": -24.01116943359375,
      "logps/rejected": -30.05943489074707,
      "loss": 0.6203,
      "losses/dpo": 0.5889841318130493,
      "losses/sft": 0.8877280354499817,
      "losses/total": 0.5889841318130493,
      "ref_logps/chosen": -22.781108856201172,
      "ref_logps/rejected": -27.01274871826172,
      "rewards/accuracies": 0.734375,
      "rewards/chosen": -0.12300599366426468,
      "rewards/margins": 0.18166252970695496,
      "rewards/rejected": -0.3046685457229614,
      "step": 185
    },
    {
      "epoch": 1.4,
      "learning_rate": 2.9494382022471906e-07,
      "logps/chosen": -22.79621124267578,
      "logps/rejected": -28.1258544921875,
      "loss": 0.6198,
      "losses/dpo": 0.6025291681289673,
      "losses/sft": 0.93308424949646,
      "losses/total": 0.6025291681289673,
      "ref_logps/chosen": -21.591278076171875,
      "ref_logps/rejected": -25.04897689819336,
      "rewards/accuracies": 0.7421875,
      "rewards/chosen": -0.12049318104982376,
      "rewards/margins": 0.18719442188739777,
      "rewards/rejected": -0.3076876401901245,
      "step": 186
    },
    {
      "epoch": 1.41,
      "learning_rate": 2.935393258426966e-07,
      "logps/chosen": -24.246837615966797,
      "logps/rejected": -30.58446502685547,
      "loss": 0.6277,
      "losses/dpo": 0.5978178977966309,
      "losses/sft": 0.7778979539871216,
      "losses/total": 0.5978178977966309,
      "ref_logps/chosen": -23.127248764038086,
      "ref_logps/rejected": -27.7061767578125,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.11195877939462662,
      "rewards/margins": 0.17587023973464966,
      "rewards/rejected": -0.2878290116786957,
      "step": 187
    },
    {
      "epoch": 1.42,
      "learning_rate": 2.921348314606741e-07,
      "logps/chosen": -24.55533218383789,
      "logps/rejected": -29.098743438720703,
      "loss": 0.6458,
      "losses/dpo": 0.6147331595420837,
      "losses/sft": 0.8299495577812195,
      "losses/total": 0.6147331595420837,
      "ref_logps/chosen": -23.001358032226562,
      "ref_logps/rejected": -26.009681701660156,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.1553977131843567,
      "rewards/margins": 0.15350814163684845,
      "rewards/rejected": -0.30890583992004395,
      "step": 188
    },
    {
      "epoch": 1.43,
      "learning_rate": 2.907303370786517e-07,
      "logps/chosen": -22.7973690032959,
      "logps/rejected": -30.61502456665039,
      "loss": 0.5968,
      "losses/dpo": 0.5409806370735168,
      "losses/sft": 0.8110998272895813,
      "losses/total": 0.5409806370735168,
      "ref_logps/chosen": -21.900728225708008,
      "ref_logps/rejected": -27.346271514892578,
      "rewards/accuracies": 0.7578125,
      "rewards/chosen": -0.08966411650180817,
      "rewards/margins": 0.23721098899841309,
      "rewards/rejected": -0.32687509059906006,
      "step": 189
    },
    {
      "epoch": 1.43,
      "learning_rate": 2.893258426966292e-07,
      "logps/chosen": -21.656837463378906,
      "logps/rejected": -28.09313201904297,
      "loss": 0.636,
      "losses/dpo": 0.6395488977432251,
      "losses/sft": 0.8838689923286438,
      "losses/total": 0.6395488977432251,
      "ref_logps/chosen": -20.17813491821289,
      "ref_logps/rejected": -25.027902603149414,
      "rewards/accuracies": 0.6484375,
      "rewards/chosen": -0.14787010848522186,
      "rewards/margins": 0.15865309536457062,
      "rewards/rejected": -0.3065232038497925,
      "step": 190
    },
    {
      "epoch": 1.44,
      "learning_rate": 2.8792134831460674e-07,
      "logps/chosen": -23.13861083984375,
      "logps/rejected": -32.06410217285156,
      "loss": 0.6131,
      "losses/dpo": 0.6822565197944641,
      "losses/sft": 0.7876338362693787,
      "losses/total": 0.6822565197944641,
      "ref_logps/chosen": -21.99342918395996,
      "ref_logps/rejected": -28.761310577392578,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.11451825499534607,
      "rewards/margins": 0.21576061844825745,
      "rewards/rejected": -0.33027884364128113,
      "step": 191
    },
    {
      "epoch": 1.45,
      "learning_rate": 2.8651685393258425e-07,
      "logps/chosen": -22.36726951599121,
      "logps/rejected": -27.791099548339844,
      "loss": 0.6132,
      "losses/dpo": 0.5694007873535156,
      "losses/sft": 0.7940797805786133,
      "losses/total": 0.5694007873535156,
      "ref_logps/chosen": -21.662071228027344,
      "ref_logps/rejected": -25.100269317626953,
      "rewards/accuracies": 0.7265625,
      "rewards/chosen": -0.07051999121904373,
      "rewards/margins": 0.19856315851211548,
      "rewards/rejected": -0.2690831422805786,
      "step": 192
    },
    {
      "epoch": 1.46,
      "learning_rate": 2.851123595505618e-07,
      "logps/chosen": -24.541927337646484,
      "logps/rejected": -30.479598999023438,
      "loss": 0.6251,
      "losses/dpo": 0.6676912307739258,
      "losses/sft": 0.8101266026496887,
      "losses/total": 0.6676912307739258,
      "ref_logps/chosen": -23.375761032104492,
      "ref_logps/rejected": -27.565099716186523,
      "rewards/accuracies": 0.7109375,
      "rewards/chosen": -0.11661653220653534,
      "rewards/margins": 0.1748332977294922,
      "rewards/rejected": -0.2914498448371887,
      "step": 193
    },
    {
      "epoch": 1.46,
      "learning_rate": 2.8370786516853936e-07,
      "logps/chosen": -23.954505920410156,
      "logps/rejected": -30.262849807739258,
      "loss": 0.6289,
      "losses/dpo": 0.6359354257583618,
      "losses/sft": 0.846460223197937,
      "losses/total": 0.6359354257583618,
      "ref_logps/chosen": -22.69145965576172,
      "ref_logps/rejected": -27.221202850341797,
      "rewards/accuracies": 0.671875,
      "rewards/chosen": -0.12630482017993927,
      "rewards/margins": 0.1778600960969925,
      "rewards/rejected": -0.30416491627693176,
      "step": 194
    },
    {
      "epoch": 1.47,
      "learning_rate": 2.823033707865168e-07,
      "logps/chosen": -25.615474700927734,
      "logps/rejected": -32.26765823364258,
      "loss": 0.6017,
      "losses/dpo": 0.6264960765838623,
      "losses/sft": 0.906339704990387,
      "losses/total": 0.6264960765838623,
      "ref_logps/chosen": -24.62253189086914,
      "ref_logps/rejected": -29.02202796936035,
      "rewards/accuracies": 0.7421875,
      "rewards/chosen": -0.09929438680410385,
      "rewards/margins": 0.22526855766773224,
      "rewards/rejected": -0.3245629370212555,
      "step": 195
    },
    {
      "epoch": 1.48,
      "learning_rate": 2.8089887640449437e-07,
      "logps/chosen": -22.84251594543457,
      "logps/rejected": -28.347021102905273,
      "loss": 0.6191,
      "losses/dpo": 0.6483104825019836,
      "losses/sft": 0.9074235558509827,
      "losses/total": 0.6483104825019836,
      "ref_logps/chosen": -21.419048309326172,
      "ref_logps/rejected": -25.032745361328125,
      "rewards/accuracies": 0.703125,
      "rewards/chosen": -0.14234672486782074,
      "rewards/margins": 0.18908075988292694,
      "rewards/rejected": -0.3314274847507477,
      "step": 196
    },
    {
      "epoch": 1.49,
      "learning_rate": 2.794943820224719e-07,
      "logps/chosen": -23.73548126220703,
      "logps/rejected": -28.329975128173828,
      "loss": 0.6238,
      "losses/dpo": 0.6014984250068665,
      "losses/sft": 0.773016631603241,
      "losses/total": 0.6014984250068665,
      "ref_logps/chosen": -22.425506591796875,
      "ref_logps/rejected": -25.099872589111328,
      "rewards/accuracies": 0.6796875,
      "rewards/chosen": -0.13099724054336548,
      "rewards/margins": 0.19201286137104034,
      "rewards/rejected": -0.323010116815567,
      "step": 197
    },
    {
      "epoch": 1.49,
      "learning_rate": 2.7808988764044943e-07,
      "logps/chosen": -26.183156967163086,
      "logps/rejected": -30.921403884887695,
      "loss": 0.625,
      "losses/dpo": 0.6309884190559387,
      "losses/sft": 0.8918415307998657,
      "losses/total": 0.6309884190559387,
      "ref_logps/chosen": -24.701202392578125,
      "ref_logps/rejected": -27.679357528686523,
      "rewards/accuracies": 0.6953125,
      "rewards/chosen": -0.14819550514221191,
      "rewards/margins": 0.17600935697555542,
      "rewards/rejected": -0.32420486211776733,
      "step": 198
    },
    {
      "epoch": 1.5,
      "learning_rate": 2.7668539325842694e-07,
      "logps/chosen": -23.88658905029297,
      "logps/rejected": -29.73432731628418,
      "loss": 0.6156,
      "losses/dpo": 0.6188192367553711,
      "losses/sft": 0.8410817384719849,
      "losses/total": 0.6188192367553711,
      "ref_logps/chosen": -22.504894256591797,
      "ref_logps/rejected": -26.384294509887695,
      "rewards/accuracies": 0.6328125,
      "rewards/chosen": -0.13816949725151062,
      "rewards/margins": 0.19683387875556946,
      "rewards/rejected": -0.3350033462047577,
      "step": 199
    },
    {
      "epoch": 1.51,
      "learning_rate": 2.752808988764045e-07,
      "logps/chosen": -23.145811080932617,
      "logps/rejected": -29.627685546875,
      "loss": 0.637,
      "losses/dpo": 0.6995939612388611,
      "losses/sft": 0.9283435344696045,
      "losses/total": 0.6995939612388611,
      "ref_logps/chosen": -21.45529556274414,
      "ref_logps/rejected": -26.336702346801758,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.1690514236688614,
      "rewards/margins": 0.16004663705825806,
      "rewards/rejected": -0.32909804582595825,
      "step": 200
    },
    {
      "epoch": 1.52,
      "learning_rate": 2.73876404494382e-07,
      "logps/chosen": -22.545406341552734,
      "logps/rejected": -30.04849624633789,
      "loss": 0.608,
      "losses/dpo": 0.6513813734054565,
      "losses/sft": 0.9403305649757385,
      "losses/total": 0.6513813734054565,
      "ref_logps/chosen": -21.010854721069336,
      "ref_logps/rejected": -26.149032592773438,
      "rewards/accuracies": 0.671875,
      "rewards/chosen": -0.15345513820648193,
      "rewards/margins": 0.23649117350578308,
      "rewards/rejected": -0.3899462819099426,
      "step": 201
    },
    {
      "epoch": 1.52,
      "learning_rate": 2.7247191011235955e-07,
      "logps/chosen": -22.640438079833984,
      "logps/rejected": -28.583681106567383,
      "loss": 0.6007,
      "losses/dpo": 0.5443820357322693,
      "losses/sft": 0.8517413139343262,
      "losses/total": 0.5443820357322693,
      "ref_logps/chosen": -21.29751968383789,
      "ref_logps/rejected": -24.850605010986328,
      "rewards/accuracies": 0.765625,
      "rewards/chosen": -0.1342916190624237,
      "rewards/margins": 0.23901620507240295,
      "rewards/rejected": -0.37330782413482666,
      "step": 202
    },
    {
      "epoch": 1.53,
      "learning_rate": 2.710674157303371e-07,
      "logps/chosen": -25.259624481201172,
      "logps/rejected": -32.96052551269531,
      "loss": 0.6029,
      "losses/dpo": 0.5749891996383667,
      "losses/sft": 0.9417051672935486,
      "losses/total": 0.5749891996383667,
      "ref_logps/chosen": -23.468887329101562,
      "ref_logps/rejected": -28.776565551757812,
      "rewards/accuracies": 0.7421875,
      "rewards/chosen": -0.17907381057739258,
      "rewards/margins": 0.23932181298732758,
      "rewards/rejected": -0.41839560866355896,
      "step": 203
    },
    {
      "epoch": 1.54,
      "learning_rate": 2.6966292134831456e-07,
      "logps/chosen": -24.431142807006836,
      "logps/rejected": -31.409852981567383,
      "loss": 0.6256,
      "losses/dpo": 0.6045551896095276,
      "losses/sft": 0.8162484169006348,
      "losses/total": 0.6045551896095276,
      "ref_logps/chosen": -22.7187442779541,
      "ref_logps/rejected": -27.74604606628418,
      "rewards/accuracies": 0.703125,
      "rewards/chosen": -0.1712397187948227,
      "rewards/margins": 0.19514092803001404,
      "rewards/rejected": -0.36638063192367554,
      "step": 204
    },
    {
      "epoch": 1.55,
      "learning_rate": 2.682584269662921e-07,
      "logps/chosen": -22.776988983154297,
      "logps/rejected": -30.418426513671875,
      "loss": 0.6093,
      "losses/dpo": 0.630817711353302,
      "losses/sft": 0.907343327999115,
      "losses/total": 0.630817711353302,
      "ref_logps/chosen": -21.150266647338867,
      "ref_logps/rejected": -26.52399444580078,
      "rewards/accuracies": 0.734375,
      "rewards/chosen": -0.16267219185829163,
      "rewards/margins": 0.22677099704742432,
      "rewards/rejected": -0.38944315910339355,
      "step": 205
    },
    {
      "epoch": 1.55,
      "learning_rate": 2.668539325842696e-07,
      "logps/chosen": -24.300395965576172,
      "logps/rejected": -32.63694763183594,
      "loss": 0.5834,
      "losses/dpo": 0.5977815389633179,
      "losses/sft": 0.8870611190795898,
      "losses/total": 0.5977815389633179,
      "ref_logps/chosen": -22.923202514648438,
      "ref_logps/rejected": -28.30066680908203,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.13771943747997284,
      "rewards/margins": 0.29590874910354614,
      "rewards/rejected": -0.4336281716823578,
      "step": 206
    },
    {
      "epoch": 1.56,
      "learning_rate": 2.654494382022472e-07,
      "logps/chosen": -25.562063217163086,
      "logps/rejected": -28.686279296875,
      "loss": 0.6248,
      "losses/dpo": 0.593975841999054,
      "losses/sft": 0.8298511505126953,
      "losses/total": 0.593975841999054,
      "ref_logps/chosen": -23.863605499267578,
      "ref_logps/rejected": -24.980735778808594,
      "rewards/accuracies": 0.6171875,
      "rewards/chosen": -0.16984564065933228,
      "rewards/margins": 0.2007087767124176,
      "rewards/rejected": -0.3705544173717499,
      "step": 207
    },
    {
      "epoch": 1.57,
      "learning_rate": 2.640449438202247e-07,
      "logps/chosen": -24.133087158203125,
      "logps/rejected": -32.86896514892578,
      "loss": 0.6072,
      "losses/dpo": 0.5785881280899048,
      "losses/sft": 0.9283973574638367,
      "losses/total": 0.5785881280899048,
      "ref_logps/chosen": -22.546520233154297,
      "ref_logps/rejected": -28.988473892211914,
      "rewards/accuracies": 0.671875,
      "rewards/chosen": -0.158656507730484,
      "rewards/margins": 0.2293928861618042,
      "rewards/rejected": -0.3880493640899658,
      "step": 208
    },
    {
      "epoch": 1.58,
      "learning_rate": 2.6264044943820224e-07,
      "logps/chosen": -21.610166549682617,
      "logps/rejected": -33.77753448486328,
      "loss": 0.5743,
      "losses/dpo": 0.5111271142959595,
      "losses/sft": 0.7807843685150146,
      "losses/total": 0.5111271142959595,
      "ref_logps/chosen": -20.26101303100586,
      "ref_logps/rejected": -29.390432357788086,
      "rewards/accuracies": 0.7578125,
      "rewards/chosen": -0.1349155306816101,
      "rewards/margins": 0.3037945628166199,
      "rewards/rejected": -0.43871009349823,
      "step": 209
    },
    {
      "epoch": 1.58,
      "learning_rate": 2.612359550561798e-07,
      "logps/chosen": -24.600027084350586,
      "logps/rejected": -28.993408203125,
      "loss": 0.621,
      "losses/dpo": 0.6254321336746216,
      "losses/sft": 0.7647839188575745,
      "losses/total": 0.6254321336746216,
      "ref_logps/chosen": -22.838638305664062,
      "ref_logps/rejected": -25.172962188720703,
      "rewards/accuracies": 0.6796875,
      "rewards/chosen": -0.1761387437582016,
      "rewards/margins": 0.20590564608573914,
      "rewards/rejected": -0.38204440474510193,
      "step": 210
    },
    {
      "epoch": 1.59,
      "learning_rate": 2.598314606741573e-07,
      "logps/chosen": -25.24309730529785,
      "logps/rejected": -32.02477264404297,
      "loss": 0.6078,
      "losses/dpo": 0.6571998000144958,
      "losses/sft": 0.8880329728126526,
      "losses/total": 0.6571998000144958,
      "ref_logps/chosen": -23.272363662719727,
      "ref_logps/rejected": -27.582080841064453,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.19707328081130981,
      "rewards/margins": 0.24719560146331787,
      "rewards/rejected": -0.4442688822746277,
      "step": 211
    },
    {
      "epoch": 1.6,
      "learning_rate": 2.5842696629213486e-07,
      "logps/chosen": -23.570541381835938,
      "logps/rejected": -31.662994384765625,
      "loss": 0.5954,
      "losses/dpo": 0.6153095960617065,
      "losses/sft": 0.7867841720581055,
      "losses/total": 0.6153095960617065,
      "ref_logps/chosen": -21.58125114440918,
      "ref_logps/rejected": -27.1029052734375,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.19892916083335876,
      "rewards/margins": 0.2570798993110657,
      "rewards/rejected": -0.45600906014442444,
      "step": 212
    },
    {
      "epoch": 1.61,
      "learning_rate": 2.5702247191011236e-07,
      "logps/chosen": -26.515090942382812,
      "logps/rejected": -33.26690673828125,
      "loss": 0.5944,
      "losses/dpo": 0.559239387512207,
      "losses/sft": 0.8030417561531067,
      "losses/total": 0.559239387512207,
      "ref_logps/chosen": -24.70389175415039,
      "ref_logps/rejected": -28.788631439208984,
      "rewards/accuracies": 0.7421875,
      "rewards/chosen": -0.18111974000930786,
      "rewards/margins": 0.2667076587677002,
      "rewards/rejected": -0.44782739877700806,
      "step": 213
    },
    {
      "epoch": 1.62,
      "learning_rate": 2.5561797752808987e-07,
      "logps/chosen": -23.109725952148438,
      "logps/rejected": -30.950822830200195,
      "loss": 0.6028,
      "losses/dpo": 0.6463332772254944,
      "losses/sft": 0.867030918598175,
      "losses/total": 0.6463332772254944,
      "ref_logps/chosen": -21.529489517211914,
      "ref_logps/rejected": -26.95291519165039,
      "rewards/accuracies": 0.7421875,
      "rewards/chosen": -0.15802377462387085,
      "rewards/margins": 0.24176692962646484,
      "rewards/rejected": -0.3997907340526581,
      "step": 214
    },
    {
      "epoch": 1.62,
      "learning_rate": 2.5421348314606737e-07,
      "logps/chosen": -22.45772933959961,
      "logps/rejected": -30.6645450592041,
      "loss": 0.548,
      "losses/dpo": 0.49787038564682007,
      "losses/sft": 0.9076435565948486,
      "losses/total": 0.49787038564682007,
      "ref_logps/chosen": -21.689294815063477,
      "ref_logps/rejected": -26.167482376098633,
      "rewards/accuracies": 0.796875,
      "rewards/chosen": -0.07684363424777985,
      "rewards/margins": 0.37286245822906494,
      "rewards/rejected": -0.449706107378006,
      "step": 215
    },
    {
      "epoch": 1.63,
      "learning_rate": 2.5280898876404493e-07,
      "logps/chosen": -23.930644989013672,
      "logps/rejected": -31.34885597229004,
      "loss": 0.5791,
      "losses/dpo": 0.6228358745574951,
      "losses/sft": 0.894844651222229,
      "losses/total": 0.6228358745574951,
      "ref_logps/chosen": -22.003002166748047,
      "ref_logps/rejected": -26.41282844543457,
      "rewards/accuracies": 0.7265625,
      "rewards/chosen": -0.19276437163352966,
      "rewards/margins": 0.3008383512496948,
      "rewards/rejected": -0.4936027228832245,
      "step": 216
    },
    {
      "epoch": 1.64,
      "learning_rate": 2.5140449438202243e-07,
      "logps/chosen": -25.59225082397461,
      "logps/rejected": -30.82415199279785,
      "loss": 0.5571,
      "losses/dpo": 0.5233840942382812,
      "losses/sft": 0.8860921263694763,
      "losses/total": 0.5233840942382812,
      "ref_logps/chosen": -23.89864730834961,
      "ref_logps/rejected": -25.53179168701172,
      "rewards/accuracies": 0.765625,
      "rewards/chosen": -0.16936028003692627,
      "rewards/margins": 0.35987579822540283,
      "rewards/rejected": -0.5292361378669739,
      "step": 217
    },
    {
      "epoch": 1.65,
      "learning_rate": 2.5e-07,
      "logps/chosen": -26.896615982055664,
      "logps/rejected": -32.64814376831055,
      "loss": 0.5821,
      "losses/dpo": 0.5345016121864319,
      "losses/sft": 0.9819333553314209,
      "losses/total": 0.5345016121864319,
      "ref_logps/chosen": -24.95808982849121,
      "ref_logps/rejected": -27.606571197509766,
      "rewards/accuracies": 0.703125,
      "rewards/chosen": -0.19385257363319397,
      "rewards/margins": 0.3103046417236328,
      "rewards/rejected": -0.5041571855545044,
      "step": 218
    },
    {
      "epoch": 1.65,
      "learning_rate": 2.485955056179775e-07,
      "logps/chosen": -21.461519241333008,
      "logps/rejected": -29.887657165527344,
      "loss": 0.5621,
      "losses/dpo": 0.5603345632553101,
      "losses/sft": 0.7855640649795532,
      "losses/total": 0.5603345632553101,
      "ref_logps/chosen": -20.122406005859375,
      "ref_logps/rejected": -24.85255241394043,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.13391147553920746,
      "rewards/margins": 0.3695991039276123,
      "rewards/rejected": -0.503510594367981,
      "step": 219
    },
    {
      "epoch": 1.66,
      "learning_rate": 2.4719101123595505e-07,
      "logps/chosen": -22.143098831176758,
      "logps/rejected": -34.4566764831543,
      "loss": 0.5923,
      "losses/dpo": 0.5465586185455322,
      "losses/sft": 1.051912546157837,
      "losses/total": 0.5465586185455322,
      "ref_logps/chosen": -20.453866958618164,
      "ref_logps/rejected": -30.012981414794922,
      "rewards/accuracies": 0.6796875,
      "rewards/chosen": -0.16892319917678833,
      "rewards/margins": 0.27544665336608887,
      "rewards/rejected": -0.4443698525428772,
      "step": 220
    },
    {
      "epoch": 1.67,
      "learning_rate": 2.4578651685393255e-07,
      "logps/chosen": -24.042566299438477,
      "logps/rejected": -29.772445678710938,
      "loss": 0.6149,
      "losses/dpo": 0.6469910144805908,
      "losses/sft": 1.0151987075805664,
      "losses/total": 0.6469910144805908,
      "ref_logps/chosen": -21.836162567138672,
      "ref_logps/rejected": -25.394845962524414,
      "rewards/accuracies": 0.6328125,
      "rewards/chosen": -0.220640629529953,
      "rewards/margins": 0.2171194702386856,
      "rewards/rejected": -0.4377601146697998,
      "step": 221
    },
    {
      "epoch": 1.68,
      "learning_rate": 2.443820224719101e-07,
      "logps/chosen": -24.834793090820312,
      "logps/rejected": -33.834083557128906,
      "loss": 0.5676,
      "losses/dpo": 0.6051491498947144,
      "losses/sft": 0.8380707502365112,
      "losses/total": 0.6051491498947144,
      "ref_logps/chosen": -22.940967559814453,
      "ref_logps/rejected": -28.49428939819336,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.18938273191452026,
      "rewards/margins": 0.3445969223976135,
      "rewards/rejected": -0.5339796543121338,
      "step": 222
    },
    {
      "epoch": 1.68,
      "learning_rate": 2.429775280898876e-07,
      "logps/chosen": -25.5327091217041,
      "logps/rejected": -30.429113388061523,
      "loss": 0.6089,
      "losses/dpo": 0.5853685140609741,
      "losses/sft": 0.6926910877227783,
      "losses/total": 0.5853685140609741,
      "ref_logps/chosen": -23.138214111328125,
      "ref_logps/rejected": -25.674575805664062,
      "rewards/accuracies": 0.7109375,
      "rewards/chosen": -0.23944953083992004,
      "rewards/margins": 0.2360040545463562,
      "rewards/rejected": -0.47545361518859863,
      "step": 223
    },
    {
      "epoch": 1.69,
      "learning_rate": 2.4157303370786517e-07,
      "logps/chosen": -24.123153686523438,
      "logps/rejected": -29.51090431213379,
      "loss": 0.6134,
      "losses/dpo": 0.7566800117492676,
      "losses/sft": 0.9139145612716675,
      "losses/total": 0.7566800117492676,
      "ref_logps/chosen": -22.065155029296875,
      "ref_logps/rejected": -24.94894027709961,
      "rewards/accuracies": 0.7109375,
      "rewards/chosen": -0.20579975843429565,
      "rewards/margins": 0.250396728515625,
      "rewards/rejected": -0.45619648694992065,
      "step": 224
    },
    {
      "epoch": 1.7,
      "learning_rate": 2.401685393258427e-07,
      "logps/chosen": -26.274799346923828,
      "logps/rejected": -32.90815734863281,
      "loss": 0.6145,
      "losses/dpo": 0.6078730225563049,
      "losses/sft": 1.1017650365829468,
      "losses/total": 0.6078730225563049,
      "ref_logps/chosen": -23.55907440185547,
      "ref_logps/rejected": -27.880718231201172,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.2715725004673004,
      "rewards/margins": 0.23117120563983917,
      "rewards/rejected": -0.5027437210083008,
      "step": 225
    },
    {
      "epoch": 1.71,
      "learning_rate": 2.3876404494382023e-07,
      "logps/chosen": -25.727689743041992,
      "logps/rejected": -30.410335540771484,
      "loss": 0.6292,
      "losses/dpo": 0.6031284332275391,
      "losses/sft": 0.7834776639938354,
      "losses/total": 0.6031284332275391,
      "ref_logps/chosen": -23.430198669433594,
      "ref_logps/rejected": -26.02400779724121,
      "rewards/accuracies": 0.6796875,
      "rewards/chosen": -0.22974896430969238,
      "rewards/margins": 0.20888389647006989,
      "rewards/rejected": -0.4386328458786011,
      "step": 226
    },
    {
      "epoch": 1.71,
      "learning_rate": 2.3735955056179774e-07,
      "logps/chosen": -25.917598724365234,
      "logps/rejected": -31.14261245727539,
      "loss": 0.5928,
      "losses/dpo": 0.5714601874351501,
      "losses/sft": 0.8888335227966309,
      "losses/total": 0.5714601874351501,
      "ref_logps/chosen": -23.436574935913086,
      "ref_logps/rejected": -25.756431579589844,
      "rewards/accuracies": 0.7265625,
      "rewards/chosen": -0.24810227751731873,
      "rewards/margins": 0.29051584005355835,
      "rewards/rejected": -0.5386180877685547,
      "step": 227
    },
    {
      "epoch": 1.72,
      "learning_rate": 2.3595505617977527e-07,
      "logps/chosen": -25.50743865966797,
      "logps/rejected": -34.945220947265625,
      "loss": 0.5505,
      "losses/dpo": 0.5715539455413818,
      "losses/sft": 0.8663308620452881,
      "losses/total": 0.5715539455413818,
      "ref_logps/chosen": -23.417984008789062,
      "ref_logps/rejected": -28.690208435058594,
      "rewards/accuracies": 0.8203125,
      "rewards/chosen": -0.20894566178321838,
      "rewards/margins": 0.41655558347702026,
      "rewards/rejected": -0.625501275062561,
      "step": 228
    },
    {
      "epoch": 1.73,
      "learning_rate": 2.345505617977528e-07,
      "logps/chosen": -23.620698928833008,
      "logps/rejected": -34.89327621459961,
      "loss": 0.571,
      "losses/dpo": 0.6053493022918701,
      "losses/sft": 0.8246825933456421,
      "losses/total": 0.6053493022918701,
      "ref_logps/chosen": -21.27004623413086,
      "ref_logps/rejected": -29.035568237304688,
      "rewards/accuracies": 0.7109375,
      "rewards/chosen": -0.23506540060043335,
      "rewards/margins": 0.35070547461509705,
      "rewards/rejected": -0.585770845413208,
      "step": 229
    },
    {
      "epoch": 1.74,
      "learning_rate": 2.331460674157303e-07,
      "logps/chosen": -21.874225616455078,
      "logps/rejected": -34.58841323852539,
      "loss": 0.5745,
      "losses/dpo": 0.5964910984039307,
      "losses/sft": 0.842921793460846,
      "losses/total": 0.5964910984039307,
      "ref_logps/chosen": -19.500164031982422,
      "ref_logps/rejected": -28.771209716796875,
      "rewards/accuracies": 0.6796875,
      "rewards/chosen": -0.23740598559379578,
      "rewards/margins": 0.34431448578834534,
      "rewards/rejected": -0.5817204713821411,
      "step": 230
    },
    {
      "epoch": 1.74,
      "learning_rate": 2.3174157303370786e-07,
      "logps/chosen": -24.84224510192871,
      "logps/rejected": -32.233497619628906,
      "loss": 0.6064,
      "losses/dpo": 0.5861349105834961,
      "losses/sft": 0.9263943433761597,
      "losses/total": 0.5861349105834961,
      "ref_logps/chosen": -21.88359832763672,
      "ref_logps/rejected": -26.701745986938477,
      "rewards/accuracies": 0.6796875,
      "rewards/chosen": -0.29586488008499146,
      "rewards/margins": 0.2573099732398987,
      "rewards/rejected": -0.5531748533248901,
      "step": 231
    },
    {
      "epoch": 1.75,
      "learning_rate": 2.303370786516854e-07,
      "logps/chosen": -25.4254207611084,
      "logps/rejected": -34.96025085449219,
      "loss": 0.5747,
      "losses/dpo": 0.5563768744468689,
      "losses/sft": 0.9355225563049316,
      "losses/total": 0.5563768744468689,
      "ref_logps/chosen": -22.772850036621094,
      "ref_logps/rejected": -28.902484893798828,
      "rewards/accuracies": 0.734375,
      "rewards/chosen": -0.26525697112083435,
      "rewards/margins": 0.3405200242996216,
      "rewards/rejected": -0.6057769656181335,
      "step": 232
    },
    {
      "epoch": 1.76,
      "learning_rate": 2.2893258426966292e-07,
      "logps/chosen": -26.856834411621094,
      "logps/rejected": -34.212364196777344,
      "loss": 0.6228,
      "losses/dpo": 0.6681157946586609,
      "losses/sft": 1.0442770719528198,
      "losses/total": 0.6681157946586609,
      "ref_logps/chosen": -23.627426147460938,
      "ref_logps/rejected": -28.62677001953125,
      "rewards/accuracies": 0.640625,
      "rewards/chosen": -0.32294073700904846,
      "rewards/margins": 0.23561875522136688,
      "rewards/rejected": -0.5585595369338989,
      "step": 233
    },
    {
      "epoch": 1.77,
      "learning_rate": 2.2752808988764045e-07,
      "logps/chosen": -26.366958618164062,
      "logps/rejected": -33.41276550292969,
      "loss": 0.6217,
      "losses/dpo": 0.6866650581359863,
      "losses/sft": 0.8693393468856812,
      "losses/total": 0.6866650581359863,
      "ref_logps/chosen": -23.189382553100586,
      "ref_logps/rejected": -27.676807403564453,
      "rewards/accuracies": 0.703125,
      "rewards/chosen": -0.31775763630867004,
      "rewards/margins": 0.25583818554878235,
      "rewards/rejected": -0.5735958218574524,
      "step": 234
    },
    {
      "epoch": 1.77,
      "learning_rate": 2.2612359550561795e-07,
      "logps/chosen": -24.26227569580078,
      "logps/rejected": -32.4229736328125,
      "loss": 0.604,
      "losses/dpo": 0.5642524361610413,
      "losses/sft": 0.9980260133743286,
      "losses/total": 0.5642524361610413,
      "ref_logps/chosen": -21.425315856933594,
      "ref_logps/rejected": -26.820331573486328,
      "rewards/accuracies": 0.609375,
      "rewards/chosen": -0.28369593620300293,
      "rewards/margins": 0.2765684127807617,
      "rewards/rejected": -0.5602643489837646,
      "step": 235
    },
    {
      "epoch": 1.78,
      "learning_rate": 2.2471910112359549e-07,
      "logps/chosen": -27.912431716918945,
      "logps/rejected": -31.85492706298828,
      "loss": 0.6448,
      "losses/dpo": 0.5940742492675781,
      "losses/sft": 0.969171404838562,
      "losses/total": 0.5940742492675781,
      "ref_logps/chosen": -24.59956932067871,
      "ref_logps/rejected": -26.790037155151367,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.3312861919403076,
      "rewards/margins": 0.17520278692245483,
      "rewards/rejected": -0.5064890384674072,
      "step": 236
    },
    {
      "epoch": 1.79,
      "learning_rate": 2.2331460674157302e-07,
      "logps/chosen": -27.303508758544922,
      "logps/rejected": -37.65882110595703,
      "loss": 0.5545,
      "losses/dpo": 0.5936781764030457,
      "losses/sft": 1.015429139137268,
      "losses/total": 0.5936781764030457,
      "ref_logps/chosen": -24.510639190673828,
      "ref_logps/rejected": -30.55707550048828,
      "rewards/accuracies": 0.796875,
      "rewards/chosen": -0.27928683161735535,
      "rewards/margins": 0.4308881163597107,
      "rewards/rejected": -0.7101750373840332,
      "step": 237
    },
    {
      "epoch": 1.8,
      "learning_rate": 2.2191011235955055e-07,
      "logps/chosen": -24.99541473388672,
      "logps/rejected": -30.256423950195312,
      "loss": 0.6034,
      "losses/dpo": 0.608791172504425,
      "losses/sft": 0.9114975929260254,
      "losses/total": 0.608791172504425,
      "ref_logps/chosen": -22.079914093017578,
      "ref_logps/rejected": -24.779722213745117,
      "rewards/accuracies": 0.6953125,
      "rewards/chosen": -0.2915502190589905,
      "rewards/margins": 0.2561199963092804,
      "rewards/rejected": -0.5476702451705933,
      "step": 238
    },
    {
      "epoch": 1.8,
      "learning_rate": 2.205056179775281e-07,
      "logps/chosen": -27.542556762695312,
      "logps/rejected": -34.17859649658203,
      "loss": 0.574,
      "losses/dpo": 0.5037014484405518,
      "losses/sft": 0.8922078609466553,
      "losses/total": 0.5037014484405518,
      "ref_logps/chosen": -24.774127960205078,
      "ref_logps/rejected": -27.759735107421875,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.2768429219722748,
      "rewards/margins": 0.3650434911251068,
      "rewards/rejected": -0.6418864727020264,
      "step": 239
    },
    {
      "epoch": 1.81,
      "learning_rate": 2.191011235955056e-07,
      "logps/chosen": -25.87149429321289,
      "logps/rejected": -34.46807861328125,
      "loss": 0.6117,
      "losses/dpo": 0.7050824165344238,
      "losses/sft": 0.9497538208961487,
      "losses/total": 0.7050824165344238,
      "ref_logps/chosen": -23.14657211303711,
      "ref_logps/rejected": -28.885162353515625,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.27249252796173096,
      "rewards/margins": 0.2857990562915802,
      "rewards/rejected": -0.5582915544509888,
      "step": 240
    },
    {
      "epoch": 1.82,
      "learning_rate": 2.1769662921348314e-07,
      "logps/chosen": -24.89635467529297,
      "logps/rejected": -33.622718811035156,
      "loss": 0.5808,
      "losses/dpo": 0.5883455276489258,
      "losses/sft": 0.9948925375938416,
      "losses/total": 0.5883455276489258,
      "ref_logps/chosen": -21.52194595336914,
      "ref_logps/rejected": -26.93505859375,
      "rewards/accuracies": 0.7265625,
      "rewards/chosen": -0.3374406695365906,
      "rewards/margins": 0.3313255310058594,
      "rewards/rejected": -0.66876620054245,
      "step": 241
    },
    {
      "epoch": 1.83,
      "learning_rate": 2.1629213483146067e-07,
      "logps/chosen": -24.833309173583984,
      "logps/rejected": -30.974327087402344,
      "loss": 0.6136,
      "losses/dpo": 0.6376237869262695,
      "losses/sft": 0.9374114274978638,
      "losses/total": 0.6376237869262695,
      "ref_logps/chosen": -21.7708683013916,
      "ref_logps/rejected": -25.24457359313965,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.30624428391456604,
      "rewards/margins": 0.2667309045791626,
      "rewards/rejected": -0.5729751586914062,
      "step": 242
    },
    {
      "epoch": 1.83,
      "learning_rate": 2.148876404494382e-07,
      "logps/chosen": -24.04471778869629,
      "logps/rejected": -34.610633850097656,
      "loss": 0.6133,
      "losses/dpo": 0.645912766456604,
      "losses/sft": 0.9913955926895142,
      "losses/total": 0.645912766456604,
      "ref_logps/chosen": -20.834651947021484,
      "ref_logps/rejected": -28.77642059326172,
      "rewards/accuracies": 0.6953125,
      "rewards/chosen": -0.32100653648376465,
      "rewards/margins": 0.26241475343704224,
      "rewards/rejected": -0.5834212899208069,
      "step": 243
    },
    {
      "epoch": 1.84,
      "learning_rate": 2.134831460674157e-07,
      "logps/chosen": -26.419416427612305,
      "logps/rejected": -34.56787109375,
      "loss": 0.5713,
      "losses/dpo": 0.6227866411209106,
      "losses/sft": 0.9809292554855347,
      "losses/total": 0.6227866411209106,
      "ref_logps/chosen": -23.278644561767578,
      "ref_logps/rejected": -27.596946716308594,
      "rewards/accuracies": 0.7265625,
      "rewards/chosen": -0.3140770494937897,
      "rewards/margins": 0.38301563262939453,
      "rewards/rejected": -0.6970926523208618,
      "step": 244
    },
    {
      "epoch": 1.85,
      "learning_rate": 2.1207865168539323e-07,
      "logps/chosen": -26.64739990234375,
      "logps/rejected": -33.21559524536133,
      "loss": 0.59,
      "losses/dpo": 0.6351089477539062,
      "losses/sft": 0.9912072420120239,
      "losses/total": 0.6351089477539062,
      "ref_logps/chosen": -23.281349182128906,
      "ref_logps/rejected": -26.577198028564453,
      "rewards/accuracies": 0.7109375,
      "rewards/chosen": -0.3366050124168396,
      "rewards/margins": 0.32723480463027954,
      "rewards/rejected": -0.6638398170471191,
      "step": 245
    },
    {
      "epoch": 1.86,
      "learning_rate": 2.1067415730337076e-07,
      "logps/chosen": -27.422582626342773,
      "logps/rejected": -35.08824920654297,
      "loss": 0.6064,
      "losses/dpo": 0.5233859419822693,
      "losses/sft": 0.8136109709739685,
      "losses/total": 0.5233859419822693,
      "ref_logps/chosen": -24.274629592895508,
      "ref_logps/rejected": -28.790220260620117,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.3147951364517212,
      "rewards/margins": 0.3150079846382141,
      "rewards/rejected": -0.6298030614852905,
      "step": 246
    },
    {
      "epoch": 1.86,
      "learning_rate": 2.0926966292134832e-07,
      "logps/chosen": -26.381507873535156,
      "logps/rejected": -31.576181411743164,
      "loss": 0.5829,
      "losses/dpo": 0.5970532894134521,
      "losses/sft": 0.8552703261375427,
      "losses/total": 0.5970532894134521,
      "ref_logps/chosen": -23.155136108398438,
      "ref_logps/rejected": -24.94633674621582,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.32263678312301636,
      "rewards/margins": 0.34034764766693115,
      "rewards/rejected": -0.6629844903945923,
      "step": 247
    },
    {
      "epoch": 1.87,
      "learning_rate": 2.0786516853932585e-07,
      "logps/chosen": -24.061811447143555,
      "logps/rejected": -29.508312225341797,
      "loss": 0.6137,
      "losses/dpo": 0.6248607039451599,
      "losses/sft": 0.8072177767753601,
      "losses/total": 0.6248607039451599,
      "ref_logps/chosen": -20.497760772705078,
      "ref_logps/rejected": -23.47817611694336,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.35640496015548706,
      "rewards/margins": 0.24660846590995789,
      "rewards/rejected": -0.6030134558677673,
      "step": 248
    },
    {
      "epoch": 1.88,
      "learning_rate": 2.0646067415730336e-07,
      "logps/chosen": -29.165149688720703,
      "logps/rejected": -35.16246032714844,
      "loss": 0.5826,
      "losses/dpo": 0.5271694660186768,
      "losses/sft": 1.0120395421981812,
      "losses/total": 0.5271694660186768,
      "ref_logps/chosen": -25.856834411621094,
      "ref_logps/rejected": -28.48740005493164,
      "rewards/accuracies": 0.7421875,
      "rewards/chosen": -0.3308315873146057,
      "rewards/margins": 0.33667463064193726,
      "rewards/rejected": -0.667506217956543,
      "step": 249
    },
    {
      "epoch": 1.89,
      "learning_rate": 2.0505617977528089e-07,
      "logps/chosen": -26.1055965423584,
      "logps/rejected": -36.45195770263672,
      "loss": 0.5345,
      "losses/dpo": 0.5425952076911926,
      "losses/sft": 0.9156839847564697,
      "losses/total": 0.5425952076911926,
      "ref_logps/chosen": -23.200654983520508,
      "ref_logps/rejected": -28.661373138427734,
      "rewards/accuracies": 0.7578125,
      "rewards/chosen": -0.2904941737651825,
      "rewards/margins": 0.48856407403945923,
      "rewards/rejected": -0.7790582776069641,
      "step": 250
    },
    {
      "epoch": 1.89,
      "learning_rate": 2.0365168539325842e-07,
      "logps/chosen": -24.59746551513672,
      "logps/rejected": -36.00947570800781,
      "loss": 0.5622,
      "losses/dpo": 0.6595858335494995,
      "losses/sft": 0.8320033550262451,
      "losses/total": 0.6595858335494995,
      "ref_logps/chosen": -21.081745147705078,
      "ref_logps/rejected": -28.422481536865234,
      "rewards/accuracies": 0.703125,
      "rewards/chosen": -0.3515721559524536,
      "rewards/margins": 0.4071270823478699,
      "rewards/rejected": -0.7586992383003235,
      "step": 251
    },
    {
      "epoch": 1.9,
      "learning_rate": 2.0224719101123595e-07,
      "logps/chosen": -25.407838821411133,
      "logps/rejected": -33.07604217529297,
      "loss": 0.5892,
      "losses/dpo": 0.5324288606643677,
      "losses/sft": 1.0311552286148071,
      "losses/total": 0.5324288606643677,
      "ref_logps/chosen": -22.188087463378906,
      "ref_logps/rejected": -26.633270263671875,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.3219751715660095,
      "rewards/margins": 0.322301983833313,
      "rewards/rejected": -0.6442771553993225,
      "step": 252
    },
    {
      "epoch": 1.91,
      "learning_rate": 2.0084269662921348e-07,
      "logps/chosen": -26.190311431884766,
      "logps/rejected": -33.34137725830078,
      "loss": 0.5861,
      "losses/dpo": 0.6612842082977295,
      "losses/sft": 0.8551939129829407,
      "losses/total": 0.6612842082977295,
      "ref_logps/chosen": -22.73942756652832,
      "ref_logps/rejected": -26.702760696411133,
      "rewards/accuracies": 0.671875,
      "rewards/chosen": -0.345088392496109,
      "rewards/margins": 0.3187733292579651,
      "rewards/rejected": -0.6638616919517517,
      "step": 253
    },
    {
      "epoch": 1.92,
      "learning_rate": 1.9943820224719098e-07,
      "logps/chosen": -27.615928649902344,
      "logps/rejected": -33.776695251464844,
      "loss": 0.5511,
      "losses/dpo": 0.6082693338394165,
      "losses/sft": 1.0973209142684937,
      "losses/total": 0.6082693338394165,
      "ref_logps/chosen": -24.300251007080078,
      "ref_logps/rejected": -26.21588897705078,
      "rewards/accuracies": 0.7578125,
      "rewards/chosen": -0.3315678834915161,
      "rewards/margins": 0.42451295256614685,
      "rewards/rejected": -0.7560808658599854,
      "step": 254
    },
    {
      "epoch": 1.92,
      "learning_rate": 1.9803370786516854e-07,
      "logps/chosen": -28.257335662841797,
      "logps/rejected": -35.67947769165039,
      "loss": 0.5919,
      "losses/dpo": 0.6389520168304443,
      "losses/sft": 1.087360143661499,
      "losses/total": 0.6389520168304443,
      "ref_logps/chosen": -24.031015396118164,
      "ref_logps/rejected": -28.11650276184082,
      "rewards/accuracies": 0.6796875,
      "rewards/chosen": -0.4226321578025818,
      "rewards/margins": 0.33366525173187256,
      "rewards/rejected": -0.7562973499298096,
      "step": 255
    },
    {
      "epoch": 1.93,
      "learning_rate": 1.9662921348314607e-07,
      "logps/chosen": -27.326435089111328,
      "logps/rejected": -33.91853713989258,
      "loss": 0.5884,
      "losses/dpo": 0.5772832632064819,
      "losses/sft": 1.0057258605957031,
      "losses/total": 0.5772832632064819,
      "ref_logps/chosen": -23.13665008544922,
      "ref_logps/rejected": -26.448516845703125,
      "rewards/accuracies": 0.6796875,
      "rewards/chosen": -0.4189784526824951,
      "rewards/margins": 0.32802364230155945,
      "rewards/rejected": -0.747002124786377,
      "step": 256
    },
    {
      "epoch": 1.94,
      "learning_rate": 1.952247191011236e-07,
      "logps/chosen": -24.134462356567383,
      "logps/rejected": -34.923095703125,
      "loss": 0.5245,
      "losses/dpo": 0.5826983451843262,
      "losses/sft": 0.7670709490776062,
      "losses/total": 0.5826983451843262,
      "ref_logps/chosen": -21.35719108581543,
      "ref_logps/rejected": -27.226768493652344,
      "rewards/accuracies": 0.7578125,
      "rewards/chosen": -0.2777270972728729,
      "rewards/margins": 0.49190521240234375,
      "rewards/rejected": -0.7696323394775391,
      "step": 257
    },
    {
      "epoch": 1.95,
      "learning_rate": 1.938202247191011e-07,
      "logps/chosen": -24.894744873046875,
      "logps/rejected": -36.34782791137695,
      "loss": 0.5654,
      "losses/dpo": 0.5832593441009521,
      "losses/sft": 0.8260340094566345,
      "losses/total": 0.5832593441009521,
      "ref_logps/chosen": -21.5096435546875,
      "ref_logps/rejected": -28.905296325683594,
      "rewards/accuracies": 0.7734375,
      "rewards/chosen": -0.3385101854801178,
      "rewards/margins": 0.4057431221008301,
      "rewards/rejected": -0.7442533373832703,
      "step": 258
    },
    {
      "epoch": 1.95,
      "learning_rate": 1.9241573033707863e-07,
      "logps/chosen": -29.12051773071289,
      "logps/rejected": -33.72222900390625,
      "loss": 0.6189,
      "losses/dpo": 0.5586456060409546,
      "losses/sft": 1.1363164186477661,
      "losses/total": 0.5586456060409546,
      "ref_logps/chosen": -24.818958282470703,
      "ref_logps/rejected": -26.637435913085938,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.430155873298645,
      "rewards/margins": 0.27832359075546265,
      "rewards/rejected": -0.7084795236587524,
      "step": 259
    },
    {
      "epoch": 1.96,
      "learning_rate": 1.9101123595505617e-07,
      "logps/chosen": -25.77654266357422,
      "logps/rejected": -32.80144119262695,
      "loss": 0.5647,
      "losses/dpo": 0.6132915616035461,
      "losses/sft": 0.8355939984321594,
      "losses/total": 0.6132915616035461,
      "ref_logps/chosen": -22.049232482910156,
      "ref_logps/rejected": -25.218961715698242,
      "rewards/accuracies": 0.765625,
      "rewards/chosen": -0.37273097038269043,
      "rewards/margins": 0.3855169415473938,
      "rewards/rejected": -0.758247971534729,
      "step": 260
    },
    {
      "epoch": 1.97,
      "learning_rate": 1.896067415730337e-07,
      "logps/chosen": -27.173105239868164,
      "logps/rejected": -33.18789291381836,
      "loss": 0.5757,
      "losses/dpo": 0.6402326822280884,
      "losses/sft": 0.9358000159263611,
      "losses/total": 0.6402326822280884,
      "ref_logps/chosen": -24.05023956298828,
      "ref_logps/rejected": -26.557050704956055,
      "rewards/accuracies": 0.6796875,
      "rewards/chosen": -0.3122865557670593,
      "rewards/margins": 0.3507978618144989,
      "rewards/rejected": -0.6630844473838806,
      "step": 261
    },
    {
      "epoch": 1.98,
      "learning_rate": 1.8820224719101123e-07,
      "logps/chosen": -25.127092361450195,
      "logps/rejected": -34.0608024597168,
      "loss": 0.5844,
      "losses/dpo": 0.576771080493927,
      "losses/sft": 0.8823024034500122,
      "losses/total": 0.576771080493927,
      "ref_logps/chosen": -21.552627563476562,
      "ref_logps/rejected": -27.01715087890625,
      "rewards/accuracies": 0.671875,
      "rewards/chosen": -0.35744667053222656,
      "rewards/margins": 0.346918523311615,
      "rewards/rejected": -0.7043651938438416,
      "step": 262
    },
    {
      "epoch": 1.98,
      "learning_rate": 1.8679775280898876e-07,
      "logps/chosen": -25.840179443359375,
      "logps/rejected": -34.11262893676758,
      "loss": 0.5675,
      "losses/dpo": 0.5643225312232971,
      "losses/sft": 0.7924672365188599,
      "losses/total": 0.5643225312232971,
      "ref_logps/chosen": -22.366439819335938,
      "ref_logps/rejected": -26.873088836669922,
      "rewards/accuracies": 0.7109375,
      "rewards/chosen": -0.3473738133907318,
      "rewards/margins": 0.37658050656318665,
      "rewards/rejected": -0.7239543199539185,
      "step": 263
    },
    {
      "epoch": 1.99,
      "learning_rate": 1.853932584269663e-07,
      "logps/chosen": -24.64289093017578,
      "logps/rejected": -30.944011688232422,
      "loss": 0.5768,
      "losses/dpo": 0.6149911880493164,
      "losses/sft": 0.9512190222740173,
      "losses/total": 0.6149911880493164,
      "ref_logps/chosen": -21.270837783813477,
      "ref_logps/rejected": -24.173620223999023,
      "rewards/accuracies": 0.734375,
      "rewards/chosen": -0.33720535039901733,
      "rewards/margins": 0.3398338854312897,
      "rewards/rejected": -0.6770392656326294,
      "step": 264
    },
    {
      "epoch": 2.0,
      "learning_rate": 1.8398876404494382e-07,
      "logps/chosen": -26.05956268310547,
      "logps/rejected": -35.905609130859375,
      "loss": 0.5407,
      "losses/dpo": 0.49823397397994995,
      "losses/sft": 0.8145182132720947,
      "losses/total": 0.49823397397994995,
      "ref_logps/chosen": -23.054027557373047,
      "ref_logps/rejected": -28.27884864807129,
      "rewards/accuracies": 0.796875,
      "rewards/chosen": -0.3005535304546356,
      "rewards/margins": 0.4621226191520691,
      "rewards/rejected": -0.7626761794090271,
      "step": 265
    },
    {
      "epoch": 2.01,
      "learning_rate": 1.8258426966292135e-07,
      "logps/chosen": -24.907108306884766,
      "logps/rejected": -33.71357345581055,
      "loss": 0.5301,
      "losses/dpo": 0.49924543499946594,
      "losses/sft": 0.9444026350975037,
      "losses/total": 0.49924543499946594,
      "ref_logps/chosen": -21.562763214111328,
      "ref_logps/rejected": -25.69823455810547,
      "rewards/accuracies": 0.7578125,
      "rewards/chosen": -0.3344343304634094,
      "rewards/margins": 0.4670996069908142,
      "rewards/rejected": -0.8015338778495789,
      "step": 266
    },
    {
      "epoch": 2.02,
      "learning_rate": 1.8117977528089888e-07,
      "logps/chosen": -24.61281967163086,
      "logps/rejected": -33.178436279296875,
      "loss": 0.5843,
      "losses/dpo": 0.6827691793441772,
      "losses/sft": 0.9820384979248047,
      "losses/total": 0.6827691793441772,
      "ref_logps/chosen": -20.769065856933594,
      "ref_logps/rejected": -25.740190505981445,
      "rewards/accuracies": 0.6953125,
      "rewards/chosen": -0.3843753933906555,
      "rewards/margins": 0.35944926738739014,
      "rewards/rejected": -0.7438246607780457,
      "step": 267
    },
    {
      "epoch": 2.02,
      "learning_rate": 1.7977528089887638e-07,
      "logps/chosen": -25.742042541503906,
      "logps/rejected": -31.92254638671875,
      "loss": 0.5765,
      "losses/dpo": 0.48391562700271606,
      "losses/sft": 0.9694733619689941,
      "losses/total": 0.48391562700271606,
      "ref_logps/chosen": -22.27023696899414,
      "ref_logps/rejected": -24.869842529296875,
      "rewards/accuracies": 0.734375,
      "rewards/chosen": -0.34718072414398193,
      "rewards/margins": 0.3580899238586426,
      "rewards/rejected": -0.7052706480026245,
      "step": 268
    },
    {
      "epoch": 2.03,
      "learning_rate": 1.7837078651685391e-07,
      "logps/chosen": -24.76668930053711,
      "logps/rejected": -33.92596435546875,
      "loss": 0.5197,
      "losses/dpo": 0.566383957862854,
      "losses/sft": 1.056198239326477,
      "losses/total": 0.566383957862854,
      "ref_logps/chosen": -21.79462432861328,
      "ref_logps/rejected": -26.1394100189209,
      "rewards/accuracies": 0.796875,
      "rewards/chosen": -0.2972065806388855,
      "rewards/margins": 0.48144853115081787,
      "rewards/rejected": -0.7786551713943481,
      "step": 269
    },
    {
      "epoch": 2.04,
      "learning_rate": 1.7696629213483144e-07,
      "logps/chosen": -25.022621154785156,
      "logps/rejected": -32.83625030517578,
      "loss": 0.554,
      "losses/dpo": 0.5455434322357178,
      "losses/sft": 0.9091237783432007,
      "losses/total": 0.5455434322357178,
      "ref_logps/chosen": -21.205692291259766,
      "ref_logps/rejected": -24.853519439697266,
      "rewards/accuracies": 0.765625,
      "rewards/chosen": -0.3816927969455719,
      "rewards/margins": 0.4165803790092468,
      "rewards/rejected": -0.7982731461524963,
      "step": 270
    },
    {
      "epoch": 2.05,
      "learning_rate": 1.75561797752809e-07,
      "logps/chosen": -27.038639068603516,
      "logps/rejected": -35.007415771484375,
      "loss": 0.5526,
      "losses/dpo": 0.7876778841018677,
      "losses/sft": 1.1023296117782593,
      "losses/total": 0.7876778841018677,
      "ref_logps/chosen": -23.039878845214844,
      "ref_logps/rejected": -26.884708404541016,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.39987578988075256,
      "rewards/margins": 0.41239458322525024,
      "rewards/rejected": -0.8122704029083252,
      "step": 271
    },
    {
      "epoch": 2.05,
      "learning_rate": 1.741573033707865e-07,
      "logps/chosen": -26.300579071044922,
      "logps/rejected": -36.53676223754883,
      "loss": 0.5444,
      "losses/dpo": 0.4805631637573242,
      "losses/sft": 0.8787716031074524,
      "losses/total": 0.4805631637573242,
      "ref_logps/chosen": -22.55372428894043,
      "ref_logps/rejected": -27.711929321289062,
      "rewards/accuracies": 0.7578125,
      "rewards/chosen": -0.3746855556964874,
      "rewards/margins": 0.5077978372573853,
      "rewards/rejected": -0.8824833631515503,
      "step": 272
    },
    {
      "epoch": 2.06,
      "learning_rate": 1.7275280898876404e-07,
      "logps/chosen": -27.612911224365234,
      "logps/rejected": -39.48854064941406,
      "loss": 0.4883,
      "losses/dpo": 0.5499591827392578,
      "losses/sft": 1.1995720863342285,
      "losses/total": 0.5499591827392578,
      "ref_logps/chosen": -23.866586685180664,
      "ref_logps/rejected": -29.748516082763672,
      "rewards/accuracies": 0.828125,
      "rewards/chosen": -0.37463241815567017,
      "rewards/margins": 0.5993699431419373,
      "rewards/rejected": -0.9740023612976074,
      "step": 273
    },
    {
      "epoch": 2.07,
      "learning_rate": 1.7134831460674157e-07,
      "logps/chosen": -28.848485946655273,
      "logps/rejected": -38.46211242675781,
      "loss": 0.5223,
      "losses/dpo": 0.5853086113929749,
      "losses/sft": 0.9450937509536743,
      "losses/total": 0.5853086113929749,
      "ref_logps/chosen": -24.71368980407715,
      "ref_logps/rejected": -29.095096588134766,
      "rewards/accuracies": 0.796875,
      "rewards/chosen": -0.4134795069694519,
      "rewards/margins": 0.5232220888137817,
      "rewards/rejected": -0.9367015957832336,
      "step": 274
    },
    {
      "epoch": 2.08,
      "learning_rate": 1.699438202247191e-07,
      "logps/chosen": -26.53584861755371,
      "logps/rejected": -33.2642707824707,
      "loss": 0.5583,
      "losses/dpo": 0.6550332307815552,
      "losses/sft": 0.844421923160553,
      "losses/total": 0.6550332307815552,
      "ref_logps/chosen": -22.528621673583984,
      "ref_logps/rejected": -25.07666778564453,
      "rewards/accuracies": 0.765625,
      "rewards/chosen": -0.4007226824760437,
      "rewards/margins": 0.4180375933647156,
      "rewards/rejected": -0.8187602758407593,
      "step": 275
    },
    {
      "epoch": 2.08,
      "learning_rate": 1.6853932584269663e-07,
      "logps/chosen": -26.93305778503418,
      "logps/rejected": -36.43919372558594,
      "loss": 0.5267,
      "losses/dpo": 0.37509262561798096,
      "losses/sft": 0.9286944270133972,
      "losses/total": 0.37509262561798096,
      "ref_logps/chosen": -22.965662002563477,
      "ref_logps/rejected": -27.516300201416016,
      "rewards/accuracies": 0.765625,
      "rewards/chosen": -0.39673954248428345,
      "rewards/margins": 0.4955495595932007,
      "rewards/rejected": -0.8922891616821289,
      "step": 276
    },
    {
      "epoch": 2.09,
      "learning_rate": 1.6713483146067413e-07,
      "logps/chosen": -27.517230987548828,
      "logps/rejected": -33.23160934448242,
      "loss": 0.585,
      "losses/dpo": 0.45891374349594116,
      "losses/sft": 0.8818660378456116,
      "losses/total": 0.45891374349594116,
      "ref_logps/chosen": -23.37508773803711,
      "ref_logps/rejected": -25.319503784179688,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.4142143726348877,
      "rewards/margins": 0.37699633836746216,
      "rewards/rejected": -0.7912107110023499,
      "step": 277
    },
    {
      "epoch": 2.1,
      "learning_rate": 1.6573033707865166e-07,
      "logps/chosen": -29.848949432373047,
      "logps/rejected": -37.009605407714844,
      "loss": 0.5569,
      "losses/dpo": 0.6695871353149414,
      "losses/sft": 1.1478632688522339,
      "losses/total": 0.6695871353149414,
      "ref_logps/chosen": -25.79513168334961,
      "ref_logps/rejected": -28.52492332458496,
      "rewards/accuracies": 0.7421875,
      "rewards/chosen": -0.4053817391395569,
      "rewards/margins": 0.4430864751338959,
      "rewards/rejected": -0.8484681844711304,
      "step": 278
    },
    {
      "epoch": 2.11,
      "learning_rate": 1.6432584269662922e-07,
      "logps/chosen": -26.847461700439453,
      "logps/rejected": -33.84664535522461,
      "loss": 0.5853,
      "losses/dpo": 0.6266674995422363,
      "losses/sft": 0.9419240951538086,
      "losses/total": 0.6266674995422363,
      "ref_logps/chosen": -23.1273193359375,
      "ref_logps/rejected": -26.497238159179688,
      "rewards/accuracies": 0.671875,
      "rewards/chosen": -0.3720143437385559,
      "rewards/margins": 0.3629264533519745,
      "rewards/rejected": -0.734940767288208,
      "step": 279
    },
    {
      "epoch": 2.11,
      "learning_rate": 1.6292134831460675e-07,
      "logps/chosen": -25.227951049804688,
      "logps/rejected": -37.79768371582031,
      "loss": 0.5277,
      "losses/dpo": 0.5965819358825684,
      "losses/sft": 1.0364360809326172,
      "losses/total": 0.5965819358825684,
      "ref_logps/chosen": -20.85896873474121,
      "ref_logps/rejected": -28.49602508544922,
      "rewards/accuracies": 0.7890625,
      "rewards/chosen": -0.43689805269241333,
      "rewards/margins": 0.49326756596565247,
      "rewards/rejected": -0.9301656484603882,
      "step": 280
    },
    {
      "epoch": 2.12,
      "learning_rate": 1.6151685393258428e-07,
      "logps/chosen": -25.945070266723633,
      "logps/rejected": -35.28973388671875,
      "loss": 0.5305,
      "losses/dpo": 0.5456879138946533,
      "losses/sft": 0.8692267537117004,
      "losses/total": 0.5456879138946533,
      "ref_logps/chosen": -22.303909301757812,
      "ref_logps/rejected": -26.77994155883789,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.36411628127098083,
      "rewards/margins": 0.4868628680706024,
      "rewards/rejected": -0.850979208946228,
      "step": 281
    },
    {
      "epoch": 2.13,
      "learning_rate": 1.6011235955056178e-07,
      "logps/chosen": -28.660266876220703,
      "logps/rejected": -36.41142272949219,
      "loss": 0.5766,
      "losses/dpo": 0.6054384708404541,
      "losses/sft": 0.9599564671516418,
      "losses/total": 0.6054384708404541,
      "ref_logps/chosen": -24.156015396118164,
      "ref_logps/rejected": -28.24047088623047,
      "rewards/accuracies": 0.734375,
      "rewards/chosen": -0.4504254460334778,
      "rewards/margins": 0.36666956543922424,
      "rewards/rejected": -0.8170950412750244,
      "step": 282
    },
    {
      "epoch": 2.14,
      "learning_rate": 1.5870786516853931e-07,
      "logps/chosen": -27.74228858947754,
      "logps/rejected": -38.50691604614258,
      "loss": 0.5215,
      "losses/dpo": 0.463223397731781,
      "losses/sft": 1.041387915611267,
      "losses/total": 0.463223397731781,
      "ref_logps/chosen": -23.603931427001953,
      "ref_logps/rejected": -28.803909301757812,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.4138358533382416,
      "rewards/margins": 0.5564644932746887,
      "rewards/rejected": -0.9703004360198975,
      "step": 283
    },
    {
      "epoch": 2.14,
      "learning_rate": 1.5730337078651685e-07,
      "logps/chosen": -24.93131446838379,
      "logps/rejected": -35.91729736328125,
      "loss": 0.5266,
      "losses/dpo": 0.6279169321060181,
      "losses/sft": 0.8709256052970886,
      "losses/total": 0.6279169321060181,
      "ref_logps/chosen": -20.768774032592773,
      "ref_logps/rejected": -26.83118438720703,
      "rewards/accuracies": 0.7734375,
      "rewards/chosen": -0.4162542223930359,
      "rewards/margins": 0.4923573136329651,
      "rewards/rejected": -0.9086115956306458,
      "step": 284
    },
    {
      "epoch": 2.15,
      "learning_rate": 1.5589887640449438e-07,
      "logps/chosen": -27.571338653564453,
      "logps/rejected": -38.57915496826172,
      "loss": 0.5687,
      "losses/dpo": 0.5966840386390686,
      "losses/sft": 0.9412966966629028,
      "losses/total": 0.5966840386390686,
      "ref_logps/chosen": -22.682205200195312,
      "ref_logps/rejected": -29.58980369567871,
      "rewards/accuracies": 0.7265625,
      "rewards/chosen": -0.48891347646713257,
      "rewards/margins": 0.4100216031074524,
      "rewards/rejected": -0.898935079574585,
      "step": 285
    },
    {
      "epoch": 2.16,
      "learning_rate": 1.5449438202247188e-07,
      "logps/chosen": -25.46674346923828,
      "logps/rejected": -33.395118713378906,
      "loss": 0.5902,
      "losses/dpo": 0.7199227213859558,
      "losses/sft": 0.9989073276519775,
      "losses/total": 0.7199227213859558,
      "ref_logps/chosen": -21.129976272583008,
      "ref_logps/rejected": -25.61261749267578,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.4336766302585602,
      "rewards/margins": 0.3445735573768616,
      "rewards/rejected": -0.7782501578330994,
      "step": 286
    },
    {
      "epoch": 2.17,
      "learning_rate": 1.5308988764044944e-07,
      "logps/chosen": -26.559568405151367,
      "logps/rejected": -39.50514221191406,
      "loss": 0.5101,
      "losses/dpo": 0.42156773805618286,
      "losses/sft": 0.824786365032196,
      "losses/total": 0.42156773805618286,
      "ref_logps/chosen": -21.88895606994629,
      "ref_logps/rejected": -29.256093978881836,
      "rewards/accuracies": 0.8046875,
      "rewards/chosen": -0.46706122159957886,
      "rewards/margins": 0.557843804359436,
      "rewards/rejected": -1.0249050855636597,
      "step": 287
    },
    {
      "epoch": 2.17,
      "learning_rate": 1.5168539325842697e-07,
      "logps/chosen": -27.430574417114258,
      "logps/rejected": -35.846214294433594,
      "loss": 0.5852,
      "losses/dpo": 0.7073966264724731,
      "losses/sft": 0.959773600101471,
      "losses/total": 0.7073966264724731,
      "ref_logps/chosen": -22.33085060119629,
      "ref_logps/rejected": -26.85816192626953,
      "rewards/accuracies": 0.7109375,
      "rewards/chosen": -0.509972333908081,
      "rewards/margins": 0.38883259892463684,
      "rewards/rejected": -0.8988049030303955,
      "step": 288
    },
    {
      "epoch": 2.18,
      "learning_rate": 1.502808988764045e-07,
      "logps/chosen": -25.799861907958984,
      "logps/rejected": -37.50861358642578,
      "loss": 0.5553,
      "losses/dpo": 0.5419721603393555,
      "losses/sft": 0.940202534198761,
      "losses/total": 0.5419721603393555,
      "ref_logps/chosen": -21.224933624267578,
      "ref_logps/rejected": -28.002174377441406,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.4574929475784302,
      "rewards/margins": 0.4931509494781494,
      "rewards/rejected": -0.9506438970565796,
      "step": 289
    },
    {
      "epoch": 2.19,
      "learning_rate": 1.4887640449438203e-07,
      "logps/chosen": -27.79110336303711,
      "logps/rejected": -34.21430206298828,
      "loss": 0.5921,
      "losses/dpo": 0.6595107913017273,
      "losses/sft": 1.0057413578033447,
      "losses/total": 0.6595107913017273,
      "ref_logps/chosen": -23.073078155517578,
      "ref_logps/rejected": -26.093578338623047,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.4718025326728821,
      "rewards/margins": 0.34026968479156494,
      "rewards/rejected": -0.8120721578598022,
      "step": 290
    },
    {
      "epoch": 2.2,
      "learning_rate": 1.4747191011235953e-07,
      "logps/chosen": -26.380355834960938,
      "logps/rejected": -37.56932830810547,
      "loss": 0.5263,
      "losses/dpo": 0.47728973627090454,
      "losses/sft": 1.0133030414581299,
      "losses/total": 0.47728973627090454,
      "ref_logps/chosen": -22.488906860351562,
      "ref_logps/rejected": -28.34372329711914,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.38914480805397034,
      "rewards/margins": 0.5334160327911377,
      "rewards/rejected": -0.9225608110427856,
      "step": 291
    },
    {
      "epoch": 2.2,
      "learning_rate": 1.4606741573033706e-07,
      "logps/chosen": -27.006134033203125,
      "logps/rejected": -37.092594146728516,
      "loss": 0.5417,
      "losses/dpo": 0.7257384061813354,
      "losses/sft": 1.2120591402053833,
      "losses/total": 0.7257384061813354,
      "ref_logps/chosen": -22.130794525146484,
      "ref_logps/rejected": -27.460662841796875,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.48753368854522705,
      "rewards/margins": 0.47565943002700806,
      "rewards/rejected": -0.9631930589675903,
      "step": 292
    },
    {
      "epoch": 2.21,
      "learning_rate": 1.446629213483146e-07,
      "logps/chosen": -25.57880210876465,
      "logps/rejected": -37.16014099121094,
      "loss": 0.5381,
      "losses/dpo": 0.6313049793243408,
      "losses/sft": 0.9201721549034119,
      "losses/total": 0.6313049793243408,
      "ref_logps/chosen": -21.904296875,
      "ref_logps/rejected": -28.32394027709961,
      "rewards/accuracies": 0.6953125,
      "rewards/chosen": -0.3674505054950714,
      "rewards/margins": 0.5161697268486023,
      "rewards/rejected": -0.8836201429367065,
      "step": 293
    },
    {
      "epoch": 2.22,
      "learning_rate": 1.4325842696629212e-07,
      "logps/chosen": -24.601829528808594,
      "logps/rejected": -37.283538818359375,
      "loss": 0.5429,
      "losses/dpo": 0.5216307044029236,
      "losses/sft": 1.0138075351715088,
      "losses/total": 0.5216307044029236,
      "ref_logps/chosen": -20.36395263671875,
      "ref_logps/rejected": -28.042282104492188,
      "rewards/accuracies": 0.7734375,
      "rewards/chosen": -0.4237874746322632,
      "rewards/margins": 0.5003381967544556,
      "rewards/rejected": -0.9241256713867188,
      "step": 294
    },
    {
      "epoch": 2.23,
      "learning_rate": 1.4185393258426968e-07,
      "logps/chosen": -28.58258819580078,
      "logps/rejected": -36.47199249267578,
      "loss": 0.5894,
      "losses/dpo": 0.43838924169540405,
      "losses/sft": 1.2099077701568604,
      "losses/total": 0.43838924169540405,
      "ref_logps/chosen": -23.226455688476562,
      "ref_logps/rejected": -26.880578994750977,
      "rewards/accuracies": 0.6953125,
      "rewards/chosen": -0.5356131792068481,
      "rewards/margins": 0.423528254032135,
      "rewards/rejected": -0.9591414332389832,
      "step": 295
    },
    {
      "epoch": 2.23,
      "learning_rate": 1.4044943820224718e-07,
      "logps/chosen": -28.39300537109375,
      "logps/rejected": -36.5651741027832,
      "loss": 0.5472,
      "losses/dpo": 0.6286274790763855,
      "losses/sft": 1.1655751466751099,
      "losses/total": 0.6286274790763855,
      "ref_logps/chosen": -23.700767517089844,
      "ref_logps/rejected": -26.989849090576172,
      "rewards/accuracies": 0.7265625,
      "rewards/chosen": -0.4692240357398987,
      "rewards/margins": 0.48830845952033997,
      "rewards/rejected": -0.9575324058532715,
      "step": 296
    },
    {
      "epoch": 2.24,
      "learning_rate": 1.3904494382022472e-07,
      "logps/chosen": -27.01761245727539,
      "logps/rejected": -34.147830963134766,
      "loss": 0.5581,
      "losses/dpo": 0.425361692905426,
      "losses/sft": 1.129596471786499,
      "losses/total": 0.425361692905426,
      "ref_logps/chosen": -22.251991271972656,
      "ref_logps/rejected": -24.953710556030273,
      "rewards/accuracies": 0.671875,
      "rewards/chosen": -0.47656214237213135,
      "rewards/margins": 0.4428498148918152,
      "rewards/rejected": -0.9194119572639465,
      "step": 297
    },
    {
      "epoch": 2.25,
      "learning_rate": 1.3764044943820225e-07,
      "logps/chosen": -27.92938995361328,
      "logps/rejected": -39.81676483154297,
      "loss": 0.5111,
      "losses/dpo": 0.5578055381774902,
      "losses/sft": 1.1197444200515747,
      "losses/total": 0.5578055381774902,
      "ref_logps/chosen": -23.314014434814453,
      "ref_logps/rejected": -29.507539749145508,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.4615376591682434,
      "rewards/margins": 0.5693849325180054,
      "rewards/rejected": -1.0309226512908936,
      "step": 298
    },
    {
      "epoch": 2.26,
      "learning_rate": 1.3623595505617978e-07,
      "logps/chosen": -27.910144805908203,
      "logps/rejected": -35.69133758544922,
      "loss": 0.5499,
      "losses/dpo": 0.4847102165222168,
      "losses/sft": 0.989621639251709,
      "losses/total": 0.4847102165222168,
      "ref_logps/chosen": -23.326908111572266,
      "ref_logps/rejected": -26.762676239013672,
      "rewards/accuracies": 0.734375,
      "rewards/chosen": -0.45832377672195435,
      "rewards/margins": 0.43454277515411377,
      "rewards/rejected": -0.8928664922714233,
      "step": 299
    },
    {
      "epoch": 2.26,
      "learning_rate": 1.3483146067415728e-07,
      "logps/chosen": -28.233020782470703,
      "logps/rejected": -37.3542366027832,
      "loss": 0.5935,
      "losses/dpo": 0.5905570983886719,
      "losses/sft": 1.0464057922363281,
      "losses/total": 0.5905570983886719,
      "ref_logps/chosen": -23.20905303955078,
      "ref_logps/rejected": -28.49638557434082,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.5023964643478394,
      "rewards/margins": 0.38338857889175415,
      "rewards/rejected": -0.8857850432395935,
      "step": 300
    },
    {
      "epoch": 2.27,
      "learning_rate": 1.334269662921348e-07,
      "logps/chosen": -29.44438934326172,
      "logps/rejected": -36.25569152832031,
      "loss": 0.5608,
      "losses/dpo": 0.5518324375152588,
      "losses/sft": 0.9761526584625244,
      "losses/total": 0.5518324375152588,
      "ref_logps/chosen": -24.553022384643555,
      "ref_logps/rejected": -26.846464157104492,
      "rewards/accuracies": 0.7109375,
      "rewards/chosen": -0.4891367554664612,
      "rewards/margins": 0.451786071062088,
      "rewards/rejected": -0.9409228563308716,
      "step": 301
    },
    {
      "epoch": 2.28,
      "learning_rate": 1.3202247191011234e-07,
      "logps/chosen": -29.32979393005371,
      "logps/rejected": -37.83529281616211,
      "loss": 0.5463,
      "losses/dpo": 0.5125599503517151,
      "losses/sft": 0.9747940897941589,
      "losses/total": 0.5125599503517151,
      "ref_logps/chosen": -24.211776733398438,
      "ref_logps/rejected": -27.92011260986328,
      "rewards/accuracies": 0.734375,
      "rewards/chosen": -0.5118017196655273,
      "rewards/margins": 0.47971609234809875,
      "rewards/rejected": -0.9915178418159485,
      "step": 302
    },
    {
      "epoch": 2.29,
      "learning_rate": 1.306179775280899e-07,
      "logps/chosen": -27.11697769165039,
      "logps/rejected": -35.418338775634766,
      "loss": 0.575,
      "losses/dpo": 0.5703020095825195,
      "losses/sft": 0.9395530223846436,
      "losses/total": 0.5703020095825195,
      "ref_logps/chosen": -22.12503433227539,
      "ref_logps/rejected": -26.28829574584961,
      "rewards/accuracies": 0.7265625,
      "rewards/chosen": -0.49919426441192627,
      "rewards/margins": 0.4138101041316986,
      "rewards/rejected": -0.9130042791366577,
      "step": 303
    },
    {
      "epoch": 2.29,
      "learning_rate": 1.2921348314606743e-07,
      "logps/chosen": -31.24747085571289,
      "logps/rejected": -38.8961181640625,
      "loss": 0.5843,
      "losses/dpo": 0.4914831221103668,
      "losses/sft": 0.9517439603805542,
      "losses/total": 0.4914831221103668,
      "ref_logps/chosen": -25.351207733154297,
      "ref_logps/rejected": -29.048057556152344,
      "rewards/accuracies": 0.734375,
      "rewards/chosen": -0.5896263122558594,
      "rewards/margins": 0.3951793909072876,
      "rewards/rejected": -0.9848057627677917,
      "step": 304
    },
    {
      "epoch": 2.3,
      "learning_rate": 1.2780898876404493e-07,
      "logps/chosen": -27.757152557373047,
      "logps/rejected": -37.481964111328125,
      "loss": 0.5261,
      "losses/dpo": 0.4620394706726074,
      "losses/sft": 1.0134756565093994,
      "losses/total": 0.4620394706726074,
      "ref_logps/chosen": -23.13431167602539,
      "ref_logps/rejected": -27.492977142333984,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.4622839689254761,
      "rewards/margins": 0.5366144776344299,
      "rewards/rejected": -0.9988985061645508,
      "step": 305
    },
    {
      "epoch": 2.31,
      "learning_rate": 1.2640449438202246e-07,
      "logps/chosen": -26.990705490112305,
      "logps/rejected": -34.95043182373047,
      "loss": 0.5636,
      "losses/dpo": 0.5714951753616333,
      "losses/sft": 0.9859296679496765,
      "losses/total": 0.5714951753616333,
      "ref_logps/chosen": -22.465744018554688,
      "ref_logps/rejected": -26.120864868164062,
      "rewards/accuracies": 0.7421875,
      "rewards/chosen": -0.452495813369751,
      "rewards/margins": 0.4304611086845398,
      "rewards/rejected": -0.882956862449646,
      "step": 306
    },
    {
      "epoch": 2.32,
      "learning_rate": 1.25e-07,
      "logps/chosen": -29.839576721191406,
      "logps/rejected": -40.363712310791016,
      "loss": 0.5508,
      "losses/dpo": 0.5849748253822327,
      "losses/sft": 0.9925932288169861,
      "losses/total": 0.5849748253822327,
      "ref_logps/chosen": -24.343534469604492,
      "ref_logps/rejected": -29.49897003173828,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5496042370796204,
      "rewards/margins": 0.5368699431419373,
      "rewards/rejected": -1.0864741802215576,
      "step": 307
    },
    {
      "epoch": 2.32,
      "learning_rate": 1.2359550561797752e-07,
      "logps/chosen": -26.925609588623047,
      "logps/rejected": -35.485931396484375,
      "loss": 0.5537,
      "losses/dpo": 0.43898260593414307,
      "losses/sft": 0.8520787954330444,
      "losses/total": 0.43898260593414307,
      "ref_logps/chosen": -22.369102478027344,
      "ref_logps/rejected": -26.426111221313477,
      "rewards/accuracies": 0.7734375,
      "rewards/chosen": -0.4556504487991333,
      "rewards/margins": 0.450331449508667,
      "rewards/rejected": -0.9059818387031555,
      "step": 308
    },
    {
      "epoch": 2.33,
      "learning_rate": 1.2219101123595506e-07,
      "logps/chosen": -27.308425903320312,
      "logps/rejected": -34.86455535888672,
      "loss": 0.6099,
      "losses/dpo": 0.6877168416976929,
      "losses/sft": 0.8925371766090393,
      "losses/total": 0.6877168416976929,
      "ref_logps/chosen": -22.015674591064453,
      "ref_logps/rejected": -26.024978637695312,
      "rewards/accuracies": 0.6796875,
      "rewards/chosen": -0.5292750597000122,
      "rewards/margins": 0.3546826243400574,
      "rewards/rejected": -0.8839576840400696,
      "step": 309
    },
    {
      "epoch": 2.34,
      "learning_rate": 1.2078651685393259e-07,
      "logps/chosen": -27.23873519897461,
      "logps/rejected": -34.355613708496094,
      "loss": 0.5451,
      "losses/dpo": 0.4608323574066162,
      "losses/sft": 1.068372130393982,
      "losses/total": 0.4608323574066162,
      "ref_logps/chosen": -22.93021011352539,
      "ref_logps/rejected": -25.29645538330078,
      "rewards/accuracies": 0.7421875,
      "rewards/chosen": -0.4308522939682007,
      "rewards/margins": 0.4750638008117676,
      "rewards/rejected": -0.9059160947799683,
      "step": 310
    },
    {
      "epoch": 2.35,
      "learning_rate": 1.1938202247191012e-07,
      "logps/chosen": -27.410297393798828,
      "logps/rejected": -36.60034942626953,
      "loss": 0.5435,
      "losses/dpo": 0.49991002678871155,
      "losses/sft": 0.9416501522064209,
      "losses/total": 0.49991002678871155,
      "ref_logps/chosen": -22.95693016052246,
      "ref_logps/rejected": -27.357826232910156,
      "rewards/accuracies": 0.765625,
      "rewards/chosen": -0.4453369379043579,
      "rewards/margins": 0.47891533374786377,
      "rewards/rejected": -0.9242523312568665,
      "step": 311
    },
    {
      "epoch": 2.35,
      "learning_rate": 1.1797752808988763e-07,
      "logps/chosen": -26.075031280517578,
      "logps/rejected": -35.989906311035156,
      "loss": 0.5108,
      "losses/dpo": 0.49980589747428894,
      "losses/sft": 0.8830540776252747,
      "losses/total": 0.49980589747428894,
      "ref_logps/chosen": -21.97342300415039,
      "ref_logps/rejected": -26.20843505859375,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.4101608395576477,
      "rewards/margins": 0.5679866075515747,
      "rewards/rejected": -0.9781473875045776,
      "step": 312
    },
    {
      "epoch": 2.36,
      "learning_rate": 1.1657303370786515e-07,
      "logps/chosen": -27.034866333007812,
      "logps/rejected": -36.756141662597656,
      "loss": 0.535,
      "losses/dpo": 0.5506036281585693,
      "losses/sft": 0.842628002166748,
      "losses/total": 0.5506036281585693,
      "ref_logps/chosen": -22.406818389892578,
      "ref_logps/rejected": -26.883403778076172,
      "rewards/accuracies": 0.7734375,
      "rewards/chosen": -0.4628047049045563,
      "rewards/margins": 0.5244689583778381,
      "rewards/rejected": -0.9872736930847168,
      "step": 313
    },
    {
      "epoch": 2.37,
      "learning_rate": 1.151685393258427e-07,
      "logps/chosen": -24.555809020996094,
      "logps/rejected": -32.58570861816406,
      "loss": 0.6002,
      "losses/dpo": 0.643724262714386,
      "losses/sft": 0.86636883020401,
      "losses/total": 0.643724262714386,
      "ref_logps/chosen": -20.17746353149414,
      "ref_logps/rejected": -24.76491928100586,
      "rewards/accuracies": 0.6640625,
      "rewards/chosen": -0.4378345012664795,
      "rewards/margins": 0.3442443907260895,
      "rewards/rejected": -0.7820788621902466,
      "step": 314
    },
    {
      "epoch": 2.38,
      "learning_rate": 1.1376404494382023e-07,
      "logps/chosen": -23.88672637939453,
      "logps/rejected": -34.9397087097168,
      "loss": 0.529,
      "losses/dpo": 0.5676740407943726,
      "losses/sft": 0.8977797627449036,
      "losses/total": 0.5676740407943726,
      "ref_logps/chosen": -19.26026153564453,
      "ref_logps/rejected": -25.219451904296875,
      "rewards/accuracies": 0.7578125,
      "rewards/chosen": -0.4626464247703552,
      "rewards/margins": 0.5093792676925659,
      "rewards/rejected": -0.9720257520675659,
      "step": 315
    },
    {
      "epoch": 2.38,
      "learning_rate": 1.1235955056179774e-07,
      "logps/chosen": -28.308603286743164,
      "logps/rejected": -38.62759017944336,
      "loss": 0.5376,
      "losses/dpo": 0.4975647032260895,
      "losses/sft": 1.098832130432129,
      "losses/total": 0.4975647032260895,
      "ref_logps/chosen": -22.983192443847656,
      "ref_logps/rejected": -28.174081802368164,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5325409173965454,
      "rewards/margins": 0.5128099918365479,
      "rewards/rejected": -1.0453509092330933,
      "step": 316
    },
    {
      "epoch": 2.39,
      "learning_rate": 1.1095505617977527e-07,
      "logps/chosen": -29.64281463623047,
      "logps/rejected": -34.92308807373047,
      "loss": 0.6085,
      "losses/dpo": 0.6205140352249146,
      "losses/sft": 1.0714130401611328,
      "losses/total": 0.6205140352249146,
      "ref_logps/chosen": -23.70892333984375,
      "ref_logps/rejected": -25.790882110595703,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.5933888554573059,
      "rewards/margins": 0.31983205676078796,
      "rewards/rejected": -0.9132209420204163,
      "step": 317
    },
    {
      "epoch": 2.4,
      "learning_rate": 1.095505617977528e-07,
      "logps/chosen": -27.25480079650879,
      "logps/rejected": -35.160831451416016,
      "loss": 0.5594,
      "losses/dpo": 0.46756136417388916,
      "losses/sft": 1.0184146165847778,
      "losses/total": 0.46756136417388916,
      "ref_logps/chosen": -22.160459518432617,
      "ref_logps/rejected": -25.47940444946289,
      "rewards/accuracies": 0.7109375,
      "rewards/chosen": -0.5094340443611145,
      "rewards/margins": 0.45870864391326904,
      "rewards/rejected": -0.9681426882743835,
      "step": 318
    },
    {
      "epoch": 2.41,
      "learning_rate": 1.0814606741573033e-07,
      "logps/chosen": -28.060367584228516,
      "logps/rejected": -35.89076232910156,
      "loss": 0.5957,
      "losses/dpo": 0.6158527135848999,
      "losses/sft": 0.9492118954658508,
      "losses/total": 0.6158527135848999,
      "ref_logps/chosen": -22.560768127441406,
      "ref_logps/rejected": -26.671247482299805,
      "rewards/accuracies": 0.7265625,
      "rewards/chosen": -0.5499600172042847,
      "rewards/margins": 0.3719918131828308,
      "rewards/rejected": -0.9219518899917603,
      "step": 319
    },
    {
      "epoch": 2.42,
      "learning_rate": 1.0674157303370785e-07,
      "logps/chosen": -28.077362060546875,
      "logps/rejected": -32.179466247558594,
      "loss": 0.579,
      "losses/dpo": 0.5990191698074341,
      "losses/sft": 1.0173970460891724,
      "losses/total": 0.5990191698074341,
      "ref_logps/chosen": -23.08903694152832,
      "ref_logps/rejected": -23.298954010009766,
      "rewards/accuracies": 0.6796875,
      "rewards/chosen": -0.4988323450088501,
      "rewards/margins": 0.38921868801116943,
      "rewards/rejected": -0.8880510330200195,
      "step": 320
    },
    {
      "epoch": 2.42,
      "learning_rate": 1.0533707865168538e-07,
      "logps/chosen": -28.690523147583008,
      "logps/rejected": -34.72178268432617,
      "loss": 0.5694,
      "losses/dpo": 0.6682005524635315,
      "losses/sft": 0.9579723477363586,
      "losses/total": 0.6682005524635315,
      "ref_logps/chosen": -23.495473861694336,
      "ref_logps/rejected": -25.300090789794922,
      "rewards/accuracies": 0.7265625,
      "rewards/chosen": -0.519504964351654,
      "rewards/margins": 0.42266416549682617,
      "rewards/rejected": -0.942169189453125,
      "step": 321
    },
    {
      "epoch": 2.43,
      "learning_rate": 1.0393258426966293e-07,
      "logps/chosen": -25.57655143737793,
      "logps/rejected": -33.412723541259766,
      "loss": 0.5471,
      "losses/dpo": 0.6572707891464233,
      "losses/sft": 1.028795599937439,
      "losses/total": 0.6572707891464233,
      "ref_logps/chosen": -21.189647674560547,
      "ref_logps/rejected": -24.523921966552734,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.4386903643608093,
      "rewards/margins": 0.45018988847732544,
      "rewards/rejected": -0.8888803124427795,
      "step": 322
    },
    {
      "epoch": 2.44,
      "learning_rate": 1.0252808988764044e-07,
      "logps/chosen": -29.36073112487793,
      "logps/rejected": -33.21003723144531,
      "loss": 0.5813,
      "losses/dpo": 0.5923129916191101,
      "losses/sft": 0.9457908272743225,
      "losses/total": 0.5923129916191101,
      "ref_logps/chosen": -24.347349166870117,
      "ref_logps/rejected": -24.210060119628906,
      "rewards/accuracies": 0.7265625,
      "rewards/chosen": -0.5013381838798523,
      "rewards/margins": 0.3986593186855316,
      "rewards/rejected": -0.8999974727630615,
      "step": 323
    },
    {
      "epoch": 2.45,
      "learning_rate": 1.0112359550561797e-07,
      "logps/chosen": -25.595857620239258,
      "logps/rejected": -36.034000396728516,
      "loss": 0.5458,
      "losses/dpo": 0.5657480359077454,
      "losses/sft": 0.9646883606910706,
      "losses/total": 0.5657480359077454,
      "ref_logps/chosen": -21.133930206298828,
      "ref_logps/rejected": -26.436534881591797,
      "rewards/accuracies": 0.7265625,
      "rewards/chosen": -0.44619300961494446,
      "rewards/margins": 0.5135533809661865,
      "rewards/rejected": -0.9597463607788086,
      "step": 324
    },
    {
      "epoch": 2.45,
      "learning_rate": 9.971910112359549e-08,
      "logps/chosen": -25.68283462524414,
      "logps/rejected": -35.9984130859375,
      "loss": 0.5137,
      "losses/dpo": 0.5698142051696777,
      "losses/sft": 0.9736462235450745,
      "losses/total": 0.5698142051696777,
      "ref_logps/chosen": -21.976032257080078,
      "ref_logps/rejected": -26.659069061279297,
      "rewards/accuracies": 0.796875,
      "rewards/chosen": -0.370680034160614,
      "rewards/margins": 0.5632542371749878,
      "rewards/rejected": -0.9339342713356018,
      "step": 325
    },
    {
      "epoch": 2.46,
      "learning_rate": 9.831460674157303e-08,
      "logps/chosen": -27.503202438354492,
      "logps/rejected": -37.614471435546875,
      "loss": 0.5574,
      "losses/dpo": 0.6440725326538086,
      "losses/sft": 0.963034987449646,
      "losses/total": 0.6440725326538086,
      "ref_logps/chosen": -22.742584228515625,
      "ref_logps/rejected": -28.33091163635254,
      "rewards/accuracies": 0.703125,
      "rewards/chosen": -0.47606179118156433,
      "rewards/margins": 0.45229417085647583,
      "rewards/rejected": -0.9283559918403625,
      "step": 326
    },
    {
      "epoch": 2.47,
      "learning_rate": 9.691011235955055e-08,
      "logps/chosen": -27.817134857177734,
      "logps/rejected": -34.62038803100586,
      "loss": 0.5561,
      "losses/dpo": 0.6037241816520691,
      "losses/sft": 0.9915317296981812,
      "losses/total": 0.6037241816520691,
      "ref_logps/chosen": -23.01769256591797,
      "ref_logps/rejected": -25.366844177246094,
      "rewards/accuracies": 0.7421875,
      "rewards/chosen": -0.47994428873062134,
      "rewards/margins": 0.44541001319885254,
      "rewards/rejected": -0.9253543019294739,
      "step": 327
    },
    {
      "epoch": 2.48,
      "learning_rate": 9.550561797752808e-08,
      "logps/chosen": -29.053791046142578,
      "logps/rejected": -36.763641357421875,
      "loss": 0.5811,
      "losses/dpo": 0.5880983471870422,
      "losses/sft": 1.1540213823318481,
      "losses/total": 0.5880983471870422,
      "ref_logps/chosen": -23.688079833984375,
      "ref_logps/rejected": -27.189102172851562,
      "rewards/accuracies": 0.6796875,
      "rewards/chosen": -0.5365712642669678,
      "rewards/margins": 0.42088285088539124,
      "rewards/rejected": -0.9574541449546814,
      "step": 328
    },
    {
      "epoch": 2.48,
      "learning_rate": 9.410112359550561e-08,
      "logps/chosen": -25.49301528930664,
      "logps/rejected": -34.453372955322266,
      "loss": 0.56,
      "losses/dpo": 0.4375653862953186,
      "losses/sft": 1.0353739261627197,
      "losses/total": 0.4375653862953186,
      "ref_logps/chosen": -21.16693115234375,
      "ref_logps/rejected": -25.54058837890625,
      "rewards/accuracies": 0.7265625,
      "rewards/chosen": -0.43260854482650757,
      "rewards/margins": 0.45866984128952026,
      "rewards/rejected": -0.8912783861160278,
      "step": 329
    },
    {
      "epoch": 2.49,
      "learning_rate": 9.269662921348314e-08,
      "logps/chosen": -27.841381072998047,
      "logps/rejected": -31.897401809692383,
      "loss": 0.5787,
      "losses/dpo": 0.6422601938247681,
      "losses/sft": 0.9122541546821594,
      "losses/total": 0.6422601938247681,
      "ref_logps/chosen": -23.25330352783203,
      "ref_logps/rejected": -23.617647171020508,
      "rewards/accuracies": 0.7109375,
      "rewards/chosen": -0.45880773663520813,
      "rewards/margins": 0.3691678047180176,
      "rewards/rejected": -0.8279755115509033,
      "step": 330
    },
    {
      "epoch": 2.5,
      "learning_rate": 9.129213483146067e-08,
      "logps/chosen": -28.4913330078125,
      "logps/rejected": -37.258323669433594,
      "loss": 0.5962,
      "losses/dpo": 0.7063708901405334,
      "losses/sft": 1.0378127098083496,
      "losses/total": 0.7063708901405334,
      "ref_logps/chosen": -22.901752471923828,
      "ref_logps/rejected": -27.694263458251953,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.558958113193512,
      "rewards/margins": 0.3974474370479584,
      "rewards/rejected": -0.956405520439148,
      "step": 331
    },
    {
      "epoch": 2.51,
      "learning_rate": 8.988764044943819e-08,
      "logps/chosen": -28.342041015625,
      "logps/rejected": -36.76737976074219,
      "loss": 0.5382,
      "losses/dpo": 0.6065940856933594,
      "losses/sft": 1.061606526374817,
      "losses/total": 0.6065940856933594,
      "ref_logps/chosen": -22.77169418334961,
      "ref_logps/rejected": -26.355728149414062,
      "rewards/accuracies": 0.7578125,
      "rewards/chosen": -0.5570348501205444,
      "rewards/margins": 0.48413002490997314,
      "rewards/rejected": -1.0411648750305176,
      "step": 332
    },
    {
      "epoch": 2.51,
      "learning_rate": 8.848314606741572e-08,
      "logps/chosen": -28.1925048828125,
      "logps/rejected": -36.757789611816406,
      "loss": 0.542,
      "losses/dpo": 0.5146865844726562,
      "losses/sft": 0.82643723487854,
      "losses/total": 0.5146865844726562,
      "ref_logps/chosen": -22.9779052734375,
      "ref_logps/rejected": -26.85318374633789,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5214601755142212,
      "rewards/margins": 0.4690002501010895,
      "rewards/rejected": -0.9904604554176331,
      "step": 333
    },
    {
      "epoch": 2.52,
      "learning_rate": 8.707865168539325e-08,
      "logps/chosen": -26.62852668762207,
      "logps/rejected": -34.3861083984375,
      "loss": 0.5288,
      "losses/dpo": 0.554874062538147,
      "losses/sft": 0.9589724540710449,
      "losses/total": 0.554874062538147,
      "ref_logps/chosen": -22.37548065185547,
      "ref_logps/rejected": -25.256423950195312,
      "rewards/accuracies": 0.7578125,
      "rewards/chosen": -0.4253048598766327,
      "rewards/margins": 0.4876634180545807,
      "rewards/rejected": -0.9129682183265686,
      "step": 334
    },
    {
      "epoch": 2.53,
      "learning_rate": 8.567415730337078e-08,
      "logps/chosen": -30.657304763793945,
      "logps/rejected": -35.13941192626953,
      "loss": 0.564,
      "losses/dpo": 0.5628042817115784,
      "losses/sft": 0.9816582202911377,
      "losses/total": 0.5628042817115784,
      "ref_logps/chosen": -25.568553924560547,
      "ref_logps/rejected": -26.13833236694336,
      "rewards/accuracies": 0.765625,
      "rewards/chosen": -0.5088753700256348,
      "rewards/margins": 0.3912326395511627,
      "rewards/rejected": -0.9001079797744751,
      "step": 335
    },
    {
      "epoch": 2.54,
      "learning_rate": 8.426966292134831e-08,
      "logps/chosen": -27.922801971435547,
      "logps/rejected": -35.52781677246094,
      "loss": 0.5751,
      "losses/dpo": 0.5604207515716553,
      "losses/sft": 0.9308174848556519,
      "losses/total": 0.5604207515716553,
      "ref_logps/chosen": -22.82427978515625,
      "ref_logps/rejected": -26.594511032104492,
      "rewards/accuracies": 0.7578125,
      "rewards/chosen": -0.5098517537117004,
      "rewards/margins": 0.38347893953323364,
      "rewards/rejected": -0.8933306932449341,
      "step": 336
    },
    {
      "epoch": 2.54,
      "learning_rate": 8.286516853932583e-08,
      "logps/chosen": -26.58924674987793,
      "logps/rejected": -37.19468688964844,
      "loss": 0.5092,
      "losses/dpo": 0.5189211368560791,
      "losses/sft": 0.9888613224029541,
      "losses/total": 0.5189211368560791,
      "ref_logps/chosen": -22.07261085510254,
      "ref_logps/rejected": -26.852371215820312,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.45166367292404175,
      "rewards/margins": 0.5825679302215576,
      "rewards/rejected": -1.0342316627502441,
      "step": 337
    },
    {
      "epoch": 2.55,
      "learning_rate": 8.146067415730337e-08,
      "logps/chosen": -26.25957489013672,
      "logps/rejected": -34.435699462890625,
      "loss": 0.5636,
      "losses/dpo": 0.5064201951026917,
      "losses/sft": 1.0053503513336182,
      "losses/total": 0.5064201951026917,
      "ref_logps/chosen": -21.541664123535156,
      "ref_logps/rejected": -25.595813751220703,
      "rewards/accuracies": 0.7265625,
      "rewards/chosen": -0.47179120779037476,
      "rewards/margins": 0.41219767928123474,
      "rewards/rejected": -0.8839888572692871,
      "step": 338
    },
    {
      "epoch": 2.56,
      "learning_rate": 8.005617977528089e-08,
      "logps/chosen": -28.839031219482422,
      "logps/rejected": -37.7884635925293,
      "loss": 0.5099,
      "losses/dpo": 0.5300096273422241,
      "losses/sft": 0.9853606224060059,
      "losses/total": 0.5300096273422241,
      "ref_logps/chosen": -24.249427795410156,
      "ref_logps/rejected": -27.3421688079834,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.4589604139328003,
      "rewards/margins": 0.5856689810752869,
      "rewards/rejected": -1.0446293354034424,
      "step": 339
    },
    {
      "epoch": 2.57,
      "learning_rate": 7.865168539325842e-08,
      "logps/chosen": -24.501375198364258,
      "logps/rejected": -35.306766510009766,
      "loss": 0.5489,
      "losses/dpo": 0.5961363315582275,
      "losses/sft": 1.0056707859039307,
      "losses/total": 0.5961363315582275,
      "ref_logps/chosen": -19.69840431213379,
      "ref_logps/rejected": -26.113300323486328,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.4802970886230469,
      "rewards/margins": 0.43904954195022583,
      "rewards/rejected": -0.9193466305732727,
      "step": 340
    },
    {
      "epoch": 2.57,
      "learning_rate": 7.724719101123594e-08,
      "logps/chosen": -28.589330673217773,
      "logps/rejected": -37.83360290527344,
      "loss": 0.5047,
      "losses/dpo": 0.584464430809021,
      "losses/sft": 1.1327065229415894,
      "losses/total": 0.584464430809021,
      "ref_logps/chosen": -23.609786987304688,
      "ref_logps/rejected": -27.048458099365234,
      "rewards/accuracies": 0.7734375,
      "rewards/chosen": -0.49795451760292053,
      "rewards/margins": 0.5805596113204956,
      "rewards/rejected": -1.0785142183303833,
      "step": 341
    },
    {
      "epoch": 2.58,
      "learning_rate": 7.584269662921348e-08,
      "logps/chosen": -26.895156860351562,
      "logps/rejected": -36.51441192626953,
      "loss": 0.5664,
      "losses/dpo": 0.5289937257766724,
      "losses/sft": 0.9366389513015747,
      "losses/total": 0.5289937257766724,
      "ref_logps/chosen": -21.687744140625,
      "ref_logps/rejected": -27.214290618896484,
      "rewards/accuracies": 0.6953125,
      "rewards/chosen": -0.5207412838935852,
      "rewards/margins": 0.40927091240882874,
      "rewards/rejected": -0.9300122261047363,
      "step": 342
    },
    {
      "epoch": 2.59,
      "learning_rate": 7.443820224719101e-08,
      "logps/chosen": -26.770729064941406,
      "logps/rejected": -37.010292053222656,
      "loss": 0.5614,
      "losses/dpo": 0.5385686159133911,
      "losses/sft": 1.026196002960205,
      "losses/total": 0.5385686159133911,
      "ref_logps/chosen": -21.59899139404297,
      "ref_logps/rejected": -27.5603084564209,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.5171737670898438,
      "rewards/margins": 0.4278249144554138,
      "rewards/rejected": -0.9449986815452576,
      "step": 343
    },
    {
      "epoch": 2.6,
      "learning_rate": 7.303370786516853e-08,
      "logps/chosen": -28.282766342163086,
      "logps/rejected": -37.94123840332031,
      "loss": 0.5574,
      "losses/dpo": 0.4933924973011017,
      "losses/sft": 1.0346543788909912,
      "losses/total": 0.4933924973011017,
      "ref_logps/chosen": -23.10584831237793,
      "ref_logps/rejected": -27.940820693969727,
      "rewards/accuracies": 0.7265625,
      "rewards/chosen": -0.5176920294761658,
      "rewards/margins": 0.48235008120536804,
      "rewards/rejected": -1.0000420808792114,
      "step": 344
    },
    {
      "epoch": 2.6,
      "learning_rate": 7.162921348314606e-08,
      "logps/chosen": -28.72567367553711,
      "logps/rejected": -39.72615432739258,
      "loss": 0.5095,
      "losses/dpo": 0.46729788184165955,
      "losses/sft": 1.0185084342956543,
      "losses/total": 0.46729788184165955,
      "ref_logps/chosen": -23.77487564086914,
      "ref_logps/rejected": -29.071794509887695,
      "rewards/accuracies": 0.8046875,
      "rewards/chosen": -0.4950796663761139,
      "rewards/margins": 0.5703563690185547,
      "rewards/rejected": -1.0654358863830566,
      "step": 345
    },
    {
      "epoch": 2.61,
      "learning_rate": 7.022471910112359e-08,
      "logps/chosen": -25.868453979492188,
      "logps/rejected": -38.04795455932617,
      "loss": 0.5312,
      "losses/dpo": 0.5006756782531738,
      "losses/sft": 0.9233719110488892,
      "losses/total": 0.5006756782531738,
      "ref_logps/chosen": -21.202287673950195,
      "ref_logps/rejected": -28.387357711791992,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.4666164517402649,
      "rewards/margins": 0.49944305419921875,
      "rewards/rejected": -0.9660595059394836,
      "step": 346
    },
    {
      "epoch": 2.62,
      "learning_rate": 6.882022471910112e-08,
      "logps/chosen": -30.674976348876953,
      "logps/rejected": -37.18801498413086,
      "loss": 0.5609,
      "losses/dpo": 0.53383469581604,
      "losses/sft": 1.0966167449951172,
      "losses/total": 0.53383469581604,
      "ref_logps/chosen": -25.279560089111328,
      "ref_logps/rejected": -27.363300323486328,
      "rewards/accuracies": 0.734375,
      "rewards/chosen": -0.5395419001579285,
      "rewards/margins": 0.4429297149181366,
      "rewards/rejected": -0.9824715852737427,
      "step": 347
    },
    {
      "epoch": 2.63,
      "learning_rate": 6.741573033707864e-08,
      "logps/chosen": -27.533077239990234,
      "logps/rejected": -36.4505729675293,
      "loss": 0.5459,
      "losses/dpo": 0.5303448438644409,
      "losses/sft": 1.0059340000152588,
      "losses/total": 0.5303448438644409,
      "ref_logps/chosen": -23.062650680541992,
      "ref_logps/rejected": -27.07624053955078,
      "rewards/accuracies": 0.734375,
      "rewards/chosen": -0.4470424950122833,
      "rewards/margins": 0.49039074778556824,
      "rewards/rejected": -0.9374332427978516,
      "step": 348
    },
    {
      "epoch": 2.63,
      "learning_rate": 6.601123595505617e-08,
      "logps/chosen": -27.371315002441406,
      "logps/rejected": -37.460845947265625,
      "loss": 0.5395,
      "losses/dpo": 0.46134790778160095,
      "losses/sft": 1.0326218605041504,
      "losses/total": 0.46134790778160095,
      "ref_logps/chosen": -21.61273765563965,
      "ref_logps/rejected": -26.598316192626953,
      "rewards/accuracies": 0.7265625,
      "rewards/chosen": -0.5758577585220337,
      "rewards/margins": 0.5103954076766968,
      "rewards/rejected": -1.0862531661987305,
      "step": 349
    },
    {
      "epoch": 2.64,
      "learning_rate": 6.460674157303371e-08,
      "logps/chosen": -26.683940887451172,
      "logps/rejected": -35.51959228515625,
      "loss": 0.531,
      "losses/dpo": 0.5929858684539795,
      "losses/sft": 0.8796969056129456,
      "losses/total": 0.5929858684539795,
      "ref_logps/chosen": -21.84777069091797,
      "ref_logps/rejected": -25.34250259399414,
      "rewards/accuracies": 0.7578125,
      "rewards/chosen": -0.48361673951148987,
      "rewards/margins": 0.5340923070907593,
      "rewards/rejected": -1.0177090167999268,
      "step": 350
    },
    {
      "epoch": 2.65,
      "learning_rate": 6.320224719101123e-08,
      "logps/chosen": -27.784767150878906,
      "logps/rejected": -36.31642150878906,
      "loss": 0.5638,
      "losses/dpo": 0.46039754152297974,
      "losses/sft": 1.014696478843689,
      "losses/total": 0.46039754152297974,
      "ref_logps/chosen": -22.830692291259766,
      "ref_logps/rejected": -26.99026107788086,
      "rewards/accuracies": 0.6484375,
      "rewards/chosen": -0.4954075217247009,
      "rewards/margins": 0.4372091293334961,
      "rewards/rejected": -0.9326165914535522,
      "step": 351
    },
    {
      "epoch": 2.66,
      "learning_rate": 6.179775280898876e-08,
      "logps/chosen": -23.8892765045166,
      "logps/rejected": -32.802425384521484,
      "loss": 0.5307,
      "losses/dpo": 0.5120245218276978,
      "losses/sft": 0.9590541124343872,
      "losses/total": 0.5120245218276978,
      "ref_logps/chosen": -19.922191619873047,
      "ref_logps/rejected": -23.754770278930664,
      "rewards/accuracies": 0.7890625,
      "rewards/chosen": -0.3967083692550659,
      "rewards/margins": 0.5080575346946716,
      "rewards/rejected": -0.9047658443450928,
      "step": 352
    },
    {
      "epoch": 2.66,
      "learning_rate": 6.039325842696629e-08,
      "logps/chosen": -27.902587890625,
      "logps/rejected": -39.759193420410156,
      "loss": 0.5216,
      "losses/dpo": 0.5157948136329651,
      "losses/sft": 0.8797988891601562,
      "losses/total": 0.5157948136329651,
      "ref_logps/chosen": -22.233232498168945,
      "ref_logps/rejected": -28.421966552734375,
      "rewards/accuracies": 0.796875,
      "rewards/chosen": -0.5669355988502502,
      "rewards/margins": 0.5667868852615356,
      "rewards/rejected": -1.1337225437164307,
      "step": 353
    },
    {
      "epoch": 2.67,
      "learning_rate": 5.898876404494382e-08,
      "logps/chosen": -27.200105667114258,
      "logps/rejected": -38.05504608154297,
      "loss": 0.5154,
      "losses/dpo": 0.6272658705711365,
      "losses/sft": 0.901512086391449,
      "losses/total": 0.6272658705711365,
      "ref_logps/chosen": -22.22509765625,
      "ref_logps/rejected": -27.425247192382812,
      "rewards/accuracies": 0.7421875,
      "rewards/chosen": -0.49750083684921265,
      "rewards/margins": 0.5654786825180054,
      "rewards/rejected": -1.0629794597625732,
      "step": 354
    },
    {
      "epoch": 2.68,
      "learning_rate": 5.758426966292135e-08,
      "logps/chosen": -29.658336639404297,
      "logps/rejected": -38.597557067871094,
      "loss": 0.5507,
      "losses/dpo": 0.4642670750617981,
      "losses/sft": 1.0486382246017456,
      "losses/total": 0.4642670750617981,
      "ref_logps/chosen": -23.814481735229492,
      "ref_logps/rejected": -27.686166763305664,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.5843857526779175,
      "rewards/margins": 0.5067534446716309,
      "rewards/rejected": -1.0911391973495483,
      "step": 355
    },
    {
      "epoch": 2.69,
      "learning_rate": 5.617977528089887e-08,
      "logps/chosen": -23.939620971679688,
      "logps/rejected": -33.06968688964844,
      "loss": 0.5631,
      "losses/dpo": 0.5865851640701294,
      "losses/sft": 1.1602400541305542,
      "losses/total": 0.5865851640701294,
      "ref_logps/chosen": -19.01073455810547,
      "ref_logps/rejected": -23.945262908935547,
      "rewards/accuracies": 0.7265625,
      "rewards/chosen": -0.4928884506225586,
      "rewards/margins": 0.4195541441440582,
      "rewards/rejected": -0.9124425649642944,
      "step": 356
    },
    {
      "epoch": 2.69,
      "learning_rate": 5.47752808988764e-08,
      "logps/chosen": -27.94991683959961,
      "logps/rejected": -36.65930938720703,
      "loss": 0.5461,
      "losses/dpo": 0.6400465369224548,
      "losses/sft": 1.0134565830230713,
      "losses/total": 0.6400465369224548,
      "ref_logps/chosen": -23.591896057128906,
      "ref_logps/rejected": -27.34914779663086,
      "rewards/accuracies": 0.7578125,
      "rewards/chosen": -0.43580204248428345,
      "rewards/margins": 0.4952143728733063,
      "rewards/rejected": -0.9310164451599121,
      "step": 357
    },
    {
      "epoch": 2.7,
      "learning_rate": 5.3370786516853926e-08,
      "logps/chosen": -28.148937225341797,
      "logps/rejected": -34.08583450317383,
      "loss": 0.561,
      "losses/dpo": 0.47015029191970825,
      "losses/sft": 0.923213005065918,
      "losses/total": 0.47015029191970825,
      "ref_logps/chosen": -23.29110336303711,
      "ref_logps/rejected": -25.051483154296875,
      "rewards/accuracies": 0.703125,
      "rewards/chosen": -0.4857832193374634,
      "rewards/margins": 0.417651891708374,
      "rewards/rejected": -0.9034351110458374,
      "step": 358
    },
    {
      "epoch": 2.71,
      "learning_rate": 5.196629213483146e-08,
      "logps/chosen": -26.931848526000977,
      "logps/rejected": -35.78190994262695,
      "loss": 0.5196,
      "losses/dpo": 0.4919354021549225,
      "losses/sft": 0.9875601530075073,
      "losses/total": 0.4919354021549225,
      "ref_logps/chosen": -22.220352172851562,
      "ref_logps/rejected": -25.88389778137207,
      "rewards/accuracies": 0.7421875,
      "rewards/chosen": -0.4711495637893677,
      "rewards/margins": 0.51865154504776,
      "rewards/rejected": -0.9898011684417725,
      "step": 359
    },
    {
      "epoch": 2.72,
      "learning_rate": 5.056179775280899e-08,
      "logps/chosen": -26.860830307006836,
      "logps/rejected": -36.244728088378906,
      "loss": 0.5141,
      "losses/dpo": 0.5143895745277405,
      "losses/sft": 0.8888437747955322,
      "losses/total": 0.5143895745277405,
      "ref_logps/chosen": -22.12276840209961,
      "ref_logps/rejected": -25.870086669921875,
      "rewards/accuracies": 0.796875,
      "rewards/chosen": -0.4738062918186188,
      "rewards/margins": 0.5636579394340515,
      "rewards/rejected": -1.0374642610549927,
      "step": 360
    },
    {
      "epoch": 2.72,
      "learning_rate": 4.915730337078652e-08,
      "logps/chosen": -27.645713806152344,
      "logps/rejected": -35.34681701660156,
      "loss": 0.5612,
      "losses/dpo": 0.5186240077018738,
      "losses/sft": 1.109127402305603,
      "losses/total": 0.5186240077018738,
      "ref_logps/chosen": -22.531267166137695,
      "ref_logps/rejected": -25.30887222290039,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.5114448070526123,
      "rewards/margins": 0.4923498034477234,
      "rewards/rejected": -1.0037946701049805,
      "step": 361
    },
    {
      "epoch": 2.73,
      "learning_rate": 4.775280898876404e-08,
      "logps/chosen": -29.528343200683594,
      "logps/rejected": -37.111507415771484,
      "loss": 0.5701,
      "losses/dpo": 0.5167029500007629,
      "losses/sft": 1.1346383094787598,
      "losses/total": 0.5167029500007629,
      "ref_logps/chosen": -23.850698471069336,
      "ref_logps/rejected": -27.338424682617188,
      "rewards/accuracies": 0.703125,
      "rewards/chosen": -0.5677646398544312,
      "rewards/margins": 0.40954357385635376,
      "rewards/rejected": -0.9773082137107849,
      "step": 362
    },
    {
      "epoch": 2.74,
      "learning_rate": 4.634831460674157e-08,
      "logps/chosen": -27.099462509155273,
      "logps/rejected": -38.734046936035156,
      "loss": 0.5367,
      "losses/dpo": 0.6075611114501953,
      "losses/sft": 1.0922847986221313,
      "losses/total": 0.6075611114501953,
      "ref_logps/chosen": -21.647756576538086,
      "ref_logps/rejected": -27.990768432617188,
      "rewards/accuracies": 0.7265625,
      "rewards/chosen": -0.5451704859733582,
      "rewards/margins": 0.5291576385498047,
      "rewards/rejected": -1.0743281841278076,
      "step": 363
    },
    {
      "epoch": 2.75,
      "learning_rate": 4.4943820224719096e-08,
      "logps/chosen": -28.804433822631836,
      "logps/rejected": -38.87983703613281,
      "loss": 0.5448,
      "losses/dpo": 0.5679644346237183,
      "losses/sft": 1.123085618019104,
      "losses/total": 0.5679644346237183,
      "ref_logps/chosen": -23.055761337280273,
      "ref_logps/rejected": -27.842578887939453,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5748672485351562,
      "rewards/margins": 0.5288586020469666,
      "rewards/rejected": -1.1037259101867676,
      "step": 364
    },
    {
      "epoch": 2.75,
      "learning_rate": 4.3539325842696626e-08,
      "logps/chosen": -29.942031860351562,
      "logps/rejected": -37.742164611816406,
      "loss": 0.5544,
      "losses/dpo": 0.4389882981777191,
      "losses/sft": 0.9757397174835205,
      "losses/total": 0.4389882981777191,
      "ref_logps/chosen": -24.796215057373047,
      "ref_logps/rejected": -27.602325439453125,
      "rewards/accuracies": 0.6640625,
      "rewards/chosen": -0.5145817995071411,
      "rewards/margins": 0.4994018077850342,
      "rewards/rejected": -1.0139836072921753,
      "step": 365
    },
    {
      "epoch": 2.76,
      "learning_rate": 4.213483146067416e-08,
      "logps/chosen": -30.154991149902344,
      "logps/rejected": -35.81608581542969,
      "loss": 0.57,
      "losses/dpo": 0.571212887763977,
      "losses/sft": 0.8268208503723145,
      "losses/total": 0.571212887763977,
      "ref_logps/chosen": -24.421096801757812,
      "ref_logps/rejected": -25.88280487060547,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5733895897865295,
      "rewards/margins": 0.4199383854866028,
      "rewards/rejected": -0.9933279752731323,
      "step": 366
    },
    {
      "epoch": 2.77,
      "learning_rate": 4.073033707865169e-08,
      "logps/chosen": -27.25971794128418,
      "logps/rejected": -33.205955505371094,
      "loss": 0.5874,
      "losses/dpo": 0.4875527620315552,
      "losses/sft": 0.8703315854072571,
      "losses/total": 0.4875527620315552,
      "ref_logps/chosen": -22.749954223632812,
      "ref_logps/rejected": -24.781803131103516,
      "rewards/accuracies": 0.734375,
      "rewards/chosen": -0.45097634196281433,
      "rewards/margins": 0.3914392292499542,
      "rewards/rejected": -0.8424156308174133,
      "step": 367
    },
    {
      "epoch": 2.78,
      "learning_rate": 3.932584269662921e-08,
      "logps/chosen": -28.230928421020508,
      "logps/rejected": -37.86750030517578,
      "loss": 0.508,
      "losses/dpo": 0.4668968617916107,
      "losses/sft": 1.1078698635101318,
      "losses/total": 0.4668968617916107,
      "ref_logps/chosen": -23.454715728759766,
      "ref_logps/rejected": -27.212678909301758,
      "rewards/accuracies": 0.765625,
      "rewards/chosen": -0.47762107849121094,
      "rewards/margins": 0.5878612995147705,
      "rewards/rejected": -1.0654823780059814,
      "step": 368
    },
    {
      "epoch": 2.78,
      "learning_rate": 3.792134831460674e-08,
      "logps/chosen": -28.5417423248291,
      "logps/rejected": -39.07720184326172,
      "loss": 0.5722,
      "losses/dpo": 0.5119404196739197,
      "losses/sft": 1.0701940059661865,
      "losses/total": 0.5119404196739197,
      "ref_logps/chosen": -22.97249984741211,
      "ref_logps/rejected": -29.236312866210938,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.5569244623184204,
      "rewards/margins": 0.42716455459594727,
      "rewards/rejected": -0.9840888977050781,
      "step": 369
    },
    {
      "epoch": 2.79,
      "learning_rate": 3.6516853932584266e-08,
      "logps/chosen": -24.37343406677246,
      "logps/rejected": -35.577354431152344,
      "loss": 0.5144,
      "losses/dpo": 0.39502987265586853,
      "losses/sft": 1.0756311416625977,
      "losses/total": 0.39502987265586853,
      "ref_logps/chosen": -19.630115509033203,
      "ref_logps/rejected": -25.026439666748047,
      "rewards/accuracies": 0.765625,
      "rewards/chosen": -0.474331796169281,
      "rewards/margins": 0.5807597041130066,
      "rewards/rejected": -1.055091381072998,
      "step": 370
    },
    {
      "epoch": 2.8,
      "learning_rate": 3.5112359550561796e-08,
      "logps/chosen": -25.75430679321289,
      "logps/rejected": -35.49622344970703,
      "loss": 0.5757,
      "losses/dpo": 0.5865879058837891,
      "losses/sft": 1.0159986019134521,
      "losses/total": 0.5865879058837891,
      "ref_logps/chosen": -21.11154556274414,
      "ref_logps/rejected": -26.56639862060547,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.4642760157585144,
      "rewards/margins": 0.42870670557022095,
      "rewards/rejected": -0.8929827213287354,
      "step": 371
    },
    {
      "epoch": 2.81,
      "learning_rate": 3.370786516853932e-08,
      "logps/chosen": -28.015663146972656,
      "logps/rejected": -36.934810638427734,
      "loss": 0.507,
      "losses/dpo": 0.6495200395584106,
      "losses/sft": 1.097916841506958,
      "losses/total": 0.6495200395584106,
      "ref_logps/chosen": -23.48037338256836,
      "ref_logps/rejected": -26.453636169433594,
      "rewards/accuracies": 0.7734375,
      "rewards/chosen": -0.45352903008461,
      "rewards/margins": 0.5945882797241211,
      "rewards/rejected": -1.0481172800064087,
      "step": 372
    },
    {
      "epoch": 2.82,
      "learning_rate": 3.230337078651686e-08,
      "logps/chosen": -29.112939834594727,
      "logps/rejected": -35.190895080566406,
      "loss": 0.5557,
      "losses/dpo": 0.3998969793319702,
      "losses/sft": 0.9329382181167603,
      "losses/total": 0.3998969793319702,
      "ref_logps/chosen": -23.576570510864258,
      "ref_logps/rejected": -24.735258102416992,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5536371469497681,
      "rewards/margins": 0.49192649126052856,
      "rewards/rejected": -1.0455635786056519,
      "step": 373
    },
    {
      "epoch": 2.82,
      "learning_rate": 3.089887640449438e-08,
      "logps/chosen": -28.713830947875977,
      "logps/rejected": -37.038963317871094,
      "loss": 0.5536,
      "losses/dpo": 0.548796534538269,
      "losses/sft": 1.0410091876983643,
      "losses/total": 0.548796534538269,
      "ref_logps/chosen": -23.76250457763672,
      "ref_logps/rejected": -27.5520076751709,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.49513280391693115,
      "rewards/margins": 0.45356276631355286,
      "rewards/rejected": -0.9486956000328064,
      "step": 374
    },
    {
      "epoch": 2.83,
      "learning_rate": 2.949438202247191e-08,
      "logps/chosen": -29.465068817138672,
      "logps/rejected": -39.406578063964844,
      "loss": 0.5343,
      "losses/dpo": 0.6983579397201538,
      "losses/sft": 1.0986469984054565,
      "losses/total": 0.6983579397201538,
      "ref_logps/chosen": -24.418071746826172,
      "ref_logps/rejected": -28.89803695678711,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5046992897987366,
      "rewards/margins": 0.5461547374725342,
      "rewards/rejected": -1.050853967666626,
      "step": 375
    },
    {
      "epoch": 2.84,
      "learning_rate": 2.8089887640449436e-08,
      "logps/chosen": -27.72464370727539,
      "logps/rejected": -35.225887298583984,
      "loss": 0.5838,
      "losses/dpo": 0.615436851978302,
      "losses/sft": 1.064025640487671,
      "losses/total": 0.615436851978302,
      "ref_logps/chosen": -22.502582550048828,
      "ref_logps/rejected": -26.32878875732422,
      "rewards/accuracies": 0.7265625,
      "rewards/chosen": -0.5222063660621643,
      "rewards/margins": 0.36750373244285583,
      "rewards/rejected": -0.8897100687026978,
      "step": 376
    },
    {
      "epoch": 2.85,
      "learning_rate": 2.6685393258426963e-08,
      "logps/chosen": -26.368640899658203,
      "logps/rejected": -35.305564880371094,
      "loss": 0.5351,
      "losses/dpo": 0.5552591681480408,
      "losses/sft": 0.8796924352645874,
      "losses/total": 0.5552591681480408,
      "ref_logps/chosen": -21.371036529541016,
      "ref_logps/rejected": -25.149438858032227,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.49976038932800293,
      "rewards/margins": 0.5158523917198181,
      "rewards/rejected": -1.0156128406524658,
      "step": 377
    },
    {
      "epoch": 2.85,
      "learning_rate": 2.5280898876404493e-08,
      "logps/chosen": -29.838565826416016,
      "logps/rejected": -38.05325698852539,
      "loss": 0.5338,
      "losses/dpo": 0.4755927324295044,
      "losses/sft": 0.9763241410255432,
      "losses/total": 0.4755927324295044,
      "ref_logps/chosen": -24.918655395507812,
      "ref_logps/rejected": -28.04724884033203,
      "rewards/accuracies": 0.734375,
      "rewards/chosen": -0.4919911026954651,
      "rewards/margins": 0.5086094737052917,
      "rewards/rejected": -1.0006005764007568,
      "step": 378
    },
    {
      "epoch": 2.86,
      "learning_rate": 2.387640449438202e-08,
      "logps/chosen": -29.036991119384766,
      "logps/rejected": -35.4906005859375,
      "loss": 0.5335,
      "losses/dpo": 0.5385127067565918,
      "losses/sft": 1.245056390762329,
      "losses/total": 0.5385127067565918,
      "ref_logps/chosen": -23.929513931274414,
      "ref_logps/rejected": -24.878217697143555,
      "rewards/accuracies": 0.7421875,
      "rewards/chosen": -0.5107479095458984,
      "rewards/margins": 0.5504903793334961,
      "rewards/rejected": -1.0612382888793945,
      "step": 379
    },
    {
      "epoch": 2.87,
      "learning_rate": 2.2471910112359548e-08,
      "logps/chosen": -29.392702102661133,
      "logps/rejected": -38.68418884277344,
      "loss": 0.545,
      "losses/dpo": 0.43943360447883606,
      "losses/sft": 1.023887276649475,
      "losses/total": 0.43943360447883606,
      "ref_logps/chosen": -24.07533073425293,
      "ref_logps/rejected": -28.427806854248047,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5317370891571045,
      "rewards/margins": 0.49390077590942383,
      "rewards/rejected": -1.0256378650665283,
      "step": 380
    },
    {
      "epoch": 2.88,
      "learning_rate": 2.106741573033708e-08,
      "logps/chosen": -25.038589477539062,
      "logps/rejected": -32.384376525878906,
      "loss": 0.56,
      "losses/dpo": 0.6935892701148987,
      "losses/sft": 1.0011663436889648,
      "losses/total": 0.6935892701148987,
      "ref_logps/chosen": -21.044326782226562,
      "ref_logps/rejected": -23.86334991455078,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.3994261920452118,
      "rewards/margins": 0.45267629623413086,
      "rewards/rejected": -0.8521024584770203,
      "step": 381
    },
    {
      "epoch": 2.88,
      "learning_rate": 1.9662921348314606e-08,
      "logps/chosen": -30.07229995727539,
      "logps/rejected": -37.0654411315918,
      "loss": 0.5936,
      "losses/dpo": 0.547340989112854,
      "losses/sft": 1.0020110607147217,
      "losses/total": 0.547340989112854,
      "ref_logps/chosen": -23.930465698242188,
      "ref_logps/rejected": -27.052518844604492,
      "rewards/accuracies": 0.6171875,
      "rewards/chosen": -0.6141834259033203,
      "rewards/margins": 0.38710883259773254,
      "rewards/rejected": -1.0012922286987305,
      "step": 382
    },
    {
      "epoch": 2.89,
      "learning_rate": 1.8258426966292133e-08,
      "logps/chosen": -30.305606842041016,
      "logps/rejected": -40.710792541503906,
      "loss": 0.537,
      "losses/dpo": 0.5175353288650513,
      "losses/sft": 0.8916615843772888,
      "losses/total": 0.5175353288650513,
      "ref_logps/chosen": -25.279661178588867,
      "ref_logps/rejected": -29.970672607421875,
      "rewards/accuracies": 0.6796875,
      "rewards/chosen": -0.5025948286056519,
      "rewards/margins": 0.5714170932769775,
      "rewards/rejected": -1.0740119218826294,
      "step": 383
    },
    {
      "epoch": 2.9,
      "learning_rate": 1.685393258426966e-08,
      "logps/chosen": -29.87887191772461,
      "logps/rejected": -39.89691162109375,
      "loss": 0.5598,
      "losses/dpo": 0.4781198799610138,
      "losses/sft": 1.0425841808319092,
      "losses/total": 0.4781198799610138,
      "ref_logps/chosen": -23.869295120239258,
      "ref_logps/rejected": -29.154647827148438,
      "rewards/accuracies": 0.7109375,
      "rewards/chosen": -0.6009576916694641,
      "rewards/margins": 0.4732685387134552,
      "rewards/rejected": -1.0742262601852417,
      "step": 384
    },
    {
      "epoch": 2.91,
      "learning_rate": 1.544943820224719e-08,
      "logps/chosen": -26.600048065185547,
      "logps/rejected": -39.657188415527344,
      "loss": 0.5186,
      "losses/dpo": 0.5135948657989502,
      "losses/sft": 0.9224843978881836,
      "losses/total": 0.5135948657989502,
      "ref_logps/chosen": -21.754756927490234,
      "ref_logps/rejected": -29.07170867919922,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.48452913761138916,
      "rewards/margins": 0.5740190744400024,
      "rewards/rejected": -1.0585482120513916,
      "step": 385
    },
    {
      "epoch": 2.91,
      "learning_rate": 1.4044943820224718e-08,
      "logps/chosen": -27.77488136291504,
      "logps/rejected": -37.88126754760742,
      "loss": 0.5551,
      "losses/dpo": 0.5367317199707031,
      "losses/sft": 1.0271828174591064,
      "losses/total": 0.5367317199707031,
      "ref_logps/chosen": -22.3087158203125,
      "ref_logps/rejected": -27.471187591552734,
      "rewards/accuracies": 0.7578125,
      "rewards/chosen": -0.5466164350509644,
      "rewards/margins": 0.4943912625312805,
      "rewards/rejected": -1.0410076379776,
      "step": 386
    },
    {
      "epoch": 2.92,
      "learning_rate": 1.2640449438202247e-08,
      "logps/chosen": -28.72400665283203,
      "logps/rejected": -36.061241149902344,
      "loss": 0.5438,
      "losses/dpo": 0.5493422746658325,
      "losses/sft": 0.9023943543434143,
      "losses/total": 0.5493422746658325,
      "ref_logps/chosen": -23.460235595703125,
      "ref_logps/rejected": -26.00853729248047,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.526377260684967,
      "rewards/margins": 0.47889336943626404,
      "rewards/rejected": -1.0052706003189087,
      "step": 387
    },
    {
      "epoch": 2.93,
      "learning_rate": 1.1235955056179774e-08,
      "logps/chosen": -27.819026947021484,
      "logps/rejected": -37.490928649902344,
      "loss": 0.5852,
      "losses/dpo": 0.5147813558578491,
      "losses/sft": 0.8766761422157288,
      "losses/total": 0.5147813558578491,
      "ref_logps/chosen": -21.90268898010254,
      "ref_logps/rejected": -27.36888885498047,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.5916341543197632,
      "rewards/margins": 0.4205697774887085,
      "rewards/rejected": -1.0122039318084717,
      "step": 388
    },
    {
      "epoch": 2.94,
      "learning_rate": 9.831460674157303e-09,
      "logps/chosen": -26.303754806518555,
      "logps/rejected": -37.83194351196289,
      "loss": 0.524,
      "losses/dpo": 0.5489503741264343,
      "losses/sft": 0.9560513496398926,
      "losses/total": 0.5489503741264343,
      "ref_logps/chosen": -21.29248046875,
      "ref_logps/rejected": -27.238914489746094,
      "rewards/accuracies": 0.765625,
      "rewards/chosen": -0.5011276006698608,
      "rewards/margins": 0.5581751465797424,
      "rewards/rejected": -1.059302806854248,
      "step": 389
    },
    {
      "epoch": 2.94,
      "learning_rate": 8.42696629213483e-09,
      "logps/chosen": -26.287546157836914,
      "logps/rejected": -38.258975982666016,
      "loss": 0.5441,
      "losses/dpo": 0.5884628295898438,
      "losses/sft": 0.9961035251617432,
      "losses/total": 0.5884628295898438,
      "ref_logps/chosen": -21.199317932128906,
      "ref_logps/rejected": -27.755794525146484,
      "rewards/accuracies": 0.7265625,
      "rewards/chosen": -0.5088227391242981,
      "rewards/margins": 0.5414952635765076,
      "rewards/rejected": -1.0503180027008057,
      "step": 390
    },
    {
      "epoch": 2.95,
      "learning_rate": 7.022471910112359e-09,
      "logps/chosen": -29.260208129882812,
      "logps/rejected": -35.26235580444336,
      "loss": 0.5711,
      "losses/dpo": 0.6062160730361938,
      "losses/sft": 0.9891349673271179,
      "losses/total": 0.6062160730361938,
      "ref_logps/chosen": -24.228797912597656,
      "ref_logps/rejected": -25.79244613647461,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.5031411051750183,
      "rewards/margins": 0.44384992122650146,
      "rewards/rejected": -0.9469910264015198,
      "step": 391
    },
    {
      "epoch": 2.96,
      "learning_rate": 5.617977528089887e-09,
      "logps/chosen": -26.954505920410156,
      "logps/rejected": -38.197296142578125,
      "loss": 0.5188,
      "losses/dpo": 0.48179134726524353,
      "losses/sft": 1.0057315826416016,
      "losses/total": 0.48179134726524353,
      "ref_logps/chosen": -22.015995025634766,
      "ref_logps/rejected": -27.583335876464844,
      "rewards/accuracies": 0.796875,
      "rewards/chosen": -0.49385106563568115,
      "rewards/margins": 0.5675452351570129,
      "rewards/rejected": -1.0613962411880493,
      "step": 392
    },
    {
      "epoch": 2.97,
      "learning_rate": 4.213483146067415e-09,
      "logps/chosen": -25.941349029541016,
      "logps/rejected": -37.711891174316406,
      "loss": 0.5126,
      "losses/dpo": 0.47302547097206116,
      "losses/sft": 1.0042707920074463,
      "losses/total": 0.47302547097206116,
      "ref_logps/chosen": -21.42403793334961,
      "ref_logps/rejected": -27.090473175048828,
      "rewards/accuracies": 0.703125,
      "rewards/chosen": -0.45173099637031555,
      "rewards/margins": 0.610410749912262,
      "rewards/rejected": -1.0621416568756104,
      "step": 393
    },
    {
      "epoch": 2.97,
      "learning_rate": 2.8089887640449435e-09,
      "logps/chosen": -27.533342361450195,
      "logps/rejected": -40.14276123046875,
      "loss": 0.5282,
      "losses/dpo": 0.47439950704574585,
      "losses/sft": 1.004162073135376,
      "losses/total": 0.47439950704574585,
      "ref_logps/chosen": -22.44705581665039,
      "ref_logps/rejected": -29.271793365478516,
      "rewards/accuracies": 0.765625,
      "rewards/chosen": -0.5086286067962646,
      "rewards/margins": 0.5784677267074585,
      "rewards/rejected": -1.0870963335037231,
      "step": 394
    },
    {
      "epoch": 2.98,
      "learning_rate": 1.4044943820224717e-09,
      "logps/chosen": -27.44398307800293,
      "logps/rejected": -38.508323669433594,
      "loss": 0.5377,
      "losses/dpo": 0.5113502740859985,
      "losses/sft": 1.0710563659667969,
      "losses/total": 0.5113502740859985,
      "ref_logps/chosen": -22.568340301513672,
      "ref_logps/rejected": -28.23776626586914,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.4875642657279968,
      "rewards/margins": 0.5394913554191589,
      "rewards/rejected": -1.0270556211471558,
      "step": 395
    },
    {
      "epoch": 2.99,
      "learning_rate": 0.0,
      "logps/chosen": -28.845203399658203,
      "logps/rejected": -36.77953338623047,
      "loss": 0.5692,
      "losses/dpo": 0.7008877992630005,
      "losses/sft": 1.1200252771377563,
      "losses/total": 0.7008877992630005,
      "ref_logps/chosen": -23.59469985961914,
      "ref_logps/rejected": -27.211450576782227,
      "rewards/accuracies": 0.703125,
      "rewards/chosen": -0.5250504612922668,
      "rewards/margins": 0.4317581057548523,
      "rewards/rejected": -0.9568085670471191,
      "step": 396
    },
    {
      "epoch": 2.99,
      "step": 396,
      "total_flos": 0.0,
      "train_loss": 0.6152852120423558,
      "train_runtime": 11562.7876,
      "train_samples_per_second": 4.4,
      "train_steps_per_second": 0.034
    }
  ],
  "logging_steps": 1.0,
  "max_steps": 396,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 3,
  "save_steps": 200,
  "total_flos": 0.0,
  "train_batch_size": 4,
  "trial_name": null,
  "trial_params": null
}