Hugging Face's logo

g8a9
/
tweety-mistral-7b-dpo

PEFT
TensorBoard
Safetensors
mistral
alignment-handbook
Generated from Trainer
4-bit precision
bitsandbytes
Model card Files Metrics Community
tweety-mistral-7b-dpo / trainer_state.json
g8a9's picture
g8a9
Upload folder using huggingface_hub
c2f275b verified
raw
history blame contribute delete
194 kB
{
"best_metric": null,
"best_model_checkpoint": null,
"epoch": 0.9997810698387214,
"eval_steps": 100,
"global_step": 3425,
"is_hyper_param_search": false,
"is_local_process_zero": true,
"is_world_process_zero": true,
"log_history": [
{
"epoch": 0.00029190688170473617,
"grad_norm": 0.013427734375,
"learning_rate": 1.457725947521866e-08,
"logits/chosen": -2.4752657413482666,
"logits/rejected": -2.4752657413482666,
"logps/chosen": -328.9035949707031,
"logps/rejected": -328.9035949707031,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": 0.0,
"rewards/margins": 0.0,
"rewards/rejected": 0.0,
"step": 1
},
{
"epoch": 0.002919068817047362,
"grad_norm": 0.0147705078125,
"learning_rate": 1.457725947521866e-07,
"logits/chosen": -2.395798683166504,
"logits/rejected": -2.395798683166504,
"logps/chosen": -317.85565185546875,
"logps/rejected": -317.85565185546875,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.0003186435205861926,
"rewards/margins": 0.0,
"rewards/rejected": -0.0003186435205861926,
"step": 10
},
{
"epoch": 0.005838137634094724,
"grad_norm": 0.01318359375,
"learning_rate": 2.915451895043732e-07,
"logits/chosen": -2.4440758228302,
"logits/rejected": -2.4440758228302,
"logps/chosen": -301.12921142578125,
"logps/rejected": -301.12921142578125,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -7.847430242691189e-05,
"rewards/margins": 0.0,
"rewards/rejected": -7.847430242691189e-05,
"step": 20
},
{
"epoch": 0.008757206451142086,
"grad_norm": 0.01177978515625,
"learning_rate": 4.373177842565598e-07,
"logits/chosen": -2.441359519958496,
"logits/rejected": -2.441359519958496,
"logps/chosen": -317.1576843261719,
"logps/rejected": -317.1576843261719,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.00025945488596335053,
"rewards/margins": 0.0,
"rewards/rejected": -0.00025945488596335053,
"step": 30
},
{
"epoch": 0.011676275268189448,
"grad_norm": 0.0167236328125,
"learning_rate": 5.830903790087464e-07,
"logits/chosen": -2.455430269241333,
"logits/rejected": -2.455430269241333,
"logps/chosen": -328.7832946777344,
"logps/rejected": -328.7832946777344,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.00034936360316351056,
"rewards/margins": 0.0,
"rewards/rejected": -0.00034936360316351056,
"step": 40
},
{
"epoch": 0.014595344085236809,
"grad_norm": 0.012939453125,
"learning_rate": 7.288629737609331e-07,
"logits/chosen": -2.406463384628296,
"logits/rejected": -2.406463384628296,
"logps/chosen": -303.563232421875,
"logps/rejected": -303.563232421875,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": 0.0001031260471791029,
"rewards/margins": 0.0,
"rewards/rejected": 0.0001031260471791029,
"step": 50
},
{
"epoch": 0.01751441290228417,
"grad_norm": 0.016357421875,
"learning_rate": 8.746355685131196e-07,
"logits/chosen": -2.4401960372924805,
"logits/rejected": -2.4401960372924805,
"logps/chosen": -284.1253967285156,
"logps/rejected": -284.1253967285156,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.00043843849562108517,
"rewards/margins": 0.0,
"rewards/rejected": -0.00043843849562108517,
"step": 60
},
{
"epoch": 0.02043348171933153,
"grad_norm": 0.01153564453125,
"learning_rate": 1.0204081632653063e-06,
"logits/chosen": -2.423875093460083,
"logits/rejected": -2.423875093460083,
"logps/chosen": -280.09442138671875,
"logps/rejected": -280.09442138671875,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.00031216375646181405,
"rewards/margins": 0.0,
"rewards/rejected": -0.00031216375646181405,
"step": 70
},
{
"epoch": 0.023352550536378896,
"grad_norm": 0.01214599609375,
"learning_rate": 1.1661807580174927e-06,
"logits/chosen": -2.404435396194458,
"logits/rejected": -2.404435396194458,
"logps/chosen": -267.2549743652344,
"logps/rejected": -267.2549743652344,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": 0.0006922121392562985,
"rewards/margins": 0.0,
"rewards/rejected": 0.0006922121392562985,
"step": 80
},
{
"epoch": 0.026271619353426257,
"grad_norm": 0.0146484375,
"learning_rate": 1.3119533527696792e-06,
"logits/chosen": -2.416917324066162,
"logits/rejected": -2.416917324066162,
"logps/chosen": -333.58563232421875,
"logps/rejected": -333.58563232421875,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": 0.0005137195694260299,
"rewards/margins": 0.0,
"rewards/rejected": 0.0005137195694260299,
"step": 90
},
{
"epoch": 0.029190688170473617,
"grad_norm": 0.0189208984375,
"learning_rate": 1.4577259475218661e-06,
"logits/chosen": -2.4351730346679688,
"logits/rejected": -2.4351730346679688,
"logps/chosen": -339.3778381347656,
"logps/rejected": -339.3778381347656,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": 0.0005722829955630004,
"rewards/margins": 0.0,
"rewards/rejected": 0.0005722829955630004,
"step": 100
},
{
"epoch": 0.029190688170473617,
"eval_logits/chosen": -2.394068479537964,
"eval_logits/rejected": -2.394068479537964,
"eval_logps/chosen": -306.389892578125,
"eval_logps/rejected": -306.389892578125,
"eval_loss": 0.6931472420692444,
"eval_rewards/accuracies": 0.0,
"eval_rewards/chosen": 0.0008870832389220595,
"eval_rewards/margins": 0.0,
"eval_rewards/rejected": 0.0008870832389220595,
"eval_runtime": 2666.9983,
"eval_samples_per_second": 2.283,
"eval_steps_per_second": 0.286,
"step": 100
},
{
"epoch": 0.03210975698752098,
"grad_norm": 0.015869140625,
"learning_rate": 1.6034985422740526e-06,
"logits/chosen": -2.420276165008545,
"logits/rejected": -2.420276165008545,
"logps/chosen": -306.0760803222656,
"logps/rejected": -306.0760803222656,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": 0.0014700460014864802,
"rewards/margins": 0.0,
"rewards/rejected": 0.0014700460014864802,
"step": 110
},
{
"epoch": 0.03502882580456834,
"grad_norm": 0.01544189453125,
"learning_rate": 1.7492711370262391e-06,
"logits/chosen": -2.4616119861602783,
"logits/rejected": -2.4616119861602783,
"logps/chosen": -328.64129638671875,
"logps/rejected": -328.64129638671875,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": 0.001054848893545568,
"rewards/margins": 0.0,
"rewards/rejected": 0.001054848893545568,
"step": 120
},
{
"epoch": 0.037947894621615706,
"grad_norm": 0.0250244140625,
"learning_rate": 1.895043731778426e-06,
"logits/chosen": -2.404423236846924,
"logits/rejected": -2.404423236846924,
"logps/chosen": -339.0644836425781,
"logps/rejected": -339.0644836425781,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": 0.0002889078459702432,
"rewards/margins": 0.0,
"rewards/rejected": 0.0002889078459702432,
"step": 130
},
{
"epoch": 0.04086696343866306,
"grad_norm": 0.0137939453125,
"learning_rate": 2.0408163265306125e-06,
"logits/chosen": -2.4294090270996094,
"logits/rejected": -2.4294090270996094,
"logps/chosen": -299.0234375,
"logps/rejected": -299.0234375,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": 0.0003939162997994572,
"rewards/margins": 0.0,
"rewards/rejected": 0.0003939162997994572,
"step": 140
},
{
"epoch": 0.04378603225571043,
"grad_norm": 0.01470947265625,
"learning_rate": 2.1865889212827988e-06,
"logits/chosen": -2.4415223598480225,
"logits/rejected": -2.4415223598480225,
"logps/chosen": -317.4403991699219,
"logps/rejected": -317.4403991699219,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": 0.0011137222172692418,
"rewards/margins": 0.0,
"rewards/rejected": 0.0011137222172692418,
"step": 150
},
{
"epoch": 0.04670510107275779,
"grad_norm": 0.01202392578125,
"learning_rate": 2.3323615160349855e-06,
"logits/chosen": -2.433961868286133,
"logits/rejected": -2.433961868286133,
"logps/chosen": -315.8016662597656,
"logps/rejected": -315.8016662597656,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": 0.00034690109896473587,
"rewards/margins": 0.0,
"rewards/rejected": 0.00034690109896473587,
"step": 160
},
{
"epoch": 0.04962416988980515,
"grad_norm": 0.01226806640625,
"learning_rate": 2.478134110787172e-06,
"logits/chosen": -2.4214272499084473,
"logits/rejected": -2.4214272499084473,
"logps/chosen": -304.0071105957031,
"logps/rejected": -304.0071105957031,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": 0.00021128072694409639,
"rewards/margins": 0.0,
"rewards/rejected": 0.00021128072694409639,
"step": 170
},
{
"epoch": 0.05254323870685251,
"grad_norm": 0.01318359375,
"learning_rate": 2.6239067055393585e-06,
"logits/chosen": -2.410125255584717,
"logits/rejected": -2.410125255584717,
"logps/chosen": -329.052978515625,
"logps/rejected": -329.052978515625,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -8.667710062582046e-05,
"rewards/margins": 0.0,
"rewards/rejected": -8.667710062582046e-05,
"step": 180
},
{
"epoch": 0.05546230752389988,
"grad_norm": 0.0111083984375,
"learning_rate": 2.7696793002915456e-06,
"logits/chosen": -2.412470579147339,
"logits/rejected": -2.412470579147339,
"logps/chosen": -302.9618225097656,
"logps/rejected": -302.9618225097656,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": 0.0005764733068645,
"rewards/margins": 0.0,
"rewards/rejected": 0.0005764733068645,
"step": 190
},
{
"epoch": 0.058381376340947234,
"grad_norm": 0.013671875,
"learning_rate": 2.9154518950437323e-06,
"logits/chosen": -2.3948373794555664,
"logits/rejected": -2.3948373794555664,
"logps/chosen": -312.7694091796875,
"logps/rejected": -312.7694091796875,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.0006709109293296933,
"rewards/margins": 0.0,
"rewards/rejected": -0.0006709109293296933,
"step": 200
},
{
"epoch": 0.058381376340947234,
"eval_logits/chosen": -2.3945627212524414,
"eval_logits/rejected": -2.3945627212524414,
"eval_logps/chosen": -306.5539245605469,
"eval_logps/rejected": -306.5539245605469,
"eval_loss": 0.6931472420692444,
"eval_rewards/accuracies": 0.0,
"eval_rewards/chosen": -0.0007532919407822192,
"eval_rewards/margins": 0.0,
"eval_rewards/rejected": -0.0007532919407822192,
"eval_runtime": 2667.9395,
"eval_samples_per_second": 2.283,
"eval_steps_per_second": 0.286,
"step": 200
},
{
"epoch": 0.0613004451579946,
"grad_norm": 0.0120849609375,
"learning_rate": 3.0612244897959185e-06,
"logits/chosen": -2.445885181427002,
"logits/rejected": -2.445885181427002,
"logps/chosen": -316.7839050292969,
"logps/rejected": -316.7839050292969,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.0010994909098371863,
"rewards/margins": 0.0,
"rewards/rejected": -0.0010994909098371863,
"step": 210
},
{
"epoch": 0.06421951397504196,
"grad_norm": 0.011962890625,
"learning_rate": 3.2069970845481052e-06,
"logits/chosen": -2.4333603382110596,
"logits/rejected": -2.4333603382110596,
"logps/chosen": -277.94915771484375,
"logps/rejected": -277.94915771484375,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.00041724619222804904,
"rewards/margins": 0.0,
"rewards/rejected": -0.00041724619222804904,
"step": 220
},
{
"epoch": 0.06713858279208933,
"grad_norm": 0.0145263671875,
"learning_rate": 3.352769679300292e-06,
"logits/chosen": -2.4338879585266113,
"logits/rejected": -2.4338879585266113,
"logps/chosen": -325.23455810546875,
"logps/rejected": -325.23455810546875,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.0012703577522188425,
"rewards/margins": 0.0,
"rewards/rejected": -0.0012703577522188425,
"step": 230
},
{
"epoch": 0.07005765160913668,
"grad_norm": 0.0157470703125,
"learning_rate": 3.4985422740524782e-06,
"logits/chosen": -2.413400173187256,
"logits/rejected": -2.413400173187256,
"logps/chosen": -309.69403076171875,
"logps/rejected": -309.69403076171875,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.001970961457118392,
"rewards/margins": 0.0,
"rewards/rejected": -0.001970961457118392,
"step": 240
},
{
"epoch": 0.07297672042618404,
"grad_norm": 0.01422119140625,
"learning_rate": 3.644314868804665e-06,
"logits/chosen": -2.4458959102630615,
"logits/rejected": -2.4458959102630615,
"logps/chosen": -304.130615234375,
"logps/rejected": -304.130615234375,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.0047234781086444855,
"rewards/margins": 0.0,
"rewards/rejected": -0.0047234781086444855,
"step": 250
},
{
"epoch": 0.07589578924323141,
"grad_norm": 0.01324462890625,
"learning_rate": 3.790087463556852e-06,
"logits/chosen": -2.4266982078552246,
"logits/rejected": -2.4266982078552246,
"logps/chosen": -286.97076416015625,
"logps/rejected": -286.97076416015625,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.006642763502895832,
"rewards/margins": 0.0,
"rewards/rejected": -0.006642763502895832,
"step": 260
},
{
"epoch": 0.07881485806027877,
"grad_norm": 0.015625,
"learning_rate": 3.935860058309039e-06,
"logits/chosen": -2.436506748199463,
"logits/rejected": -2.436506748199463,
"logps/chosen": -310.330322265625,
"logps/rejected": -310.330322265625,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.007435324601829052,
"rewards/margins": 0.0,
"rewards/rejected": -0.007435324601829052,
"step": 270
},
{
"epoch": 0.08173392687732613,
"grad_norm": 0.01495361328125,
"learning_rate": 4.081632653061225e-06,
"logits/chosen": -2.394254446029663,
"logits/rejected": -2.394254446029663,
"logps/chosen": -304.8192443847656,
"logps/rejected": -304.8192443847656,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.00737042585387826,
"rewards/margins": 0.0,
"rewards/rejected": -0.00737042585387826,
"step": 280
},
{
"epoch": 0.0846529956943735,
"grad_norm": 0.0130615234375,
"learning_rate": 4.227405247813411e-06,
"logits/chosen": -2.4005939960479736,
"logits/rejected": -2.4005939960479736,
"logps/chosen": -288.9790954589844,
"logps/rejected": -288.9790954589844,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.0066338046453893185,
"rewards/margins": 0.0,
"rewards/rejected": -0.0066338046453893185,
"step": 290
},
{
"epoch": 0.08757206451142086,
"grad_norm": 0.01458740234375,
"learning_rate": 4.3731778425655976e-06,
"logits/chosen": -2.4416656494140625,
"logits/rejected": -2.4416656494140625,
"logps/chosen": -288.1855773925781,
"logps/rejected": -288.1855773925781,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.0070955632254481316,
"rewards/margins": 0.0,
"rewards/rejected": -0.0070955632254481316,
"step": 300
},
{
"epoch": 0.08757206451142086,
"eval_logits/chosen": -2.3941876888275146,
"eval_logits/rejected": -2.3941876888275146,
"eval_logps/chosen": -307.0490417480469,
"eval_logps/rejected": -307.0490417480469,
"eval_loss": 0.6931472420692444,
"eval_rewards/accuracies": 0.0,
"eval_rewards/chosen": -0.005704815499484539,
"eval_rewards/margins": 0.0,
"eval_rewards/rejected": -0.005704815499484539,
"eval_runtime": 2667.7916,
"eval_samples_per_second": 2.283,
"eval_steps_per_second": 0.286,
"step": 300
},
{
"epoch": 0.09049113332846821,
"grad_norm": 0.01153564453125,
"learning_rate": 4.518950437317785e-06,
"logits/chosen": -2.420503854751587,
"logits/rejected": -2.420503854751587,
"logps/chosen": -276.64093017578125,
"logps/rejected": -276.64093017578125,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.007363935001194477,
"rewards/margins": 0.0,
"rewards/rejected": -0.007363935001194477,
"step": 310
},
{
"epoch": 0.09341020214551558,
"grad_norm": 0.0185546875,
"learning_rate": 4.664723032069971e-06,
"logits/chosen": -2.4066500663757324,
"logits/rejected": -2.4066500663757324,
"logps/chosen": -315.653076171875,
"logps/rejected": -315.653076171875,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.006720393896102905,
"rewards/margins": 0.0,
"rewards/rejected": -0.006720393896102905,
"step": 320
},
{
"epoch": 0.09632927096256294,
"grad_norm": 0.015625,
"learning_rate": 4.810495626822158e-06,
"logits/chosen": -2.445965528488159,
"logits/rejected": -2.445965528488159,
"logps/chosen": -324.6703796386719,
"logps/rejected": -324.6703796386719,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.007294761948287487,
"rewards/margins": 0.0,
"rewards/rejected": -0.007294761948287487,
"step": 330
},
{
"epoch": 0.0992483397796103,
"grad_norm": 0.01446533203125,
"learning_rate": 4.956268221574344e-06,
"logits/chosen": -2.4288485050201416,
"logits/rejected": -2.4288485050201416,
"logps/chosen": -323.6286926269531,
"logps/rejected": -323.6286926269531,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.009048479609191418,
"rewards/margins": 0.0,
"rewards/rejected": -0.009048479609191418,
"step": 340
},
{
"epoch": 0.10216740859665767,
"grad_norm": 0.01458740234375,
"learning_rate": 4.999936358746211e-06,
"logits/chosen": -2.4309639930725098,
"logits/rejected": -2.4309639930725098,
"logps/chosen": -271.655029296875,
"logps/rejected": -271.655029296875,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.009400355629622936,
"rewards/margins": 0.0,
"rewards/rejected": -0.009400355629622936,
"step": 350
},
{
"epoch": 0.10508647741370503,
"grad_norm": 0.0152587890625,
"learning_rate": 4.99962465428288e-06,
"logits/chosen": -2.4447290897369385,
"logits/rejected": -2.4447290897369385,
"logps/chosen": -303.4416198730469,
"logps/rejected": -303.4416198730469,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.013941009528934956,
"rewards/margins": 0.0,
"rewards/rejected": -0.013941009528934956,
"step": 360
},
{
"epoch": 0.10800554623075238,
"grad_norm": 0.0185546875,
"learning_rate": 4.999053229746866e-06,
"logits/chosen": -2.440117359161377,
"logits/rejected": -2.440117359161377,
"logps/chosen": -290.806884765625,
"logps/rejected": -290.806884765625,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.01759205386042595,
"rewards/margins": 0.0,
"rewards/rejected": -0.01759205386042595,
"step": 370
},
{
"epoch": 0.11092461504779975,
"grad_norm": 0.01263427734375,
"learning_rate": 4.9982221445112535e-06,
"logits/chosen": -2.4275150299072266,
"logits/rejected": -2.4275150299072266,
"logps/chosen": -320.67938232421875,
"logps/rejected": -320.67938232421875,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.018790820613503456,
"rewards/margins": 0.0,
"rewards/rejected": -0.018790820613503456,
"step": 380
},
{
"epoch": 0.11384368386484711,
"grad_norm": 0.01397705078125,
"learning_rate": 4.997131484928813e-06,
"logits/chosen": -2.414685010910034,
"logits/rejected": -2.414685010910034,
"logps/chosen": -301.1441650390625,
"logps/rejected": -301.1441650390625,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.015089067630469799,
"rewards/margins": 0.0,
"rewards/rejected": -0.015089067630469799,
"step": 390
},
{
"epoch": 0.11676275268189447,
"grad_norm": 0.01458740234375,
"learning_rate": 4.995781364323035e-06,
"logits/chosen": -2.391239643096924,
"logits/rejected": -2.391239643096924,
"logps/chosen": -285.70941162109375,
"logps/rejected": -285.70941162109375,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.010374903678894043,
"rewards/margins": 0.0,
"rewards/rejected": -0.010374903678894043,
"step": 400
},
{
"epoch": 0.11676275268189447,
"eval_logits/chosen": -2.393982172012329,
"eval_logits/rejected": -2.393982172012329,
"eval_logps/chosen": -307.3796081542969,
"eval_logps/rejected": -307.3796081542969,
"eval_loss": 0.6931472420692444,
"eval_rewards/accuracies": 0.0,
"eval_rewards/chosen": -0.009010241366922855,
"eval_rewards/margins": 0.0,
"eval_rewards/rejected": -0.009010241366922855,
"eval_runtime": 2667.3233,
"eval_samples_per_second": 2.283,
"eval_steps_per_second": 0.286,
"step": 400
},
{
"epoch": 0.11968182149894184,
"grad_norm": 0.01300048828125,
"learning_rate": 4.994171922976349e-06,
"logits/chosen": -2.4642019271850586,
"logits/rejected": -2.4642019271850586,
"logps/chosen": -298.46978759765625,
"logps/rejected": -298.46978759765625,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.009510824456810951,
"rewards/margins": 0.0,
"rewards/rejected": -0.009510824456810951,
"step": 410
},
{
"epoch": 0.1226008903159892,
"grad_norm": 0.0159912109375,
"learning_rate": 4.992303328115551e-06,
"logits/chosen": -2.420297145843506,
"logits/rejected": -2.420297145843506,
"logps/chosen": -306.69610595703125,
"logps/rejected": -306.69610595703125,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.0014959282707422972,
"rewards/margins": 0.0,
"rewards/rejected": -0.0014959282707422972,
"step": 420
},
{
"epoch": 0.12551995913303657,
"grad_norm": 0.0159912109375,
"learning_rate": 4.990175773894428e-06,
"logits/chosen": -2.46386981010437,
"logits/rejected": -2.46386981010437,
"logps/chosen": -281.81097412109375,
"logps/rejected": -281.81097412109375,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.008724676445126534,
"rewards/margins": 0.0,
"rewards/rejected": -0.008724676445126534,
"step": 430
},
{
"epoch": 0.1284390279500839,
"grad_norm": 0.01287841796875,
"learning_rate": 4.987789481373586e-06,
"logits/chosen": -2.406324625015259,
"logits/rejected": -2.406324625015259,
"logps/chosen": -297.7574157714844,
"logps/rejected": -297.7574157714844,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.006952273193746805,
"rewards/margins": 0.0,
"rewards/rejected": -0.006952273193746805,
"step": 440
},
{
"epoch": 0.13135809676713128,
"grad_norm": 0.015869140625,
"learning_rate": 4.985144698497477e-06,
"logits/chosen": -2.4094862937927246,
"logits/rejected": -2.4094862937927246,
"logps/chosen": -294.4402160644531,
"logps/rejected": -294.4402160644531,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.009783747605979443,
"rewards/margins": 0.0,
"rewards/rejected": -0.009783747605979443,
"step": 450
},
{
"epoch": 0.13427716558417865,
"grad_norm": 0.015625,
"learning_rate": 4.982241700068639e-06,
"logits/chosen": -2.448880434036255,
"logits/rejected": -2.448880434036255,
"logps/chosen": -312.9103088378906,
"logps/rejected": -312.9103088378906,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.010099256411194801,
"rewards/margins": 0.0,
"rewards/rejected": -0.010099256411194801,
"step": 460
},
{
"epoch": 0.137196234401226,
"grad_norm": 0.014404296875,
"learning_rate": 4.979080787719144e-06,
"logits/chosen": -2.4513556957244873,
"logits/rejected": -2.4513556957244873,
"logps/chosen": -330.3889465332031,
"logps/rejected": -330.3889465332031,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.012815428897738457,
"rewards/margins": 0.0,
"rewards/rejected": -0.012815428897738457,
"step": 470
},
{
"epoch": 0.14011530321827337,
"grad_norm": 0.013427734375,
"learning_rate": 4.975662289879257e-06,
"logits/chosen": -2.3824195861816406,
"logits/rejected": -2.3824195861816406,
"logps/chosen": -324.45654296875,
"logps/rejected": -324.45654296875,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.010385606437921524,
"rewards/margins": 0.0,
"rewards/rejected": -0.010385606437921524,
"step": 480
},
{
"epoch": 0.14303437203532074,
"grad_norm": 0.016845703125,
"learning_rate": 4.971986561743308e-06,
"logits/chosen": -2.388378620147705,
"logits/rejected": -2.388378620147705,
"logps/chosen": -292.9872131347656,
"logps/rejected": -292.9872131347656,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.00819515809416771,
"rewards/margins": 0.0,
"rewards/rejected": -0.00819515809416771,
"step": 490
},
{
"epoch": 0.14595344085236808,
"grad_norm": 0.01348876953125,
"learning_rate": 4.96805398523279e-06,
"logits/chosen": -2.438722610473633,
"logits/rejected": -2.438722610473633,
"logps/chosen": -333.7470397949219,
"logps/rejected": -333.7470397949219,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.007836517877876759,
"rewards/margins": 0.0,
"rewards/rejected": -0.007836517877876759,
"step": 500
},
{
"epoch": 0.14595344085236808,
"eval_logits/chosen": -2.3937265872955322,
"eval_logits/rejected": -2.3937265872955322,
"eval_logps/chosen": -307.1580505371094,
"eval_logps/rejected": -307.1580505371094,
"eval_loss": 0.6931472420692444,
"eval_rewards/accuracies": 0.0,
"eval_rewards/chosen": -0.006794503424316645,
"eval_rewards/margins": 0.0,
"eval_rewards/rejected": -0.006794503424316645,
"eval_runtime": 2668.7964,
"eval_samples_per_second": 2.282,
"eval_steps_per_second": 0.286,
"step": 500
},
{
"epoch": 0.14887250966941545,
"grad_norm": 0.0146484375,
"learning_rate": 4.963864968956674e-06,
"logits/chosen": -2.4363291263580322,
"logits/rejected": -2.4363291263580322,
"logps/chosen": -295.4735412597656,
"logps/rejected": -295.4735412597656,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.008334552869200706,
"rewards/margins": 0.0,
"rewards/rejected": -0.008334552869200706,
"step": 510
},
{
"epoch": 0.15179157848646282,
"grad_norm": 0.0113525390625,
"learning_rate": 4.959419948168952e-06,
"logits/chosen": -2.4209957122802734,
"logits/rejected": -2.4209957122802734,
"logps/chosen": -252.09475708007812,
"logps/rejected": -252.09475708007812,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.005244333762675524,
"rewards/margins": 0.0,
"rewards/rejected": -0.005244333762675524,
"step": 520
},
{
"epoch": 0.15471064730351017,
"grad_norm": 0.011962890625,
"learning_rate": 4.954719384723416e-06,
"logits/chosen": -2.4421539306640625,
"logits/rejected": -2.4421539306640625,
"logps/chosen": -290.62939453125,
"logps/rejected": -290.62939453125,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.006143758539110422,
"rewards/margins": 0.0,
"rewards/rejected": -0.006143758539110422,
"step": 530
},
{
"epoch": 0.15762971612055754,
"grad_norm": 0.0155029296875,
"learning_rate": 4.949763767025665e-06,
"logits/chosen": -2.433292865753174,
"logits/rejected": -2.433292865753174,
"logps/chosen": -301.56488037109375,
"logps/rejected": -301.56488037109375,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.007085380610078573,
"rewards/margins": 0.0,
"rewards/rejected": -0.007085380610078573,
"step": 540
},
{
"epoch": 0.1605487849376049,
"grad_norm": 0.01513671875,
"learning_rate": 4.944553609982363e-06,
"logits/chosen": -2.397106647491455,
"logits/rejected": -2.397106647491455,
"logps/chosen": -274.3099670410156,
"logps/rejected": -274.3099670410156,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.002214896958321333,
"rewards/margins": 0.0,
"rewards/rejected": -0.002214896958321333,
"step": 550
},
{
"epoch": 0.16346785375465225,
"grad_norm": 0.0152587890625,
"learning_rate": 4.939089454947734e-06,
"logits/chosen": -2.417797088623047,
"logits/rejected": -2.417797088623047,
"logps/chosen": -299.5130615234375,
"logps/rejected": -299.5130615234375,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.005161653272807598,
"rewards/margins": 0.0,
"rewards/rejected": -0.005161653272807598,
"step": 560
},
{
"epoch": 0.16638692257169962,
"grad_norm": 0.01507568359375,
"learning_rate": 4.933371869667315e-06,
"logits/chosen": -2.4109036922454834,
"logits/rejected": -2.4109036922454834,
"logps/chosen": -279.4015808105469,
"logps/rejected": -279.4015808105469,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.0024168032687157393,
"rewards/margins": 0.0,
"rewards/rejected": -0.0024168032687157393,
"step": 570
},
{
"epoch": 0.169305991388747,
"grad_norm": 0.00885009765625,
"learning_rate": 4.9274014482189654e-06,
"logits/chosen": -2.4315690994262695,
"logits/rejected": -2.4315690994262695,
"logps/chosen": -309.34234619140625,
"logps/rejected": -309.34234619140625,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.004016817547380924,
"rewards/margins": 0.0,
"rewards/rejected": -0.004016817547380924,
"step": 580
},
{
"epoch": 0.17222506020579434,
"grad_norm": 0.017578125,
"learning_rate": 4.9211788109511405e-06,
"logits/chosen": -2.460508108139038,
"logits/rejected": -2.460508108139038,
"logps/chosen": -334.00933837890625,
"logps/rejected": -334.00933837890625,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.005641533527523279,
"rewards/margins": 0.0,
"rewards/rejected": -0.005641533527523279,
"step": 590
},
{
"epoch": 0.1751441290228417,
"grad_norm": 0.016845703125,
"learning_rate": 4.914704604418435e-06,
"logits/chosen": -2.4566855430603027,
"logits/rejected": -2.4566855430603027,
"logps/chosen": -307.21331787109375,
"logps/rejected": -307.21331787109375,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.0077440254390239716,
"rewards/margins": 0.0,
"rewards/rejected": -0.0077440254390239716,
"step": 600
},
{
"epoch": 0.1751441290228417,
"eval_logits/chosen": -2.394993782043457,
"eval_logits/rejected": -2.394993782043457,
"eval_logps/chosen": -306.9631042480469,
"eval_logps/rejected": -306.9631042480469,
"eval_loss": 0.6931472420692444,
"eval_rewards/accuracies": 0.0,
"eval_rewards/chosen": -0.004845078103244305,
"eval_rewards/margins": 0.0,
"eval_rewards/rejected": -0.004845078103244305,
"eval_runtime": 2667.3075,
"eval_samples_per_second": 2.283,
"eval_steps_per_second": 0.286,
"step": 600
},
{
"epoch": 0.17806319783988908,
"grad_norm": 0.01312255859375,
"learning_rate": 4.907979501314402e-06,
"logits/chosen": -2.452761173248291,
"logits/rejected": -2.452761173248291,
"logps/chosen": -293.330078125,
"logps/rejected": -293.330078125,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.005413960665464401,
"rewards/margins": 0.0,
"rewards/rejected": -0.005413960665464401,
"step": 610
},
{
"epoch": 0.18098226665693642,
"grad_norm": 0.013427734375,
"learning_rate": 4.901004200401659e-06,
"logits/chosen": -2.415590763092041,
"logits/rejected": -2.415590763092041,
"logps/chosen": -316.59185791015625,
"logps/rejected": -316.59185791015625,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.009168794378638268,
"rewards/margins": 0.0,
"rewards/rejected": -0.009168794378638268,
"step": 620
},
{
"epoch": 0.1839013354739838,
"grad_norm": 0.017333984375,
"learning_rate": 4.893779426439285e-06,
"logits/chosen": -2.4269957542419434,
"logits/rejected": -2.4269957542419434,
"logps/chosen": -330.297607421875,
"logps/rejected": -330.297607421875,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.008635496720671654,
"rewards/margins": 0.0,
"rewards/rejected": -0.008635496720671654,
"step": 630
},
{
"epoch": 0.18682040429103117,
"grad_norm": 0.0137939453125,
"learning_rate": 4.886305930107512e-06,
"logits/chosen": -2.4132332801818848,
"logits/rejected": -2.4132332801818848,
"logps/chosen": -334.0628967285156,
"logps/rejected": -334.0628967285156,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.00843154825270176,
"rewards/margins": 0.0,
"rewards/rejected": -0.00843154825270176,
"step": 640
},
{
"epoch": 0.1897394731080785,
"grad_norm": 0.0162353515625,
"learning_rate": 4.878584487929731e-06,
"logits/chosen": -2.393531084060669,
"logits/rejected": -2.393531084060669,
"logps/chosen": -312.2678527832031,
"logps/rejected": -312.2678527832031,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.008157333359122276,
"rewards/margins": 0.0,
"rewards/rejected": -0.008157333359122276,
"step": 650
},
{
"epoch": 0.19265854192512588,
"grad_norm": 0.01141357421875,
"learning_rate": 4.8706159021918046e-06,
"logits/chosen": -2.4334394931793213,
"logits/rejected": -2.4334394931793213,
"logps/chosen": -313.9178466796875,
"logps/rejected": -313.9178466796875,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.010157248005270958,
"rewards/margins": 0.0,
"rewards/rejected": -0.010157248005270958,
"step": 660
},
{
"epoch": 0.19557761074217325,
"grad_norm": 0.01446533203125,
"learning_rate": 4.86240100085871e-06,
"logits/chosen": -2.4123024940490723,
"logits/rejected": -2.4123024940490723,
"logps/chosen": -330.71856689453125,
"logps/rejected": -330.71856689453125,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.01049681194126606,
"rewards/margins": 0.0,
"rewards/rejected": -0.01049681194126606,
"step": 670
},
{
"epoch": 0.1984966795592206,
"grad_norm": 0.0145263671875,
"learning_rate": 4.853940637488505e-06,
"logits/chosen": -2.4219470024108887,
"logits/rejected": -2.4219470024108887,
"logps/chosen": -347.1614990234375,
"logps/rejected": -347.1614990234375,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.010124921798706055,
"rewards/margins": 0.0,
"rewards/rejected": -0.010124921798706055,
"step": 680
},
{
"epoch": 0.20141574837626797,
"grad_norm": 0.0140380859375,
"learning_rate": 4.84523569114365e-06,
"logits/chosen": -2.441845417022705,
"logits/rejected": -2.441845417022705,
"logps/chosen": -268.2397766113281,
"logps/rejected": -268.2397766113281,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.013552245683968067,
"rewards/margins": 0.0,
"rewards/rejected": -0.013552245683968067,
"step": 690
},
{
"epoch": 0.20433481719331534,
"grad_norm": 0.020751953125,
"learning_rate": 4.8362870662996574e-06,
"logits/chosen": -2.408205509185791,
"logits/rejected": -2.408205509185791,
"logps/chosen": -313.0887756347656,
"logps/rejected": -313.0887756347656,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.01138794980943203,
"rewards/margins": 0.0,
"rewards/rejected": -0.01138794980943203,
"step": 700
},
{
"epoch": 0.20433481719331534,
"eval_logits/chosen": -2.394869565963745,
"eval_logits/rejected": -2.394869565963745,
"eval_logps/chosen": -307.6349182128906,
"eval_logps/rejected": -307.6349182128906,
"eval_loss": 0.6931472420692444,
"eval_rewards/accuracies": 0.0,
"eval_rewards/chosen": -0.011563203297555447,
"eval_rewards/margins": 0.0,
"eval_rewards/rejected": -0.011563203297555447,
"eval_runtime": 2685.1829,
"eval_samples_per_second": 2.268,
"eval_steps_per_second": 0.284,
"step": 700
},
{
"epoch": 0.20725388601036268,
"grad_norm": 0.015380859375,
"learning_rate": 4.827095692751124e-06,
"logits/chosen": -2.4306788444519043,
"logits/rejected": -2.4306788444519043,
"logps/chosen": -295.8254089355469,
"logps/rejected": -295.8254089355469,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.009687040001153946,
"rewards/margins": 0.0,
"rewards/rejected": -0.009687040001153946,
"step": 710
},
{
"epoch": 0.21017295482741005,
"grad_norm": 0.0135498046875,
"learning_rate": 4.817662525515116e-06,
"logits/chosen": -2.399963855743408,
"logits/rejected": -2.399963855743408,
"logps/chosen": -285.0207824707031,
"logps/rejected": -285.0207824707031,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.010509965009987354,
"rewards/margins": 0.0,
"rewards/rejected": -0.010509965009987354,
"step": 720
},
{
"epoch": 0.21309202364445742,
"grad_norm": 0.01275634765625,
"learning_rate": 4.807988544731944e-06,
"logits/chosen": -2.4015610218048096,
"logits/rejected": -2.4015610218048096,
"logps/chosen": -301.6191711425781,
"logps/rejected": -301.6191711425781,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.0023958988022059202,
"rewards/margins": 0.0,
"rewards/rejected": -0.0023958988022059202,
"step": 730
},
{
"epoch": 0.21601109246150477,
"grad_norm": 0.0120849609375,
"learning_rate": 4.7980747555633174e-06,
"logits/chosen": -2.421522617340088,
"logits/rejected": -2.421522617340088,
"logps/chosen": -300.5765380859375,
"logps/rejected": -300.5765380859375,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.0066505610011518,
"rewards/margins": 0.0,
"rewards/rejected": -0.0066505610011518,
"step": 740
},
{
"epoch": 0.21893016127855214,
"grad_norm": 0.0167236328125,
"learning_rate": 4.787922188087907e-06,
"logits/chosen": -2.3898696899414062,
"logits/rejected": -2.3898696899414062,
"logps/chosen": -312.099853515625,
"logps/rejected": -312.099853515625,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.009563307277858257,
"rewards/margins": 0.0,
"rewards/rejected": -0.009563307277858257,
"step": 750
},
{
"epoch": 0.2218492300955995,
"grad_norm": 0.0185546875,
"learning_rate": 4.7775318971943165e-06,
"logits/chosen": -2.368053674697876,
"logits/rejected": -2.368053674697876,
"logps/chosen": -280.77703857421875,
"logps/rejected": -280.77703857421875,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.008711813017725945,
"rewards/margins": 0.0,
"rewards/rejected": -0.008711813017725945,
"step": 760
},
{
"epoch": 0.22476829891264685,
"grad_norm": 0.01434326171875,
"learning_rate": 4.766904962471477e-06,
"logits/chosen": -2.428321361541748,
"logits/rejected": -2.428321361541748,
"logps/chosen": -283.40704345703125,
"logps/rejected": -283.40704345703125,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.0074463835917413235,
"rewards/margins": 0.0,
"rewards/rejected": -0.0074463835917413235,
"step": 770
},
{
"epoch": 0.22768736772969422,
"grad_norm": 0.020751953125,
"learning_rate": 4.756042488096472e-06,
"logits/chosen": -2.421441078186035,
"logits/rejected": -2.421441078186035,
"logps/chosen": -283.1347961425781,
"logps/rejected": -283.1347961425781,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.009277506731450558,
"rewards/margins": 0.0,
"rewards/rejected": -0.009277506731450558,
"step": 780
},
{
"epoch": 0.2306064365467416,
"grad_norm": 0.0169677734375,
"learning_rate": 4.744945602719806e-06,
"logits/chosen": -2.4225807189941406,
"logits/rejected": -2.4225807189941406,
"logps/chosen": -296.5173645019531,
"logps/rejected": -296.5173645019531,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.009408360347151756,
"rewards/margins": 0.0,
"rewards/rejected": -0.009408360347151756,
"step": 790
},
{
"epoch": 0.23352550536378894,
"grad_norm": 0.01495361328125,
"learning_rate": 4.733615459348143e-06,
"logits/chosen": -2.3777918815612793,
"logits/rejected": -2.3777918815612793,
"logps/chosen": -337.0318298339844,
"logps/rejected": -337.0318298339844,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.012588550336658955,
"rewards/margins": 0.0,
"rewards/rejected": -0.012588550336658955,
"step": 800
},
{
"epoch": 0.23352550536378894,
"eval_logits/chosen": -2.394713878631592,
"eval_logits/rejected": -2.394713878631592,
"eval_logps/chosen": -307.6956787109375,
"eval_logps/rejected": -307.6956787109375,
"eval_loss": 0.6931472420692444,
"eval_rewards/accuracies": 0.0,
"eval_rewards/chosen": -0.012170875445008278,
"eval_rewards/margins": 0.0,
"eval_rewards/rejected": -0.012170875445008278,
"eval_runtime": 2762.1462,
"eval_samples_per_second": 2.205,
"eval_steps_per_second": 0.276,
"step": 800
},
{
"epoch": 0.2364445741808363,
"grad_norm": 0.0145263671875,
"learning_rate": 4.722053235224495e-06,
"logits/chosen": -2.4402616024017334,
"logits/rejected": -2.4402616024017334,
"logps/chosen": -333.5353698730469,
"logps/rejected": -333.5353698730469,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.008296088315546513,
"rewards/margins": 0.0,
"rewards/rejected": -0.008296088315546513,
"step": 810
},
{
"epoch": 0.23936364299788368,
"grad_norm": 0.0128173828125,
"learning_rate": 4.710260131705908e-06,
"logits/chosen": -2.411567211151123,
"logits/rejected": -2.411567211151123,
"logps/chosen": -274.9350280761719,
"logps/rejected": -274.9350280761719,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.015997527167201042,
"rewards/margins": 0.0,
"rewards/rejected": -0.015997527167201042,
"step": 820
},
{
"epoch": 0.24228271181493102,
"grad_norm": 0.01531982421875,
"learning_rate": 4.698237374138634e-06,
"logits/chosen": -2.420203447341919,
"logits/rejected": -2.420203447341919,
"logps/chosen": -312.3550720214844,
"logps/rejected": -312.3550720214844,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.015846502035856247,
"rewards/margins": 0.0,
"rewards/rejected": -0.015846502035856247,
"step": 830
},
{
"epoch": 0.2452017806319784,
"grad_norm": 0.01513671875,
"learning_rate": 4.685986211730816e-06,
"logits/chosen": -2.3960068225860596,
"logits/rejected": -2.3960068225860596,
"logps/chosen": -331.6641845703125,
"logps/rejected": -331.6641845703125,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.01894356682896614,
"rewards/margins": 0.0,
"rewards/rejected": -0.01894356682896614,
"step": 840
},
{
"epoch": 0.24812084944902577,
"grad_norm": 0.01165771484375,
"learning_rate": 4.6735079174226864e-06,
"logits/chosen": -2.408433198928833,
"logits/rejected": -2.408433198928833,
"logps/chosen": -269.3624572753906,
"logps/rejected": -269.3624572753906,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.009970271959900856,
"rewards/margins": 0.0,
"rewards/rejected": -0.009970271959900856,
"step": 850
},
{
"epoch": 0.25103991826607314,
"grad_norm": 0.01483154296875,
"learning_rate": 4.660803787754306e-06,
"logits/chosen": -2.416790723800659,
"logits/rejected": -2.416790723800659,
"logps/chosen": -302.0819396972656,
"logps/rejected": -302.0819396972656,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.010707431472837925,
"rewards/margins": 0.0,
"rewards/rejected": -0.010707431472837925,
"step": 860
},
{
"epoch": 0.2539589870831205,
"grad_norm": 0.0157470703125,
"learning_rate": 4.647875142730853e-06,
"logits/chosen": -2.3868987560272217,
"logits/rejected": -2.3868987560272217,
"logps/chosen": -299.74444580078125,
"logps/rejected": -299.74444580078125,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.012594198808073997,
"rewards/margins": 0.0,
"rewards/rejected": -0.012594198808073997,
"step": 870
},
{
"epoch": 0.2568780559001678,
"grad_norm": 0.0140380859375,
"learning_rate": 4.634723325685462e-06,
"logits/chosen": -2.442610263824463,
"logits/rejected": -2.442610263824463,
"logps/chosen": -308.396240234375,
"logps/rejected": -308.396240234375,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.011057281866669655,
"rewards/margins": 0.0,
"rewards/rejected": -0.011057281866669655,
"step": 880
},
{
"epoch": 0.2597971247172152,
"grad_norm": 0.0157470703125,
"learning_rate": 4.621349703139651e-06,
"logits/chosen": -2.4502758979797363,
"logits/rejected": -2.4502758979797363,
"logps/chosen": -327.5845031738281,
"logps/rejected": -327.5845031738281,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.012428502552211285,
"rewards/margins": 0.0,
"rewards/rejected": -0.012428502552211285,
"step": 890
},
{
"epoch": 0.26271619353426257,
"grad_norm": 0.01519775390625,
"learning_rate": 4.6077556646613365e-06,
"logits/chosen": -2.4429335594177246,
"logits/rejected": -2.4429335594177246,
"logps/chosen": -309.44598388671875,
"logps/rejected": -309.44598388671875,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.008300786837935448,
"rewards/margins": 0.0,
"rewards/rejected": -0.008300786837935448,
"step": 900
},
{
"epoch": 0.26271619353426257,
"eval_logits/chosen": -2.396768093109131,
"eval_logits/rejected": -2.396768093109131,
"eval_logps/chosen": -307.1708068847656,
"eval_logps/rejected": -307.1708068847656,
"eval_loss": 0.6931472420692444,
"eval_rewards/accuracies": 0.0,
"eval_rewards/chosen": -0.0069224112667143345,
"eval_rewards/margins": 0.0,
"eval_rewards/rejected": -0.0069224112667143345,
"eval_runtime": 2667.1988,
"eval_samples_per_second": 2.283,
"eval_steps_per_second": 0.286,
"step": 900
},
{
"epoch": 0.2656352623513099,
"grad_norm": 0.0150146484375,
"learning_rate": 4.593942622720449e-06,
"logits/chosen": -2.431570529937744,
"logits/rejected": -2.431570529937744,
"logps/chosen": -333.9033203125,
"logps/rejected": -333.9033203125,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.008790754713118076,
"rewards/margins": 0.0,
"rewards/rejected": -0.008790754713118076,
"step": 910
},
{
"epoch": 0.2685543311683573,
"grad_norm": 0.011474609375,
"learning_rate": 4.579912012542172e-06,
"logits/chosen": -2.4538259506225586,
"logits/rejected": -2.4538259506225586,
"logps/chosen": -330.14776611328125,
"logps/rejected": -330.14776611328125,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.010161884129047394,
"rewards/margins": 0.0,
"rewards/rejected": -0.010161884129047394,
"step": 920
},
{
"epoch": 0.27147339998540465,
"grad_norm": 0.0164794921875,
"learning_rate": 4.565665291957821e-06,
"logits/chosen": -2.412051200866699,
"logits/rejected": -2.412051200866699,
"logps/chosen": -300.0600891113281,
"logps/rejected": -300.0600891113281,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.010114507749676704,
"rewards/margins": 0.0,
"rewards/rejected": -0.010114507749676704,
"step": 930
},
{
"epoch": 0.274392468802452,
"grad_norm": 0.0125732421875,
"learning_rate": 4.551203941253367e-06,
"logits/chosen": -2.4353108406066895,
"logits/rejected": -2.4353108406066895,
"logps/chosen": -288.15032958984375,
"logps/rejected": -288.15032958984375,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.01036372222006321,
"rewards/margins": 0.0,
"rewards/rejected": -0.01036372222006321,
"step": 940
},
{
"epoch": 0.2773115376194994,
"grad_norm": 0.01434326171875,
"learning_rate": 4.5365294630156264e-06,
"logits/chosen": -2.4350383281707764,
"logits/rejected": -2.4350383281707764,
"logps/chosen": -319.06195068359375,
"logps/rejected": -319.06195068359375,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.011402562260627747,
"rewards/margins": 0.0,
"rewards/rejected": -0.011402562260627747,
"step": 950
},
{
"epoch": 0.28023060643654674,
"grad_norm": 0.012451171875,
"learning_rate": 4.521643381976142e-06,
"logits/chosen": -2.428330898284912,
"logits/rejected": -2.428330898284912,
"logps/chosen": -322.0547790527344,
"logps/rejected": -322.0547790527344,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.012878289446234703,
"rewards/margins": 0.0,
"rewards/rejected": -0.012878289446234703,
"step": 960
},
{
"epoch": 0.2831496752535941,
"grad_norm": 0.013671875,
"learning_rate": 4.506547244852756e-06,
"logits/chosen": -2.4220213890075684,
"logits/rejected": -2.4220213890075684,
"logps/chosen": -298.77056884765625,
"logps/rejected": -298.77056884765625,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.012091143056750298,
"rewards/margins": 0.0,
"rewards/rejected": -0.012091143056750298,
"step": 970
},
{
"epoch": 0.2860687440706415,
"grad_norm": 0.0145263671875,
"learning_rate": 4.491242620188898e-06,
"logits/chosen": -2.400778293609619,
"logits/rejected": -2.400778293609619,
"logps/chosen": -302.7762756347656,
"logps/rejected": -302.7762756347656,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.01696743816137314,
"rewards/margins": 0.0,
"rewards/rejected": -0.01696743816137314,
"step": 980
},
{
"epoch": 0.2889878128876888,
"grad_norm": 0.012451171875,
"learning_rate": 4.475731098190611e-06,
"logits/chosen": -2.4159862995147705,
"logits/rejected": -2.4159862995147705,
"logps/chosen": -278.34356689453125,
"logps/rejected": -278.34356689453125,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.014010600745677948,
"rewards/margins": 0.0,
"rewards/rejected": -0.014010600745677948,
"step": 990
},
{
"epoch": 0.29190688170473617,
"grad_norm": 0.0145263671875,
"learning_rate": 4.4600142905613216e-06,
"logits/chosen": -2.416891098022461,
"logits/rejected": -2.416891098022461,
"logps/chosen": -310.4523620605469,
"logps/rejected": -310.4523620605469,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.01909947767853737,
"rewards/margins": 0.0,
"rewards/rejected": -0.01909947767853737,
"step": 1000
},
{
"epoch": 0.29190688170473617,
"eval_logits/chosen": -2.3967111110687256,
"eval_logits/rejected": -2.3967111110687256,
"eval_logps/chosen": -308.2130432128906,
"eval_logps/rejected": -308.2130432128906,
"eval_loss": 0.6931472420692444,
"eval_rewards/accuracies": 0.0,
"eval_rewards/chosen": -0.017344659194350243,
"eval_rewards/margins": 0.0,
"eval_rewards/rejected": -0.017344659194350243,
"eval_runtime": 2668.0913,
"eval_samples_per_second": 2.283,
"eval_steps_per_second": 0.286,
"step": 1000
},
{
"epoch": 0.29482595052178356,
"grad_norm": 0.029296875,
"learning_rate": 4.444093830334381e-06,
"logits/chosen": -2.395017147064209,
"logits/rejected": -2.395017147064209,
"logps/chosen": -330.1224670410156,
"logps/rejected": -330.1224670410156,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.013958754017949104,
"rewards/margins": 0.0,
"rewards/rejected": -0.013958754017949104,
"step": 1010
},
{
"epoch": 0.2977450193388309,
"grad_norm": 0.01611328125,
"learning_rate": 4.427971371703378e-06,
"logits/chosen": -2.4404492378234863,
"logits/rejected": -2.4404492378234863,
"logps/chosen": -314.79888916015625,
"logps/rejected": -314.79888916015625,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.027685949578881264,
"rewards/margins": 0.0,
"rewards/rejected": -0.027685949578881264,
"step": 1020
},
{
"epoch": 0.30066408815587825,
"grad_norm": 0.01263427734375,
"learning_rate": 4.411648589850276e-06,
"logits/chosen": -2.4368889331817627,
"logits/rejected": -2.4368889331817627,
"logps/chosen": -299.6970520019531,
"logps/rejected": -299.6970520019531,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.01648426428437233,
"rewards/margins": 0.0,
"rewards/rejected": -0.01648426428437233,
"step": 1030
},
{
"epoch": 0.30358315697292565,
"grad_norm": 0.01416015625,
"learning_rate": 4.395127180771342e-06,
"logits/chosen": -2.4541175365448,
"logits/rejected": -2.4541175365448,
"logps/chosen": -326.87841796875,
"logps/rejected": -326.87841796875,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.020237499848008156,
"rewards/margins": 0.0,
"rewards/rejected": -0.020237499848008156,
"step": 1040
},
{
"epoch": 0.306502225789973,
"grad_norm": 0.01318359375,
"learning_rate": 4.378408861100937e-06,
"logits/chosen": -2.415283203125,
"logits/rejected": -2.415283203125,
"logps/chosen": -261.1552429199219,
"logps/rejected": -261.1552429199219,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.01741962879896164,
"rewards/margins": 0.0,
"rewards/rejected": -0.01741962879896164,
"step": 1050
},
{
"epoch": 0.30942129460702034,
"grad_norm": 0.01416015625,
"learning_rate": 4.361495367933144e-06,
"logits/chosen": -2.396031141281128,
"logits/rejected": -2.396031141281128,
"logps/chosen": -322.30377197265625,
"logps/rejected": -322.30377197265625,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.014474359340965748,
"rewards/margins": 0.0,
"rewards/rejected": -0.014474359340965748,
"step": 1060
},
{
"epoch": 0.31234036342406774,
"grad_norm": 0.0181884765625,
"learning_rate": 4.344388458641283e-06,
"logits/chosen": -2.4288814067840576,
"logits/rejected": -2.4288814067840576,
"logps/chosen": -324.64501953125,
"logps/rejected": -324.64501953125,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.025701653212308884,
"rewards/margins": 0.0,
"rewards/rejected": -0.025701653212308884,
"step": 1070
},
{
"epoch": 0.3152594322411151,
"grad_norm": 0.0164794921875,
"learning_rate": 4.32708991069531e-06,
"logits/chosen": -2.411003589630127,
"logits/rejected": -2.411003589630127,
"logps/chosen": -318.289794921875,
"logps/rejected": -318.289794921875,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.02725202962756157,
"rewards/margins": 0.0,
"rewards/rejected": -0.02725202962756157,
"step": 1080
},
{
"epoch": 0.3181785010581624,
"grad_norm": 0.01312255859375,
"learning_rate": 4.309601521477134e-06,
"logits/chosen": -2.437730550765991,
"logits/rejected": -2.437730550765991,
"logps/chosen": -318.1125793457031,
"logps/rejected": -318.1125793457031,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.035508617758750916,
"rewards/margins": 0.0,
"rewards/rejected": -0.035508617758750916,
"step": 1090
},
{
"epoch": 0.3210975698752098,
"grad_norm": 0.01373291015625,
"learning_rate": 4.291925108093856e-06,
"logits/chosen": -2.4134514331817627,
"logits/rejected": -2.4134514331817627,
"logps/chosen": -306.98712158203125,
"logps/rejected": -306.98712158203125,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.02741456963121891,
"rewards/margins": 0.0,
"rewards/rejected": -0.02741456963121891,
"step": 1100
},
{
"epoch": 0.3210975698752098,
"eval_logits/chosen": -2.3970751762390137,
"eval_logits/rejected": -2.3970751762390137,
"eval_logps/chosen": -309.472412109375,
"eval_logps/rejected": -309.472412109375,
"eval_loss": 0.6931472420692444,
"eval_rewards/accuracies": 0.0,
"eval_rewards/chosen": -0.029938040301203728,
"eval_rewards/margins": 0.0,
"eval_rewards/rejected": -0.029938040301203728,
"eval_runtime": 2667.8688,
"eval_samples_per_second": 2.283,
"eval_steps_per_second": 0.286,
"step": 1100
},
{
"epoch": 0.32401663869225716,
"grad_norm": 0.0120849609375,
"learning_rate": 4.274062507188978e-06,
"logits/chosen": -2.413846492767334,
"logits/rejected": -2.413846492767334,
"logps/chosen": -319.53887939453125,
"logps/rejected": -319.53887939453125,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.03637847676873207,
"rewards/margins": 0.0,
"rewards/rejected": -0.03637847676873207,
"step": 1110
},
{
"epoch": 0.3269357075093045,
"grad_norm": 0.0130615234375,
"learning_rate": 4.256015574751555e-06,
"logits/chosen": -2.443239212036133,
"logits/rejected": -2.443239212036133,
"logps/chosen": -302.9671630859375,
"logps/rejected": -302.9671630859375,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.032804206013679504,
"rewards/margins": 0.0,
"rewards/rejected": -0.032804206013679504,
"step": 1120
},
{
"epoch": 0.3298547763263519,
"grad_norm": 0.0159912109375,
"learning_rate": 4.2377861859233604e-06,
"logits/chosen": -2.4368813037872314,
"logits/rejected": -2.4368813037872314,
"logps/chosen": -277.4005126953125,
"logps/rejected": -277.4005126953125,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.030909573659300804,
"rewards/margins": 0.0,
"rewards/rejected": -0.030909573659300804,
"step": 1130
},
{
"epoch": 0.33277384514339925,
"grad_norm": 0.01263427734375,
"learning_rate": 4.219376234804047e-06,
"logits/chosen": -2.4358789920806885,
"logits/rejected": -2.4358789920806885,
"logps/chosen": -294.87567138671875,
"logps/rejected": -294.87567138671875,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.033531349152326584,
"rewards/margins": 0.0,
"rewards/rejected": -0.033531349152326584,
"step": 1140
},
{
"epoch": 0.3356929139604466,
"grad_norm": 0.01519775390625,
"learning_rate": 4.200787634254345e-06,
"logits/chosen": -2.458458662033081,
"logits/rejected": -2.458458662033081,
"logps/chosen": -284.5567321777344,
"logps/rejected": -284.5567321777344,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.029438916593790054,
"rewards/margins": 0.0,
"rewards/rejected": -0.029438916593790054,
"step": 1150
},
{
"epoch": 0.338611982777494,
"grad_norm": 0.0157470703125,
"learning_rate": 4.18202231569731e-06,
"logits/chosen": -2.465770721435547,
"logits/rejected": -2.465770721435547,
"logps/chosen": -325.60443115234375,
"logps/rejected": -325.60443115234375,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.03488076478242874,
"rewards/margins": 0.0,
"rewards/rejected": -0.03488076478242874,
"step": 1160
},
{
"epoch": 0.34153105159454133,
"grad_norm": 0.0194091796875,
"learning_rate": 4.163082228917639e-06,
"logits/chosen": -2.42230224609375,
"logits/rejected": -2.42230224609375,
"logps/chosen": -332.96807861328125,
"logps/rejected": -332.96807861328125,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.03761008754372597,
"rewards/margins": 0.0,
"rewards/rejected": -0.03761008754372597,
"step": 1170
},
{
"epoch": 0.3444501204115887,
"grad_norm": 0.01519775390625,
"learning_rate": 4.143969341859083e-06,
"logits/chosen": -2.4006218910217285,
"logits/rejected": -2.4006218910217285,
"logps/chosen": -298.38372802734375,
"logps/rejected": -298.38372802734375,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.027944693341851234,
"rewards/margins": 0.0,
"rewards/rejected": -0.027944693341851234,
"step": 1180
},
{
"epoch": 0.3473691892286361,
"grad_norm": 0.0167236328125,
"learning_rate": 4.124685640419967e-06,
"logits/chosen": -2.4376044273376465,
"logits/rejected": -2.4376044273376465,
"logps/chosen": -339.3370666503906,
"logps/rejected": -339.3370666503906,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.04471305012702942,
"rewards/margins": 0.0,
"rewards/rejected": -0.04471305012702942,
"step": 1190
},
{
"epoch": 0.3502882580456834,
"grad_norm": 0.015625,
"learning_rate": 4.105233128246849e-06,
"logits/chosen": -2.4307379722595215,
"logits/rejected": -2.4307379722595215,
"logps/chosen": -314.7157287597656,
"logps/rejected": -314.7157287597656,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.04377968981862068,
"rewards/margins": 0.0,
"rewards/rejected": -0.04377968981862068,
"step": 1200
},
{
"epoch": 0.3502882580456834,
"eval_logits/chosen": -2.3975985050201416,
"eval_logits/rejected": -2.3975985050201416,
"eval_logps/chosen": -310.0194091796875,
"eval_logps/rejected": -310.0194091796875,
"eval_loss": 0.6931472420692444,
"eval_rewards/accuracies": 0.0,
"eval_rewards/chosen": -0.035407647490501404,
"eval_rewards/margins": 0.0,
"eval_rewards/rejected": -0.035407647490501404,
"eval_runtime": 2667.8638,
"eval_samples_per_second": 2.283,
"eval_steps_per_second": 0.286,
"step": 1200
},
{
"epoch": 0.35320732686273076,
"grad_norm": 0.01373291015625,
"learning_rate": 4.085613826526338e-06,
"logits/chosen": -2.4104952812194824,
"logits/rejected": -2.4104952812194824,
"logps/chosen": -307.89056396484375,
"logps/rejected": -307.89056396484375,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.034878071397542953,
"rewards/margins": 0.0,
"rewards/rejected": -0.034878071397542953,
"step": 1210
},
{
"epoch": 0.35612639567977816,
"grad_norm": 0.0130615234375,
"learning_rate": 4.065829773775082e-06,
"logits/chosen": -2.454697847366333,
"logits/rejected": -2.454697847366333,
"logps/chosen": -331.95556640625,
"logps/rejected": -331.95556640625,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.035688284784555435,
"rewards/margins": 0.0,
"rewards/rejected": -0.035688284784555435,
"step": 1220
},
{
"epoch": 0.3590454644968255,
"grad_norm": 0.01318359375,
"learning_rate": 4.045883025627957e-06,
"logits/chosen": -2.416503429412842,
"logits/rejected": -2.416503429412842,
"logps/chosen": -317.5516662597656,
"logps/rejected": -317.5516662597656,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.036794569343328476,
"rewards/margins": 0.0,
"rewards/rejected": -0.036794569343328476,
"step": 1230
},
{
"epoch": 0.36196453331387285,
"grad_norm": 0.0159912109375,
"learning_rate": 4.025775654624481e-06,
"logits/chosen": -2.431762218475342,
"logits/rejected": -2.431762218475342,
"logps/chosen": -286.4144592285156,
"logps/rejected": -286.4144592285156,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.0327475443482399,
"rewards/margins": 0.0,
"rewards/rejected": -0.0327475443482399,
"step": 1240
},
{
"epoch": 0.36488360213092025,
"grad_norm": 0.01373291015625,
"learning_rate": 4.005509749993471e-06,
"logits/chosen": -2.4348835945129395,
"logits/rejected": -2.4348835945129395,
"logps/chosen": -264.43670654296875,
"logps/rejected": -264.43670654296875,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.03447514772415161,
"rewards/margins": 0.0,
"rewards/rejected": -0.03447514772415161,
"step": 1250
},
{
"epoch": 0.3678026709479676,
"grad_norm": 0.01544189453125,
"learning_rate": 3.985087417435964e-06,
"logits/chosen": -2.4379494190216064,
"logits/rejected": -2.4379494190216064,
"logps/chosen": -306.0783386230469,
"logps/rejected": -306.0783386230469,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.03243451565504074,
"rewards/margins": 0.0,
"rewards/rejected": -0.03243451565504074,
"step": 1260
},
{
"epoch": 0.37072173976501493,
"grad_norm": 0.01318359375,
"learning_rate": 3.964510778906425e-06,
"logits/chosen": -2.434380292892456,
"logits/rejected": -2.434380292892456,
"logps/chosen": -316.9388427734375,
"logps/rejected": -316.9388427734375,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.038867734372615814,
"rewards/margins": 0.0,
"rewards/rejected": -0.038867734372615814,
"step": 1270
},
{
"epoch": 0.37364080858206233,
"grad_norm": 0.0142822265625,
"learning_rate": 3.943781972392269e-06,
"logits/chosen": -2.4212710857391357,
"logits/rejected": -2.4212710857391357,
"logps/chosen": -326.74237060546875,
"logps/rejected": -326.74237060546875,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.038525618612766266,
"rewards/margins": 0.0,
"rewards/rejected": -0.038525618612766266,
"step": 1280
},
{
"epoch": 0.3765598773991097,
"grad_norm": 0.016357421875,
"learning_rate": 3.922903151691716e-06,
"logits/chosen": -2.450032949447632,
"logits/rejected": -2.450032949447632,
"logps/chosen": -329.82073974609375,
"logps/rejected": -329.82073974609375,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.037473224103450775,
"rewards/margins": 0.0,
"rewards/rejected": -0.037473224103450775,
"step": 1290
},
{
"epoch": 0.379478946216157,
"grad_norm": 0.018310546875,
"learning_rate": 3.901876486190008e-06,
"logits/chosen": -2.4351401329040527,
"logits/rejected": -2.4351401329040527,
"logps/chosen": -315.5516662597656,
"logps/rejected": -315.5516662597656,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.03452508896589279,
"rewards/margins": 0.0,
"rewards/rejected": -0.03452508896589279,
"step": 1300
},
{
"epoch": 0.379478946216157,
"eval_logits/chosen": -2.3963370323181152,
"eval_logits/rejected": -2.3963370323181152,
"eval_logps/chosen": -309.5113525390625,
"eval_logps/rejected": -309.5113525390625,
"eval_loss": 0.6931472420692444,
"eval_rewards/accuracies": 0.0,
"eval_rewards/chosen": -0.030327608808875084,
"eval_rewards/margins": 0.0,
"eval_rewards/rejected": -0.030327608808875084,
"eval_runtime": 2666.8225,
"eval_samples_per_second": 2.284,
"eval_steps_per_second": 0.286,
"step": 1300
},
{
"epoch": 0.3823980150332044,
"grad_norm": 0.018798828125,
"learning_rate": 3.880704160633995e-06,
"logits/chosen": -2.4444994926452637,
"logits/rejected": -2.4444994926452637,
"logps/chosen": -295.88348388671875,
"logps/rejected": -295.88348388671875,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.036420173943042755,
"rewards/margins": 0.0,
"rewards/rejected": -0.036420173943042755,
"step": 1310
},
{
"epoch": 0.38531708385025176,
"grad_norm": 0.013671875,
"learning_rate": 3.859388374905136e-06,
"logits/chosen": -2.41549015045166,
"logits/rejected": -2.41549015045166,
"logps/chosen": -291.2346496582031,
"logps/rejected": -291.2346496582031,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.03046022728085518,
"rewards/margins": 0.0,
"rewards/rejected": -0.03046022728085518,
"step": 1320
},
{
"epoch": 0.3882361526672991,
"grad_norm": 0.0152587890625,
"learning_rate": 3.837931343790924e-06,
"logits/chosen": -2.4401891231536865,
"logits/rejected": -2.4401891231536865,
"logps/chosen": -297.060791015625,
"logps/rejected": -297.060791015625,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.02374189719557762,
"rewards/margins": 0.0,
"rewards/rejected": -0.02374189719557762,
"step": 1330
},
{
"epoch": 0.3911552214843465,
"grad_norm": 0.0152587890625,
"learning_rate": 3.8163352967547575e-06,
"logits/chosen": -2.4282491207122803,
"logits/rejected": -2.4282491207122803,
"logps/chosen": -350.7884216308594,
"logps/rejected": -350.7884216308594,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.031809043139219284,
"rewards/margins": 0.0,
"rewards/rejected": -0.031809043139219284,
"step": 1340
},
{
"epoch": 0.39407429030139385,
"grad_norm": 0.01190185546875,
"learning_rate": 3.7946024777042974e-06,
"logits/chosen": -2.423346996307373,
"logits/rejected": -2.423346996307373,
"logps/chosen": -300.26800537109375,
"logps/rejected": -300.26800537109375,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.029370862990617752,
"rewards/margins": 0.0,
"rewards/rejected": -0.029370862990617752,
"step": 1350
},
{
"epoch": 0.3969933591184412,
"grad_norm": 0.01953125,
"learning_rate": 3.7727351447583095e-06,
"logits/chosen": -2.397026538848877,
"logits/rejected": -2.397026538848877,
"logps/chosen": -318.9501647949219,
"logps/rejected": -318.9501647949219,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.030634000897407532,
"rewards/margins": 0.0,
"rewards/rejected": -0.030634000897407532,
"step": 1360
},
{
"epoch": 0.3999124279354886,
"grad_norm": 0.01385498046875,
"learning_rate": 3.750735570012043e-06,
"logits/chosen": -2.438441276550293,
"logits/rejected": -2.438441276550293,
"logps/chosen": -330.5710754394531,
"logps/rejected": -330.5710754394531,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.03722671791911125,
"rewards/margins": 0.0,
"rewards/rejected": -0.03722671791911125,
"step": 1370
},
{
"epoch": 0.40283149675253593,
"grad_norm": 0.01806640625,
"learning_rate": 3.7286060393011513e-06,
"logits/chosen": -2.419067144393921,
"logits/rejected": -2.419067144393921,
"logps/chosen": -314.528564453125,
"logps/rejected": -314.528564453125,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.032639987766742706,
"rewards/margins": 0.0,
"rewards/rejected": -0.032639987766742706,
"step": 1380
},
{
"epoch": 0.4057505655695833,
"grad_norm": 0.01904296875,
"learning_rate": 3.7063488519641825e-06,
"logits/chosen": -2.4223015308380127,
"logits/rejected": -2.4223015308380127,
"logps/chosen": -329.4114685058594,
"logps/rejected": -329.4114685058594,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.03504698723554611,
"rewards/margins": 0.0,
"rewards/rejected": -0.03504698723554611,
"step": 1390
},
{
"epoch": 0.4086696343866307,
"grad_norm": 0.0162353515625,
"learning_rate": 3.6839663206036715e-06,
"logits/chosen": -2.4432168006896973,
"logits/rejected": -2.4432168006896973,
"logps/chosen": -293.8369445800781,
"logps/rejected": -293.8369445800781,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.031177738681435585,
"rewards/margins": 0.0,
"rewards/rejected": -0.031177738681435585,
"step": 1400
},
{
"epoch": 0.4086696343866307,
"eval_logits/chosen": -2.395508050918579,
"eval_logits/rejected": -2.395508050918579,
"eval_logps/chosen": -309.2061462402344,
"eval_logps/rejected": -309.2061462402344,
"eval_loss": 0.6931472420692444,
"eval_rewards/accuracies": 0.0,
"eval_rewards/chosen": -0.02727527543902397,
"eval_rewards/margins": 0.0,
"eval_rewards/rejected": -0.02727527543902397,
"eval_runtime": 2667.2175,
"eval_samples_per_second": 2.283,
"eval_steps_per_second": 0.286,
"step": 1400
},
{
"epoch": 0.411588703203678,
"grad_norm": 0.01239013671875,
"learning_rate": 3.6614607708458532e-06,
"logits/chosen": -2.418804883956909,
"logits/rejected": -2.418804883956909,
"logps/chosen": -295.696533203125,
"logps/rejected": -295.696533203125,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.024944758042693138,
"rewards/margins": 0.0,
"rewards/rejected": -0.024944758042693138,
"step": 1410
},
{
"epoch": 0.41450777202072536,
"grad_norm": 0.0146484375,
"learning_rate": 3.6388345410990195e-06,
"logits/chosen": -2.4199652671813965,
"logits/rejected": -2.4199652671813965,
"logps/chosen": -341.0202331542969,
"logps/rejected": -341.0202331542969,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.030235985293984413,
"rewards/margins": 0.0,
"rewards/rejected": -0.030235985293984413,
"step": 1420
},
{
"epoch": 0.41742684083777276,
"grad_norm": 0.01141357421875,
"learning_rate": 3.6160899823105518e-06,
"logits/chosen": -2.4291069507598877,
"logits/rejected": -2.4291069507598877,
"logps/chosen": -287.2336730957031,
"logps/rejected": -287.2336730957031,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.0277925543487072,
"rewards/margins": 0.0,
"rewards/rejected": -0.0277925543487072,
"step": 1430
},
{
"epoch": 0.4203459096548201,
"grad_norm": 0.0140380859375,
"learning_rate": 3.5932294577226468e-06,
"logits/chosen": -2.440561532974243,
"logits/rejected": -2.440561532974243,
"logps/chosen": -276.7684020996094,
"logps/rejected": -276.7684020996094,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.017870336771011353,
"rewards/margins": 0.0,
"rewards/rejected": -0.017870336771011353,
"step": 1440
},
{
"epoch": 0.42326497847186745,
"grad_norm": 0.0118408203125,
"learning_rate": 3.5702553426267704e-06,
"logits/chosen": -2.449218988418579,
"logits/rejected": -2.449218988418579,
"logps/chosen": -305.78814697265625,
"logps/rejected": -305.78814697265625,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.024231892079114914,
"rewards/margins": 0.0,
"rewards/rejected": -0.024231892079114914,
"step": 1450
},
{
"epoch": 0.42618404728891485,
"grad_norm": 0.015625,
"learning_rate": 3.547170024116854e-06,
"logits/chosen": -2.4015636444091797,
"logits/rejected": -2.4015636444091797,
"logps/chosen": -281.1402893066406,
"logps/rejected": -281.1402893066406,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.027333328500390053,
"rewards/margins": 0.0,
"rewards/rejected": -0.027333328500390053,
"step": 1460
},
{
"epoch": 0.4291031161059622,
"grad_norm": 0.0164794921875,
"learning_rate": 3.5239759008412666e-06,
"logits/chosen": -2.461341381072998,
"logits/rejected": -2.461341381072998,
"logps/chosen": -315.0804443359375,
"logps/rejected": -315.0804443359375,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.026240995153784752,
"rewards/margins": 0.0,
"rewards/rejected": -0.026240995153784752,
"step": 1470
},
{
"epoch": 0.43202218492300953,
"grad_norm": 0.0164794921875,
"learning_rate": 3.500675382753588e-06,
"logits/chosen": -2.420381784439087,
"logits/rejected": -2.420381784439087,
"logps/chosen": -310.7515563964844,
"logps/rejected": -310.7515563964844,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.023152858018875122,
"rewards/margins": 0.0,
"rewards/rejected": -0.023152858018875122,
"step": 1480
},
{
"epoch": 0.43494125374005693,
"grad_norm": 0.01336669921875,
"learning_rate": 3.477270890862204e-06,
"logits/chosen": -2.3881866931915283,
"logits/rejected": -2.3881866931915283,
"logps/chosen": -318.3128356933594,
"logps/rejected": -318.3128356933594,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.030725980177521706,
"rewards/margins": 0.0,
"rewards/rejected": -0.030725980177521706,
"step": 1490
},
{
"epoch": 0.4378603225571043,
"grad_norm": 0.0140380859375,
"learning_rate": 3.453764856978758e-06,
"logits/chosen": -2.409209728240967,
"logits/rejected": -2.409209728240967,
"logps/chosen": -331.4593200683594,
"logps/rejected": -331.4593200683594,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.022285277023911476,
"rewards/margins": 0.0,
"rewards/rejected": -0.022285277023911476,
"step": 1500
},
{
"epoch": 0.4378603225571043,
"eval_logits/chosen": -2.394321918487549,
"eval_logits/rejected": -2.394321918487549,
"eval_logps/chosen": -308.9651794433594,
"eval_logps/rejected": -308.9651794433594,
"eval_loss": 0.6931472420692444,
"eval_rewards/accuracies": 0.0,
"eval_rewards/chosen": -0.024866018444299698,
"eval_rewards/margins": 0.0,
"eval_rewards/rejected": -0.024866018444299698,
"eval_runtime": 2666.7789,
"eval_samples_per_second": 2.284,
"eval_steps_per_second": 0.286,
"step": 1500
},
{
"epoch": 0.4407793913741516,
"grad_norm": 0.01312255859375,
"learning_rate": 3.4301597234654733e-06,
"logits/chosen": -2.4193215370178223,
"logits/rejected": -2.4193215370178223,
"logps/chosen": -304.951171875,
"logps/rejected": -304.951171875,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.02852988801896572,
"rewards/margins": 0.0,
"rewards/rejected": -0.02852988801896572,
"step": 1510
},
{
"epoch": 0.443698460191199,
"grad_norm": 0.0177001953125,
"learning_rate": 3.406457942981384e-06,
"logits/chosen": -2.430614948272705,
"logits/rejected": -2.430614948272705,
"logps/chosen": -333.06988525390625,
"logps/rejected": -333.06988525390625,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.024759288877248764,
"rewards/margins": 0.0,
"rewards/rejected": -0.024759288877248764,
"step": 1520
},
{
"epoch": 0.44661752900824636,
"grad_norm": 0.0133056640625,
"learning_rate": 3.3826619782274954e-06,
"logits/chosen": -2.43021559715271,
"logits/rejected": -2.43021559715271,
"logps/chosen": -284.0345153808594,
"logps/rejected": -284.0345153808594,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.025433775037527084,
"rewards/margins": 0.0,
"rewards/rejected": -0.025433775037527084,
"step": 1530
},
{
"epoch": 0.4495365978252937,
"grad_norm": 0.0142822265625,
"learning_rate": 3.3587743016909013e-06,
"logits/chosen": -2.439312219619751,
"logits/rejected": -2.439312219619751,
"logps/chosen": -320.015380859375,
"logps/rejected": -320.015380859375,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.02944205328822136,
"rewards/margins": 0.0,
"rewards/rejected": -0.02944205328822136,
"step": 1540
},
{
"epoch": 0.4524556666423411,
"grad_norm": 0.044677734375,
"learning_rate": 3.334797395387882e-06,
"logits/chosen": -2.4262938499450684,
"logits/rejected": -2.4262938499450684,
"logps/chosen": -329.60504150390625,
"logps/rejected": -329.60504150390625,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.027106398716568947,
"rewards/margins": 0.0,
"rewards/rejected": -0.027106398716568947,
"step": 1550
},
{
"epoch": 0.45537473545938845,
"grad_norm": 0.01226806640625,
"learning_rate": 3.3107337506060145e-06,
"logits/chosen": -2.4414420127868652,
"logits/rejected": -2.4414420127868652,
"logps/chosen": -289.9877014160156,
"logps/rejected": -289.9877014160156,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.028158003464341164,
"rewards/margins": 0.0,
"rewards/rejected": -0.028158003464341164,
"step": 1560
},
{
"epoch": 0.4582938042764358,
"grad_norm": 0.0301513671875,
"learning_rate": 3.2865858676453172e-06,
"logits/chosen": -2.434182643890381,
"logits/rejected": -2.434182643890381,
"logps/chosen": -306.0428466796875,
"logps/rejected": -306.0428466796875,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.02475564181804657,
"rewards/margins": 0.0,
"rewards/rejected": -0.02475564181804657,
"step": 1570
},
{
"epoch": 0.4612128730934832,
"grad_norm": 0.0098876953125,
"learning_rate": 3.2623562555584633e-06,
"logits/chosen": -2.430816411972046,
"logits/rejected": -2.430816411972046,
"logps/chosen": -281.2196960449219,
"logps/rejected": -281.2196960449219,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.02929893136024475,
"rewards/margins": 0.0,
"rewards/rejected": -0.02929893136024475,
"step": 1580
},
{
"epoch": 0.46413194191053053,
"grad_norm": 0.024658203125,
"learning_rate": 3.2380474318900766e-06,
"logits/chosen": -2.4165406227111816,
"logits/rejected": -2.4165406227111816,
"logps/chosen": -310.68511962890625,
"logps/rejected": -310.68511962890625,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.03376628831028938,
"rewards/margins": 0.0,
"rewards/rejected": -0.03376628831028938,
"step": 1590
},
{
"epoch": 0.4670510107275779,
"grad_norm": 0.016845703125,
"learning_rate": 3.2136619224151533e-06,
"logits/chosen": -2.4508678913116455,
"logits/rejected": -2.4508678913116455,
"logps/chosen": -327.84619140625,
"logps/rejected": -327.84619140625,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.03426826745271683,
"rewards/margins": 0.0,
"rewards/rejected": -0.03426826745271683,
"step": 1600
},
{
"epoch": 0.4670510107275779,
"eval_logits/chosen": -2.3953943252563477,
"eval_logits/rejected": -2.3953943252563477,
"eval_logps/chosen": -309.15863037109375,
"eval_logps/rejected": -309.15863037109375,
"eval_loss": 0.6931472420692444,
"eval_rewards/accuracies": 0.0,
"eval_rewards/chosen": -0.026800233870744705,
"eval_rewards/margins": 0.0,
"eval_rewards/rejected": -0.026800233870744705,
"eval_runtime": 2666.9806,
"eval_samples_per_second": 2.283,
"eval_steps_per_second": 0.286,
"step": 1600
},
{
"epoch": 0.4699700795446253,
"grad_norm": 0.014892578125,
"learning_rate": 3.1892022608766215e-06,
"logits/chosen": -2.361971378326416,
"logits/rejected": -2.361971378326416,
"logps/chosen": -299.3944396972656,
"logps/rejected": -299.3944396972656,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.0262086633592844,
"rewards/margins": 0.0,
"rewards/rejected": -0.0262086633592844,
"step": 1610
},
{
"epoch": 0.4728891483616726,
"grad_norm": 0.01422119140625,
"learning_rate": 3.16467098872208e-06,
"logits/chosen": -2.4706971645355225,
"logits/rejected": -2.4706971645355225,
"logps/chosen": -332.5861511230469,
"logps/rejected": -332.5861511230469,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.034811943769454956,
"rewards/margins": 0.0,
"rewards/rejected": -0.034811943769454956,
"step": 1620
},
{
"epoch": 0.47580821717871996,
"grad_norm": 0.032470703125,
"learning_rate": 3.140070654839728e-06,
"logits/chosen": -2.4026148319244385,
"logits/rejected": -2.4026148319244385,
"logps/chosen": -296.76605224609375,
"logps/rejected": -296.76605224609375,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.02368254028260708,
"rewards/margins": 0.0,
"rewards/rejected": -0.02368254028260708,
"step": 1630
},
{
"epoch": 0.47872728599576736,
"grad_norm": 0.0242919921875,
"learning_rate": 3.115403815293532e-06,
"logits/chosen": -2.43617582321167,
"logits/rejected": -2.43617582321167,
"logps/chosen": -342.2427062988281,
"logps/rejected": -342.2427062988281,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.036105576902627945,
"rewards/margins": 0.0,
"rewards/rejected": -0.036105576902627945,
"step": 1640
},
{
"epoch": 0.4816463548128147,
"grad_norm": 0.0113525390625,
"learning_rate": 3.0906730330576345e-06,
"logits/chosen": -2.4739155769348145,
"logits/rejected": -2.4739155769348145,
"logps/chosen": -332.26678466796875,
"logps/rejected": -332.26678466796875,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.028261488303542137,
"rewards/margins": 0.0,
"rewards/rejected": -0.028261488303542137,
"step": 1650
},
{
"epoch": 0.48456542362986205,
"grad_norm": 0.017333984375,
"learning_rate": 3.065880877750059e-06,
"logits/chosen": -2.427436351776123,
"logits/rejected": -2.427436351776123,
"logps/chosen": -304.4495544433594,
"logps/rejected": -304.4495544433594,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.03417867794632912,
"rewards/margins": 0.0,
"rewards/rejected": -0.03417867794632912,
"step": 1660
},
{
"epoch": 0.48748449244690945,
"grad_norm": 0.01226806640625,
"learning_rate": 3.041029925365711e-06,
"logits/chosen": -2.4058425426483154,
"logits/rejected": -2.4058425426483154,
"logps/chosen": -308.30072021484375,
"logps/rejected": -308.30072021484375,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.035630594938993454,
"rewards/margins": 0.0,
"rewards/rejected": -0.035630594938993454,
"step": 1670
},
{
"epoch": 0.4904035612639568,
"grad_norm": 0.0126953125,
"learning_rate": 3.0161227580087282e-06,
"logits/chosen": -2.433281421661377,
"logits/rejected": -2.433281421661377,
"logps/chosen": -342.0614013671875,
"logps/rejected": -342.0614013671875,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.03289630264043808,
"rewards/margins": 0.0,
"rewards/rejected": -0.03289630264043808,
"step": 1680
},
{
"epoch": 0.49332263008100413,
"grad_norm": 0.0123291015625,
"learning_rate": 2.9911619636241862e-06,
"logits/chosen": -2.4333884716033936,
"logits/rejected": -2.4333884716033936,
"logps/chosen": -322.1616516113281,
"logps/rejected": -322.1616516113281,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.034327663481235504,
"rewards/margins": 0.0,
"rewards/rejected": -0.034327663481235504,
"step": 1690
},
{
"epoch": 0.49624169889805153,
"grad_norm": 0.01275634765625,
"learning_rate": 2.966150135729203e-06,
"logits/chosen": -2.38623046875,
"logits/rejected": -2.38623046875,
"logps/chosen": -335.8984680175781,
"logps/rejected": -335.8984680175781,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.03050742670893669,
"rewards/margins": 0.0,
"rewards/rejected": -0.03050742670893669,
"step": 1700
},
{
"epoch": 0.49624169889805153,
"eval_logits/chosen": -2.3913044929504395,
"eval_logits/rejected": -2.3913044929504395,
"eval_logps/chosen": -309.405517578125,
"eval_logps/rejected": -309.405517578125,
"eval_loss": 0.6931472420692444,
"eval_rewards/accuracies": 0.0,
"eval_rewards/chosen": -0.029269486665725708,
"eval_rewards/margins": 0.0,
"eval_rewards/rejected": -0.029269486665725708,
"eval_runtime": 2669.612,
"eval_samples_per_second": 2.281,
"eval_steps_per_second": 0.285,
"step": 1700
},
{
"epoch": 0.4991607677150989,
"grad_norm": 0.01324462890625,
"learning_rate": 2.9410898731434667e-06,
"logits/chosen": -2.41214919090271,
"logits/rejected": -2.41214919090271,
"logps/chosen": -302.40887451171875,
"logps/rejected": -302.40887451171875,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.028926188126206398,
"rewards/margins": 0.0,
"rewards/rejected": -0.028926188126206398,
"step": 1710
},
{
"epoch": 0.5020798365321463,
"grad_norm": 0.0152587890625,
"learning_rate": 2.9159837797192003e-06,
"logits/chosen": -2.415527820587158,
"logits/rejected": -2.415527820587158,
"logps/chosen": -329.7999267578125,
"logps/rejected": -329.7999267578125,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.03768650442361832,
"rewards/margins": 0.0,
"rewards/rejected": -0.03768650442361832,
"step": 1720
},
{
"epoch": 0.5049989053491936,
"grad_norm": 0.014404296875,
"learning_rate": 2.890834464070623e-06,
"logits/chosen": -2.4205574989318848,
"logits/rejected": -2.4205574989318848,
"logps/chosen": -309.94329833984375,
"logps/rejected": -309.94329833984375,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.03702525794506073,
"rewards/margins": 0.0,
"rewards/rejected": -0.03702525794506073,
"step": 1730
},
{
"epoch": 0.507917974166241,
"grad_norm": 0.013671875,
"learning_rate": 2.865644539302896e-06,
"logits/chosen": -2.389092206954956,
"logits/rejected": -2.389092206954956,
"logps/chosen": -339.6660461425781,
"logps/rejected": -339.6660461425781,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.029835382476449013,
"rewards/margins": 0.0,
"rewards/rejected": -0.029835382476449013,
"step": 1740
},
{
"epoch": 0.5108370429832884,
"grad_norm": 0.01300048828125,
"learning_rate": 2.840416622740617e-06,
"logits/chosen": -2.444392681121826,
"logits/rejected": -2.444392681121826,
"logps/chosen": -318.47296142578125,
"logps/rejected": -318.47296142578125,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.03087993524968624,
"rewards/margins": 0.0,
"rewards/rejected": -0.03087993524968624,
"step": 1750
},
{
"epoch": 0.5137561118003356,
"grad_norm": 0.01263427734375,
"learning_rate": 2.8151533356558673e-06,
"logits/chosen": -2.4179341793060303,
"logits/rejected": -2.4179341793060303,
"logps/chosen": -295.8548889160156,
"logps/rejected": -295.8548889160156,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.032246123999357224,
"rewards/margins": 0.0,
"rewards/rejected": -0.032246123999357224,
"step": 1760
},
{
"epoch": 0.516675180617383,
"grad_norm": 0.014892578125,
"learning_rate": 2.7898573029958563e-06,
"logits/chosen": -2.377382516860962,
"logits/rejected": -2.377382516860962,
"logps/chosen": -305.41656494140625,
"logps/rejected": -305.41656494140625,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.03165289759635925,
"rewards/margins": 0.0,
"rewards/rejected": -0.03165289759635925,
"step": 1770
},
{
"epoch": 0.5195942494344304,
"grad_norm": 0.0103759765625,
"learning_rate": 2.7645311531101763e-06,
"logits/chosen": -2.412802219390869,
"logits/rejected": -2.412802219390869,
"logps/chosen": -312.50067138671875,
"logps/rejected": -312.50067138671875,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.034763775765895844,
"rewards/margins": 0.0,
"rewards/rejected": -0.034763775765895844,
"step": 1780
},
{
"epoch": 0.5225133182514777,
"grad_norm": 0.0135498046875,
"learning_rate": 2.7391775174777084e-06,
"logits/chosen": -2.419868230819702,
"logits/rejected": -2.419868230819702,
"logps/chosen": -310.26922607421875,
"logps/rejected": -310.26922607421875,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.035530101507902145,
"rewards/margins": 0.0,
"rewards/rejected": -0.035530101507902145,
"step": 1790
},
{
"epoch": 0.5254323870685251,
"grad_norm": 0.0167236328125,
"learning_rate": 2.713799030433203e-06,
"logits/chosen": -2.423767566680908,
"logits/rejected": -2.423767566680908,
"logps/chosen": -308.0718688964844,
"logps/rejected": -308.0718688964844,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.03834807127714157,
"rewards/margins": 0.0,
"rewards/rejected": -0.03834807127714157,
"step": 1800
},
{
"epoch": 0.5254323870685251,
"eval_logits/chosen": -2.392709732055664,
"eval_logits/rejected": -2.392709732055664,
"eval_logps/chosen": -310.26434326171875,
"eval_logps/rejected": -310.26434326171875,
"eval_loss": 0.6931472420692444,
"eval_rewards/accuracies": 0.0,
"eval_rewards/chosen": -0.037857454270124435,
"eval_rewards/margins": 0.0,
"eval_rewards/rejected": -0.037857454270124435,
"eval_runtime": 2669.2359,
"eval_samples_per_second": 2.282,
"eval_steps_per_second": 0.285,
"step": 1800
},
{
"epoch": 0.5283514558855725,
"grad_norm": 0.01373291015625,
"learning_rate": 2.688398328893561e-06,
"logits/chosen": -2.4216887950897217,
"logits/rejected": -2.4216887950897217,
"logps/chosen": -307.491455078125,
"logps/rejected": -307.491455078125,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.03987189009785652,
"rewards/margins": 0.0,
"rewards/rejected": -0.03987189009785652,
"step": 1810
},
{
"epoch": 0.5312705247026198,
"grad_norm": 0.013916015625,
"learning_rate": 2.6629780520838526e-06,
"logits/chosen": -2.389004945755005,
"logits/rejected": -2.389004945755005,
"logps/chosen": -314.912353515625,
"logps/rejected": -314.912353515625,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.03697946295142174,
"rewards/margins": 0.0,
"rewards/rejected": -0.03697946295142174,
"step": 1820
},
{
"epoch": 0.5341895935196672,
"grad_norm": 0.016845703125,
"learning_rate": 2.637540841263088e-06,
"logits/chosen": -2.4251251220703125,
"logits/rejected": -2.4251251220703125,
"logps/chosen": -309.82611083984375,
"logps/rejected": -309.82611083984375,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.041506171226501465,
"rewards/margins": 0.0,
"rewards/rejected": -0.041506171226501465,
"step": 1830
},
{
"epoch": 0.5371086623367146,
"grad_norm": 0.0130615234375,
"learning_rate": 2.6120893394497825e-06,
"logits/chosen": -2.4095826148986816,
"logits/rejected": -2.4095826148986816,
"logps/chosen": -290.29876708984375,
"logps/rejected": -290.29876708984375,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.03885159641504288,
"rewards/margins": 0.0,
"rewards/rejected": -0.03885159641504288,
"step": 1840
},
{
"epoch": 0.5400277311537619,
"grad_norm": 0.0203857421875,
"learning_rate": 2.586626191147337e-06,
"logits/chosen": -2.414461612701416,
"logits/rejected": -2.414461612701416,
"logps/chosen": -298.74444580078125,
"logps/rejected": -298.74444580078125,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.035465486347675323,
"rewards/margins": 0.0,
"rewards/rejected": -0.035465486347675323,
"step": 1850
},
{
"epoch": 0.5429467999708093,
"grad_norm": 0.0142822265625,
"learning_rate": 2.5611540420692666e-06,
"logits/chosen": -2.4189705848693848,
"logits/rejected": -2.4189705848693848,
"logps/chosen": -361.6686706542969,
"logps/rejected": -361.6686706542969,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.04054202139377594,
"rewards/margins": 0.0,
"rewards/rejected": -0.04054202139377594,
"step": 1860
},
{
"epoch": 0.5458658687878567,
"grad_norm": 0.01446533203125,
"learning_rate": 2.5356755388642973e-06,
"logits/chosen": -2.4053876399993896,
"logits/rejected": -2.4053876399993896,
"logps/chosen": -290.9534606933594,
"logps/rejected": -290.9534606933594,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.037081241607666016,
"rewards/margins": 0.0,
"rewards/rejected": -0.037081241607666016,
"step": 1870
},
{
"epoch": 0.548784937604904,
"grad_norm": 0.01611328125,
"learning_rate": 2.510193328841375e-06,
"logits/chosen": -2.4209909439086914,
"logits/rejected": -2.4209909439086914,
"logps/chosen": -304.0765075683594,
"logps/rejected": -304.0765075683594,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.03415703400969505,
"rewards/margins": 0.0,
"rewards/rejected": -0.03415703400969505,
"step": 1880
},
{
"epoch": 0.5517040064219514,
"grad_norm": 0.0164794921875,
"learning_rate": 2.484710059694594e-06,
"logits/chosen": -2.4459662437438965,
"logits/rejected": -2.4459662437438965,
"logps/chosen": -274.7349548339844,
"logps/rejected": -274.7349548339844,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.03464942425489426,
"rewards/margins": 0.0,
"rewards/rejected": -0.03464942425489426,
"step": 1890
},
{
"epoch": 0.5546230752389988,
"grad_norm": 0.01348876953125,
"learning_rate": 2.4592283792280977e-06,
"logits/chosen": -2.384141206741333,
"logits/rejected": -2.384141206741333,
"logps/chosen": -293.96533203125,
"logps/rejected": -293.96533203125,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.04034542292356491,
"rewards/margins": 0.0,
"rewards/rejected": -0.04034542292356491,
"step": 1900
},
{
"epoch": 0.5546230752389988,
"eval_logits/chosen": -2.3927481174468994,
"eval_logits/rejected": -2.3927481174468994,
"eval_logps/chosen": -310.4163818359375,
"eval_logps/rejected": -310.4163818359375,
"eval_loss": 0.6931472420692444,
"eval_rewards/accuracies": 0.0,
"eval_rewards/chosen": -0.03937768191099167,
"eval_rewards/margins": 0.0,
"eval_rewards/rejected": -0.03937768191099167,
"eval_runtime": 2717.9911,
"eval_samples_per_second": 2.241,
"eval_steps_per_second": 0.28,
"step": 1900
},
{
"epoch": 0.5575421440560461,
"grad_norm": 0.01123046875,
"learning_rate": 2.433750935080959e-06,
"logits/chosen": -2.438390016555786,
"logits/rejected": -2.438390016555786,
"logps/chosen": -282.78106689453125,
"logps/rejected": -282.78106689453125,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.05149908736348152,
"rewards/margins": 0.0,
"rewards/rejected": -0.05149908736348152,
"step": 1910
},
{
"epoch": 0.5604612128730935,
"grad_norm": 0.011962890625,
"learning_rate": 2.408280374452083e-06,
"logits/chosen": -2.4534342288970947,
"logits/rejected": -2.4534342288970947,
"logps/chosen": -306.63946533203125,
"logps/rejected": -306.63946533203125,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.04204695671796799,
"rewards/margins": 0.0,
"rewards/rejected": -0.04204695671796799,
"step": 1920
},
{
"epoch": 0.5633802816901409,
"grad_norm": 0.01385498046875,
"learning_rate": 2.3828193438251497e-06,
"logits/chosen": -2.4302496910095215,
"logits/rejected": -2.4302496910095215,
"logps/chosen": -328.1105651855469,
"logps/rejected": -328.1105651855469,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.03839876502752304,
"rewards/margins": 0.0,
"rewards/rejected": -0.03839876502752304,
"step": 1930
},
{
"epoch": 0.5662993505071882,
"grad_norm": 0.01513671875,
"learning_rate": 2.3573704886936414e-06,
"logits/chosen": -2.4566609859466553,
"logits/rejected": -2.4566609859466553,
"logps/chosen": -314.76910400390625,
"logps/rejected": -314.76910400390625,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.04071163386106491,
"rewards/margins": 0.0,
"rewards/rejected": -0.04071163386106491,
"step": 1940
},
{
"epoch": 0.5692184193242356,
"grad_norm": 0.01397705078125,
"learning_rate": 2.331936453285957e-06,
"logits/chosen": -2.414055109024048,
"logits/rejected": -2.414055109024048,
"logps/chosen": -346.7576904296875,
"logps/rejected": -346.7576904296875,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.036326270550489426,
"rewards/margins": 0.0,
"rewards/rejected": -0.036326270550489426,
"step": 1950
},
{
"epoch": 0.572137488141283,
"grad_norm": 0.0157470703125,
"learning_rate": 2.3065198802906767e-06,
"logits/chosen": -2.4286112785339355,
"logits/rejected": -2.4286112785339355,
"logps/chosen": -339.60064697265625,
"logps/rejected": -339.60064697265625,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.04548191279172897,
"rewards/margins": 0.0,
"rewards/rejected": -0.04548191279172897,
"step": 1960
},
{
"epoch": 0.5750565569583302,
"grad_norm": 0.01141357421875,
"learning_rate": 2.2811234105819714e-06,
"logits/chosen": -2.4342637062072754,
"logits/rejected": -2.4342637062072754,
"logps/chosen": -314.4915771484375,
"logps/rejected": -314.4915771484375,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.03697306662797928,
"rewards/margins": 0.0,
"rewards/rejected": -0.03697306662797928,
"step": 1970
},
{
"epoch": 0.5779756257753776,
"grad_norm": 0.01495361328125,
"learning_rate": 2.2557496829452056e-06,
"logits/chosen": -2.387324810028076,
"logits/rejected": -2.387324810028076,
"logps/chosen": -349.37835693359375,
"logps/rejected": -349.37835693359375,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.04250973090529442,
"rewards/margins": 0.0,
"rewards/rejected": -0.04250973090529442,
"step": 1980
},
{
"epoch": 0.580894694592425,
"grad_norm": 0.0152587890625,
"learning_rate": 2.230401333802763e-06,
"logits/chosen": -2.412137985229492,
"logits/rejected": -2.412137985229492,
"logps/chosen": -310.9895324707031,
"logps/rejected": -310.9895324707031,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.040226660668849945,
"rewards/margins": 0.0,
"rewards/rejected": -0.040226660668849945,
"step": 1990
},
{
"epoch": 0.5838137634094723,
"grad_norm": 0.01483154296875,
"learning_rate": 2.205080996940108e-06,
"logits/chosen": -2.4124810695648193,
"logits/rejected": -2.4124810695648193,
"logps/chosen": -273.5890197753906,
"logps/rejected": -273.5890197753906,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.04197770729660988,
"rewards/margins": 0.0,
"rewards/rejected": -0.04197770729660988,
"step": 2000
},
{
"epoch": 0.5838137634094723,
"eval_logits/chosen": -2.392037868499756,
"eval_logits/rejected": -2.392037868499756,
"eval_logps/chosen": -310.4427185058594,
"eval_logps/rejected": -310.4427185058594,
"eval_loss": 0.6931472420692444,
"eval_rewards/accuracies": 0.0,
"eval_rewards/chosen": -0.039641354233026505,
"eval_rewards/margins": 0.0,
"eval_rewards/rejected": -0.039641354233026505,
"eval_runtime": 2711.551,
"eval_samples_per_second": 2.246,
"eval_steps_per_second": 0.281,
"step": 2000
},
{
"epoch": 0.5867328322265197,
"grad_norm": 0.01214599609375,
"learning_rate": 2.1797913032321283e-06,
"logits/chosen": -2.420572519302368,
"logits/rejected": -2.420572519302368,
"logps/chosen": -277.4279479980469,
"logps/rejected": -277.4279479980469,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.03539072722196579,
"rewards/margins": 0.0,
"rewards/rejected": -0.03539072722196579,
"step": 2010
},
{
"epoch": 0.5896519010435671,
"grad_norm": 0.0157470703125,
"learning_rate": 2.1545348803697745e-06,
"logits/chosen": -2.4433321952819824,
"logits/rejected": -2.4433321952819824,
"logps/chosen": -281.5128479003906,
"logps/rejected": -281.5128479003906,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": 0.016543438658118248,
"rewards/margins": 0.0,
"rewards/rejected": 0.016543438658118248,
"step": 2020
},
{
"epoch": 0.5925709698606144,
"grad_norm": 0.015869140625,
"learning_rate": 2.1293143525870396e-06,
"logits/chosen": -2.435228109359741,
"logits/rejected": -2.435228109359741,
"logps/chosen": -315.1198425292969,
"logps/rejected": -315.1198425292969,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.04324204847216606,
"rewards/margins": 0.0,
"rewards/rejected": -0.04324204847216606,
"step": 2030
},
{
"epoch": 0.5954900386776618,
"grad_norm": 0.0133056640625,
"learning_rate": 2.1041323403882836e-06,
"logits/chosen": -2.458317995071411,
"logits/rejected": -2.458317995071411,
"logps/chosen": -314.63482666015625,
"logps/rejected": -314.63482666015625,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.039002105593681335,
"rewards/margins": 0.0,
"rewards/rejected": -0.039002105593681335,
"step": 2040
},
{
"epoch": 0.5984091074947092,
"grad_norm": 0.0164794921875,
"learning_rate": 2.078991460275958e-06,
"logits/chosen": -2.4496326446533203,
"logits/rejected": -2.4496326446533203,
"logps/chosen": -295.86199951171875,
"logps/rejected": -295.86199951171875,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.03856682404875755,
"rewards/margins": 0.0,
"rewards/rejected": -0.03856682404875755,
"step": 2050
},
{
"epoch": 0.6013281763117565,
"grad_norm": 0.01409912109375,
"learning_rate": 2.0538943244787452e-06,
"logits/chosen": -2.440256118774414,
"logits/rejected": -2.440256118774414,
"logps/chosen": -302.68463134765625,
"logps/rejected": -302.68463134765625,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.040991030633449554,
"rewards/margins": 0.0,
"rewards/rejected": -0.040991030633449554,
"step": 2060
},
{
"epoch": 0.6042472451288039,
"grad_norm": 0.01226806640625,
"learning_rate": 2.0288435406801293e-06,
"logits/chosen": -2.4207422733306885,
"logits/rejected": -2.4207422733306885,
"logps/chosen": -347.23297119140625,
"logps/rejected": -347.23297119140625,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.03826383873820305,
"rewards/margins": 0.0,
"rewards/rejected": -0.03826383873820305,
"step": 2070
},
{
"epoch": 0.6071663139458513,
"grad_norm": 0.01275634765625,
"learning_rate": 2.0038417117474574e-06,
"logits/chosen": -2.4277267456054688,
"logits/rejected": -2.4277267456054688,
"logps/chosen": -314.09674072265625,
"logps/rejected": -314.09674072265625,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.05563684552907944,
"rewards/margins": 0.0,
"rewards/rejected": -0.05563684552907944,
"step": 2080
},
{
"epoch": 0.6100853827628986,
"grad_norm": 0.01251220703125,
"learning_rate": 1.9788914354614853e-06,
"logits/chosen": -2.4430274963378906,
"logits/rejected": -2.4430274963378906,
"logps/chosen": -280.791015625,
"logps/rejected": -280.791015625,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.039162371307611465,
"rewards/margins": 0.0,
"rewards/rejected": -0.039162371307611465,
"step": 2090
},
{
"epoch": 0.613004451579946,
"grad_norm": 0.0159912109375,
"learning_rate": 1.9539953042464656e-06,
"logits/chosen": -2.4126973152160645,
"logits/rejected": -2.4126973152160645,
"logps/chosen": -341.8514709472656,
"logps/rejected": -341.8514709472656,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.04444648697972298,
"rewards/margins": 0.0,
"rewards/rejected": -0.04444648697972298,
"step": 2100
},
{
"epoch": 0.613004451579946,
"eval_logits/chosen": -2.390094041824341,
"eval_logits/rejected": -2.390094041824341,
"eval_logps/chosen": -310.71502685546875,
"eval_logps/rejected": -310.71502685546875,
"eval_loss": 0.6931472420692444,
"eval_rewards/accuracies": 0.0,
"eval_rewards/chosen": -0.042364299297332764,
"eval_rewards/margins": 0.0,
"eval_rewards/rejected": -0.042364299297332764,
"eval_runtime": 2698.9127,
"eval_samples_per_second": 2.256,
"eval_steps_per_second": 0.282,
"step": 2100
},
{
"epoch": 0.6159235203969934,
"grad_norm": 0.0126953125,
"learning_rate": 1.929155904900778e-06,
"logits/chosen": -2.442920207977295,
"logits/rejected": -2.442920207977295,
"logps/chosen": -336.13153076171875,
"logps/rejected": -336.13153076171875,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.04605261981487274,
"rewards/margins": 0.0,
"rewards/rejected": -0.04605261981487274,
"step": 2110
},
{
"epoch": 0.6188425892140407,
"grad_norm": 0.0128173828125,
"learning_rate": 1.9043758183281548e-06,
"logits/chosen": -2.398139476776123,
"logits/rejected": -2.398139476776123,
"logps/chosen": -297.93353271484375,
"logps/rejected": -297.93353271484375,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.03661734238266945,
"rewards/margins": 0.0,
"rewards/rejected": -0.03661734238266945,
"step": 2120
},
{
"epoch": 0.6217616580310881,
"grad_norm": 0.0162353515625,
"learning_rate": 1.8796576192695198e-06,
"logits/chosen": -2.4115586280822754,
"logits/rejected": -2.4115586280822754,
"logps/chosen": -283.5032653808594,
"logps/rejected": -283.5032653808594,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.05024952441453934,
"rewards/margins": 0.0,
"rewards/rejected": -0.05024952441453934,
"step": 2130
},
{
"epoch": 0.6246807268481355,
"grad_norm": 0.01611328125,
"learning_rate": 1.8550038760354559e-06,
"logits/chosen": -2.4140570163726807,
"logits/rejected": -2.4140570163726807,
"logps/chosen": -328.29241943359375,
"logps/rejected": -328.29241943359375,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.03783569857478142,
"rewards/margins": 0.0,
"rewards/rejected": -0.03783569857478142,
"step": 2140
},
{
"epoch": 0.6275997956651828,
"grad_norm": 0.01470947265625,
"learning_rate": 1.8304171502393542e-06,
"logits/chosen": -2.4498252868652344,
"logits/rejected": -2.4498252868652344,
"logps/chosen": -333.46807861328125,
"logps/rejected": -333.46807861328125,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.05009561777114868,
"rewards/margins": 0.0,
"rewards/rejected": -0.05009561777114868,
"step": 2150
},
{
"epoch": 0.6305188644822302,
"grad_norm": 0.0198974609375,
"learning_rate": 1.8058999965312484e-06,
"logits/chosen": -2.3965957164764404,
"logits/rejected": -2.3965957164764404,
"logps/chosen": -306.3211669921875,
"logps/rejected": -306.3211669921875,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.04475449398159981,
"rewards/margins": 0.0,
"rewards/rejected": -0.04475449398159981,
"step": 2160
},
{
"epoch": 0.6334379332992776,
"grad_norm": 0.016357421875,
"learning_rate": 1.7814549623323828e-06,
"logits/chosen": -2.400684356689453,
"logits/rejected": -2.400684356689453,
"logps/chosen": -286.625,
"logps/rejected": -286.625,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.043368883430957794,
"rewards/margins": 0.0,
"rewards/rejected": -0.043368883430957794,
"step": 2170
},
{
"epoch": 0.6363570021163248,
"grad_norm": 0.01531982421875,
"learning_rate": 1.7570845875705205e-06,
"logits/chosen": -2.4366753101348877,
"logits/rejected": -2.4366753101348877,
"logps/chosen": -338.27679443359375,
"logps/rejected": -338.27679443359375,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.05562018230557442,
"rewards/margins": 0.0,
"rewards/rejected": -0.05562018230557442,
"step": 2180
},
{
"epoch": 0.6392760709333722,
"grad_norm": 0.0162353515625,
"learning_rate": 1.7327914044160388e-06,
"logits/chosen": -2.449612617492676,
"logits/rejected": -2.449612617492676,
"logps/chosen": -316.91766357421875,
"logps/rejected": -316.91766357421875,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.04444233328104019,
"rewards/margins": 0.0,
"rewards/rejected": -0.04444233328104019,
"step": 2190
},
{
"epoch": 0.6421951397504196,
"grad_norm": 0.0145263671875,
"learning_rate": 1.7085779370188276e-06,
"logits/chosen": -2.3980746269226074,
"logits/rejected": -2.3980746269226074,
"logps/chosen": -308.85906982421875,
"logps/rejected": -308.85906982421875,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.0481327660381794,
"rewards/margins": 0.0,
"rewards/rejected": -0.0481327660381794,
"step": 2200
},
{
"epoch": 0.6421951397504196,
"eval_logits/chosen": -2.3910679817199707,
"eval_logits/rejected": -2.3910679817199707,
"eval_logps/chosen": -311.0310363769531,
"eval_logps/rejected": -311.0310363769531,
"eval_loss": 0.6931472420692444,
"eval_rewards/accuracies": 0.0,
"eval_rewards/chosen": -0.04552413523197174,
"eval_rewards/margins": 0.0,
"eval_rewards/rejected": -0.04552413523197174,
"eval_runtime": 2707.3295,
"eval_samples_per_second": 2.249,
"eval_steps_per_second": 0.281,
"step": 2200
},
{
"epoch": 0.6451142085674669,
"grad_norm": 0.016845703125,
"learning_rate": 1.6844467012460193e-06,
"logits/chosen": -2.429086446762085,
"logits/rejected": -2.429086446762085,
"logps/chosen": -306.8155822753906,
"logps/rejected": -306.8155822753906,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.04282836988568306,
"rewards/margins": 0.0,
"rewards/rejected": -0.04282836988568306,
"step": 2210
},
{
"epoch": 0.6480332773845143,
"grad_norm": 0.014404296875,
"learning_rate": 1.6604002044205825e-06,
"logits/chosen": -2.4325811862945557,
"logits/rejected": -2.4325811862945557,
"logps/chosen": -337.0578308105469,
"logps/rejected": -337.0578308105469,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.04457175359129906,
"rewards/margins": 0.0,
"rewards/rejected": -0.04457175359129906,
"step": 2220
},
{
"epoch": 0.6509523462015617,
"grad_norm": 0.01397705078125,
"learning_rate": 1.6364409450608018e-06,
"logits/chosen": -2.4428985118865967,
"logits/rejected": -2.4428985118865967,
"logps/chosen": -308.55657958984375,
"logps/rejected": -308.55657958984375,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.04731472209095955,
"rewards/margins": 0.0,
"rewards/rejected": -0.04731472209095955,
"step": 2230
},
{
"epoch": 0.653871415018609,
"grad_norm": 0.013427734375,
"learning_rate": 1.6125714126206736e-06,
"logits/chosen": -2.4196009635925293,
"logits/rejected": -2.4196009635925293,
"logps/chosen": -348.8056335449219,
"logps/rejected": -348.8056335449219,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.049455009400844574,
"rewards/margins": 0.0,
"rewards/rejected": -0.049455009400844574,
"step": 2240
},
{
"epoch": 0.6567904838356564,
"grad_norm": 0.01556396484375,
"learning_rate": 1.5887940872312391e-06,
"logits/chosen": -2.4100897312164307,
"logits/rejected": -2.4100897312164307,
"logps/chosen": -320.3233642578125,
"logps/rejected": -320.3233642578125,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.04676957428455353,
"rewards/margins": 0.0,
"rewards/rejected": -0.04676957428455353,
"step": 2250
},
{
"epoch": 0.6597095526527038,
"grad_norm": 0.0147705078125,
"learning_rate": 1.5651114394428955e-06,
"logits/chosen": -2.4624266624450684,
"logits/rejected": -2.4624266624450684,
"logps/chosen": -344.6718444824219,
"logps/rejected": -344.6718444824219,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.0538489893078804,
"rewards/margins": 0.0,
"rewards/rejected": -0.0538489893078804,
"step": 2260
},
{
"epoch": 0.6626286214697511,
"grad_norm": 0.01251220703125,
"learning_rate": 1.5415259299686903e-06,
"logits/chosen": -2.4147191047668457,
"logits/rejected": -2.4147191047668457,
"logps/chosen": -316.6529235839844,
"logps/rejected": -316.6529235839844,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.041484713554382324,
"rewards/margins": 0.0,
"rewards/rejected": -0.041484713554382324,
"step": 2270
},
{
"epoch": 0.6655476902867985,
"grad_norm": 0.01348876953125,
"learning_rate": 1.5180400094286496e-06,
"logits/chosen": -2.440053939819336,
"logits/rejected": -2.440053939819336,
"logps/chosen": -309.5370178222656,
"logps/rejected": -309.5370178222656,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.04570756107568741,
"rewards/margins": 0.0,
"rewards/rejected": -0.04570756107568741,
"step": 2280
},
{
"epoch": 0.6684667591038459,
"grad_norm": 0.017822265625,
"learning_rate": 1.494656118095149e-06,
"logits/chosen": -2.407764434814453,
"logits/rejected": -2.407764434814453,
"logps/chosen": -320.51263427734375,
"logps/rejected": -320.51263427734375,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.04706931859254837,
"rewards/margins": 0.0,
"rewards/rejected": -0.04706931859254837,
"step": 2290
},
{
"epoch": 0.6713858279208932,
"grad_norm": 0.0120849609375,
"learning_rate": 1.4713766856393557e-06,
"logits/chosen": -2.420919895172119,
"logits/rejected": -2.420919895172119,
"logps/chosen": -295.04547119140625,
"logps/rejected": -295.04547119140625,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.05071335285902023,
"rewards/margins": 0.0,
"rewards/rejected": -0.05071335285902023,
"step": 2300
},
{
"epoch": 0.6713858279208932,
"eval_logits/chosen": -2.391244411468506,
"eval_logits/rejected": -2.391244411468506,
"eval_logps/chosen": -310.7880554199219,
"eval_logps/rejected": -310.7880554199219,
"eval_loss": 0.6931472420692444,
"eval_rewards/accuracies": 0.0,
"eval_rewards/chosen": -0.04309455305337906,
"eval_rewards/margins": 0.0,
"eval_rewards/rejected": -0.04309455305337906,
"eval_runtime": 2670.058,
"eval_samples_per_second": 2.281,
"eval_steps_per_second": 0.285,
"step": 2300
},
{
"epoch": 0.6743048967379406,
"grad_norm": 0.0198974609375,
"learning_rate": 1.448204130878785e-06,
"logits/chosen": -2.3968968391418457,
"logits/rejected": -2.3968968391418457,
"logps/chosen": -287.2406005859375,
"logps/rejected": -287.2406005859375,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.04974811524152756,
"rewards/margins": 0.0,
"rewards/rejected": -0.04974811524152756,
"step": 2310
},
{
"epoch": 0.677223965554988,
"grad_norm": 0.013916015625,
"learning_rate": 1.425140861525967e-06,
"logits/chosen": -2.407982587814331,
"logits/rejected": -2.407982587814331,
"logps/chosen": -346.8302307128906,
"logps/rejected": -346.8302307128906,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.045040082186460495,
"rewards/margins": 0.0,
"rewards/rejected": -0.045040082186460495,
"step": 2320
},
{
"epoch": 0.6801430343720353,
"grad_norm": 0.01531982421875,
"learning_rate": 1.4021892739382853e-06,
"logits/chosen": -2.4366557598114014,
"logits/rejected": -2.4366557598114014,
"logps/chosen": -315.5507507324219,
"logps/rejected": -315.5507507324219,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.053034014999866486,
"rewards/margins": 0.0,
"rewards/rejected": -0.053034014999866486,
"step": 2330
},
{
"epoch": 0.6830621031890827,
"grad_norm": 0.013916015625,
"learning_rate": 1.3793517528689804e-06,
"logits/chosen": -2.40993070602417,
"logits/rejected": -2.40993070602417,
"logps/chosen": -322.5754699707031,
"logps/rejected": -322.5754699707031,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.04859765246510506,
"rewards/margins": 0.0,
"rewards/rejected": -0.04859765246510506,
"step": 2340
},
{
"epoch": 0.6859811720061301,
"grad_norm": 0.0167236328125,
"learning_rate": 1.3566306712193704e-06,
"logits/chosen": -2.4204134941101074,
"logits/rejected": -2.4204134941101074,
"logps/chosen": -349.4993896484375,
"logps/rejected": -349.4993896484375,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.05103806406259537,
"rewards/margins": 0.0,
"rewards/rejected": -0.05103806406259537,
"step": 2350
},
{
"epoch": 0.6889002408231774,
"grad_norm": 0.01531982421875,
"learning_rate": 1.3340283897922911e-06,
"logits/chosen": -2.4295237064361572,
"logits/rejected": -2.4295237064361572,
"logps/chosen": -330.99005126953125,
"logps/rejected": -330.99005126953125,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.04879484325647354,
"rewards/margins": 0.0,
"rewards/rejected": -0.04879484325647354,
"step": 2360
},
{
"epoch": 0.6918193096402248,
"grad_norm": 0.0146484375,
"learning_rate": 1.3115472570468058e-06,
"logits/chosen": -2.4285712242126465,
"logits/rejected": -2.4285712242126465,
"logps/chosen": -336.67364501953125,
"logps/rejected": -336.67364501953125,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.04440216347575188,
"rewards/margins": 0.0,
"rewards/rejected": -0.04440216347575188,
"step": 2370
},
{
"epoch": 0.6947383784572722,
"grad_norm": 0.0162353515625,
"learning_rate": 1.2891896088541928e-06,
"logits/chosen": -2.405956745147705,
"logits/rejected": -2.405956745147705,
"logps/chosen": -338.88739013671875,
"logps/rejected": -338.88739013671875,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.047105275094509125,
"rewards/margins": 0.0,
"rewards/rejected": -0.047105275094509125,
"step": 2380
},
{
"epoch": 0.6976574472743194,
"grad_norm": 0.0169677734375,
"learning_rate": 1.266957768255232e-06,
"logits/chosen": -2.422194719314575,
"logits/rejected": -2.422194719314575,
"logps/chosen": -318.286865234375,
"logps/rejected": -318.286865234375,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.04666005074977875,
"rewards/margins": 0.0,
"rewards/rejected": -0.04666005074977875,
"step": 2390
},
{
"epoch": 0.7005765160913668,
"grad_norm": 0.0142822265625,
"learning_rate": 1.2448540452188432e-06,
"logits/chosen": -2.3955206871032715,
"logits/rejected": -2.3955206871032715,
"logps/chosen": -314.3586120605469,
"logps/rejected": -314.3586120605469,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.04044215753674507,
"rewards/margins": 0.0,
"rewards/rejected": -0.04044215753674507,
"step": 2400
},
{
"epoch": 0.7005765160913668,
"eval_logits/chosen": -2.3899266719818115,
"eval_logits/rejected": -2.3899266719818115,
"eval_logps/chosen": -310.6455078125,
"eval_logps/rejected": -310.6455078125,
"eval_loss": 0.6931472420692444,
"eval_rewards/accuracies": 0.0,
"eval_rewards/chosen": -0.04166920483112335,
"eval_rewards/margins": 0.0,
"eval_rewards/rejected": -0.04166920483112335,
"eval_runtime": 2668.7744,
"eval_samples_per_second": 2.282,
"eval_steps_per_second": 0.286,
"step": 2400
},
{
"epoch": 0.7034955849084142,
"grad_norm": 0.01446533203125,
"learning_rate": 1.2228807364020617e-06,
"logits/chosen": -2.4090027809143066,
"logits/rejected": -2.4090027809143066,
"logps/chosen": -268.48944091796875,
"logps/rejected": -268.48944091796875,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.037643421441316605,
"rewards/margins": 0.0,
"rewards/rejected": -0.037643421441316605,
"step": 2410
},
{
"epoch": 0.7064146537254615,
"grad_norm": 0.012451171875,
"learning_rate": 1.2010401249114166e-06,
"logits/chosen": -2.4060184955596924,
"logits/rejected": -2.4060184955596924,
"logps/chosen": -338.2677001953125,
"logps/rejected": -338.2677001953125,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.035085879266262054,
"rewards/margins": 0.0,
"rewards/rejected": -0.035085879266262054,
"step": 2420
},
{
"epoch": 0.7093337225425089,
"grad_norm": 0.0206298828125,
"learning_rate": 1.1793344800656995e-06,
"logits/chosen": -2.3857572078704834,
"logits/rejected": -2.3857572078704834,
"logps/chosen": -325.4837646484375,
"logps/rejected": -325.4837646484375,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.03704181686043739,
"rewards/margins": 0.0,
"rewards/rejected": -0.03704181686043739,
"step": 2430
},
{
"epoch": 0.7122527913595563,
"grad_norm": 0.01544189453125,
"learning_rate": 1.1577660571601796e-06,
"logits/chosen": -2.396127223968506,
"logits/rejected": -2.396127223968506,
"logps/chosen": -321.38897705078125,
"logps/rejected": -321.38897705078125,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.0452260822057724,
"rewards/margins": 0.0,
"rewards/rejected": -0.0452260822057724,
"step": 2440
},
{
"epoch": 0.7151718601766036,
"grad_norm": 0.0137939453125,
"learning_rate": 1.1363370972322694e-06,
"logits/chosen": -2.4177489280700684,
"logits/rejected": -2.4177489280700684,
"logps/chosen": -296.6512756347656,
"logps/rejected": -296.6512756347656,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.04763947054743767,
"rewards/margins": 0.0,
"rewards/rejected": -0.04763947054743767,
"step": 2450
},
{
"epoch": 0.718090928993651,
"grad_norm": 0.0142822265625,
"learning_rate": 1.115049826828669e-06,
"logits/chosen": -2.4321625232696533,
"logits/rejected": -2.4321625232696533,
"logps/chosen": -306.14141845703125,
"logps/rejected": -306.14141845703125,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.04333222657442093,
"rewards/margins": 0.0,
"rewards/rejected": -0.04333222657442093,
"step": 2460
},
{
"epoch": 0.7210099978106984,
"grad_norm": 0.01483154296875,
"learning_rate": 1.0939064577740266e-06,
"logits/chosen": -2.4054694175720215,
"logits/rejected": -2.4054694175720215,
"logps/chosen": -301.36334228515625,
"logps/rejected": -301.36334228515625,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.041240572929382324,
"rewards/margins": 0.0,
"rewards/rejected": -0.041240572929382324,
"step": 2470
},
{
"epoch": 0.7239290666277457,
"grad_norm": 0.0159912109375,
"learning_rate": 1.0729091869411137e-06,
"logits/chosen": -2.4020252227783203,
"logits/rejected": -2.4020252227783203,
"logps/chosen": -332.1387023925781,
"logps/rejected": -332.1387023925781,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.043921731412410736,
"rewards/margins": 0.0,
"rewards/rejected": -0.043921731412410736,
"step": 2480
},
{
"epoch": 0.7268481354447931,
"grad_norm": 0.013427734375,
"learning_rate": 1.0520601960225708e-06,
"logits/chosen": -2.421534299850464,
"logits/rejected": -2.421534299850464,
"logps/chosen": -314.00311279296875,
"logps/rejected": -314.00311279296875,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.01183334831148386,
"rewards/margins": 0.0,
"rewards/rejected": -0.01183334831148386,
"step": 2490
},
{
"epoch": 0.7297672042618405,
"grad_norm": 0.020751953125,
"learning_rate": 1.0313616513042133e-06,
"logits/chosen": -2.4747350215911865,
"logits/rejected": -2.4747350215911865,
"logps/chosen": -319.47918701171875,
"logps/rejected": -319.47918701171875,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.055976878851652145,
"rewards/margins": 0.0,
"rewards/rejected": -0.055976878851652145,
"step": 2500
},
{
"epoch": 0.7297672042618405,
"eval_logits/chosen": -2.3914709091186523,
"eval_logits/rejected": -2.3914709091186523,
"eval_logps/chosen": -310.819580078125,
"eval_logps/rejected": -310.819580078125,
"eval_loss": 0.6931472420692444,
"eval_rewards/accuracies": 0.0,
"eval_rewards/chosen": -0.04341000318527222,
"eval_rewards/margins": 0.0,
"eval_rewards/rejected": -0.04341000318527222,
"eval_runtime": 2669.3967,
"eval_samples_per_second": 2.281,
"eval_steps_per_second": 0.285,
"step": 2500
},
{
"epoch": 0.7326862730788878,
"grad_norm": 0.0145263671875,
"learning_rate": 1.0108157034399532e-06,
"logits/chosen": -2.4052977561950684,
"logits/rejected": -2.4052977561950684,
"logps/chosen": -298.67474365234375,
"logps/rejected": -298.67474365234375,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.04686864838004112,
"rewards/margins": 0.0,
"rewards/rejected": -0.04686864838004112,
"step": 2510
},
{
"epoch": 0.7356053418959352,
"grad_norm": 0.0179443359375,
"learning_rate": 9.90424487228334e-07,
"logits/chosen": -2.411712646484375,
"logits/rejected": -2.411712646484375,
"logps/chosen": -322.70428466796875,
"logps/rejected": -322.70428466796875,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.043312918394804,
"rewards/margins": 0.0,
"rewards/rejected": -0.043312918394804,
"step": 2520
},
{
"epoch": 0.7385244107129826,
"grad_norm": 0.01611328125,
"learning_rate": 9.701901213907192e-07,
"logits/chosen": -2.4330382347106934,
"logits/rejected": -2.4330382347106934,
"logps/chosen": -324.5224609375,
"logps/rejected": -324.5224609375,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.05447854846715927,
"rewards/margins": 0.0,
"rewards/rejected": -0.05447854846715927,
"step": 2530
},
{
"epoch": 0.7414434795300299,
"grad_norm": 0.01416015625,
"learning_rate": 9.501147083511511e-07,
"logits/chosen": -2.45332407951355,
"logits/rejected": -2.45332407951355,
"logps/chosen": -321.7140808105469,
"logps/rejected": -321.7140808105469,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.0516216978430748,
"rewards/margins": 0.0,
"rewards/rejected": -0.0516216978430748,
"step": 2540
},
{
"epoch": 0.7443625483470773,
"grad_norm": 0.015625,
"learning_rate": 9.302003340178962e-07,
"logits/chosen": -2.417236804962158,
"logits/rejected": -2.417236804962158,
"logps/chosen": -333.95574951171875,
"logps/rejected": -333.95574951171875,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.0465201810002327,
"rewards/margins": 0.0,
"rewards/rejected": -0.0465201810002327,
"step": 2550
},
{
"epoch": 0.7472816171641247,
"grad_norm": 0.01422119140625,
"learning_rate": 9.10449067566718e-07,
"logits/chosen": -2.459394931793213,
"logits/rejected": -2.459394931793213,
"logps/chosen": -303.9725646972656,
"logps/rejected": -303.9725646972656,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.04736841470003128,
"rewards/margins": 0.0,
"rewards/rejected": -0.04736841470003128,
"step": 2560
},
{
"epoch": 0.750200685981172,
"grad_norm": 0.01513671875,
"learning_rate": 8.908629612258765e-07,
"logits/chosen": -2.435121774673462,
"logits/rejected": -2.435121774673462,
"logps/chosen": -300.51055908203125,
"logps/rejected": -300.51055908203125,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.04969844967126846,
"rewards/margins": 0.0,
"rewards/rejected": -0.04969844967126846,
"step": 2570
},
{
"epoch": 0.7531197547982194,
"grad_norm": 0.0145263671875,
"learning_rate": 8.714440500628999e-07,
"logits/chosen": -2.393557071685791,
"logits/rejected": -2.393557071685791,
"logps/chosen": -305.946044921875,
"logps/rejected": -305.946044921875,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.035433102399110794,
"rewards/margins": 0.0,
"rewards/rejected": -0.035433102399110794,
"step": 2580
},
{
"epoch": 0.7560388236152668,
"grad_norm": 0.01385498046875,
"learning_rate": 8.521943517731276e-07,
"logits/chosen": -2.394944667816162,
"logits/rejected": -2.394944667816162,
"logps/chosen": -329.5417175292969,
"logps/rejected": -329.5417175292969,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.029879886656999588,
"rewards/margins": 0.0,
"rewards/rejected": -0.029879886656999588,
"step": 2590
},
{
"epoch": 0.758957892432314,
"grad_norm": 0.01513671875,
"learning_rate": 8.33115866470069e-07,
"logits/chosen": -2.3986093997955322,
"logits/rejected": -2.3986093997955322,
"logps/chosen": -297.0606994628906,
"logps/rejected": -297.0606994628906,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.04173046723008156,
"rewards/margins": 0.0,
"rewards/rejected": -0.04173046723008156,
"step": 2600
},
{
"epoch": 0.758957892432314,
"eval_logits/chosen": -2.3918919563293457,
"eval_logits/rejected": -2.3918919563293457,
"eval_logps/chosen": -310.8546447753906,
"eval_logps/rejected": -310.8546447753906,
"eval_loss": 0.6931472420692444,
"eval_rewards/accuracies": 0.0,
"eval_rewards/chosen": -0.0437602661550045,
"eval_rewards/margins": 0.0,
"eval_rewards/rejected": -0.0437602661550045,
"eval_runtime": 2682.4018,
"eval_samples_per_second": 2.27,
"eval_steps_per_second": 0.284,
"step": 2600
},
{
"epoch": 0.7618769612493614,
"grad_norm": 0.01544189453125,
"learning_rate": 8.142105764775824e-07,
"logits/chosen": -2.384005546569824,
"logits/rejected": -2.384005546569824,
"logps/chosen": -327.1615295410156,
"logps/rejected": -327.1615295410156,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.051810719072818756,
"rewards/margins": 0.0,
"rewards/rejected": -0.051810719072818756,
"step": 2610
},
{
"epoch": 0.7647960300664088,
"grad_norm": 0.01458740234375,
"learning_rate": 7.954804461239054e-07,
"logits/chosen": -2.444282054901123,
"logits/rejected": -2.444282054901123,
"logps/chosen": -314.5889587402344,
"logps/rejected": -314.5889587402344,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.04727676510810852,
"rewards/margins": 0.0,
"rewards/rejected": -0.04727676510810852,
"step": 2620
},
{
"epoch": 0.7677150988834561,
"grad_norm": 0.016357421875,
"learning_rate": 7.769274215375544e-07,
"logits/chosen": -2.432978391647339,
"logits/rejected": -2.432978391647339,
"logps/chosen": -293.0484924316406,
"logps/rejected": -293.0484924316406,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.0413900688290596,
"rewards/margins": 0.0,
"rewards/rejected": -0.0413900688290596,
"step": 2630
},
{
"epoch": 0.7706341677005035,
"grad_norm": 0.01446533203125,
"learning_rate": 7.585534304451103e-07,
"logits/chosen": -2.444913387298584,
"logits/rejected": -2.444913387298584,
"logps/chosen": -330.8976135253906,
"logps/rejected": -330.8976135253906,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.043237775564193726,
"rewards/margins": 0.0,
"rewards/rejected": -0.043237775564193726,
"step": 2640
},
{
"epoch": 0.7735532365175509,
"grad_norm": 0.01312255859375,
"learning_rate": 7.403603819709288e-07,
"logits/chosen": -2.4194247722625732,
"logits/rejected": -2.4194247722625732,
"logps/chosen": -302.08465576171875,
"logps/rejected": -302.08465576171875,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.04694979637861252,
"rewards/margins": 0.0,
"rewards/rejected": -0.04694979637861252,
"step": 2650
},
{
"epoch": 0.7764723053345982,
"grad_norm": 0.014404296875,
"learning_rate": 7.223501664387664e-07,
"logits/chosen": -2.440764904022217,
"logits/rejected": -2.440764904022217,
"logps/chosen": -280.7825622558594,
"logps/rejected": -280.7825622558594,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.036083877086639404,
"rewards/margins": 0.0,
"rewards/rejected": -0.036083877086639404,
"step": 2660
},
{
"epoch": 0.7793913741516456,
"grad_norm": 0.01458740234375,
"learning_rate": 7.045246551753779e-07,
"logits/chosen": -2.4197888374328613,
"logits/rejected": -2.4197888374328613,
"logps/chosen": -323.67938232421875,
"logps/rejected": -323.67938232421875,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.043979812413454056,
"rewards/margins": 0.0,
"rewards/rejected": -0.043979812413454056,
"step": 2670
},
{
"epoch": 0.782310442968693,
"grad_norm": 0.0142822265625,
"learning_rate": 6.868857003160709e-07,
"logits/chosen": -2.470567226409912,
"logits/rejected": -2.470567226409912,
"logps/chosen": -356.6578369140625,
"logps/rejected": -356.6578369140625,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.05309978872537613,
"rewards/margins": 0.0,
"rewards/rejected": -0.05309978872537613,
"step": 2680
},
{
"epoch": 0.7852295117857403,
"grad_norm": 0.0150146484375,
"learning_rate": 6.69435134612266e-07,
"logits/chosen": -2.4125561714172363,
"logits/rejected": -2.4125561714172363,
"logps/chosen": -302.1919250488281,
"logps/rejected": -302.1919250488281,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.04638701677322388,
"rewards/margins": 0.0,
"rewards/rejected": -0.04638701677322388,
"step": 2690
},
{
"epoch": 0.7881485806027877,
"grad_norm": 0.013427734375,
"learning_rate": 6.521747712410687e-07,
"logits/chosen": -2.431802988052368,
"logits/rejected": -2.431802988052368,
"logps/chosen": -319.6323547363281,
"logps/rejected": -319.6323547363281,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.04552530124783516,
"rewards/margins": 0.0,
"rewards/rejected": -0.04552530124783516,
"step": 2700
},
{
"epoch": 0.7881485806027877,
"eval_logits/chosen": -2.3916165828704834,
"eval_logits/rejected": -2.3916165828704834,
"eval_logps/chosen": -310.8406677246094,
"eval_logps/rejected": -310.8406677246094,
"eval_loss": 0.6931472420692444,
"eval_rewards/accuracies": 0.0,
"eval_rewards/chosen": -0.04362065717577934,
"eval_rewards/margins": 0.0,
"eval_rewards/rejected": -0.04362065717577934,
"eval_runtime": 2682.1268,
"eval_samples_per_second": 2.271,
"eval_steps_per_second": 0.284,
"step": 2700
},
{
"epoch": 0.7910676494198351,
"grad_norm": 0.0250244140625,
"learning_rate": 6.351064036168708e-07,
"logits/chosen": -2.4238877296447754,
"logits/rejected": -2.4238877296447754,
"logps/chosen": -338.21759033203125,
"logps/rejected": -338.21759033203125,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.048554692417383194,
"rewards/margins": 0.0,
"rewards/rejected": -0.048554692417383194,
"step": 2710
},
{
"epoch": 0.7939867182368824,
"grad_norm": 0.01416015625,
"learning_rate": 6.182318052050102e-07,
"logits/chosen": -2.398974895477295,
"logits/rejected": -2.398974895477295,
"logps/chosen": -329.53106689453125,
"logps/rejected": -329.53106689453125,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.05249527841806412,
"rewards/margins": 0.0,
"rewards/rejected": -0.05249527841806412,
"step": 2720
},
{
"epoch": 0.7969057870539298,
"grad_norm": 0.019287109375,
"learning_rate": 6.015527293374979e-07,
"logits/chosen": -2.4338581562042236,
"logits/rejected": -2.4338581562042236,
"logps/chosen": -334.1202087402344,
"logps/rejected": -334.1202087402344,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.04841463267803192,
"rewards/margins": 0.0,
"rewards/rejected": -0.04841463267803192,
"step": 2730
},
{
"epoch": 0.7998248558709772,
"grad_norm": 0.014404296875,
"learning_rate": 5.850709090308459e-07,
"logits/chosen": -2.4255330562591553,
"logits/rejected": -2.4255330562591553,
"logps/chosen": -295.30523681640625,
"logps/rejected": -295.30523681640625,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.0433431938290596,
"rewards/margins": 0.0,
"rewards/rejected": -0.0433431938290596,
"step": 2740
},
{
"epoch": 0.8027439246880245,
"grad_norm": 0.0133056640625,
"learning_rate": 5.687880568059961e-07,
"logits/chosen": -2.3997416496276855,
"logits/rejected": -2.3997416496276855,
"logps/chosen": -314.76361083984375,
"logps/rejected": -314.76361083984375,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.04872073233127594,
"rewards/margins": 0.0,
"rewards/rejected": -0.04872073233127594,
"step": 2750
},
{
"epoch": 0.8056629935050719,
"grad_norm": 0.01422119140625,
"learning_rate": 5.527058645103842e-07,
"logits/chosen": -2.3996376991271973,
"logits/rejected": -2.3996376991271973,
"logps/chosen": -376.6802673339844,
"logps/rejected": -376.6802673339844,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.0522351935505867,
"rewards/margins": 0.0,
"rewards/rejected": -0.0522351935505867,
"step": 2760
},
{
"epoch": 0.8085820623221193,
"grad_norm": 0.0159912109375,
"learning_rate": 5.368260031421526e-07,
"logits/chosen": -2.4533755779266357,
"logits/rejected": -2.4533755779266357,
"logps/chosen": -338.7648010253906,
"logps/rejected": -338.7648010253906,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.04105439782142639,
"rewards/margins": 0.0,
"rewards/rejected": -0.04105439782142639,
"step": 2770
},
{
"epoch": 0.8115011311391666,
"grad_norm": 0.01263427734375,
"learning_rate": 5.211501226765242e-07,
"logits/chosen": -2.43373441696167,
"logits/rejected": -2.43373441696167,
"logps/chosen": -285.7012023925781,
"logps/rejected": -285.7012023925781,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.02375207468867302,
"rewards/margins": 0.0,
"rewards/rejected": -0.02375207468867302,
"step": 2780
},
{
"epoch": 0.814420199956214,
"grad_norm": 0.0184326171875,
"learning_rate": 5.056798518943678e-07,
"logits/chosen": -2.4133718013763428,
"logits/rejected": -2.4133718013763428,
"logps/chosen": -315.09210205078125,
"logps/rejected": -315.09210205078125,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.053018081933259964,
"rewards/margins": 0.0,
"rewards/rejected": -0.053018081933259964,
"step": 2790
},
{
"epoch": 0.8173392687732614,
"grad_norm": 0.01397705078125,
"learning_rate": 4.904167982129591e-07,
"logits/chosen": -2.423839569091797,
"logits/rejected": -2.423839569091797,
"logps/chosen": -294.44683837890625,
"logps/rejected": -294.44683837890625,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.04831403121352196,
"rewards/margins": 0.0,
"rewards/rejected": -0.04831403121352196,
"step": 2800
},
{
"epoch": 0.8173392687732614,
"eval_logits/chosen": -2.3914895057678223,
"eval_logits/rejected": -2.3914895057678223,
"eval_logps/chosen": -310.798095703125,
"eval_logps/rejected": -310.798095703125,
"eval_loss": 0.6931472420692444,
"eval_rewards/accuracies": 0.0,
"eval_rewards/chosen": -0.04319505766034126,
"eval_rewards/margins": 0.0,
"eval_rewards/rejected": -0.04319505766034126,
"eval_runtime": 2682.0962,
"eval_samples_per_second": 2.271,
"eval_steps_per_second": 0.284,
"step": 2800
},
{
"epoch": 0.8202583375903086,
"grad_norm": 0.0126953125,
"learning_rate": 4.7536254751896493e-07,
"logits/chosen": -2.4333229064941406,
"logits/rejected": -2.4333229064941406,
"logps/chosen": -315.96234130859375,
"logps/rejected": -315.96234130859375,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.05122748017311096,
"rewards/margins": 0.0,
"rewards/rejected": -0.05122748017311096,
"step": 2810
},
{
"epoch": 0.823177406407356,
"grad_norm": 0.0167236328125,
"learning_rate": 4.6051866400366354e-07,
"logits/chosen": -2.4289793968200684,
"logits/rejected": -2.4289793968200684,
"logps/chosen": -344.29608154296875,
"logps/rejected": -344.29608154296875,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.04764155298471451,
"rewards/margins": 0.0,
"rewards/rejected": -0.04764155298471451,
"step": 2820
},
{
"epoch": 0.8260964752244034,
"grad_norm": 0.0166015625,
"learning_rate": 4.4588669000042133e-07,
"logits/chosen": -2.4046084880828857,
"logits/rejected": -2.4046084880828857,
"logps/chosen": -325.74957275390625,
"logps/rejected": -325.74957275390625,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.035486068576574326,
"rewards/margins": 0.0,
"rewards/rejected": -0.035486068576574326,
"step": 2830
},
{
"epoch": 0.8290155440414507,
"grad_norm": 0.016845703125,
"learning_rate": 4.3146814582443605e-07,
"logits/chosen": -2.418729066848755,
"logits/rejected": -2.418729066848755,
"logps/chosen": -327.8818359375,
"logps/rejected": -327.8818359375,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.046850480139255524,
"rewards/margins": 0.0,
"rewards/rejected": -0.046850480139255524,
"step": 2840
},
{
"epoch": 0.8319346128584981,
"grad_norm": 0.0135498046875,
"learning_rate": 4.1726452961477147e-07,
"logits/chosen": -2.416329860687256,
"logits/rejected": -2.416329860687256,
"logps/chosen": -319.5370178222656,
"logps/rejected": -319.5370178222656,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.0506584569811821,
"rewards/margins": 0.0,
"rewards/rejected": -0.0506584569811821,
"step": 2850
},
{
"epoch": 0.8348536816755455,
"grad_norm": 0.0146484375,
"learning_rate": 4.0327731717869775e-07,
"logits/chosen": -2.4376559257507324,
"logits/rejected": -2.4376559257507324,
"logps/chosen": -272.7819519042969,
"logps/rejected": -272.7819519042969,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.05310596153140068,
"rewards/margins": 0.0,
"rewards/rejected": -0.05310596153140068,
"step": 2860
},
{
"epoch": 0.8377727504925928,
"grad_norm": 0.0191650390625,
"learning_rate": 3.8950796183834516e-07,
"logits/chosen": -2.4388468265533447,
"logits/rejected": -2.4388468265533447,
"logps/chosen": -345.3861389160156,
"logps/rejected": -345.3861389160156,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.05019260570406914,
"rewards/margins": 0.0,
"rewards/rejected": -0.05019260570406914,
"step": 2870
},
{
"epoch": 0.8406918193096402,
"grad_norm": 0.01495361328125,
"learning_rate": 3.759578942797029e-07,
"logits/chosen": -2.4550201892852783,
"logits/rejected": -2.4550201892852783,
"logps/chosen": -306.2907409667969,
"logps/rejected": -306.2907409667969,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.046652454882860184,
"rewards/margins": 0.0,
"rewards/rejected": -0.046652454882860184,
"step": 2880
},
{
"epoch": 0.8436108881266876,
"grad_norm": 0.0126953125,
"learning_rate": 3.6262852240396356e-07,
"logits/chosen": -2.446690082550049,
"logits/rejected": -2.446690082550049,
"logps/chosen": -310.69708251953125,
"logps/rejected": -310.69708251953125,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.04568660259246826,
"rewards/margins": 0.0,
"rewards/rejected": -0.04568660259246826,
"step": 2890
},
{
"epoch": 0.8465299569437349,
"grad_norm": 0.014404296875,
"learning_rate": 3.4952123118123735e-07,
"logits/chosen": -2.402627468109131,
"logits/rejected": -2.402627468109131,
"logps/chosen": -312.1624755859375,
"logps/rejected": -312.1624755859375,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.045014895498752594,
"rewards/margins": 0.0,
"rewards/rejected": -0.045014895498752594,
"step": 2900
},
{
"epoch": 0.8465299569437349,
"eval_logits/chosen": -2.391954183578491,
"eval_logits/rejected": -2.391954183578491,
"eval_logps/chosen": -310.79425048828125,
"eval_logps/rejected": -310.79425048828125,
"eval_loss": 0.6931472420692444,
"eval_rewards/accuracies": 0.0,
"eval_rewards/chosen": -0.043156567960977554,
"eval_rewards/margins": 0.0,
"eval_rewards/rejected": -0.043156567960977554,
"eval_runtime": 2682.3759,
"eval_samples_per_second": 2.27,
"eval_steps_per_second": 0.284,
"step": 2900
},
{
"epoch": 0.8494490257607823,
"grad_norm": 0.01470947265625,
"learning_rate": 3.3663738250664853e-07,
"logits/chosen": -2.416839122772217,
"logits/rejected": -2.416839122772217,
"logps/chosen": -342.91815185546875,
"logps/rejected": -342.91815185546875,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.051371246576309204,
"rewards/margins": 0.0,
"rewards/rejected": -0.051371246576309204,
"step": 2910
},
{
"epoch": 0.8523680945778297,
"grad_norm": 0.0159912109375,
"learning_rate": 3.239783150588283e-07,
"logits/chosen": -2.3476662635803223,
"logits/rejected": -2.3476662635803223,
"logps/chosen": -304.71368408203125,
"logps/rejected": -304.71368408203125,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": 0.005339882802218199,
"rewards/margins": 0.0,
"rewards/rejected": 0.005339882802218199,
"step": 2920
},
{
"epoch": 0.855287163394877,
"grad_norm": 0.01409912109375,
"learning_rate": 3.1154534416082573e-07,
"logits/chosen": -2.416965961456299,
"logits/rejected": -2.416965961456299,
"logps/chosen": -299.3199157714844,
"logps/rejected": -299.3199157714844,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.04180007427930832,
"rewards/margins": 0.0,
"rewards/rejected": -0.04180007427930832,
"step": 2930
},
{
"epoch": 0.8582062322119244,
"grad_norm": 0.01055908203125,
"learning_rate": 2.9933976164343514e-07,
"logits/chosen": -2.4285387992858887,
"logits/rejected": -2.4285387992858887,
"logps/chosen": -303.32183837890625,
"logps/rejected": -303.32183837890625,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.04410778731107712,
"rewards/margins": 0.0,
"rewards/rejected": -0.04410778731107712,
"step": 2940
},
{
"epoch": 0.8611253010289718,
"grad_norm": 0.0162353515625,
"learning_rate": 2.873628357109745e-07,
"logits/chosen": -2.4083211421966553,
"logits/rejected": -2.4083211421966553,
"logps/chosen": -326.7142028808594,
"logps/rejected": -326.7142028808594,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.046554580330848694,
"rewards/margins": 0.0,
"rewards/rejected": -0.046554580330848694,
"step": 2950
},
{
"epoch": 0.8640443698460191,
"grad_norm": 0.01324462890625,
"learning_rate": 2.7561581080951195e-07,
"logits/chosen": -2.4226157665252686,
"logits/rejected": -2.4226157665252686,
"logps/chosen": -292.55767822265625,
"logps/rejected": -292.55767822265625,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.04246639460325241,
"rewards/margins": 0.0,
"rewards/rejected": -0.04246639460325241,
"step": 2960
},
{
"epoch": 0.8669634386630665,
"grad_norm": 0.01361083984375,
"learning_rate": 2.640999074975645e-07,
"logits/chosen": -2.43457293510437,
"logits/rejected": -2.43457293510437,
"logps/chosen": -298.2882385253906,
"logps/rejected": -298.2882385253906,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.04169774800539017,
"rewards/margins": 0.0,
"rewards/rejected": -0.04169774800539017,
"step": 2970
},
{
"epoch": 0.8698825074801139,
"grad_norm": 0.01708984375,
"learning_rate": 2.5281632231927786e-07,
"logits/chosen": -2.473017930984497,
"logits/rejected": -2.473017930984497,
"logps/chosen": -307.8494567871094,
"logps/rejected": -307.8494567871094,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.045904386788606644,
"rewards/margins": 0.0,
"rewards/rejected": -0.045904386788606644,
"step": 2980
},
{
"epoch": 0.8728015762971612,
"grad_norm": 0.014404296875,
"learning_rate": 2.417662276800997e-07,
"logits/chosen": -2.4377925395965576,
"logits/rejected": -2.4377925395965576,
"logps/chosen": -329.8043518066406,
"logps/rejected": -329.8043518066406,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.052566416561603546,
"rewards/margins": 0.0,
"rewards/rejected": -0.052566416561603546,
"step": 2990
},
{
"epoch": 0.8757206451142086,
"grad_norm": 0.01226806640625,
"learning_rate": 2.30950771724964e-07,
"logits/chosen": -2.4452061653137207,
"logits/rejected": -2.4452061653137207,
"logps/chosen": -316.7723388671875,
"logps/rejected": -316.7723388671875,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.048340607434511185,
"rewards/margins": 0.0,
"rewards/rejected": -0.048340607434511185,
"step": 3000
},
{
"epoch": 0.8757206451142086,
"eval_logits/chosen": -2.3918232917785645,
"eval_logits/rejected": -2.3918232917785645,
"eval_logps/chosen": -310.78662109375,
"eval_logps/rejected": -310.78662109375,
"eval_loss": 0.6931472420692444,
"eval_rewards/accuracies": 0.0,
"eval_rewards/chosen": -0.04308019578456879,
"eval_rewards/margins": 0.0,
"eval_rewards/rejected": -0.04308019578456879,
"eval_runtime": 2681.8561,
"eval_samples_per_second": 2.271,
"eval_steps_per_second": 0.284,
"step": 3000
},
{
"epoch": 0.878639713931256,
"grad_norm": 0.017822265625,
"learning_rate": 2.2037107821899272e-07,
"logits/chosen": -2.414727210998535,
"logits/rejected": -2.414727210998535,
"logps/chosen": -343.22796630859375,
"logps/rejected": -343.22796630859375,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.047485075891017914,
"rewards/margins": 0.0,
"rewards/rejected": -0.047485075891017914,
"step": 3010
},
{
"epoch": 0.8815587827483032,
"grad_norm": 0.01708984375,
"learning_rate": 2.100282464307357e-07,
"logits/chosen": -2.4386258125305176,
"logits/rejected": -2.4386258125305176,
"logps/chosen": -305.25250244140625,
"logps/rejected": -305.25250244140625,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.04185379669070244,
"rewards/margins": 0.0,
"rewards/rejected": -0.04185379669070244,
"step": 3020
},
{
"epoch": 0.8844778515653506,
"grad_norm": 0.016357421875,
"learning_rate": 1.999233510179488e-07,
"logits/chosen": -2.4112370014190674,
"logits/rejected": -2.4112370014190674,
"logps/chosen": -339.65093994140625,
"logps/rejected": -339.65093994140625,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.044059764593839645,
"rewards/margins": 0.0,
"rewards/rejected": -0.044059764593839645,
"step": 3030
},
{
"epoch": 0.887396920382398,
"grad_norm": 0.012939453125,
"learning_rate": 1.9005744191593678e-07,
"logits/chosen": -2.4179887771606445,
"logits/rejected": -2.4179887771606445,
"logps/chosen": -297.5303649902344,
"logps/rejected": -297.5303649902344,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.03384246677160263,
"rewards/margins": 0.0,
"rewards/rejected": -0.03384246677160263,
"step": 3040
},
{
"epoch": 0.8903159891994453,
"grad_norm": 0.0120849609375,
"learning_rate": 1.8043154422845794e-07,
"logits/chosen": -2.4646730422973633,
"logits/rejected": -2.4646730422973633,
"logps/chosen": -295.91790771484375,
"logps/rejected": -295.91790771484375,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.04882458597421646,
"rewards/margins": 0.0,
"rewards/rejected": -0.04882458597421646,
"step": 3050
},
{
"epoch": 0.8932350580164927,
"grad_norm": 0.0186767578125,
"learning_rate": 1.7104665812121445e-07,
"logits/chosen": -2.423285961151123,
"logits/rejected": -2.423285961151123,
"logps/chosen": -297.9593505859375,
"logps/rejected": -297.9593505859375,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.04218859225511551,
"rewards/margins": 0.0,
"rewards/rejected": -0.04218859225511551,
"step": 3060
},
{
"epoch": 0.8961541268335401,
"grad_norm": 0.0164794921875,
"learning_rate": 1.619037587179309e-07,
"logits/chosen": -2.3985249996185303,
"logits/rejected": -2.3985249996185303,
"logps/chosen": -332.85809326171875,
"logps/rejected": -332.85809326171875,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.048540227115154266,
"rewards/margins": 0.0,
"rewards/rejected": -0.048540227115154266,
"step": 3070
},
{
"epoch": 0.8990731956505874,
"grad_norm": 0.0172119140625,
"learning_rate": 1.5300379599903408e-07,
"logits/chosen": -2.4070308208465576,
"logits/rejected": -2.4070308208465576,
"logps/chosen": -310.7314147949219,
"logps/rejected": -310.7314147949219,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.03895800933241844,
"rewards/margins": 0.0,
"rewards/rejected": -0.03895800933241844,
"step": 3080
},
{
"epoch": 0.9019922644676348,
"grad_norm": 0.013671875,
"learning_rate": 1.44347694702949e-07,
"logits/chosen": -2.3916313648223877,
"logits/rejected": -2.3916313648223877,
"logps/chosen": -288.28106689453125,
"logps/rejected": -288.28106689453125,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.03293871134519577,
"rewards/margins": 0.0,
"rewards/rejected": -0.03293871134519577,
"step": 3090
},
{
"epoch": 0.9049113332846822,
"grad_norm": 0.017822265625,
"learning_rate": 1.359363542300124e-07,
"logits/chosen": -2.4147801399230957,
"logits/rejected": -2.4147801399230957,
"logps/chosen": -295.56768798828125,
"logps/rejected": -295.56768798828125,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.04364749416708946,
"rewards/margins": 0.0,
"rewards/rejected": -0.04364749416708946,
"step": 3100
},
{
"epoch": 0.9049113332846822,
"eval_logits/chosen": -2.390821933746338,
"eval_logits/rejected": -2.390821933746338,
"eval_logps/chosen": -310.7793884277344,
"eval_logps/rejected": -310.7793884277344,
"eval_loss": 0.6931472420692444,
"eval_rewards/accuracies": 0.0,
"eval_rewards/chosen": -0.04300786182284355,
"eval_rewards/margins": 0.0,
"eval_rewards/rejected": -0.04300786182284355,
"eval_runtime": 2681.9583,
"eval_samples_per_second": 2.271,
"eval_steps_per_second": 0.284,
"step": 3100
},
{
"epoch": 0.9078304021017295,
"grad_norm": 0.0152587890625,
"learning_rate": 1.2777064854902487e-07,
"logits/chosen": -2.44869065284729,
"logits/rejected": -2.44869065284729,
"logps/chosen": -324.82257080078125,
"logps/rejected": -324.82257080078125,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.04247719421982765,
"rewards/margins": 0.0,
"rewards/rejected": -0.04247719421982765,
"step": 3110
},
{
"epoch": 0.9107494709187769,
"grad_norm": 0.023681640625,
"learning_rate": 1.1985142610643902e-07,
"logits/chosen": -2.4080257415771484,
"logits/rejected": -2.4080257415771484,
"logps/chosen": -321.1974792480469,
"logps/rejected": -321.1974792480469,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.05160089209675789,
"rewards/margins": 0.0,
"rewards/rejected": -0.05160089209675789,
"step": 3120
},
{
"epoch": 0.9136685397358243,
"grad_norm": 0.01275634765625,
"learning_rate": 1.121795097382064e-07,
"logits/chosen": -2.422560691833496,
"logits/rejected": -2.422560691833496,
"logps/chosen": -335.0086975097656,
"logps/rejected": -335.0086975097656,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.04736005887389183,
"rewards/margins": 0.0,
"rewards/rejected": -0.04736005887389183,
"step": 3130
},
{
"epoch": 0.9165876085528716,
"grad_norm": 0.0169677734375,
"learning_rate": 1.0475569658427803e-07,
"logits/chosen": -2.438781261444092,
"logits/rejected": -2.438781261444092,
"logps/chosen": -311.33868408203125,
"logps/rejected": -311.33868408203125,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.03844516724348068,
"rewards/margins": 0.0,
"rewards/rejected": -0.03844516724348068,
"step": 3140
},
{
"epoch": 0.919506677369919,
"grad_norm": 0.02001953125,
"learning_rate": 9.758075800578193e-08,
"logits/chosen": -2.4374260902404785,
"logits/rejected": -2.4374260902404785,
"logps/chosen": -300.9288635253906,
"logps/rejected": -300.9288635253906,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.0434752032160759,
"rewards/margins": 0.0,
"rewards/rejected": -0.0434752032160759,
"step": 3150
},
{
"epoch": 0.9224257461869664,
"grad_norm": 0.01544189453125,
"learning_rate": 9.06554395048742e-08,
"logits/chosen": -2.4104561805725098,
"logits/rejected": -2.4104561805725098,
"logps/chosen": -310.27789306640625,
"logps/rejected": -310.27789306640625,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.04003220796585083,
"rewards/margins": 0.0,
"rewards/rejected": -0.04003220796585083,
"step": 3160
},
{
"epoch": 0.9253448150040137,
"grad_norm": 0.01416015625,
"learning_rate": 8.398046064727855e-08,
"logits/chosen": -2.448122262954712,
"logits/rejected": -2.448122262954712,
"logps/chosen": -303.9940185546875,
"logps/rejected": -303.9940185546875,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.04497329518198967,
"rewards/margins": 0.0,
"rewards/rejected": -0.04497329518198967,
"step": 3170
},
{
"epoch": 0.9282638838210611,
"grad_norm": 0.0140380859375,
"learning_rate": 7.755651498752265e-08,
"logits/chosen": -2.4395852088928223,
"logits/rejected": -2.4395852088928223,
"logps/chosen": -292.140380859375,
"logps/rejected": -292.140380859375,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.04522908851504326,
"rewards/margins": 0.0,
"rewards/rejected": -0.04522908851504326,
"step": 3180
},
{
"epoch": 0.9311829526381085,
"grad_norm": 0.016357421875,
"learning_rate": 7.138426999687171e-08,
"logits/chosen": -2.4227964878082275,
"logits/rejected": -2.4227964878082275,
"logps/chosen": -333.205810546875,
"logps/rejected": -333.205810546875,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.04331531375646591,
"rewards/margins": 0.0,
"rewards/rejected": -0.04331531375646591,
"step": 3190
},
{
"epoch": 0.9341020214551558,
"grad_norm": 0.0177001953125,
"learning_rate": 6.546436699398029e-08,
"logits/chosen": -2.4100470542907715,
"logits/rejected": -2.4100470542907715,
"logps/chosen": -334.2508850097656,
"logps/rejected": -334.2508850097656,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.043238040059804916,
"rewards/margins": 0.0,
"rewards/rejected": -0.043238040059804916,
"step": 3200
},
{
"epoch": 0.9341020214551558,
"eval_logits/chosen": -2.391075849533081,
"eval_logits/rejected": -2.391075849533081,
"eval_logps/chosen": -310.7811584472656,
"eval_logps/rejected": -310.7811584472656,
"eval_loss": 0.6931472420692444,
"eval_rewards/accuracies": 0.0,
"eval_rewards/chosen": -0.043025679886341095,
"eval_rewards/margins": 0.0,
"eval_rewards/rejected": -0.043025679886341095,
"eval_runtime": 2682.606,
"eval_samples_per_second": 2.27,
"eval_steps_per_second": 0.284,
"step": 3200
},
{
"epoch": 0.9370210902722031,
"grad_norm": 0.0150146484375,
"learning_rate": 5.979742107825287e-08,
"logits/chosen": -2.3894600868225098,
"logits/rejected": -2.3894600868225098,
"logps/chosen": -313.91131591796875,
"logps/rejected": -313.91131591796875,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.04388252645730972,
"rewards/margins": 0.0,
"rewards/rejected": -0.04388252645730972,
"step": 3210
},
{
"epoch": 0.9399401590892505,
"grad_norm": 0.01446533203125,
"learning_rate": 5.4384021065936045e-08,
"logits/chosen": -2.408024549484253,
"logits/rejected": -2.408024549484253,
"logps/chosen": -288.5419006347656,
"logps/rejected": -288.5419006347656,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.0435100793838501,
"rewards/margins": 0.0,
"rewards/rejected": -0.0435100793838501,
"step": 3220
},
{
"epoch": 0.9428592279062978,
"grad_norm": 0.033447265625,
"learning_rate": 4.9224729428935806e-08,
"logits/chosen": -2.423318862915039,
"logits/rejected": -2.423318862915039,
"logps/chosen": -309.74176025390625,
"logps/rejected": -309.74176025390625,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.04652589559555054,
"rewards/margins": 0.0,
"rewards/rejected": -0.04652589559555054,
"step": 3230
},
{
"epoch": 0.9457782967233452,
"grad_norm": 0.011962890625,
"learning_rate": 4.432008223637596e-08,
"logits/chosen": -2.4209766387939453,
"logits/rejected": -2.4209766387939453,
"logps/chosen": -299.3330078125,
"logps/rejected": -299.3330078125,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.04555036872625351,
"rewards/margins": 0.0,
"rewards/rejected": -0.04555036872625351,
"step": 3240
},
{
"epoch": 0.9486973655403926,
"grad_norm": 0.01953125,
"learning_rate": 3.967058909889937e-08,
"logits/chosen": -2.397352457046509,
"logits/rejected": -2.397352457046509,
"logps/chosen": -313.8124694824219,
"logps/rejected": -313.8124694824219,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.03916158154606819,
"rewards/margins": 0.0,
"rewards/rejected": -0.03916158154606819,
"step": 3250
},
{
"epoch": 0.9516164343574399,
"grad_norm": 0.014404296875,
"learning_rate": 3.5276733115715556e-08,
"logits/chosen": -2.448172092437744,
"logits/rejected": -2.448172092437744,
"logps/chosen": -305.734130859375,
"logps/rejected": -305.734130859375,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.050101179629564285,
"rewards/margins": 0.0,
"rewards/rejected": -0.050101179629564285,
"step": 3260
},
{
"epoch": 0.9545355031744873,
"grad_norm": 0.01397705078125,
"learning_rate": 3.11389708244067e-08,
"logits/chosen": -2.4387991428375244,
"logits/rejected": -2.4387991428375244,
"logps/chosen": -325.77374267578125,
"logps/rejected": -325.77374267578125,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.04632676765322685,
"rewards/margins": 0.0,
"rewards/rejected": -0.04632676765322685,
"step": 3270
},
{
"epoch": 0.9574545719915347,
"grad_norm": 0.0130615234375,
"learning_rate": 2.7257732153490313e-08,
"logits/chosen": -2.3997585773468018,
"logits/rejected": -2.3997585773468018,
"logps/chosen": -323.36962890625,
"logps/rejected": -323.36962890625,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.04246259480714798,
"rewards/margins": 0.0,
"rewards/rejected": -0.04246259480714798,
"step": 3280
},
{
"epoch": 0.960373640808582,
"grad_norm": 0.01226806640625,
"learning_rate": 2.3633420377749684e-08,
"logits/chosen": -2.404913902282715,
"logits/rejected": -2.404913902282715,
"logps/chosen": -309.89715576171875,
"logps/rejected": -309.89715576171875,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.05698453634977341,
"rewards/margins": 0.0,
"rewards/rejected": -0.05698453634977341,
"step": 3290
},
{
"epoch": 0.9632927096256294,
"grad_norm": 0.013671875,
"learning_rate": 2.0266412076330457e-08,
"logits/chosen": -2.431570529937744,
"logits/rejected": -2.431570529937744,
"logps/chosen": -297.8599548339844,
"logps/rejected": -297.8599548339844,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.052738986909389496,
"rewards/margins": 0.0,
"rewards/rejected": -0.052738986909389496,
"step": 3300
},
{
"epoch": 0.9632927096256294,
"eval_logits/chosen": -2.3914639949798584,
"eval_logits/rejected": -2.3914639949798584,
"eval_logps/chosen": -310.7767333984375,
"eval_logps/rejected": -310.7767333984375,
"eval_loss": 0.6931472420692444,
"eval_rewards/accuracies": 0.0,
"eval_rewards/chosen": -0.0429811105132103,
"eval_rewards/margins": 0.0,
"eval_rewards/rejected": -0.0429811105132103,
"eval_runtime": 2682.6978,
"eval_samples_per_second": 2.27,
"eval_steps_per_second": 0.284,
"step": 3300
},
{
"epoch": 0.9662117784426768,
"grad_norm": 0.01361083984375,
"learning_rate": 1.7157057093614704e-08,
"logits/chosen": -2.452519178390503,
"logits/rejected": -2.452519178390503,
"logps/chosen": -296.8190002441406,
"logps/rejected": -296.8190002441406,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.041969865560531616,
"rewards/margins": 0.0,
"rewards/rejected": -0.041969865560531616,
"step": 3310
},
{
"epoch": 0.9691308472597241,
"grad_norm": 0.01422119140625,
"learning_rate": 1.430567850286807e-08,
"logits/chosen": -2.4390811920166016,
"logits/rejected": -2.4390811920166016,
"logps/chosen": -339.14801025390625,
"logps/rejected": -339.14801025390625,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.0495796874165535,
"rewards/margins": 0.0,
"rewards/rejected": -0.0495796874165535,
"step": 3320
},
{
"epoch": 0.9720499160767715,
"grad_norm": 0.017333984375,
"learning_rate": 1.1712572572674386e-08,
"logits/chosen": -2.3779425621032715,
"logits/rejected": -2.3779425621032715,
"logps/chosen": -342.52117919921875,
"logps/rejected": -342.52117919921875,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.03777734562754631,
"rewards/margins": 0.0,
"rewards/rejected": -0.03777734562754631,
"step": 3330
},
{
"epoch": 0.9749689848938189,
"grad_norm": 0.0166015625,
"learning_rate": 9.378008736149746e-09,
"logits/chosen": -2.408357620239258,
"logits/rejected": -2.408357620239258,
"logps/chosen": -321.5648193359375,
"logps/rejected": -321.5648193359375,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.03947510942816734,
"rewards/margins": 0.0,
"rewards/rejected": -0.03947510942816734,
"step": 3340
},
{
"epoch": 0.9778880537108662,
"grad_norm": 0.01275634765625,
"learning_rate": 7.30222956294907e-09,
"logits/chosen": -2.456228733062744,
"logits/rejected": -2.456228733062744,
"logps/chosen": -322.9805603027344,
"logps/rejected": -322.9805603027344,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.04836040362715721,
"rewards/margins": 0.0,
"rewards/rejected": -0.04836040362715721,
"step": 3350
},
{
"epoch": 0.9808071225279136,
"grad_norm": 0.015380859375,
"learning_rate": 5.485450734061259e-09,
"logits/chosen": -2.395473003387451,
"logits/rejected": -2.395473003387451,
"logps/chosen": -292.87994384765625,
"logps/rejected": -292.87994384765625,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.0445740707218647,
"rewards/margins": 0.0,
"rewards/rejected": -0.0445740707218647,
"step": 3360
},
{
"epoch": 0.983726191344961,
"grad_norm": 0.01544189453125,
"learning_rate": 3.927861019399903e-09,
"logits/chosen": -2.406294345855713,
"logits/rejected": -2.406294345855713,
"logps/chosen": -288.55987548828125,
"logps/rejected": -288.55987548828125,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.03885042294859886,
"rewards/margins": 0.0,
"rewards/rejected": -0.03885042294859886,
"step": 3370
},
{
"epoch": 0.9866452601620083,
"grad_norm": 0.0155029296875,
"learning_rate": 2.629622258188691e-09,
"logits/chosen": -2.4149577617645264,
"logits/rejected": -2.4149577617645264,
"logps/chosen": -282.56585693359375,
"logps/rejected": -282.56585693359375,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.0403311550617218,
"rewards/margins": 0.0,
"rewards/rejected": -0.0403311550617218,
"step": 3380
},
{
"epoch": 0.9895643289790557,
"grad_norm": 0.01544189453125,
"learning_rate": 1.5908693421465282e-09,
"logits/chosen": -2.4097559452056885,
"logits/rejected": -2.4097559452056885,
"logps/chosen": -284.1174011230469,
"logps/rejected": -284.1174011230469,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.04213656857609749,
"rewards/margins": 0.0,
"rewards/rejected": -0.04213656857609749,
"step": 3390
},
{
"epoch": 0.9924833977961031,
"grad_norm": 0.0137939453125,
"learning_rate": 8.11710201470417e-10,
"logits/chosen": -2.4348714351654053,
"logits/rejected": -2.4348714351654053,
"logps/chosen": -325.41339111328125,
"logps/rejected": -325.41339111328125,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.04575268179178238,
"rewards/margins": 0.0,
"rewards/rejected": -0.04575268179178238,
"step": 3400
},
{
"epoch": 0.9924833977961031,
"eval_logits/chosen": -2.3908708095550537,
"eval_logits/rejected": -2.3908708095550537,
"eval_logps/chosen": -310.7832336425781,
"eval_logps/rejected": -310.7832336425781,
"eval_loss": 0.6931472420692444,
"eval_rewards/accuracies": 0.0,
"eval_rewards/chosen": -0.04304642230272293,
"eval_rewards/margins": 0.0,
"eval_rewards/rejected": -0.04304642230272293,
"eval_runtime": 2747.5083,
"eval_samples_per_second": 2.217,
"eval_steps_per_second": 0.277,
"step": 3400
},
{
"epoch": 0.9954024666131503,
"grad_norm": 0.01165771484375,
"learning_rate": 2.922257936230355e-10,
"logits/chosen": -2.409545421600342,
"logits/rejected": -2.409545421600342,
"logps/chosen": -264.7913818359375,
"logps/rejected": -264.7913818359375,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.038275159895420074,
"rewards/margins": 0.0,
"rewards/rejected": -0.038275159895420074,
"step": 3410
},
{
"epoch": 0.9983215354301977,
"grad_norm": 0.0167236328125,
"learning_rate": 3.247009491946784e-11,
"logits/chosen": -2.429719924926758,
"logits/rejected": -2.429719924926758,
"logps/chosen": -340.23126220703125,
"logps/rejected": -340.23126220703125,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.04289505258202553,
"rewards/margins": 0.0,
"rewards/rejected": -0.04289505258202553,
"step": 3420
},
{
"epoch": 0.9997810698387214,
"step": 3425,
"total_flos": 0.0,
"train_loss": 0.18720042388804636,
"train_runtime": 41876.0871,
"train_samples_per_second": 1.309,
"train_steps_per_second": 0.082
}
],
"logging_steps": 10,
"max_steps": 3425,
"num_input_tokens_seen": 0,
"num_train_epochs": 1,
"save_steps": 100,
"total_flos": 0.0,
"train_batch_size": 4,
"trial_name": null,
"trial_params": null
}