| { | |
| "best_metric": null, | |
| "best_model_checkpoint": null, | |
| "epoch": 0.9995965030262273, | |
| "eval_steps": 300, | |
| "global_step": 1858, | |
| "is_hyper_param_search": false, | |
| "is_local_process_zero": true, | |
| "is_world_process_zero": true, | |
| "log_history": [ | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 2.6881720430107528e-09, | |
| "logits/chosen": -2.670260429382324, | |
| "logits/rejected": -2.1533777713775635, | |
| "logps/chosen": -299.33551025390625, | |
| "logps/rejected": -186.81130981445312, | |
| "loss": 13125.0, | |
| "rewards/accuracies": 0.0, | |
| "rewards/chosen": 0.0, | |
| "rewards/margins": 0.0, | |
| "rewards/rejected": 0.0, | |
| "rewards/safe_rewards": 0.0, | |
| "rewards/unsafe_rewards": 0.0, | |
| "step": 1 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 2.6881720430107527e-08, | |
| "logits/chosen": -2.4763858318328857, | |
| "logits/rejected": -2.354341745376587, | |
| "logps/chosen": -201.82504272460938, | |
| "logps/rejected": -189.45822143554688, | |
| "loss": 14971.9097, | |
| "rewards/accuracies": 0.4930555522441864, | |
| "rewards/chosen": 9.199242413160391e-06, | |
| "rewards/margins": -8.218608854804188e-05, | |
| "rewards/rejected": 9.138535824604332e-05, | |
| "rewards/safe_rewards": 1.8223654478788376e-05, | |
| "rewards/unsafe_rewards": 1.7482994962847442e-07, | |
| "step": 10 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 5.3763440860215054e-08, | |
| "logits/chosen": -2.44279408454895, | |
| "logits/rejected": -2.218726873397827, | |
| "logps/chosen": -226.34970092773438, | |
| "logps/rejected": -181.1803436279297, | |
| "loss": 15239.6516, | |
| "rewards/accuracies": 0.5, | |
| "rewards/chosen": -0.0002221722388640046, | |
| "rewards/margins": -0.0001655549422139302, | |
| "rewards/rejected": -5.6617216614540666e-05, | |
| "rewards/safe_rewards": -0.0002462912234477699, | |
| "rewards/unsafe_rewards": -0.0001980532251764089, | |
| "step": 20 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 8.064516129032257e-08, | |
| "logits/chosen": -2.4326512813568115, | |
| "logits/rejected": -2.2918035984039307, | |
| "logps/chosen": -215.0783233642578, | |
| "logps/rejected": -189.28309631347656, | |
| "loss": 14993.0922, | |
| "rewards/accuracies": 0.375, | |
| "rewards/chosen": -0.0004244670271873474, | |
| "rewards/margins": -0.000839566346257925, | |
| "rewards/rejected": 0.0004150994645897299, | |
| "rewards/safe_rewards": -0.0002224749478045851, | |
| "rewards/unsafe_rewards": -0.0006264590774662793, | |
| "step": 30 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 1.0752688172043011e-07, | |
| "logits/chosen": -2.375096082687378, | |
| "logits/rejected": -2.157045841217041, | |
| "logps/chosen": -180.7657470703125, | |
| "logps/rejected": -173.87054443359375, | |
| "loss": 14723.7563, | |
| "rewards/accuracies": 0.543749988079071, | |
| "rewards/chosen": -0.00011304272629786283, | |
| "rewards/margins": 0.0008593280799686909, | |
| "rewards/rejected": -0.0009723707917146385, | |
| "rewards/safe_rewards": 0.0004990470479242504, | |
| "rewards/unsafe_rewards": -0.0007251326460391283, | |
| "step": 40 | |
| }, | |
| { | |
| "epoch": 0.03, | |
| "learning_rate": 1.3440860215053762e-07, | |
| "logits/chosen": -2.489865303039551, | |
| "logits/rejected": -2.1804325580596924, | |
| "logps/chosen": -209.59664916992188, | |
| "logps/rejected": -167.6984100341797, | |
| "loss": 14816.0938, | |
| "rewards/accuracies": 0.643750011920929, | |
| "rewards/chosen": 0.0011160348076373339, | |
| "rewards/margins": 0.0046949503012001514, | |
| "rewards/rejected": -0.0035789154935628176, | |
| "rewards/safe_rewards": 0.00042248546378687024, | |
| "rewards/unsafe_rewards": 0.0018095843261107802, | |
| "step": 50 | |
| }, | |
| { | |
| "epoch": 0.03, | |
| "learning_rate": 1.6129032258064515e-07, | |
| "logits/chosen": -2.4289777278900146, | |
| "logits/rejected": -2.2897238731384277, | |
| "logps/chosen": -185.92630004882812, | |
| "logps/rejected": -185.1912841796875, | |
| "loss": 14593.3875, | |
| "rewards/accuracies": 0.5562499761581421, | |
| "rewards/chosen": -0.0020237788558006287, | |
| "rewards/margins": 0.002217882312834263, | |
| "rewards/rejected": -0.004241660702973604, | |
| "rewards/safe_rewards": -0.001639070687815547, | |
| "rewards/unsafe_rewards": -0.002408486558124423, | |
| "step": 60 | |
| }, | |
| { | |
| "epoch": 0.04, | |
| "learning_rate": 1.8817204301075268e-07, | |
| "logits/chosen": -2.423598289489746, | |
| "logits/rejected": -2.2320618629455566, | |
| "logps/chosen": -202.7837677001953, | |
| "logps/rejected": -184.62033081054688, | |
| "loss": 14276.2172, | |
| "rewards/accuracies": 0.5375000238418579, | |
| "rewards/chosen": -0.005977807100862265, | |
| "rewards/margins": 0.005661749746650457, | |
| "rewards/rejected": -0.011639557778835297, | |
| "rewards/safe_rewards": -0.00685838982462883, | |
| "rewards/unsafe_rewards": -0.005097225774079561, | |
| "step": 70 | |
| }, | |
| { | |
| "epoch": 0.04, | |
| "learning_rate": 2.1505376344086022e-07, | |
| "logits/chosen": -2.4342398643493652, | |
| "logits/rejected": -2.271530866622925, | |
| "logps/chosen": -221.3511505126953, | |
| "logps/rejected": -196.20684814453125, | |
| "loss": 14430.5766, | |
| "rewards/accuracies": 0.6312500238418579, | |
| "rewards/chosen": -0.017918918281793594, | |
| "rewards/margins": 0.013557764701545238, | |
| "rewards/rejected": -0.03147668391466141, | |
| "rewards/safe_rewards": -0.020387938246130943, | |
| "rewards/unsafe_rewards": -0.015449894592165947, | |
| "step": 80 | |
| }, | |
| { | |
| "epoch": 0.05, | |
| "learning_rate": 2.4193548387096775e-07, | |
| "logits/chosen": -2.4415786266326904, | |
| "logits/rejected": -2.301713466644287, | |
| "logps/chosen": -211.0481414794922, | |
| "logps/rejected": -171.99159240722656, | |
| "loss": 14501.5938, | |
| "rewards/accuracies": 0.625, | |
| "rewards/chosen": -0.05115891247987747, | |
| "rewards/margins": 0.025721188634634018, | |
| "rewards/rejected": -0.07688009738922119, | |
| "rewards/safe_rewards": -0.04981667920947075, | |
| "rewards/unsafe_rewards": -0.05250114947557449, | |
| "step": 90 | |
| }, | |
| { | |
| "epoch": 0.05, | |
| "learning_rate": 2.6881720430107523e-07, | |
| "logits/chosen": -2.4574618339538574, | |
| "logits/rejected": -2.3291263580322266, | |
| "logps/chosen": -204.63087463378906, | |
| "logps/rejected": -179.63002014160156, | |
| "loss": 14364.2516, | |
| "rewards/accuracies": 0.59375, | |
| "rewards/chosen": -0.08542316406965256, | |
| "rewards/margins": 0.03905550017952919, | |
| "rewards/rejected": -0.12447866052389145, | |
| "rewards/safe_rewards": -0.08581903576850891, | |
| "rewards/unsafe_rewards": -0.0850272923707962, | |
| "step": 100 | |
| }, | |
| { | |
| "epoch": 0.06, | |
| "learning_rate": 2.956989247311828e-07, | |
| "logits/chosen": -2.429097890853882, | |
| "logits/rejected": -2.2684311866760254, | |
| "logps/chosen": -222.95913696289062, | |
| "logps/rejected": -187.91293334960938, | |
| "loss": 14761.6688, | |
| "rewards/accuracies": 0.6812499761581421, | |
| "rewards/chosen": -0.11019313335418701, | |
| "rewards/margins": 0.048916272819042206, | |
| "rewards/rejected": -0.159109428524971, | |
| "rewards/safe_rewards": -0.10632093250751495, | |
| "rewards/unsafe_rewards": -0.11406532675027847, | |
| "step": 110 | |
| }, | |
| { | |
| "epoch": 0.06, | |
| "learning_rate": 3.225806451612903e-07, | |
| "logits/chosen": -2.4783873558044434, | |
| "logits/rejected": -2.262389659881592, | |
| "logps/chosen": -220.35800170898438, | |
| "logps/rejected": -169.4020233154297, | |
| "loss": 13980.0906, | |
| "rewards/accuracies": 0.731249988079071, | |
| "rewards/chosen": -0.09087369590997696, | |
| "rewards/margins": 0.07458756119012833, | |
| "rewards/rejected": -0.1654612421989441, | |
| "rewards/safe_rewards": -0.08417809009552002, | |
| "rewards/unsafe_rewards": -0.0975693017244339, | |
| "step": 120 | |
| }, | |
| { | |
| "epoch": 0.07, | |
| "learning_rate": 3.4946236559139783e-07, | |
| "logits/chosen": -2.47499942779541, | |
| "logits/rejected": -2.3391237258911133, | |
| "logps/chosen": -224.6880645751953, | |
| "logps/rejected": -181.26388549804688, | |
| "loss": 13439.1469, | |
| "rewards/accuracies": 0.675000011920929, | |
| "rewards/chosen": -0.10466276109218597, | |
| "rewards/margins": 0.0889045000076294, | |
| "rewards/rejected": -0.19356727600097656, | |
| "rewards/safe_rewards": -0.07197071611881256, | |
| "rewards/unsafe_rewards": -0.13735483586788177, | |
| "step": 130 | |
| }, | |
| { | |
| "epoch": 0.08, | |
| "learning_rate": 3.7634408602150537e-07, | |
| "logits/chosen": -2.5111544132232666, | |
| "logits/rejected": -2.323676347732544, | |
| "logps/chosen": -232.02120971679688, | |
| "logps/rejected": -207.3649444580078, | |
| "loss": 13156.9813, | |
| "rewards/accuracies": 0.6812499761581421, | |
| "rewards/chosen": -0.20595864951610565, | |
| "rewards/margins": 0.0673115998506546, | |
| "rewards/rejected": -0.27327024936676025, | |
| "rewards/safe_rewards": -0.19269119203090668, | |
| "rewards/unsafe_rewards": -0.21922609210014343, | |
| "step": 140 | |
| }, | |
| { | |
| "epoch": 0.08, | |
| "learning_rate": 4.0322580645161285e-07, | |
| "logits/chosen": -2.440796375274658, | |
| "logits/rejected": -2.294173002243042, | |
| "logps/chosen": -229.3905792236328, | |
| "logps/rejected": -201.4041748046875, | |
| "loss": 12286.168, | |
| "rewards/accuracies": 0.675000011920929, | |
| "rewards/chosen": -0.2244747132062912, | |
| "rewards/margins": 0.09208513051271439, | |
| "rewards/rejected": -0.3165598511695862, | |
| "rewards/safe_rewards": -0.2293473184108734, | |
| "rewards/unsafe_rewards": -0.21960210800170898, | |
| "step": 150 | |
| }, | |
| { | |
| "epoch": 0.09, | |
| "learning_rate": 4.3010752688172043e-07, | |
| "logits/chosen": -2.4001574516296387, | |
| "logits/rejected": -2.2244515419006348, | |
| "logps/chosen": -242.0061492919922, | |
| "logps/rejected": -220.7064666748047, | |
| "loss": 13450.5672, | |
| "rewards/accuracies": 0.643750011920929, | |
| "rewards/chosen": -0.2182481735944748, | |
| "rewards/margins": 0.09150904417037964, | |
| "rewards/rejected": -0.30975720286369324, | |
| "rewards/safe_rewards": -0.20791450142860413, | |
| "rewards/unsafe_rewards": -0.22858186066150665, | |
| "step": 160 | |
| }, | |
| { | |
| "epoch": 0.09, | |
| "learning_rate": 4.569892473118279e-07, | |
| "logits/chosen": -2.4316458702087402, | |
| "logits/rejected": -2.3187336921691895, | |
| "logps/chosen": -229.7932891845703, | |
| "logps/rejected": -191.04086303710938, | |
| "loss": 12580.4609, | |
| "rewards/accuracies": 0.699999988079071, | |
| "rewards/chosen": -0.15405751764774323, | |
| "rewards/margins": 0.08396536856889725, | |
| "rewards/rejected": -0.23802292346954346, | |
| "rewards/safe_rewards": -0.13653624057769775, | |
| "rewards/unsafe_rewards": -0.1715788096189499, | |
| "step": 170 | |
| }, | |
| { | |
| "epoch": 0.1, | |
| "learning_rate": 4.838709677419355e-07, | |
| "logits/chosen": -2.473118305206299, | |
| "logits/rejected": -2.3191967010498047, | |
| "logps/chosen": -217.89785766601562, | |
| "logps/rejected": -209.47775268554688, | |
| "loss": 12886.1875, | |
| "rewards/accuracies": 0.731249988079071, | |
| "rewards/chosen": -0.2591598629951477, | |
| "rewards/margins": 0.1223045364022255, | |
| "rewards/rejected": -0.3814643919467926, | |
| "rewards/safe_rewards": -0.2669682800769806, | |
| "rewards/unsafe_rewards": -0.25135138630867004, | |
| "step": 180 | |
| }, | |
| { | |
| "epoch": 0.1, | |
| "learning_rate": 4.999929391798331e-07, | |
| "logits/chosen": -2.4944000244140625, | |
| "logits/rejected": -2.323819637298584, | |
| "logps/chosen": -224.4022674560547, | |
| "logps/rejected": -209.09341430664062, | |
| "loss": 12964.0625, | |
| "rewards/accuracies": 0.71875, | |
| "rewards/chosen": -0.30063313245773315, | |
| "rewards/margins": 0.11855147778987885, | |
| "rewards/rejected": -0.4191845953464508, | |
| "rewards/safe_rewards": -0.30268028378486633, | |
| "rewards/unsafe_rewards": -0.2985859215259552, | |
| "step": 190 | |
| }, | |
| { | |
| "epoch": 0.11, | |
| "learning_rate": 4.9991350953333e-07, | |
| "logits/chosen": -2.4175021648406982, | |
| "logits/rejected": -2.261751413345337, | |
| "logps/chosen": -257.97100830078125, | |
| "logps/rejected": -248.92697143554688, | |
| "loss": 12408.5, | |
| "rewards/accuracies": 0.675000011920929, | |
| "rewards/chosen": -0.3828560709953308, | |
| "rewards/margins": 0.09374293684959412, | |
| "rewards/rejected": -0.4765990376472473, | |
| "rewards/safe_rewards": -0.37296319007873535, | |
| "rewards/unsafe_rewards": -0.39274901151657104, | |
| "step": 200 | |
| }, | |
| { | |
| "epoch": 0.11, | |
| "learning_rate": 4.997458523498236e-07, | |
| "logits/chosen": -2.4530012607574463, | |
| "logits/rejected": -2.3131752014160156, | |
| "logps/chosen": -222.4829559326172, | |
| "logps/rejected": -198.89447021484375, | |
| "loss": 12109.7687, | |
| "rewards/accuracies": 0.574999988079071, | |
| "rewards/chosen": -0.3033100664615631, | |
| "rewards/margins": 0.09060867875814438, | |
| "rewards/rejected": -0.3939187526702881, | |
| "rewards/safe_rewards": -0.29972246289253235, | |
| "rewards/unsafe_rewards": -0.30689769983291626, | |
| "step": 210 | |
| }, | |
| { | |
| "epoch": 0.12, | |
| "learning_rate": 4.99490026817712e-07, | |
| "logits/chosen": -2.415255308151245, | |
| "logits/rejected": -2.290353775024414, | |
| "logps/chosen": -225.1759490966797, | |
| "logps/rejected": -215.8201446533203, | |
| "loss": 12286.7969, | |
| "rewards/accuracies": 0.668749988079071, | |
| "rewards/chosen": -0.308417946100235, | |
| "rewards/margins": 0.15336203575134277, | |
| "rewards/rejected": -0.46177998185157776, | |
| "rewards/safe_rewards": -0.2879489064216614, | |
| "rewards/unsafe_rewards": -0.3288869559764862, | |
| "step": 220 | |
| }, | |
| { | |
| "epoch": 0.12, | |
| "learning_rate": 4.991461232516674e-07, | |
| "logits/chosen": -2.403742551803589, | |
| "logits/rejected": -2.2323994636535645, | |
| "logps/chosen": -265.6297607421875, | |
| "logps/rejected": -247.0828399658203, | |
| "loss": 13271.4391, | |
| "rewards/accuracies": 0.675000011920929, | |
| "rewards/chosen": -0.35555627942085266, | |
| "rewards/margins": 0.13768498599529266, | |
| "rewards/rejected": -0.49324122071266174, | |
| "rewards/safe_rewards": -0.34540650248527527, | |
| "rewards/unsafe_rewards": -0.36570602655410767, | |
| "step": 230 | |
| }, | |
| { | |
| "epoch": 0.13, | |
| "learning_rate": 4.98714263060751e-07, | |
| "logits/chosen": -2.5071444511413574, | |
| "logits/rejected": -2.31742525100708, | |
| "logps/chosen": -209.05905151367188, | |
| "logps/rejected": -179.74855041503906, | |
| "loss": 12854.6859, | |
| "rewards/accuracies": 0.6499999761581421, | |
| "rewards/chosen": -0.2065323293209076, | |
| "rewards/margins": 0.12446670234203339, | |
| "rewards/rejected": -0.33099907636642456, | |
| "rewards/safe_rewards": -0.21961939334869385, | |
| "rewards/unsafe_rewards": -0.19344526529312134, | |
| "step": 240 | |
| }, | |
| { | |
| "epoch": 0.13, | |
| "learning_rate": 4.98194598705552e-07, | |
| "logits/chosen": -2.460813045501709, | |
| "logits/rejected": -2.3778514862060547, | |
| "logps/chosen": -238.38693237304688, | |
| "logps/rejected": -223.18795776367188, | |
| "loss": 13078.5102, | |
| "rewards/accuracies": 0.574999988079071, | |
| "rewards/chosen": -0.48650604486465454, | |
| "rewards/margins": 0.07239247858524323, | |
| "rewards/rejected": -0.5588985681533813, | |
| "rewards/safe_rewards": -0.4960268437862396, | |
| "rewards/unsafe_rewards": -0.47698527574539185, | |
| "step": 250 | |
| }, | |
| { | |
| "epoch": 0.14, | |
| "learning_rate": 4.975873136443648e-07, | |
| "logits/chosen": -2.5023112297058105, | |
| "logits/rejected": -2.350739002227783, | |
| "logps/chosen": -268.8946228027344, | |
| "logps/rejected": -247.2296600341797, | |
| "loss": 11995.3344, | |
| "rewards/accuracies": 0.6937500238418579, | |
| "rewards/chosen": -0.3785567581653595, | |
| "rewards/margins": 0.12816599011421204, | |
| "rewards/rejected": -0.5067228078842163, | |
| "rewards/safe_rewards": -0.39122840762138367, | |
| "rewards/unsafe_rewards": -0.36588507890701294, | |
| "step": 260 | |
| }, | |
| { | |
| "epoch": 0.15, | |
| "learning_rate": 4.968926222684212e-07, | |
| "logits/chosen": -2.4438796043395996, | |
| "logits/rejected": -2.3461410999298096, | |
| "logps/chosen": -229.61386108398438, | |
| "logps/rejected": -223.92630004882812, | |
| "loss": 12123.7047, | |
| "rewards/accuracies": 0.6875, | |
| "rewards/chosen": -0.294075608253479, | |
| "rewards/margins": 0.13579340279102325, | |
| "rewards/rejected": -0.42986902594566345, | |
| "rewards/safe_rewards": -0.300296813249588, | |
| "rewards/unsafe_rewards": -0.2878544330596924, | |
| "step": 270 | |
| }, | |
| { | |
| "epoch": 0.15, | |
| "learning_rate": 4.961107698262044e-07, | |
| "logits/chosen": -2.4015512466430664, | |
| "logits/rejected": -2.263901948928833, | |
| "logps/chosen": -255.80538940429688, | |
| "logps/rejected": -228.9020233154297, | |
| "loss": 12352.2758, | |
| "rewards/accuracies": 0.65625, | |
| "rewards/chosen": -0.375516414642334, | |
| "rewards/margins": 0.10626471042633057, | |
| "rewards/rejected": -0.48178109526634216, | |
| "rewards/safe_rewards": -0.37126559019088745, | |
| "rewards/unsafe_rewards": -0.37976714968681335, | |
| "step": 280 | |
| }, | |
| { | |
| "epoch": 0.16, | |
| "learning_rate": 4.952420323368673e-07, | |
| "logits/chosen": -2.4498801231384277, | |
| "logits/rejected": -2.382075548171997, | |
| "logps/chosen": -239.36386108398438, | |
| "logps/rejected": -241.67861938476562, | |
| "loss": 12562.6453, | |
| "rewards/accuracies": 0.675000011920929, | |
| "rewards/chosen": -0.47904032468795776, | |
| "rewards/margins": 0.12548959255218506, | |
| "rewards/rejected": -0.6045299172401428, | |
| "rewards/safe_rewards": -0.4896617829799652, | |
| "rewards/unsafe_rewards": -0.46841883659362793, | |
| "step": 290 | |
| }, | |
| { | |
| "epoch": 0.16, | |
| "learning_rate": 4.942867164927899e-07, | |
| "logits/chosen": -2.448484182357788, | |
| "logits/rejected": -2.3499300479888916, | |
| "logps/chosen": -233.24447631835938, | |
| "logps/rejected": -213.6923828125, | |
| "loss": 13096.7359, | |
| "rewards/accuracies": 0.668749988079071, | |
| "rewards/chosen": -0.3095199167728424, | |
| "rewards/margins": 0.13528046011924744, | |
| "rewards/rejected": -0.44480031728744507, | |
| "rewards/safe_rewards": -0.2981899082660675, | |
| "rewards/unsafe_rewards": -0.3208498954772949, | |
| "step": 300 | |
| }, | |
| { | |
| "epoch": 0.16, | |
| "eval_logits/chosen": -2.319474458694458, | |
| "eval_logits/rejected": -2.1815290451049805, | |
| "eval_logps/chosen": -170.00274658203125, | |
| "eval_logps/rejected": -140.18304443359375, | |
| "eval_loss": 4529.67333984375, | |
| "eval_rewards/accuracies": 0.6583978533744812, | |
| "eval_rewards/chosen": -0.39572206139564514, | |
| "eval_rewards/margins": 0.08145187050104141, | |
| "eval_rewards/rejected": -0.4771738648414612, | |
| "eval_rewards/safe_rewards": -0.39296460151672363, | |
| "eval_rewards/unsafe_rewards": -0.3956325650215149, | |
| "eval_runtime": 996.6135, | |
| "eval_samples_per_second": 33.156, | |
| "eval_steps_per_second": 1.037, | |
| "step": 300 | |
| }, | |
| { | |
| "epoch": 0.17, | |
| "learning_rate": 4.932451595513062e-07, | |
| "logits/chosen": -2.464543342590332, | |
| "logits/rejected": -2.3167717456817627, | |
| "logps/chosen": -244.10421752929688, | |
| "logps/rejected": -229.1746368408203, | |
| "loss": 11442.1586, | |
| "rewards/accuracies": 0.75, | |
| "rewards/chosen": -0.25108686089515686, | |
| "rewards/margins": 0.15766267478466034, | |
| "rewards/rejected": -0.408749520778656, | |
| "rewards/safe_rewards": -0.2550010681152344, | |
| "rewards/unsafe_rewards": -0.24717266857624054, | |
| "step": 310 | |
| }, | |
| { | |
| "epoch": 0.17, | |
| "learning_rate": 4.921177292156419e-07, | |
| "logits/chosen": -2.486952543258667, | |
| "logits/rejected": -2.317573070526123, | |
| "logps/chosen": -234.8188934326172, | |
| "logps/rejected": -228.7853240966797, | |
| "loss": 11535.2578, | |
| "rewards/accuracies": 0.7437499761581421, | |
| "rewards/chosen": -0.2879149317741394, | |
| "rewards/margins": 0.18317629396915436, | |
| "rewards/rejected": -0.47109121084213257, | |
| "rewards/safe_rewards": -0.2716086506843567, | |
| "rewards/unsafe_rewards": -0.30422115325927734, | |
| "step": 320 | |
| }, | |
| { | |
| "epoch": 0.18, | |
| "learning_rate": 4.909048235051033e-07, | |
| "logits/chosen": -2.408730983734131, | |
| "logits/rejected": -2.313384532928467, | |
| "logps/chosen": -240.1051483154297, | |
| "logps/rejected": -238.6286163330078, | |
| "loss": 11998.6938, | |
| "rewards/accuracies": 0.6875, | |
| "rewards/chosen": -0.4070356488227844, | |
| "rewards/margins": 0.15228822827339172, | |
| "rewards/rejected": -0.559323787689209, | |
| "rewards/safe_rewards": -0.38683491945266724, | |
| "rewards/unsafe_rewards": -0.42723625898361206, | |
| "step": 330 | |
| }, | |
| { | |
| "epoch": 0.18, | |
| "learning_rate": 4.896068706145631e-07, | |
| "logits/chosen": -2.430915355682373, | |
| "logits/rejected": -2.3069653511047363, | |
| "logps/chosen": -263.82440185546875, | |
| "logps/rejected": -228.1202392578125, | |
| "loss": 12415.2109, | |
| "rewards/accuracies": 0.668749988079071, | |
| "rewards/chosen": -0.5314391255378723, | |
| "rewards/margins": 0.14294834434986115, | |
| "rewards/rejected": -0.674387514591217, | |
| "rewards/safe_rewards": -0.5249046087265015, | |
| "rewards/unsafe_rewards": -0.5379736423492432, | |
| "step": 340 | |
| }, | |
| { | |
| "epoch": 0.19, | |
| "learning_rate": 4.882243287632946e-07, | |
| "logits/chosen": -2.5057172775268555, | |
| "logits/rejected": -2.3827576637268066, | |
| "logps/chosen": -237.1790771484375, | |
| "logps/rejected": -237.27392578125, | |
| "loss": 11917.3391, | |
| "rewards/accuracies": 0.6625000238418579, | |
| "rewards/chosen": -0.44174814224243164, | |
| "rewards/margins": 0.1188526302576065, | |
| "rewards/rejected": -0.5606008172035217, | |
| "rewards/safe_rewards": -0.4454471170902252, | |
| "rewards/unsafe_rewards": -0.4380492568016052, | |
| "step": 350 | |
| }, | |
| { | |
| "epoch": 0.19, | |
| "learning_rate": 4.867576860332048e-07, | |
| "logits/chosen": -2.5187079906463623, | |
| "logits/rejected": -2.4128689765930176, | |
| "logps/chosen": -207.39404296875, | |
| "logps/rejected": -220.6639404296875, | |
| "loss": 11619.4062, | |
| "rewards/accuracies": 0.706250011920929, | |
| "rewards/chosen": -0.353740930557251, | |
| "rewards/margins": 0.16230328381061554, | |
| "rewards/rejected": -0.5160442590713501, | |
| "rewards/safe_rewards": -0.37073829770088196, | |
| "rewards/unsafe_rewards": -0.33674362301826477, | |
| "step": 360 | |
| }, | |
| { | |
| "epoch": 0.2, | |
| "learning_rate": 4.85207460196526e-07, | |
| "logits/chosen": -2.5497612953186035, | |
| "logits/rejected": -2.4256086349487305, | |
| "logps/chosen": -251.84036254882812, | |
| "logps/rejected": -246.384033203125, | |
| "loss": 11846.7016, | |
| "rewards/accuracies": 0.668749988079071, | |
| "rewards/chosen": -0.47489339113235474, | |
| "rewards/margins": 0.13903862237930298, | |
| "rewards/rejected": -0.6139320135116577, | |
| "rewards/safe_rewards": -0.4880181849002838, | |
| "rewards/unsafe_rewards": -0.46176856756210327, | |
| "step": 370 | |
| }, | |
| { | |
| "epoch": 0.2, | |
| "learning_rate": 4.835741985330259e-07, | |
| "logits/chosen": -2.5404162406921387, | |
| "logits/rejected": -2.4239704608917236, | |
| "logps/chosen": -246.53939819335938, | |
| "logps/rejected": -233.34353637695312, | |
| "loss": 11351.725, | |
| "rewards/accuracies": 0.7437499761581421, | |
| "rewards/chosen": -0.45782560110092163, | |
| "rewards/margins": 0.15662749111652374, | |
| "rewards/rejected": -0.614453136920929, | |
| "rewards/safe_rewards": -0.42890676856040955, | |
| "rewards/unsafe_rewards": -0.48674440383911133, | |
| "step": 380 | |
| }, | |
| { | |
| "epoch": 0.21, | |
| "learning_rate": 4.818584776367992e-07, | |
| "logits/chosen": -2.5068519115448, | |
| "logits/rejected": -2.4303596019744873, | |
| "logps/chosen": -244.3930206298828, | |
| "logps/rejected": -249.14187622070312, | |
| "loss": 11726.7609, | |
| "rewards/accuracies": 0.75, | |
| "rewards/chosen": -0.40478819608688354, | |
| "rewards/margins": 0.16650350391864777, | |
| "rewards/rejected": -0.5712917447090149, | |
| "rewards/safe_rewards": -0.43173927068710327, | |
| "rewards/unsafe_rewards": -0.377837210893631, | |
| "step": 390 | |
| }, | |
| { | |
| "epoch": 0.22, | |
| "learning_rate": 4.800609032127122e-07, | |
| "logits/chosen": -2.4628491401672363, | |
| "logits/rejected": -2.374331474304199, | |
| "logps/chosen": -254.27822875976562, | |
| "logps/rejected": -228.73330688476562, | |
| "loss": 12046.5695, | |
| "rewards/accuracies": 0.6312500238418579, | |
| "rewards/chosen": -0.48730865120887756, | |
| "rewards/margins": 0.11220350116491318, | |
| "rewards/rejected": -0.5995121598243713, | |
| "rewards/safe_rewards": -0.4591635763645172, | |
| "rewards/unsafe_rewards": -0.5154536962509155, | |
| "step": 400 | |
| }, | |
| { | |
| "epoch": 0.22, | |
| "learning_rate": 4.78182109862569e-07, | |
| "logits/chosen": -2.4446866512298584, | |
| "logits/rejected": -2.408660650253296, | |
| "logps/chosen": -230.39208984375, | |
| "logps/rejected": -231.1884765625, | |
| "loss": 12854.2617, | |
| "rewards/accuracies": 0.6625000238418579, | |
| "rewards/chosen": -0.3896816670894623, | |
| "rewards/margins": 0.12279312312602997, | |
| "rewards/rejected": -0.5124748945236206, | |
| "rewards/safe_rewards": -0.35095858573913574, | |
| "rewards/unsafe_rewards": -0.4284047484397888, | |
| "step": 410 | |
| }, | |
| { | |
| "epoch": 0.23, | |
| "learning_rate": 4.7622276086107677e-07, | |
| "logits/chosen": -2.4900124073028564, | |
| "logits/rejected": -2.3827693462371826, | |
| "logps/chosen": -251.73007202148438, | |
| "logps/rejected": -243.5518341064453, | |
| "loss": 12053.4797, | |
| "rewards/accuracies": 0.6812499761581421, | |
| "rewards/chosen": -0.4146080017089844, | |
| "rewards/margins": 0.16079458594322205, | |
| "rewards/rejected": -0.5754026174545288, | |
| "rewards/safe_rewards": -0.4179013669490814, | |
| "rewards/unsafe_rewards": -0.41131457686424255, | |
| "step": 420 | |
| }, | |
| { | |
| "epoch": 0.23, | |
| "learning_rate": 4.741835479216879e-07, | |
| "logits/chosen": -2.445971727371216, | |
| "logits/rejected": -2.286686420440674, | |
| "logps/chosen": -294.9362487792969, | |
| "logps/rejected": -262.58404541015625, | |
| "loss": 11785.9859, | |
| "rewards/accuracies": 0.668749988079071, | |
| "rewards/chosen": -0.48062044382095337, | |
| "rewards/margins": 0.16858412325382233, | |
| "rewards/rejected": -0.6492044925689697, | |
| "rewards/safe_rewards": -0.5107543468475342, | |
| "rewards/unsafe_rewards": -0.450486421585083, | |
| "step": 430 | |
| }, | |
| { | |
| "epoch": 0.24, | |
| "learning_rate": 4.720651909524036e-07, | |
| "logits/chosen": -2.3577795028686523, | |
| "logits/rejected": -2.2147059440612793, | |
| "logps/chosen": -236.0032196044922, | |
| "logps/rejected": -218.00820922851562, | |
| "loss": 11980.575, | |
| "rewards/accuracies": 0.706250011920929, | |
| "rewards/chosen": -0.3994632661342621, | |
| "rewards/margins": 0.1237812489271164, | |
| "rewards/rejected": -0.5232445001602173, | |
| "rewards/safe_rewards": -0.4401687681674957, | |
| "rewards/unsafe_rewards": -0.3587578237056732, | |
| "step": 440 | |
| }, | |
| { | |
| "epoch": 0.24, | |
| "learning_rate": 4.698684378016222e-07, | |
| "logits/chosen": -2.2142910957336426, | |
| "logits/rejected": -2.0808310508728027, | |
| "logps/chosen": -249.1070556640625, | |
| "logps/rejected": -246.5348663330078, | |
| "loss": 12188.9062, | |
| "rewards/accuracies": 0.7250000238418579, | |
| "rewards/chosen": -0.6178480982780457, | |
| "rewards/margins": 0.13151735067367554, | |
| "rewards/rejected": -0.7493655681610107, | |
| "rewards/safe_rewards": -0.6169500350952148, | |
| "rewards/unsafe_rewards": -0.6187463402748108, | |
| "step": 450 | |
| }, | |
| { | |
| "epoch": 0.25, | |
| "learning_rate": 4.675940639941256e-07, | |
| "logits/chosen": -2.2552475929260254, | |
| "logits/rejected": -2.087047815322876, | |
| "logps/chosen": -268.5553283691406, | |
| "logps/rejected": -256.51849365234375, | |
| "loss": 11744.7563, | |
| "rewards/accuracies": 0.706250011920929, | |
| "rewards/chosen": -0.6386964917182922, | |
| "rewards/margins": 0.15222086012363434, | |
| "rewards/rejected": -0.7909173965454102, | |
| "rewards/safe_rewards": -0.6151617765426636, | |
| "rewards/unsafe_rewards": -0.6622311472892761, | |
| "step": 460 | |
| }, | |
| { | |
| "epoch": 0.25, | |
| "learning_rate": 4.6524287245729286e-07, | |
| "logits/chosen": -2.2173266410827637, | |
| "logits/rejected": -2.0759875774383545, | |
| "logps/chosen": -247.02444458007812, | |
| "logps/rejected": -235.70431518554688, | |
| "loss": 11539.7117, | |
| "rewards/accuracies": 0.6937500238418579, | |
| "rewards/chosen": -0.5241425633430481, | |
| "rewards/margins": 0.1680615395307541, | |
| "rewards/rejected": -0.6922041177749634, | |
| "rewards/safe_rewards": -0.5475858449935913, | |
| "rewards/unsafe_rewards": -0.5006993412971497, | |
| "step": 470 | |
| }, | |
| { | |
| "epoch": 0.26, | |
| "learning_rate": 4.628156932376418e-07, | |
| "logits/chosen": -2.254505157470703, | |
| "logits/rejected": -2.057737112045288, | |
| "logps/chosen": -252.7125244140625, | |
| "logps/rejected": -227.06204223632812, | |
| "loss": 11626.5016, | |
| "rewards/accuracies": 0.6937500238418579, | |
| "rewards/chosen": -0.5887488126754761, | |
| "rewards/margins": 0.16838543117046356, | |
| "rewards/rejected": -0.7571342587471008, | |
| "rewards/safe_rewards": -0.5993281602859497, | |
| "rewards/unsafe_rewards": -0.5781695246696472, | |
| "step": 480 | |
| }, | |
| { | |
| "epoch": 0.26, | |
| "learning_rate": 4.603133832077953e-07, | |
| "logits/chosen": -2.2982699871063232, | |
| "logits/rejected": -2.1872057914733887, | |
| "logps/chosen": -293.5793762207031, | |
| "logps/rejected": -291.3560791015625, | |
| "loss": 11510.6367, | |
| "rewards/accuracies": 0.643750011920929, | |
| "rewards/chosen": -0.609082043170929, | |
| "rewards/margins": 0.1590423583984375, | |
| "rewards/rejected": -0.7681244015693665, | |
| "rewards/safe_rewards": -0.6237068176269531, | |
| "rewards/unsafe_rewards": -0.5944572687149048, | |
| "step": 490 | |
| }, | |
| { | |
| "epoch": 0.27, | |
| "learning_rate": 4.5773682576397776e-07, | |
| "logits/chosen": -2.2795727252960205, | |
| "logits/rejected": -2.150864839553833, | |
| "logps/chosen": -247.63088989257812, | |
| "logps/rejected": -234.78408813476562, | |
| "loss": 11965.7203, | |
| "rewards/accuracies": 0.65625, | |
| "rewards/chosen": -0.47748714685440063, | |
| "rewards/margins": 0.13509830832481384, | |
| "rewards/rejected": -0.6125854849815369, | |
| "rewards/safe_rewards": -0.48745498061180115, | |
| "rewards/unsafe_rewards": -0.4675193428993225, | |
| "step": 500 | |
| }, | |
| { | |
| "epoch": 0.27, | |
| "learning_rate": 4.5508693051414774e-07, | |
| "logits/chosen": -2.3894762992858887, | |
| "logits/rejected": -2.2791035175323486, | |
| "logps/chosen": -240.63394165039062, | |
| "logps/rejected": -236.6797332763672, | |
| "loss": 11414.6078, | |
| "rewards/accuracies": 0.6812499761581421, | |
| "rewards/chosen": -0.4428652822971344, | |
| "rewards/margins": 0.16295991837978363, | |
| "rewards/rejected": -0.6058252453804016, | |
| "rewards/safe_rewards": -0.4437985420227051, | |
| "rewards/unsafe_rewards": -0.44193196296691895, | |
| "step": 510 | |
| }, | |
| { | |
| "epoch": 0.28, | |
| "learning_rate": 4.52364632956877e-07, | |
| "logits/chosen": -2.376743793487549, | |
| "logits/rejected": -2.2216954231262207, | |
| "logps/chosen": -262.9234313964844, | |
| "logps/rejected": -226.41116333007812, | |
| "loss": 12766.6898, | |
| "rewards/accuracies": 0.637499988079071, | |
| "rewards/chosen": -0.4680143892765045, | |
| "rewards/margins": 0.11080136150121689, | |
| "rewards/rejected": -0.578815758228302, | |
| "rewards/safe_rewards": -0.4691530764102936, | |
| "rewards/unsafe_rewards": -0.4668757915496826, | |
| "step": 520 | |
| }, | |
| { | |
| "epoch": 0.29, | |
| "learning_rate": 4.4957089415108895e-07, | |
| "logits/chosen": -2.3553097248077393, | |
| "logits/rejected": -2.2464723587036133, | |
| "logps/chosen": -242.263671875, | |
| "logps/rejected": -255.6951141357422, | |
| "loss": 11431.9516, | |
| "rewards/accuracies": 0.731249988079071, | |
| "rewards/chosen": -0.4732840061187744, | |
| "rewards/margins": 0.19163894653320312, | |
| "rewards/rejected": -0.6649229526519775, | |
| "rewards/safe_rewards": -0.4660876393318176, | |
| "rewards/unsafe_rewards": -0.4804803729057312, | |
| "step": 530 | |
| }, | |
| { | |
| "epoch": 0.29, | |
| "learning_rate": 4.467067003767745e-07, | |
| "logits/chosen": -2.3562865257263184, | |
| "logits/rejected": -2.181304931640625, | |
| "logps/chosen": -247.70449829101562, | |
| "logps/rejected": -240.8135528564453, | |
| "loss": 12058.2938, | |
| "rewards/accuracies": 0.7250000238418579, | |
| "rewards/chosen": -0.5476623177528381, | |
| "rewards/margins": 0.19659535586833954, | |
| "rewards/rejected": -0.7442576885223389, | |
| "rewards/safe_rewards": -0.5530461072921753, | |
| "rewards/unsafe_rewards": -0.5422784686088562, | |
| "step": 540 | |
| }, | |
| { | |
| "epoch": 0.3, | |
| "learning_rate": 4.437730627868027e-07, | |
| "logits/chosen": -2.3266587257385254, | |
| "logits/rejected": -2.120724678039551, | |
| "logps/chosen": -231.5970916748047, | |
| "logps/rejected": -221.3756103515625, | |
| "loss": 11200.6187, | |
| "rewards/accuracies": 0.7250000238418579, | |
| "rewards/chosen": -0.5469081997871399, | |
| "rewards/margins": 0.20715579390525818, | |
| "rewards/rejected": -0.7540639638900757, | |
| "rewards/safe_rewards": -0.5482960939407349, | |
| "rewards/unsafe_rewards": -0.5455203652381897, | |
| "step": 550 | |
| }, | |
| { | |
| "epoch": 0.3, | |
| "learning_rate": 4.4077101704995163e-07, | |
| "logits/chosen": -2.3897128105163574, | |
| "logits/rejected": -2.2368927001953125, | |
| "logps/chosen": -238.13796997070312, | |
| "logps/rejected": -232.872314453125, | |
| "loss": 11633.2375, | |
| "rewards/accuracies": 0.65625, | |
| "rewards/chosen": -0.4596262574195862, | |
| "rewards/margins": 0.13524024188518524, | |
| "rewards/rejected": -0.5948664546012878, | |
| "rewards/safe_rewards": -0.4546341300010681, | |
| "rewards/unsafe_rewards": -0.4646182954311371, | |
| "step": 560 | |
| }, | |
| { | |
| "epoch": 0.31, | |
| "learning_rate": 4.3770162298528356e-07, | |
| "logits/chosen": -2.396641969680786, | |
| "logits/rejected": -2.2687771320343018, | |
| "logps/chosen": -244.2262420654297, | |
| "logps/rejected": -219.45248413085938, | |
| "loss": 12088.3172, | |
| "rewards/accuracies": 0.612500011920929, | |
| "rewards/chosen": -0.493887335062027, | |
| "rewards/margins": 0.15973922610282898, | |
| "rewards/rejected": -0.653626561164856, | |
| "rewards/safe_rewards": -0.48037633299827576, | |
| "rewards/unsafe_rewards": -0.5073983073234558, | |
| "step": 570 | |
| }, | |
| { | |
| "epoch": 0.31, | |
| "learning_rate": 4.3456596418799476e-07, | |
| "logits/chosen": -2.3043763637542725, | |
| "logits/rejected": -2.173457145690918, | |
| "logps/chosen": -257.493896484375, | |
| "logps/rejected": -241.50222778320312, | |
| "loss": 10751.843, | |
| "rewards/accuracies": 0.699999988079071, | |
| "rewards/chosen": -0.5369631052017212, | |
| "rewards/margins": 0.15752388536930084, | |
| "rewards/rejected": -0.6944869756698608, | |
| "rewards/safe_rewards": -0.5310293436050415, | |
| "rewards/unsafe_rewards": -0.5428968667984009, | |
| "step": 580 | |
| }, | |
| { | |
| "epoch": 0.32, | |
| "learning_rate": 4.313651476468715e-07, | |
| "logits/chosen": -2.270984172821045, | |
| "logits/rejected": -2.1114678382873535, | |
| "logps/chosen": -245.8052520751953, | |
| "logps/rejected": -238.4367218017578, | |
| "loss": 11156.9102, | |
| "rewards/accuracies": 0.699999988079071, | |
| "rewards/chosen": -0.4919458329677582, | |
| "rewards/margins": 0.16596952080726624, | |
| "rewards/rejected": -0.6579153537750244, | |
| "rewards/safe_rewards": -0.533513605594635, | |
| "rewards/unsafe_rewards": -0.45037803053855896, | |
| "step": 590 | |
| }, | |
| { | |
| "epoch": 0.32, | |
| "learning_rate": 4.2810030335348693e-07, | |
| "logits/chosen": -2.199920177459717, | |
| "logits/rejected": -2.001626491546631, | |
| "logps/chosen": -263.06634521484375, | |
| "logps/rejected": -236.2901153564453, | |
| "loss": 11584.7875, | |
| "rewards/accuracies": 0.625, | |
| "rewards/chosen": -0.5653510689735413, | |
| "rewards/margins": 0.14779801666736603, | |
| "rewards/rejected": -0.7131490707397461, | |
| "rewards/safe_rewards": -0.5651000142097473, | |
| "rewards/unsafe_rewards": -0.56560218334198, | |
| "step": 600 | |
| }, | |
| { | |
| "epoch": 0.32, | |
| "eval_logits/chosen": -1.993375301361084, | |
| "eval_logits/rejected": -1.7938207387924194, | |
| "eval_logps/chosen": -211.25747680664062, | |
| "eval_logps/rejected": -180.6571044921875, | |
| "eval_loss": 4406.71337890625, | |
| "eval_rewards/accuracies": 0.6338334679603577, | |
| "eval_rewards/chosen": -0.8082689642906189, | |
| "eval_rewards/margins": 0.07364560663700104, | |
| "eval_rewards/rejected": -0.8819145560264587, | |
| "eval_rewards/safe_rewards": -0.8027816414833069, | |
| "eval_rewards/unsafe_rewards": -0.8050090670585632, | |
| "eval_runtime": 995.4426, | |
| "eval_samples_per_second": 33.195, | |
| "eval_steps_per_second": 1.038, | |
| "step": 600 | |
| }, | |
| { | |
| "epoch": 0.33, | |
| "learning_rate": 4.2477258390327806e-07, | |
| "logits/chosen": -2.2489686012268066, | |
| "logits/rejected": -2.072410821914673, | |
| "logps/chosen": -241.9006805419922, | |
| "logps/rejected": -251.07376098632812, | |
| "loss": 11304.5898, | |
| "rewards/accuracies": 0.7437499761581421, | |
| "rewards/chosen": -0.5690335035324097, | |
| "rewards/margins": 0.20418229699134827, | |
| "rewards/rejected": -0.7732157707214355, | |
| "rewards/safe_rewards": -0.5881134271621704, | |
| "rewards/unsafe_rewards": -0.5499535799026489, | |
| "step": 610 | |
| }, | |
| { | |
| "epoch": 0.33, | |
| "learning_rate": 4.2138316408864197e-07, | |
| "logits/chosen": -2.334721803665161, | |
| "logits/rejected": -2.150754928588867, | |
| "logps/chosen": -246.8868865966797, | |
| "logps/rejected": -237.44033813476562, | |
| "loss": 10365.4461, | |
| "rewards/accuracies": 0.731249988079071, | |
| "rewards/chosen": -0.48651427030563354, | |
| "rewards/margins": 0.2631617486476898, | |
| "rewards/rejected": -0.7496760487556458, | |
| "rewards/safe_rewards": -0.4882384240627289, | |
| "rewards/unsafe_rewards": -0.4847901463508606, | |
| "step": 620 | |
| }, | |
| { | |
| "epoch": 0.34, | |
| "learning_rate": 4.179332404841962e-07, | |
| "logits/chosen": -2.2644031047821045, | |
| "logits/rejected": -2.0903661251068115, | |
| "logps/chosen": -267.8985290527344, | |
| "logps/rejected": -251.5312042236328, | |
| "loss": 10894.2219, | |
| "rewards/accuracies": 0.7250000238418579, | |
| "rewards/chosen": -0.5778486728668213, | |
| "rewards/margins": 0.22166796028614044, | |
| "rewards/rejected": -0.7995165586471558, | |
| "rewards/safe_rewards": -0.5826883316040039, | |
| "rewards/unsafe_rewards": -0.5730089545249939, | |
| "step": 630 | |
| }, | |
| { | |
| "epoch": 0.34, | |
| "learning_rate": 4.1442403102434954e-07, | |
| "logits/chosen": -2.282378911972046, | |
| "logits/rejected": -2.113847255706787, | |
| "logps/chosen": -270.49737548828125, | |
| "logps/rejected": -250.124755859375, | |
| "loss": 11471.2906, | |
| "rewards/accuracies": 0.699999988079071, | |
| "rewards/chosen": -0.5048245787620544, | |
| "rewards/margins": 0.20923128724098206, | |
| "rewards/rejected": -0.7140558958053589, | |
| "rewards/safe_rewards": -0.5091570615768433, | |
| "rewards/unsafe_rewards": -0.5004920959472656, | |
| "step": 640 | |
| }, | |
| { | |
| "epoch": 0.35, | |
| "learning_rate": 4.108567745733318e-07, | |
| "logits/chosen": -2.283024311065674, | |
| "logits/rejected": -2.1171913146972656, | |
| "logps/chosen": -216.49264526367188, | |
| "logps/rejected": -221.82421875, | |
| "loss": 11842.7375, | |
| "rewards/accuracies": 0.668749988079071, | |
| "rewards/chosen": -0.4298115372657776, | |
| "rewards/margins": 0.178109809756279, | |
| "rewards/rejected": -0.6079213619232178, | |
| "rewards/safe_rewards": -0.41675862669944763, | |
| "rewards/unsafe_rewards": -0.4428643584251404, | |
| "step": 650 | |
| }, | |
| { | |
| "epoch": 0.36, | |
| "learning_rate": 4.0723273048783426e-07, | |
| "logits/chosen": -2.3322463035583496, | |
| "logits/rejected": -2.1912810802459717, | |
| "logps/chosen": -271.88299560546875, | |
| "logps/rejected": -232.814208984375, | |
| "loss": 11859.6484, | |
| "rewards/accuracies": 0.643750011920929, | |
| "rewards/chosen": -0.47567135095596313, | |
| "rewards/margins": 0.16224288940429688, | |
| "rewards/rejected": -0.6379141807556152, | |
| "rewards/safe_rewards": -0.4415673315525055, | |
| "rewards/unsafe_rewards": -0.5097752809524536, | |
| "step": 660 | |
| }, | |
| { | |
| "epoch": 0.36, | |
| "learning_rate": 4.0355317817241697e-07, | |
| "logits/chosen": -2.3272366523742676, | |
| "logits/rejected": -2.1486003398895264, | |
| "logps/chosen": -275.4678039550781, | |
| "logps/rejected": -224.7365264892578, | |
| "loss": 11542.4758, | |
| "rewards/accuracies": 0.675000011920929, | |
| "rewards/chosen": -0.465405136346817, | |
| "rewards/margins": 0.16741091012954712, | |
| "rewards/rejected": -0.6328160166740417, | |
| "rewards/safe_rewards": -0.43667951226234436, | |
| "rewards/unsafe_rewards": -0.4941307008266449, | |
| "step": 670 | |
| }, | |
| { | |
| "epoch": 0.37, | |
| "learning_rate": 3.998194166278367e-07, | |
| "logits/chosen": -2.378793239593506, | |
| "logits/rejected": -2.2765145301818848, | |
| "logps/chosen": -242.05722045898438, | |
| "logps/rejected": -228.02822875976562, | |
| "loss": 12028.9203, | |
| "rewards/accuracies": 0.581250011920929, | |
| "rewards/chosen": -0.5077196359634399, | |
| "rewards/margins": 0.10326583683490753, | |
| "rewards/rejected": -0.6109854578971863, | |
| "rewards/safe_rewards": -0.519899845123291, | |
| "rewards/unsafe_rewards": -0.4955393671989441, | |
| "step": 680 | |
| }, | |
| { | |
| "epoch": 0.37, | |
| "learning_rate": 3.9603276399245855e-07, | |
| "logits/chosen": -2.3500876426696777, | |
| "logits/rejected": -2.2054431438446045, | |
| "logps/chosen": -266.26800537109375, | |
| "logps/rejected": -240.6906280517578, | |
| "loss": 11937.7102, | |
| "rewards/accuracies": 0.668749988079071, | |
| "rewards/chosen": -0.5368236303329468, | |
| "rewards/margins": 0.19003283977508545, | |
| "rewards/rejected": -0.7268564105033875, | |
| "rewards/safe_rewards": -0.5230244994163513, | |
| "rewards/unsafe_rewards": -0.5506226420402527, | |
| "step": 690 | |
| }, | |
| { | |
| "epoch": 0.38, | |
| "learning_rate": 3.9219455707691e-07, | |
| "logits/chosen": -2.3385093212127686, | |
| "logits/rejected": -2.194088935852051, | |
| "logps/chosen": -262.49957275390625, | |
| "logps/rejected": -248.57937622070312, | |
| "loss": 11237.9461, | |
| "rewards/accuracies": 0.643750011920929, | |
| "rewards/chosen": -0.5866261720657349, | |
| "rewards/margins": 0.18088310956954956, | |
| "rewards/rejected": -0.7675093412399292, | |
| "rewards/safe_rewards": -0.5994827747344971, | |
| "rewards/unsafe_rewards": -0.5737696886062622, | |
| "step": 700 | |
| }, | |
| { | |
| "epoch": 0.38, | |
| "learning_rate": 3.883061508921439e-07, | |
| "logits/chosen": -2.4109256267547607, | |
| "logits/rejected": -2.309866428375244, | |
| "logps/chosen": -253.4579620361328, | |
| "logps/rejected": -269.43060302734375, | |
| "loss": 11465.2805, | |
| "rewards/accuracies": 0.65625, | |
| "rewards/chosen": -0.5475128889083862, | |
| "rewards/margins": 0.13771310448646545, | |
| "rewards/rejected": -0.6852259635925293, | |
| "rewards/safe_rewards": -0.5719602704048157, | |
| "rewards/unsafe_rewards": -0.5230655670166016, | |
| "step": 710 | |
| }, | |
| { | |
| "epoch": 0.39, | |
| "learning_rate": 3.8436891817107555e-07, | |
| "logits/chosen": -2.3683512210845947, | |
| "logits/rejected": -2.29433012008667, | |
| "logps/chosen": -238.59597778320312, | |
| "logps/rejected": -241.7389678955078, | |
| "loss": 11748.5664, | |
| "rewards/accuracies": 0.706250011920929, | |
| "rewards/chosen": -0.481682687997818, | |
| "rewards/margins": 0.18611426651477814, | |
| "rewards/rejected": -0.6677969694137573, | |
| "rewards/safe_rewards": -0.4765700697898865, | |
| "rewards/unsafe_rewards": -0.4867952764034271, | |
| "step": 720 | |
| }, | |
| { | |
| "epoch": 0.39, | |
| "learning_rate": 3.8038424888396414e-07, | |
| "logits/chosen": -2.3680453300476074, | |
| "logits/rejected": -2.233586072921753, | |
| "logps/chosen": -255.9365692138672, | |
| "logps/rejected": -252.37393188476562, | |
| "loss": 10985.4922, | |
| "rewards/accuracies": 0.637499988079071, | |
| "rewards/chosen": -0.5011948347091675, | |
| "rewards/margins": 0.1678631603717804, | |
| "rewards/rejected": -0.6690580248832703, | |
| "rewards/safe_rewards": -0.5107179880142212, | |
| "rewards/unsafe_rewards": -0.49167174100875854, | |
| "step": 730 | |
| }, | |
| { | |
| "epoch": 0.4, | |
| "learning_rate": 3.763535497477079e-07, | |
| "logits/chosen": -2.376601457595825, | |
| "logits/rejected": -2.2197787761688232, | |
| "logps/chosen": -260.691650390625, | |
| "logps/rejected": -245.3418731689453, | |
| "loss": 11114.0781, | |
| "rewards/accuracies": 0.706250011920929, | |
| "rewards/chosen": -0.5256686210632324, | |
| "rewards/margins": 0.21716609597206116, | |
| "rewards/rejected": -0.742834746837616, | |
| "rewards/safe_rewards": -0.5268703103065491, | |
| "rewards/unsafe_rewards": -0.524466872215271, | |
| "step": 740 | |
| }, | |
| { | |
| "epoch": 0.4, | |
| "learning_rate": 3.7227824372922795e-07, | |
| "logits/chosen": -2.3355045318603516, | |
| "logits/rejected": -2.202324867248535, | |
| "logps/chosen": -244.21817016601562, | |
| "logps/rejected": -234.26766967773438, | |
| "loss": 11000.375, | |
| "rewards/accuracies": 0.643750011920929, | |
| "rewards/chosen": -0.5600683093070984, | |
| "rewards/margins": 0.15717127919197083, | |
| "rewards/rejected": -0.7172395586967468, | |
| "rewards/safe_rewards": -0.5515316724777222, | |
| "rewards/unsafe_rewards": -0.5686048269271851, | |
| "step": 750 | |
| }, | |
| { | |
| "epoch": 0.41, | |
| "learning_rate": 3.681597695431148e-07, | |
| "logits/chosen": -2.351304292678833, | |
| "logits/rejected": -2.2070152759552, | |
| "logps/chosen": -249.6879425048828, | |
| "logps/rejected": -252.49722290039062, | |
| "loss": 11395.6094, | |
| "rewards/accuracies": 0.706250011920929, | |
| "rewards/chosen": -0.5104349851608276, | |
| "rewards/margins": 0.22875532507896423, | |
| "rewards/rejected": -0.7391902804374695, | |
| "rewards/safe_rewards": -0.5327340364456177, | |
| "rewards/unsafe_rewards": -0.4881359040737152, | |
| "step": 760 | |
| }, | |
| { | |
| "epoch": 0.41, | |
| "learning_rate": 3.639995811437159e-07, | |
| "logits/chosen": -2.3204877376556396, | |
| "logits/rejected": -2.2027907371520996, | |
| "logps/chosen": -252.81082153320312, | |
| "logps/rejected": -258.1892395019531, | |
| "loss": 11175.6797, | |
| "rewards/accuracies": 0.6937500238418579, | |
| "rewards/chosen": -0.5397385358810425, | |
| "rewards/margins": 0.19889900088310242, | |
| "rewards/rejected": -0.7386375069618225, | |
| "rewards/safe_rewards": -0.5588012337684631, | |
| "rewards/unsafe_rewards": -0.520675778388977, | |
| "step": 770 | |
| }, | |
| { | |
| "epoch": 0.42, | |
| "learning_rate": 3.597991472118426e-07, | |
| "logits/chosen": -2.3824949264526367, | |
| "logits/rejected": -2.2542223930358887, | |
| "logps/chosen": -267.04437255859375, | |
| "logps/rejected": -254.86337280273438, | |
| "loss": 11511.8547, | |
| "rewards/accuracies": 0.6875, | |
| "rewards/chosen": -0.5231243371963501, | |
| "rewards/margins": 0.17550811171531677, | |
| "rewards/rejected": -0.6986324191093445, | |
| "rewards/safe_rewards": -0.5508791208267212, | |
| "rewards/unsafe_rewards": -0.49536967277526855, | |
| "step": 780 | |
| }, | |
| { | |
| "epoch": 0.43, | |
| "learning_rate": 3.5555995063627836e-07, | |
| "logits/chosen": -2.3838717937469482, | |
| "logits/rejected": -2.264705181121826, | |
| "logps/chosen": -288.45098876953125, | |
| "logps/rejected": -261.7911071777344, | |
| "loss": 11126.725, | |
| "rewards/accuracies": 0.6875, | |
| "rewards/chosen": -0.5807030200958252, | |
| "rewards/margins": 0.1779719889163971, | |
| "rewards/rejected": -0.7586749792098999, | |
| "rewards/safe_rewards": -0.5821424722671509, | |
| "rewards/unsafe_rewards": -0.5792635679244995, | |
| "step": 790 | |
| }, | |
| { | |
| "epoch": 0.43, | |
| "learning_rate": 3.512834879902715e-07, | |
| "logits/chosen": -2.325700521469116, | |
| "logits/rejected": -2.20318603515625, | |
| "logps/chosen": -262.4079284667969, | |
| "logps/rejected": -256.8218078613281, | |
| "loss": 11113.3211, | |
| "rewards/accuracies": 0.7250000238418579, | |
| "rewards/chosen": -0.6240984797477722, | |
| "rewards/margins": 0.1907762587070465, | |
| "rewards/rejected": -0.8148747682571411, | |
| "rewards/safe_rewards": -0.6181318759918213, | |
| "rewards/unsafe_rewards": -0.6300650835037231, | |
| "step": 800 | |
| }, | |
| { | |
| "epoch": 0.44, | |
| "learning_rate": 3.4697126900319616e-07, | |
| "logits/chosen": -2.2967851161956787, | |
| "logits/rejected": -2.1394267082214355, | |
| "logps/chosen": -259.38018798828125, | |
| "logps/rejected": -245.7625732421875, | |
| "loss": 11771.4461, | |
| "rewards/accuracies": 0.768750011920929, | |
| "rewards/chosen": -0.5871919393539429, | |
| "rewards/margins": 0.2232244461774826, | |
| "rewards/rejected": -0.8104164004325867, | |
| "rewards/safe_rewards": -0.5566633939743042, | |
| "rewards/unsafe_rewards": -0.6177204251289368, | |
| "step": 810 | |
| }, | |
| { | |
| "epoch": 0.44, | |
| "learning_rate": 3.426248160275693e-07, | |
| "logits/chosen": -2.33948016166687, | |
| "logits/rejected": -2.2020907402038574, | |
| "logps/chosen": -251.8537139892578, | |
| "logps/rejected": -247.29443359375, | |
| "loss": 11619.0344, | |
| "rewards/accuracies": 0.65625, | |
| "rewards/chosen": -0.5438165068626404, | |
| "rewards/margins": 0.17501583695411682, | |
| "rewards/rejected": -0.7188323736190796, | |
| "rewards/safe_rewards": -0.5689524412155151, | |
| "rewards/unsafe_rewards": -0.5186805725097656, | |
| "step": 820 | |
| }, | |
| { | |
| "epoch": 0.45, | |
| "learning_rate": 3.3824566350161094e-07, | |
| "logits/chosen": -2.3640289306640625, | |
| "logits/rejected": -2.195350408554077, | |
| "logps/chosen": -256.7790222167969, | |
| "logps/rejected": -235.6261444091797, | |
| "loss": 11238.1625, | |
| "rewards/accuracies": 0.731249988079071, | |
| "rewards/chosen": -0.4845482409000397, | |
| "rewards/margins": 0.1882091611623764, | |
| "rewards/rejected": -0.6727573871612549, | |
| "rewards/safe_rewards": -0.48699751496315, | |
| "rewards/unsafe_rewards": -0.48209887742996216, | |
| "step": 830 | |
| }, | |
| { | |
| "epoch": 0.45, | |
| "learning_rate": 3.338353574075381e-07, | |
| "logits/chosen": -2.319687843322754, | |
| "logits/rejected": -2.2427151203155518, | |
| "logps/chosen": -229.2741241455078, | |
| "logps/rejected": -230.6369171142578, | |
| "loss": 13030.1109, | |
| "rewards/accuracies": 0.675000011920929, | |
| "rewards/chosen": -0.5387877225875854, | |
| "rewards/margins": 0.1425672322511673, | |
| "rewards/rejected": -0.6813548803329468, | |
| "rewards/safe_rewards": -0.5679432153701782, | |
| "rewards/unsafe_rewards": -0.5096321702003479, | |
| "step": 840 | |
| }, | |
| { | |
| "epoch": 0.46, | |
| "learning_rate": 3.2939545472578314e-07, | |
| "logits/chosen": -2.3902430534362793, | |
| "logits/rejected": -2.1937272548675537, | |
| "logps/chosen": -279.47705078125, | |
| "logps/rejected": -250.4775848388672, | |
| "loss": 11581.1313, | |
| "rewards/accuracies": 0.6312500238418579, | |
| "rewards/chosen": -0.5328287482261658, | |
| "rewards/margins": 0.13031499087810516, | |
| "rewards/rejected": -0.6631438136100769, | |
| "rewards/safe_rewards": -0.5133041143417358, | |
| "rewards/unsafe_rewards": -0.5523533821105957, | |
| "step": 850 | |
| }, | |
| { | |
| "epoch": 0.46, | |
| "learning_rate": 3.2492752288532916e-07, | |
| "logits/chosen": -2.325867176055908, | |
| "logits/rejected": -2.1522111892700195, | |
| "logps/chosen": -253.1811981201172, | |
| "logps/rejected": -233.44174194335938, | |
| "loss": 11311.8109, | |
| "rewards/accuracies": 0.668749988079071, | |
| "rewards/chosen": -0.4949401319026947, | |
| "rewards/margins": 0.15677447617053986, | |
| "rewards/rejected": -0.6517146825790405, | |
| "rewards/safe_rewards": -0.47377505898475647, | |
| "rewards/unsafe_rewards": -0.5161052942276001, | |
| "step": 860 | |
| }, | |
| { | |
| "epoch": 0.47, | |
| "learning_rate": 3.204331392103574e-07, | |
| "logits/chosen": -2.337698459625244, | |
| "logits/rejected": -2.1352732181549072, | |
| "logps/chosen": -268.2446594238281, | |
| "logps/rejected": -234.0172882080078, | |
| "loss": 11545.3641, | |
| "rewards/accuracies": 0.71875, | |
| "rewards/chosen": -0.5469434261322021, | |
| "rewards/margins": 0.15391257405281067, | |
| "rewards/rejected": -0.7008560299873352, | |
| "rewards/safe_rewards": -0.5454758405685425, | |
| "rewards/unsafe_rewards": -0.5484111905097961, | |
| "step": 870 | |
| }, | |
| { | |
| "epoch": 0.47, | |
| "learning_rate": 3.159138903634006e-07, | |
| "logits/chosen": -2.1978166103363037, | |
| "logits/rejected": -2.0318470001220703, | |
| "logps/chosen": -280.46319580078125, | |
| "logps/rejected": -256.2702941894531, | |
| "loss": 11248.9852, | |
| "rewards/accuracies": 0.625, | |
| "rewards/chosen": -0.7179843187332153, | |
| "rewards/margins": 0.11917723715305328, | |
| "rewards/rejected": -0.837161660194397, | |
| "rewards/safe_rewards": -0.7153393030166626, | |
| "rewards/unsafe_rewards": -0.7206293940544128, | |
| "step": 880 | |
| }, | |
| { | |
| "epoch": 0.48, | |
| "learning_rate": 3.1137137178519977e-07, | |
| "logits/chosen": -2.1384453773498535, | |
| "logits/rejected": -2.003512382507324, | |
| "logps/chosen": -253.1010284423828, | |
| "logps/rejected": -262.6402893066406, | |
| "loss": 11450.3414, | |
| "rewards/accuracies": 0.6625000238418579, | |
| "rewards/chosen": -0.7104918956756592, | |
| "rewards/margins": 0.18712057173252106, | |
| "rewards/rejected": -0.897612452507019, | |
| "rewards/safe_rewards": -0.6939632296562195, | |
| "rewards/unsafe_rewards": -0.7270206212997437, | |
| "step": 890 | |
| }, | |
| { | |
| "epoch": 0.48, | |
| "learning_rate": 3.068071871314626e-07, | |
| "logits/chosen": -2.135709524154663, | |
| "logits/rejected": -2.028604507446289, | |
| "logps/chosen": -242.0713653564453, | |
| "logps/rejected": -236.3900909423828, | |
| "loss": 10862.3484, | |
| "rewards/accuracies": 0.643750011920929, | |
| "rewards/chosen": -0.6334865689277649, | |
| "rewards/margins": 0.13563254475593567, | |
| "rewards/rejected": -0.7691190838813782, | |
| "rewards/safe_rewards": -0.63458251953125, | |
| "rewards/unsafe_rewards": -0.6323906183242798, | |
| "step": 900 | |
| }, | |
| { | |
| "epoch": 0.48, | |
| "eval_logits/chosen": -1.9376695156097412, | |
| "eval_logits/rejected": -1.7468173503875732, | |
| "eval_logps/chosen": -218.71400451660156, | |
| "eval_logps/rejected": -187.7608642578125, | |
| "eval_loss": 4377.5634765625, | |
| "eval_rewards/accuracies": 0.6195546984672546, | |
| "eval_rewards/chosen": -0.8828346133232117, | |
| "eval_rewards/margins": 0.07011755555868149, | |
| "eval_rewards/rejected": -0.9529521465301514, | |
| "eval_rewards/safe_rewards": -0.8774688839912415, | |
| "eval_rewards/unsafe_rewards": -0.8777852058410645, | |
| "eval_runtime": 994.9448, | |
| "eval_samples_per_second": 33.212, | |
| "eval_steps_per_second": 1.038, | |
| "step": 900 | |
| }, | |
| { | |
| "epoch": 0.49, | |
| "learning_rate": 3.022229477067205e-07, | |
| "logits/chosen": -2.1737711429595947, | |
| "logits/rejected": -1.9998136758804321, | |
| "logps/chosen": -283.02313232421875, | |
| "logps/rejected": -255.57656860351562, | |
| "loss": 10507.2695, | |
| "rewards/accuracies": 0.699999988079071, | |
| "rewards/chosen": -0.6587027311325073, | |
| "rewards/margins": 0.193914994597435, | |
| "rewards/rejected": -0.8526177406311035, | |
| "rewards/safe_rewards": -0.6458637714385986, | |
| "rewards/unsafe_rewards": -0.6715416312217712, | |
| "step": 910 | |
| }, | |
| { | |
| "epoch": 0.49, | |
| "learning_rate": 2.976202718954869e-07, | |
| "logits/chosen": -2.183156728744507, | |
| "logits/rejected": -1.9853588342666626, | |
| "logps/chosen": -282.3530578613281, | |
| "logps/rejected": -271.27069091796875, | |
| "loss": 11861.3852, | |
| "rewards/accuracies": 0.65625, | |
| "rewards/chosen": -0.7082624435424805, | |
| "rewards/margins": 0.18091826140880585, | |
| "rewards/rejected": -0.8891807794570923, | |
| "rewards/safe_rewards": -0.7145110368728638, | |
| "rewards/unsafe_rewards": -0.7020138502120972, | |
| "step": 920 | |
| }, | |
| { | |
| "epoch": 0.5, | |
| "learning_rate": 2.930007845909146e-07, | |
| "logits/chosen": -2.177462339401245, | |
| "logits/rejected": -2.0282797813415527, | |
| "logps/chosen": -268.3103942871094, | |
| "logps/rejected": -266.30548095703125, | |
| "loss": 11683.3781, | |
| "rewards/accuracies": 0.637499988079071, | |
| "rewards/chosen": -0.7064036726951599, | |
| "rewards/margins": 0.13596151769161224, | |
| "rewards/rejected": -0.8423651456832886, | |
| "rewards/safe_rewards": -0.7049628496170044, | |
| "rewards/unsafe_rewards": -0.7078445553779602, | |
| "step": 930 | |
| }, | |
| { | |
| "epoch": 0.51, | |
| "learning_rate": 2.8836611662115634e-07, | |
| "logits/chosen": -2.1535885334014893, | |
| "logits/rejected": -1.9535331726074219, | |
| "logps/chosen": -287.9471130371094, | |
| "logps/rejected": -254.46157836914062, | |
| "loss": 12032.6703, | |
| "rewards/accuracies": 0.637499988079071, | |
| "rewards/chosen": -0.6779440641403198, | |
| "rewards/margins": 0.1851780265569687, | |
| "rewards/rejected": -0.8631221652030945, | |
| "rewards/safe_rewards": -0.6615537405014038, | |
| "rewards/unsafe_rewards": -0.6943344473838806, | |
| "step": 940 | |
| }, | |
| { | |
| "epoch": 0.51, | |
| "learning_rate": 2.8371790417362986e-07, | |
| "logits/chosen": -2.1404051780700684, | |
| "logits/rejected": -1.9921302795410156, | |
| "logps/chosen": -250.58895874023438, | |
| "logps/rejected": -252.75021362304688, | |
| "loss": 12575.8664, | |
| "rewards/accuracies": 0.625, | |
| "rewards/chosen": -0.6325485110282898, | |
| "rewards/margins": 0.13314202427864075, | |
| "rewards/rejected": -0.7656905651092529, | |
| "rewards/safe_rewards": -0.615986168384552, | |
| "rewards/unsafe_rewards": -0.6491108536720276, | |
| "step": 950 | |
| }, | |
| { | |
| "epoch": 0.52, | |
| "learning_rate": 2.7905778821739056e-07, | |
| "logits/chosen": -2.1147265434265137, | |
| "logits/rejected": -1.9771724939346313, | |
| "logps/chosen": -253.51351928710938, | |
| "logps/rejected": -233.7639923095703, | |
| "loss": 11670.4188, | |
| "rewards/accuracies": 0.675000011920929, | |
| "rewards/chosen": -0.5872529745101929, | |
| "rewards/margins": 0.1395425647497177, | |
| "rewards/rejected": -0.7267955541610718, | |
| "rewards/safe_rewards": -0.5655093193054199, | |
| "rewards/unsafe_rewards": -0.6089966297149658, | |
| "step": 960 | |
| }, | |
| { | |
| "epoch": 0.52, | |
| "learning_rate": 2.74387413923817e-07, | |
| "logits/chosen": -2.056212902069092, | |
| "logits/rejected": -1.9303762912750244, | |
| "logps/chosen": -294.0152893066406, | |
| "logps/rejected": -272.24615478515625, | |
| "loss": 11716.1664, | |
| "rewards/accuracies": 0.6812499761581421, | |
| "rewards/chosen": -0.6957898736000061, | |
| "rewards/margins": 0.17264129221439362, | |
| "rewards/rejected": -0.8684310913085938, | |
| "rewards/safe_rewards": -0.688141405582428, | |
| "rewards/unsafe_rewards": -0.7034383416175842, | |
| "step": 970 | |
| }, | |
| { | |
| "epoch": 0.53, | |
| "learning_rate": 2.69708430085812e-07, | |
| "logits/chosen": -2.0949666500091553, | |
| "logits/rejected": -1.8358476161956787, | |
| "logps/chosen": -303.7754821777344, | |
| "logps/rejected": -279.3113708496094, | |
| "loss": 12070.975, | |
| "rewards/accuracies": 0.731249988079071, | |
| "rewards/chosen": -0.8039963841438293, | |
| "rewards/margins": 0.2226048707962036, | |
| "rewards/rejected": -1.0266011953353882, | |
| "rewards/safe_rewards": -0.7967602014541626, | |
| "rewards/unsafe_rewards": -0.8112322688102722, | |
| "step": 980 | |
| }, | |
| { | |
| "epoch": 0.53, | |
| "learning_rate": 2.6502248853572504e-07, | |
| "logits/chosen": -2.018751859664917, | |
| "logits/rejected": -1.81708562374115, | |
| "logps/chosen": -268.05535888671875, | |
| "logps/rejected": -263.68487548828125, | |
| "loss": 12117.2781, | |
| "rewards/accuracies": 0.668749988079071, | |
| "rewards/chosen": -0.8013098835945129, | |
| "rewards/margins": 0.21267572045326233, | |
| "rewards/rejected": -1.013985514640808, | |
| "rewards/safe_rewards": -0.7930777668952942, | |
| "rewards/unsafe_rewards": -0.8095420002937317, | |
| "step": 990 | |
| }, | |
| { | |
| "epoch": 0.54, | |
| "learning_rate": 2.6033124356220325e-07, | |
| "logits/chosen": -2.040283441543579, | |
| "logits/rejected": -1.8361635208129883, | |
| "logps/chosen": -266.68841552734375, | |
| "logps/rejected": -244.3878631591797, | |
| "loss": 11242.6422, | |
| "rewards/accuracies": 0.737500011920929, | |
| "rewards/chosen": -0.639050304889679, | |
| "rewards/margins": 0.21101757884025574, | |
| "rewards/rejected": -0.8500677943229675, | |
| "rewards/safe_rewards": -0.6206936836242676, | |
| "rewards/unsafe_rewards": -0.6574069261550903, | |
| "step": 1000 | |
| }, | |
| { | |
| "epoch": 0.54, | |
| "learning_rate": 2.55636351326173e-07, | |
| "logits/chosen": -2.074812412261963, | |
| "logits/rejected": -1.9070053100585938, | |
| "logps/chosen": -267.3968505859375, | |
| "logps/rejected": -242.2158660888672, | |
| "loss": 10413.1977, | |
| "rewards/accuracies": 0.612500011920929, | |
| "rewards/chosen": -0.6132654547691345, | |
| "rewards/margins": 0.21324153244495392, | |
| "rewards/rejected": -0.8265069723129272, | |
| "rewards/safe_rewards": -0.638529360294342, | |
| "rewards/unsafe_rewards": -0.588001549243927, | |
| "step": 1010 | |
| }, | |
| { | |
| "epoch": 0.55, | |
| "learning_rate": 2.509394692761622e-07, | |
| "logits/chosen": -2.1109747886657715, | |
| "logits/rejected": -1.8711456060409546, | |
| "logps/chosen": -266.0138244628906, | |
| "logps/rejected": -246.04159545898438, | |
| "loss": 11730.4734, | |
| "rewards/accuracies": 0.699999988079071, | |
| "rewards/chosen": -0.6028767824172974, | |
| "rewards/margins": 0.20766966044902802, | |
| "rewards/rejected": -0.8105465173721313, | |
| "rewards/safe_rewards": -0.5901221632957458, | |
| "rewards/unsafe_rewards": -0.6156314611434937, | |
| "step": 1020 | |
| }, | |
| { | |
| "epoch": 0.55, | |
| "learning_rate": 2.462422555631674e-07, | |
| "logits/chosen": -2.073171854019165, | |
| "logits/rejected": -1.8314094543457031, | |
| "logps/chosen": -267.2673034667969, | |
| "logps/rejected": -234.9337921142578, | |
| "loss": 11384.8359, | |
| "rewards/accuracies": 0.6812499761581421, | |
| "rewards/chosen": -0.6479398012161255, | |
| "rewards/margins": 0.20152795314788818, | |
| "rewards/rejected": -0.8494676351547241, | |
| "rewards/safe_rewards": -0.6472254991531372, | |
| "rewards/unsafe_rewards": -0.6486541032791138, | |
| "step": 1030 | |
| }, | |
| { | |
| "epoch": 0.56, | |
| "learning_rate": 2.415463684552728e-07, | |
| "logits/chosen": -2.0377871990203857, | |
| "logits/rejected": -1.8571240901947021, | |
| "logps/chosen": -264.3353576660156, | |
| "logps/rejected": -250.5574493408203, | |
| "loss": 11869.4391, | |
| "rewards/accuracies": 0.643750011920929, | |
| "rewards/chosen": -0.6739367246627808, | |
| "rewards/margins": 0.1584298014640808, | |
| "rewards/rejected": -0.8323665857315063, | |
| "rewards/safe_rewards": -0.6686100363731384, | |
| "rewards/unsafe_rewards": -0.6792632341384888, | |
| "step": 1040 | |
| }, | |
| { | |
| "epoch": 0.56, | |
| "learning_rate": 2.3685346575222807e-07, | |
| "logits/chosen": -2.111082077026367, | |
| "logits/rejected": -1.8854366540908813, | |
| "logps/chosen": -280.7798767089844, | |
| "logps/rejected": -254.7810821533203, | |
| "loss": 11278.0977, | |
| "rewards/accuracies": 0.643750011920929, | |
| "rewards/chosen": -0.6307443976402283, | |
| "rewards/margins": 0.16730864346027374, | |
| "rewards/rejected": -0.7980531454086304, | |
| "rewards/safe_rewards": -0.6317691802978516, | |
| "rewards/unsafe_rewards": -0.6297196745872498, | |
| "step": 1050 | |
| }, | |
| { | |
| "epoch": 0.57, | |
| "learning_rate": 2.321652042001919e-07, | |
| "logits/chosen": -2.124929189682007, | |
| "logits/rejected": -1.8838306665420532, | |
| "logps/chosen": -272.2227783203125, | |
| "logps/rejected": -273.8869323730469, | |
| "loss": 11306.2313, | |
| "rewards/accuracies": 0.643750011920929, | |
| "rewards/chosen": -0.6101773381233215, | |
| "rewards/margins": 0.20483005046844482, | |
| "rewards/rejected": -0.8150073885917664, | |
| "rewards/safe_rewards": -0.6046205759048462, | |
| "rewards/unsafe_rewards": -0.6157340407371521, | |
| "step": 1060 | |
| }, | |
| { | |
| "epoch": 0.58, | |
| "learning_rate": 2.2748323890684662e-07, | |
| "logits/chosen": -2.1761631965637207, | |
| "logits/rejected": -1.8986377716064453, | |
| "logps/chosen": -270.1111145019531, | |
| "logps/rejected": -247.43576049804688, | |
| "loss": 11255.1672, | |
| "rewards/accuracies": 0.7124999761581421, | |
| "rewards/chosen": -0.5767061710357666, | |
| "rewards/margins": 0.2480432540178299, | |
| "rewards/rejected": -0.8247492909431458, | |
| "rewards/safe_rewards": -0.5869646072387695, | |
| "rewards/unsafe_rewards": -0.5664476156234741, | |
| "step": 1070 | |
| }, | |
| { | |
| "epoch": 0.58, | |
| "learning_rate": 2.2280922275709213e-07, | |
| "logits/chosen": -2.1585605144500732, | |
| "logits/rejected": -2.006892681121826, | |
| "logps/chosen": -272.51593017578125, | |
| "logps/rejected": -259.193603515625, | |
| "loss": 11610.1609, | |
| "rewards/accuracies": 0.643750011920929, | |
| "rewards/chosen": -0.6135168671607971, | |
| "rewards/margins": 0.16986827552318573, | |
| "rewards/rejected": -0.783385157585144, | |
| "rewards/safe_rewards": -0.5932785272598267, | |
| "rewards/unsafe_rewards": -0.6337552666664124, | |
| "step": 1080 | |
| }, | |
| { | |
| "epoch": 0.59, | |
| "learning_rate": 2.1814480582952375e-07, | |
| "logits/chosen": -2.176574230194092, | |
| "logits/rejected": -2.0007762908935547, | |
| "logps/chosen": -264.99627685546875, | |
| "logps/rejected": -260.7152099609375, | |
| "loss": 11244.9641, | |
| "rewards/accuracies": 0.6812499761581421, | |
| "rewards/chosen": -0.6294008493423462, | |
| "rewards/margins": 0.1786467581987381, | |
| "rewards/rejected": -0.8080476522445679, | |
| "rewards/safe_rewards": -0.610293984413147, | |
| "rewards/unsafe_rewards": -0.6485077738761902, | |
| "step": 1090 | |
| }, | |
| { | |
| "epoch": 0.59, | |
| "learning_rate": 2.1349163481390187e-07, | |
| "logits/chosen": -2.1403450965881348, | |
| "logits/rejected": -1.9742145538330078, | |
| "logps/chosen": -258.41851806640625, | |
| "logps/rejected": -253.95596313476562, | |
| "loss": 11432.9805, | |
| "rewards/accuracies": 0.699999988079071, | |
| "rewards/chosen": -0.579275369644165, | |
| "rewards/margins": 0.1978916972875595, | |
| "rewards/rejected": -0.7771671414375305, | |
| "rewards/safe_rewards": -0.5507979393005371, | |
| "rewards/unsafe_rewards": -0.6077528595924377, | |
| "step": 1100 | |
| }, | |
| { | |
| "epoch": 0.6, | |
| "learning_rate": 2.0885135242981647e-07, | |
| "logits/chosen": -2.110644817352295, | |
| "logits/rejected": -1.8921115398406982, | |
| "logps/chosen": -295.12310791015625, | |
| "logps/rejected": -242.7633056640625, | |
| "loss": 10866.6008, | |
| "rewards/accuracies": 0.7124999761581421, | |
| "rewards/chosen": -0.6292051672935486, | |
| "rewards/margins": 0.1930016279220581, | |
| "rewards/rejected": -0.8222068548202515, | |
| "rewards/safe_rewards": -0.6636418104171753, | |
| "rewards/unsafe_rewards": -0.5947686433792114, | |
| "step": 1110 | |
| }, | |
| { | |
| "epoch": 0.6, | |
| "learning_rate": 2.0422559684675494e-07, | |
| "logits/chosen": -2.07668137550354, | |
| "logits/rejected": -1.870774269104004, | |
| "logps/chosen": -279.4585876464844, | |
| "logps/rejected": -252.98422241210938, | |
| "loss": 11123.7453, | |
| "rewards/accuracies": 0.706250011920929, | |
| "rewards/chosen": -0.622269868850708, | |
| "rewards/margins": 0.18196968734264374, | |
| "rewards/rejected": -0.8042396306991577, | |
| "rewards/safe_rewards": -0.6149351596832275, | |
| "rewards/unsafe_rewards": -0.629604697227478, | |
| "step": 1120 | |
| }, | |
| { | |
| "epoch": 0.61, | |
| "learning_rate": 1.9961600110577457e-07, | |
| "logits/chosen": -2.0477161407470703, | |
| "logits/rejected": -1.8454376459121704, | |
| "logps/chosen": -281.2010803222656, | |
| "logps/rejected": -276.59503173828125, | |
| "loss": 11451.4273, | |
| "rewards/accuracies": 0.6812499761581421, | |
| "rewards/chosen": -0.7229688763618469, | |
| "rewards/margins": 0.15784484148025513, | |
| "rewards/rejected": -0.880813717842102, | |
| "rewards/safe_rewards": -0.6920644640922546, | |
| "rewards/unsafe_rewards": -0.7538732290267944, | |
| "step": 1130 | |
| }, | |
| { | |
| "epoch": 0.61, | |
| "learning_rate": 1.950241925429867e-07, | |
| "logits/chosen": -2.1420865058898926, | |
| "logits/rejected": -1.9303150177001953, | |
| "logps/chosen": -264.2613830566406, | |
| "logps/rejected": -247.3074493408203, | |
| "loss": 11185.8914, | |
| "rewards/accuracies": 0.6812499761581421, | |
| "rewards/chosen": -0.6415218114852905, | |
| "rewards/margins": 0.21913623809814453, | |
| "rewards/rejected": -0.8606580495834351, | |
| "rewards/safe_rewards": -0.6352616548538208, | |
| "rewards/unsafe_rewards": -0.647782027721405, | |
| "step": 1140 | |
| }, | |
| { | |
| "epoch": 0.62, | |
| "learning_rate": 1.9045179221505495e-07, | |
| "logits/chosen": -2.1051979064941406, | |
| "logits/rejected": -1.9602015018463135, | |
| "logps/chosen": -292.8426513671875, | |
| "logps/rejected": -272.5882568359375, | |
| "loss": 10919.0359, | |
| "rewards/accuracies": 0.7437499761581421, | |
| "rewards/chosen": -0.6359490156173706, | |
| "rewards/margins": 0.21196213364601135, | |
| "rewards/rejected": -0.8479111790657043, | |
| "rewards/safe_rewards": -0.605754554271698, | |
| "rewards/unsafe_rewards": -0.666143536567688, | |
| "step": 1150 | |
| }, | |
| { | |
| "epoch": 0.62, | |
| "learning_rate": 1.8590041432690893e-07, | |
| "logits/chosen": -2.0412395000457764, | |
| "logits/rejected": -1.9340324401855469, | |
| "logps/chosen": -256.9598693847656, | |
| "logps/rejected": -250.4047393798828, | |
| "loss": 11277.7938, | |
| "rewards/accuracies": 0.581250011920929, | |
| "rewards/chosen": -0.6930029988288879, | |
| "rewards/margins": 0.13424697518348694, | |
| "rewards/rejected": -0.8272498846054077, | |
| "rewards/safe_rewards": -0.6941550970077515, | |
| "rewards/unsafe_rewards": -0.6918508410453796, | |
| "step": 1160 | |
| }, | |
| { | |
| "epoch": 0.63, | |
| "learning_rate": 1.813716656618788e-07, | |
| "logits/chosen": -2.0102741718292236, | |
| "logits/rejected": -1.9039026498794556, | |
| "logps/chosen": -259.2917175292969, | |
| "logps/rejected": -252.873046875, | |
| "loss": 11988.2266, | |
| "rewards/accuracies": 0.7250000238418579, | |
| "rewards/chosen": -0.7098476886749268, | |
| "rewards/margins": 0.1709776222705841, | |
| "rewards/rejected": -0.8808252215385437, | |
| "rewards/safe_rewards": -0.7011247873306274, | |
| "rewards/unsafe_rewards": -0.7185705304145813, | |
| "step": 1170 | |
| }, | |
| { | |
| "epoch": 0.63, | |
| "learning_rate": 1.7686714501444788e-07, | |
| "logits/chosen": -2.084367275238037, | |
| "logits/rejected": -1.8202540874481201, | |
| "logps/chosen": -288.35626220703125, | |
| "logps/rejected": -254.33651733398438, | |
| "loss": 11612.3062, | |
| "rewards/accuracies": 0.6499999761581421, | |
| "rewards/chosen": -0.7599023580551147, | |
| "rewards/margins": 0.16640828549861908, | |
| "rewards/rejected": -0.926310658454895, | |
| "rewards/safe_rewards": -0.7765257954597473, | |
| "rewards/unsafe_rewards": -0.7432790994644165, | |
| "step": 1180 | |
| }, | |
| { | |
| "epoch": 0.64, | |
| "learning_rate": 1.7238844262582768e-07, | |
| "logits/chosen": -2.052704095840454, | |
| "logits/rejected": -1.9864280223846436, | |
| "logps/chosen": -275.9404602050781, | |
| "logps/rejected": -276.5259094238281, | |
| "loss": 10682.7672, | |
| "rewards/accuracies": 0.6937500238418579, | |
| "rewards/chosen": -0.6891142129898071, | |
| "rewards/margins": 0.15954537689685822, | |
| "rewards/rejected": -0.8486596345901489, | |
| "rewards/safe_rewards": -0.6521228551864624, | |
| "rewards/unsafe_rewards": -0.7261057496070862, | |
| "step": 1190 | |
| }, | |
| { | |
| "epoch": 0.65, | |
| "learning_rate": 1.679371396225504e-07, | |
| "logits/chosen": -2.0941474437713623, | |
| "logits/rejected": -1.8897063732147217, | |
| "logps/chosen": -271.2891540527344, | |
| "logps/rejected": -270.78924560546875, | |
| "loss": 11671.4219, | |
| "rewards/accuracies": 0.6875, | |
| "rewards/chosen": -0.7015270590782166, | |
| "rewards/margins": 0.19374321401119232, | |
| "rewards/rejected": -0.8952702283859253, | |
| "rewards/safe_rewards": -0.6884527802467346, | |
| "rewards/unsafe_rewards": -0.7146013379096985, | |
| "step": 1200 | |
| }, | |
| { | |
| "epoch": 0.65, | |
| "eval_logits/chosen": -1.8665262460708618, | |
| "eval_logits/rejected": -1.6740189790725708, | |
| "eval_logps/chosen": -228.5369415283203, | |
| "eval_logps/rejected": -197.5587921142578, | |
| "eval_loss": 4346.4052734375, | |
| "eval_rewards/accuracies": 0.6158034801483154, | |
| "eval_rewards/chosen": -0.9810636043548584, | |
| "eval_rewards/margins": 0.06986771523952484, | |
| "eval_rewards/rejected": -1.0509313344955444, | |
| "eval_rewards/safe_rewards": -0.9763943552970886, | |
| "eval_rewards/unsafe_rewards": -0.9767957329750061, | |
| "eval_runtime": 992.8374, | |
| "eval_samples_per_second": 33.282, | |
| "eval_steps_per_second": 1.04, | |
| "step": 1200 | |
| }, | |
| { | |
| "epoch": 0.65, | |
| "learning_rate": 1.6351480745828096e-07, | |
| "logits/chosen": -2.1039538383483887, | |
| "logits/rejected": -1.9394419193267822, | |
| "logps/chosen": -265.77984619140625, | |
| "logps/rejected": -253.25137329101562, | |
| "loss": 10276.6594, | |
| "rewards/accuracies": 0.6499999761581421, | |
| "rewards/chosen": -0.6483036279678345, | |
| "rewards/margins": 0.1812000572681427, | |
| "rewards/rejected": -0.8295037150382996, | |
| "rewards/safe_rewards": -0.6423134803771973, | |
| "rewards/unsafe_rewards": -0.6542937755584717, | |
| "step": 1210 | |
| }, | |
| { | |
| "epoch": 0.66, | |
| "learning_rate": 1.5912300735904248e-07, | |
| "logits/chosen": -2.1374449729919434, | |
| "logits/rejected": -1.983070969581604, | |
| "logps/chosen": -290.1751403808594, | |
| "logps/rejected": -259.19097900390625, | |
| "loss": 11657.7609, | |
| "rewards/accuracies": 0.675000011920929, | |
| "rewards/chosen": -0.6737682223320007, | |
| "rewards/margins": 0.14703810214996338, | |
| "rewards/rejected": -0.8208063244819641, | |
| "rewards/safe_rewards": -0.6786805987358093, | |
| "rewards/unsafe_rewards": -0.6688558459281921, | |
| "step": 1220 | |
| }, | |
| { | |
| "epoch": 0.66, | |
| "learning_rate": 1.5476328977205395e-07, | |
| "logits/chosen": -2.1445086002349854, | |
| "logits/rejected": -1.9767332077026367, | |
| "logps/chosen": -271.8307189941406, | |
| "logps/rejected": -251.6882781982422, | |
| "loss": 11008.7156, | |
| "rewards/accuracies": 0.6937500238418579, | |
| "rewards/chosen": -0.6544319987297058, | |
| "rewards/margins": 0.21836614608764648, | |
| "rewards/rejected": -0.8727981448173523, | |
| "rewards/safe_rewards": -0.6966055631637573, | |
| "rewards/unsafe_rewards": -0.6122584342956543, | |
| "step": 1230 | |
| }, | |
| { | |
| "epoch": 0.67, | |
| "learning_rate": 1.5043719381837112e-07, | |
| "logits/chosen": -2.08951473236084, | |
| "logits/rejected": -1.9547208547592163, | |
| "logps/chosen": -287.0835266113281, | |
| "logps/rejected": -271.43670654296875, | |
| "loss": 11362.9422, | |
| "rewards/accuracies": 0.6812499761581421, | |
| "rewards/chosen": -0.6643306612968445, | |
| "rewards/margins": 0.17835856974124908, | |
| "rewards/rejected": -0.8426891565322876, | |
| "rewards/safe_rewards": -0.6574736833572388, | |
| "rewards/unsafe_rewards": -0.6711875200271606, | |
| "step": 1240 | |
| }, | |
| { | |
| "epoch": 0.67, | |
| "learning_rate": 1.461462467495284e-07, | |
| "logits/chosen": -2.0471625328063965, | |
| "logits/rejected": -1.8923746347427368, | |
| "logps/chosen": -252.0188446044922, | |
| "logps/rejected": -260.54522705078125, | |
| "loss": 10503.6422, | |
| "rewards/accuracies": 0.6875, | |
| "rewards/chosen": -0.7411776185035706, | |
| "rewards/margins": 0.21499836444854736, | |
| "rewards/rejected": -0.9561759829521179, | |
| "rewards/safe_rewards": -0.7554097175598145, | |
| "rewards/unsafe_rewards": -0.7269455194473267, | |
| "step": 1250 | |
| }, | |
| { | |
| "epoch": 0.68, | |
| "learning_rate": 1.4189196340836865e-07, | |
| "logits/chosen": -2.1763851642608643, | |
| "logits/rejected": -1.9493157863616943, | |
| "logps/chosen": -269.4758605957031, | |
| "logps/rejected": -258.123291015625, | |
| "loss": 10853.5781, | |
| "rewards/accuracies": 0.6812499761581421, | |
| "rewards/chosen": -0.7069443464279175, | |
| "rewards/margins": 0.17843110859394073, | |
| "rewards/rejected": -0.8853754997253418, | |
| "rewards/safe_rewards": -0.7389132380485535, | |
| "rewards/unsafe_rewards": -0.6749754548072815, | |
| "step": 1260 | |
| }, | |
| { | |
| "epoch": 0.68, | |
| "learning_rate": 1.3767584569425561e-07, | |
| "logits/chosen": -2.19474458694458, | |
| "logits/rejected": -1.9729722738265991, | |
| "logps/chosen": -277.60992431640625, | |
| "logps/rejected": -258.0458984375, | |
| "loss": 11364.5312, | |
| "rewards/accuracies": 0.675000011920929, | |
| "rewards/chosen": -0.6931368708610535, | |
| "rewards/margins": 0.18750238418579102, | |
| "rewards/rejected": -0.8806392550468445, | |
| "rewards/safe_rewards": -0.7000595927238464, | |
| "rewards/unsafe_rewards": -0.6862143278121948, | |
| "step": 1270 | |
| }, | |
| { | |
| "epoch": 0.69, | |
| "learning_rate": 1.334993820328541e-07, | |
| "logits/chosen": -2.0907111167907715, | |
| "logits/rejected": -1.9239038228988647, | |
| "logps/chosen": -246.83642578125, | |
| "logps/rejected": -245.0505828857422, | |
| "loss": 10652.2328, | |
| "rewards/accuracies": 0.7437499761581421, | |
| "rewards/chosen": -0.7036882638931274, | |
| "rewards/margins": 0.24492530524730682, | |
| "rewards/rejected": -0.9486135244369507, | |
| "rewards/safe_rewards": -0.6736655235290527, | |
| "rewards/unsafe_rewards": -0.7337108850479126, | |
| "step": 1280 | |
| }, | |
| { | |
| "epoch": 0.69, | |
| "learning_rate": 1.2936404685066852e-07, | |
| "logits/chosen": -2.0505154132843018, | |
| "logits/rejected": -1.9092705249786377, | |
| "logps/chosen": -286.58990478515625, | |
| "logps/rejected": -280.3271789550781, | |
| "loss": 11526.9, | |
| "rewards/accuracies": 0.6812499761581421, | |
| "rewards/chosen": -0.68525230884552, | |
| "rewards/margins": 0.18679992854595184, | |
| "rewards/rejected": -0.8720521926879883, | |
| "rewards/safe_rewards": -0.7125786542892456, | |
| "rewards/unsafe_rewards": -0.6579257249832153, | |
| "step": 1290 | |
| }, | |
| { | |
| "epoch": 0.7, | |
| "learning_rate": 1.252713000545221e-07, | |
| "logits/chosen": -2.2164149284362793, | |
| "logits/rejected": -2.019528388977051, | |
| "logps/chosen": -282.0648498535156, | |
| "logps/rejected": -258.7922058105469, | |
| "loss": 10187.7117, | |
| "rewards/accuracies": 0.6937500238418579, | |
| "rewards/chosen": -0.6330300569534302, | |
| "rewards/margins": 0.23740728199481964, | |
| "rewards/rejected": -0.8704373240470886, | |
| "rewards/safe_rewards": -0.6281386613845825, | |
| "rewards/unsafe_rewards": -0.6379214525222778, | |
| "step": 1300 | |
| }, | |
| { | |
| "epoch": 0.7, | |
| "learning_rate": 1.2122258651616304e-07, | |
| "logits/chosen": -2.183960437774658, | |
| "logits/rejected": -1.976144790649414, | |
| "logps/chosen": -275.6617126464844, | |
| "logps/rejected": -244.86434936523438, | |
| "loss": 10813.9688, | |
| "rewards/accuracies": 0.75, | |
| "rewards/chosen": -0.6538507342338562, | |
| "rewards/margins": 0.17538292706012726, | |
| "rewards/rejected": -0.8292337656021118, | |
| "rewards/safe_rewards": -0.6337584853172302, | |
| "rewards/unsafe_rewards": -0.673943042755127, | |
| "step": 1310 | |
| }, | |
| { | |
| "epoch": 0.71, | |
| "learning_rate": 1.1721933556217792e-07, | |
| "logits/chosen": -2.167842388153076, | |
| "logits/rejected": -2.0031533241271973, | |
| "logps/chosen": -265.863525390625, | |
| "logps/rejected": -261.1900329589844, | |
| "loss": 11171.5523, | |
| "rewards/accuracies": 0.6812499761581421, | |
| "rewards/chosen": -0.6531133651733398, | |
| "rewards/margins": 0.19861795008182526, | |
| "rewards/rejected": -0.8517313003540039, | |
| "rewards/safe_rewards": -0.6670490503311157, | |
| "rewards/unsafe_rewards": -0.639177680015564, | |
| "step": 1320 | |
| }, | |
| { | |
| "epoch": 0.72, | |
| "learning_rate": 1.1326296046939333e-07, | |
| "logits/chosen": -2.134734869003296, | |
| "logits/rejected": -1.9698076248168945, | |
| "logps/chosen": -257.0758972167969, | |
| "logps/rejected": -242.34640502929688, | |
| "loss": 10608.8219, | |
| "rewards/accuracies": 0.6625000238418579, | |
| "rewards/chosen": -0.6214333772659302, | |
| "rewards/margins": 0.21942774951457977, | |
| "rewards/rejected": -0.8408611416816711, | |
| "rewards/safe_rewards": -0.6008444428443909, | |
| "rewards/unsafe_rewards": -0.6420222520828247, | |
| "step": 1330 | |
| }, | |
| { | |
| "epoch": 0.72, | |
| "learning_rate": 1.0935485796594351e-07, | |
| "logits/chosen": -2.174132823944092, | |
| "logits/rejected": -1.9792499542236328, | |
| "logps/chosen": -291.65814208984375, | |
| "logps/rejected": -260.0698547363281, | |
| "loss": 11895.8789, | |
| "rewards/accuracies": 0.71875, | |
| "rewards/chosen": -0.6717311143875122, | |
| "rewards/margins": 0.22725185751914978, | |
| "rewards/rejected": -0.8989830017089844, | |
| "rewards/safe_rewards": -0.6783491373062134, | |
| "rewards/unsafe_rewards": -0.665113091468811, | |
| "step": 1340 | |
| }, | |
| { | |
| "epoch": 0.73, | |
| "learning_rate": 1.0549640773818028e-07, | |
| "logits/chosen": -2.139070749282837, | |
| "logits/rejected": -2.040194034576416, | |
| "logps/chosen": -272.8217468261719, | |
| "logps/rejected": -252.80606079101562, | |
| "loss": 11776.8328, | |
| "rewards/accuracies": 0.637499988079071, | |
| "rewards/chosen": -0.724553108215332, | |
| "rewards/margins": 0.14907190203666687, | |
| "rewards/rejected": -0.8736250996589661, | |
| "rewards/safe_rewards": -0.764702320098877, | |
| "rewards/unsafe_rewards": -0.6844038963317871, | |
| "step": 1350 | |
| }, | |
| { | |
| "epoch": 0.73, | |
| "learning_rate": 1.0168897194359921e-07, | |
| "logits/chosen": -2.1789050102233887, | |
| "logits/rejected": -1.9954169988632202, | |
| "logps/chosen": -300.97711181640625, | |
| "logps/rejected": -274.7542724609375, | |
| "loss": 10877.8953, | |
| "rewards/accuracies": 0.643750011920929, | |
| "rewards/chosen": -0.7322704195976257, | |
| "rewards/margins": 0.15806125104427338, | |
| "rewards/rejected": -0.8903317451477051, | |
| "rewards/safe_rewards": -0.6902467608451843, | |
| "rewards/unsafe_rewards": -0.7742940783500671, | |
| "step": 1360 | |
| }, | |
| { | |
| "epoch": 0.74, | |
| "learning_rate": 9.793389472995392e-08, | |
| "logits/chosen": -2.155348539352417, | |
| "logits/rejected": -1.9421207904815674, | |
| "logps/chosen": -279.4602966308594, | |
| "logps/rejected": -240.51791381835938, | |
| "loss": 10089.3242, | |
| "rewards/accuracies": 0.7437499761581421, | |
| "rewards/chosen": -0.639030933380127, | |
| "rewards/margins": 0.252288281917572, | |
| "rewards/rejected": -0.8913192749023438, | |
| "rewards/safe_rewards": -0.6204690933227539, | |
| "rewards/unsafe_rewards": -0.6575928330421448, | |
| "step": 1370 | |
| }, | |
| { | |
| "epoch": 0.74, | |
| "learning_rate": 9.423250176072874e-08, | |
| "logits/chosen": -2.1420297622680664, | |
| "logits/rejected": -1.946412444114685, | |
| "logps/chosen": -260.1219177246094, | |
| "logps/rejected": -234.1204833984375, | |
| "loss": 12708.7586, | |
| "rewards/accuracies": 0.6187499761581421, | |
| "rewards/chosen": -0.71236252784729, | |
| "rewards/margins": 0.1536431610584259, | |
| "rewards/rejected": -0.8660055994987488, | |
| "rewards/safe_rewards": -0.6938896179199219, | |
| "rewards/unsafe_rewards": -0.7308354377746582, | |
| "step": 1380 | |
| }, | |
| { | |
| "epoch": 0.75, | |
| "learning_rate": 9.058609974713654e-08, | |
| "logits/chosen": -2.1647183895111084, | |
| "logits/rejected": -1.9520107507705688, | |
| "logps/chosen": -270.0279235839844, | |
| "logps/rejected": -260.9911193847656, | |
| "loss": 10654.6578, | |
| "rewards/accuracies": 0.706250011920929, | |
| "rewards/chosen": -0.6530768275260925, | |
| "rewards/margins": 0.2193538248538971, | |
| "rewards/rejected": -0.872430682182312, | |
| "rewards/safe_rewards": -0.6400060653686523, | |
| "rewards/unsafe_rewards": -0.6661475896835327, | |
| "step": 1390 | |
| }, | |
| { | |
| "epoch": 0.75, | |
| "learning_rate": 8.699597598680753e-08, | |
| "logits/chosen": -2.1252872943878174, | |
| "logits/rejected": -1.9515492916107178, | |
| "logps/chosen": -257.53680419921875, | |
| "logps/rejected": -247.7872314453125, | |
| "loss": 10438.4352, | |
| "rewards/accuracies": 0.6625000238418579, | |
| "rewards/chosen": -0.6546493768692017, | |
| "rewards/margins": 0.19867801666259766, | |
| "rewards/rejected": -0.8533273935317993, | |
| "rewards/safe_rewards": -0.6472882032394409, | |
| "rewards/unsafe_rewards": -0.6620105504989624, | |
| "step": 1400 | |
| }, | |
| { | |
| "epoch": 0.76, | |
| "learning_rate": 8.346339790933166e-08, | |
| "logits/chosen": -2.163567543029785, | |
| "logits/rejected": -1.9771531820297241, | |
| "logps/chosen": -261.906982421875, | |
| "logps/rejected": -245.43344116210938, | |
| "loss": 11502.7297, | |
| "rewards/accuracies": 0.699999988079071, | |
| "rewards/chosen": -0.6902798414230347, | |
| "rewards/margins": 0.19094708561897278, | |
| "rewards/rejected": -0.8812268376350403, | |
| "rewards/safe_rewards": -0.6901260018348694, | |
| "rewards/unsafe_rewards": -0.6904336214065552, | |
| "step": 1410 | |
| }, | |
| { | |
| "epoch": 0.76, | |
| "learning_rate": 7.998961262881506e-08, | |
| "logits/chosen": -2.136033535003662, | |
| "logits/rejected": -1.915771245956421, | |
| "logps/chosen": -282.3092346191406, | |
| "logps/rejected": -245.1779022216797, | |
| "loss": 10907.2641, | |
| "rewards/accuracies": 0.737500011920929, | |
| "rewards/chosen": -0.6517400741577148, | |
| "rewards/margins": 0.2158946692943573, | |
| "rewards/rejected": -0.8676347732543945, | |
| "rewards/safe_rewards": -0.6872564554214478, | |
| "rewards/unsafe_rewards": -0.6162236928939819, | |
| "step": 1420 | |
| }, | |
| { | |
| "epoch": 0.77, | |
| "learning_rate": 7.657584650360846e-08, | |
| "logits/chosen": -2.068126678466797, | |
| "logits/rejected": -1.9587552547454834, | |
| "logps/chosen": -251.7869873046875, | |
| "logps/rejected": -236.9489288330078, | |
| "loss": 11409.3359, | |
| "rewards/accuracies": 0.6499999761581421, | |
| "rewards/chosen": -0.6544846296310425, | |
| "rewards/margins": 0.2086174488067627, | |
| "rewards/rejected": -0.8631020784378052, | |
| "rewards/safe_rewards": -0.6824710965156555, | |
| "rewards/unsafe_rewards": -0.6264981627464294, | |
| "step": 1430 | |
| }, | |
| { | |
| "epoch": 0.77, | |
| "learning_rate": 7.322330470336313e-08, | |
| "logits/chosen": -2.122067928314209, | |
| "logits/rejected": -1.9209505319595337, | |
| "logps/chosen": -269.77520751953125, | |
| "logps/rejected": -268.40972900390625, | |
| "loss": 10821.843, | |
| "rewards/accuracies": 0.7250000238418579, | |
| "rewards/chosen": -0.6080222129821777, | |
| "rewards/margins": 0.25041937828063965, | |
| "rewards/rejected": -0.8584416508674622, | |
| "rewards/safe_rewards": -0.5671194195747375, | |
| "rewards/unsafe_rewards": -0.6489250659942627, | |
| "step": 1440 | |
| }, | |
| { | |
| "epoch": 0.78, | |
| "learning_rate": 6.993317078356709e-08, | |
| "logits/chosen": -2.1524910926818848, | |
| "logits/rejected": -2.056199789047241, | |
| "logps/chosen": -279.29583740234375, | |
| "logps/rejected": -255.84521484375, | |
| "loss": 11391.4539, | |
| "rewards/accuracies": 0.6499999761581421, | |
| "rewards/chosen": -0.691224217414856, | |
| "rewards/margins": 0.13461162149906158, | |
| "rewards/rejected": -0.8258358240127563, | |
| "rewards/safe_rewards": -0.7272582054138184, | |
| "rewards/unsafe_rewards": -0.6551901698112488, | |
| "step": 1450 | |
| }, | |
| { | |
| "epoch": 0.79, | |
| "learning_rate": 6.67066062677118e-08, | |
| "logits/chosen": -2.191707134246826, | |
| "logits/rejected": -2.0195794105529785, | |
| "logps/chosen": -263.27642822265625, | |
| "logps/rejected": -235.50973510742188, | |
| "loss": 11926.3453, | |
| "rewards/accuracies": 0.6625000238418579, | |
| "rewards/chosen": -0.6236655116081238, | |
| "rewards/margins": 0.16336306929588318, | |
| "rewards/rejected": -0.7870286107063293, | |
| "rewards/safe_rewards": -0.6308915615081787, | |
| "rewards/unsafe_rewards": -0.6164394617080688, | |
| "step": 1460 | |
| }, | |
| { | |
| "epoch": 0.79, | |
| "learning_rate": 6.354475023723685e-08, | |
| "logits/chosen": -2.1155574321746826, | |
| "logits/rejected": -1.9720067977905273, | |
| "logps/chosen": -301.7005920410156, | |
| "logps/rejected": -267.5169372558594, | |
| "loss": 11094.9625, | |
| "rewards/accuracies": 0.706250011920929, | |
| "rewards/chosen": -0.6763675808906555, | |
| "rewards/margins": 0.23334193229675293, | |
| "rewards/rejected": -0.9097094535827637, | |
| "rewards/safe_rewards": -0.6679859757423401, | |
| "rewards/unsafe_rewards": -0.6847492456436157, | |
| "step": 1470 | |
| }, | |
| { | |
| "epoch": 0.8, | |
| "learning_rate": 6.044871892939746e-08, | |
| "logits/chosen": -2.184211254119873, | |
| "logits/rejected": -1.9999923706054688, | |
| "logps/chosen": -281.41900634765625, | |
| "logps/rejected": -268.82464599609375, | |
| "loss": 10924.2703, | |
| "rewards/accuracies": 0.7437499761581421, | |
| "rewards/chosen": -0.6950246691703796, | |
| "rewards/margins": 0.19827482104301453, | |
| "rewards/rejected": -0.8932995796203613, | |
| "rewards/safe_rewards": -0.6960557699203491, | |
| "rewards/unsafe_rewards": -0.6939936280250549, | |
| "step": 1480 | |
| }, | |
| { | |
| "epoch": 0.8, | |
| "learning_rate": 5.741960534319676e-08, | |
| "logits/chosen": -2.134298801422119, | |
| "logits/rejected": -2.02238130569458, | |
| "logps/chosen": -241.8282012939453, | |
| "logps/rejected": -240.76290893554688, | |
| "loss": 11064.8711, | |
| "rewards/accuracies": 0.643750011920929, | |
| "rewards/chosen": -0.687341570854187, | |
| "rewards/margins": 0.16167931258678436, | |
| "rewards/rejected": -0.8490209579467773, | |
| "rewards/safe_rewards": -0.7233562469482422, | |
| "rewards/unsafe_rewards": -0.6513269543647766, | |
| "step": 1490 | |
| }, | |
| { | |
| "epoch": 0.81, | |
| "learning_rate": 5.44584788535217e-08, | |
| "logits/chosen": -2.1548972129821777, | |
| "logits/rejected": -2.0020337104797363, | |
| "logps/chosen": -288.1227111816406, | |
| "logps/rejected": -265.52337646484375, | |
| "loss": 10202.4125, | |
| "rewards/accuracies": 0.6937500238418579, | |
| "rewards/chosen": -0.7084766626358032, | |
| "rewards/margins": 0.21978096663951874, | |
| "rewards/rejected": -0.9282576441764832, | |
| "rewards/safe_rewards": -0.6754325032234192, | |
| "rewards/unsafe_rewards": -0.741520881652832, | |
| "step": 1500 | |
| }, | |
| { | |
| "epoch": 0.81, | |
| "eval_logits/chosen": -1.9419655799865723, | |
| "eval_logits/rejected": -1.764477252960205, | |
| "eval_logps/chosen": -226.97747802734375, | |
| "eval_logps/rejected": -195.17942810058594, | |
| "eval_loss": 4320.98779296875, | |
| "eval_rewards/accuracies": 0.6022507548332214, | |
| "eval_rewards/chosen": -0.9654689431190491, | |
| "eval_rewards/margins": 0.061668772250413895, | |
| "eval_rewards/rejected": -1.0271376371383667, | |
| "eval_rewards/safe_rewards": -0.9610660672187805, | |
| "eval_rewards/unsafe_rewards": -0.9618369936943054, | |
| "eval_runtime": 993.2958, | |
| "eval_samples_per_second": 33.267, | |
| "eval_steps_per_second": 1.04, | |
| "step": 1500 | |
| }, | |
| { | |
| "epoch": 0.81, | |
| "learning_rate": 5.156638483361933e-08, | |
| "logits/chosen": -2.2049641609191895, | |
| "logits/rejected": -2.0106749534606934, | |
| "logps/chosen": -285.2719421386719, | |
| "logps/rejected": -273.9066467285156, | |
| "loss": 11107.9672, | |
| "rewards/accuracies": 0.7124999761581421, | |
| "rewards/chosen": -0.6739949584007263, | |
| "rewards/margins": 0.23203198611736298, | |
| "rewards/rejected": -0.9060269594192505, | |
| "rewards/safe_rewards": -0.6972517967224121, | |
| "rewards/unsafe_rewards": -0.6507382988929749, | |
| "step": 1510 | |
| }, | |
| { | |
| "epoch": 0.82, | |
| "learning_rate": 4.8744344286046236e-08, | |
| "logits/chosen": -2.148716926574707, | |
| "logits/rejected": -2.0059280395507812, | |
| "logps/chosen": -285.7190856933594, | |
| "logps/rejected": -266.8186950683594, | |
| "loss": 11427.2711, | |
| "rewards/accuracies": 0.668749988079071, | |
| "rewards/chosen": -0.6893398761749268, | |
| "rewards/margins": 0.15746144950389862, | |
| "rewards/rejected": -0.8468014001846313, | |
| "rewards/safe_rewards": -0.7316367030143738, | |
| "rewards/unsafe_rewards": -0.6470431089401245, | |
| "step": 1520 | |
| }, | |
| { | |
| "epoch": 0.82, | |
| "learning_rate": 4.599335348222169e-08, | |
| "logits/chosen": -2.174553871154785, | |
| "logits/rejected": -2.05711030960083, | |
| "logps/chosen": -288.70849609375, | |
| "logps/rejected": -289.8013610839844, | |
| "loss": 10682.943, | |
| "rewards/accuracies": 0.7124999761581421, | |
| "rewards/chosen": -0.6674157977104187, | |
| "rewards/margins": 0.21851542592048645, | |
| "rewards/rejected": -0.8859313130378723, | |
| "rewards/safe_rewards": -0.6756128668785095, | |
| "rewards/unsafe_rewards": -0.6592189073562622, | |
| "step": 1530 | |
| }, | |
| { | |
| "epoch": 0.83, | |
| "learning_rate": 4.331438361071163e-08, | |
| "logits/chosen": -2.1525204181671143, | |
| "logits/rejected": -2.059199571609497, | |
| "logps/chosen": -302.69195556640625, | |
| "logps/rejected": -287.97161865234375, | |
| "loss": 11478.0016, | |
| "rewards/accuracies": 0.6499999761581421, | |
| "rewards/chosen": -0.6294053792953491, | |
| "rewards/margins": 0.17254997789859772, | |
| "rewards/rejected": -0.8019553422927856, | |
| "rewards/safe_rewards": -0.6268101334571838, | |
| "rewards/unsafe_rewards": -0.6320004463195801, | |
| "step": 1540 | |
| }, | |
| { | |
| "epoch": 0.83, | |
| "learning_rate": 4.0708380434367864e-08, | |
| "logits/chosen": -2.1821987628936768, | |
| "logits/rejected": -1.9993159770965576, | |
| "logps/chosen": -265.56854248046875, | |
| "logps/rejected": -261.4121398925781, | |
| "loss": 10532.0812, | |
| "rewards/accuracies": 0.706250011920929, | |
| "rewards/chosen": -0.6764332056045532, | |
| "rewards/margins": 0.19545750319957733, | |
| "rewards/rejected": -0.8718908429145813, | |
| "rewards/safe_rewards": -0.6992810964584351, | |
| "rewards/unsafe_rewards": -0.6535855531692505, | |
| "step": 1550 | |
| }, | |
| { | |
| "epoch": 0.84, | |
| "learning_rate": 3.817626395644305e-08, | |
| "logits/chosen": -2.1872050762176514, | |
| "logits/rejected": -2.0328307151794434, | |
| "logps/chosen": -265.10205078125, | |
| "logps/rejected": -248.8824462890625, | |
| "loss": 11308.0203, | |
| "rewards/accuracies": 0.606249988079071, | |
| "rewards/chosen": -0.6716850399971008, | |
| "rewards/margins": 0.13508987426757812, | |
| "rewards/rejected": -0.806774914264679, | |
| "rewards/safe_rewards": -0.6651741862297058, | |
| "rewards/unsafe_rewards": -0.6781958341598511, | |
| "step": 1560 | |
| }, | |
| { | |
| "epoch": 0.84, | |
| "learning_rate": 3.571892809580013e-08, | |
| "logits/chosen": -2.157172203063965, | |
| "logits/rejected": -2.0089964866638184, | |
| "logps/chosen": -260.57525634765625, | |
| "logps/rejected": -251.6233367919922, | |
| "loss": 11400.9055, | |
| "rewards/accuracies": 0.6187499761581421, | |
| "rewards/chosen": -0.6940633058547974, | |
| "rewards/margins": 0.136986643075943, | |
| "rewards/rejected": -0.831049919128418, | |
| "rewards/safe_rewards": -0.6672950983047485, | |
| "rewards/unsafe_rewards": -0.7208314538002014, | |
| "step": 1570 | |
| }, | |
| { | |
| "epoch": 0.85, | |
| "learning_rate": 3.333724037132976e-08, | |
| "logits/chosen": -2.180004596710205, | |
| "logits/rejected": -2.0329430103302, | |
| "logps/chosen": -274.8149108886719, | |
| "logps/rejected": -270.170654296875, | |
| "loss": 11022.3203, | |
| "rewards/accuracies": 0.643750011920929, | |
| "rewards/chosen": -0.6869722008705139, | |
| "rewards/margins": 0.17142841219902039, | |
| "rewards/rejected": -0.8584005236625671, | |
| "rewards/safe_rewards": -0.664495587348938, | |
| "rewards/unsafe_rewards": -0.7094486951828003, | |
| "step": 1580 | |
| }, | |
| { | |
| "epoch": 0.86, | |
| "learning_rate": 3.1032041595688506e-08, | |
| "logits/chosen": -2.107234477996826, | |
| "logits/rejected": -1.9175293445587158, | |
| "logps/chosen": -265.2936706542969, | |
| "logps/rejected": -258.67987060546875, | |
| "loss": 10853.2336, | |
| "rewards/accuracies": 0.6812499761581421, | |
| "rewards/chosen": -0.6839567422866821, | |
| "rewards/margins": 0.1879226267337799, | |
| "rewards/rejected": -0.8718793988227844, | |
| "rewards/safe_rewards": -0.6944083571434021, | |
| "rewards/unsafe_rewards": -0.6735051870346069, | |
| "step": 1590 | |
| }, | |
| { | |
| "epoch": 0.86, | |
| "learning_rate": 2.880414557846453e-08, | |
| "logits/chosen": -2.131779432296753, | |
| "logits/rejected": -2.045048236846924, | |
| "logps/chosen": -249.33114624023438, | |
| "logps/rejected": -240.8230438232422, | |
| "loss": 10239.6547, | |
| "rewards/accuracies": 0.643750011920929, | |
| "rewards/chosen": -0.6429909467697144, | |
| "rewards/margins": 0.1874755173921585, | |
| "rewards/rejected": -0.8304663896560669, | |
| "rewards/safe_rewards": -0.6290788650512695, | |
| "rewards/unsafe_rewards": -0.6569029092788696, | |
| "step": 1600 | |
| }, | |
| { | |
| "epoch": 0.87, | |
| "learning_rate": 2.6654338838876662e-08, | |
| "logits/chosen": -2.2182207107543945, | |
| "logits/rejected": -1.985608458518982, | |
| "logps/chosen": -280.660400390625, | |
| "logps/rejected": -240.7994384765625, | |
| "loss": 11038.0031, | |
| "rewards/accuracies": 0.668749988079071, | |
| "rewards/chosen": -0.6624557971954346, | |
| "rewards/margins": 0.23739679157733917, | |
| "rewards/rejected": -0.8998525738716125, | |
| "rewards/safe_rewards": -0.68045574426651, | |
| "rewards/unsafe_rewards": -0.6444558501243591, | |
| "step": 1610 | |
| }, | |
| { | |
| "epoch": 0.87, | |
| "learning_rate": 2.4583380328107805e-08, | |
| "logits/chosen": -2.1470413208007812, | |
| "logits/rejected": -1.985733985900879, | |
| "logps/chosen": -291.9235534667969, | |
| "logps/rejected": -258.4531555175781, | |
| "loss": 11528.2828, | |
| "rewards/accuracies": 0.6499999761581421, | |
| "rewards/chosen": -0.6623538136482239, | |
| "rewards/margins": 0.20262321829795837, | |
| "rewards/rejected": -0.8649770021438599, | |
| "rewards/safe_rewards": -0.6641503572463989, | |
| "rewards/unsafe_rewards": -0.660557210445404, | |
| "step": 1620 | |
| }, | |
| { | |
| "epoch": 0.88, | |
| "learning_rate": 2.259200116137039e-08, | |
| "logits/chosen": -2.1394782066345215, | |
| "logits/rejected": -1.9972803592681885, | |
| "logps/chosen": -284.652099609375, | |
| "logps/rejected": -278.02423095703125, | |
| "loss": 11545.0234, | |
| "rewards/accuracies": 0.643750011920929, | |
| "rewards/chosen": -0.6758708357810974, | |
| "rewards/margins": 0.17819495499134064, | |
| "rewards/rejected": -0.8540657758712769, | |
| "rewards/safe_rewards": -0.6568797826766968, | |
| "rewards/unsafe_rewards": -0.6948619484901428, | |
| "step": 1630 | |
| }, | |
| { | |
| "epoch": 0.88, | |
| "learning_rate": 2.068090435979958e-08, | |
| "logits/chosen": -2.1189610958099365, | |
| "logits/rejected": -2.0093135833740234, | |
| "logps/chosen": -268.87371826171875, | |
| "logps/rejected": -250.8845672607422, | |
| "loss": 11243.7469, | |
| "rewards/accuracies": 0.6625000238418579, | |
| "rewards/chosen": -0.6307465434074402, | |
| "rewards/margins": 0.16744786500930786, | |
| "rewards/rejected": -0.798194408416748, | |
| "rewards/safe_rewards": -0.6202843189239502, | |
| "rewards/unsafe_rewards": -0.6412087678909302, | |
| "step": 1640 | |
| }, | |
| { | |
| "epoch": 0.89, | |
| "learning_rate": 1.8850764602263423e-08, | |
| "logits/chosen": -2.1497740745544434, | |
| "logits/rejected": -1.9683231115341187, | |
| "logps/chosen": -263.89495849609375, | |
| "logps/rejected": -266.2047424316406, | |
| "loss": 11196.7602, | |
| "rewards/accuracies": 0.6625000238418579, | |
| "rewards/chosen": -0.6697946786880493, | |
| "rewards/margins": 0.1570041924715042, | |
| "rewards/rejected": -0.8267987966537476, | |
| "rewards/safe_rewards": -0.6728789806365967, | |
| "rewards/unsafe_rewards": -0.666710376739502, | |
| "step": 1650 | |
| }, | |
| { | |
| "epoch": 0.89, | |
| "learning_rate": 1.710222798718028e-08, | |
| "logits/chosen": -2.16522479057312, | |
| "logits/rejected": -2.045431613922119, | |
| "logps/chosen": -276.67889404296875, | |
| "logps/rejected": -280.6698303222656, | |
| "loss": 10997.7469, | |
| "rewards/accuracies": 0.6875, | |
| "rewards/chosen": -0.6575156450271606, | |
| "rewards/margins": 0.1838151067495346, | |
| "rewards/rejected": -0.8413307070732117, | |
| "rewards/safe_rewards": -0.6694937944412231, | |
| "rewards/unsafe_rewards": -0.6455374360084534, | |
| "step": 1660 | |
| }, | |
| { | |
| "epoch": 0.9, | |
| "learning_rate": 1.5435911804424356e-08, | |
| "logits/chosen": -2.159721851348877, | |
| "logits/rejected": -2.048668384552002, | |
| "logps/chosen": -290.70916748046875, | |
| "logps/rejected": -270.44683837890625, | |
| "loss": 11924.0805, | |
| "rewards/accuracies": 0.6499999761581421, | |
| "rewards/chosen": -0.6324605345726013, | |
| "rewards/margins": 0.19088464975357056, | |
| "rewards/rejected": -0.8233451843261719, | |
| "rewards/safe_rewards": -0.6401357054710388, | |
| "rewards/unsafe_rewards": -0.624785304069519, | |
| "step": 1670 | |
| }, | |
| { | |
| "epoch": 0.9, | |
| "learning_rate": 1.3852404317403199e-08, | |
| "logits/chosen": -2.108271360397339, | |
| "logits/rejected": -1.9915637969970703, | |
| "logps/chosen": -250.95361328125, | |
| "logps/rejected": -268.6639099121094, | |
| "loss": 11402.2625, | |
| "rewards/accuracies": 0.668749988079071, | |
| "rewards/chosen": -0.6399115324020386, | |
| "rewards/margins": 0.16807249188423157, | |
| "rewards/rejected": -0.8079840540885925, | |
| "rewards/safe_rewards": -0.6549087762832642, | |
| "rewards/unsafe_rewards": -0.6249145269393921, | |
| "step": 1680 | |
| }, | |
| { | |
| "epoch": 0.91, | |
| "learning_rate": 1.235226455538113e-08, | |
| "logits/chosen": -2.1952316761016846, | |
| "logits/rejected": -2.0874345302581787, | |
| "logps/chosen": -270.4241943359375, | |
| "logps/rejected": -263.0255432128906, | |
| "loss": 11680.2766, | |
| "rewards/accuracies": 0.6312500238418579, | |
| "rewards/chosen": -0.6519337892532349, | |
| "rewards/margins": 0.18009743094444275, | |
| "rewards/rejected": -0.8320311307907104, | |
| "rewards/safe_rewards": -0.662136435508728, | |
| "rewards/unsafe_rewards": -0.6417311429977417, | |
| "step": 1690 | |
| }, | |
| { | |
| "epoch": 0.91, | |
| "learning_rate": 1.0936022116124321e-08, | |
| "logits/chosen": -2.155649185180664, | |
| "logits/rejected": -2.0004172325134277, | |
| "logps/chosen": -262.668212890625, | |
| "logps/rejected": -252.78994750976562, | |
| "loss": 10840.3375, | |
| "rewards/accuracies": 0.706250011920929, | |
| "rewards/chosen": -0.6261528134346008, | |
| "rewards/margins": 0.2126920521259308, | |
| "rewards/rejected": -0.838844895362854, | |
| "rewards/safe_rewards": -0.6410089731216431, | |
| "rewards/unsafe_rewards": -0.6112965941429138, | |
| "step": 1700 | |
| }, | |
| { | |
| "epoch": 0.92, | |
| "learning_rate": 9.60417697893534e-09, | |
| "logits/chosen": -2.155482292175293, | |
| "logits/rejected": -2.0352649688720703, | |
| "logps/chosen": -259.88397216796875, | |
| "logps/rejected": -262.71746826171875, | |
| "loss": 11428.7422, | |
| "rewards/accuracies": 0.6625000238418579, | |
| "rewards/chosen": -0.632883608341217, | |
| "rewards/margins": 0.17666089534759521, | |
| "rewards/rejected": -0.8095444440841675, | |
| "rewards/safe_rewards": -0.5953446626663208, | |
| "rewards/unsafe_rewards": -0.6704224348068237, | |
| "step": 1710 | |
| }, | |
| { | |
| "epoch": 0.93, | |
| "learning_rate": 8.357199328144576e-09, | |
| "logits/chosen": -2.1424155235290527, | |
| "logits/rejected": -2.0332658290863037, | |
| "logps/chosen": -308.6863708496094, | |
| "logps/rejected": -289.75830078125, | |
| "loss": 10265.0281, | |
| "rewards/accuracies": 0.643750011920929, | |
| "rewards/chosen": -0.6669701337814331, | |
| "rewards/margins": 0.17594949901103973, | |
| "rewards/rejected": -0.8429197072982788, | |
| "rewards/safe_rewards": -0.7015780210494995, | |
| "rewards/unsafe_rewards": -0.6323622465133667, | |
| "step": 1720 | |
| }, | |
| { | |
| "epoch": 0.93, | |
| "learning_rate": 7.1955293871198144e-09, | |
| "logits/chosen": -2.112140417098999, | |
| "logits/rejected": -2.030024528503418, | |
| "logps/chosen": -241.85476684570312, | |
| "logps/rejected": -241.1482391357422, | |
| "loss": 11823.4781, | |
| "rewards/accuracies": 0.6187499761581421, | |
| "rewards/chosen": -0.6754211187362671, | |
| "rewards/margins": 0.13721489906311035, | |
| "rewards/rejected": -0.8126360177993774, | |
| "rewards/safe_rewards": -0.6722853779792786, | |
| "rewards/unsafe_rewards": -0.6785567998886108, | |
| "step": 1730 | |
| }, | |
| { | |
| "epoch": 0.94, | |
| "learning_rate": 6.119577262853254e-09, | |
| "logits/chosen": -2.1218502521514893, | |
| "logits/rejected": -1.944239854812622, | |
| "logps/chosen": -251.3634490966797, | |
| "logps/rejected": -230.8335723876953, | |
| "loss": 11556.3273, | |
| "rewards/accuracies": 0.65625, | |
| "rewards/chosen": -0.638350248336792, | |
| "rewards/margins": 0.21106202900409698, | |
| "rewards/rejected": -0.8494122624397278, | |
| "rewards/safe_rewards": -0.613598108291626, | |
| "rewards/unsafe_rewards": -0.6631024479866028, | |
| "step": 1740 | |
| }, | |
| { | |
| "epoch": 0.94, | |
| "learning_rate": 5.129722801180542e-09, | |
| "logits/chosen": -2.1147098541259766, | |
| "logits/rejected": -2.0030481815338135, | |
| "logps/chosen": -270.8954162597656, | |
| "logps/rejected": -255.99600219726562, | |
| "loss": 10173.3906, | |
| "rewards/accuracies": 0.625, | |
| "rewards/chosen": -0.6771684885025024, | |
| "rewards/margins": 0.1781720221042633, | |
| "rewards/rejected": -0.8553404808044434, | |
| "rewards/safe_rewards": -0.7092324495315552, | |
| "rewards/unsafe_rewards": -0.6451044082641602, | |
| "step": 1750 | |
| }, | |
| { | |
| "epoch": 0.95, | |
| "learning_rate": 4.226315452682816e-09, | |
| "logits/chosen": -2.1494641304016113, | |
| "logits/rejected": -2.025930404663086, | |
| "logps/chosen": -264.47100830078125, | |
| "logps/rejected": -256.65093994140625, | |
| "loss": 10950.5703, | |
| "rewards/accuracies": 0.7124999761581421, | |
| "rewards/chosen": -0.6349852681159973, | |
| "rewards/margins": 0.19823208451271057, | |
| "rewards/rejected": -0.833217442035675, | |
| "rewards/safe_rewards": -0.6437097787857056, | |
| "rewards/unsafe_rewards": -0.6262607574462891, | |
| "step": 1760 | |
| }, | |
| { | |
| "epoch": 0.95, | |
| "learning_rate": 3.4096741493194193e-09, | |
| "logits/chosen": -2.2127685546875, | |
| "logits/rejected": -2.0972578525543213, | |
| "logps/chosen": -264.87530517578125, | |
| "logps/rejected": -257.4256286621094, | |
| "loss": 11913.2609, | |
| "rewards/accuracies": 0.6625000238418579, | |
| "rewards/chosen": -0.6370722651481628, | |
| "rewards/margins": 0.15080325305461884, | |
| "rewards/rejected": -0.7878755927085876, | |
| "rewards/safe_rewards": -0.617976725101471, | |
| "rewards/unsafe_rewards": -0.6561679840087891, | |
| "step": 1770 | |
| }, | |
| { | |
| "epoch": 0.96, | |
| "learning_rate": 2.6800871918346846e-09, | |
| "logits/chosen": -2.220578670501709, | |
| "logits/rejected": -2.04236102104187, | |
| "logps/chosen": -274.6324768066406, | |
| "logps/rejected": -260.865234375, | |
| "loss": 11005.4078, | |
| "rewards/accuracies": 0.668749988079071, | |
| "rewards/chosen": -0.6086142659187317, | |
| "rewards/margins": 0.19890496134757996, | |
| "rewards/rejected": -0.8075191378593445, | |
| "rewards/safe_rewards": -0.6277071237564087, | |
| "rewards/unsafe_rewards": -0.5895212888717651, | |
| "step": 1780 | |
| }, | |
| { | |
| "epoch": 0.96, | |
| "learning_rate": 2.0378121479783796e-09, | |
| "logits/chosen": -2.132037401199341, | |
| "logits/rejected": -1.972651481628418, | |
| "logps/chosen": -265.41961669921875, | |
| "logps/rejected": -256.274658203125, | |
| "loss": 11459.243, | |
| "rewards/accuracies": 0.675000011920929, | |
| "rewards/chosen": -0.6822707056999207, | |
| "rewards/margins": 0.2049897462129593, | |
| "rewards/rejected": -0.8872605562210083, | |
| "rewards/safe_rewards": -0.6853871941566467, | |
| "rewards/unsafe_rewards": -0.6791542768478394, | |
| "step": 1790 | |
| }, | |
| { | |
| "epoch": 0.97, | |
| "learning_rate": 1.4830757615760247e-09, | |
| "logits/chosen": -2.1159682273864746, | |
| "logits/rejected": -1.9551427364349365, | |
| "logps/chosen": -281.0558166503906, | |
| "logps/rejected": -260.3757629394531, | |
| "loss": 11785.8336, | |
| "rewards/accuracies": 0.6187499761581421, | |
| "rewards/chosen": -0.6576239466667175, | |
| "rewards/margins": 0.15307986736297607, | |
| "rewards/rejected": -0.8107039332389832, | |
| "rewards/safe_rewards": -0.689802348613739, | |
| "rewards/unsafe_rewards": -0.6254457235336304, | |
| "step": 1800 | |
| }, | |
| { | |
| "epoch": 0.97, | |
| "eval_logits/chosen": -1.954970121383667, | |
| "eval_logits/rejected": -1.7745355367660522, | |
| "eval_logps/chosen": -224.6013946533203, | |
| "eval_logps/rejected": -193.1151123046875, | |
| "eval_loss": 4320.82080078125, | |
| "eval_rewards/accuracies": 0.6027347445487976, | |
| "eval_rewards/chosen": -0.9417080879211426, | |
| "eval_rewards/margins": 0.06478659808635712, | |
| "eval_rewards/rejected": -1.0064946413040161, | |
| "eval_rewards/safe_rewards": -0.9369282722473145, | |
| "eval_rewards/unsafe_rewards": -0.9372634291648865, | |
| "eval_runtime": 993.1393, | |
| "eval_samples_per_second": 33.272, | |
| "eval_steps_per_second": 1.04, | |
| "step": 1800 | |
| }, | |
| { | |
| "epoch": 0.97, | |
| "learning_rate": 1.0160738724809548e-09, | |
| "logits/chosen": -2.1704697608947754, | |
| "logits/rejected": -1.9731838703155518, | |
| "logps/chosen": -259.1123046875, | |
| "logps/rejected": -257.42254638671875, | |
| "loss": 10364.2625, | |
| "rewards/accuracies": 0.675000011920929, | |
| "rewards/chosen": -0.6492170691490173, | |
| "rewards/margins": 0.19766607880592346, | |
| "rewards/rejected": -0.8468831777572632, | |
| "rewards/safe_rewards": -0.6583928465843201, | |
| "rewards/unsafe_rewards": -0.6400412321090698, | |
| "step": 1810 | |
| }, | |
| { | |
| "epoch": 0.98, | |
| "learning_rate": 6.369713474366212e-10, | |
| "logits/chosen": -2.1475830078125, | |
| "logits/rejected": -1.9969937801361084, | |
| "logps/chosen": -294.9414367675781, | |
| "logps/rejected": -288.337890625, | |
| "loss": 10068.2195, | |
| "rewards/accuracies": 0.6875, | |
| "rewards/chosen": -0.6980792880058289, | |
| "rewards/margins": 0.2098824679851532, | |
| "rewards/rejected": -0.9079617261886597, | |
| "rewards/safe_rewards": -0.7020989656448364, | |
| "rewards/unsafe_rewards": -0.6940596699714661, | |
| "step": 1820 | |
| }, | |
| { | |
| "epoch": 0.98, | |
| "learning_rate": 3.459020218731512e-10, | |
| "logits/chosen": -2.1376261711120605, | |
| "logits/rejected": -2.0157809257507324, | |
| "logps/chosen": -254.5150604248047, | |
| "logps/rejected": -243.1461181640625, | |
| "loss": 10131.9508, | |
| "rewards/accuracies": 0.6312500238418579, | |
| "rewards/chosen": -0.6527233719825745, | |
| "rewards/margins": 0.2155275046825409, | |
| "rewards/rejected": -0.8682507276535034, | |
| "rewards/safe_rewards": -0.6437439322471619, | |
| "rewards/unsafe_rewards": -0.6617026329040527, | |
| "step": 1830 | |
| }, | |
| { | |
| "epoch": 0.99, | |
| "learning_rate": 1.429686526593088e-10, | |
| "logits/chosen": -2.116459608078003, | |
| "logits/rejected": -1.9903713464736938, | |
| "logps/chosen": -270.08428955078125, | |
| "logps/rejected": -265.38336181640625, | |
| "loss": 11322.0812, | |
| "rewards/accuracies": 0.668749988079071, | |
| "rewards/chosen": -0.6661794185638428, | |
| "rewards/margins": 0.18438491225242615, | |
| "rewards/rejected": -0.8505643606185913, | |
| "rewards/safe_rewards": -0.6831308603286743, | |
| "rewards/unsafe_rewards": -0.6492279767990112, | |
| "step": 1840 | |
| }, | |
| { | |
| "epoch": 1.0, | |
| "learning_rate": 2.824288182584622e-11, | |
| "logits/chosen": -2.1975011825561523, | |
| "logits/rejected": -2.020158290863037, | |
| "logps/chosen": -284.1047668457031, | |
| "logps/rejected": -259.551513671875, | |
| "loss": 10472.5969, | |
| "rewards/accuracies": 0.643750011920929, | |
| "rewards/chosen": -0.6722114682197571, | |
| "rewards/margins": 0.17123940587043762, | |
| "rewards/rejected": -0.8434508442878723, | |
| "rewards/safe_rewards": -0.6627709865570068, | |
| "rewards/unsafe_rewards": -0.6816519498825073, | |
| "step": 1850 | |
| }, | |
| { | |
| "epoch": 1.0, | |
| "step": 1858, | |
| "total_flos": 0.0, | |
| "train_loss": 11692.422362755651, | |
| "train_runtime": 23508.9776, | |
| "train_samples_per_second": 2.53, | |
| "train_steps_per_second": 0.079 | |
| } | |
| ], | |
| "logging_steps": 10, | |
| "max_steps": 1858, | |
| "num_input_tokens_seen": 0, | |
| "num_train_epochs": 1, | |
| "save_steps": 300, | |
| "total_flos": 0.0, | |
| "train_batch_size": 2, | |
| "trial_name": null, | |
| "trial_params": null | |
| } | |