| { | |
| "best_metric": null, | |
| "best_model_checkpoint": null, | |
| "epoch": 1.0, | |
| "eval_steps": 100, | |
| "global_step": 37, | |
| "is_hyper_param_search": false, | |
| "is_local_process_zero": true, | |
| "is_world_process_zero": true, | |
| "log_history": [ | |
| { | |
| "debug/policy_chosen_logits": -0.8575258255004883, | |
| "debug/policy_chosen_logps": -145.75564575195312, | |
| "debug/policy_rejected_logits": -0.7912808060646057, | |
| "debug/policy_rejected_logps": -155.6038818359375, | |
| "debug/reference_chosen_logps": -145.75564575195312, | |
| "debug/reference_rejected_logps": -155.6038818359375, | |
| "epoch": 0.02702702702702703, | |
| "grad_norm": 10.590267262290878, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": -0.8575258255004883, | |
| "logits/rejected": -0.7912808060646057, | |
| "logps/chosen": -145.75564575195312, | |
| "logps/rejected": -155.6038818359375, | |
| "loss": 0.5, | |
| "rewards/accuracies": 0.0, | |
| "rewards/chosen": 0.0, | |
| "rewards/margins": 0.0, | |
| "rewards/rejected": 0.0, | |
| "step": 1 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": -0.7469339966773987, | |
| "debug/policy_chosen_logps": -161.53433227539062, | |
| "debug/policy_rejected_logits": -0.7776355147361755, | |
| "debug/policy_rejected_logps": -170.75521850585938, | |
| "debug/reference_chosen_logps": -161.53152465820312, | |
| "debug/reference_rejected_logps": -170.4493408203125, | |
| "epoch": 0.05405405405405406, | |
| "grad_norm": 10.000972104848039, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": -0.7469339966773987, | |
| "logits/rejected": -0.7776355147361755, | |
| "logps/chosen": -161.53433227539062, | |
| "logps/rejected": -170.75521850585938, | |
| "loss": 0.4999, | |
| "rewards/accuracies": 0.875, | |
| "rewards/chosen": -2.8095149900764227e-05, | |
| "rewards/margins": 0.0030306337866932154, | |
| "rewards/rejected": -0.0030587289948016405, | |
| "step": 2 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": -0.8981359004974365, | |
| "debug/policy_chosen_logps": -176.5548095703125, | |
| "debug/policy_rejected_logits": -0.8165794014930725, | |
| "debug/policy_rejected_logps": -174.51219177246094, | |
| "debug/reference_chosen_logps": -176.45700073242188, | |
| "debug/reference_rejected_logps": -174.48855590820312, | |
| "epoch": 0.08108108108108109, | |
| "grad_norm": 11.901363041409127, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": -0.8981359004974365, | |
| "logits/rejected": -0.8165794014930725, | |
| "logps/chosen": -176.5548095703125, | |
| "logps/rejected": -174.51219177246094, | |
| "loss": 0.4991, | |
| "rewards/accuracies": 0.5, | |
| "rewards/chosen": -0.0009781360859051347, | |
| "rewards/margins": -0.000741882249712944, | |
| "rewards/rejected": -0.00023625371977686882, | |
| "step": 3 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": -0.9096183776855469, | |
| "debug/policy_chosen_logps": -166.55648803710938, | |
| "debug/policy_rejected_logits": -0.9066538214683533, | |
| "debug/policy_rejected_logps": -164.07363891601562, | |
| "debug/reference_chosen_logps": -166.54132080078125, | |
| "debug/reference_rejected_logps": -164.5047607421875, | |
| "epoch": 0.10810810810810811, | |
| "grad_norm": 11.906594144254457, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": -0.9096183776855469, | |
| "logits/rejected": -0.9066538214683533, | |
| "logps/chosen": -166.55648803710938, | |
| "logps/rejected": -164.07363891601562, | |
| "loss": 0.4985, | |
| "rewards/accuracies": 0.5, | |
| "rewards/chosen": -0.00015165336662903428, | |
| "rewards/margins": -0.004462923854589462, | |
| "rewards/rejected": 0.004311270546168089, | |
| "step": 4 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": -0.7255207896232605, | |
| "debug/policy_chosen_logps": -197.11146545410156, | |
| "debug/policy_rejected_logits": -0.6460945010185242, | |
| "debug/policy_rejected_logps": -199.1307830810547, | |
| "debug/reference_chosen_logps": -196.3956298828125, | |
| "debug/reference_rejected_logps": -198.6553192138672, | |
| "epoch": 0.13513513513513514, | |
| "grad_norm": 12.022558068888802, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": -0.7255207896232605, | |
| "logits/rejected": -0.6460945010185242, | |
| "logps/chosen": -197.11146545410156, | |
| "logps/rejected": -199.1307830810547, | |
| "loss": 0.4968, | |
| "rewards/accuracies": 0.375, | |
| "rewards/chosen": -0.007158536929637194, | |
| "rewards/margins": -0.00240384042263031, | |
| "rewards/rejected": -0.004754695575684309, | |
| "step": 5 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": -0.9493230581283569, | |
| "debug/policy_chosen_logps": -129.1173553466797, | |
| "debug/policy_rejected_logits": -0.8156340718269348, | |
| "debug/policy_rejected_logps": -164.30892944335938, | |
| "debug/reference_chosen_logps": -127.74217987060547, | |
| "debug/reference_rejected_logps": -162.74935913085938, | |
| "epoch": 0.16216216216216217, | |
| "grad_norm": 12.824448923481688, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": -0.9493230581283569, | |
| "logits/rejected": -0.8156340718269348, | |
| "logps/chosen": -129.1173553466797, | |
| "logps/rejected": -164.30892944335938, | |
| "loss": 0.5049, | |
| "rewards/accuracies": 0.625, | |
| "rewards/chosen": -0.013751697726547718, | |
| "rewards/margins": 0.0018438897095620632, | |
| "rewards/rejected": -0.015595588833093643, | |
| "step": 6 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": -0.7272260785102844, | |
| "debug/policy_chosen_logps": -184.9543914794922, | |
| "debug/policy_rejected_logits": -0.6780456900596619, | |
| "debug/policy_rejected_logps": -187.98831176757812, | |
| "debug/reference_chosen_logps": -184.20895385742188, | |
| "debug/reference_rejected_logps": -188.1205291748047, | |
| "epoch": 0.1891891891891892, | |
| "grad_norm": 12.347680907504191, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": -0.7272260785102844, | |
| "logits/rejected": -0.6780456900596619, | |
| "logps/chosen": -184.9543914794922, | |
| "logps/rejected": -187.98831176757812, | |
| "loss": 0.4986, | |
| "rewards/accuracies": 0.375, | |
| "rewards/chosen": -0.007454394828528166, | |
| "rewards/margins": -0.008776684291660786, | |
| "rewards/rejected": 0.0013222885318100452, | |
| "step": 7 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": -0.8364517688751221, | |
| "debug/policy_chosen_logps": -189.10472106933594, | |
| "debug/policy_rejected_logits": -0.7381677627563477, | |
| "debug/policy_rejected_logps": -188.10775756835938, | |
| "debug/reference_chosen_logps": -187.6288299560547, | |
| "debug/reference_rejected_logps": -188.3019256591797, | |
| "epoch": 0.21621621621621623, | |
| "grad_norm": 12.404458206833132, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": -0.8364517688751221, | |
| "logits/rejected": -0.7381677627563477, | |
| "logps/chosen": -189.10472106933594, | |
| "logps/rejected": -188.10775756835938, | |
| "loss": 0.4937, | |
| "rewards/accuracies": 0.375, | |
| "rewards/chosen": -0.014758807606995106, | |
| "rewards/margins": -0.016700536012649536, | |
| "rewards/rejected": 0.0019417284056544304, | |
| "step": 8 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": -0.9120941758155823, | |
| "debug/policy_chosen_logps": -163.34133911132812, | |
| "debug/policy_rejected_logits": -0.976243257522583, | |
| "debug/policy_rejected_logps": -167.19322204589844, | |
| "debug/reference_chosen_logps": -161.68980407714844, | |
| "debug/reference_rejected_logps": -164.67279052734375, | |
| "epoch": 0.24324324324324326, | |
| "grad_norm": 12.727135168469278, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": -0.9120941758155823, | |
| "logits/rejected": -0.976243257522583, | |
| "logps/chosen": -163.34133911132812, | |
| "logps/rejected": -167.19322204589844, | |
| "loss": 0.4976, | |
| "rewards/accuracies": 0.5, | |
| "rewards/chosen": -0.0165153406560421, | |
| "rewards/margins": 0.008688842877745628, | |
| "rewards/rejected": -0.025204181671142578, | |
| "step": 9 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": -0.760608434677124, | |
| "debug/policy_chosen_logps": -172.08404541015625, | |
| "debug/policy_rejected_logits": -0.7949759364128113, | |
| "debug/policy_rejected_logps": -178.0618133544922, | |
| "debug/reference_chosen_logps": -171.69947814941406, | |
| "debug/reference_rejected_logps": -175.53970336914062, | |
| "epoch": 0.2702702702702703, | |
| "grad_norm": 12.198398917852398, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": -0.760608434677124, | |
| "logits/rejected": -0.7949759364128113, | |
| "logps/chosen": -172.08404541015625, | |
| "logps/rejected": -178.0618133544922, | |
| "loss": 0.49, | |
| "rewards/accuracies": 0.875, | |
| "rewards/chosen": -0.003845730097964406, | |
| "rewards/margins": 0.021375417709350586, | |
| "rewards/rejected": -0.025221146643161774, | |
| "step": 10 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": -0.7727924585342407, | |
| "debug/policy_chosen_logps": -174.89260864257812, | |
| "debug/policy_rejected_logits": -0.8414075970649719, | |
| "debug/policy_rejected_logps": -191.7485809326172, | |
| "debug/reference_chosen_logps": -174.308349609375, | |
| "debug/reference_rejected_logps": -189.44308471679688, | |
| "epoch": 0.2972972972972973, | |
| "grad_norm": 12.869944279488237, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": -0.7727924585342407, | |
| "logits/rejected": -0.8414075970649719, | |
| "logps/chosen": -174.89260864257812, | |
| "logps/rejected": -191.7485809326172, | |
| "loss": 0.4961, | |
| "rewards/accuracies": 0.75, | |
| "rewards/chosen": -0.005842561833560467, | |
| "rewards/margins": 0.0172123983502388, | |
| "rewards/rejected": -0.02305496111512184, | |
| "step": 11 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": -0.9232946038246155, | |
| "debug/policy_chosen_logps": -172.6126708984375, | |
| "debug/policy_rejected_logits": -0.8749657273292542, | |
| "debug/policy_rejected_logps": -176.91111755371094, | |
| "debug/reference_chosen_logps": -170.90977478027344, | |
| "debug/reference_rejected_logps": -176.2742156982422, | |
| "epoch": 0.32432432432432434, | |
| "grad_norm": 13.637899725211922, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": -0.9232946038246155, | |
| "logits/rejected": -0.8749657273292542, | |
| "logps/chosen": -172.6126708984375, | |
| "logps/rejected": -176.91111755371094, | |
| "loss": 0.5005, | |
| "rewards/accuracies": 0.625, | |
| "rewards/chosen": -0.017028970643877983, | |
| "rewards/margins": -0.010659895837306976, | |
| "rewards/rejected": -0.006369075272232294, | |
| "step": 12 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": -0.7241289615631104, | |
| "debug/policy_chosen_logps": -170.33370971679688, | |
| "debug/policy_rejected_logits": -0.7508945465087891, | |
| "debug/policy_rejected_logps": -174.8129119873047, | |
| "debug/reference_chosen_logps": -169.210205078125, | |
| "debug/reference_rejected_logps": -175.19602966308594, | |
| "epoch": 0.35135135135135137, | |
| "grad_norm": 12.678047953411022, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": -0.7241289615631104, | |
| "logits/rejected": -0.7508945465087891, | |
| "logps/chosen": -170.33370971679688, | |
| "logps/rejected": -174.8129119873047, | |
| "loss": 0.4867, | |
| "rewards/accuracies": 0.125, | |
| "rewards/chosen": -0.011235074140131474, | |
| "rewards/margins": -0.015066290274262428, | |
| "rewards/rejected": 0.0038312142714858055, | |
| "step": 13 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": -0.9043007493019104, | |
| "debug/policy_chosen_logps": -165.7388916015625, | |
| "debug/policy_rejected_logits": -0.8851659893989563, | |
| "debug/policy_rejected_logps": -185.2967529296875, | |
| "debug/reference_chosen_logps": -169.01324462890625, | |
| "debug/reference_rejected_logps": -184.21755981445312, | |
| "epoch": 0.3783783783783784, | |
| "grad_norm": 14.228799593284224, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": -0.9043007493019104, | |
| "logits/rejected": -0.8851659893989563, | |
| "logps/chosen": -165.7388916015625, | |
| "logps/rejected": -185.2967529296875, | |
| "loss": 0.4838, | |
| "rewards/accuracies": 0.875, | |
| "rewards/chosen": 0.032743629068136215, | |
| "rewards/margins": 0.04353557527065277, | |
| "rewards/rejected": -0.010791949927806854, | |
| "step": 14 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": -0.8794234991073608, | |
| "debug/policy_chosen_logps": -190.51846313476562, | |
| "debug/policy_rejected_logits": -0.9416622519493103, | |
| "debug/policy_rejected_logps": -182.75303649902344, | |
| "debug/reference_chosen_logps": -191.22064208984375, | |
| "debug/reference_rejected_logps": -180.401611328125, | |
| "epoch": 0.40540540540540543, | |
| "grad_norm": 13.867700853582742, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": -0.8794234991073608, | |
| "logits/rejected": -0.9416622519493103, | |
| "logps/chosen": -190.51846313476562, | |
| "logps/rejected": -182.75303649902344, | |
| "loss": 0.5002, | |
| "rewards/accuracies": 0.875, | |
| "rewards/chosen": 0.0070218658074736595, | |
| "rewards/margins": 0.03053615428507328, | |
| "rewards/rejected": -0.023514289408922195, | |
| "step": 15 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": -0.9163352251052856, | |
| "debug/policy_chosen_logps": -161.2454833984375, | |
| "debug/policy_rejected_logits": -0.8506691455841064, | |
| "debug/policy_rejected_logps": -174.76438903808594, | |
| "debug/reference_chosen_logps": -164.75534057617188, | |
| "debug/reference_rejected_logps": -172.7283172607422, | |
| "epoch": 0.43243243243243246, | |
| "grad_norm": 15.950956103697601, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": -0.9163352251052856, | |
| "logits/rejected": -0.8506691455841064, | |
| "logps/chosen": -161.2454833984375, | |
| "logps/rejected": -174.76438903808594, | |
| "loss": 0.4953, | |
| "rewards/accuracies": 0.75, | |
| "rewards/chosen": 0.035098638385534286, | |
| "rewards/margins": 0.055459294468164444, | |
| "rewards/rejected": -0.020360659807920456, | |
| "step": 16 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": -0.886226236820221, | |
| "debug/policy_chosen_logps": -152.4619598388672, | |
| "debug/policy_rejected_logits": -0.8994572162628174, | |
| "debug/policy_rejected_logps": -186.78121948242188, | |
| "debug/reference_chosen_logps": -153.10696411132812, | |
| "debug/reference_rejected_logps": -188.9115447998047, | |
| "epoch": 0.4594594594594595, | |
| "grad_norm": 14.555594379538805, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": -0.886226236820221, | |
| "logits/rejected": -0.8994572162628174, | |
| "logps/chosen": -152.4619598388672, | |
| "logps/rejected": -186.78121948242188, | |
| "loss": 0.4997, | |
| "rewards/accuracies": 0.375, | |
| "rewards/chosen": 0.006449948064982891, | |
| "rewards/margins": -0.014853332191705704, | |
| "rewards/rejected": 0.02130328118801117, | |
| "step": 17 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": -0.791405200958252, | |
| "debug/policy_chosen_logps": -184.97439575195312, | |
| "debug/policy_rejected_logits": -0.7767256498336792, | |
| "debug/policy_rejected_logps": -168.33358764648438, | |
| "debug/reference_chosen_logps": -185.62191772460938, | |
| "debug/reference_rejected_logps": -161.90869140625, | |
| "epoch": 0.4864864864864865, | |
| "grad_norm": 17.299703458305174, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": -0.791405200958252, | |
| "logits/rejected": -0.7767256498336792, | |
| "logps/chosen": -184.97439575195312, | |
| "logps/rejected": -168.33358764648438, | |
| "loss": 0.4931, | |
| "rewards/accuracies": 0.75, | |
| "rewards/chosen": 0.006475199945271015, | |
| "rewards/margins": 0.0707239881157875, | |
| "rewards/rejected": -0.06424878537654877, | |
| "step": 18 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": -0.8800061345100403, | |
| "debug/policy_chosen_logps": -148.43475341796875, | |
| "debug/policy_rejected_logits": -0.9134210348129272, | |
| "debug/policy_rejected_logps": -142.7111358642578, | |
| "debug/reference_chosen_logps": -149.74551391601562, | |
| "debug/reference_rejected_logps": -140.6129150390625, | |
| "epoch": 0.5135135135135135, | |
| "grad_norm": 15.389494933483391, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": -0.8800061345100403, | |
| "logits/rejected": -0.9134210348129272, | |
| "logps/chosen": -148.43475341796875, | |
| "logps/rejected": -142.7111358642578, | |
| "loss": 0.4953, | |
| "rewards/accuracies": 0.75, | |
| "rewards/chosen": 0.013107641600072384, | |
| "rewards/margins": 0.03408981114625931, | |
| "rewards/rejected": -0.02098216861486435, | |
| "step": 19 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": -0.95091712474823, | |
| "debug/policy_chosen_logps": -152.49203491210938, | |
| "debug/policy_rejected_logits": -0.8285200595855713, | |
| "debug/policy_rejected_logps": -152.87184143066406, | |
| "debug/reference_chosen_logps": -151.83273315429688, | |
| "debug/reference_rejected_logps": -152.75054931640625, | |
| "epoch": 0.5405405405405406, | |
| "grad_norm": 13.903822325437991, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": -0.95091712474823, | |
| "logits/rejected": -0.8285200595855713, | |
| "logps/chosen": -152.49203491210938, | |
| "logps/rejected": -152.87184143066406, | |
| "loss": 0.5003, | |
| "rewards/accuracies": 0.5, | |
| "rewards/chosen": -0.006593028549104929, | |
| "rewards/margins": -0.005380069836974144, | |
| "rewards/rejected": -0.0012129591777920723, | |
| "step": 20 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": -0.8724645972251892, | |
| "debug/policy_chosen_logps": -142.1439666748047, | |
| "debug/policy_rejected_logits": -0.7575433850288391, | |
| "debug/policy_rejected_logps": -175.90309143066406, | |
| "debug/reference_chosen_logps": -140.2176513671875, | |
| "debug/reference_rejected_logps": -174.95127868652344, | |
| "epoch": 0.5675675675675675, | |
| "grad_norm": 14.060737834168805, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": -0.8724645972251892, | |
| "logits/rejected": -0.7575433850288391, | |
| "logps/chosen": -142.1439666748047, | |
| "logps/rejected": -175.90309143066406, | |
| "loss": 0.4882, | |
| "rewards/accuracies": 0.5, | |
| "rewards/chosen": -0.01926323026418686, | |
| "rewards/margins": -0.00974507350474596, | |
| "rewards/rejected": -0.009518155828118324, | |
| "step": 21 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": -1.0412858724594116, | |
| "debug/policy_chosen_logps": -135.55929565429688, | |
| "debug/policy_rejected_logits": -1.00175142288208, | |
| "debug/policy_rejected_logps": -188.136962890625, | |
| "debug/reference_chosen_logps": -135.7158203125, | |
| "debug/reference_rejected_logps": -186.81320190429688, | |
| "epoch": 0.5945945945945946, | |
| "grad_norm": 14.183550914621547, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": -1.0412858724594116, | |
| "logits/rejected": -1.00175142288208, | |
| "logps/chosen": -135.55929565429688, | |
| "logps/rejected": -188.136962890625, | |
| "loss": 0.4911, | |
| "rewards/accuracies": 0.625, | |
| "rewards/chosen": 0.0015652086585760117, | |
| "rewards/margins": 0.014802752062678337, | |
| "rewards/rejected": -0.013237543404102325, | |
| "step": 22 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": -0.8892878293991089, | |
| "debug/policy_chosen_logps": -152.1015625, | |
| "debug/policy_rejected_logits": -0.7090870141983032, | |
| "debug/policy_rejected_logps": -186.6959686279297, | |
| "debug/reference_chosen_logps": -147.5272216796875, | |
| "debug/reference_rejected_logps": -184.30201721191406, | |
| "epoch": 0.6216216216216216, | |
| "grad_norm": 14.276125809311413, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": -0.8892878293991089, | |
| "logits/rejected": -0.7090870141983032, | |
| "logps/chosen": -152.1015625, | |
| "logps/rejected": -186.6959686279297, | |
| "loss": 0.5004, | |
| "rewards/accuracies": 0.5, | |
| "rewards/chosen": -0.04574331268668175, | |
| "rewards/margins": -0.021803725510835648, | |
| "rewards/rejected": -0.0239395871758461, | |
| "step": 23 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": -0.8802600502967834, | |
| "debug/policy_chosen_logps": -159.21139526367188, | |
| "debug/policy_rejected_logits": -0.9364652037620544, | |
| "debug/policy_rejected_logps": -164.08383178710938, | |
| "debug/reference_chosen_logps": -155.6630859375, | |
| "debug/reference_rejected_logps": -161.748291015625, | |
| "epoch": 0.6486486486486487, | |
| "grad_norm": 15.603022958174197, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": -0.8802600502967834, | |
| "logits/rejected": -0.9364652037620544, | |
| "logps/chosen": -159.21139526367188, | |
| "logps/rejected": -164.08383178710938, | |
| "loss": 0.4987, | |
| "rewards/accuracies": 0.375, | |
| "rewards/chosen": -0.035483140498399734, | |
| "rewards/margins": -0.012127798981964588, | |
| "rewards/rejected": -0.02335534058511257, | |
| "step": 24 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": -0.9097840785980225, | |
| "debug/policy_chosen_logps": -161.55003356933594, | |
| "debug/policy_rejected_logits": -0.8685249090194702, | |
| "debug/policy_rejected_logps": -179.54681396484375, | |
| "debug/reference_chosen_logps": -162.00999450683594, | |
| "debug/reference_rejected_logps": -181.08807373046875, | |
| "epoch": 0.6756756756756757, | |
| "grad_norm": 14.622466903613363, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": -0.9097840785980225, | |
| "logits/rejected": -0.8685249090194702, | |
| "logps/chosen": -161.55003356933594, | |
| "logps/rejected": -179.54681396484375, | |
| "loss": 0.5001, | |
| "rewards/accuracies": 0.5, | |
| "rewards/chosen": 0.004599475301802158, | |
| "rewards/margins": -0.010813076049089432, | |
| "rewards/rejected": 0.015412550419569016, | |
| "step": 25 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": -0.8982373476028442, | |
| "debug/policy_chosen_logps": -155.50433349609375, | |
| "debug/policy_rejected_logits": -0.8736176490783691, | |
| "debug/policy_rejected_logps": -176.95404052734375, | |
| "debug/reference_chosen_logps": -155.57803344726562, | |
| "debug/reference_rejected_logps": -176.65240478515625, | |
| "epoch": 0.7027027027027027, | |
| "grad_norm": 14.148910548336614, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": -0.8982373476028442, | |
| "logits/rejected": -0.8736176490783691, | |
| "logps/chosen": -155.50433349609375, | |
| "logps/rejected": -176.95404052734375, | |
| "loss": 0.5059, | |
| "rewards/accuracies": 0.625, | |
| "rewards/chosen": 0.0007370477542281151, | |
| "rewards/margins": 0.003753413911908865, | |
| "rewards/rejected": -0.003016366856172681, | |
| "step": 26 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": -0.9832875728607178, | |
| "debug/policy_chosen_logps": -159.749267578125, | |
| "debug/policy_rejected_logits": -0.9118414521217346, | |
| "debug/policy_rejected_logps": -151.69076538085938, | |
| "debug/reference_chosen_logps": -162.98480224609375, | |
| "debug/reference_rejected_logps": -154.2532501220703, | |
| "epoch": 0.7297297297297297, | |
| "grad_norm": 15.17612475288469, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": -0.9832875728607178, | |
| "logits/rejected": -0.9118414521217346, | |
| "logps/chosen": -159.749267578125, | |
| "logps/rejected": -151.69076538085938, | |
| "loss": 0.5099, | |
| "rewards/accuracies": 0.375, | |
| "rewards/chosen": 0.03235547989606857, | |
| "rewards/margins": 0.006730623543262482, | |
| "rewards/rejected": 0.02562485635280609, | |
| "step": 27 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": -0.8673918843269348, | |
| "debug/policy_chosen_logps": -147.43087768554688, | |
| "debug/policy_rejected_logits": -0.7803842425346375, | |
| "debug/policy_rejected_logps": -188.6826629638672, | |
| "debug/reference_chosen_logps": -148.56741333007812, | |
| "debug/reference_rejected_logps": -186.77401733398438, | |
| "epoch": 0.7567567567567568, | |
| "grad_norm": 13.774005492067339, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": -0.8673918843269348, | |
| "logits/rejected": -0.7803842425346375, | |
| "logps/chosen": -147.43087768554688, | |
| "logps/rejected": -188.6826629638672, | |
| "loss": 0.4971, | |
| "rewards/accuracies": 0.5, | |
| "rewards/chosen": 0.01136524323374033, | |
| "rewards/margins": 0.030451610684394836, | |
| "rewards/rejected": -0.01908636838197708, | |
| "step": 28 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": -0.8602281808853149, | |
| "debug/policy_chosen_logps": -153.77182006835938, | |
| "debug/policy_rejected_logits": -0.8865491151809692, | |
| "debug/policy_rejected_logps": -184.97218322753906, | |
| "debug/reference_chosen_logps": -154.79705810546875, | |
| "debug/reference_rejected_logps": -183.60633850097656, | |
| "epoch": 0.7837837837837838, | |
| "grad_norm": 15.257213384975808, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": -0.8602281808853149, | |
| "logits/rejected": -0.8865491151809692, | |
| "logps/chosen": -153.77182006835938, | |
| "logps/rejected": -184.97218322753906, | |
| "loss": 0.5138, | |
| "rewards/accuracies": 0.625, | |
| "rewards/chosen": 0.01025250181555748, | |
| "rewards/margins": 0.02391086146235466, | |
| "rewards/rejected": -0.01365836150944233, | |
| "step": 29 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": -0.9129707217216492, | |
| "debug/policy_chosen_logps": -184.99423217773438, | |
| "debug/policy_rejected_logits": -1.0919617414474487, | |
| "debug/policy_rejected_logps": -141.35853576660156, | |
| "debug/reference_chosen_logps": -188.018798828125, | |
| "debug/reference_rejected_logps": -136.7606201171875, | |
| "epoch": 0.8108108108108109, | |
| "grad_norm": 13.614612194522516, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": -0.9129707217216492, | |
| "logits/rejected": -1.0919617414474487, | |
| "logps/chosen": -184.99423217773438, | |
| "logps/rejected": -141.35853576660156, | |
| "loss": 0.4793, | |
| "rewards/accuracies": 0.75, | |
| "rewards/chosen": 0.030245695263147354, | |
| "rewards/margins": 0.07622484117746353, | |
| "rewards/rejected": -0.04597914591431618, | |
| "step": 30 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": -0.7870268225669861, | |
| "debug/policy_chosen_logps": -157.5154266357422, | |
| "debug/policy_rejected_logits": -0.8374965190887451, | |
| "debug/policy_rejected_logps": -162.17874145507812, | |
| "debug/reference_chosen_logps": -159.80447387695312, | |
| "debug/reference_rejected_logps": -163.95941162109375, | |
| "epoch": 0.8378378378378378, | |
| "grad_norm": 13.069040617397022, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": -0.7870268225669861, | |
| "logits/rejected": -0.8374965190887451, | |
| "logps/chosen": -157.5154266357422, | |
| "logps/rejected": -162.17874145507812, | |
| "loss": 0.48, | |
| "rewards/accuracies": 0.625, | |
| "rewards/chosen": 0.022890347987413406, | |
| "rewards/margins": 0.0050835697911679745, | |
| "rewards/rejected": 0.017806777730584145, | |
| "step": 31 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": -0.8162547945976257, | |
| "debug/policy_chosen_logps": -171.15731811523438, | |
| "debug/policy_rejected_logits": -0.8792607188224792, | |
| "debug/policy_rejected_logps": -170.6051483154297, | |
| "debug/reference_chosen_logps": -174.41046142578125, | |
| "debug/reference_rejected_logps": -171.5959930419922, | |
| "epoch": 0.8648648648648649, | |
| "grad_norm": 13.409519517707912, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": -0.8162547945976257, | |
| "logits/rejected": -0.8792607188224792, | |
| "logps/chosen": -171.15731811523438, | |
| "logps/rejected": -170.6051483154297, | |
| "loss": 0.4797, | |
| "rewards/accuracies": 0.75, | |
| "rewards/chosen": 0.03253144025802612, | |
| "rewards/margins": 0.02262299507856369, | |
| "rewards/rejected": 0.009908447973430157, | |
| "step": 32 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": -0.8389750719070435, | |
| "debug/policy_chosen_logps": -130.0693359375, | |
| "debug/policy_rejected_logits": -0.8284645676612854, | |
| "debug/policy_rejected_logps": -172.84576416015625, | |
| "debug/reference_chosen_logps": -128.82723999023438, | |
| "debug/reference_rejected_logps": -173.13807678222656, | |
| "epoch": 0.8918918918918919, | |
| "grad_norm": 14.49702969131408, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": -0.8389750719070435, | |
| "logits/rejected": -0.8284645676612854, | |
| "logps/chosen": -130.0693359375, | |
| "logps/rejected": -172.84576416015625, | |
| "loss": 0.5019, | |
| "rewards/accuracies": 0.25, | |
| "rewards/chosen": -0.01242092065513134, | |
| "rewards/margins": -0.01534400973469019, | |
| "rewards/rejected": 0.0029230881482362747, | |
| "step": 33 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": -0.7313972115516663, | |
| "debug/policy_chosen_logps": -206.6120147705078, | |
| "debug/policy_rejected_logits": -0.8341861367225647, | |
| "debug/policy_rejected_logps": -174.07289123535156, | |
| "debug/reference_chosen_logps": -204.1991729736328, | |
| "debug/reference_rejected_logps": -172.9614715576172, | |
| "epoch": 0.918918918918919, | |
| "grad_norm": 13.848425223482339, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": -0.7313972115516663, | |
| "logits/rejected": -0.8341861367225647, | |
| "logps/chosen": -206.6120147705078, | |
| "logps/rejected": -174.07289123535156, | |
| "loss": 0.4918, | |
| "rewards/accuracies": 0.375, | |
| "rewards/chosen": -0.024128342047333717, | |
| "rewards/margins": -0.013014238327741623, | |
| "rewards/rejected": -0.01111410278826952, | |
| "step": 34 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": -0.7511980533599854, | |
| "debug/policy_chosen_logps": -169.01324462890625, | |
| "debug/policy_rejected_logits": -0.8619469404220581, | |
| "debug/policy_rejected_logps": -155.92356872558594, | |
| "debug/reference_chosen_logps": -174.2823028564453, | |
| "debug/reference_rejected_logps": -151.06353759765625, | |
| "epoch": 0.9459459459459459, | |
| "grad_norm": 13.804422393772251, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": -0.7511980533599854, | |
| "logits/rejected": -0.8619469404220581, | |
| "logps/chosen": -169.01324462890625, | |
| "logps/rejected": -155.92356872558594, | |
| "loss": 0.491, | |
| "rewards/accuracies": 0.875, | |
| "rewards/chosen": 0.052690617740154266, | |
| "rewards/margins": 0.10129091143608093, | |
| "rewards/rejected": -0.04860030114650726, | |
| "step": 35 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": -0.9176344275474548, | |
| "debug/policy_chosen_logps": -142.12106323242188, | |
| "debug/policy_rejected_logits": -0.8687180876731873, | |
| "debug/policy_rejected_logps": -179.96939086914062, | |
| "debug/reference_chosen_logps": -143.38665771484375, | |
| "debug/reference_rejected_logps": -179.182861328125, | |
| "epoch": 0.972972972972973, | |
| "grad_norm": 12.092026182468153, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": -0.9176344275474548, | |
| "logits/rejected": -0.8687180876731873, | |
| "logps/chosen": -142.12106323242188, | |
| "logps/rejected": -179.96939086914062, | |
| "loss": 0.4918, | |
| "rewards/accuracies": 0.625, | |
| "rewards/chosen": 0.012656106613576412, | |
| "rewards/margins": 0.02052140235900879, | |
| "rewards/rejected": -0.007865296676754951, | |
| "step": 36 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": -0.8388864398002625, | |
| "debug/policy_chosen_logps": -190.0416717529297, | |
| "debug/policy_rejected_logits": -0.9188035726547241, | |
| "debug/policy_rejected_logps": -160.5929412841797, | |
| "debug/reference_chosen_logps": -190.74612426757812, | |
| "debug/reference_rejected_logps": -159.9268798828125, | |
| "epoch": 1.0, | |
| "grad_norm": 12.215368730436227, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": -0.8388864398002625, | |
| "logits/rejected": -0.9188035726547241, | |
| "logps/chosen": -190.0416717529297, | |
| "logps/rejected": -160.5929412841797, | |
| "loss": 0.4773, | |
| "rewards/accuracies": 0.625, | |
| "rewards/chosen": 0.00704436469823122, | |
| "rewards/margins": 0.013704795390367508, | |
| "rewards/rejected": -0.006660431623458862, | |
| "step": 37 | |
| }, | |
| { | |
| "epoch": 1.0, | |
| "step": 37, | |
| "total_flos": 0.0, | |
| "train_loss": 0.4953597542401907, | |
| "train_runtime": 139.3771, | |
| "train_samples_per_second": 16.897, | |
| "train_steps_per_second": 0.265 | |
| } | |
| ], | |
| "logging_steps": 1, | |
| "max_steps": 37, | |
| "num_input_tokens_seen": 0, | |
| "num_train_epochs": 1, | |
| "save_steps": 500, | |
| "stateful_callbacks": { | |
| "TrainerControl": { | |
| "args": { | |
| "should_epoch_stop": false, | |
| "should_evaluate": false, | |
| "should_log": false, | |
| "should_save": true, | |
| "should_training_stop": true | |
| }, | |
| "attributes": {} | |
| } | |
| }, | |
| "total_flos": 0.0, | |
| "train_batch_size": 8, | |
| "trial_name": null, | |
| "trial_params": null | |
| } | |